♻️ Refactor docker devenv and image building.

.gitignore

@@ -25,7 +25,7 @@ node_modules
 /frontend/resources/public/*
 /exporter/target
 /exporter/.shadow-cljs
-/docker/testenv/bundle
+/docker/images/bundle
 /bundle*
 /media
 /deploy

docker/devenv/Dockerfile

@@ -2,10 +2,9 @@ FROM debian:buster
 LABEL maintainer="Andrey Antukh <niwi@niwi.nz>"
 
 ARG DEBIAN_FRONTEND=noninteractive
-ARG EXTERNAL_UID=1000
 
-ENV NODE_VERSION=v14.15.0 \
+ENV NODE_VERSION=v14.15.1 \
-    CLOJURE_VERSION=1.10.1.727 \
+    CLOJURE_VERSION=1.10.1.739 \
     LANG=en_US.UTF-8 \
     LC_ALL=en_US.UTF-8
 
@@ -31,7 +30,7 @@ RUN set -ex; \
     rm -rf /var/lib/apt/lists/*;
 
 RUN set -ex; \
-    useradd -m -g users -s /bin/bash -u $EXTERNAL_UID penpot; \
+    useradd -m -g users -s /bin/bash penpot; \
     passwd penpot -d; \
     echo "penpot ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
 
@@ -120,19 +119,20 @@ COPY files/phantomjs-mock /usr/bin/phantomjs
 COPY files/bashrc         /root/.bashrc
 COPY files/vimrc          /root/.vimrc
 COPY files/tmux.conf      /root/.tmux.conf
-COPY files/start-tmux.sh  /home/start-tmux.sh
-COPY files/entrypoint.sh  /home/entrypoint.sh
-COPY files/init.sh        /home/init.sh
 
-USER penpot
+WORKDIR /home
-WORKDIR /home/penpot
 
 RUN set -ex; \
-    git clone https://github.com/creationix/nvm.git .nvm; \
+    mkdir -p /tmp/node; \
-    bash -c "source .nvm/nvm.sh && nvm install $NODE_VERSION"; \
+    cd /tmp/node; \
-    bash -c "source .nvm/nvm.sh && nvm alias default $NODE_VERSION"; \
+    export PATH="$PATH:/usr/local/nodejs/bin"; \
-    bash -c "source .nvm/nvm.sh && nvm use default"; \
+    wget https://nodejs.org/dist/$NODE_VERSION/node-$NODE_VERSION-linux-x64.tar.xz; \
-    bash -c "source .nvm/nvm.sh && npm install -g yarn";
+    tar xvf node-$NODE_VERSION-linux-x64.tar.xz; \
+    mv /tmp/node/node-$NODE_VERSION-linux-x64 /usr/local/nodejs; \
+    chown -R root /usr/local/nodejs; \
+    /usr/local/nodejs/bin/npm install -g yarn; \
+    /usr/local/nodejs/bin/yarn global add clj-kondo; \
+    rm -rf /tmp/node;
 
 EXPOSE 3447
 EXPOSE 3448
@@ -140,5 +140,9 @@ EXPOSE 3449
 EXPOSE 6060
 EXPOSE 9090
 
+COPY files/start-tmux.sh  /home/start-tmux.sh
+COPY files/entrypoint.sh  /home/entrypoint.sh
+COPY files/init.sh        /home/init.sh
+
 ENTRYPOINT ["/home/entrypoint.sh"]
 CMD ["/home/init.sh"]

docker/devenv/docker-compose.yaml

@@ -14,7 +14,7 @@ volumes:
 services:
   main:
     privileged: true
-    image: "penpot-devenv"
+    image: "penpotapp/devenv:latest"
     build:
       context: "."
     container_name: "penpot-devenv-main"
@@ -42,6 +42,7 @@ services:
       - APP_DATABASE_USERNAME=penpot
       - APP_DATABASE_PASSWORD=penpot
       - APP_REDIS_URI=redis://redis/0
+      - EXTERNAL_UID=${CURRENT_USER_ID}
 
   postgres:
     image: postgres:13

docker/devenv/files/bashrc

@@ -1,12 +1,7 @@
-export PATH=$HOME/.local/bin:$PATH
+export PATH=$HOME/.local/bin:/usr/local/nodejs/bin:$PATH
 
 alias l='ls --color -GFlh'
 alias rm='rm -r'
 alias ls='ls --color -F'
 alias lsd='ls -d *(/)'
 alias lsf='ls -h *(.)'
-
-export LEIN_FAST_TRAMPOLINE=y
-
-export NVM_DIR="$HOME/.nvm"
-[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"  # This loads nvm

docker/devenv/files/entrypoint.sh

@@ -1,12 +1,6 @@
 #!/usr/bin/env bash
 
 set -e
-
+usermod -u ${EXTERNAL_UID:-1000} penpot
-sudo cp /root/.bashrc /home/penpot/.bashrc
-sudo cp /root/.vimrc /home/penpot/.vimrc
-sudo cp /root/.tmux.conf /home/penpot/.tmux.conf
-
-source /home/penpot/.bashrc
-sudo chown penpot:users /home/penpot
 
 exec "$@"

docker/devenv/files/init.sh

@@ -1,10 +1,5 @@
 #!/usr/bin/env bash
 
-set -e;
+set -e
-source ~/.bashrc
+nginx
-
-echo "[init.sh] Start nginx."
-sudo nginx
-
-echo "[init.sh] Ready!"
 tail -f /dev/null

docker/devenv/files/start-tmux.sh

@@ -1,5 +1,12 @@
 #!/usr/bin/env bash
 
+sudo cp /root/.bashrc /home/penpot/.bashrc
+sudo cp /root/.vimrc /home/penpot/.vimrc
+sudo cp /root/.tmux.conf /home/penpot/.tmux.conf
+
+source /home/penpot/.bashrc
+sudo chown penpot:users /home/penpot
+
 cd ~;
 
 set -e;

docker/images/Dockerfile.backend

@@ -0,0 +1,21 @@
+FROM debian:buster-slim
+LABEL maintainer="Andrey Antukh <niwi@niwi.nz>"
+
+ENV LANG='en_US.UTF-8' LC_ALL='en_US.UTF-8'
+
+RUN set -ex; \
+    apt-get -qq update; \
+    apt-get -qqy --no-install-recommends install wget locales ca-certificates imagemagick webp gnupg2; \
+    echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen; \
+    locale-gen; \
+    mkdir -p /usr/share/man/man1; \
+    mkdir -p /usr/share/man/man7; \
+    wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | apt-key add -; \
+    echo "deb https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ buster main" >> /etc/apt/sources.list.d/adoptopenjdk.list; \
+    apt-get -qq update; \
+    apt-get -qqy install adoptopenjdk-15-hotspot; \
+    rm -rf /var/lib/apt/lists/*;
+
+ADD ./bundle/backend/ /opt/bundle/
+WORKDIR /opt/bundle
+CMD ["/bin/bash", "run.sh"]

docker/images/Dockerfile.exporter

@@ -5,7 +5,7 @@ ARG DEBIAN_FRONTEND=noninteractive
 
 ENV LANG=en_US.UTF-8 \
     LC_ALL=en_US.UTF-8 \
-    NODE_VERSION=v12.18.4
+    NODE_VERSION=v14.15.1
 
 RUN set -ex; \
     mkdir -p /etc/resolvconf/resolv.conf.d; \

docker/images/docker-compose.yaml

@@ -0,0 +1,68 @@
+---
+version: "3"
+
+networks:
+  default:
+    driver: bridge
+    ipam:
+      driver: default
+      config:
+        - subnet: 172.177.99.0/24
+
+volumes:
+  postgres_data:
+  user_data:
+  backend_data:
+
+services:
+  penpot-frontend:
+    image: "penpotapp/frontend:develop"
+    ports:
+      - 8080:80
+
+    volumes:
+      - backend_data:/opt/data
+
+    depends_on:
+      - penpot-backend
+      - penpot-exporter
+
+  penpot-backend:
+    image: "penpotapp/backend:develop"
+    volumes:
+      - backend_data:/opt/data
+
+    depends_on:
+      - penpot-postgres
+      - penpot-redis
+
+    environment:
+      - APP_DATABASE_URI=postgresql://penpot-postgres/penpot
+      - APP_DATABASE_USERNAME=penpot
+      - APP_DATABASE_PASSWORD=penpot
+      - APP_SMTP_ENABLED=false
+      - APP_REDIS_URI=redis://penpot-redis/0
+      - APP_MEDIA_DIRECTORY=/opt/data/media
+
+  penpot-exporter:
+    image: "penpotapp/exporter:develop"
+    environment:
+      - APP_PUBLIC_URI=http://penpot-frontend
+
+  penpot-postgres:
+    image: "postgres:13"
+    restart: always
+    stop_signal: SIGINT
+
+    environment:
+      - POSTGRES_INITDB_ARGS=--data-checksums
+      - POSTGRES_DB=penpot
+      - POSTGRES_USER=penpot
+      - POSTGRES_PASSWORD=penpot
+
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+
+  penpot-redis:
+    image: redis:6
+    restart: always

docker/images/files/nginx.conf

@@ -26,9 +26,9 @@ http {
     access_log /dev/stdout;
 
     gzip on;
-
     gzip_vary on;
     gzip_proxied any;
+    gzip_static on;
     gzip_comp_level 4;
     gzip_buffers 16 8k;
     gzip_http_version 1.1;
@@ -70,21 +70,21 @@ http {
         }
 
         location /api {
-            proxy_pass http://172.177.99.3:6060/api;
+            proxy_pass http://penpot-backend:6060/api;
         }
 
         location /export {
-            proxy_pass http://172.177.99.4:6061;
+            proxy_pass http://penpot-exporter:6061;
         }
 
         location /ws/notifications {
             proxy_set_header Upgrade $http_upgrade;
             proxy_set_header Connection 'upgrade';
-            proxy_pass http://172.177.99.3:6060/ws/notifications;
+            proxy_pass http://penpot-backend:6060/ws/notifications;
         }
 
         location /media {
-            alias  /mount/backend/media;
+            alias  /opt/data/media;
         }
     }
 }

docker/testenv/Dockerfile-backend

@@ -1,5 +0,0 @@
-FROM adoptopenjdk/openjdk15:debianslim-jre
-LABEL maintainer="Andrey Antukh <niwi@niwi.nz>"
-ADD ./bundle/backend/ /opt/bundle/
-WORKDIR /opt/bundle
-CMD ["/bin/bash", "run.sh"]

docker/testenv/docker-compose.yaml

@@ -1,112 +0,0 @@
----
-version: "3"
-
-networks:
-  default:
-    driver: bridge
-    ipam:
-      driver: default
-      config:
-        - subnet: 172.177.99.0/24
-
-volumes:
-  postgres_data:
-  user_data:
-  backend_data:
-
-services:
-  nginx:
-    image: "uxbox-testenv-nginx"
-    build:
-      context: "."
-      dockerfile: "Dockerfile-nginx"
-
-    ports:
-      - 8080:80
-
-    networks:
-      default:
-        ipv4_address: 172.177.99.2
-
-  backend:
-    image: "uxbox-testenv-backend"
-    build:
-      context: "."
-      dockerfile: "Dockerfile-backend"
-
-    volumes:
-      - backend_data:/opt/data
-
-    depends_on:
-      - postgres
-      - smtp
-      - redis
-
-    environment:
-      - APP_DATABASE_URI=postgresql://postgres/uxbox
-      - APP_DATABASE_USERNAME=uxbox
-      - APP_DATABASE_PASSWORD=uxbox
-      - APP_SENDMAIL_BACKEND=smtp
-      - APP_SMTP_HOST=smtp
-      - APP_SMTP_PORT=25
-      - APP_MEDIA_DIRECTORY=/opt/data/media
-
-    networks:
-      default:
-        ipv4_address: 172.177.99.3
-
-
-  exporter:
-    image: "uxbox-testenv-exporter"
-    build:
-      context: "."
-      dockerfile: "Dockerfile-exporter"
-
-    environment:
-      - APP_PUBLIC_URI=http://nginx
-
-    depends_on:
-      - backend
-      - nginx
-
-    networks:
-      default:
-        ipv4_address: 172.177.99.4
-
-  smtp:
-    image: mwader/postfix-relay:latest
-    restart: always
-    environment:
-      - POSTFIX_myhostname=smtp.testing.uxbox.io
-      - OPENDKIM_DOMAINS=smtp.testing.uxbox.io
-
-    networks:
-      default:
-        ipv4_address: 172.177.99.5
-
-  postgres:
-    image: "postgres:12"
-    restart: always
-    stop_signal: SIGINT
-
-    environment:
-      - POSTGRES_INITDB_ARGS=--data-checksums
-      - POSTGRES_DB=uxbox
-      - POSTGRES_USER=uxbox
-      - POSTGRES_PASSWORD=uxbox
-
-    volumes:
-      - postgres_data:/var/lib/postgresql/data
-
-    networks:
-      default:
-        ipv4_address: 172.177.99.6
-
-  redis:
-    image: redis:6
-    restart: always
-
-    networks:
-      default:
-        ipv4_address: 172.177.99.7
-

manage.sh

@@ -1,58 +1,76 @@
 #!/usr/bin/env bash
 set -e
 
-REV=`git log -n 1 --pretty=format:%h -- docker/`
+export ORGANIZATION="penpotapp";
-DEVENV_IMGNAME="penpot-devenv"
+export DEVENV_IMGNAME="$ORGANIZATION/devenv";
+export DEVENV_PNAME="penpotdev";
+
+export CURRENT_USER_ID=$(id -u);
+export CURRENT_GIT_TAG=$(git describe --tags);
+export CURRENT_GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD);
 
 function build-devenv {
-    echo "Building development image $DEVENV_IMGNAME:latest with UID $EXTERNAL_UID..."
+    echo "Building development image $DEVENV_IMGNAME:latest..."
-    local EXTERNAL_UID=${1:-$(id -u)}
+
-    docker-compose -p penpotdev -f docker/devenv/docker-compose.yaml build \
+    pushd docker/devenv;
-                   --force-rm --build-arg EXTERNAL_UID=$EXTERNAL_UID
+    docker build -t $DEVENV_IMGNAME:latest .
+    popd;
 }
 
-function build-devenv-if-not-exists {
+function publish-devenv {
+    docker push $DEVENV_IMGNAME:latest
+}
+
+function pull-devenv {
+    set -ex
+    docker pull $DEVENV_IMGNAME:latest
+}
+
+function pull-devenv-if-not-exists {
     if [[ ! $(docker images $DEVENV_IMGNAME:latest -q) ]]; then
-        build-devenv $@
+        pull-devenv $@
     fi
 }
 
 function start-devenv {
-    build-devenv-if-not-exists $@;
+    pull-devenv-if-not-exists $@;
-    docker-compose -p penpotdev -f docker/devenv/docker-compose.yaml up -d;
+    docker-compose -p $DEVENV_PNAME -f docker/devenv/docker-compose.yaml up -d;
 }
 
 function stop-devenv {
-    docker-compose -p penpotdev -f docker/devenv/docker-compose.yaml stop -t 2;
+    docker-compose -p $DEVENV_PNAME -f docker/devenv/docker-compose.yaml stop -t 2;
 }
 
 function drop-devenv {
-    docker-compose -p penpotdev -f docker/devenv/docker-compose.yaml down -t 2 -v;
+    docker-compose -p $DEVENV_PNAME -f docker/devenv/docker-compose.yaml down -t 2 -v;
 
     echo "Clean old development image $DEVENV_IMGNAME..."
     docker images $DEVENV_IMGNAME -q | awk '{print $3}' | xargs --no-run-if-empty docker rmi
 }
 
+function log-devenv {
+    docker-compose -p $DEVENV_PNAME -f docker/devenv/docker-compose.yaml logs -f --tail=50
+}
+
 function run-devenv {
     if [[ ! $(docker ps -f "name=penpot-devenv-main" -q) ]]; then
         start-devenv
     fi
 
-    docker exec -ti penpot-devenv-main /home/start-tmux.sh
+    docker exec -ti penpot-devenv-main sudo -EH -u penpot /home/start-tmux.sh
 }
 
 function build {
-    build-devenv-if-not-exists;
+    pull-devenv-if-not-exists;
-    local IMAGE=$DEVENV_IMGNAME:latest;
+    docker volume create $DEVENV_PNAME_user_data;
-
-    docker volume create penpotdev_user_data;
 
     echo "Running development image $IMAGE to build frontend."
     docker run -t --rm \
-           --mount source=penpotdev_user_data,type=volume,target=/home/penpot/ \
+           --mount source=$DEVENV_PNAME_user_data,type=volume,target=/home/penpot/ \
            --mount source=`pwd`,type=bind,target=/home/penpot/penpot \
+           -e EXTERNAL_UID=$CURRENT_USER_ID \
            -w /home/penpot/penpot/$1 \
-           $IMAGE ./scripts/build.sh
+           $DEVENV_IMGNAME:latest sudo -u penpot ./scripts/build.sh
 }
 
 function build-frontend {
@@ -68,7 +86,6 @@ function build-backend {
 }
 
 function build-bundle {
-
     build "frontend";
     build "exporter";
     build "backend";
@@ -79,70 +96,103 @@ function build-bundle {
     mv ./backend/target/dist ./bundle/backend
     mv ./exporter/target ./bundle/exporter
 
-    NAME="penpot-$(date '+%Y.%m.%d-%H%M')"
+    local name="penpot-$CURRENT_GIT_TAG";
+
+    echo $CURRENT_GIT_TAG > ./bundle/frontend/version.txt
+    echo $CURRENT_GIT_TAG > ./bundle/backend/main/version.txt
+    echo $CURRENT_GIT_TAG > ./bundle/exporter/version.txt
+    echo $CURRENT_GIT_TAG > ./bundle/version.txt
 
     pushd bundle/
-    tar -cvf ../$NAME.tar *;
+    tar -cvf ../$name.tar *;
     popd
 
-    xz -vez4f -T4 $NAME.tar
+    xz -vez1f -T4 $name.tar
+
+    echo "##############################################################";
+    echo "# Generated $name.tar.xz";
+    echo "##############################################################";
 }
 
-function log-devenv {
+function build-image {
-    docker-compose -p penpotdev -f docker/devenv/docker-compose.yaml logs -f --tail=50
+    set -ex;
+
+    local image=$1;
+
+    pushd ./docker/images;
+    local docker_image="$ORGANIZATION/$image";
+    docker build -t $docker_image:$CURRENT_GIT_TAG -f Dockerfile.$image .;
+    popd;
 }
 
-function build-testenv {
+function build-images {
-    local BUNDLE_FILE=$1;
+    local bundle_file="penpot-$CURRENT_GIT_TAG.tar.xz";
-    local BUNDLE_FILE_PATH=`readlink -f $BUNDLE_FILE`;
 
-    echo "Building testenv with bundle: $BUNDLE_FILE_PATH."
+    if [ ! -f $bundle_file ]; then
-
+        echo "File '$bundle_file' does not exists.";
-    if [ ! -f $BUNDLE_FILE ]; then
+        exit 1;
-        echo "File $BUNDLE_FILE does not exists."
     fi
 
-    rm -rf ./docker/testenv/bundle;
+    rm -rf ./docker/images/bundle;
-    mkdir -p ./docker/testenv/bundle;
+    mkdir -p ./docker/images/bundle;
 
-    pushd ./docker/testenv/bundle;
+    local bundle_file_path=`readlink -f $bundle_file`;
-    tar xvf $BUNDLE_FILE_PATH;
+    echo "Building docker image from: $bundle_file_path.";
+
+    pushd ./docker/images/bundle;
+    tar xvf $bundle_file_path;
     popd
 
-    pushd ./docker/testenv;
+    build-image "backend";
-    docker-compose -p penpot-testenv -f ./docker-compose.yaml build
+    build-image "frontend";
-    popd
+    build-image "exporter";
 }
 
-function start-testenv {
+function publish-snapshot {
-    pushd ./docker/testenv;
+    set -x
-    docker-compose -p penpot-testenv -f ./docker-compose.yaml up
+    docker tag $ORGANIZATION/frontend:$CURRENT_GIT_TAG $ORGANIZATION/frontend:$CURRENT_GIT_BRANCH
-    popd
+    docker tag $ORGANIZATION/backend:$CURRENT_GIT_TAG $ORGANIZATION/backend:$CURRENT_GIT_BRANCH
+    docker tag $ORGANIZATION/exporter:$CURRENT_GIT_TAG $ORGANIZATION/exporter:$CURRENT_GIT_BRANCH
+
+    docker push $ORGANIZATION/frontend:$CURRENT_GIT_BRANCH;
+    docker push $ORGANIZATION/backend:$CURRENT_GIT_BRANCH;
+    docker push $ORGANIZATION/exporter:$CURRENT_GIT_BRANCH;
 }
 
 function usage {
-    echo "PENPOT build & release manager v$REV"
+    echo "PENPOT build & release manager"
     echo "USAGE: $0 OPTION"
     echo "Options:"
     # echo "- clean                            Stop and clean up docker containers"
     # echo ""
-    echo "- build-devenv                     Build docker development oriented image; (can specify external user id in parameter)"
+    echo "- pull-devenv                      Pulls docker development oriented image"
+    echo "- build-devenv                     Build docker development oriented image"
     echo "- start-devenv                     Start the development oriented docker-compose service."
     echo "- stop-devenv                      Stops the development oriented docker-compose service."
     echo "- drop-devenv                      Remove the development oriented docker-compose containers, volumes and clean images."
     echo "- run-devenv                       Attaches to the running devenv container and starts development environment"
     echo "                                   based on tmux (frontend at localhost:3449, backend at localhost:6060)."
     echo ""
-    echo "- run-all-tests                    Execute unit tests for both backend and frontend."
+    # echo "- run-all-tests                    Execute unit tests for both backend and frontend."
-    echo "- run-frontend-tests               Execute unit tests for frontend only."
+    # echo "- run-frontend-tests               Execute unit tests for frontend only."
-    echo "- run-backend-tests                Execute unit tests for backend only."
+    # echo "- run-backend-tests                Execute unit tests for backend only."
 }
 
 case $1 in
     ## devenv related commands
+    pull-devenv)
+        pull-devenv ${@:2};
+        ;;
+
     build-devenv)
         build-devenv ${@:2}
         ;;
+
+
+    publish-devenv)
+        publish-devenv ${@:2}
+        ;;
+
     start-devenv)
         start-devenv ${@:2}
         ;;
@@ -159,16 +209,6 @@ case $1 in
         log-devenv ${@:2}
         ;;
 
-
-    # Test Env
-    start-testenv)
-        start-testenv
-        ;;
-
-    build-testenv)
-        build-testenv ${@:2}
-        ;;
-
     ## testin related commands
 
     # run-all-tests)
@@ -198,6 +238,15 @@ case $1 in
         build-bundle
         ;;
 
+    # Docker Image Tasks
+    build-images)
+        build-images;
+        ;;
+
+    publish-snapshot)
+        publish-snapshot ${@:2}
+        ;;
+
     *)
         usage
         ;;