🎉 Add testenv.

.gitignore

@@ -25,8 +25,8 @@ node_modules
 /frontend/resources/public/*
 /exporter/target
 /exporter/.shadow-cljs
-/docker/frontend/dist
-/docker/backend/dist
+/docker/testenv/bundle
+/bundle*
 /media
 /deploy
 /web

backend/deps.edn

@@ -98,6 +98,10 @@
   {:extra-deps {olical/depot {:mvn/version "1.8.4"}}
    :main-opts ["-m" "depot.outdated.main"]}
 
+  :jar
+  {:extra-deps {seancorfield/depstar {:mvn/version "RELEASE"}}
+   :main-opts ["-m" "hf.depstar.jar" "-S" "target/app.jar"]}
+
   :jmx-remote
   {:jvm-opts ["-Dcom.sun.management.jmxremote"
               "-Dcom.sun.management.jmxremote.port=9090"

backend/resources/log4j2-bundle.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="info" monitorInterval="60">
+  <Appenders>
+    <Console name="console" target="SYSTEM_OUT">
+      <PatternLayout pattern="[%t] %level{length=1} %logger{36} - %msg%n"/>
+    </Console>
+  </Appenders>
+
+  <Loggers>
+    <Logger name="com.zaxxer.hikari" level="error" additivity="false">
+      <AppenderRef ref="console"/>
+    </Logger>
+
+    <Logger name="org.eclipse.jetty" level="info" additivity="false">
+      <AppenderRef ref="console"/>
+    </Logger>
+
+    <Logger name="app" level="debug" additivity="false">
+      <AppenderRef ref="console"/>
+    </Logger>
+
+    <Root level="info">
+      <AppenderRef ref="console"/>
+    </Root>
+  </Loggers>
+</Configuration>

backend/scripts/build.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+CLASSPATH=`(clojure -Spath)`
+NEWCP="./resources:./app.jar"
+
+rm -rf ./target/dist
+mkdir -p ./target/dist/deps
+
+for item in $(echo $CLASSPATH | tr ":" "\n"); do
+    if [ "${item: -4}" == ".jar" ]; then
+        cp $item ./target/dist/deps/;
+        BN="$(basename -- $item)"
+        NEWCP+=":./deps/$BN"
+    fi
+done
+
+cp ./resources/log4j2-bundle.xml ./target/dist/log4j2.xml
+
+clojure -Ajar
+
+cp ./target/app.jar ./target/dist/app.jar
+echo $NEWCP > ./target/dist/classpath;
+
+tee -a ./target/dist/run.sh  >> /dev/null <<EOF
+#!/usr/bin/env bash
+
+CP="$NEWCP"
+
+# Exports
+
+# Find java executable
+set +e
+JAVA_CMD=\$(type -p java)
+
+set -e
+if [[ ! -n "\$JAVA_CMD" ]]; then
+  if [[ -n "\$JAVA_HOME" ]] && [[ -x "\$JAVA_HOME/bin/java" ]]; then
+    JAVA_CMD="\$JAVA_HOME/bin/java"
+  else
+    >&2 echo "Couldn't find 'java'. Please set JAVA_HOME."
+    exit 1
+  fi
+fi
+
+if [ -f ./environ ]; then
+   source ./environ
+fi
+
+set -x
+\$JAVA_CMD \$JVM_OPTS -classpath \$CP -Dlog4j.configurationFile=./log4j2.xml "\$@" clojure.main -m app.main
+EOF
+
+chmod +x ./target/dist/run.sh
+
+
+

docker/testenv/Dockerfile-backend

@@ -0,0 +1,5 @@
+FROM azul/zulu-openjdk-debian:14
+LABEL maintainer="Andrey Antukh <niwi@niwi.nz>"
+ADD ./bundle/backend/ /opt/bundle/
+WORKDIR /opt/bundle
+CMD ["/bin/bash", "run.sh"]

docker/testenv/Dockerfile-exporter

@@ -0,0 +1,87 @@
+FROM debian:buster
+LABEL maintainer="Andrey Antukh <niwi@niwi.nz>"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ENV LANG=en_US.UTF-8 LC_ALL=C.UTF-8
+
+RUN set -ex; \
+    mkdir -p /etc/resolvconf/resolv.conf.d; \
+    echo "nameserver 8.8.8.8" > /etc/resolvconf/resolv.conf.d/tail;
+
+RUN set -ex; \
+    apt-get update && \
+    apt-get install -yq \
+        locales \
+        gnupg2 \
+        ca-certificates \
+        wget \
+        curl \
+        bash \
+        rlwrap \
+        imagemagick \
+        netpbm \
+        potrace \
+        gconf-service \
+        libasound2 \
+        libatk1.0-0 \
+        libatk-bridge2.0-0 \
+        libcairo2 \
+        libcups2 \
+        libdbus-1-3 \
+        libexpat1 \
+        libfontconfig1 \
+        libgcc1 \
+        libgconf-2-4 \
+        libgdk-pixbuf2.0-0 \
+        libglib2.0-0 \
+        libgtk-3-0 \
+        libnspr4 \
+        libpango-1.0-0 \
+        libpangocairo-1.0-0 \
+        libx11-6 \
+        libx11-xcb1 \
+        libxcb1 \
+        libxcb-dri3-0 \
+        libxcomposite1 \
+        libxcursor1 \
+        libxdamage1 \
+        libxext6 \
+        libxfixes3 \
+        libxi6 \
+        libxrandr2 \
+        libxrender1 \
+        libxss1 \
+        libxtst6 \
+        fonts-liberation \
+        libappindicator1 \
+        libnss3 \
+        libgbm1 \
+    ; \
+    rm -rf /var/lib/apt/lists/*;
+
+RUN set -ex; \
+    wget https://github.com/RazrFalcon/svgcleaner/releases/download/v0.9.5/svgcleaner_linux_x86_64_0.9.5.tar.gz; \
+    tar xvf svgcleaner_linux_x86_64_0.9.5.tar.gz; \
+    mv svgcleaner /usr/local/bin/; \
+    rm -rf svgcleaner_linux_x86_64_0.9.5.tar.gz;
+
+RUN set -ex; \
+    mkdir -p /tmp/node; \
+    cd /tmp/node; \
+    export PATH="$PATH:/usr/local/node-v12.18.3/bin"; \
+    wget https://nodejs.org/dist/v12.18.3/node-v12.18.3-linux-x64.tar.xz; \
+    tar xvf node-v12.18.3-linux-x64.tar.xz; \
+    mv /tmp/node/node-v12.18.3-linux-x64 /usr/local/node-v12.18.3; \
+    /usr/local/node-v12.18.3/bin/npm install -g yarn; \
+    rm -rf /tmp/node;
+
+WORKDIR /opt/app
+
+ADD ./bundle/exporter/ /opt/app/
+
+RUN set -ex; \
+    export PATH="$PATH:/usr/local/node-v12.18.3/bin"; \
+    yarn install;
+
+CMD ["/usr/local/node-v12.18.3/bin/node", "app.js"]

docker/testenv/Dockerfile-nginx

@@ -0,0 +1,9 @@
+FROM nginx:latest
+LABEL maintainer="Andrey Antukh <niwi@niwi.nz>"
+
+ADD ./bundle/frontend /var/www/app/
+ADD ./files/nginx.conf /etc/nginx/nginx.conf
+ADD ./files/nginx-entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
+CMD ["nginx", "-g", "daemon off;"]

docker/testenv/docker-compose.yaml

@@ -0,0 +1,112 @@
+---
+version: "3"
+
+networks:
+  default:
+    driver: bridge
+    ipam:
+      driver: default
+      config:
+        - subnet: 172.177.99.0/24
+
+volumes:
+  postgres_data:
+  user_data:
+  backend_data:
+
+services:
+  nginx:
+    image: "uxbox-testenv-nginx"
+    build:
+      context: "."
+      dockerfile: "Dockerfile-nginx"
+
+    ports:
+      - 8080:80
+
+    networks:
+      default:
+        ipv4_address: 172.177.99.2
+
+  backend:
+    image: "uxbox-testenv-backend"
+    build:
+      context: "."
+      dockerfile: "Dockerfile-backend"
+
+    volumes:
+      - backend_data:/opt/data
+
+    depends_on:
+      - postgres
+      - smtp
+      - redis
+
+    environment:
+      - APP_DATABASE_URI=postgresql://postgres/uxbox
+      - APP_DATABASE_USERNAME=uxbox
+      - APP_DATABASE_PASSWORD=uxbox
+      - APP_SENDMAIL_BACKEND=smtp
+      - APP_SMTP_HOST=smtp
+      - APP_SMTP_PORT=25
+      - APP_MEDIA_DIRECTORY=/opt/data/media
+
+    networks:
+      default:
+        ipv4_address: 172.177.99.3
+
+
+  exporter:
+    image: "uxbox-testenv-exporter"
+    build:
+      context: "."
+      dockerfile: "Dockerfile-exporter"
+
+    environment:
+      - APP_PUBLIC_URI=http://nginx
+
+    depends_on:
+      - backend
+      - nginx
+
+    networks:
+      default:
+        ipv4_address: 172.177.99.4
+
+  smtp:
+    image: mwader/postfix-relay:latest
+    restart: always
+    environment:
+      - POSTFIX_myhostname=smtp.testing.uxbox.io
+      - OPENDKIM_DOMAINS=smtp.testing.uxbox.io
+
+    networks:
+      default:
+        ipv4_address: 172.177.99.5
+
+  postgres:
+    image: "postgres:12"
+    restart: always
+    stop_signal: SIGINT
+
+    environment:
+      - POSTGRES_INITDB_ARGS=--data-checksums
+      - POSTGRES_DB=uxbox
+      - POSTGRES_USER=uxbox
+      - POSTGRES_PASSWORD=uxbox
+
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+
+    networks:
+      default:
+        ipv4_address: 172.177.99.6
+
+  redis:
+    image: redis:6
+    restart: always
+
+    networks:
+      default:
+        ipv4_address: 172.177.99.7
+

docker/testenv/files/exporter-entrypoint.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -ex
+
+export PATH="/usr/local/node-v12.18.3/bin/:$PATH"
+# yarn install
+
+exec "$@"

docker/testenv/files/nginx-entrypoint.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+exec "$@";

docker/testenv/files/nginx.conf

@@ -0,0 +1,90 @@
+user www-data;
+worker_processes auto;
+pid /run/nginx.pid;
+include /etc/nginx/modules-enabled/*.conf;
+
+events {
+    worker_connections 768;
+    # multi_accept on;
+}
+
+http {
+    sendfile on;
+    tcp_nopush on;
+    tcp_nodelay on;
+    keepalive_timeout 65;
+    types_hash_max_size 2048;
+    # server_tokens off;
+
+    # server_names_hash_bucket_size 64;
+    # server_name_in_redirect off;
+
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    error_log /dev/stdout;
+    access_log /dev/stdout;
+
+    gzip on;
+
+    gzip_vary on;
+    gzip_proxied any;
+    gzip_comp_level 4;
+    gzip_buffers 16 8k;
+    gzip_http_version 1.1;
+
+    gzip_types text/plain text/css text/javascript application/javascript application/json application/transit+json;
+
+    map $http_upgrade $connection_upgrade {
+        default upgrade;
+    ''      close;
+    }
+
+    server {
+        listen 80 default_server;
+        server_name _;
+
+        client_max_body_size 5M;
+        charset utf-8;
+
+        proxy_http_version 1.1;
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Scheme $scheme;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+        etag off;
+
+        location / {
+            root /var/www/app/;
+            try_files $uri /index.html;
+
+            location ~* \.(js|css).*$ {
+                add_header Cache-Control "max-age=86400" always; # 24 hours
+            }
+
+            location = /index.html {
+                add_header Cache-Control "no-cache, max-age=0";
+            }
+        }
+
+        location /api {
+            proxy_pass http://172.177.99.3:6060/api;
+        }
+
+        location /export {
+            proxy_pass http://172.177.99.4:6061;
+        }
+
+        location /ws/notifications {
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection 'upgrade';
+            proxy_pass http://172.177.99.3:6060/ws/notifications;
+        }
+
+        location /media {
+            alias  /mount/backend/media;
+        }
+    }
+}

docs/01-Development-Environment.md

@@ -64,8 +64,15 @@ This will do the following:
 - Attaches to the **devenv** container and executes the tmux session.
 - The tmux session automatically starts all the necessary services.
 
+You can execute the individual steps manully if you want:
 
-## First steps with tmux ##
+```bash
+./manage.sh build-devenv # builds the devenv docker image
+./manage.sh start-devenv # starts background running containers
+./manage.sh run-devenv   # enters to new tmux session inside of one of the running containers
+./manage.sh stop-devenv  # stops background running containers
+./manage.sh drop-devenv  # removes all the volumes, containers and networks used by the devenv
+```
 
 Now having the the container running and tmux open inside the
 container, you are free to execute any commands and open many shells
@@ -78,16 +85,16 @@ current window.
 For more info: https://tmuxcheatsheet.com/
 
 
-## Inside the tmux session ##
+### Inside the tmux session
 
-### Styles ###
+#### gulp
 
 The styles and many related tasks are executed thanks to gulp and they are
 executed in the tmux **window 0**. This is a normal gulp watcher with some
 additional tasks.
 
 
-### Frontend ###
+#### shadow-cljs
 
 The frontend build process is located on the tmux **window 1**.
 **Shadow-cljs** is used for build and serve the frontend code. For
@@ -101,7 +108,7 @@ connected browser, by opening a third window with `Ctrl+c` and running
 `npx shadow-cljs cljs-repl main`.
 
 
-### Exporter ###
+#### exporter
 
 The exporter app (clojurescript app running in nodejs) is located in
 **window 2**, and you can go directly to it using `ctrl+b 2` shortcut.
@@ -115,7 +122,7 @@ If some reason scripts does not stars correctly, you can manually
 execute `node target/app.js ` to start the exporter app.
 
 
-### Backend ###
+#### backend
 
 The backend related environment is located in the tmux **window 3**,
 and you can go directly to it using `ctrl+b 2` shortcut.
@@ -137,3 +144,46 @@ later use `(restart)` again.
 
 For more information, please refer to: `03-Backend-Guide.md`.
 
+
+## Start the testenv ##
+
+The purpose of the testenv (Test Environment) is provide an easy way
+to get uxbox running in local pc without getting into the full
+development environment.
+
+As first step we still need to build devenv image because that image
+is used to produce the production-like bundle of the application:
+
+```bash
+./manage.sh build-devenv
+```
+
+Once the image is build, you no longer need to rebuilt it until the
+devenv image is changed and this happens we make some structural
+changes or upgrading some dependencies.
+
+Them, let's proceed to build the bundle (a directory that contains all
+the sources and dependencies of the platform ready to be deployed):
+
+```bash
+./manage.sh build-bundle
+```
+
+This will generate on current directory one file and one
+directory. The most important is the file like
+`uxbox-2020.09.09-1343.tar.xz`. 
+
+Then, let's proceed to build the docker images with the bundle
+generated from the previous step.
+
+```bash
+./manage.sh build-testenv ./uxbox-2020.09.09-1343.tar.xz
+```
+
+This will generate the necessary docker images ready to be executed.
+
+And finally, start the docker-compose:
+
+```bash
+./manage.sh start-devenv
+```

manage.sh

@@ -45,53 +45,85 @@ function run-devenv {
     docker exec -ti uxbox-devenv-main /home/start-tmux.sh
 }
 
-function build-frontend {
+function build {
     build-devenv-if-not-exists;
-
     local IMAGE=$DEVENV_IMGNAME:latest;
 
+    docker volume create uxboxdev_user_data;
+
     echo "Running development image $IMAGE to build frontend."
     docker run -t --rm \
+           --mount source=uxboxdev_user_data,type=volume,target=/home/uxbox/ \
            --mount source=`pwd`,type=bind,target=/home/uxbox/uxbox \
-           --mount source=${HOME}/.m2,type=bind,target=/home/uxbox/.m2 \
-           -w /home/uxbox/uxbox/frontend \
-           $IMAGE ./scripts/build-app.sh
-}
-
-function build-exporter {
-    build-devenv-if-not-exists;
-
-    local IMAGE=$DEVENV_IMGNAME:latest;
-
-    echo "Running development image $IMAGE to build frontend."
-    docker run -t --rm \
-           --mount source=`pwd`,type=bind,target=/home/uxbox/uxbox \
-           --mount source=${HOME}/.m2,type=bind,target=/home/uxbox/.m2 \
-           -w /home/uxbox/uxbox/exporter \
+           -w /home/uxbox/uxbox/$1 \
            $IMAGE ./scripts/build.sh
 }
 
+function build-frontend {
+    build "frontend";
+}
+
+function build-exporter {
+    build "exporter";
+}
+
 function build-backend {
-    rm -rf ./backend/target/dist
-    mkdir -p ./backend/target/dist
+    build "backend";
+}
 
-    rsync -ar \
-          --exclude="/tests*" \
-          --exclude="/resources/public/media" \
-          --exclude="/file-uploads" \
-          --exclude="/target" \
-          --exclude="/scripts" \
-          --exclude="/.*" \
-          ./backend/ ./backend/target/dist/
+function build-bundle {
 
-    rsync -ar \
-          ./common/ ./backend/target/dist/common/
+    build "frontend";
+    build "exporter";
+    build "backend";
+
+    rm -rf ./bundle
+    mkdir -p ./bundle
+    mv ./frontend/target/dist ./bundle/frontend
+    mv ./backend/target/dist ./bundle/backend
+    mv ./exporter/target ./bundle/exporter
+
+    NAME="uxbox-$(date '+%Y.%m.%d-%H%M')"
+
+    pushd bundle/
+    tar -cvf ../$NAME.tar *;
+    popd
+
+    xz -vez4f -T4 $NAME.tar
 }
 
 function log-devenv {
     docker-compose -p uxboxdev -f docker/devenv/docker-compose.yaml logs -f --tail=50
 }
 
+function build-testenv {
+    local BUNDLE_FILE=$1;
+    local BUNDLE_FILE_PATH=`readlink -f $BUNDLE_FILE`;
+
+    echo "Building testenv with bundle: $BUNDLE_FILE_PATH."
+
+    if [ ! -f $BUNDLE_FILE ]; then
+        echo "File $BUNDLE_FILE does not exists."
+    fi
+
+    rm -rf ./docker/testenv/bundle;
+    mkdir -p ./docker/testenv/bundle;
+
+    pushd ./docker/testenv/bundle;
+    tar xvf $BUNDLE_FILE_PATH;
+    popd
+
+    pushd ./docker/testenv;
+    docker-compose -p uxbox-testenv -f ./docker-compose.yaml build
+    popd
+}
+
+function start-testenv {
+    pushd ./docker/testenv;
+    docker-compose -p uxbox-testenv -f ./docker-compose.yaml up
+    popd
+}
+
 function usage {
     echo "UXBOX build & release manager v$REV"
     echo "USAGE: $0 OPTION"
@@ -131,6 +163,18 @@ case $1 in
         log-devenv ${@:2}
         ;;
 
+
+    # Test Env
+    start-testenv)
+        start-testenv
+        ;;
+
+    build-testenv)
+        build-testenv ${@:2}
+        ;;
+
+
+
     ## testin related commands
 
     # run-all-tests)
@@ -156,6 +200,10 @@ case $1 in
         build-exporter
         ;;
 
+    build-bundle)
+        build-bundle
+        ;;
+
     *)
         usage
         ;;