refactor: reorganize with some changes the docker images and manage.sh

.gitignore

@@ -9,6 +9,7 @@ pom.xml
 .repl
 .nrepl-port
 node_modules
+/backend/.cpcache
 /backend/target/
 /backend/resources/media
 /backend/dist/
@@ -23,6 +24,8 @@ node_modules
 /frontend/resources/public/js
 /frontend/resources/public/view
 /frontend/resources/public/index.html
+/docker/release.frontend/dist
+/docker/release.backend/dist
 /media
 /deploy
 /web

docker-compose.yml

@@ -1,85 +0,0 @@
-version: '2'
-
-volumes:
-  frontend_data:
-  backend_data:
-  db_data:
-
-services:
-    uxdb:
-        image: postgres:latest
-        container_name: uxdb
-        restart: always
-        ports:
-            - "5432:5432"
-        environment:
-            - "POSTGRES_DB=uxbox"
-            - "POSTGRES_USER=uxbox"
-            - "POSTGRES_PASSWORD=uxbox_postgres_password"
-        volumes:
-            - db_data:/var/lib/postgresql/data
-            - /etc/localtime:/etc/localtime:ro
-            - /etc/timezone:/etc/timezone:ro
-
-    uxbackend:
-        build: ./backend
-        #image: monogramm/uxbox:backend
-        #image: uxbox-backend:beta
-        container_name: uxbackend
-        #restart: always
-        depends_on:
-            - uxdb
-        ports:
-            - 6060:6060
-        environment:
-            # HTTP setup
-            - UXBOX_HTTP_SERVER_DEBUG=false
-            - UXBOX_MEDIA_URI="/media/"
-            #- UXBOX_MEDIA_URI="http://uxbox.io/media/"
-            - UXBOX_MEDIA_DIRECTORY="resources/public/media"
-            - UXBOX_ASSETS_URI="/static/"
-            #- UXBOX_ASSETS_URI="http://uxbox.io/static/"
-            - UXBOX_ASSETS_DIRECTORY="resources/public/static"
-            # Database setup
-            - UXBOX_DATABASE_USERNAME="uxbox"
-            - UXBOX_DATABASE_PASSWORD="uxbox_postgres_password"
-            - UXBOX_DATABASE_NAME="uxbox"
-            - UXBOX_DATABASE_SERVER="uxdb"
-            - UXBOX_DATABASE_PORT=5432
-            # Mail setup
-            - UXBOX_EMAIL_REPLY_TO="no-reply@uxbox.io"
-            - UXBOX_EMAIL_FROM="no-reply@uxbox.io"
-            # STMP setup
-            - UXBOX_SMTP_HOST="localhost"
-            - UXBOX_SMTP_PORT=25
-            - UXBOX_SMTP_USER="uxbox"
-            - UXBOX_SMTP_PASSWORD="youshouldoverwritethiswithsomethingelse"
-            - UXBOX_SMTP_SSL=false
-            - UXBOX_SMTP_TLS=false
-            - UXBOX_SMTP_ENABLED=false
-            # Security setup
-            - UXBOX_SECRET=""
-        volumes:
-            - /srv/uxbox/backend/resources:/srv/uxbox/resources
-            - /etc/localtime:/etc/localtime:ro
-            - /etc/timezone:/etc/timezone:ro
-
-    uxfrontend:
-        build: ./frontend
-        #image: monogramm/uxbox:frontend
-        #image: uxbox-frontend:beta
-        container_name: uxfrontend
-        #hostname: uxbox.io
-        #restart: always
-        depends_on:
-            - uxbackend
-        ports:
-            - 80:80
-        environment:
-            - UXBOX_CONFIG_URL="http://localhost:6060/api/"
-            #- UXBOX_CONFIG_URL="http://uxbox.io/api/"
-        volumes:
-            #- /srv/uxbox/frontend/html:/usr/share/nginx/html
-            - /srv/uxbox/backend/resources:/srv/uxbox/resources:ro
-            - /etc/localtime:/etc/localtime:ro
-            - /etc/timezone:/etc/timezone:ro

docker/devenv/Dockerfile

@@ -1,6 +1,13 @@
 FROM ubuntu:xenial
 LABEL maintainer="Andrey Antukh <niwi@niwi.nz>"
 
+ARG EXTERNAL_UID=1000
+
+ENV NODE_VERSION=v10.16.0 \
+    LANG=en_US.UTF-8 \
+    LC_ALL=C.UTF-8 \
+    UXBOX_CONFIG_URL="/api"
+
 RUN set -ex; \
     apt-get update && \
     apt-get install -yq \
@@ -20,28 +27,7 @@ RUN set -ex; \
         webp \
     ; \
     mkdir -p /etc/resolvconf/resolv.conf.d; \
-    echo "nameserver 8.8.8.8" > /etc/resolvconf/resolv.conf.d/tail; \
-    apt-get update -yq; \
-    apt-get install -yq \
-        libbz2-dev liblzma-dev zlib1g-dev libfftw3-dev \
-        libfreetype6-dev libfontconfig1-dev libxt-dev \
-        libexif-dev libjpeg-dev libpng-dev libtiff-dev \
-        libwmf-dev libpango1.0-dev librsvg2-bin librsvg2-dev \
-        libxml2-dev libwebp-dev webp autoconf \
-    ; \
-    git clone https://github.com/ImageMagick/ImageMagick.git imagemagick && \
-    cd imagemagick && \
-    git checkout -f 7.0.8-27 && \
-    ./configure --prefix=/opt/img && \
-    make -j2 && \
-    make install && \
-    cd .. && \
-    rm -rf ./imagemagick
-
-ENV NODE_VERSION=8.15.0 \
-    LANG=en_US.UTF-8 \
-    LC_ALL=C.UTF-8 \
-    UXBOX_CONFIG_URL="/api"
+    echo "nameserver 8.8.8.8" > /etc/resolvconf/resolv.conf.d/tail; 
 
 RUN set -ex; \
 	echo "deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main" >> /etc/apt/sources.list; \
@@ -53,10 +39,10 @@ RUN set -ex; \
         sudo \
         tmux \
         vim \
+        curl \
     ;
 
 COPY files/pg_hba.conf /etc/postgresql/9.6/main/pg_hba.conf
-# COPY files/postgresql.conf /etc/postgresql/9.5/main/postgresql.conf
 COPY files/bashrc /root/.bashrc
 COPY files/vimrc /root/.vimrc
 
@@ -72,24 +58,30 @@ EXPOSE 6060
 EXPOSE 9090
 
 RUN set -ex; \
-	useradd -m -g users -s /bin/bash uxbox; \
+	useradd -m -g users -s /bin/bash -u $EXTERNAL_UID uxbox; \
     passwd uxbox -d; \
     echo "uxbox ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
 
+RUN set -ex; \
+    wget https://download.clojure.org/install/linux-install-1.10.0.442.sh; \
+    chmod +x linux-install-1.10.0.442.sh; \
+    ./linux-install-1.10.0.442.sh; \
+    rm -rf linux-install-1.10.0.442.sh
+
 USER uxbox
 WORKDIR /home/uxbox
 
-COPY files/lein /home/uxbox/.local/bin/lein
 RUN set -ex; \
     git clone https://github.com/creationix/nvm.git .nvm; \
     bash -c "source .nvm/nvm.sh && nvm install $NODE_VERSION"; \
     bash -c "source .nvm/nvm.sh && nvm alias default $NODE_VERSION"; \
-    bash -c "source .nvm/nvm.sh && nvm use default"; \
-    bash -c "/home/uxbox/.local/bin/lein version"
+    bash -c "source .nvm/nvm.sh && nvm use default";
 
 COPY files/bashrc /home/uxbox/.bashrc
 COPY files/vimrc /home/uxbox/.vimrc
-COPY files/start.sh /home/uxbox/.start.sh
+COPY files/start.sh /home/uxbox/start-tmux.sh
 COPY files/tmux.conf /home/uxbox/.tmux.conf
+COPY files/entrypoint.sh /home/uxbox/
 
-CMD /home/uxbox/.start.sh
+ENTRYPOINT ["bash", "/home/uxbox/entrypoint.sh"]
+CMD ["/home/uxbox/start-tmux.sh"]

docker/devenv/files/build-release.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+source ~/.bashrc
+
+echo `env`
+
+cd /home/uxbox/uxbox/frontend
+npm install || exit 1;
+npm run dist
+# TODO: WIP

docker/devenv/files/entrypoint.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -e
+echo "UXBOX Docker entrypoint initialization..."
+
+exec "$@"

docker/devenv/files/start.sh

@@ -5,8 +5,7 @@ tmux -2 new-session -d -s uxbox
 tmux new-window -t uxbox:1 -n 'figwheel'
 tmux select-window -t uxbox:1
 tmux send-keys -t uxbox 'cd uxbox/frontend' enter C-l
-tmux send-keys -t uxbox "sed -i -e 's/\"uxbox.config.url\" \".*\"/\"uxbox.config.url\" \"${UXBOX_CONFIG_URL}\"/g' figwheel.clj" enter C-l
-tmux send-keys -t uxbox 'npm run figwheel' enter
+tmux send-keys -t uxbox 'npm run start' enter
 
 tmux new-window -t uxbox:2 -n 'backend'
 tmux select-window -t uxbox:2

docker/docker-compose.yml

@@ -0,0 +1,83 @@
+version: "3"
+
+volumes:
+  backend_m2:
+  backend_data:
+  db_data:
+
+networks:
+  default:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 172.177.57.0/24
+
+services:
+  uxdb:
+    image: postgres:latest
+    container_name: uxdb
+    restart: always
+    ports:
+      - "5432:5432"
+    environment:
+      - POSTGRES_INITDB_ARGS="--data-checksums"
+      - POSTGRES_DB=uxbox
+      - POSTGRES_USER=uxbox
+      - POSTGRES_PASSWORD=uxbox_postgres_password
+    volumes:
+      - db_data:/var/lib/postgresql/data
+      # - /etc/localtime:/etc/localtime:ro
+      # - /etc/timezone:/etc/timezone:ro
+
+  uxbackend:
+    image: uxbox-backend:latest
+    container_name: uxbackend
+    restart: always
+    depends_on:
+      - uxdb
+    ports:
+      - 6060:6060
+    volumes:
+      - backend_data:/srv/uxbox/resources/public
+      - backend_m2:/root/.m2
+      # - /etc/localtime:/etc/localtime:ro
+      # - /etc/timezone:/etc/timezone:ro
+
+    environment:
+      # HTTP setup
+      - UXBOX_HTTP_SERVER_DEBUG=false
+      - UXBOX_MEDIA_URI="/media/"
+      - UXBOX_ASSETS_URI="/static/"
+      # Database setup
+      - UXBOX_DATABASE_USERNAME="uxbox"
+      - UXBOX_DATABASE_PASSWORD="uxbox_postgres_password"
+      - UXBOX_DATABASE_NAME="uxbox"
+      - UXBOX_DATABASE_SERVER="uxdb"
+      - UXBOX_DATABASE_PORT=5432
+      # Mail setup
+      - UXBOX_EMAIL_REPLY_TO="no-reply@uxbox.io"
+      - UXBOX_EMAIL_FROM="no-reply@uxbox.io"
+      # STMP setup
+      - UXBOX_SMTP_HOST="localhost"
+      - UXBOX_SMTP_PORT=25
+      - UXBOX_SMTP_USER="uxbox"
+      - UXBOX_SMTP_PASSWORD="youshouldoverwritethiswithsomethingelse"
+      - UXBOX_SMTP_SSL=false
+      - UXBOX_SMTP_TLS=false
+      - UXBOX_SMTP_ENABLED=false
+      # Security setup
+      - UXBOX_SECRET="foobar"
+
+  uxfrontend:
+    image: uxbox-frontend:latest
+    container_name: uxfrontend
+    #hostname: uxbox.io
+    restart: always
+    depends_on:
+      - uxbackend
+    ports:
+      - 80:80
+    volumes:
+      - backend_data:/srv/uxbox/
+      # - /etc/localtime:/etc/localtime:ro
+      # - /etc/timezone:/etc/timezone:ro

docker/release.backend/Dockerfile

@@ -0,0 +1,34 @@
+FROM openjdk:8-jre
+
+LABEL maintainer="Monogramm Maintainers <opensource at monogramm dot io>"
+
+ENV LANG=en_US.UTF-8 \
+    LC_ALL=C.UTF-8
+
+RUN set -ex; \
+    wget https://download.clojure.org/install/linux-install-1.10.0.442.sh; \
+    chmod +x linux-install-1.10.0.442.sh; \
+    ./linux-install-1.10.0.442.sh; \
+    rm -rf linux-install-1.10.0.442.sh
+
+RUN set -ex; \
+    apt-get update -yq &&  \
+    apt-get install -yq \
+        git \
+        imagemagick \
+        webp
+
+COPY ./entrypoint.sh /entrypoint.sh
+COPY ./dist /srv/uxbox
+
+RUN set -ex; \
+    chmod 755 /entrypoint.sh; \
+    mkdir -p /srv/uxbox/resources/media
+
+VOLUME /srv/uxbox/resources/public
+WORKDIR /srv/uxbox/
+
+EXPOSE 6060
+
+ENTRYPOINT ["sh", "/entrypoint.sh"]
+CMD ["clojure", "-m", "uxbox.main"]

docker/release.backend/entrypoint.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -e
+echo "Setting up UXBOX Backend..."
+
+exec "$@"

docker/release.frontend/Dockerfile

@@ -0,0 +1,13 @@
+# Once application has been built, prepare production image
+FROM nginx:alpine
+
+LABEL maintainer="Monogramm Maintainers <opensource at monogramm dot io>"
+
+ENV LANG=en_US.UTF-8 \
+    LC_ALL=C.UTF-8
+
+# Copy built app to www root
+COPY ./dist /usr/share/nginx/html
+
+# NGINX configurations
+COPY ./nginx/conf.d /etc/nginx/conf.d

docker/release.frontend/nginx/README.md

@@ -0,0 +1,40 @@
+# Setting up NGNIX
+
+You will need to complete the following tasks to setup your dockerized proxy server:
+
+1. Include/Create SSL keys
+2. Alter your backend upstream
+3. Confirm your backend's path
+
+## Include/Create SSL Keys
+
+Have your key and csr in the nginx/keys directory as server.key and server.crt. These are copied into the docker image on build and used to serve your website or proxy your services.
+
+### Generate your own self signed certificate
+```bash
+openssl req \
+       -newkey rsa:2048 -nodes -keyout nginx/keys/server.key \
+       -x509 -out nginx/keys/server.crt
+```
+
+This command from your project root will create the keys needed to start docker with self signed certificates. Note that if you are going to deploy this site for production you will want to replace these and rebuild your image with valid (purchased) SSL certificates. All the fields are optional. Do not set any challenge passwords.
+
+If you want validated certificates but are not looking to purchase them; then checkout [Let's Encrypt](https://letsencrypt.org) which is a free SSL certification service.
+
+## Alter your backend upstream
+
+The upstream is a block used to load balance different destinations important to your proxy. In this example the upstream is used to proxy requests to your backend without worrying about XSS configurations.
+
+We have preloaded some examples of what this looks like in the `nginx/conf.d/default.conf` file. You can certainly only specify one server in the block if that is your only server.
+
+## Confirm your backend's path
+
+Assuming your website uses a backend collection of APIs, you can setup your nginx service to reverse proxy to them avoiding any XSS configuration needs. The provided default.conf includes a `/api/` location block to serve as an example. You can replace api in `/api/` with any path you want to have forwarded to your backend.
+
+There is only one setting you need to adjust in this block and that is the `proxy_cookie_domain`. Assuming you have a production domain you would change `my.uxbox.com` to be your domain. If you do not have a production domain it is safe to leave this as is or delete.
+
+## Extending the configuration
+
+You can include more servers or configuration settings by adding any named file in `nginx/conf.d`. These files are automatically consumed by nginx on startup.
+
+[Visit NGINX's beginnner's guide](http://nginx.org/en/docs/beginners_guide.html) for additional help.

docker/release.frontend/nginx/conf.d/default.conf

@@ -0,0 +1,41 @@
+server {
+  listen 80 default_server;
+  listen [::]:80 default_server;
+
+  server_name _;
+
+  # Frontend
+  location / {
+    root /usr/share/nginx/html/;
+
+    try_files $uri /index.html;
+    gzip on;
+    gzip_types text/css text/javascript application/x-javascript application/javascript application/json;
+
+    add_header Cache-Control "max-age=15552000" always;
+  }
+
+  location /view {
+    alias /usr/share/nginx/html/view;
+  }
+
+  location /media {
+    alias /srv/uxbox/media;
+  }
+
+  location /static {
+    alias /srv/uxbox/static;
+  }
+
+  # Backend
+  location /api/ {
+    proxy_pass http://uxbackend:6060/api/;
+    proxy_http_version 1.1;
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection 'upgrade';
+    proxy_set_header Host $host;
+    proxy_cache_bypass $http_upgrade;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+  }
+}

docker/release.frontend/nginx/conf.d/misc.conf

@@ -0,0 +1,3 @@
+log_format gzip '[$time_local] ' '"$request" $status $bytes_sent';
+access_log /dev/stdout;
+charset utf-8;

manage.sh

@@ -4,42 +4,41 @@ set -e
 REV=`git rev-parse --short HEAD`
 IMGNAME="uxbox"
 
-function kill_container {
+function kill-container {
     echo "Cleaning development container $IMGNAME:$REV..."
-    if $(sudo docker ps | grep -q $IMGNAME); then
-        sudo docker ps | grep $IMGNAME | awk '{print $1}' | xargs --no-run-if-empty sudo docker kill
+    if $(docker ps | grep -q $IMGNAME); then
+        docker ps | grep $IMGNAME | awk '{print $1}' | xargs --no-run-if-empty docker kill
     fi
-    if $(sudo docker ps -a | grep -q $IMGNAME); then
-        sudo docker ps -a | grep $IMGNAME | awk '{print $1}' | xargs --no-run-if-empty sudo docker rm
+    if $(docker ps -a | grep -q $IMGNAME); then
+        docker ps -a | grep $IMGNAME | awk '{print $1}' | xargs --no-run-if-empty docker rm
     fi
 }
 
-function remove_image {
+function remove-image {
     echo "Clean old development image $IMGNAME..."
-    sudo docker images | grep $IMGNAME | awk '{print $3}' | xargs --no-run-if-empty sudo docker rmi
+    docker images | grep $IMGNAME | awk '{print $3}' | xargs --no-run-if-empty docker rmi
 }
 
-function build_image {
-    kill_container
-    remove_image
+function build-devenv {
+    kill-container
     echo "Building development image $IMGNAME:$REV..."
-    sudo docker build --rm=true -t $IMGNAME:$REV docker/
+    docker build --rm=true -t $IMGNAME:$REV -t $IMGNAME:latest docker/devenv
 }
 
-function run_image {
-    kill_container
+function run-devenv {
+    kill-container
 
-    if ! $(sudo docker images | grep $IMGNAME | grep -q $REV); then
-        build_image
+    if ! $(docker images | grep $IMGNAME | grep -q $REV); then
+        build-devenv
     fi
 
     mkdir -p $HOME/.m2
     rm -rf ./frontend/node_modules
 
-    CONTAINER=$IMGNAME:$REV
+    CONTAINER=$IMGNAME:latest
 
     echo "Running development image $CONTAINER..."
-    sudo docker run -ti \
+    docker run --rm -ti \
          -v `pwd`:/home/uxbox/uxbox  \
          -v $HOME/.m2:/home/uxbox/.m2 \
          -v $HOME/.gitconfig:/home/uxbox/.gitconfig \
@@ -47,59 +46,40 @@ function run_image {
          $CONTAINER
 }
 
-function test {
-    kill_container
-
-    echo "TODO Testing backend (require running postgresql)..."
-    cd ./backend
-    #lein test
-    cd ..
-
-    echo "Testing frontend..."
-    cd ./frontend
-    ./scripts/build-tests
-    node ./out/tests.js
-    cd ..
+function build-release-frontend-local {
+    docker run -ti --rm \
+           -w /home/uxbox/uxbox/frontend \
+           -v `pwd`:/home/uxbox/uxbox  \
+           -v $HOME/.m2:/home/uxbox/.m2 \
+           -e UXBOX_API_URL="/api" \
+           -e UXBOX_VIEW_URL="/view" \
+           $IMGNAME:latest ./scripts/build-release.sh
 }
 
-function release_local {
-    cd frontend
-    echo "Building frontend release..."
-    rm -rf ./dist
-    rm -rf ./node_modules
-    npm install
-    npm run dist
-    ./scripts/dist-main
-    ./scripts/dist-view
-    ./scripts/dist-worker
-    echo "Frontend release generated in $(pwd)/dist"
-
-    cd ../backend
-    echo "Building backend release..."
-    rm -rf ./dist
-    ./scripts/dist.sh
-    echo "Backend release generated in $(pwd)/dist"
-
-    cd ..
+function build-release-frontend {
+    build-release-frontend-local || exit 1;
+    rm -rf docker/release.frontend/dist || exit 1;
+    cp -r frontend/dist docker/release.frontend/ || exit 1;
+    docker build --rm=true -t ${IMGNAME}-frontend:$REV -t ${IMGNAME}-frontend:latest docker/release.frontend/
+    rm -rf docker/release.frontend/dist || exit 1;
 }
 
-function release_image {
-    echo "Building frontend release..."
-    rm -rf ./frontend/dist ./frontend/node_modules ./frontend/dist
-    sudo docker build --rm=true -t ${IMGNAME}_frontend:$REV frontend/
-    echo "Frontend release image generated"
-
-    echo "Building backend release..."
-    rm -rf ./backend/dist
-    sudo docker build --rm=true -t ${IMGNAME}_backend:$REV backend/
-    echo "Backend release image generated"
+function build-release-backend {
+    rm -rf docker/release.backend/dist || exit 1;
+    rsync -avr \
+          --exclude="/test" \
+          --exclude="/resources/public/media" \
+          --exclude="/target" \
+          --exclude="/scripts" \
+          --exclude="/.*" \
+          backend/ docker/release.backend/dist/;
+    docker build --rm=true -t ${IMGNAME}-backend:$REV -t ${IMGNAME}-backend:latest docker/release.backend/
+    rm -rf docker/release.backend/dist || exit 1;
 }
 
-function run_release {
-    kill_container
-
-    echo "Running production images..."
-    sudo docker-compose up -d
+function build-release {
+    build-release-frontend || exit 1;
+    build-release-backend || exit 1;
 }
 
 function usage {
@@ -107,30 +87,21 @@ function usage {
     echo "USAGE: $0 [ clean | build | run | test | release-local | release-docker | run-release ]"
     echo "Options:"
     echo "- clean           Stop and clean up docker containers"
-    echo "- build           Build docker container for development with tmux"
-    echo "- run             Run (and build if necessary) development container (frontend at localhost:3449, backend at localhost:6060)"
-    echo "- test            Execute frontend unit tests (backend unit tests no available yet)"
-    echo "- release-local   Build a 'production ready' release"
-    echo "- release-docker  Build a 'production ready' docker container"
-    echo "- run-release     Run a 'production ready' docker-compose environment (frontend at localhost:80, backend at localhost:6060)"
+    echo "- build-devenv    Build docker container for development with tmux"
+    echo "- run-devenv      Run (and build if necessary) development container (frontend at localhost:3449, backend at localhost:6060)"
+    echo "- build-release   Build a 'production ready' release docker images"
 }
 
 case $1 in
     clean)
-        kill_container
-        remove_image
+        kill-container
+        remove-image
         ;;
-    build)
-        build_image
+    build-devenv)
+        build-devenv
         ;;
-    run)
-        run_image
-        ;;
-    test)
-        test
-        ;;
-    release-local)
-        release_local
+    run-devenv)
+        run-devenv
         ;;
     release-docker)
         release_image
@@ -138,6 +109,15 @@ case $1 in
     run-release)
         run_release
         ;;
+    build-release)
+        build-release
+        ;;
+    build-release-frontend)
+        build-release-frontend
+        ;;
+    build-release-backend)
+        build-release-backend
+        ;;
     *)
         usage
         ;;