Merge remote-tracking branch 'origin/staging' into develop

.gitignore

@@ -15,6 +15,7 @@ node_modules
 /backend/target/
 /backend/resources/public/media
 /backend/resources/public/assets
+/backend/assets/
 /backend/dist/
 /backend/logs/
 /backend/-

backend/scripts/build

@@ -0,0 +1,81 @@
+#!/usr/bin/env bb
+
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; This Source Code Form is "Incompatible With Secondary Licenses", as
+;; defined by the Mozilla Public License, v. 2.0.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns build
+  (:require
+   [clojure.string :as str]
+   [clojure.java.io :as io]
+   [clojure.pprint :refer [pprint]]
+   [babashka.fs :as fs]
+   [babashka.process :refer [$ check]]))
+
+(defn split-cp
+  [data]
+  (str/split data #":"))
+
+(def classpath
+  (->> ($ clojure -Spath)
+       (check)
+       (:out)
+       (slurp)
+       (split-cp)
+       (map str/trim)))
+
+(def classpath-jars
+  (let [xfm (filter #(str/ends-with? % ".jar"))]
+    (into #{} xfm classpath)))
+
+(def classpath-paths
+  (let [xfm (comp (remove #(str/ends-with? % ".jar"))
+                  (filter #(.isDirectory (io/file %))))]
+    (into #{} xfm classpath)))
+
+(def version
+  (or (first *command-line-args*) "%version%"))
+
+;; Clean previous dist
+(-> ($ rm -rf "./target/dist") check)
+
+;; Create a new dist
+(-> ($ mkdir -p "./target/dist/deps") check)
+
+;; Copy all jar deps into dist
+(run! (fn [item] (-> ($ cp ~item "./target/dist/deps/") check)) classpath-jars)
+
+;; Create the application jar
+(spit "./target/dist/version.txt" version)
+(-> ($ jar cvf "./target/dist/deps/app.jar" -C ~(first classpath-paths) ".") check)
+(-> ($ jar uvf "./target/dist/deps/app.jar" -C "./target/dist" "version.txt") check)
+(run! (fn [item]
+        (-> ($ jar uvf "./target/dist/deps/app.jar" -C ~item ".")  check))
+      (rest classpath-paths))
+
+;; Copy logging configuration
+(-> ($ cp "./resources/log4j2.xml" "./target/dist/") check)
+
+;; Create classpath file
+(let [jars (->> (into ["app.jar"] classpath-jars)
+                (map fs/file-name)
+                (map #(fs/path "deps" %))
+                (map str))]
+  (spit "./target/dist/classpath" (str/join ":" jars)))
+
+;; Copy run script template
+(-> ($ cp "./scripts/run.template.sh" "./target/dist/run.sh") check)
+
+;; Copy run script template
+(-> ($ cp "./scripts/manage.template.sh" "./target/dist/manage.sh") check)
+
+;; Add exec permisions to scripts.
+(-> ($ chmod +x "./target/dist/run.sh") check)
+(-> ($ chmod +x "./target/dist/manage.sh") check)
+
+nil

backend/scripts/build.sh

@@ -1,75 +0,0 @@
-#!/usr/bin/env bash
-
-CLASSPATH=`(clojure -Spath)`
-NEWCP="./main:./common"
-
-rm -rf ./target/dist
-mkdir -p ./target/dist/deps
-
-for item in $(echo $CLASSPATH | tr ":" "\n"); do
-    if [ "${item: -4}" == ".jar" ]; then
-        cp $item ./target/dist/deps/;
-        BN="$(basename -- $item)"
-        NEWCP+=":./deps/$BN"
-    fi
-done
-
-cp ./resources/log4j2.xml ./target/dist/log4j2.xml
-cp -r ./src ./target/dist/main
-cp -r ./resources/emails ./target/dist/main/
-cp -r ./resources/error-report.tmpl ./target/dist/main/
-cp -r ../common ./target/dist/common
-
-echo $NEWCP > ./target/dist/classpath;
-
-tee -a ./target/dist/run.sh  >> /dev/null <<EOF
-#!/usr/bin/env bash
-CP="$NEWCP"
-
-set +e
-JAVA_CMD=\$(type -p java)
-
-set -e
-if [[ ! -n "\$JAVA_CMD" ]]; then
-  if [[ -n "\$JAVA_HOME" ]] && [[ -x "\$JAVA_HOME/bin/java" ]]; then
-    JAVA_CMD="\$JAVA_HOME/bin/java"
-  else
-    >&2 echo "Couldn't find 'java'. Please set JAVA_HOME."
-    exit 1
-  fi
-fi
-
-if [ -f ./environ ]; then
-   source ./environ
-fi
-
-set -x
-exec \$JAVA_CMD \$JVM_OPTS -classpath \$CP -Dlog4j2.configurationFile=./log4j2.xml "\$@" clojure.main -m app.main
-EOF
-
-tee -a ./target/dist/manage.sh  >> /dev/null <<EOF
-#!/usr/bin/env bash
-CP="$NEWCP"
-
-set +e
-JAVA_CMD=\$(type -p java)
-
-set -e
-if [[ ! -n "\$JAVA_CMD" ]]; then
-  if [[ -n "\$JAVA_HOME" ]] && [[ -x "\$JAVA_HOME/bin/java" ]]; then
-    JAVA_CMD="\$JAVA_HOME/bin/java"
-  else
-    >&2 echo "Couldn't find 'java'. Please set JAVA_HOME."
-    exit 1
-  fi
-fi
-
-if [ -f ./environ ]; then
-   source ./environ
-fi
-
-exec \$JAVA_CMD \$JVM_OPTS -classpath \$CP -Dlog4j2.configurationFile=./log4j2.xml clojure.main -m app.cli.manage "\$@"
-EOF
-
-chmod +x ./target/dist/run.sh
-chmod +x ./target/dist/manage.sh

backend/scripts/import-generic-collections.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-clojure -Adev -m app.cli.collimp $@
-

backend/scripts/manage.template.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set +e
+JAVA_CMD=$(type -p java)
+
+set -e
+if [[ ! -n "$JAVA_CMD" ]]; then
+  if [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]]; then
+    JAVA_CMD="$JAVA_HOME/bin/java"
+  else
+    >&2 echo "Couldn't find 'java'. Please set JAVA_HOME."
+    exit 1
+  fi
+fi
+
+if [ -f ./environ ]; then
+   source ./environ
+fi
+
+exec $JAVA_CMD $JVM_OPTS -classpath $(cat classpath) -Dlog4j2.configurationFile=./log4j2.xml clojure.main -m app.cli.manage "\$@"

backend/scripts/prepare-release.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-
-if [ "$#" -e 0 ]; then
-    echo "Expecting parameters: 1=path to backend; 2=destination directory"
-    exit 1
-fi
-
-rm -rf $2 || exit 1;
-
-rsync -avr \
-      --exclude="/test" \
-      --exclude="/resources/public/media" \
-      --exclude="/target" \
-      --exclude="/scripts" \
-      --exclude="/.*" \
-      $1 $2;

backend/scripts/psql.sh

@@ -1,2 +0,0 @@
-#!/usr/bin/env bash
-PGPASSWORD=$PENPOT_DATABASE_PASSWORD psql $PENPOT_DATABASE_URI -U $PENPOT_DATABASE_USERNAME

backend/scripts/run-tests-in-docker.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-set -xe
-clojure -Adev -m app.tests.main;

backend/scripts/run.template.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set +e
+JAVA_CMD=$(type -p java)
+
+set -e
+if [[ ! -n "$JAVA_CMD" ]]; then
+  if [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]]; then
+    JAVA_CMD="$JAVA_HOME/bin/java"
+  else
+    >&2 echo "Couldn't find 'java'. Please set JAVA_HOME."
+    exit 1
+  fi
+fi
+
+if [ -f ./environ ]; then
+   source ./environ
+fi
+
+set -x
+exec $JAVA_CMD $JVM_OPTS -classpath "$(cat classpath)" -Dlog4j2.configurationFile=./log4j2.xml "$@" clojure.main -m app.main

backend/scripts/smtpd.sh

@@ -1,2 +0,0 @@
-#!/usr/bin/env bash
-python -m smtpd -n -c DebuggingServer localhost:25

backend/scripts/tests.sh

@@ -1,2 +0,0 @@
-#!/usr/bin/env sh
-exec clojure -M:dev:tests "$@"

backend/src/app/config.clj

@@ -16,6 +16,7 @@
    [app.util.time :as dt]
    [clojure.core :as c]
    [clojure.pprint :as pprint]
+   [clojure.java.io :as io]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
    [environ.core :refer [env]]))
@@ -50,7 +51,7 @@
 
    :storage-backend :fs
 
-   :storage-fs-directory "resources/public/assets"
+   :storage-fs-directory "assets"
    :storage-s3-region :eu-central-1
    :storage-s3-bucket "penpot-devenv-assets-pre"
 
@@ -246,15 +247,17 @@
      {}
      env)))
 
-
 (defn- read-config
   []
   (->> (read-env "penpot")
        (merge defaults)
        (us/conform ::config)))
 
-(def version (v/parse "%version%"))
-(def config  (atom (read-config)))
+(def version (v/parse (or (some-> (io/resource "version.txt")
+                                  (slurp)
+                                  (str/trim))
+                          "%version%")))
+(def config (atom (read-config)))
 
 (def deletion-delay
   (dt/duration {:days 7}))

docker/devenv/Dockerfile

@@ -6,7 +6,7 @@ ARG DEBIAN_FRONTEND=noninteractive
 ENV NODE_VERSION=v14.16.0 \
     CLOJURE_VERSION=1.10.3.814 \
     CLJKONDO_VERSION=2021.03.22 \
-    BABASHKA_VERSION=0.3.0 \
+    BABASHKA_VERSION=0.3.1 \
     LANG=en_US.UTF-8 \
     LC_ALL=en_US.UTF-8
 
@@ -151,6 +151,7 @@ EXPOSE 3449
 EXPOSE 6060
 EXPOSE 9090
 
+COPY files/nginx.conf /etc/nginx/nginx.conf
 COPY files/phantomjs-mock /usr/bin/phantomjs
 
 COPY files/bashrc         /root/.bashrc

docker/devenv/docker-compose.yaml

@@ -27,7 +27,6 @@ services:
     volumes:
       - "user_data:/home/penpot/"
       - "${PWD}:/home/penpot/penpot"
-      - ./files/nginx.conf:/etc/nginx/nginx.conf
 
     ports:
       - 3447:3447

docker/devenv/files/nginx.conf

@@ -91,7 +91,7 @@ http {
 
         location /internal/assets {
             internal;
-            alias /home/penpot/penpot/backend/resources/public/assets;
+            alias /home/penpot/penpot/backend/assets;
             add_header x-internal-redirect "$upstream_http_x_accel_redirect";
         }
 

docker/devenv/files/start-tmux.sh

@@ -26,7 +26,9 @@ tmux send-keys -t penpot 'npx shadow-cljs watch main' enter
 tmux new-window -t penpot:2 -n 'exporter'
 tmux select-window -t penpot:2
 tmux send-keys -t penpot 'cd penpot/exporter' enter C-l
+tmux send-keys -t penpot 'rm -f target/app.js*' enter C-l
 tmux send-keys -t penpot 'npx shadow-cljs watch main' enter
+
 tmux split-window -v
 tmux send-keys -t penpot 'cd penpot/exporter' enter C-l
 tmux send-keys -t penpot './scripts/wait-and-start.sh' enter

exporter/scripts/build

@@ -1,6 +1,5 @@
 #!/usr/bin/env bash
 
-source ~/.bashrc
 set -ex
 
 yarn install

exporter/scripts/wait-and-start.sh

@@ -1,16 +1,5 @@
 #!/usr/bin/env bash
 
-set -e
-
-wait_file() {
-  local file="$1"; shift
-  local wait_seconds="${1:-10}"; shift # 10 seconds as default timeout
-
-  until test $((wait_seconds--)) -eq 0 -o -f "$file" ; do sleep 1; done
-
-  ((++wait_seconds))
-}
-
-wait_file "target/app.js" 120 && {
-  node target/app.js
-}
+bb -i '(babashka.wait/wait-for-port "localhost" 9630)';
+bb -i '(babashka.wait/wait-for-path "target/app.js")';
+node target/app.js

frontend/scripts/build

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -ex
+
+CURRENT_VERSION=$1;
+CURRENT_HASH=$(git rev-parse --short HEAD);
+EXTRA_PARAMS=$SHADOWCLJS_EXTRA_PARAMS;
+
+yarn install || exit 1;
+npx gulp clean || exit 1;
+npx shadow-cljs release main --config-merge "{:release-version \"${CURRENT_HASH}\"}" $EXTRA_PARAMS || exit 1
+npx gulp build || exit 1;
+npx gulp dist:clean || exit 1;
+npx gulp dist:copy || exit 1;
+
+sed -i -re "s/\%version\%/$CURRENT_VERSION/g" ./target/dist/index.html;
+

frontend/scripts/build-and-run-tests.sh

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-source ~/.bashrc
-
-set -ex;
-
-yarn install
-clojure -Adev tools.clj build:tests
-node ./target/tests/main

frontend/scripts/build.sh

@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-
-source ~/.bashrc
-
-set -ex
-
-if [ -z "${TAG}" ]; then
-    export TAG=$(git log -n 1 --pretty=format:%H -- ./);
-fi
-
-yarn install
-
-export NODE_ENV=production;
-
-# Clean the output directory
-npx gulp clean || exit 1;
-
-npx shadow-cljs release main --config-merge "{:release-version \"${TAG}\"}" $SHADOWCLJS_EXTRA_PARAMS
-npx gulp build || exit 1;
-npx gulp dist:clean || exit 1;
-npx gulp dist:copy || exit 1;

frontend/src/app/main/ui/workspace/viewport/hooks.cljs

@@ -96,10 +96,11 @@
          (mf/deps page-id)
          (fn [point]
            (let [rect (gsh/center->rect point 8 8)]
-             (uw/ask! {:cmd :selection/query
-                       :page-id page-id
-                       :rect rect
-                       :include-frames? true}))))
+             (uw/ask-buffered!
+              {:cmd :selection/query
+               :page-id page-id
+               :rect rect
+               :include-frames? true}))))
 
         ;; We use ref so we don't recreate the stream on a change
         transform-ref (mf/use-ref nil)

frontend/src/app/main/worker.cljs

@@ -25,3 +25,7 @@
 (defn ask!
   [message]
   (uw/ask! instance message))
+
+(defn ask-buffered!
+  [message]
+  (uw/ask-buffered! instance message))

frontend/src/app/util/worker.cljs

@@ -17,39 +17,58 @@
 (declare handle-response)
 (defrecord Worker [instance stream])
 
-(defn ask!
-  [w message]
-  (let [sender-id (uuid/next)
-        data (t/encode {:payload message :sender-id sender-id})
-        instance (:instance w)]
+(defn- send-message! [worker {sender-id :sender-id :as message}]
+  (let [data (t/encode message)
+        instance (:instance worker)]
     (.postMessage instance data)
-    (->> (:stream w)
+    (->> (:stream worker)
          (rx/filter #(= (:reply-to %) sender-id))
-         (rx/map handle-response)
-         (rx/first))))
+         (rx/take 1)
+         (rx/map handle-response))))
+
+(defn ask!
+  [worker message]
+  (send-message!
+   worker
+   {:sender-id (uuid/next)
+    :payload message}))
+
+(defn ask-buffered!
+  [worker message]
+  (send-message!
+   worker
+   {:sender-id (uuid/next)
+    :payload message
+    :buffer? true}))
 
 (defn init
   "Return a initialized webworker instance."
   [path on-error]
-  (let [ins (js/Worker. path)
-        bus (rx/subject)
-        wrk (Worker. ins bus)]
-    (.addEventListener ins "message"
-                       (fn [event]
-                         (let [data (.-data event)
-                               data (t/decode data)]
-                           (if (:error data)
-                             (on-error (:error data))
-                             (rx/push! bus data)))))
-    (.addEventListener ins "error"
-                       (fn [error]
-                         (on-error wrk (.-data error))))
+  (let [instance (js/Worker. path)
+        bus     (rx/subject)
+        worker  (Worker. instance bus)
 
-    wrk))
+        handle-message
+        (fn [event]
+          (let [data (.-data event)
+                data (t/decode data)]
+            (if (:error data)
+              (on-error (:error data))
+              (rx/push! bus data))))
+
+        handle-error
+        (fn [error]
+          (on-error worker (.-data error)))]
+    
+    (.addEventListener instance "message" handle-message)
+    (.addEventListener instance "error" handle-error)
+
+    worker))
 
 (defn- handle-response
-  [{:keys [payload error] :as response}]
-  (if-let [{:keys [data message]} error]
-    (throw (ex-info message data))
-    payload))
+  [{:keys [payload error dropped] :as response}]
+  (when-not dropped
+    (if-let [{:keys [data message]} error]
+      (throw (ex-info message data))
+      payload)))
 

frontend/src/app/worker.cljs

@@ -28,14 +28,23 @@
 ;; --- Messages Handling
 
 (s/def ::cmd keyword?)
+
 (s/def ::payload
   (s/keys :req-un [::cmd]))
 
 (s/def ::sender-id uuid?)
+
+(s/def ::buffer? boolean?)
+
 (s/def ::message
-  (s/keys :req-un [::payload ::sender-id]))
+  (s/keys
+   :req-opt [::buffer?]
+   :req-un [::payload ::sender-id]))
+
+(def buffer (rx/subject))
 
 (defn- handle-message
+  "Process the message and returns to the client"
   [{:keys [sender-id payload] :as message}]
   (us/assert ::message message)
   (try
@@ -68,20 +77,83 @@
                              :message (ex-message e)}}]
         (.postMessage js/self (t/encode message))))))
 
+(defn- drop-message
+  "Sends to the client a notifiction that its messages have been dropped"
+  [{:keys [sender-id payload] :as message}]
+  (us/assert ::message message)
+  (.postMessage js/self (t/encode {:reply-to sender-id
+                                   :dropped true})))
+
+(defn subscribe-buffer-messages
+  "Creates a subscription to process the buffer messages"
+  []
+  (let [empty [{} [] ::clear]]
+    (->> buffer
+
+         ;; We want async processing to not block the main loop
+         (rx/observe-on :async)
+
+         ;; This scan will store the last message per type in `messages`
+         ;; when a previous message is dropped is stored in `dropped`
+         ;; we also store the last message processed in order to detect
+         ;; posible infinite loops
+         (rx/scan
+          (fn [[messages dropped last] message]
+            (let [cmd (get-in message [:payload :cmd])
+
+                  ;; The previous message is dropped
+                  dropped
+                  (cond-> dropped
+                    (contains? messages cmd)
+                    (conj (get messages cmd)))
+
+                  ;; This is the new "head" for its type
+                  messages
+                  (assoc messages cmd message)]
+
+              ;; When a "clear" message is detected we empty the buffer
+              (if (= message ::clear)
+                empty
+                [messages dropped message])))
+
+          empty)
+
+         ;; 1ms debounce, after 1ms without messages will process the buffer
+         (rx/debounce 1)
+
+         (rx/subs (fn [[messages dropped last]]
+                    ;; Send back the dropped messages replies 
+                    (doseq [msg dropped]
+                      (drop-message msg))
+
+                    ;; Process the message
+                    (doseq [msg (vals messages)]
+                      (handle-message msg))
+
+                    ;; After process the buffer we send a clear
+                    (when-not (= last ::clear)
+                      (rx/push! buffer ::clear)))))))
+
+(defonce process-message-sub (subscribe-buffer-messages))
+
 (defn- on-message
   [event]
   (when (nil? (.-source event))
     (let [message (.-data event)
           message (t/decode message)]
-      (handle-message message))))
+      (if (:buffer? message)
+        (rx/push! buffer message)
+        (handle-message message)))))
 
 (.addEventListener js/self "message" on-message)
 
 (defn ^:dev/before-load stop []
+  (rx/-dispose process-message-sub)
   (.removeEventListener js/self "message" on-message))
 
 (defn ^:dev/after-load start []
   []
+  (set! process-message-sub (subscribe-buffer-messages))
   (.addEventListener js/self "message" on-message))
 
 

manage.sh

@@ -71,6 +71,9 @@ function run-devenv {
 }
 
 function build {
+    echo ">> build start: $1"
+    local version=$(print-current-version);
+
     pull-devenv-if-not-exists;
     docker volume create ${DEVENV_PNAME}_user_data;
     docker run -t --rm \
@@ -79,10 +82,28 @@ function build {
            -e EXTERNAL_UID=$CURRENT_USER_ID \
            -e SHADOWCLJS_EXTRA_PARAMS=$SHADOWCLJS_EXTRA_PARAMS \
            -w /home/penpot/penpot/$1 \
-           $DEVENV_IMGNAME:latest sudo -EH -u penpot ./scripts/build.sh
+           $DEVENV_IMGNAME:latest sudo -EH -u penpot ./scripts/build $version
+
+    echo ">> build end: $1"
+}
+
+function put-license-file {
+    local target=$1;
+    tee -a $target/LICENSE  >> /dev/null <<EOF
+This Source Code Form is subject to the terms of the Mozilla Public
+License, v. 2.0. If a copy of the MPL was not distributed with this
+file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+This Source Code Form is "Incompatible With Secondary Licenses", as
+defined by the Mozilla Public License, v. 2.0.
+
+Copyright (c) UXBOX Labs SL
+EOF
 }
 
 function build-app-bundle {
+    echo ">> bundle app start";
+
     local version=$(print-current-version);
     local bundle_dir="./bundle-app";
 
@@ -91,25 +112,28 @@ function build-app-bundle {
 
     rm -rf $bundle_dir
     mkdir -p $bundle_dir;
-    cp -r ./frontend/target/dist $bundle_dir/frontend;
-    cp -r ./backend/target/dist $bundle_dir/backend;
+    mv ./frontend/target/dist $bundle_dir/frontend;
+    mv ./backend/target/dist $bundle_dir/backend;
 
     echo $version > $bundle_dir/version.txt
-
-    sed -i -re "s/\%version\%/$version/g" $bundle_dir/frontend/index.html;
-    sed -i -re "s/\%version\%/$version/g" $bundle_dir/backend/main/app/config.clj;
+    put-license-file $bundle_dir;
+    echo ">> bundle app end";
 }
 
 function build-exporter-bundle {
+    echo ">> bundle exporter start";
     local version=$(print-current-version);
     local bundle_dir="./bundle-exporter";
 
     build "exporter";
 
     rm -rf $bundle_dir;
-    cp -r ./exporter/target $bundle_dir;
+    mv ./exporter/target $bundle_dir;
 
     echo $version > $bundle_dir/version.txt
+    put-license-file $bundle_dir;
+
+    echo ">> bundle exporter end";
 }
 
 function usage {