Merge pull request #2817 from penpot/niwinz-docker-and-admin

✨ Improvements on PREPL

backend/scripts/build

@@ -13,9 +13,9 @@ cp ../CHANGES.md target/classes/changelog.md;
 clojure -T:build jar;
 mv target/penpot.jar target/dist/penpot.jar
 cp scripts/run.template.sh target/dist/run.sh;
-cp scripts/manage.template.sh target/dist/manage.sh;
+cp scripts/manage.py target/dist/manage.py
 chmod +x target/dist/run.sh;
-chmod +x target/dist/manage.sh;
+chmod +x target/dist/manage.py
 
 # Prefetch
 bb ./scripts/prefetch-templates.clj resources/app/onboarding.edn builtin-templates/

backend/scripts/manage.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python3
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Copyright (c) KALEIDOS INC
+
+import argparse
+import json
+import socket
+import sys
+
+from getpass import getpass
+from urllib.parse import urlparse
+
+PREPL_URI = "tcp://localhost:6063"
+
+def get_prepl_conninfo():
+    uri_data = urlparse(PREPL_URI)
+    if uri_data.scheme != "tcp":
+        raise RuntimeException(f"invalid PREPL_URI: {PREPL_URI}")
+
+    if not isinstance(uri_data.netloc, str):
+        raise RuntimeException(f"invalid PREPL_URI: {PREPL_URI}")
+
+    host, port = uri_data.netloc.split(":", 2)
+
+    if port is None:
+        port = 6063
+
+    if isinstance(port, str):
+        port = int(port)
+
+    return host, port
+
+def send_eval(expr):
+    host, port = get_prepl_conninfo()
+
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.connect((host, port))
+        s.send(expr.encode("utf-8"))
+        s.send(b":repl/quit\n\n")
+
+        with s.makefile() as f:
+            result = json.load(f)
+            tag = result.get("tag", None)
+            if tag != "ret":
+                raise RuntimeException("unexpected response from PREPL")
+            return result.get("val", None), result.get("exception", None)
+
+def encode(val):
+    return json.dumps(json.dumps(val))
+
+def print_error(res):
+    for error in res["via"]:
+        print("ERR:", error["message"])
+        break
+
+def run_cmd(params):
+    expr = "(app.srepl.ext/run-json-cmd {})".format(encode(params))
+    res, failed = send_eval(expr)
+    if failed:
+        print_error(res)
+        sys.exit(-1)
+
+    return res
+
+def create_profile(fullname, email, password):
+    params = {
+        "cmd": "create-profile",
+        "params": {
+            "fullname": fullname,
+            "email": email,
+            "password": password
+        }
+    }
+
+    res = run_cmd(params)
+    print(f"Created: {res['email']} / {res['id']}")
+
+def update_profile(email, fullname, password, is_active):
+    params = {
+        "cmd": "update-profile",
+        "params": {
+            "email": email,
+            "fullname": fullname,
+            "password": password,
+            "is_active": is_active
+        }
+    }
+
+    res = run_cmd(params)
+    if res is True:
+        print(f"Updated")
+    else:
+        print(f"No profile found with email {email}")
+
+def derive_password(password):
+    params = {
+        "cmd": "derive-password",
+        "params": {
+            "password": password,
+        }
+    }
+
+    res = run_cmd(params)
+    print(f"Derived password: \"{res}\"")
+
+available_commands = [
+    "create-profile",
+    "update-profile",
+    "derive-password"
+]
+
+parser = argparse.ArgumentParser(
+    description=(
+        "Penpot Command Line Interface (CLI)"
+    )
+)
+
+parser.add_argument("-V", "--version", action="version", version="Penpot CLI %%develop%%")
+parser.add_argument("action", action="store", choices=available_commands)
+parser.add_argument("-n", "--fullname", help="Fullname", action="store")
+parser.add_argument("-e", "--email", help="Email", action="store")
+parser.add_argument("-p", "--password", help="Password", action="store")
+parser.add_argument("-c", "--connect", help="Connect to PREPL", action="store", default="tcp://localhost:6063")
+
+args = parser.parse_args()
+
+PREPL_URI = args.connect
+
+if args.action == "create-profile":
+    email = args.email
+    password = args.password
+    fullname = args.fullname
+
+    if email is None:
+        email = input("Email: ")
+
+    if fullname is None:
+        fullname = input("Fullname: ")
+
+    if password is None:
+        password = getpass("Password: ")
+
+    create_profile(fullname, email, password)
+
+elif args.action == "update-profile":
+    email = args.email
+    password = args.password
+
+    if email is None:
+        email = input("Email: ")
+
+    if password is None:
+        password = getpass("Password: ")
+
+    update_profile(email, None, password, None)
+
+elif args.action == "derive-password":
+    password = args.password
+
+    if password is None:
+        password = getpass("Password: ")
+
+    derive_password(password)

backend/scripts/manage.template.sh

@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-set +e
-JAVA_CMD=$(type -p java)
-
-set -e
-if [[ ! -n "$JAVA_CMD" ]]; then
-  if [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]]; then
-    JAVA_CMD="$JAVA_HOME/bin/java"
-  else
-    >&2 echo "Couldn't find 'java'. Please set JAVA_HOME."
-    exit 1
-  fi
-fi
-
-if [ -f ./environ ]; then
-   source ./environ
-fi
-
-exec $JAVA_CMD $JVM_OPTS -jar penpot.jar -m app.cli.manage "$@"

backend/src/app/rpc/commands/auth.clj

@@ -313,6 +313,7 @@
             (throw e)
             (ex/raise :type :validation
                       :code :email-already-exists
+                      :hint "email already exists"
                       :cause e)))))))
 
 (defn create-profile-relations

backend/src/app/setup/keys.clj

@@ -11,7 +11,6 @@
    [app.common.spec :as us]
    [buddy.core.kdf :as bk]))
 
-
 (defn derive
   "Derive a key from secret-key"
   [secret-key & {:keys [salt size] :or {size 32}}]

backend/src/app/srepl.clj

@@ -30,7 +30,7 @@
    :init repl-init
    :read ccs/repl-read))
 
-(defn json-prepl
+(defn json-repl
   []
   (let [out  *out*
         lock (locks/create)]
@@ -61,7 +61,7 @@
   [[type _] {:keys [::flag port host] :as cfg}]
   (when (contains? cf/flags flag)
     (let [accept (case type
-                   ::prepl 'app.srepl/json-prepl
+                   ::prepl 'app.srepl/json-repl
                    ::urepl 'app.srepl/user-repl)
           params {:address host
                   :port port

backend/src/app/srepl/ext.clj

@@ -8,37 +8,70 @@
   "PREPL API for external usage (CLI or ADMIN)"
   (:require
    [app.auth :as auth]
+   [app.common.exceptions :as ex]
    [app.common.uuid :as uuid]
    [app.db :as db]
-   [app.rpc.commands.auth :as cmd.auth]))
+   [app.rpc.commands.auth :as cmd.auth]
+   [app.util.json :as json]
+   [cuerdas.core :as str]))
 
 (defn- get-current-system
   []
   (or (deref (requiring-resolve 'app.main/system))
       (deref (requiring-resolve 'user/system))))
 
-(defn derive-password
-  [password]
-  (auth/derive-password password))
+(defmulti ^:private run-json-cmd* ::cmd)
 
-(defn create-profile
-  [fullname, email, password]
+(defn run-json-cmd
+  "Entry point with external tools integrations that uses PREPL
+  interface for interacting with running penpot backend."
+  [data]
+  (let [data (json/decode data)
+        params (merge {::cmd (keyword (:cmd data "default"))}
+                      (:params data))]
+    (run-json-cmd* params)))
+
+(defmethod run-json-cmd* :create-profile
+  [{:keys [fullname email password is-active]
+    :or {is-active true}}]
   (when-let [system (get-current-system)]
     (db/with-atomic [conn (:app.db/pool system)]
       (let [params  {:id (uuid/next)
                      :email email
                      :fullname fullname
-                     :is-active true
+                     :is-active is-active
                      :password password
-                     :props {}}
-
-            profile (->> (cmd.auth/create-profile conn params)
-                         (cmd.auth/create-profile-relations conn))]
-
-        (str (:id profile))))))
-
+                     :props {}}]
+        (->> (cmd.auth/create-profile conn params)
+             (cmd.auth/create-profile-relations conn))))))
 
+(defmethod run-json-cmd* :update-profile
+  [{:keys [fullname email password is-active]}]
+  (when-let [system (get-current-system)]
+    (db/with-atomic [conn (:app.db/pool system)]
+      (let [params (cond-> {}
+                     (some? fullname)
+                     (assoc :fullname fullname)
 
+                     (some? password)
+                     (assoc :password (auth/derive-password password))
 
+                     (some? is-active)
+                     (assoc :is-active is-active))]
+        (when (seq params)
+          (let [res (db/update! conn :profile
+                                params
+                                {:email email
+                                 :deleted-at nil}
+                                {:return-keys false})]
+            (pos? (:next.jdbc/update-count res))))))))
 
+(defmethod run-json-cmd* :derive-password
+  [{:keys [password]}]
+  (auth/derive-password password))
 
+(defmethod run-json-cmd* :default
+  [{:keys [::cmd]}]
+  (ex/raise :type :internal
+            :code :not-implemented
+            :hint (str/ffmt "command '%' not implemented" (name cmd))))

common/src/app/common/exceptions.cljc

@@ -16,7 +16,7 @@
 
 (defmacro error
   [& {:keys [type hint] :as params}]
-  `(ex-info ~(or hint (pr-str type))
+  `(ex-info ~(or hint (name type))
             (merge
              ~(dissoc params :cause ::data)
              ~(::data params))

docker/images/Dockerfile.backend

@@ -74,10 +74,13 @@ RUN set -ex; \
         fontconfig \
         woff-tools \
         woff2 \
+        python3 \
         fontforge \
     ; \
     echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen; \
     locale-gen; \
+    mkdir -p /opt/penpot/assets; \
+    chown -R penpot:penpot /opt/penpot; \
     rm -rf /var/lib/apt/lists/*;
 
 COPY --chown=penpot:penpot ./bundle-backend/ /opt/penpot/backend/

docker/images/build.sh

@@ -18,8 +18,6 @@ done
 
 if [ "$PENPOT_BUILD_PUSH" = "true" ]; then
     OPTIONS="--push $OPTIONS"
-else
-    OPTIONS="--load $OPTIONS"
 fi
 
 docker buildx inspect penpot > /dev/null 2>&1;
@@ -34,4 +32,4 @@ else
 fi
 
 unset IFS;
-docker buildx build --platform ${PLATFORM// /,} $OPTIONS -f Dockerfile.$IMAGE .;
+docker buildx build --platform ${PLATFORM// /,} $OPTIONS -f Dockerfile.$IMAGE "$@" .;

docker/images/docker-compose.yaml

@@ -11,9 +11,8 @@ volumes:
   # penpot_minio:
 
 services:
-  ## Traefik service declaration example. Consider using it if you are
-  ## going to expose penpot to the internet or different host than
-  ## `localhost`.
+  ## Traefik service declaration example. Consider using it if you are going to expose
+  ## penpot to the internet or different host than `localhost`.
 
   # traefik:
   #   image: traefik:v2.9
@@ -41,7 +40,7 @@ services:
       - 9001:80
 
     volumes:
-      - penpot_assets:/opt/data
+      - penpot_assets:/opt/penpot/assets
 
     depends_on:
       - penpot-backend
@@ -53,16 +52,15 @@ services:
     labels:
       - "traefik.enable=true"
 
-      ## HTTP: example of labels for the case if you are going to
-      ## expose penpot to the internet using only HTTP (without HTTPS)
-      ## with traefik
+      ## HTTP: example of labels for the case if you are going to expose penpot to the
+      ## internet using only HTTP (without HTTPS) with traefik
 
       # - "traefik.http.routers.penpot-http.entrypoints=web"
       # - "traefik.http.routers.penpot-http.rule=Host(`<DOMAIN_NAME>`)"
       # - "traefik.http.services.penpot-http.loadbalancer.server.port=80"
 
-      ## HTTPS: example of labels for the case if you are going to
-      ## expose penpot to the internet using with HTTPS using traefik
+      ## HTTPS: example of labels for the case if you are going to expose penpot to the
+      ## internet using with HTTPS using traefik
 
       # - "traefik.http.middlewares.http-redirect.redirectscheme.scheme=https"
       # - "traefik.http.middlewares.http-redirect.redirectscheme.permanent=true"
@@ -75,32 +73,31 @@ services:
       # - "traefik.http.routers.penpot-https.tls=true"
       # - "traefik.http.routers.penpot-https.tls.certresolver=letsencrypt"
 
-    ## Configuration envronment variables for frontend the
-    ## container. In this case this container only needs the
-    ## `PENPOT_FLAGS`. This environment variable is shared with other
-    ## services but not all flags are relevant to all services.
-    ##
-    ## Relevant flags for frontend:
-    ## - demo-users
-    ## - login-with-github
-    ## - login-with-gitlab
-    ## - login-with-google
-    ## - login-with-ldap
-    ## - login-with-oidc
-    ## - login-with-password
-    ## - registration
-    ## - webhooks
-    ##
-    ## You can read more about all available flags on:
-    ## https://help.penpot.app/technical-guide/configuration/#advanced-configuration
+    ## Configuration envronment variables for frontend the container. In this case this
+    ## container only needs the `PENPOT_FLAGS`. This environment variable is shared with
+    ## other services but not all flags are relevant to all services.
 
     environment:
+      ## Relevant flags for frontend:
+      ## - demo-users
+      ## - login-with-github
+      ## - login-with-gitlab
+      ## - login-with-google
+      ## - login-with-ldap
+      ## - login-with-oidc
+      ## - login-with-password
+      ## - registration
+      ## - webhooks
+      ##
+      ## You can read more about all available flags on:
+      ## https://help.penpot.app/technical-guide/configuration/#advanced-configuration
+
       - PENPOT_FLAGS=enable-registration enable-login-with-password
 
   penpot-backend:
     image: "penpotapp/backend:latest"
     volumes:
-      - penpot_assets:/opt/data
+      - penpot_assets:/opt/penpot/assets
 
     depends_on:
       - penpot-postgres
@@ -111,58 +108,79 @@ services:
 
     ## Configuration envronment variables for backend the
     ## container.
-    ##
-    ## Relevant flags for backend:
-    ## - demo-users
-    ## - email-verification
-    ## - log-emails
-    ## - log-invitation-tokens
-    ## - login-with-github
-    ## - login-with-gitlab
-    ## - login-with-google
-    ## - login-with-ldap
-    ## - login-with-oidc
-    ## - login-with-password
-    ## - registration
-    ## - secure-session-cookies
-    ## - smtp
-    ## - smtp-debug
-    ## - telemetry
-    ## - webhooks
-    ##
-    ## You can read more about all available flags and other
-    ## environment variables for the backend here:
-    ## https://help.penpot.app/technical-guide/configuration/#advanced-configuration
 
     environment:
-      - PENPOT_FLAGS=enable-registration enable-login disable-email-verification enable-smtp
 
-      ## Public URI. If you are going to expose this instance to the
-      ## internet, or use it under different domain than 'localhost'
-      ## consider using traefik and set the
-      ## `disable-secure-session-cookies` if you are not going to
-      ## serve penpot under HTTPS.
+      ## Relevant flags for backend:
+      ## - demo-users
+      ## - email-verification
+      ## - log-emails
+      ## - log-invitation-tokens
+      ## - login-with-github
+      ## - login-with-gitlab
+      ## - login-with-google
+      ## - login-with-ldap
+      ## - login-with-oidc
+      ## - login-with-password
+      ## - registration
+      ## - secure-session-cookies
+      ## - smtp
+      ## - smtp-debug
+      ## - telemetry
+      ## - webhooks
+      ## - prepl-server
+      ##
+      ## You can read more about all available flags and other
+      ## environment variables for the backend here:
+      ## https://help.penpot.app/technical-guide/configuration/#advanced-configuration
+
+      - PENPOT_FLAGS=enable-registration enable-login disable-email-verification enable-smtp enable-prepl-server
+
+      ## Penpot SECRET KEY. It serves as a master key from which other keys for subsystems
+      ## (eg http sessions) are derived.
+      ##
+      ## Leave it comment if it is ok for you to have to login again after each backend
+      ## restart.
+      ##
+      ## If you going to uncomment this, we recommend use here a trully randomly generated
+      ## 512 bits base64 encoded string.  You can generate one with:
+      ##
+      ## python3 -c "import secrets; print(secrets.token_urlsafe(64))"
+
+      # - PENPOT_SECRET_KEY=my-insecure-key
+
+      ## The PREPL host. Mainly used for external programatic access to penpot backend
+      ## (example: admin). By default it listen on `localhost` but if you are going to use
+      ## the `admin`, you will need to uncomment this and set the host to `0.0.0.0`.
+
+      # - PENPOT_PREPL_HOST=0.0.0.0
+
+      ## Public URI. If you are going to expose this instance to the internet and use it
+      ## under different domain than 'localhost', you will need to adjust it to the final
+      ## domain.
+      ##
+      ## Consider using traefik and set the 'disable-secure-session-cookies' if you are
+      ## not going to serve penpot under HTTPS.
 
       - PENPOT_PUBLIC_URI=http://localhost:9001
 
-      ## Database connection parameters. Don't touch them unless you
-      ## are using custom postgresql connection parameters
+      ## Database connection parameters. Don't touch them unless you are using custom
+      ## postgresql connection parameters.
 
       - PENPOT_DATABASE_URI=postgresql://penpot-postgres/penpot
       - PENPOT_DATABASE_USERNAME=penpot
       - PENPOT_DATABASE_PASSWORD=penpot
 
-      ## Redis is used for the websockets notifications. Don't touch
-      ## unless the redis container has different parameters or
-      ## different name.
+      ## Redis is used for the websockets notifications. Don't touch unless the redis
+      ## container has different parameters or different name.
 
       - PENPOT_REDIS_URI=redis://penpot-redis/0
 
-      ## Default configuration for assets storage: using filesystem
-      ## based with all files stored in a docker volume.
+      ## Default configuration for assets storage: using filesystem based with all files
+      ## stored in a docker volume.
 
       - PENPOT_ASSETS_STORAGE_BACKEND=assets-fs
-      - PENPOT_STORAGE_ASSETS_FS_DIRECTORY=/opt/data/assets
+      - PENPOT_STORAGE_ASSETS_FS_DIRECTORY=/opt/penpot/assets
 
       ## Also can be configured to to use a S3 compatible storage
       ## service like MiniIO. Look below for minio service setup.
@@ -173,19 +191,18 @@ services:
       # - PENPOT_STORAGE_ASSETS_S3_ENDPOINT=http://penpot-minio:9000
       # - PENPOT_STORAGE_ASSETS_S3_BUCKET=<BUKET_NAME>
 
-      ## Telemetry. When enabled, a periodical process will send
-      ## anonymous data about this instance. Telemetry data will
-      ## enable us to learn on how the application is used, based on
-      ## real scenarios. If you want to help us, please leave it
-      ## enabled. You can audit what data we send with the code
-      ## available on github
+      ## Telemetry. When enabled, a periodical process will send anonymous data about this
+      ## instance. Telemetry data will enable us to learn on how the application is used,
+      ## based on real scenarios. If you want to help us, please leave it enabled. You can
+      ## audit what data we send with the code available on github
+
       - PENPOT_TELEMETRY_ENABLED=true
 
-      ## Example SMTP/Email configuration. By default, emails are sent
-      ## to the mailcatch service, but for production usage is
-      ## recommended to setup a real SMTP provider. Emails are used to
-      ## confirm user registrations & invitations. Look below how
-      ## mailcatch service is configured.
+      ## Example SMTP/Email configuration. By default, emails are sent to the mailcatch
+      ## service, but for production usage is recommended to setup a real SMTP
+      ## provider. Emails are used to confirm user registrations & invitations. Look below
+      ## how mailcatch service is configured.
+
       - PENPOT_SMTP_DEFAULT_FROM=no-reply@example.com
       - PENPOT_SMTP_DEFAULT_REPLY_TO=no-reply@example.com
       - PENPOT_SMTP_HOST=penpot-mailcatch
@@ -231,10 +248,9 @@ services:
     networks:
       - penpot
 
-  ## A mailcatch service, used as temporal SMTP server. You can access
-  ## via HTTP to the port 1080 for read all emails the penpot platform
-  ## has sent. Should be only used as a temporal solution meanwhile
-  ## you don't have a real SMTP provider configured.
+  ## A mailcatch service, used as temporal SMTP server. You can access via HTTP to the
+  ## port 1080 for read all emails the penpot platform has sent. Should be only used as a
+  ## temporal solution meanwhile you don't have a real SMTP provider configured.
 
   penpot-mailcatch:
     image: sj26/mailcatcher:latest
@@ -244,9 +260,42 @@ services:
     ports:
       - "1080:1080"
 
-  ## Example configuration of MiniIO (S3 compatible object storage
-  ## service); If you don't have preference, then just use filesystem,
-  ## this is here just for the completeness.
+  ## An optional admin application for pentpot. It allows manage users, teams and inspect
+  ## some parts of the database. You can read more about it on:
+  ## https://github.com/penpot/penpot-admin
+  ##
+  ## If you are going to use admin, ensure to have `enable-prepl-server` in backend flags
+  ## and uncomment the `PENPOT_PREPL_HOST` environment variable.
+  ##
+  ## Status: EXPERIMENTAL
+
+  # penpot-admin:
+  #   image: "penpotapp/admin:latest"
+  #   networks:
+  #     - penpot
+  #
+  #   depends_on:
+  #     - penpot-postgres
+  #     - penpot-backend
+  #
+  #   environment:
+  #     ## Adjust to the same value as on backend
+  #     - PENPOT_PUBLIC_URI=http://localhost:9001
+  #
+  #     ## Do not touch it, this is an internal routes
+  #     - PENPOT_API_URI=http://penpot-frontend/
+  #     - PENPOT_PREPL_URI=tcp://penpot-backend:6063/
+  #     - PENPOT_DEBUG="false"
+  #
+  #     ## Adjust to the same values as on backend
+  #     - PENPOT_DATABASE_HOST=penpot-postgres
+  #     - PENPOT_DATABASE_NAME=penpot
+  #     - PENPOT_DATABASE_USERNAME=penpot
+  #     - PENPOT_DATABASE_PASSWORD=penpot
+  #     - PENPOT_REDIS_URI=redis://penpot-redis/0
+
+  ## Example configuration of MiniIO (S3 compatible object storage service); If you don't
+  ## have preference, then just use filesystem, this is here just for the completeness.
 
   # minio:
   #   image: "minio/minio:latest"

docker/images/files/nginx.conf

@@ -90,11 +90,10 @@ http {
 
         location /internal/assets {
             internal;
-            alias /opt/data/assets;
+            alias /opt/penpot/assets;
             add_header x-internal-redirect "$upstream_http_x_accel_redirect";
         }
 
-
         location /api/export {
             proxy_pass http://penpot-exporter:6061;
         }
@@ -104,7 +103,7 @@ http {
         }
 
         location /admin {
-            proxy_pass http://penpot-admin:6063/admin;
+            proxy_pass http://penpot-admin:6065/admin;
         }
 
         location /ws/notifications {