Merge pull request #3352 from penpot/niwinz-bugfixes

✨ Improvements & bugfixes

CHANGES.md

@@ -15,6 +15,7 @@
   rendered as bitmap images.
 - Add the ability to disable google fonts provider with the `disable-google-fonts-provider` flag
 - Add the ability to disable dashboard templates section with the `disable-dashboard-templates-section` flag
+- Add the ability to use the registration whitelist with OICD [Github #3348](https://github.com/penpot/penpot/issues/3348)
 
 ### :bug: Bugs fixed
 

backend/src/app/auth.clj

@@ -6,7 +6,9 @@
 
 (ns app.auth
   (:require
+   [app.config :as cf]
    [buddy.hashers :as hashers]
+   [cuerdas.core :as str]
    [promesa.exec :as px]))
 
 (def default-params
@@ -27,3 +29,16 @@
       {:update false
        :valid false})))
 
+(defn email-domain-in-whitelist?
+  "Returns true if email's domain is in the given whitelist or if
+  given whitelist is an empty string."
+  ([email]
+   (let [domains (cf/get :registration-domain-whitelist)]
+     (email-domain-in-whitelist? domains email)))
+  ([domains email]
+   (if (or (nil? domains) (empty? domains))
+     true
+     (let [[_ candidate] (-> (str/lower email)
+                             (str/split #"@" 2))]
+       (contains? domains candidate)))))
+

backend/src/app/auth/oidc.clj

@@ -7,6 +7,7 @@
 (ns app.auth.oidc
   "OIDC client implementation."
   (:require
+   [app.auth :as auth]
    [app.auth.oidc.providers :as-alias providers]
    [app.common.data :as d]
    [app.common.data.macros :as dm]
@@ -430,10 +431,24 @@
    ::yrs/headers {"location" (str uri)}})
 
 (defn- generate-error-redirect
-  [_ error]
-  (let [uri (-> (u/uri (cf/get :public-uri))
+  [_ cause]
+  (let [data   (if (ex/error? cause) (ex-data cause) nil)
+        code   (or (:code data) :unexpected)
+        type   (or (:type data) :internal)
+        hint   (or (:hint data)
+                   (if (ex/exception? cause)
+                     (ex-message cause)
+                     (str cause)))
+
+        params {:error "unable-to-auth"
+                :hint hint
+                :type type
+                :code code}
+
+        uri    (-> (u/uri (cf/get :public-uri))
                    (assoc :path "/#/auth/login")
-                (assoc :query (u/map->query-string {:error "unable-to-auth" :hint (ex-message error)})))]
+                   (assoc :query (u/map->query-string params)))]
+
     (redirect-response uri)))
 
 (defn- generate-redirect
@@ -463,6 +478,8 @@
       (->> (redirect-response uri)
            (sxf request)))
 
+
+    (if (auth/email-domain-in-whitelist? (:email info))
       (let [info   (assoc info
                           :iss :prepared-register
                           :is-active true
@@ -475,7 +492,9 @@
                        (assoc :path "/#/auth/register/validate")
                        (assoc :query (u/map->query-string params)))]
 
-      (redirect-response uri))))
+        (redirect-response uri))
+      (generate-error-redirect cfg "email-domain-not-allowed"))))
+
 
 (defn- auth-handler
   [cfg {:keys [params] :as request}]

backend/src/app/email.clj

@@ -305,7 +305,7 @@
   (fn [params]
     (when (contains? cf/flags :smtp)
       (let [session (create-smtp-session cfg)]
-        (with-open [transport (.getTransport session (if (:ssl cfg) "smtps" "smtp"))]
+        (with-open [transport (.getTransport session (if (::ssl cfg) "smtps" "smtp"))]
           (.connect ^Transport transport
                     ^String (::username cfg)
                     ^String (::password cfg))
@@ -341,7 +341,7 @@
                                (map :content)
                                first)))
                (println "******** end email" (:id email) "**********"))]
-    (l/info ::l/raw out)))
+    (l/raw! :info out)))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; EMAIL FACTORIES

backend/src/app/rpc/commands/auth.clj

@@ -6,6 +6,7 @@
 
 (ns app.rpc.commands.auth
   (:require
+   [app.auth :as auth]
    [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
@@ -38,19 +39,6 @@
 (s/def ::invitation-token ::us/not-empty-string)
 (s/def ::token ::us/not-empty-string)
 
-;; ---- HELPERS
-
-(defn email-domain-in-whitelist?
-  "Returns true if email's domain is in the given whitelist or if
-  given whitelist is an empty string."
-  [domains email]
-  (if (or (empty? domains)
-          (nil? domains))
-    true
-    (let [[_ candidate] (-> (str/lower email)
-                            (str/split #"@" 2))]
-      (contains? domains candidate))))
-
 ;; ---- COMMAND: login with password
 
 (defn login-with-password
@@ -180,10 +168,9 @@
                   :code :email-does-not-match-invitation
                   :hint "email should match the invitation"))))
 
-  (when-let [domains (cf/get :registration-domain-whitelist)]
-    (when-not (email-domain-in-whitelist? domains (:email params))
+  (when-not (auth/email-domain-in-whitelist? (:email params))
     (ex/raise :type :validation
-                :code :email-domain-is-not-allowed)))
+              :code :email-domain-is-not-allowed))
 
   ;; Don't allow proceed in preparing registration if the profile is
   ;; already reported as spammer.

backend/test/backend_tests/rpc_profile_test.clj

@@ -10,7 +10,7 @@
    [app.config :as cf]
    [app.db :as db]
    [app.rpc :as-alias rpc]
-   [app.rpc.commands.auth :as cauth]
+   [app.auth :as auth]
    [app.tokens :as tokens]
    [app.util.time :as dt]
    [backend-tests.helpers :as th]
@@ -226,11 +226,11 @@
 (t/deftest registration-domain-whitelist
   (let [whitelist #{"gmail.com" "hey.com" "ya.ru"}]
     (t/testing "allowed email domain"
-      (t/is (true? (cauth/email-domain-in-whitelist? whitelist "username@ya.ru")))
-      (t/is (true? (cauth/email-domain-in-whitelist? #{} "username@somedomain.com"))))
+      (t/is (true? (auth/email-domain-in-whitelist? whitelist "username@ya.ru")))
+      (t/is (true? (auth/email-domain-in-whitelist? #{} "username@somedomain.com"))))
 
     (t/testing "not allowed email domain"
-      (t/is (false? (cauth/email-domain-in-whitelist? whitelist "username@somedomain.com"))))))
+      (t/is (false? (auth/email-domain-in-whitelist? whitelist "username@somedomain.com"))))))
 
 (t/deftest prepare-register-and-register-profile-1
   (let [data  {::th/type :prepare-register-profile

common/src/app/common/encoding_impl.js

@@ -0,0 +1,211 @@
+/**
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) KALEIDOS INC
+ */
+"use strict";
+
+goog.require("cljs.core");
+goog.provide("app.common.encoding_impl");
+
+goog.scope(function() {
+  const core = cljs.core;
+  const global = goog.global;
+  const self = app.common.encoding_impl;
+
+  const hexMap = [];
+  for (let i = 0; i < 256; i++) {
+    hexMap[i] = (i + 0x100).toString(16).substr(1);
+  }
+
+  function hexToBuffer(input) {
+    if (typeof input !== "string") {
+      throw new TypeError("Expected input to be a string");
+    }
+
+    // Accept UUID hex format
+    input = input.replace(/-/g, "");
+
+    if ((input.length % 2) !== 0) {
+      throw new RangeError("Expected string to be an even number of characters")
+    }
+
+    const view = new Uint8Array(input.length / 2);
+
+    for (let i = 0; i < input.length; i += 2) {
+      view[i / 2] = parseInt(input.substring(i, i + 2), 16);
+    }
+
+    return view.buffer;
+  }
+
+  function bufferToHex(source, isUuid) {
+    if (source instanceof Uint8Array) {
+    } else if (ArrayBuffer.isView(source)) {
+      source = new Uint8Array(source.buffer, source.byteOffset, source.byteLength);
+    } else if (Array.isArray(source)) {
+      source = Uint8Array.from(source);
+    }
+
+    if (source.length != 16) {
+      throw new RangeError("only 16 bytes array is allowed");
+    }
+
+    const spacer = isUuid ? "-" : "";
+
+    let i = 0;
+    return  (hexMap[source[i++]] +
+             hexMap[source[i++]] +
+             hexMap[source[i++]] +
+             hexMap[source[i++]] + spacer +
+             hexMap[source[i++]] +
+             hexMap[source[i++]] + spacer +
+             hexMap[source[i++]] +
+             hexMap[source[i++]] + spacer +
+             hexMap[source[i++]] +
+             hexMap[source[i++]] + spacer +
+             hexMap[source[i++]] +
+             hexMap[source[i++]] +
+             hexMap[source[i++]] +
+             hexMap[source[i++]] +
+             hexMap[source[i++]] +
+             hexMap[source[i++]]);
+  }
+
+  self.hexToBuffer = hexToBuffer;
+  self.bufferToHex = bufferToHex;
+
+  // base-x encoding / decoding
+  // Copyright (c) 2018 base-x contributors
+  // Copyright (c) 2014-2018 The Bitcoin Core developers (base58.cpp)
+  // Distributed under the MIT software license, see the accompanying
+  // file LICENSE or http://www.opensource.org/licenses/mit-license.php.
+
+  // WARNING: This module is NOT RFC3548 compliant, it cannot be used
+  // for base16 (hex), base32, or base64 encoding in a standards
+  // compliant manner.
+
+  function getBaseCodec (ALPHABET) {
+    if (ALPHABET.length >= 255) { throw new TypeError("Alphabet too long"); }
+    let BASE_MAP = new Uint8Array(256);
+    for (let j = 0; j < BASE_MAP.length; j++) {
+      BASE_MAP[j] = 255;
+    }
+    for (let i = 0; i < ALPHABET.length; i++) {
+      let x = ALPHABET.charAt(i);
+      let xc = x.charCodeAt(0);
+      if (BASE_MAP[xc] !== 255) { throw new TypeError(x + " is ambiguous"); }
+      BASE_MAP[xc] = i;
+    }
+    let BASE = ALPHABET.length;
+    let LEADER = ALPHABET.charAt(0);
+    let FACTOR = Math.log(BASE) / Math.log(256); // log(BASE) / log(256), rounded up
+    let iFACTOR = Math.log(256) / Math.log(BASE); // log(256) / log(BASE), rounded up
+    function encode (source) {
+      if (source instanceof Uint8Array) {
+      } else if (ArrayBuffer.isView(source)) {
+        source = new Uint8Array(source.buffer, source.byteOffset, source.byteLength);
+      } else if (Array.isArray(source)) {
+        source = Uint8Array.from(source);
+      }
+      if (!(source instanceof Uint8Array)) { throw new TypeError("Expected Uint8Array"); }
+      if (source.length === 0) { return ""; }
+      // Skip & count leading zeroes.
+      let zeroes = 0;
+      let length = 0;
+      let pbegin = 0;
+      let pend = source.length;
+      while (pbegin !== pend && source[pbegin] === 0) {
+        pbegin++;
+        zeroes++;
+      }
+      // Allocate enough space in big-endian base58 representation.
+      let size = ((pend - pbegin) * iFACTOR + 1) >>> 0;
+      let b58 = new Uint8Array(size);
+      // Process the bytes.
+      while (pbegin !== pend) {
+        let carry = source[pbegin];
+        // Apply "b58 = b58 * 256 + ch".
+        let i = 0;
+        for (let it1 = size - 1; (carry !== 0 || i < length) && (it1 !== -1); it1--, i++) {
+          carry += (256 * b58[it1]) >>> 0;
+          b58[it1] = (carry % BASE) >>> 0;
+          carry = (carry / BASE) >>> 0;
+        }
+        if (carry !== 0) { throw new Error("Non-zero carry"); }
+        length = i;
+        pbegin++;
+      }
+      // Skip leading zeroes in base58 result.
+      let it2 = size - length;
+      while (it2 !== size && b58[it2] === 0) {
+        it2++;
+      }
+      // Translate the result into a string.
+      let str = LEADER.repeat(zeroes);
+      for (; it2 < size; ++it2) { str += ALPHABET.charAt(b58[it2]); }
+      return str;
+    }
+
+    function decodeUnsafe (source) {
+      if (typeof source !== "string") { throw new TypeError("Expected String"); }
+      if (source.length === 0) { return new Uint8Array(); }
+      let psz = 0;
+      // Skip and count leading '1's.
+      let zeroes = 0;
+      let length = 0;
+      while (source[psz] === LEADER) {
+        zeroes++;
+        psz++;
+      }
+      // Allocate enough space in big-endian base256 representation.
+      let size = (((source.length - psz) * FACTOR) + 1) >>> 0; // log(58) / log(256), rounded up.
+      let b256 = new Uint8Array(size);
+      // Process the characters.
+      while (source[psz]) {
+        // Decode character
+        let carry = BASE_MAP[source.charCodeAt(psz)];
+        // Invalid character
+        if (carry === 255) { return; }
+        let i = 0;
+        for (let it3 = size - 1; (carry !== 0 || i < length) && (it3 !== -1); it3--, i++) {
+          carry += (BASE * b256[it3]) >>> 0;
+          b256[it3] = (carry % 256) >>> 0;
+          carry = (carry / 256) >>> 0;
+        }
+        if (carry !== 0) { throw new Error("Non-zero carry"); }
+        length = i;
+        psz++;
+      }
+      // Skip leading zeroes in b256.
+      let it4 = size - length;
+      while (it4 !== size && b256[it4] === 0) {
+        it4++;
+      }
+      let vch = new Uint8Array(zeroes + (size - it4));
+      let j = zeroes;
+      while (it4 !== size) {
+        vch[j++] = b256[it4++];
+      }
+      return vch;
+    }
+
+    function decode (string) {
+      let buffer = decodeUnsafe(string);
+      if (buffer) { return buffer; }
+      throw new Error("Non-base" + BASE + " character");
+    }
+
+    return {
+      encode: encode,
+      decodeUnsafe: decodeUnsafe,
+      decode: decode
+    };
+  }
+  // MORE bases here: https://github.com/cryptocoinjs/base-x/tree/master
+  const BASE62 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+  self.bufferToBase62 = getBaseCodec(BASE62).encode;
+
+});

common/src/app/common/logging.cljc

@@ -239,7 +239,7 @@
 #?(:clj
    (defn slf4j-log-handler
      {:no-doc true}
-     [_ _ _ {:keys [::logger ::level ::props ::cause ::trace ::message]}]
+     [_ _ _ {:keys [::logger ::level ::trace ::message] }]
      (when-let [logger (enabled? logger level)]
        (let [message (cond-> @message
                        (some? trace)
@@ -307,6 +307,18 @@
                (l/set-level! logger level)))
            config)))
 
+(defmacro raw!
+  [level message]
+  (let [cljs? (:ns &env)]
+    `(do
+       (~(if cljs?
+         `(partial console-log-handler nil nil nil)
+         `(partial slf4j-log-handler nil nil nil))
+        {::logger ~(str *ns*)
+         ::level ~level
+         ::message (delay ~message)})
+       nil)))
+
 (defmacro info
   [& params]
   `(do

common/src/app/common/uuid.cljc

@@ -4,9 +4,11 @@
 ;;
 ;; Copyright (c) KALEIDOS INC
 
+#_:clj-kondo/ignore
 (ns app.common.uuid
-  (:refer-clojure :exclude [next uuid zero?])
+  (:refer-clojure :exclude [next uuid zero? short])
   (:require
+   [app.common.data.macros :as dm]
    #?(:clj [clojure.core :as c])
    #?(:cljs [app.common.uuid-impl :as impl])
    #?(:cljs [cljs.core :as c]))
@@ -66,3 +68,10 @@
      (let [buf (ByteBuffer/wrap o)]
        (UUID. ^long (.getLong buf)
               ^long (.getLong buf)))))
+
+#?(:cljs
+   (defn uuid->short-id
+     "Return a shorter string of a safe subset of bytes of an uuid encoded
+     with base62. It is only safe to use with uuid v4 and penpot custom v8"
+     [id]
+     (impl/short-v8 (dm/str id))))

common/src/app/common/uuid_impl.js

@@ -8,13 +8,17 @@
 "use strict";
 
 goog.require("cljs.core");
+goog.require("app.common.encoding_impl");
 goog.provide("app.common.uuid_impl");
 
 goog.scope(function() {
   const core = cljs.core;
   const global = goog.global;
+  const encoding  = app.common.encoding_impl;
   const self = app.common.uuid_impl;
 
+  const timeRef = 1640995200000; // ms since 2022-01-01T00:00:00
+
   const fill = (() => {
     if (typeof global.crypto !== "undefined" &&
         typeof global.crypto.getRandomValues !== "undefined") {
@@ -45,12 +49,8 @@ goog.scope(function() {
     }
   })();
 
-  const hexMap = [];
-  for (let i = 0; i < 256; i++) {
-    hexMap[i] = (i + 0x100).toString(16).substr(1);
-  }
-
   function toHexString(buf) {
+    const hexMap = encoding.hexMap;
     let i = 0;
     return  (hexMap[buf[i++]] +
              hexMap[buf[i++]] +
@@ -68,18 +68,7 @@ goog.scope(function() {
              hexMap[buf[i++]] +
              hexMap[buf[i++]] +
              hexMap[buf[i++]]);
-  }
-
-  self.v4 = (function () {
-    const buff8 = new Uint8Array(16);
-
-    return function v4() {
-      fill(buff8);
-      buff8[6] = (buff8[6] & 0x0f) | 0x40;
-      buff8[8] = (buff8[8] & 0x3f) | 0x80;
-      return core.uuid(toHexString(buff8));
   };
-  })();
 
   function getBigUint64(view, byteOffset, le) {
     const a = view.getUint32(byteOffset, le);
@@ -103,16 +92,53 @@ goog.scope(function() {
     }
   }
 
-  self.v8 = (function () {
-    const buff  = new ArrayBuffer(16);
-    const int8 = new Uint8Array(buff);
-    const view  = new DataView(buff);
+  function currentTimestamp(timeRef) {
+    return BigInt.asUintN(64, "" + (Date.now() - timeRef));
+  }
 
   const tmpBuff = new ArrayBuffer(8);
   const tmpView = new DataView(tmpBuff);
   const tmpInt8 = new Uint8Array(tmpBuff);
 
-    const timeRef = 1640995200000; // ms since 2022-01-01T00:00:00
+  function nextLong() {
+    fill(tmpInt8);
+    return getBigUint64(tmpView, 0, false);
+  }
+
+  self.shortID = (function () {
+    const buff  = new ArrayBuffer(8);
+    const int8 = new Uint8Array(buff);
+    const view  = new DataView(buff);
+
+    const base = 0x0000_0000_0000_0000n;
+
+    return function shortID(ts) {
+      const tss = currentTimestamp(timeRef);
+      const msb = (base
+                   | (nextLong() & 0xffff_ffff_0000_0000n)
+                   | (tss & 0x0000_0000_ffff_ffffn));
+      setBigUint64(view, 0, msb, false);
+      return encoding.toBase62(int8);
+    };
+  })();
+
+
+  self.v4 = (function () {
+    const arr = new Uint8Array(16);
+
+    return function v4() {
+      fill(arr);
+      arr[6] = (arr[6] & 0x0f) | 0x40;
+      arr[8] = (arr[8] & 0x3f) | 0x80;
+      return core.uuid(encoding.bufferToHex(arr, true));
+    };
+  })();
+
+  self.v8 = (function () {
+    const buff = new ArrayBuffer(16);
+    const int8 = new Uint8Array(buff);
+    const view = new DataView(buff);
+
     const maxCs = 0x0000_0000_0000_3fffn; // 14 bits space
 
     let countCs = 0n;
@@ -122,15 +148,6 @@ goog.scope(function() {
     let baseMsb = 0x0000_0000_0000_8000n;
     let baseLsb = 0x8000_0000_0000_0000n;
 
-    const currentTimestamp = () => {
-      return BigInt.asUintN(64, "" + (Date.now() - timeRef));
-    };
-
-    const nextLong = () => {
-      fill(tmpInt8);
-      return getBigUint64(tmpView, 0, false);
-    };
-
     lastRd = nextLong() & 0xffff_ffff_ffff_f0ffn;
     lastCs = nextLong() & maxCs;
 
@@ -145,12 +162,12 @@ goog.scope(function() {
       setBigUint64(view, 0, msb, false);
       setBigUint64(view, 8, lsb, false);
 
-      return core.uuid(toHexString(int8));
+      return core.uuid(encoding.bufferToHex(int8, true));
     };
 
     const factory = function v8() {
       while (true) {
-        let ts = currentTimestamp();
+        let ts = currentTimestamp(timeRef);
 
         // Protect from clock regression
         if ((ts - lastTs) < 0) {
@@ -195,6 +212,12 @@ goog.scope(function() {
   })();
 
 
+  self.short_v8 = function(uuid) {
+    const buff = encoding.hexToBuffer(uuid);
+    const short =  new Uint8Array(buff, 4);
+    return encoding.bufferToBase62(short);
+  };
+
   self.custom = function formatAsUUID(mostSigBits, leastSigBits) {
     const most = mostSigBits.toString("16").padStart(16, "0");
     const least = leastSigBits.toString("16").padStart(16, "0");

frontend/src/app/main/thumbnail_renderer.cljs

@@ -78,7 +78,7 @@
          (rx/mapcat (fn [msg]
                       (case (unchecked-get msg "type")
                         "success" (rx/of (unchecked-get msg "payload"))
-                        "failure" (rx/throw (unchecked-get msg "payload")))))
+                        "failure" (rx/throw (js/Error. (unchecked-get msg "payload"))))))
          (rx/take 1))))
 
 (defn init!
@@ -88,6 +88,6 @@
     (dom/set-attribute! iframe "src" origin)
     (dom/set-attribute! iframe "hidden" true)
     (dom/append-child! js/document.body iframe)
-
+    (.addEventListener js/window "message" on-message)
     (set! instance iframe)
-    (.addEventListener js/window "message" on-message)))
+    ))

frontend/src/app/main/ui/dashboard/grid.cljs

@@ -76,7 +76,12 @@
       (when (and visible? (not thumbnail-uri))
         (->> (ask-for-thumbnail file-id revn)
              (rx/subs (fn [url]
-                        (st/emit! (dd/set-file-thumbnail file-id url)))))))
+                        (st/emit! (dd/set-file-thumbnail file-id url)))
+                      (fn [cause]
+                        (log/error :hint "unable to render thumbnail"
+                                   :file-if file-id
+                                   :revn revn
+                                   :message (ex-message cause)))))))
 
     [:div.grid-item-th
      {:style {:background-color background-color}

frontend/src/app/thumbnail_renderer.cljs

@@ -229,8 +229,8 @@
 
 (defn- send-failure!
   "Sends a failure message."
-  [id payload]
-  (send-answer! id "failure" payload))
+  [id cause]
+  (send-answer! id "failure" (ex-message cause)))
 
 (defn- send-ready!
   "Sends a ready message."