From 3011d2490539a835432ebadb97b0b6c94a9e8ef5 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 16 Mar 2023 19:36:18 +0100 Subject: [PATCH 1/8] :paperclip: Enable storage features on start-dev and repl scripts --- backend/scripts/repl | 2 ++ backend/scripts/start-dev | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/backend/scripts/repl b/backend/scripts/repl index 5cae7d7be..286111b72 100755 --- a/backend/scripts/repl +++ b/backend/scripts/repl @@ -5,6 +5,8 @@ export PENPOT_TENANT=dev export PENPOT_FLAGS="\ $PENPOT_FLAGS \ enable-backend-asserts \ + enable-fdata-storage-pointer-map \ + enable-fdata-storage-objets-map \ enable-audit-log \ enable-transit-readable-response \ enable-demo-users \ diff --git a/backend/scripts/start-dev b/backend/scripts/start-dev index 86edb2d70..a7f0045d1 100755 --- a/backend/scripts/start-dev +++ b/backend/scripts/start-dev @@ -2,7 +2,20 @@ export PENPOT_HOST=devenv export PENPOT_TENANT=dev -export PENPOT_FLAGS="$PENPOT_FLAGS enable-backend-asserts enable-audit-log enable-transit-readable-response enable-demo-users disable-secure-session-cookies enable-smtp enable-webhooks" +export PENPOT_FLAGS="\ + $PENPOT_FLAGS \ + enable-prepl-server \ + enable-urepl-server \ + enable-webhooks \ + enable-backend-asserts \ + enable-audit-log \ + enable-transit-readable-response \ + enable-demo-users \ + enable-fdata-storage-pointer-map \ + enable-fdata-storage-objets-map \ + disable-secure-session-cookies \ + enable-smtp \ + enable-webhooks"; set -ex From b462ac019a9fea631a88edfefbd50cc2fea86859 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 16 Mar 2023 19:39:19 +0100 Subject: [PATCH 2/8] :bug: Fix typo on error type --- backend/src/app/http/errors.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/app/http/errors.clj b/backend/src/app/http/errors.clj index 23aae0c39..9fa4e7a78 100644 --- a/backend/src/app/http/errors.clj +++ b/backend/src/app/http/errors.clj @@ -112,7 +112,7 @@ (l/error :hint "Internal error" :message (ex-message error) :cause error) {::yrs/status 500 ::yrs/body {:type :server-error - :code :unhandloed + :code :unhandled :hint (ex-message error) :data (ex-data error)}})) From 69c8a89dd2ab356427c0b50f3ed89d139794848d Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 16 Mar 2023 19:40:07 +0100 Subject: [PATCH 3/8] :tada: Add the ability to specify the output format from query string --- backend/src/app/http/middleware.clj | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/src/app/http/middleware.clj b/backend/src/app/http/middleware.clj index 7e5cdc7aa..64f2b5d54 100644 --- a/backend/src/app/http/middleware.clj +++ b/backend/src/app/http/middleware.clj @@ -158,8 +158,13 @@ (assoc ::yrs/body (transit-streamable-body body opts)))) response))) + (format-from-params [{:keys [query-params] :as request}] + (and (= "json" (get query-params :_fmt)) + "application/json")) + (format-response [response request] - (let [accept (yrq/get-header request "accept")] + (let [accept (or (format-from-params request) + (yrq/get-header request "accept"))] (cond (or (= accept "application/transit+json") (str/includes? accept "application/transit+json")) From 9173c73eca20f0fc35f909a49770003494037a51 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 16 Mar 2023 19:40:28 +0100 Subject: [PATCH 4/8] :bug: Forward var bindings on climit submit operation --- backend/src/app/rpc/climit.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/app/rpc/climit.clj b/backend/src/app/rpc/climit.clj index bbd5179d4..060ab0015 100644 --- a/backend/src/app/rpc/climit.clj +++ b/backend/src/app/rpc/climit.clj @@ -168,7 +168,7 @@ (defn submit! [{:keys [::id ::cache ::wrk/executor ::mtx/metrics]} f] - (let [f (partial px/submit! executor f)] + (let [f (partial px/submit! executor (px/wrap-bindings f))] (if (and cache id) (p/await! (invoke! cache metrics id nil f)) (p/await! (f))))) From 938890c04ccf0f0329f84ec1860a0f4b902aebee Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 16 Mar 2023 19:40:58 +0100 Subject: [PATCH 5/8] :bug: Fix vthread pining on get-file-data-for-thumbnail rpc method --- backend/src/app/rpc/commands/files.clj | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/backend/src/app/rpc/commands/files.clj b/backend/src/app/rpc/commands/files.clj index 719f63598..d8dc77086 100644 --- a/backend/src/app/rpc/commands/files.clj +++ b/backend/src/app/rpc/commands/files.clj @@ -670,12 +670,30 @@ ;; --- COMMAND QUERY: get-file-data-for-thumbnail +;; FIXME: performance issue +;; +;; We need to improve how we set frame for thumbnail in order to avoid +;; loading all pages into memory for find the frame set for thumbnail. + (defn get-file-data-for-thumbnail [conn {:keys [data id] :as file}] (letfn [;; function responsible on finding the frame marked to be ;; used as thumbnail; the returned frame always have ;; the :page-id set to the page that it belongs. + (get-thumbnail-frame [data] + ;; NOTE: this is a hack for avoid perform blocking + ;; operation inside the for loop, clojure lazy-seq uses + ;; synchronized blocks that does not plays well with + ;; virtual threads, so we need to perform the load + ;; operation first. This operation forces all pointer maps + ;; load into the memory. + (->> (-> data :pages-index vals) + (filter pmap/pointer-map?) + (run! pmap/load!)) + + ;; Then proceed to find the frame set for thumbnail + (d/seek :use-for-thumbnail? (for [page (-> data :pages-index vals) frame (-> page :objects ctt/get-frames)] From f18d2ea629b6b412c596d100ac83563e6cdba68e Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 16 Mar 2023 19:41:30 +0100 Subject: [PATCH 6/8] :bug: Add missing fragment persistence on creating file Related with storage/pointer-map feature. --- backend/src/app/rpc/commands/files_create.clj | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/backend/src/app/rpc/commands/files_create.clj b/backend/src/app/rpc/commands/files_create.clj index 772d7aa25..423ea3c57 100644 --- a/backend/src/app/rpc/commands/files_create.clj +++ b/backend/src/app/rpc/commands/files_create.clj @@ -33,22 +33,26 @@ (db/insert! conn :file-profile-rel)))) (defn create-file - [conn {:keys [id name project-id is-shared data revn + [conn {:keys [id name project-id is-shared revn modified-at deleted-at create-page ignore-sync-until features] :or {is-shared false revn 0 create-page true} :as params}] - (let [id (or id (:id data) (uuid/next)) + + (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED;"]) + (let [id (or id (uuid/next)) features (-> (into files/default-features features) (files/check-features-compatibility!)) - data (or data - (binding [ffeat/*current* features - ffeat/*wrap-with-objects-map-fn* (if (features "storate/objects-map") omap/wrap identity) - ffeat/*wrap-with-pointer-map-fn* (if (features "storage/pointer-map") pmap/wrap identity)] - (if create-page - (ctf/make-file-data id) - (ctf/make-file-data id nil)))) + data (binding [pmap/*tracked* (atom {}) + ffeat/*current* features + ffeat/*wrap-with-objects-map-fn* (if (features "storate/objects-map") omap/wrap identity) + ffeat/*wrap-with-pointer-map-fn* (if (features "storage/pointer-map") pmap/wrap identity)] + (let [data (if create-page + (ctf/make-file-data id) + (ctf/make-file-data id nil))] + (files/persist-pointers! conn id) + data)) features (db/create-array conn "text" features) file (db/insert! conn :file From 9143639357d5c47cb3be49560f8058be48244d49 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 16 Mar 2023 19:42:17 +0100 Subject: [PATCH 7/8] :bug: Fix incorrect webhook url validation --- backend/src/app/rpc/commands/webhooks.clj | 27 +++++++++++------------ 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/backend/src/app/rpc/commands/webhooks.clj b/backend/src/app/rpc/commands/webhooks.clj index d07d9ca33..1c6b812c5 100644 --- a/backend/src/app/rpc/commands/webhooks.clj +++ b/backend/src/app/rpc/commands/webhooks.clj @@ -48,25 +48,24 @@ (defn- validate-webhook! [cfg whook params] (when (not= (:uri whook) (:uri params)) - (try - (let [response (http/req! cfg - {:method :head - :uri (str (:uri params)) - :timeout (dt/duration "3s")} - {:sync? true})] - (when-let [hint (webhooks/interpret-response response)] - (ex/raise :type :validation - :code :webhook-validation - :hint hint))) - - (catch Throwable cause - (if-let [hint (webhooks/interpret-exception cause)] + (let [response (ex/try! + (http/req! cfg + {:method :head + :uri (str (:uri params)) + :timeout (dt/duration "3s")} + {:sync? true}))] + (if (ex/exception? response) + (if-let [hint (webhooks/interpret-exception response)] (ex/raise :type :validation :code :webhook-validation :hint hint) (ex/raise :type :internal :code :webhook-validation - :cause cause)))))) + :cause response)) + (when-let [hint (webhooks/interpret-response response)] + (ex/raise :type :validation + :code :webhook-validation + :hint hint)))))) (defn- validate-quotes! [{:keys [::db/pool]} {:keys [team-id]}] From 2818666a1a1c22830cde1f925c75230f511ce808 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 16 Mar 2023 19:42:33 +0100 Subject: [PATCH 8/8] :paperclip: Fix minnor cosmetic issue on instant and duration pretty printing --- backend/src/app/util/time.clj | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/src/app/util/time.clj b/backend/src/app/util/time.clj index a2cd237ad..69d401efa 100644 --- a/backend/src/app/util/time.clj +++ b/backend/src/app/util/time.clj @@ -128,7 +128,8 @@ (extend-protocol fez/IEdn Duration - (-edn [o] (pr-str o))) + (-edn [o] + (tagged-literal 'app/duration (str o)))) (defn format-duration [o] @@ -218,7 +219,7 @@ (extend-protocol fez/IEdn Instant - (-edn [o] (pr-str o))) + (-edn [o] (tagged-literal 'app/instant (format-instant o)))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Cron Expression