From ecd4f326892bf6dd2996b2b6060486c367a83d6b Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 15 Nov 2023 13:24:21 +0100 Subject: [PATCH 1/8] :bug: Add missing features set forwarding on libraries fetching on workspace initialization --- frontend/src/app/main/data/workspace.cljs | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/app/main/data/workspace.cljs b/frontend/src/app/main/data/workspace.cljs index 89e3c86a7..cefd01dd4 100644 --- a/frontend/src/app/main/data/workspace.cljs +++ b/frontend/src/app/main/data/workspace.cljs @@ -298,6 +298,7 @@ (rx/map (fn [[file thumbnails team-users comments-users]] (let [bundle (-> bundle (assoc :file file) + (assoc :features features) (assoc :thumbnails thumbnails) (assoc :team-users team-users) (assoc :comments-users comments-users))] From ed1c7dcc12e4c42bc5533ae6132820d122527a62 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 15 Nov 2023 13:25:09 +0100 Subject: [PATCH 2/8] :sparkles: Set a concrete version of minio on devenv compose file --- docker/devenv/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/devenv/docker-compose.yaml b/docker/devenv/docker-compose.yaml index 9d70fa7f1..90fd8b232 100644 --- a/docker/devenv/docker-compose.yaml +++ b/docker/devenv/docker-compose.yaml @@ -70,7 +70,7 @@ services: - PENPOT_LDAP_ATTRS_PHOTO=jpegPhoto minio: - image: "minio/minio:latest" + image: "minio/minio:RELEASE.2023-11-11T08-14-41Z" command: minio server /mnt/data --console-address ":9001" volumes: From 4c190e385e0b33b8d0e319b7c9d2a674c59d530a Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 15 Nov 2023 13:26:52 +0100 Subject: [PATCH 3/8] :sparkles: Make the s3 client io-threads configurable and adaptable --- backend/src/app/config.clj | 2 ++ backend/src/app/main.clj | 8 ++++---- backend/src/app/storage/s3.clj | 15 ++++++--------- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index 3d109abc4..f5a1602af 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -203,6 +203,7 @@ (s/def ::storage-assets-s3-bucket ::us/string) (s/def ::storage-assets-s3-region ::us/keyword) (s/def ::storage-assets-s3-endpoint ::us/string) +(s/def ::storage-assets-s3-io-threads ::us/integer) (s/def ::telemetry-uri ::us/string) (s/def ::telemetry-with-taiga ::us/boolean) (s/def ::tenant ::us/string) @@ -320,6 +321,7 @@ ::storage-assets-s3-bucket ::storage-assets-s3-region ::storage-assets-s3-endpoint + ::storage-assets-s3-io-threads ::telemetry-enabled ::telemetry-uri ::telemetry-referer diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index fd2360be1..4767911bf 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -450,10 +450,10 @@ :assets-fs (ig/ref [::assets :app.storage.fs/backend])}} [::assets :app.storage.s3/backend] - {::sto.s3/region (cf/get :storage-assets-s3-region) - ::sto.s3/endpoint (cf/get :storage-assets-s3-endpoint) - ::sto.s3/bucket (cf/get :storage-assets-s3-bucket) - ::wrk/executor (ig/ref ::wrk/executor)} + {::sto.s3/region (cf/get :storage-assets-s3-region) + ::sto.s3/endpoint (cf/get :storage-assets-s3-endpoint) + ::sto.s3/bucket (cf/get :storage-assets-s3-bucket) + ::sto.s3/io-threads (cf/get :storage-assets-s3-io-threads)} [::assets :app.storage.fs/backend] {::sto.fs/directory (cf/get :storage-assets-fs-directory)} diff --git a/backend/src/app/storage/s3.clj b/backend/src/app/storage/s3.clj index 3a5cdacd0..10436cad2 100644 --- a/backend/src/app/storage/s3.clj +++ b/backend/src/app/storage/s3.clj @@ -17,7 +17,6 @@ [app.storage.impl :as impl] [app.storage.tmp :as tmp] [app.util.time :as dt] - [app.worker :as wrk] [clojure.java.io :as io] [clojure.spec.alpha :as s] [datoteka.fs :as fs] @@ -40,7 +39,6 @@ software.amazon.awssdk.core.async.AsyncRequestBody software.amazon.awssdk.core.async.AsyncResponseTransformer software.amazon.awssdk.core.client.config.ClientAsyncConfiguration - software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient software.amazon.awssdk.http.nio.netty.SdkEventLoopGroup software.amazon.awssdk.regions.Region @@ -77,9 +75,10 @@ (s/def ::bucket ::us/string) (s/def ::prefix ::us/string) (s/def ::endpoint ::us/string) +(s/def ::io-threads ::us/integer) (defmethod ig/pre-init-spec ::backend [_] - (s/keys :opt [::region ::bucket ::prefix ::endpoint ::wrk/executor])) + (s/keys :opt [::region ::bucket ::prefix ::endpoint ::io-threads])) (defmethod ig/prep-key ::backend [_ {:keys [::prefix ::region] :as cfg}] @@ -114,8 +113,7 @@ ::client ::presigner] :opt [::prefix - ::sto/id - ::wrk/executor])) + ::sto/id])) ;; --- API IMPL @@ -161,7 +159,6 @@ ;; --- HELPERS -(def default-eventloop-threads 4) (def default-timeout (dt/duration {:seconds 30})) @@ -171,18 +168,18 @@ (Region/of (name region))) (defn- build-s3-client - [{:keys [::region ::endpoint ::wrk/executor]}] + [{:keys [::region ::endpoint ::io-threads]}] (let [aconfig (-> (ClientAsyncConfiguration/builder) - (.advancedOption SdkAdvancedAsyncClientOption/FUTURE_COMPLETION_EXECUTOR executor) (.build)) sconfig (-> (S3Configuration/builder) (cond-> (some? endpoint) (.pathStyleAccessEnabled true)) (.build)) + thr-num (or io-threads (min 16 (px/get-available-processors))) hclient (-> (NettyNioAsyncHttpClient/builder) (.eventLoopGroupBuilder (-> (SdkEventLoopGroup/builder) - (.numberOfThreads (int default-eventloop-threads)))) + (.numberOfThreads (int thr-num)))) (.connectionAcquisitionTimeout default-timeout) (.connectionTimeout default-timeout) (.readTimeout default-timeout) From 1457b7cf385f08c1a948f67104f7ba7ad35bb95f Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 15 Nov 2023 13:27:40 +0100 Subject: [PATCH 4/8] :sparkles: Add performance and reporting improvements to migration script --- backend/src/app/features/components_v2.clj | 243 +++++++++++---------- backend/src/app/srepl/components_v2.clj | 93 ++++---- 2 files changed, 165 insertions(+), 171 deletions(-) diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index 20bea852d..0b7f1553e 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -45,6 +45,7 @@ [buddy.core.codecs :as bc] [cuerdas.core :as str] [datoteka.io :as io] + [promesa.exec :as px] [promesa.exec.semaphore :as ps])) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -53,6 +54,8 @@ (def ^:dynamic *system* nil) (def ^:dynamic *stats* nil) +(def ^:dynamic *file-stats* nil) +(def ^:dynamic *team-stats* nil) (def ^:dynamic *semaphore* nil) (def ^:dynamic *skip-on-error* true) @@ -380,12 +383,10 @@ fdata (d/zip components positions))))] - (when (some? *stats*) - (let [total (count components)] - (swap! *stats* (fn [stats] - (-> stats - (update :processed/components (fnil + 0) total) - (assoc :current/components total)))))) + (let [total (count components)] + (some-> *stats* (swap! update :processed/components (fnil + 0) total)) + (some-> *team-stats* (swap! update :processed/components (fnil + 0) total)) + (some-> *file-stats* (swap! assoc :processed/components total))) (-> file-data (prepare-file-data libraries) @@ -575,8 +576,7 @@ cfsh/prepare-create-artboard-from-selection) changes (pcb/concat-changes changes changes2)] - (cp/process-changes (assoc-in fdata [:options :components-v2] true) ; Process component creation in v2 way - (:redo-changes changes) false))) + (:redo-changes changes))) (defn- migrate-graphics [fdata] @@ -591,30 +591,48 @@ (grc/rect->points))] (assoc media :points points))))) - ;; FIXME: think about what to do with existing media entries ?? grid (ctst/generate-shape-grid media position grid-gap)] - (when (some? *stats*) - (let [total (count media)] - (swap! *stats* (fn [stats] - (-> stats - (update :processed/graphics (fnil + 0) total) - (assoc :current/graphics total)))))) + (let [total (count media)] + (some-> *stats* (swap! update :processed/graphics (fnil + 0) total)) + (some-> *team-stats* (swap! update :processed/graphics (fnil + 0) total)) + (some-> *file-stats* (swap! assoc :processed/graphics total))) - (->> (d/zip media grid) - (reduce (fn [fdata [mobj position]] - (try - (process-media-object fdata page-id mobj position) - (catch Throwable cause - (l/warn :hint "unable to process file media object (skiping)" - :file-id (str (:id fdata)) - :id (str (:id mobj)) - :cause cause) + (let [factory (px/thread-factory :virtual true) + executor (px/fixed-executor :parallelism 10 :factory factory) + process (fn [mobj position] + (let [tp1 (dt/tpoint)] + (try + (process-media-object fdata page-id mobj position) + (catch Throwable cause + (l/wrn :hint "unable to process file media object (skiping)" + :file-id (str (:id fdata)) + :id (str (:id mobj)) + :cause cause) - (if-not *skip-on-error* - (throw cause) - fdata)))) - fdata))))) + (if-not *skip-on-error* + (throw cause) + fdata)) + (finally + (l/trc :hint "graphic processed" + :file-id (str (:id fdata)) + :media-id (str (:id mobj)) + :elapsed (dt/format-duration (tp1))))))) + + process (px/wrap-bindings process)] + + (try + (->> (d/zip media grid) + (map (fn [[mobj position]] + (l/trc :hint "submit graphic processing" :file-id (str (:id fdata)) :id (str (:id mobj))) + (px/submit! executor (partial process mobj position)))) + (reduce (fn [fdata promise] + (if-let [changes (deref promise)] + (cp/process-changes fdata changes false) + fdata)) + fdata)) + (finally + (.close ^java.lang.AutoCloseable executor))))))) (defn- migrate-file-data [fdata libs] @@ -665,7 +683,7 @@ (when validate? (let [errors (cfv/validate-file file libs)] (when (seq errors) - (l/err :hint "migrate:file:validation-error" + (l/wrn :hint "migrate:file:validation-error" :file-id (str (:id file)) :file-name (:name file) :errors errors)))) @@ -674,43 +692,38 @@ (defn migrate-file! [system file-id & {:keys [validate?]}] + (let [tpoint (dt/tpoint) file-id (if (string? file-id) (parse-uuid file-id) file-id)] - (try - (l/dbg :hint "migrate:file:start" :file-id (str file-id)) - (let [system (update system ::sto/storage media/configure-assets-storage)] - (db/tx-run! system - (fn [{:keys [::db/conn] :as system}] - (fsnap/take-file-snapshot! system {:file-id file-id - :label "migration/components-v2"}) + (binding [*file-stats* (atom {})] + (try + (l/dbg :hint "migrate:file:start" :file-id (str file-id)) - (binding [*system* system] - (-> (db/get conn :file {:id file-id}) - (update :features db/decode-pgarray #{}) - (process-file :validate? validate?)))))) + (let [system (update system ::sto/storage media/configure-assets-storage)] + (db/tx-run! system + (fn [{:keys [::db/conn] :as system}] + (binding [*system* system] + (fsnap/take-file-snapshot! system {:file-id file-id + :label "migration/components-v2"}) + (-> (db/get conn :file {:id file-id}) + (update :features db/decode-pgarray #{}) + (process-file :validate? validate?)))))) - (finally - (let [elapsed (tpoint) - stats (some-> *stats* deref)] - (l/dbg :hint "migrate:file:end" - :file-id (str file-id) - :components (:current/components stats 0) - :graphics (:current/graphics stats 0) - :elapsed (dt/format-duration elapsed)) + (finally + (let [elapsed (tpoint) + components (get @*file-stats* :processed/components 0) + graphics (get @*file-stats* :processed/graphics 0)] - (when (some? *stats*) - (swap! *stats* (fn [stats] - (let [elapsed (inst-ms elapsed) - completed (inc (get stats :processed/files 0)) - total (+ (get stats :elapsed/total-by-file 0) elapsed) - avg (/ (double elapsed) completed)] - (-> stats - (update :elapsed/max-by-file (fnil max 0) elapsed) - (assoc :elapsed/avg-by-file avg) - (assoc :elapsed/total-by-file total) - (assoc :processed/files completed))))))))))) + (l/dbg :hint "migrate:file:end" + :file-id (str file-id) + :graphics graphics + :components components + :elapsed (dt/format-duration elapsed)) + + (some-> *stats* (swap! update :processed/files (fnil inc 0))) + (some-> *team-stats* (swap! update :processed/files (fnil inc 0))))))))) (defn migrate-team! [system team-id & {:keys [validate?]}] @@ -719,72 +732,66 @@ (parse-uuid team-id) team-id)] (l/dbg :hint "migrate:team:start" :team-id (dm/str team-id)) - (try - ;; We execute this out of transaction because we want this - ;; change to be visible to all other sessions before starting - ;; the migration - (let [sql (str "UPDATE team SET features = " - " array_append(features, 'ephimeral/v2-migration') " - " WHERE id = ?")] - (db/exec-one! system [sql team-id])) + (binding [*team-stats* (atom {})] + (try + ;; We execute this out of transaction because we want this + ;; change to be visible to all other sessions before starting + ;; the migration + (let [sql (str "UPDATE team SET features = " + " array_append(features, 'ephimeral/v2-migration') " + " WHERE id = ?")] + (db/exec-one! system [sql team-id])) - (db/tx-run! system - (fn [{:keys [::db/conn] :as system}] - ;; Lock the team - (db/exec-one! conn ["SET idle_in_transaction_session_timeout = 0"]) + (db/tx-run! system + (fn [{:keys [::db/conn] :as system}] + ;; Lock the team + (db/exec-one! conn ["SET idle_in_transaction_session_timeout = 0"]) - (let [{:keys [features] :as team} (-> (db/get conn :team {:id team-id}) - (update :features db/decode-pgarray #{}))] + (let [{:keys [features] :as team} (-> (db/get conn :team {:id team-id}) + (update :features db/decode-pgarray #{}))] - (if (contains? features "components/v2") - (l/dbg :hint "team already migrated") - (let [sql (str/concat - "SELECT f.id FROM file AS f " - " JOIN project AS p ON (p.id = f.project_id) " - "WHERE p.team_id = ? AND f.deleted_at IS NULL AND p.deleted_at IS NULL " - "FOR UPDATE") + (if (contains? features "components/v2") + (l/dbg :hint "team already migrated") + (let [sql (str/concat + "SELECT f.id FROM file AS f " + " JOIN project AS p ON (p.id = f.project_id) " + "WHERE p.team_id = ? AND f.deleted_at IS NULL AND p.deleted_at IS NULL " + "FOR UPDATE") - rows (->> (db/exec! conn [sql team-id]) - (map :id))] + rows (->> (db/exec! conn [sql team-id]) + (map :id))] - (run! #(migrate-file! system % :validate? validate?) rows) - (some-> *stats* (swap! assoc :current/files (count rows))) + (run! #(migrate-file! system % :validate? validate?) rows) - (let [features (-> features - (disj "ephimeral/v2-migration") - (conj "components/v2") - (conj "layout/grid") - (conj "styles/v2"))] - (db/update! conn :team - {:features (db/create-array conn "text" features)} - {:id team-id}))))))) - (finally - (some-> *semaphore* ps/release!) - (let [elapsed (tpoint) - stats (some-> *stats* deref)] - (when (some? *stats*) - (swap! *stats* (fn [stats] - (let [elapsed (inst-ms elapsed) - completed (inc (get stats :processed/teams 0)) - total (+ (get stats :elapsed/total-by-team 0) elapsed) - avg (/ (double elapsed) completed)] - (-> stats - (update :elapsed/max-by-team (fnil max 0) elapsed) - (assoc :elapsed/avg-by-team avg) - (assoc :elapsed/total-by-team total) - (assoc :processed/teams completed)))))) + (let [features (-> features + (disj "ephimeral/v2-migration") + (conj "components/v2") + (conj "layout/grid") + (conj "styles/v2"))] + (db/update! conn :team + {:features (db/create-array conn "text" features)} + {:id team-id}))))))) + (finally + (some-> *semaphore* ps/release!) + (let [elapsed (tpoint)] + (some-> *stats* (swap! update :processed/teams (fnil inc 0))) - ;; We execute this out of transaction because we want this - ;; change to be visible to all other sessions before starting - ;; the migration - (let [sql (str "UPDATE team SET features = " - " array_remove(features, 'ephimeral/v2-migration') " - " WHERE id = ?")] - (db/exec-one! system [sql team-id])) + ;; We execute this out of transaction because we want this + ;; change to be visible to all other sessions before starting + ;; the migration + (let [sql (str "UPDATE team SET features = " + " array_remove(features, 'ephimeral/v2-migration') " + " WHERE id = ?")] + (db/exec-one! system [sql team-id])) - (l/dbg :hint "migrate:team:end" - :team-id (dm/str team-id) - :files (:current/files stats 0) - :elapsed (dt/format-duration elapsed))))))) + (let [components (get @*team-stats* :processed/components 0) + graphics (get @*team-stats* :processed/graphics 0) + files (get @*team-stats* :processed/files 0)] + (l/dbg :hint "migrate:team:end" + :team-id (dm/str team-id) + :files files + :components components + :graphics graphics + :elapsed (dt/format-duration elapsed))))))))) diff --git a/backend/src/app/srepl/components_v2.clj b/backend/src/app/srepl/components_v2.clj index e8ffb1a7f..8a14ccbc5 100644 --- a/backend/src/app/srepl/components_v2.clj +++ b/backend/src/app/srepl/components_v2.clj @@ -21,17 +21,9 @@ (defn- print-stats! [stats] - (let [stats (-> stats - (d/update-when :elapsed/max-by-team (comp dt/format-duration dt/duration int)) - (d/update-when :elapsed/avg-by-team (comp dt/format-duration dt/duration int)) - (d/update-when :elapsed/total-by-team (comp dt/format-duration dt/duration int)) - (d/update-when :elapsed/max-by-file (comp dt/format-duration dt/duration int)) - (d/update-when :elapsed/avg-by-file (comp dt/format-duration dt/duration int)) - (d/update-when :elapsed/total-by-file (comp dt/format-duration dt/duration int)) - )] - (->> stats - (into (sorted-map)) - (pp/pprint)))) + (->> stats + (into (sorted-map)) + (pp/pprint))) (defn- report-progress-files [tpoint] @@ -42,7 +34,7 @@ completed (:processed/files newv) progress (/ (* completed 100.0) total) elapsed (tpoint)] - (l/trc :hint "progress" + (l/dbg :hint "progress" :completed (:processed/files newv) :total (:total/files newv) :progress (str (int progress) "%") @@ -57,8 +49,11 @@ completed (:processed/teams newv) progress (/ (* completed 100.0) total) elapsed (tpoint)] - (l/trc :hint "progress" - :completed (:processed/teams newv) + (l/dbg :hint "progress" + :completed-teams (:processed/teams newv) + :completed-files (:processed/files newv) + :completed-graphics (:processed/graphics newv) + :completed-components (:processed/components newv) :progress (str (int progress) "%") :elapsed (dt/format-duration elapsed)))))) @@ -88,36 +83,35 @@ (:count res))) (defn migrate-file! - [system file-id & {:keys [rollback] :or {rollback true}}] + [system file-id & {:keys [rollback?] :or {rollback? true}}] (l/dbg :hint "migrate:start") (let [tpoint (dt/tpoint)] (try (binding [feat/*stats* (atom {})] - (-> (assoc system ::db/rollback rollback) + (-> (assoc system ::db/rollback rollback?) (feat/migrate-file! file-id)) + (-> (deref feat/*stats*) - (assoc :elapsed (dt/format-duration (tpoint))) - (dissoc :current/graphics) - (dissoc :current/components) - (dissoc :current/files))) + (assoc :elapsed (dt/format-duration (tpoint))))) (catch Throwable cause - (l/dbg :hint "migrate:error" :cause cause)) + (l/wrn :hint "migrate:error" :cause cause)) (finally (let [elapsed (dt/format-duration (tpoint))] (l/dbg :hint "migrate:end" :elapsed elapsed)))))) (defn migrate-files! - [{:keys [::db/pool] :as system} & {:keys [chunk-size max-jobs max-items start-at preset rollback skip-on-error validate] - :or {chunk-size 10 - skip-on-error true - max-jobs 10 - max-items Long/MAX_VALUE - preset :shutdown-on-failure - rollback true - validate false}}] + [{:keys [::db/pool] :as system} + & {:keys [chunk-size max-jobs max-items start-at preset rollback? skip-on-error validate?] + :or {chunk-size 10 + skip-on-error true + max-jobs 10 + max-items Long/MAX_VALUE + preset :shutdown-on-failure + rollback? true + validate? false}}] (letfn [(get-chunk [cursor] (let [sql (str/concat "SELECT id, created_at FROM file " @@ -151,17 +145,14 @@ (run! (fn [file-id] (ps/acquire! feat/*semaphore*) (px/submit! scope (fn [] - (-> (assoc system ::db/rollback rollback) - (feat/migrate-file! file-id :validate? validate))))) + (-> (assoc system ::db/rollback rollback?) + (feat/migrate-file! file-id :validate? validate?))))) (get-candidates)) (p/await! scope)) (-> (deref feat/*stats*) - (assoc :elapsed (dt/format-duration (tpoint))) - (dissoc :current/graphics) - (dissoc :current/components) - (dissoc :current/files)) + (assoc :elapsed (dt/format-duration (tpoint)))) (catch Throwable cause (l/dbg :hint "migrate:error" :cause cause)) @@ -172,8 +163,8 @@ (defn migrate-team! [{:keys [::db/pool] :as system} team-id - & {:keys [rollback skip-on-error validate] - :or {rollback true skip-on-error true validate false}}] + & {:keys [rollback? skip-on-error validate?] + :or {rollback? true skip-on-error true validate? false}}] (l/dbg :hint "migrate:start") (let [total (get-total-files pool :team-id team-id) @@ -185,15 +176,13 @@ (try (binding [feat/*stats* stats feat/*skip-on-error* skip-on-error] - (-> (assoc system ::db/rollback rollback) - (feat/migrate-team! team-id :validate? validate)) + (-> (assoc system ::db/rollback rollback?) + (feat/migrate-team! team-id :validate? validate?)) (print-stats! (-> (deref feat/*stats*) (dissoc :total/files) - (dissoc :current/graphics) - (dissoc :current/components) - (dissoc :current/files)))) + (assoc :elapsed (dt/format-duration (tpoint)))))) (catch Throwable cause (l/dbg :hint "migrate:error" :cause cause)) @@ -204,14 +193,14 @@ (defn migrate-teams! [{:keys [::db/pool] :as system} - & {:keys [chunk-size max-jobs max-items start-at rollback preset skip-on-error max-time validate] + & {:keys [chunk-size max-jobs max-items start-at rollback? preset skip-on-error max-time validate?] :or {chunk-size 10000 - rollback true + validate? false + rollback? true skip-on-error true preset :shutdown-on-failure max-jobs Integer/MAX_VALUE - max-items Long/MAX_VALUE - validate false}}] + max-items Long/MAX_VALUE}}] (letfn [(get-chunk [cursor] (let [sql (str/concat @@ -233,8 +222,8 @@ (migrate-team [team-id] (try - (-> (assoc system ::db/rollback rollback) - (feat/migrate-team! team-id :validate? validate)) + (-> (assoc system ::db/rollback rollback?) + (feat/migrate-team! team-id :validate? validate?)) (catch Throwable cause (l/err :hint "unexpected error on processing team" :team-id (dm/str team-id) :cause cause)))) @@ -242,7 +231,7 @@ (ps/acquire! feat/*semaphore*) (let [ts (tpoint)] (if (and mtime (neg? (compare mtime ts))) - (l/trc :hint "max time constraint reached" :elapsed (dt/format-duration ts)) + (l/inf :hint "max time constraint reached" :elapsed (dt/format-duration ts)) (px/submit! scope (partial migrate-team team-id)))))] (l/dbg :hint "migrate:start") @@ -270,10 +259,8 @@ (print-stats! (-> (deref feat/*stats*) - (dissoc :total/teams) - (dissoc :current/graphics) - (dissoc :current/components) - (dissoc :current/files))) + (assoc :elapsed/total (dt/format-duration (tpoint))) + (dissoc :total/teams))) (catch Throwable cause (l/dbg :hint "migrate:error" :cause cause)) From f1782f746da2fd94794f63529d66a6fba35f5282 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 15 Nov 2023 13:28:06 +0100 Subject: [PATCH 5/8] :sparkles: Add better error reporting on load-pointer function --- backend/src/app/rpc/commands/files.clj | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/backend/src/app/rpc/commands/files.clj b/backend/src/app/rpc/commands/files.clj index 7db6cd78a..7835a71ca 100644 --- a/backend/src/app/rpc/commands/files.clj +++ b/backend/src/app/rpc/commands/files.clj @@ -170,11 +170,18 @@ (defn load-pointer [conn file-id id] - (let [row (db/get conn :file-data-fragment - {:id id :file-id file-id} - {:columns [:content] - ::db/check-deleted? false})] - (blob/decode (:content row)))) + (let [{:keys [content]} (db/get conn :file-data-fragment + {:id id :file-id file-id} + {:columns [:content] + ::db/check-deleted? false})] + (when-not content + (ex/raise :type :internal + :code :fragment-not-found + :hint "fragment not found" + :file-id file-id + :fragment-id id)) + + (blob/decode content))) (defn- load-all-pointers! [{:keys [data] :as file}] From c022b71b5912220ea77bf8d5c50a0aa4dc6a4292 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 15 Nov 2023 13:28:44 +0100 Subject: [PATCH 6/8] :sparkles: Add better error reporting on template clone operation --- frontend/src/app/main/data/dashboard.cljs | 2 +- frontend/src/app/main/errors.cljs | 7 +++++++ frontend/src/app/main/ui/dashboard/import.cljs | 12 +++++++----- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/frontend/src/app/main/data/dashboard.cljs b/frontend/src/app/main/data/dashboard.cljs index 52e9ea67a..de6480c43 100644 --- a/frontend/src/app/main/data/dashboard.cljs +++ b/frontend/src/app/main/data/dashboard.cljs @@ -920,8 +920,8 @@ (rx/tap on-success) (rx/catch on-error)))))) - ;; --- EVENT: clone-template + (defn clone-template [{:keys [template-id project-id] :as params}] (dm/assert! (uuid? project-id)) diff --git a/frontend/src/app/main/errors.cljs b/frontend/src/app/main/errors.cljs index a17b5bada..a9880d326 100644 --- a/frontend/src/app/main/errors.cljs +++ b/frontend/src/app/main/errors.cljs @@ -50,6 +50,13 @@ (finally (js/console.groupEnd message)))) +(defn print-cause! + [message cause] + (print-group! message (fn [] + (print-data! cause) + (print-explain! cause) + (print-trace! cause)))) + (defn on-error "A general purpose error handler." [error] diff --git a/frontend/src/app/main/ui/dashboard/import.cljs b/frontend/src/app/main/ui/dashboard/import.cljs index 4edb59bf6..0125aaab7 100644 --- a/frontend/src/app/main/ui/dashboard/import.cljs +++ b/frontend/src/app/main/ui/dashboard/import.cljs @@ -13,6 +13,7 @@ [app.main.data.events :as ev] [app.main.data.messages :as msg] [app.main.data.modal :as modal] + [app.main.errors :as errors] [app.main.features :as features] [app.main.store :as st] [app.main.ui.components.file-uploader :refer [file-uploader]] @@ -296,10 +297,10 @@ (st/emit! (dd/fetch-recent-files))) on-template-cloned-error - (fn [] - (st/emit! - (modal/hide) - (msg/error (tr "dashboard.libraries-and-templates.import-error")))) + (fn [cause] + (errors/print-cause! "Template Clone Error" cause) + (st/emit! (modal/hide) + (msg/error (tr "dashboard.libraries-and-templates.import-error")))) continue-files (fn [] @@ -314,7 +315,8 @@ continue-template (fn [] - (let [mdata {:on-success on-template-cloned-success :on-error on-template-cloned-error} + (let [mdata {:on-success on-template-cloned-success + :on-error on-template-cloned-error} params {:project-id project-id :template-id (:id template)}] (swap! state (fn [state] From aee516e642ef16a76e5bced7ccc543785495309b Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 15 Nov 2023 13:35:34 +0100 Subject: [PATCH 7/8] :sparkles: Unify system binding on devenv repl and nrepl --- backend/dev/user.clj | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/backend/dev/user.clj b/backend/dev/user.clj index 043679aeb..697826f42 100644 --- a/backend/dev/user.clj +++ b/backend/dev/user.clj @@ -94,22 +94,14 @@ (defn- start [] (try - (alter-var-root #'system (fn [sys] - (when sys (ig/halt! sys)) - (-> main/system-config - (cond-> (contains? cf/flags :backend-worker) - (merge main/worker-config)) - (ig/prep) - (ig/init)))) + (main/start) :started (catch Throwable cause (ex/print-throwable cause)))) (defn- stop [] - (alter-var-root #'system (fn [sys] - (when sys (ig/halt! sys)) - nil)) + (main/stop) :stopped) (defn restart From d03577987ed2fe20789419318244028a5183614f Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 15 Nov 2023 13:38:57 +0100 Subject: [PATCH 8/8] :sparkles: Ignore recent colors on importing a library --- backend/src/app/rpc/commands/binfile.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/app/rpc/commands/binfile.clj b/backend/src/app/rpc/commands/binfile.clj index 852516ebf..83f5e554a 100644 --- a/backend/src/app/rpc/commands/binfile.clj +++ b/backend/src/app/rpc/commands/binfile.clj @@ -731,6 +731,7 @@ (assoc :modified-at timestamp) (update :data (fn [data] (-> data + (dissoc :recent-colors) (assoc :id file-id') (cond-> (> (:version data) cfd/version) (assoc :version cfd/version))