From 0f04b863167422d2ebbe2c760a2c2980e3ee1238 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 19 Jan 2022 16:08:08 +0100 Subject: [PATCH] :zap: Improve performance on import .penpot files. --- CHANGES.md | 1 + backend/src/app/rpc/mutations/files.clj | 46 ++++++++++++++++++++++--- frontend/src/app/worker/import.cljs | 46 ++++++++++--------------- 3 files changed, 62 insertions(+), 31 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index bfa08ef7f..1ef7929b0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -22,6 +22,7 @@ - Add actions to go to main component context menu option [Taiga #2053](https://tree.taiga.io/project/penpot/us/2053). - Add contrast between component select color and shape select color [Taiga #2121](https://tree.taiga.io/project/penpot/issue/2121). - Add animations in interactions [Taiga #2244](https://tree.taiga.io/project/penpot/us/2244). +- Add performance improvements on .penpot file import process [Taiga 2497](https://tree.taiga.io/project/penpot/us/2497). ### :bug: Bugs fixed diff --git a/backend/src/app/rpc/mutations/files.clj b/backend/src/app/rpc/mutations/files.clj index 4e787d43d..c145e2ceb 100644 --- a/backend/src/app/rpc/mutations/files.clj +++ b/backend/src/app/rpc/mutations/files.clj @@ -414,7 +414,9 @@ [conn project-id] (:team-id (db/get-by-id conn :project project-id {:columns [:team-id]}))) -;; TEMPORARY FILE CREATION +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; TEMPORARY FILES (behaves differently) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (s/def ::create-temp-file ::create-file) @@ -424,6 +426,23 @@ (proj/check-edition-permissions! conn profile-id project-id) (create-file conn (assoc params :deleted-at (dt/in-future {:days 1}))))) +(s/def ::update-temp-file + (s/keys :req-un [::changes ::revn ::session-id ::id])) + +(sv/defmethod ::update-temp-file + [{:keys [pool] :as cfg} {:keys [profile-id session-id id revn changes] :as params}] + (db/with-atomic [conn pool] + (db/insert! conn :file-change + {:id (uuid/next) + :session-id session-id + :profile-id profile-id + :created-at (dt/now) + :file-id id + :revn revn + :data nil + :changes (blob/encode changes)}) + nil)) + (s/def ::persist-temp-file (s/keys :req-un [::id ::profile-id])) @@ -431,6 +450,25 @@ [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}] (db/with-atomic [conn pool] (files/check-edition-permissions! conn profile-id id) - (db/update! conn :file - {:deleted-at nil} - {:id id}))) + (let [file (db/get-by-id conn :file id) + revs (db/query conn :file-change + {:file-id id} + {:order-by [[:revn :asc]]}) + revn (count revs)] + + (when (nil? (:deleted-at file)) + (ex/raise :type :validation + :code :cant-persist-already-persisted-file)) + + (loop [revs (seq revs) + data (blob/decode (:data file))] + (if-let [rev (first revs)] + (recur (rest revs) + (->> rev :changes blob/decode (cp/process-changes data))) + (db/update! conn :file + {:deleted-at nil + :revn revn + :data (blob/encode data)} + {:id id}))) + + nil))) diff --git a/frontend/src/app/worker/import.cljs b/frontend/src/app/worker/import.cljs index 5dcfa3f52..983e5d83d 100644 --- a/frontend/src/app/worker/import.cljs +++ b/frontend/src/app/worker/import.cljs @@ -141,38 +141,30 @@ (rx/map #(hash-map :file-id file-id :library-id %)) (rx/flat-map (partial rp/mutation :link-file-to-library))))) -(defn persist-file [file] - (rp/mutation :persist-temp-file {:id (:id file)})) - (defn send-changes "Creates batches of changes to be sent to the backend" [context file] - (let [revn (atom (:revn file)) - file-id (:id file) + (let [file-id (:id file) session-id (uuid/next) - changes-batches - (->> (fb/generate-changes file) - (partition change-batch-size change-batch-size nil) - (mapv vec)) + batches (->> (fb/generate-changes file) + (partition change-batch-size change-batch-size nil) + (mapv vec)) - current (atom 0) - total (count changes-batches)] + processed (atom 0) + total (count batches)] (rx/concat - (->> (rx/from changes-batches) - (rx/mapcat - (fn [change-batch] - (->> (rp/mutation :update-file + (->> (rx/from (d/enumerate batches)) + (rx/merge-map + (fn [[i change-batch]] + (->> (rp/mutation :update-temp-file {:id file-id :session-id session-id - :revn @revn + :revn i :changes change-batch}) - (rx/tap #(do (swap! current inc) - (progress! context - :upload-data @current total)))))) - + (rx/tap #(do (swap! processed inc) + (progress! context :upload-data @processed total)))))) (rx/map first) - (rx/tap #(reset! revn (:revn %))) (rx/ignore)) (->> (rp/mutation :persist-temp-file {:id file-id}) @@ -340,7 +332,7 @@ file (-> file (fb/add-page page-data))] (->> (rx/from nodes) (rx/filter cip/shape?) - (rx/mapcat (partial resolve-media context file-id)) + (rx/merge-map (partial resolve-media context file-id)) (rx/reduce (partial process-import-node context) file) (rx/map (comp fb/close-page setup-interactions))))) @@ -362,7 +354,7 @@ (rx/filter cip/shape?) (rx/skip 1) (rx/skip-last 1) - (rx/mapcat (partial resolve-media context file-id)) + (rx/merge-map (partial resolve-media context file-id)) (rx/reduce (partial process-import-node context) file) (rx/map fb/finish-component)))) @@ -382,6 +374,7 @@ (fn [[page-id page-name]] (->> (get-file context :page page-id) (rx/map (fn [page-data] [page-id page-name page-data]))))) + (rx/concat-reduce (partial import-page context) file)))) (defn process-library-colors @@ -420,7 +413,7 @@ (let [resolve (:resolve context)] (->> (get-file context :media-list) (rx/flat-map (comp d/kebab-keys cip/string->uuid)) - (rx/mapcat + (rx/merge-map (fn [[id media]] (let [media (assoc media :id (resolve id))] (->> (get-file context :media id media) @@ -432,12 +425,11 @@ :content content :is-local false}))) (rx/tap #(progress! context :upload-media (:name %))) - (rx/flat-map #(rp/mutation! :upload-file-media-object %)) + (rx/merge-map #(rp/mutation! :upload-file-media-object %)) (rx/map (constantly media)) (rx/catch #(do (.error js/console (str "Error uploading media: " (:name media)) ) (rx/empty))))))) (rx/reduce fb/add-library-media file))) - (rx/of file))) (defn process-library-components @@ -511,7 +503,7 @@ (fn [[file data]] (->> (uz/load-from-url (:uri data)) (rx/map #(-> context (assoc :zip %) (merge data))) - (rx/flat-map + (rx/merge-map (fn [context] ;; process file retrieves a stream that will emit progress notifications ;; and other that will emit the files once imported