Penpot/penpot
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-03-20 19:51:23 -05:00
Code Issues Activity

⚡ Improve performance on import .penpot files.

Browse source
This commit is contained in:
Andrey Antukh 2022-01-19 16:08:08 +01:00 • committed by Andrés Moya
parent 1dae8a0771
commit 0f04b86316
3 changed files with 62 additions and 31 deletions
CHANGES.md
backend/src/app/rpc/mutations
files.clj
frontend/src/app/worker
import.cljs

1
CHANGES.md
View file

@ -22,6 +22,7 @@
- Add actions to go to main component context menu option [Taiga #2053](https://tree.taiga.io/project/penpot/us/2053).
- Add contrast between component select color and shape select color [Taiga #2121](https://tree.taiga.io/project/penpot/issue/2121).
- Add animations in interactions [Taiga #2244](https://tree.taiga.io/project/penpot/us/2244).
- Add performance improvements on .penpot file import process [Taiga 2497](https://tree.taiga.io/project/penpot/us/2497).
### :bug: Bugs fixed

46
backend/src/app/rpc/mutations/files.clj
View file

@ -414,7 +414,9 @@
[conn project-id]
(:team-id (db/get-by-id conn :project project-id {:columns [:team-id]})))
;; TEMPORARY FILE CREATION
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; TEMPORARY FILES (behaves differently)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(s/def ::create-temp-file ::create-file)
@ -424,6 +426,23 @@
(proj/check-edition-permissions! conn profile-id project-id)
(create-file conn (assoc params :deleted-at (dt/in-future {:days 1})))))
(s/def ::update-temp-file
(s/keys :req-un [::changes ::revn ::session-id ::id]))
(sv/defmethod ::update-temp-file
[{:keys [pool] :as cfg} {:keys [profile-id session-id id revn changes] :as params}]
(db/with-atomic [conn pool]
(db/insert! conn :file-change
{:id (uuid/next)
:session-id session-id
:profile-id profile-id
:created-at (dt/now)
:file-id id
:revn revn
:data nil
:changes (blob/encode changes)})
nil))
(s/def ::persist-temp-file
(s/keys :req-un [::id ::profile-id]))
@ -431,6 +450,25 @@
[{:keys [pool] :as cfg} {:keys [id profile-id] :as params}]
(db/with-atomic [conn pool]
(files/check-edition-permissions! conn profile-id id)
(db/update! conn :file
{:deleted-at nil}
{:id id})))
(let [file (db/get-by-id conn :file id)
revs (db/query conn :file-change
{:file-id id}
{:order-by [[:revn :asc]]})
revn (count revs)]
(when (nil? (:deleted-at file))
(ex/raise :type :validation
:code :cant-persist-already-persisted-file))
(loop [revs (seq revs)
data (blob/decode (:data file))]
(if-let [rev (first revs)]
(recur (rest revs)
(->> rev :changes blob/decode (cp/process-changes data)))
(db/update! conn :file
{:deleted-at nil
:revn revn
:data (blob/encode data)}
{:id id})))
nil)))

46
frontend/src/app/worker/import.cljs
View file

@ -141,38 +141,30 @@
(rx/map #(hash-map :file-id file-id :library-id %))
(rx/flat-map (partial rp/mutation :link-file-to-library)))))
(defn persist-file [file]
(rp/mutation :persist-temp-file {:id (:id file)}))
(defn send-changes
"Creates batches of changes to be sent to the backend"
[context file]
(let [revn (atom (:revn file))
file-id (:id file)
(let [file-id (:id file)
session-id (uuid/next)
changes-batches
(->> (fb/generate-changes file)
(partition change-batch-size change-batch-size nil)
(mapv vec))
batches (->> (fb/generate-changes file)
(partition change-batch-size change-batch-size nil)
(mapv vec))
current (atom 0)
total (count changes-batches)]
processed (atom 0)
total (count batches)]
(rx/concat
(->> (rx/from changes-batches)
(rx/mapcat
(fn [change-batch]
(->> (rp/mutation :update-file
(->> (rx/from (d/enumerate batches))
(rx/merge-map
(fn [[i change-batch]]
(->> (rp/mutation :update-temp-file
{:id file-id
:session-id session-id
:revn @revn
:revn i
:changes change-batch})
(rx/tap #(do (swap! current inc)
(progress! context
:upload-data @current total))))))
(rx/tap #(do (swap! processed inc)
(progress! context :upload-data @processed total))))))
(rx/map first)
(rx/tap #(reset! revn (:revn %)))
(rx/ignore))
(->> (rp/mutation :persist-temp-file {:id file-id})
@ -340,7 +332,7 @@
file (-> file (fb/add-page page-data))]
(->> (rx/from nodes)
(rx/filter cip/shape?)
(rx/mapcat (partial resolve-media context file-id))
(rx/merge-map (partial resolve-media context file-id))
(rx/reduce (partial process-import-node context) file)
(rx/map (comp fb/close-page setup-interactions)))))
@ -362,7 +354,7 @@
(rx/filter cip/shape?)
(rx/skip 1)
(rx/skip-last 1)
(rx/mapcat (partial resolve-media context file-id))
(rx/merge-map (partial resolve-media context file-id))
(rx/reduce (partial process-import-node context) file)
(rx/map fb/finish-component))))
@ -382,6 +374,7 @@
(fn [[page-id page-name]]
(->> (get-file context :page page-id)
(rx/map (fn [page-data] [page-id page-name page-data])))))
(rx/concat-reduce (partial import-page context) file))))
(defn process-library-colors
@ -420,7 +413,7 @@
(let [resolve (:resolve context)]
(->> (get-file context :media-list)
(rx/flat-map (comp d/kebab-keys cip/string->uuid))
(rx/mapcat
(rx/merge-map
(fn [[id media]]
(let [media (assoc media :id (resolve id))]
(->> (get-file context :media id media)
@ -432,12 +425,11 @@
:content content
:is-local false})))
(rx/tap #(progress! context :upload-media (:name %)))
(rx/flat-map #(rp/mutation! :upload-file-media-object %))
(rx/merge-map #(rp/mutation! :upload-file-media-object %))
(rx/map (constantly media))
(rx/catch #(do (.error js/console (str "Error uploading media: " (:name media)) )
(rx/empty)))))))
(rx/reduce fb/add-library-media file)))
(rx/of file)))
(defn process-library-components
@ -511,7 +503,7 @@
(fn [[file data]]
(->> (uz/load-from-url (:uri data))
(rx/map #(-> context (assoc :zip %) (merge data)))
(rx/flat-map
(rx/merge-map
(fn [context]
;; process file retrieves a stream that will emit progress notifications
;; and other that will emit the files once imported