From a77f9eae7c3e4f530618ab9d857112eda3039a60 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 11 Aug 2022 07:44:47 +0200 Subject: [PATCH] :tada: Backport binfile improvements from develop --- backend/src/app/rpc/commands/binfile.clj | 712 ++++++++++---------- backend/test/app/test_files/template.penpot | Bin 0 -> 8815 bytes 2 files changed, 361 insertions(+), 351 deletions(-) create mode 100644 backend/test/app/test_files/template.penpot diff --git a/backend/src/app/rpc/commands/binfile.clj b/backend/src/app/rpc/commands/binfile.clj index e10abf969..6885f5cf5 100644 --- a/backend/src/app/rpc/commands/binfile.clj +++ b/backend/src/app/rpc/commands/binfile.clj @@ -42,7 +42,7 @@ (set! *warn-on-reflection* true) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; VARS & DEFAULTS +;; DEFAULTS ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Threshold in MiB when we pass from using @@ -50,22 +50,6 @@ (def temp-file-threshold (* 1024 1024 2)) -;; Represents the current processing file-id on -;; export process. -(def ^:dynamic *file-id*) - -;; Stores all media file object references of -;; processed files on import process. -(def ^:dynamic *media*) - -;; Stores the objects index on reamping subprocess -;; part of the import process. -(def ^:dynamic *index*) - -;; Has the current connection used on the import -;; process. -(def ^:dynamic *conn*) - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; LOW LEVEL STREAM IO API ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -211,33 +195,33 @@ (read-obj! input))) (defn write-header! - [^DataOutputStream output & {:keys [version metadata]}] + [^OutputStream output version] (l/trace :fn "write-header!" :version version - :metadata metadata :position @*position* ::l/async false) - - (doto output - (write-byte! (get-mark :header)) - (write-long! penpot-magic-number) - (write-long! version) - (write-obj! metadata))) + (let [vers (-> version name (subs 1) parse-long) + output (bs/data-output-stream output)] + (doto output + (write-byte! (get-mark :header)) + (write-long! penpot-magic-number) + (write-long! vers)))) (defn read-header! - [^DataInputStream input] + [^InputStream input] (l/trace :fn "read-header!" :position @*position* ::l/async false) - (let [mark (read-byte! input) + (let [input (bs/data-input-stream input) + mark (read-byte! input) mnum (read-long! input) vers (read-long! input)] (when (or (not= mark (get-mark :header)) (not= mnum penpot-magic-number)) (ex/raise :type :validation - :code :invalid-penpot-file)) + :code :invalid-penpot-file + :hint "invalid penpot file")) - (-> (read-obj! input) - (assoc ::version vers)))) + (keyword (str "v" vers)))) (defn copy-stream! [^OutputStream output ^InputStream input ^long size] @@ -349,8 +333,84 @@ (with-open [^AutoCloseable conn (db/open pool)] (db/exec! conn [sql:file-library-rels (db/create-array conn "uuid" ids)]))) + +(defn- create-or-update-file + [conn params] + (let [sql (str "INSERT INTO file (id, project_id, name, revn, is_shared, data, created_at, modified_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?) " + "ON CONFLICT (id) DO UPDATE SET data=?")] + (db/exec-one! conn [sql + (:id params) + (:project-id params) + (:name params) + (:revn params) + (:is-shared params) + (:data params) + (:created-at params) + (:modified-at params) + (:data params)]))) + +;; --- GENERAL PURPOSE DYNAMIC VARS + +(def ^:dynamic *state*) +(def ^:dynamic *options*) + ;; --- EXPORT WRITTER +(defn- embed-file-assets + [data conn file-id] + (letfn [(walk-map-form [form state] + (cond + (uuid? (:fill-color-ref-file form)) + (do + (vswap! state conj [(:fill-color-ref-file form) :colors (:fill-color-ref-id form)]) + (assoc form :fill-color-ref-file file-id)) + + (uuid? (:stroke-color-ref-file form)) + (do + (vswap! state conj [(:stroke-color-ref-file form) :colors (:stroke-color-ref-id form)]) + (assoc form :stroke-color-ref-file file-id)) + + (uuid? (:typography-ref-file form)) + (do + (vswap! state conj [(:typography-ref-file form) :typographies (:typography-ref-id form)]) + (assoc form :typography-ref-file file-id)) + + (uuid? (:component-file form)) + (do + (vswap! state conj [(:component-file form) :components (:component-id form)]) + (assoc form :component-file file-id)) + + :else + form)) + + (process-group-of-assets [data [lib-id items]] + ;; NOTE: there are a posibility that shape refers to a not + ;; existing file because the file was removed. In this + ;; case we just ignore the asset. + (if-let [lib (retrieve-file conn lib-id)] + (reduce (partial process-asset lib) data items) + data)) + + (process-asset [lib data [bucket asset-id]] + (let [asset (get-in lib [:data bucket asset-id]) + ;; Add a special case for colors that need to have + ;; correctly set the :file-id prop (pending of the + ;; refactor that will remove it). + asset (cond-> asset + (= bucket :colors) (assoc :file-id file-id))] + (update data bucket assoc asset-id asset)))] + + (let [assets (volatile! [])] + (walk/postwalk #(cond-> % (map? %) (walk-map-form assets)) data) + (->> (deref assets) + (filter #(as-> (first %) $ (and (uuid? $) (not= $ file-id)))) + (d/group-by first rest) + (reduce (partial process-group-of-assets) data))))) + +(defmulti write-export ::version) +(defmulti write-section ::section) + (s/def ::output bs/output-stream?) (s/def ::file-ids (s/every ::us/uuid :kind vector? :min-count 1)) (s/def ::include-libraries? (s/nilable ::us/boolean)) @@ -370,147 +430,104 @@ dependencies). `::embed-assets?`: instead of including the libraryes, embedd in the - same file library all assets used from external libraries. - " - - [{:keys [pool storage ::output ::file-ids ::include-libraries? ::embed-assets?] :as options}] - + same file library all assets used from external libraries." + [{:keys [::include-libraries? ::embed-assets?] :as options}] (us/assert! ::write-export-options options) - (us/verify! :expr (not (and include-libraries? embed-assets?)) :hint "the `include-libraries?` and `embed-assets?` are mutally excluding options") + (write-export options)) - (letfn [(write-header [output files] - (let [sections [:v1/files :v1/rels :v1/sobjects] - mdata {:penpot-version (:full cf/version) - :sections sections - :files files}] - (write-header! output :version 1 :metadata mdata))) +(defmethod write-export :default + [{:keys [::output] :as options}] + (write-header! output :v1) + (with-open [output (bs/zstd-output-stream output :level 12)] + (with-open [output (bs/data-output-stream output)] + (binding [*state* (volatile! {})] + (run! (fn [section] + (l/debug :hint "write section" :section section ::l/async false) + (write-label! output section) + (let [options (-> options + (assoc ::output output) + (assoc ::section section))] + (binding [*options* options] + (write-section options)))) - (write-files [output files sids] - (l/debug :hint "write section" :section :v1/files :total (count files) ::l/async false) - (write-label! output :v1/files) - (doseq [file-id files] - (let [file (cond-> (retrieve-file pool file-id) - embed-assets? (update :data embed-file-assets file-id)) - media (retrieve-file-media pool file)] + [:v1/metadata :v1/files :v1/rels :v1/sobjects]))))) - ;; Collect all storage ids for later write them all under - ;; specific storage objects section. - (vswap! sids into (sequence storage-object-id-xf media)) +(defmethod write-section :v1/metadata + [{:keys [pool ::output ::file-ids ::include-libraries?]}] + (let [libs (when include-libraries? + (retrieve-libraries pool file-ids)) + files (into file-ids libs)] + (write-obj! output {:version cf/version :files files}) + (vswap! *state* assoc :files files))) - (l/trace :hint "write penpot file" - :id file-id - :media (count media) - ::l/async false) +(defmethod write-section :v1/files + [{:keys [pool ::output ::embed-assets?]}] - (doto output - (write-obj! file) - (write-obj! media))))) + ;; Initialize SIDS with empty vector + (vswap! *state* assoc :sids []) - (write-rels [output files] - (let [rels (when include-libraries? (retrieve-library-relations pool files))] - (l/debug :hint "write section" :section :v1/rels :total (count rels) ::l/async false) - (doto output - (write-label! :v1/rels) - (write-obj! rels)))) + (doseq [file-id (-> *state* deref :files)] + (let [file (cond-> (retrieve-file pool file-id) + embed-assets? + (update :data embed-file-assets pool file-id)) - (write-sobjects [output sids] - (l/debug :hint "write section" - :section :v1/sobjects - :items (count sids) - ::l/async false) + media (retrieve-file-media pool file)] - ;; Write all collected storage objects - (doto output - (write-label! :v1/sobjects) - (write-obj! sids)) + (l/debug :hint "write penpot file" + :id file-id + :media (count media) + ::l/async false) - (let [storage (media/configure-assets-storage storage)] - (doseq [id sids] - (let [{:keys [size] :as obj} @(sto/get-object storage id)] - (l/trace :hint "write sobject" :id id ::l/async false) + (doto output + (write-obj! file) + (write-obj! media)) - (doto output - (write-uuid! id) - (write-obj! (meta obj))) + (vswap! *state* update :sids into storage-object-id-xf media)))) - (with-open [^InputStream stream @(sto/get-object-data storage obj)] - (let [written (write-stream! output stream size)] - (when (not= written size) - (ex/raise :type :validation - :code :mismatch-readed-size - :hint (str/ffmt "found unexpected object size; size=% written=%" size written))))))))) +(defmethod write-section :v1/rels + [{:keys [pool ::output ::include-libraries?]}] + (let [rels (when include-libraries? + (retrieve-library-relations pool (-> *state* deref :files)))] + (l/debug :hint "found rels" :total (count rels) ::l/async false) + (write-obj! output rels))) - (embed-file-assets [data file-id] - (binding [*file-id* file-id] - (let [assets (volatile! [])] - (walk/postwalk #(cond-> % (map? %) (walk-map-form assets)) data) - (->> (deref assets) - (filter #(as-> (first %) $ (and (uuid? $) (not= $ file-id)))) - (d/group-by first rest) - (reduce process-group-of-assets data))))) +(defmethod write-section :v1/sobjects + [{:keys [storage ::output]}] + (let [sids (-> *state* deref :sids) + storage (media/configure-assets-storage storage)] + (l/debug :hint "found sobjects" + :items (count sids) + ::l/async false) - (walk-map-form [form state] - (cond - (uuid? (:fill-color-ref-file form)) - (do - (vswap! state conj [(:fill-color-ref-file form) :colors (:fill-color-ref-id form)]) - (assoc form :fill-color-ref-file *file-id*)) + ;; Write all collected storage objects + (write-obj! output sids) - (uuid? (:stroke-color-ref-file form)) - (do - (vswap! state conj [(:stroke-color-ref-file form) :colors (:stroke-color-ref-id form)]) - (assoc form :stroke-color-ref-file *file-id*)) + (doseq [id sids] + (let [{:keys [size] :as obj} @(sto/get-object storage id)] + (l/debug :hint "write sobject" :id id ::l/async false) + (doto output + (write-uuid! id) + (write-obj! (meta obj))) - (uuid? (:typography-ref-file form)) - (do - (vswap! state conj [(:typography-ref-file form) :typographies (:typography-ref-id form)]) - (assoc form :typography-ref-file *file-id*)) + (with-open [^InputStream stream @(sto/get-object-data storage obj)] + (let [written (write-stream! output stream size)] + (when (not= written size) + (ex/raise :type :validation + :code :mismatch-readed-size + :hint (str/ffmt "found unexpected object size; size=% written=%" size written))))))))) - (uuid? (:component-file form)) - (do - (vswap! state conj [(:component-file form) :components (:component-id form)]) - (assoc form :component-file *file-id*)) +;; --- EXPORT READER - :else - form)) +(declare lookup-index) +(declare update-index) +(declare relink-media) +(declare relink-shapes) - (process-group-of-assets [data [lib-id items]] - ;; NOTE: there are a posibility that shape refers to a not - ;; existing file because the file was removed. In this - ;; case we just ignore the asset. - (if-let [lib (retrieve-file pool lib-id)] - (reduce #(process-asset %1 lib %2) data items) - data)) - - (process-asset [data lib [bucket asset-id]] - (let [asset (get-in lib [:data bucket asset-id]) - ;; Add a special case for colors that need to have - ;; correctly set the :file-id prop (pending of the - ;; refactor that will remove it). - asset (cond-> asset - (= bucket :colors) (assoc :file-id *file-id*))] - (update data bucket assoc asset-id asset)))] - - (with-open [output (bs/zstd-output-stream output :level 12)] - (with-open [output (bs/data-output-stream output)] - (let [libs (when include-libraries? (retrieve-libraries pool file-ids)) - files (into file-ids libs) - sids (volatile! #{})] - - ;; Write header with metadata - (l/debug :hint "exportation summary" - :files (count files) - :embed-assets? embed-assets? - :include-libs? include-libraries? - ::l/async false) - - (write-header output files) - (write-files output files sids) - (write-rels output files) - (write-sobjects output (vec @sids))))))) +(defmulti read-import ::version) +(defmulti read-section ::section) (s/def ::project-id ::us/uuid) (s/def ::input bs/input-stream?) @@ -538,31 +555,178 @@ happen with broken files; defaults to: `false`. " - [{:keys [pool storage ::project-id ::timestamp ::input ::overwrite? ::migrate? ::ignore-index-errors?] - :or {overwrite? false migrate? false timestamp (dt/now)} - :as options}] + [{:keys [::input ::timestamp] :or {timestamp (dt/now)} :as options}] + (us/verify! ::read-import-options options) + (let [version (read-header! input)] + (read-import (assoc options ::version version ::timestamp timestamp)))) - (us/assert! ::read-import-options options) +(defmethod read-import :v1 + [{:keys [pool ::input] :as options}] + (with-open [input (bs/zstd-input-stream input)] + (with-open [input (bs/data-input-stream input)] + (db/with-atomic [conn pool] + (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED;"]) + (binding [*state* (volatile! {:media [] :index {}})] + (run! (fn [section] + (l/debug :hint "reading section" :section section ::l/async false) + (assert-read-label! input section) + (let [options (-> options + (assoc ::section section) + (assoc ::input input) + (assoc :conn conn))] + (binding [*options* options] + (read-section options)))) + [:v1/metadata :v1/files :v1/rels :v1/sobjects]) - (letfn [(lookup-index [id] - (let [val (get @*index* id)] - (l/trace :fn "lookup-index" :id id :val val ::l/async false) - (when (and (not ignore-index-errors?) (not val)) - (ex/raise :type :validation - :code :incomplete-index - :hint "looks like index has missing data")) - (or val id))) - (update-index [index coll] - (loop [items (seq coll) - index index] - (if-let [id (first items)] - (let [new-id (if overwrite? id (uuid/next))] - (l/trace :fn "update-index" :id id :new-id new-id ::l/async false) - (recur (rest items) - (assoc index id new-id))) - index))) + ;; Knowing that the ids of the created files are in + ;; index, just lookup them and return it as a set + (let [files (-> *state* deref :files)] + (into #{} (keep #(get-in @*state* [:index %])) files))))))) - (process-map-form [form] +(defmethod read-section :v1/metadata + [{:keys [::input]}] + (let [{:keys [version files]} (read-obj! input)] + (l/debug :hint "metadata readed" :version (:full version) :files files ::l/async false) + (vswap! *state* update :index update-index files) + (vswap! *state* assoc :version version :files files))) + +(defmethod read-section :v1/files + [{:keys [conn ::input ::migrate? ::project-id ::timestamp ::overwrite?]}] + (doseq [expected-file-id (-> *state* deref :files)] + (let [file (read-obj! input) + media' (read-obj! input) + file-id (:id file)] + + (when (not= file-id expected-file-id) + (ex/raise :type :validation + :code :inconsistent-penpot-file + :hint "the penpot file seems corrupt, found unexpected uuid (file-id)")) + + ;; Update index using with media + (l/debug :hint "update index with media" ::l/async false) + (vswap! *state* update :index update-index (map :id media')) + + ;; Store file media for later insertion + (l/debug :hint "update media references" ::l/async false) + (vswap! *state* update :media into (map #(update % :id lookup-index)) media') + + (l/debug :hint "procesing file" :file-id file-id ::l/async false) + + (let [file-id' (lookup-index file-id) + data (-> (:data file) + (assoc :id file-id') + (cond-> migrate? (pmg/migrate-data)) + (update :pages-index relink-shapes) + (update :components relink-shapes) + (update :media relink-media)) + + params {:id file-id' + :project-id project-id + :name (str "Imported: " (:name file)) + :revn (:revn file) + :is-shared (:is-shared file) + :data (blob/encode data) + :created-at timestamp + :modified-at timestamp}] + + (l/debug :hint "create file" :id file-id' ::l/async false) + + (if overwrite? + (create-or-update-file conn params) + (db/insert! conn :file params)) + + (when overwrite? + (db/delete! conn :file-thumbnail {:file-id file-id'})))))) + +(defmethod read-section :v1/rels + [{:keys [conn ::input ::timestamp]}] + (let [rels (read-obj! input)] + ;; Insert all file relations + (doseq [rel rels] + (let [rel (-> rel + (assoc :synced-at timestamp) + (update :file-id lookup-index) + (update :library-file-id lookup-index))] + (l/debug :hint "create file library link" + :file-id (:file-id rel) + :lib-id (:library-file-id rel) + ::l/async false) + (db/insert! conn :file-library-rel rel))))) + +(defmethod read-section :v1/sobjects + [{:keys [storage conn ::input ::overwrite?]}] + (let [storage (media/configure-assets-storage storage) + ids (read-obj! input)] + + (doseq [expected-storage-id ids] + (let [id (read-uuid! input) + mdata (read-obj! input)] + + (when (not= id expected-storage-id) + (ex/raise :type :validation + :code :inconsistent-penpot-file + :hint "the penpot file seems corrupt, found unexpected uuid (storage-object-id)")) + + (l/debug :hint "readed storage object" :id id ::l/async false) + + (let [[size resource] (read-stream! input) + hash (sto/calculate-hash resource) + content (-> (sto/content resource size) + (sto/wrap-with-hash hash)) + params (-> mdata + (assoc ::sto/deduplicate? true) + (assoc ::sto/content content) + (assoc ::sto/touched-at (dt/now)) + (assoc :bucket "file-media-object")) + + sobject @(sto/put-object! storage params)] + + (l/debug :hint "persisted storage object" :id id :new-id (:id sobject) ::l/async false) + (vswap! *state* update :index assoc id (:id sobject))))) + + (doseq [item (:media @*state*)] + (l/debug :hint "inserting file media object" + :id (:id item) + :file-id (:file-id item) + ::l/async false) + + (let [file-id (lookup-index (:file-id item))] + (if (= file-id (:file-id item)) + (l/warn :hint "ignoring file media object" :file-id (:file-id item) ::l/async false) + (db/insert! conn :file-media-object + (-> item + (assoc :file-id file-id) + (d/update-when :media-id lookup-index) + (d/update-when :thumbnail-id lookup-index)) + {:on-conflict-do-nothing overwrite?})))))) + +(defn- lookup-index + [id] + (let [val (get-in @*state* [:index id])] + (l/trace :fn "lookup-index" :id id :val val ::l/async false) + (when (and (not (::ignore-index-errors? *options*)) (not val)) + (ex/raise :type :validation + :code :incomplete-index + :hint "looks like index has missing data")) + (or val id))) + +(defn- update-index + [index coll] + (loop [items (seq coll) + index index] + (if-let [id (first items)] + (let [new-id (if (::overwrite? *options*) id (uuid/next))] + (l/trace :fn "update-index" :id id :new-id new-id ::l/async false) + (recur (rest items) + (assoc index id new-id))) + index))) + +(defn- relink-shapes + "A function responsible to analyze all file data and + replace the old :component-file reference with the new + ones, using the provided file-index." + [data] + (letfn [(process-map-form [form] (cond-> form ;; Relink Image Shapes (and (map? (:metadata form)) @@ -584,189 +748,35 @@ ;; This covers the shadows and grids (they have directly ;; the :file-id prop) (uuid? (:file-id form)) - (update :file-id lookup-index))) + (update :file-id lookup-index)))] - ;; a function responsible to analyze all file data and - ;; replace the old :component-file reference with the new - ;; ones, using the provided file-index - (relink-shapes [data] - (walk/postwalk (fn [form] - (if (map? form) - (try - (process-map-form form) - (catch Throwable cause - (l/trace :hint "failed form" :form (pr-str form) ::l/async false) - (throw cause))) - form)) - data)) + (walk/postwalk (fn [form] + (if (map? form) + (try + (process-map-form form) + (catch Throwable cause + (l/warn :hint "failed form" :form (pr-str form) ::l/async false) + (throw cause))) + form)) + data))) - ;; A function responsible of process the :media attr of file - ;; data and remap the old ids with the new ones. - (relink-media [media] - (reduce-kv (fn [res k v] - (let [id (lookup-index k)] - (if (uuid? id) - (-> res - (assoc id (assoc v :id id)) - (dissoc k)) - res))) - media - media)) +(defn- relink-media + "A function responsible of process the :media attr of file data and + remap the old ids with the new ones." + [media] + (reduce-kv (fn [res k v] + (let [id (lookup-index k)] + (if (uuid? id) + (-> res + (assoc id (assoc v :id id)) + (dissoc k)) + res))) + media + media)) - (create-or-update-file [params] - (let [sql (str "INSERT INTO file (id, project_id, name, revn, is_shared, data, created_at, modified_at) " - "VALUES (?, ?, ?, ?, ?, ?, ?, ?) " - "ON CONFLICT (id) DO UPDATE SET data=?")] - (db/exec-one! *conn* [sql - (:id params) - (:project-id params) - (:name params) - (:revn params) - (:is-shared params) - (:data params) - (:created-at params) - (:modified-at params) - (:data params)]))) - - (read-files-section! [input expected-files] - (l/debug :hint "reading section" :section :v1/files ::l/async false) - (assert-read-label! input :v1/files) - - ;; Process/Read all file - (doseq [expected-file-id expected-files] - (let [file (read-obj! input) - media' (read-obj! input) - file-id (:id file)] - - (when (not= file-id expected-file-id) - (ex/raise :type :validation - :code :inconsistent-penpot-file - :hint "the penpot file seems corrupt, found unexpected uuid (file-id)")) - - - ;; Update index using with media - (l/trace :hint "update index with media" ::l/async false) - (vswap! *index* update-index (map :id media')) - - ;; Store file media for later insertion - (l/trace :hint "update media references" ::l/async false) - (vswap! *media* into (map #(update % :id lookup-index)) media') - - (l/trace :hint "procesing file" :file-id file-id ::l/async false) - - (let [file-id' (lookup-index file-id) - data (-> (:data file) - (assoc :id file-id') - (cond-> migrate? (pmg/migrate-data)) - (update :pages-index relink-shapes) - (update :components relink-shapes) - (update :media relink-media)) - - params {:id file-id' - :project-id project-id - :name (str "Imported: " (:name file)) - :revn (:revn file) - :is-shared (:is-shared file) - :data (blob/encode data) - :created-at timestamp - :modified-at timestamp}] - - (l/trace :hint "create file" :id file-id' ::l/async false) - - (if overwrite? - (create-or-update-file params) - (db/insert! *conn* :file params)) - - (when overwrite? - (db/delete! *conn* :file-thumbnail {:file-id file-id'})))))) - - (read-rels-section! [input] - (l/debug :hint "reading section" :section :v1/rels ::l/async false) - (assert-read-label! input :v1/rels) - - (let [rels (read-obj! input)] - ;; Insert all file relations - (doseq [rel rels] - (let [rel (-> rel - (assoc :synced-at timestamp) - (update :file-id lookup-index) - (update :library-file-id lookup-index))] - (l/trace :hint "create file library link" - :file-id (:file-id rel) - :lib-id (:library-file-id rel) - ::l/async false) - (db/insert! *conn* :file-library-rel rel))))) - - (read-sobjects-section! [input] - (l/debug :hint "reading section" :section :v1/sobjects ::l/async false) - (assert-read-label! input :v1/sobjects) - - (let [storage (media/configure-assets-storage storage) - ids (read-obj! input)] - - (doseq [expected-storage-id ids] - (let [id (read-uuid! input) - mdata (read-obj! input)] - - (when (not= id expected-storage-id) - (ex/raise :type :validation - :code :inconsistent-penpot-file - :hint "the penpot file seems corrupt, found unexpected uuid (storage-object-id)")) - - (l/trace :hint "readed storage object" :id id ::l/async false) - - (let [[size resource] (read-stream! input) - hash (sto/calculate-hash resource) - content (-> (sto/content resource size) - (sto/wrap-with-hash hash)) - params (-> mdata - (assoc ::sto/deduplicate? true) - (assoc ::sto/content content) - (assoc ::sto/touched-at (dt/now))) - sobject @(sto/put-object! storage params)] - (l/trace :hint "persisted storage object" :id id :new-id (:id sobject) ::l/async false) - (vswap! *index* assoc id (:id sobject))))))) - - (persist-file-media-objects! [] - (l/debug :hint "processing file media objects" :section :v1/sobjects ::l/async false) - - ;; Step 2: insert all file-media-object rows with correct - ;; storage-id reference. - (doseq [item @*media*] - (l/trace :hint "inserting file media object" - :id (:id item) - :file-id (:file-id item) - ::l/async false) - - (let [file-id (lookup-index (:file-id item))] - (if (= file-id (:file-id item)) - (l/warn :hint "ignoring file media object" :file-id (:file-id item) ::l/async false) - (db/insert! *conn* :file-media-object - (-> item - (assoc :file-id file-id) - (d/update-when :media-id lookup-index) - (d/update-when :thumbnail-id lookup-index)) - {:on-conflict-do-nothing overwrite?})))))] - - (with-open [input (bs/zstd-input-stream input)] - (with-open [input (bs/data-input-stream input)] - (db/with-atomic [conn pool] - (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED;"]) - - ;; Verify that we received a proper .penpot file - (let [{:keys [sections files]} (read-header! input)] - (l/debug :hint "import verified" :files files :overwrite? overwrite?) - (binding [*index* (volatile! (update-index {} files)) - *media* (volatile! []) - *conn* conn] - - (doseq [section sections] - (case section - :v1/rels (read-rels-section! input) - :v1/files (read-files-section! input files) - :v1/sobjects (do - (read-sobjects-section! input) - (persist-file-media-objects!))))))))))) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; HIGH LEVEL API +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (defn export! [cfg] diff --git a/backend/test/app/test_files/template.penpot b/backend/test/app/test_files/template.penpot new file mode 100644 index 0000000000000000000000000000000000000000..e0c81bb83ef474a9e190d2f99a335e58422ae9e0 GIT binary patch literal 8815 zcmX9^2RzjO|Nq<_?yTHdNygcfan6=;MxyLJ3g;ZocFvBDtca|JQf8_ziKvWZlaN`7 z>{V7|@Bhc||GE3z`~7;)*X#9uzTWTG`}H^oBX_(hA>XMD`2PnY{!ZoyaL!_dc z=B@PP@AQB@{d{~#6e$mUpvMlv$Iq4IMk12O-GaQmr(B7_L~lR;({>a7cnZN|6yt;U z^dnF1kbK8o@PWh$j5J2tg^c$lc#QktNxowsedI>+CI2R7Gs=C`KuguPZSW)QX=!1yoPq zq@!0O#^21{R)* zE4FmKk?T2X<@QNiQExD|=BQ^hY#--_{6!9(*a16%JX7#wX-|Lm5Rxm!!yM=q^ufFP z`;mMpfrY<8sr-q){(cl0$h^sJ(Y#q)tO*M!gufs8lsPGHKZ4g*l0TkI^qoOW1yaa< zUf{g5p(eW!qBj}jGH<0@JWRG*8XWi1uTa@A?5|L)*`x=N<&!GtDFDCVvIsK0#&JELL4b+`=frz&^%O8sJc4{&eDNf2 zL~0sSd{4o-aWc_6kYzSdVIKD1_$iQPO!Zy7@m@r%iz-&pO+k@JP*G4KDq#q21UZ7T z98ra+pyG;gRhE^gPg~ z1`#O}iuae1mBz>bYMP<`_*16PE<|^due#XZx&|>c$yHs|EWo$Lw&q`164!S#ZtS7s=shaERfr+|UNJxlu zh@7+^*W!RpGZ$>{nKf==z!Ko@<= zJDC{h7epoy)x}PSaMcn+%Mi(=V4|xY+0O?AY9P@PM4P&pxeO>JD6fnL1JM^$8u?T^ zC!>H40D*!F3Wb8dP$&!vqk++Y4=wo8A?QyVoYui$FtCIcK}UO9*bJZ$Pyhk@njf{aM|lL_>}ucH zU?@JH^J6{>l3c$Vu)Oyd0I=OrTS2@hfD8`m^gxKV3j;)aiYiIsmHQS&1$eCjc=q0W zKG=Y5ZhubD?R1Px1Av=O4T@s9ic5JwZ~8Y2)CdSkCb(o#ucR};Pg>}IFECI?e@|Ao zNz%k-_?GZ3nG51R$8iVnxmiDrkd(+>-lCN+`%XVJsFNAbqg%1PRIFI8V^4zKQ9U2q zzn1rbG7`88h?UBsAOL?aFe7V?3P^A*dnhmATE`F1ao#)u=*%}(ZIQn{xYNG;4}ajo5Y*(%~r=nZJE`K)eMeoc30BrH&(f4Ilry!3% z7ZTljG#nPIL3tqyFN=Mkxv26)|OiXUXexWjT%*R*6Z?dF;40e11)Mcdi;Y)mS8=>RJ z{=fl?M#7)p%#lAkKW^cex_CL9cqen_?meAL2gu&spDM@fMYCPjnBL~WWcTt*@%1pL zV6vaODGb|j*`^~@Oogxz=8EHaLieV0$NEmO?MIm8OQ1W-!;8xg_hT&QNv|c)UD0o~ zK2HDDc5mDc2^VcDSyKO2KzbkpbX}0+f6U!}zE5GtU3B9cASX$&x)~A^_&_7%fi9%8 z+;hJ-$m13WxCGxOGR+wTFsOH%Nmr=gWt&fcnYo`GE%GwfWA*3or1@71vC?&w)@ zY2$MA#6MdCP2YL1;wM$+8H<)OLQp0xl)=^fte7CXiO=-u%7L%+d+Mqt-rdD%VG;5? z&QLNvRMU~I#P~>@O(^vQc%S-8cn>+e*(BDPG-xE>pK>>!1I8*UUcI4n=M(2YSY-U< z)9Y6j_1dX0u2xe3b1mlm(E)zhG{;xz+%A2OU{(qXAM&LnT1~+848wF*O52~}+QmmA zedR(WJwmI4KTr#0LcQ0oJgdH^9>5nqt+v$~wv@QhdMC#}{rX`pzFItOr+FetpEgE( zX6r1(xrsMPXY2$piOyUOYsU@o@L9n1YvsQw9i(TVH7UQn*Lvy~>>BSL zbk)B-{P>0M>BIWnBK5JhZ~6(fZ+EhF2Hzezfw)pbfg2nREhB`EmKFkTXXpSLq?|k% zpkX=4XH1(rc**VND@B)Hb+WDM97+ks;xEpFhPJqc>np~|+Jx}cbCdR z@qRnMirTqK{ij&?Joj8jrL~`Y@px{PS0_C^z^q-m^Ca@;o2sXAxA=XP+cF-P3aEay z`xN{5B0Bob)>F!VSP71_K!0rL2R11&jNUV@Sx@5Spv33XACU{q`kEeT2O&n}uIXl#n02X`T3z|Q$l`3Hvnfpt@wVy#d$*MHKCPFddigCp*v*WGywfSBT zQlBR3O)q;-7OnY{2u)B51VUeb=wgwQ4kyub2O*M5BV2~c?K z%E#^AtvPRb!Cpc#X3&nW#yw|rwI|Y|{I#QUrlO}uUe>Yj+!GIpf(m->6Tq%1H}Kih z;*?9EEGoImk1slP;O!4*5kf;Yh6M#_pYxr{EFY^5%Y*7@y>a-n&^Vj0!E1h7b3vU! zUSIR6Q)9~b*Qi2VWkTz@4NxhXK9@+xSCHM=AI9wut`39Bipe!HI+t!Yr}jkA!s>pB zT?(TFI)xzlEr(X(-kVkK$=m>}=+bV;n#*_kLhgGrr{ZDrFGsS; z$U(&80n6K<$h+r$C?#3&n}F+5l$CL&qIql^LArK0Q5C*<0$9yE$0}Zpf5*}hSEp1N zyT!j)f603`)S)s$*~AR;wF(z0e_JNm^f<3;dPwsncK032BGV@+o&DRQ?DNd`m-C$; zj}8o1Yz~OCTt`P<_A?$;;p1E$|H?Lm8~*PDw6Kg3f}q-S{fhdRlooU28uKVP-vVp^jIddG2~8pi1D6=)`-}!W;FL~ zkr!)u5X-ZgAF2zEdf996423&u@V9EPJo_tDO5>;ov^V-ul-YwQ+qqocz_OsPH9xK| zwUWOWwaIo#D6w})7X7l|d7BwvI1kRJfdqXZC@q4P0S<-0&=oYaNOB;65->Hp{kDl! zSXR!QjYCL53-1!0p204N!OCk-;bQ1C!P8@Mm-<3K8JDWp)%d!0WFF%p+XCj9Kf(xhAFqZ>b%&CVH#9dtEo zeCz!0BCOx1X#Zd)AR(c6@YSOZA?KCflZUL}%9S%@3Xc=R)$HRCRJVa+r8h*hK#ro&dgU zY9cGwEUn}Uwu^5lX1*GbfBT7JZAo<4HQcwhhDv5o{zIu6;2ANEI+J?C{BEsHdj#G@ zyxu%LfBomWYwX1V)`5#&RaY{#jtABrR*-{}cIN}0?-!2|7F6}aLo!Q}zyoJ=@vQ>(v;p3GAEtSp6k%^DJ~f=Q zuDKE#?xUK0mrTAJuJbNIO{YXo^uA+jSC^VPIV1bm?iE))uL8&gnSZ}@ojyoU9)Ihq z)_g}^Wm&micl*F*BS@t3s@&3VgUh!BPJnWxpj*+q;=55txoRmN;Caq9cY`l@R{d)k z)H^Q*(Ga#YzV)v^*E?LyfaBlz7O(XxWS!;kT~Nzo&kz1AiOyyhnyLi)UggTze%R{z zE_FoSO1^cy-euy4h-dP(v%hq|7Cez+X&bnFPvSt;@z>ArwuUmt3csa?r4IXZOPt3w z`UcO>R``^Z@eEqkQE%$diXj&mhehPyzaA&wNxps2b16aoo9`q)vdN%TrFom)NU9_x zr@&?M%Si21kAg!rycJ^}aK^tr#^Rdi*n;?@YV7;6>OiAYE}7VKbaG^zvj#>!gnbc} zW_qnT4}pUI-v2Hc5Lq*D%P?INz-Z;;1CUMq6*%YUv~*2SA^97(B_Xeb^X}X<9RN-D ziy~mrJ1uHq&;0TPC|SFbt+}AqCKA{MCZV26a_~O|KKzLH|n-3Vu1e`m0%7Cq}ovkz{fbFJp`!()@MM+sS zo0C@ih=ut%;T@haD27;@P!0JkV0tqXA{V;`&!jQGicg!8gSxhD&=xt%@@QptW3}S5 z-p)aDg@je$LrCY@y@lw{j@3116Cq)u!uVCE-vR#~XT@pzL;4thD;N2j_K8d{J+ni(Y5S!3_#kkLmPcg>Jty&NQcbQ zUHzf@E47_s<~#rB^%(jfhMb$DQ=NN1joFVSY)$}W>JvI4{&X2p0v8~BOmBKT3Q(yG z2GOq-AC<6-aT5YW;b}A>=gtnfI+>+M>RsMy>BAVSn*s(6D{8nnMz(xYsqmKcqI6Bm zB5vVq>McPepnd{;c&VTF#B?oIN>ujlb#o;w%;50fD=&jiO9Iz-6|D0|=1#%8Gus@S z63vNU-p;1$1spK!vrp*~?a2Ct%13dL4E_b!(i&m2oT%ncamr@rtgmN?f7Qf0Y3GRF zEs-{0cjqsElW)wfXZ27xN$%dTP2UR!p$lCZm(O)9U;J$mhq_P9oh<^uhzEcKXsFKU zJiwp75!%Os1N_0D2@ezp01N<(3IYZP@KAR-sNz5=f(ivVfIKijVoC?(IRk(;fIAO> zZD%+$fKC}a$s|D^6`+UFf_HNWfJC8X&GIYO z1hh=?0qOng(M^~soHG)<^UUB#BGe#lv_ve|?cVpy80wErJ_}N6`p2=hnyKl*>-*Pg zaNCU-NI3#0kYH+BD&>~?`beem%V0Upw_?F(fBz89>NE|f=0i1Jq*w6n)xWeqv#|WL zuU!7Is?z3SEYg-Dlsy>fhg%lBiDqJo2u)3|8cR9YfJE5dji1fybyn+s_Iy37ACsDB zxfq^v6w4y14bWx8xc;RuH!$y?0Q8&O2}08Cw9wm#<}B0hMwVsQ0Zx(21Dxi&pb!yC)?8{!?T1Zq<`NkLkqKNiL%}NeTTQXkp|gk z+MTj`yJ6~su|81&bLP2%x%$H|e)nWt-9y0e`cpGYdD0y4gLm;ozj|52{2EDpr z6CoE>zAL2E%&M`Tx|;G9a|~&;aGBXZaaCP$ZT`uh2zX>O@~c{@y4$nY^|fe#ju~)# z;RaE^J)x`g`jWA}hVigojG_SKZl!*BZ$)4zXY_66-Kl%8CVfp@xoY{0XSXhb*HZax z8mw>+>gZYVM9Ui*i26i2GT~dH$3>>s=MhGsE)vU!%i-%(R$B!TXyX9s_vmOz3^yzk zzJ9<@zs^#$n=o)w=Z}WKW-fByBr#H~RU>nn-@xl|3mHYX%1`%r;ropwmiMSw-o)wF zM;_v-dDOM7l2My_o6M%MatZm8b5lH-IojgD|CqKu~CmtSO*{6aX=K_OPxi zjO9PyjNH5z(7Gri9TI%cn>FX^M?MkmMQ#0{kA?q+gqa{=YR^o>61AJnim%#cS~`n| za`gX8-FD%oiKa?SL7D!26;ZzLf=nw+G|5}LAZK(9KJLhY7wFdPSj?S~ZfLyFMy|PD zW`n%aM?ZI(vkot1E)~itfq5YoE;Nobt9&U}EKemLcY#b^1UEJShJetb1k3=mENDde zYw5^<=>ByKuE{xV3S>He%{8NK4W{6|6PH63ZAf~DK66!tE6m=C#*8Um<-Yp6Ir4&j zT;|R5DDMknk5GW3Bx8Ft5}i1$=qpUiVL_4mxl-o-lETJGH6jtBaagEfKs5Em%QVwy zHschFN>$z1%r9$Rl)1F*&)sbvH=reX5TnI~y7AAv&gfVa59&KkMIc`^sCBJe6ts7y zC7H|3Z09@Y(3kXQ-U$z^rD0V{A@YwWE+&^=xqSjWfcM*G3d^*V&>N(4WW;3mg%UG4kc8ndS;g@{vwcw_KO|`y&2_wr`^2^@26EX|#kV4=<@(GT@Bt0K4ok@!_-%`0QA6 z-+fd&Om(;cg1=Wq5&_e`&8^w@~RyZEjTH=0~=%Ord$oeT?ystGt{BHYu{l@B; z;YYfSHilYDKoSy=O^HJ!B3`MgtZr8~V$_%(-FAvTJPO}h`tioGKKwj;Q+B654@-S> zPf{nn;x6yxu-vGuF7oHVSE%>Pm2=&A+Z*N{_RXVyoTH)dN88?yOTAzB`H<}M(ZOf> ze}z>{@Z|hGb;0!uY0~~Dz{3w7V+w}9Y=AD|4~tQ+_j604UMx<$+OKT?vlyG`EU}pG z>mcH)tXsZ}NM$>+70G%86^wX1sP~sO_@7Ka0cH zMw%x-VHkpvqB35^OoigIpv1w}b*f@F4vaxe> zpio=_T-?08{QUfEX9a}?`Gk4+`1!zIa60%I1mX-IJBpp}q!Em11FnE8H)u)VSxX0} zfkDAtKpngr!XPv-W;z=9X+RNvX?C@cumT4cgpevgl<59WyM}rhENkvHYz2m-*Iy9{blCjw|7YvcF zPWd7etid&Xx8YXFg$GJ)7pK#cIx4KnJl6_}3%aE&d*tN5kN+}JZSaN|}V`&m9Xyz<5*DLg0Oy2gWo{{+Q$TT9<_ z@qT}DiDi1LWT*PWhl^5gJT%I@%z8kMDg@fN#>-G_)t~Sjj<4MYN{obpokHNVbSD6N z$i5~yq_&}};;8q-k-ennIgirzzM-K52iqs#Yr0#e7M;SUtyac-w`_;o?gxc_h=@G* zV9|TGbd(&fZ)Gjq)O~p<>Cx2w*?1Xbokhq;D|ch3EN2Q80t|SISk`?sv*M5-Hd!1b zZF&1f+5LI8{Exo)(DsB0Y`GDk_U6kyzS-@Ni2Gj}0(EEqI4f8wWlyMIYROoWe-@b&hUjcc2@E13ppK?! z?#l`N6LAlRY)S}${~8Q;+~p6wBa7$l?iDIKzPXZNy26{zUUi_;r1v(tPUF_Yy1MRu z6;)ffoT|x3l{YYhg`XX1cXO0~C|RdbMdlzywvoKMc9ECE-z#Ze-0TewG->Y`@g+Xm zmy2863==ui{}Xl1q)Lt5OT4afVK1WYV-H*Ab84Vm(Lu}O7ED0tiW0Uv@?7&)xLQVn z$8zHH7bAQO3RyRPZXDK+jl13H9B5R@9*CUk3FS7?_^KA`I@MlNTNMkD5l1o$w*3Cv z+_iY|TSd*DQ*lJ)BjxtsOqP?M(Zb>o+I9?rw{(DxmFgVLIVT<)!)0+-IZjtN4W}D3 zn;yfi3}^eU_~|yj+g_xOS3}8{UHt_OvhcDy-~Er6&+${|8+Mm|`YC_%tD>lJ6EnE!X{8HRC)TY7hkjw6v6n|rYP`NcfpjNG^DgW;mm z8Ewwhbo4I+h`+2ZnY(mlS`(u6=q}KQyN^s5GM_)k+End(dC*t~@={n8O?oB!)Xi_J z!kGOp7pYkJG4C0t`8HY)<+w9UTVDPPbcIS4b|rxfdq&kTyL)kdxI*7GGaEjpq(PH= zIgcNu)i!l12s&PSc~<17Z(3ao_`d}e0?mlQh9mOb^1oFY1ifMgTb^iew*x`QhTLyI zGVSg73!yA&`lQ3P-XSe`BZ~#1u_);S!HA}h*-av72WyZ@p~dnmg7iOMI<;UAjC3X) zZ^XL0vKX(m8+02o#MzTA5!GX9-qB8DA{|9)@g0V%3|}i4l_9SpAAUD{itGM+G4O-4 zkw%#^ozkWI9d|34M<|Wcp2!9%g+8V9#;IsD(_!RqJ z`&rj!8WD0pN}>Q89KlLW`p)toM&v`B3t^rO#Z#6Y?^xcQ(}f=GyYsJ#P4PTh68HR)HGQ7(+w8fIF;6L`oAIuI zhqQkJyYh99`(Sv_D`Frf`ialTH-Op@M?