Penpot/penpot
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-04-06 12:01:19 -05:00
Code Issues Activity

✨ Add optional validation when migrating files to components-v2

Browse source
This commit is contained in:
Andrés Moya 2023-11-13 17:24:43 +01:00 • committed by Andrey Antukh
parent 3dfd54d8e2
commit 451d6c1d7b
2 changed files with 20 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
backend/src/app/features/components_v2.clj
View file

@ -13,6 +13,7 @@
[app.common.files.libraries-helpers :as cflh]
[app.common.files.migrations :as pmg]
[app.common.files.shapes-helpers :as cfsh]
[app.common.files.validate :as cfv]
[app.common.geom.point :as gpt]
[app.common.geom.rect :as grc]
[app.common.geom.shapes :as gsh]
@ -623,7 +624,7 @@
(update fdata :options assoc :components-v2 true)))))
(defn- process-file
[{:keys [id] :as file}]
[{:keys [id] :as file} & {:keys [validate?]}]
(let [conn (::db/conn *system*)]
(binding [pmap/*tracked* (atom {})
pmap/*load-fn* (partial files/load-pointer conn id)
@ -659,10 +660,13 @@
:revn (:revn file)}
{:id (:id file)})
(when validate?
(cfv/validate-file file libs :throw? true))
(dissoc file :data)))))
(defn migrate-file!
[system file-id]
[system file-id & {:keys [validate?]}]
(let [tpoint (dt/tpoint)
file-id (if (string? file-id)
(parse-uuid file-id)
@ -678,7 +682,7 @@
(binding [*system* system]
(-> (db/get conn :file {:id file-id})
(update :features db/decode-pgarray #{})
(process-file))))))
(process-file :validate? validate?))))))
(finally
(let [elapsed (tpoint)
@ -701,9 +705,8 @@
(assoc :elapsed/total-by-file total)
(assoc :processed/files completed)))))))))))
(defn migrate-team!
[system team-id]
[system team-id & {:keys [validate?]}]
(let [tpoint (dt/tpoint)
team-id (if (string? team-id)
(parse-uuid team-id)
@ -737,7 +740,7 @@
rows (->> (db/exec! conn [sql team-id])
(map :id))]
(run! (partial migrate-file! system) rows)
(run! #(migrate-file! system % :validate? validate?) rows)
(some-> *stats* (swap! assoc :current/files (count rows)))
(let [features (-> features

20
backend/src/app/srepl/components_v2.clj
View file

@ -110,13 +110,14 @@
(l/dbg :hint "migrate:end" :elapsed elapsed))))))
(defn migrate-files!
[{:keys [::db/pool] :as system} & {:keys [chunk-size max-jobs max-items start-at preset rollback skip-on-error]
[{:keys [::db/pool] :as system} & {:keys [chunk-size max-jobs max-items start-at preset rollback skip-on-error validate]
:or {chunk-size 10
skip-on-error true
max-jobs 10
max-items Long/MAX_VALUE
preset :shutdown-on-failure
rollback true}}]
rollback true
validate false}}]
(letfn [(get-chunk [cursor]
(let [sql (str/concat
"SELECT id, created_at FROM file "
@ -151,7 +152,7 @@
(ps/acquire! feat/*semaphore*)
(px/submit! scope (fn []
(-> (assoc system ::db/rollback rollback)
(feat/migrate-file! file-id)))))
(feat/migrate-file! file-id :validate? validate)))))
(get-candidates))
(p/await! scope))
@ -171,8 +172,8 @@
(defn migrate-team!
[{:keys [::db/pool] :as system} team-id
& {:keys [rollback skip-on-error]
:or {rollback true skip-on-error true}}]
& {:keys [rollback skip-on-error validate]
:or {rollback true skip-on-error true validate false}}]
(l/dbg :hint "migrate:start")
(let [total (get-total-files pool :team-id team-id)
@ -185,7 +186,7 @@
(binding [feat/*stats* stats
feat/*skip-on-error* skip-on-error]
(-> (assoc system ::db/rollback rollback)
(feat/migrate-team! team-id))
(feat/migrate-team! team-id :validate? validate))
(print-stats!
(-> (deref feat/*stats*)
@ -203,13 +204,14 @@
(defn migrate-teams!
[{:keys [::db/pool] :as system}
& {:keys [chunk-size max-jobs max-items start-at rollback preset skip-on-error max-time]
& {:keys [chunk-size max-jobs max-items start-at rollback preset skip-on-error max-time validate]
:or {chunk-size 10000
rollback true
skip-on-error true
preset :shutdown-on-failure
max-jobs Integer/MAX_VALUE
max-items Long/MAX_VALUE}}]
max-items Long/MAX_VALUE
validate false}}]
(letfn [(get-chunk [cursor]
(let [sql (str/concat
@ -232,7 +234,7 @@
(migrate-team [team-id]
(try
(-> (assoc system ::db/rollback rollback)
(feat/migrate-team! team-id))
(feat/migrate-team! team-id :validate? validate))
(catch Throwable cause
(l/err :hint "unexpected error on processing team" :team-id (dm/str team-id) :cause cause))))