Penpot/penpot
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-03-11 23:31:21 -05:00
Code Issues Activity

📎 Update migration scripts

Browse source
This commit is contained in:
Andrey Antukh 2024-04-07 14:06:42 +02:00
parent da5f452db5
commit e01f8d6fdf
2 changed files with 200 additions and 121 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

111
backend/src/app/features/components_v2.clj
View file

@ -16,6 +16,7 @@
[app.common.files.migrations :as fmg] [app.common.files.migrations :as fmg]
[app.common.files.shapes-helpers :as cfsh] [app.common.files.shapes-helpers :as cfsh]
[app.common.files.validate :as cfv] [app.common.files.validate :as cfv]
[app.common.fressian :as fres]
[app.common.geom.matrix :as gmt] [app.common.geom.matrix :as gmt]
[app.common.geom.point :as gpt] [app.common.geom.point :as gpt]
[app.common.geom.rect :as grc] [app.common.geom.rect :as grc]
@ -48,18 +49,18 @@
[app.rpc.commands.files-snapshot :as fsnap] [app.rpc.commands.files-snapshot :as fsnap]
[app.rpc.commands.media :as cmd.media] [app.rpc.commands.media :as cmd.media]
[app.storage :as sto] [app.storage :as sto]
[app.storage.impl :as impl]
[app.storage.tmp :as tmp] [app.storage.tmp :as tmp]
[app.svgo :as svgo] [app.svgo :as svgo]
[app.util.blob :as blob] [app.util.blob :as blob]
[app.util.cache :as cache]
[app.util.events :as events] [app.util.events :as events]
[app.util.pointer-map :as pmap] [app.util.pointer-map :as pmap]
[app.util.time :as dt] [app.util.time :as dt]
[buddy.core.codecs :as bc] [buddy.core.codecs :as bc]
[clojure.set :refer [rename-keys]] [clojure.set :refer [rename-keys]]
[cuerdas.core :as str] [cuerdas.core :as str]
[datoteka.fs :as fs]
[datoteka.io :as io] [datoteka.io :as io]
[promesa.exec :as px]
[promesa.util :as pu])) [promesa.util :as pu]))
(def ^:dynamic *stats* (def ^:dynamic *stats*
@ -68,7 +69,7 @@
(def ^:dynamic *cache* (def ^:dynamic *cache*
"A dynamic var for setting up a cache instance." "A dynamic var for setting up a cache instance."
nil) false)
(def ^:dynamic *skip-on-graphic-error* (def ^:dynamic *skip-on-graphic-error*
"A dynamic var for setting up the default error behavior for graphics processing." "A dynamic var for setting up the default error behavior for graphics processing."
@ -100,6 +101,8 @@
(some? data) (some? data)
(assoc :data (blob/decode data)))) (assoc :data (blob/decode data))))
(set! *warn-on-reflection* true)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; FILE PREPARATION BEFORE MIGRATION ;; FILE PREPARATION BEFORE MIGRATION
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@ -1296,7 +1299,7 @@
(try (try
(let [item (if (str/starts-with? href "data:") (let [item (if (str/starts-with? href "data:")
(let [[mtype data] (parse-datauri href) (let [[mtype data] (parse-datauri href)
size (alength data) size (alength ^bytes data)
path (tmp/tempfile :prefix "penpot.media.download.") path (tmp/tempfile :prefix "penpot.media.download.")
written (io/write-to-file! data path :size size)] written (io/write-to-file! data path :size size)]
@ -1365,27 +1368,49 @@
{::sql/columns [:media-id]})] {::sql/columns [:media-id]})]
(:media-id fmobject))) (:media-id fmobject)))
(defn- get-sobject-content (defn get-sobject-content
[id] [id]
(let [storage (::sto/storage *system*) (let [storage (::sto/storage *system*)
sobject (sto/get-object storage id)] sobject (sto/get-object storage id)]
(when-not sobject
(throw (RuntimeException. "sobject is nil")))
(when (> (:size sobject) 1135899)
(throw (RuntimeException. "svg too big")))
(with-open [stream (sto/get-object-data storage sobject)] (with-open [stream (sto/get-object-data storage sobject)]
(slurp stream)))) (slurp stream))))
(defn get-optimized-svg
[sid]
(let [svg-text (get-sobject-content sid)
svg-text (svgo/optimize *system* svg-text)]
(csvg/parse svg-text)))
(def base-path "/data/cache")
(defn get-sobject-cache-path
[sid]
(let [path (impl/id->path sid)]
(fs/join base-path path)))
(defn get-cached-svg
[sid]
(let [path (get-sobject-cache-path sid)]
(if (fs/exists? path)
(with-open [^java.lang.AutoCloseable stream (io/input-stream path)]
(let [reader (fres/reader stream)]
(fres/read! reader)))
(get-optimized-svg sid))))
(defn- create-shapes-for-svg (defn- create-shapes-for-svg
[{:keys [id] :as mobj} file-id objects frame-id position] [{:keys [id] :as mobj} file-id objects frame-id position]
(let [get-svg (fn [sid] (let [sid (resolve-sobject-id id)
(let [svg-text (get-sobject-content sid) svg-data (if *cache*
svg-text (svgo/optimize *system* svg-text)] (get-cached-svg sid)
(-> (csvg/parse svg-text) (get-optimized-svg sid))
(assoc :name (:name mobj))))) svg-data (collect-and-persist-images svg-data file-id id)
svg-data (assoc svg-data :name (:name mobj))]
sid (resolve-sobject-id id)
svg-data (if (cache/cache? *cache*)
(cache/get *cache* sid (px/wrap-bindings get-svg))
(get-svg sid))
svg-data (collect-and-persist-images svg-data file-id id)]
(sbuilder/create-svg-shapes svg-data position objects frame-id frame-id #{} false))) (sbuilder/create-svg-shapes svg-data position objects frame-id frame-id #{} false)))
@ -1714,7 +1739,7 @@
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn migrate-file! (defn migrate-file!
[system file-id & {:keys [validate? skip-on-graphic-error? label]}] [system file-id & {:keys [validate? skip-on-graphic-error? label rown]}]
(let [tpoint (dt/tpoint) (let [tpoint (dt/tpoint)
err (volatile! false)] err (volatile! false)]
@ -1754,24 +1779,14 @@
components (get @*file-stats* :processed-components 0) components (get @*file-stats* :processed-components 0)
graphics (get @*file-stats* :processed-graphics 0)] graphics (get @*file-stats* :processed-graphics 0)]
(if (cache/cache? *cache*) (l/dbg :hint "migrate:file:end"
(let [cache-stats (cache/stats *cache*)] :file-id (str file-id)
(l/dbg :hint "migrate:file:end" :graphics graphics
:file-id (str file-id) :components components
:graphics graphics :validate validate?
:components components :rown rown
:validate validate? :error @err
:crt (mth/to-fixed (:hit-rate cache-stats) 2) :elapsed (dt/format-duration elapsed))
:crq (str (:req-count cache-stats))
:error @err
:elapsed (dt/format-duration elapsed)))
(l/dbg :hint "migrate:file:end"
:file-id (str file-id)
:graphics graphics
:components components
:validate validate?
:error @err
:elapsed (dt/format-duration elapsed)))
(some-> *stats* (swap! update :processed-files (fnil inc 0))) (some-> *stats* (swap! update :processed-files (fnil inc 0)))
(some-> *team-stats* (swap! update :processed-files (fnil inc 0))))))))) (some-> *team-stats* (swap! update :processed-files (fnil inc 0)))))))))
@ -1833,21 +1848,9 @@
(when-not @err (when-not @err
(some-> *stats* (swap! update :processed-teams (fnil inc 0)))) (some-> *stats* (swap! update :processed-teams (fnil inc 0))))
(if (cache/cache? *cache*) (l/dbg :hint "migrate:team:end"
(let [cache-stats (cache/stats *cache*)] :team-id (dm/str team-id)
(l/dbg :hint "migrate:team:end" :files files
:team-id (dm/str team-id) :components components
:files files :graphics graphics
:components components :elapsed (dt/format-duration elapsed))))))))
:graphics graphics
:crt (mth/to-fixed (:hit-rate cache-stats) 2)
:crq (str (:req-count cache-stats))
:error @err
:elapsed (dt/format-duration elapsed)))
(l/dbg :hint "migrate:team:end"
:team-id (dm/str team-id)
:files files
:components components
:graphics graphics
:elapsed (dt/format-duration elapsed)))))))))

210
backend/src/app/srepl/components_v2.clj
View file

@ -7,18 +7,19 @@
(ns app.srepl.components-v2 (ns app.srepl.components-v2
(:require (:require
[app.common.data :as d] [app.common.data :as d]
[app.common.fressian :as fres]
[app.common.logging :as l] [app.common.logging :as l]
[app.common.uuid :as uuid]
[app.db :as db] [app.db :as db]
[app.features.components-v2 :as feat] [app.features.components-v2 :as feat]
[app.main :as main] [app.main :as main]
[app.srepl.helpers :as h] [app.srepl.helpers :as h]
[app.svgo :as svgo] [app.svgo :as svgo]
[app.util.cache :as cache]
[app.util.events :as events] [app.util.events :as events]
[app.util.time :as dt] [app.util.time :as dt]
[app.worker :as-alias wrk] [app.worker :as-alias wrk]
[cuerdas.core :as str] [cuerdas.core :as str]
[datoteka.fs :as fs]
[datoteka.io :as io]
[promesa.exec :as px] [promesa.exec :as px]
[promesa.exec.semaphore :as ps] [promesa.exec.semaphore :as ps]
[promesa.util :as pu])) [promesa.util :as pu]))
@ -112,14 +113,14 @@
(def ^:private sql:get-files-by-created-at (def ^:private sql:get-files-by-created-at
"SELECT id, features, "SELECT id, features,
row_number() OVER (ORDER BY created_at) AS rown row_number() OVER (ORDER BY created_at DESC) AS rown
FROM file FROM file
WHERE deleted_at IS NULL WHERE deleted_at IS NULL
ORDER BY created_at DESC") ORDER BY created_at DESC")
(def ^:private sql:get-files-by-modified-at (def ^:private sql:get-files-by-modified-at
"SELECT id, features "SELECT id, features
row_number() OVER (ORDER BY modified_at) AS rown row_number() OVER (ORDER BY modified_at DESC) AS rown
FROM file FROM file
WHERE deleted_at IS NULL WHERE deleted_at IS NULL
ORDER BY modified_at DESC") ORDER BY modified_at DESC")
@ -210,11 +211,7 @@
skip-on-graphic-error? true}}] skip-on-graphic-error? true}}]
(l/dbg :hint "migrate:start" :rollback rollback?) (l/dbg :hint "migrate:start" :rollback rollback?)
(let [tpoint (dt/tpoint) (let [tpoint (dt/tpoint)
file-id (h/parse-uuid file-id) file-id (h/parse-uuid file-id)]
cache (if (int? cache)
(cache/create :executor (::wrk/executor main/system)
:max-items cache)
nil)]
(binding [feat/*stats* (atom {}) (binding [feat/*stats* (atom {})
feat/*cache* cache] feat/*cache* cache]
@ -245,12 +242,7 @@
(let [team-id (h/parse-uuid team-id) (let [team-id (h/parse-uuid team-id)
stats (atom {}) stats (atom {})
tpoint (dt/tpoint) tpoint (dt/tpoint)]
cache (if (int? cache)
(cache/create :executor (::wrk/executor main/system)
:max-items cache)
nil)]
(add-watch stats :progress-report (report-progress-files tpoint)) (add-watch stats :progress-report (report-progress-files tpoint))
@ -313,35 +305,30 @@
sjobs (ps/create :permits max-jobs) sjobs (ps/create :permits max-jobs)
sprocs (ps/create :permits max-procs) sprocs (ps/create :permits max-procs)
cache (if (int? cache)
(cache/create :executor (::wrk/executor main/system)
:max-items cache)
nil)
migrate-team migrate-team
(fn [team-id] (fn [team-id]
(let [tpoint (dt/tpoint)] (try
(try (db/tx-run! (assoc main/system ::db/rollback rollback?)
(db/tx-run! (assoc main/system ::db/rollback rollback?) (fn [system]
(fn [system] (db/exec-one! system ["SET LOCAL idle_in_transaction_session_timeout = 0"])
(db/exec-one! system ["SET LOCAL idle_in_transaction_session_timeout = 0"]) (feat/migrate-team! system team-id
(feat/migrate-team! system team-id :label label
:label label :validate? validate?
:validate? validate? :skip-on-graphic-error? skip-on-graphic-error?)))
:skip-on-graphic-error? skip-on-graphic-error?)))
(catch Throwable cause (catch Throwable cause
(l/wrn :hint "unexpected error on processing team (skiping)" (l/wrn :hint "unexpected error on processing team (skiping)"
:team-id (str team-id)) :team-id (str team-id))
(events/tap :error (events/tap :error
(ex-info "unexpected error on processing team (skiping)" (ex-info "unexpected error on processing team (skiping)"
{:team-id team-id} {:team-id team-id}
cause)) cause))
(swap! stats update :errors (fnil inc 0))) (swap! stats update :errors (fnil inc 0)))
(finally (finally
(ps/release! sjobs))))) (ps/release! sjobs))))
process-team process-team
(fn [team-id] (fn [team-id]
@ -439,50 +426,45 @@
sjobs (ps/create :permits max-jobs) sjobs (ps/create :permits max-jobs)
sprocs (ps/create :permits max-procs) sprocs (ps/create :permits max-procs)
cache (if (int? cache)
(cache/create :executor (::wrk/executor main/system)
:max-items cache)
nil)
migrate-file migrate-file
(fn [file-id] (fn [file-id rown]
(let [tpoint (dt/tpoint)] (try
(try (db/tx-run! (assoc main/system ::db/rollback rollback?)
(db/tx-run! (assoc main/system ::db/rollback rollback?) (fn [system]
(fn [system] (db/exec-one! system ["SET LOCAL idle_in_transaction_session_timeout = 0"])
(db/exec-one! system ["SET LOCAL idle_in_transaction_session_timeout = 0"]) (feat/migrate-file! system file-id
(feat/migrate-file! system file-id :rown rown
:label label :label label
:validate? validate? :validate? validate?
:skip-on-graphic-error? skip-on-graphic-error?))) :skip-on-graphic-error? skip-on-graphic-error?)))
(catch Throwable cause (catch Throwable cause
(l/wrn :hint "unexpected error on processing file (skiping)" (l/wrn :hint "unexpected error on processing file (skiping)"
:file-id (str file-id)) :file-id (str file-id))
(events/tap :error (events/tap :error
(ex-info "unexpected error on processing file (skiping)" (ex-info "unexpected error on processing file (skiping)"
{:file-id file-id} {:file-id file-id}
cause)) cause))
(swap! stats update :errors (fnil inc 0))) (swap! stats update :errors (fnil inc 0)))
(finally (finally
(ps/release! sjobs))))) (ps/release! sjobs))))
process-file process-file
(fn [file-id] (fn [{:keys [id rown]}]
(ps/acquire! sjobs) (ps/acquire! sjobs)
(let [ts (tpoint)] (let [ts (tpoint)]
(if (and mtime (neg? (compare mtime ts))) (if (and mtime (neg? (compare mtime ts)))
(do (do
(l/inf :hint "max time constraint reached" (l/inf :hint "max time constraint reached"
:file-id (str file-id) :file-id (str id)
:elapsed (dt/format-duration ts)) :elapsed (dt/format-duration ts))
(ps/release! sjobs) (ps/release! sjobs)
(reduced nil)) (reduced nil))
(px/run! executor (partial migrate-file file-id)))))] (px/run! executor (partial migrate-file id rown)))))]
(l/dbg :hint "migrate:start" (l/dbg :hint "migrate:start"
:label label :label label
@ -507,7 +489,6 @@
(if (int? partitions) (if (int? partitions)
(= current-partition (inc (mod rown partitions))) (= current-partition (inc (mod rown partitions)))
true))) true)))
(map :id)
(take max-items))) (take max-items)))
;; Close and await tasks ;; Close and await tasks
@ -526,6 +507,101 @@
:rollback rollback? :rollback rollback?
:elapsed elapsed))))))) :elapsed elapsed)))))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; CACHE POPULATE
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(def sql:sobjects-for-cache
"SELECT id,
row_number() OVER (ORDER BY created_at) AS index
FROM storage_object
WHERE (metadata->>'~:bucket' = 'file-media-object' OR
metadata->>'~:bucket' IS NULL)
AND metadata->>'~:content-type' = 'image/svg+xml'
AND deleted_at IS NULL
AND size < 1135899
ORDER BY created_at ASC")
(defn populate-cache!
"A REPL helper for migrate all files.
This function starts multiple concurrent file migration processes
until thw maximum number of jobs is reached which by default has the
value of `1`. This is controled with the `:max-jobs` option.
If you want to run this on multiple machines you will need to specify
the total number of partitions and the current partition.
In order to get the report table populated, you will need to provide
a correct `:label`. That label is also used for persist a file
snaphot before continue with the migration."
[& {:keys [max-jobs] :or {max-jobs 1}}]
(let [tpoint (dt/tpoint)
factory (px/thread-factory :virtual false :prefix "penpot/cache/")
executor (px/cached-executor :factory factory)
sjobs (ps/create :permits max-jobs)
retrieve-sobject
(fn [id index]
(let [path (feat/get-sobject-cache-path id)
parent (fs/parent path)]
(try
(when-not (fs/exists? parent)
(fs/create-dir parent))
(if (fs/exists? path)
(l/inf :hint "create cache entry" :status "exists" :index index :id (str id) :path (str path))
(let [svg-data (feat/get-optimized-svg id)]
(with-open [^java.lang.AutoCloseable stream (io/output-stream path)]
(let [writer (fres/writer stream)]
(fres/write! writer svg-data)))
(l/inf :hint "create cache entry" :status "created"
:index index
:id (str id)
:path (str path))))
(catch Throwable cause
(l/wrn :hint "create cache entry"
:status "error"
:index index
:id (str id)
:path (str path)
:cause cause))
(finally
(ps/release! sjobs)))))
process-sobject
(fn [{:keys [id index]}]
(ps/acquire! sjobs)
(px/run! executor (partial retrieve-sobject id index)))]
(l/dbg :hint "migrate:start"
:max-jobs max-jobs)
(try
(binding [feat/*system* main/system]
(run! process-sobject
(db/exec! main/system [sql:sobjects-for-cache]))
;; Close and await tasks
(pu/close! executor))
{:elapsed (dt/format-duration (tpoint))}
(catch Throwable cause
(l/dbg :hint "populate:error" :cause cause))
(finally
(let [elapsed (dt/format-duration (tpoint))]
(l/dbg :hint "populate:end"
:elapsed elapsed))))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; FILE PROCESS HELPERS ;; FILE PROCESS HELPERS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;