mirror of
https://github.com/penpot/penpot.git
synced 2025-02-08 16:18:11 -05:00
✨ Simplify SVGO module API
This commit is contained in:
Andrey Antukh
parent
4fc391763e
commit
944d167bbb
6 changed files with 205 additions and 237 deletions
|
|
@ -209,7 +209,6 @@
|
|||
(s/def ::telemetry-uri ::us/string)
|
||||
(s/def ::telemetry-with-taiga ::us/boolean)
|
||||
(s/def ::tenant ::us/string)
|
||||
(s/def ::svgo-max-procs ::us/integer)
|
||||
|
||||
(s/def ::config
|
||||
(s/keys :opt-un [::secret-key
|
||||
|
|
@ -329,9 +328,7 @@
|
|||
::telemetry-uri
|
||||
::telemetry-referer
|
||||
::telemetry-with-taiga
|
||||
::tenant
|
||||
|
||||
::svgo-max-procs]))
|
||||
::tenant]))
|
||||
|
||||
(def default-flags
|
||||
[:enable-backend-api-doc
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@
|
|||
[app.common.types.shape-tree :as ctst]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.db :as db]
|
||||
[app.db.sql :as sql]
|
||||
[app.features.fdata :as fdata]
|
||||
[app.http.sse :as sse]
|
||||
[app.media :as media]
|
||||
|
|
@ -41,6 +42,7 @@
|
|||
[app.storage.tmp :as tmp]
|
||||
[app.svgo :as svgo]
|
||||
[app.util.blob :as blob]
|
||||
[app.util.cache :as cache]
|
||||
[app.util.pointer-map :as pmap]
|
||||
[app.util.time :as dt]
|
||||
[buddy.core.codecs :as bc]
|
||||
|
|
@ -52,20 +54,19 @@
|
|||
"A dynamic var for setting up state for collect stats globally."
|
||||
nil)
|
||||
|
||||
(def ^:dynamic *skip-on-error*
|
||||
"A dynamic var for setting up the default error behavior."
|
||||
true)
|
||||
(def ^:dynamic *cache*
|
||||
"A dynamic var for setting up a cache instance."
|
||||
nil)
|
||||
|
||||
(def ^:dynamic *skip-on-graphic-error*
|
||||
"A dynamic var for setting up the default error behavior for graphics processing."
|
||||
nil)
|
||||
|
||||
(def ^:dynamic ^:private *system*
|
||||
"An internal var for making the current `system` available to all
|
||||
internal functions without the need to explicitly pass it top down."
|
||||
nil)
|
||||
|
||||
(def ^:dynamic ^:private *max-procs*
|
||||
"A dynamic variable that can optionally indicates the maxumum number
|
||||
of concurrent graphics migration processes."
|
||||
nil)
|
||||
|
||||
(def ^:dynamic ^:private *file-stats*
|
||||
"An internal dynamic var for collect stats by file."
|
||||
nil)
|
||||
|
|
@ -576,37 +577,30 @@
|
|||
(defn- collect-and-persist-images
|
||||
[svg-data file-id]
|
||||
(letfn [(process-image [{:keys [href] :as item}]
|
||||
(try
|
||||
(let [item (if (str/starts-with? href "data:")
|
||||
(let [[mtype data] (parse-datauri href)
|
||||
size (alength data)
|
||||
path (tmp/tempfile :prefix "penpot.media.download.")
|
||||
written (io/write-to-file! data path :size size)]
|
||||
(let [item (if (str/starts-with? href "data:")
|
||||
(let [[mtype data] (parse-datauri href)
|
||||
size (alength data)
|
||||
path (tmp/tempfile :prefix "penpot.media.download.")
|
||||
written (io/write-to-file! data path :size size)]
|
||||
|
||||
(when (not= written size)
|
||||
(ex/raise :type :internal
|
||||
:code :mismatch-write-size
|
||||
:hint "unexpected state: unable to write to file"))
|
||||
(when (not= written size)
|
||||
(ex/raise :type :internal
|
||||
:code :mismatch-write-size
|
||||
:hint "unexpected state: unable to write to file"))
|
||||
|
||||
(-> item
|
||||
(assoc :size size)
|
||||
(assoc :path path)
|
||||
(assoc :filename "tempfile")
|
||||
(assoc :mtype mtype)))
|
||||
(-> item
|
||||
(assoc :size size)
|
||||
(assoc :path path)
|
||||
(assoc :filename "tempfile")
|
||||
(assoc :mtype mtype)))
|
||||
|
||||
(let [result (cmd.media/download-image *system* href)]
|
||||
(-> (merge item result)
|
||||
(assoc :name (extract-name href)))))]
|
||||
(let [result (cmd.media/download-image *system* href)]
|
||||
(-> (merge item result)
|
||||
(assoc :name (extract-name href)))))]
|
||||
|
||||
;; The media processing adds the data to the
|
||||
;; input map and returns it.
|
||||
(media/run {:cmd :info :input item}))
|
||||
|
||||
(catch Throwable cause
|
||||
(l/warn :hint "unexpected exception on processing internal image shape (skiping)"
|
||||
:cause cause)
|
||||
(when-not *skip-on-error*
|
||||
(throw cause)))))
|
||||
;; The media processing adds the data to the
|
||||
;; input map and returns it.
|
||||
(media/run {:cmd :info :input item})))
|
||||
|
||||
(persist-image [acc {:keys [path size width height mtype href] :as item}]
|
||||
(let [storage (::sto/storage *system*)
|
||||
|
|
@ -642,23 +636,36 @@
|
|||
(completing persist-image) {}))]
|
||||
(assoc svg-data :image-data images))))
|
||||
|
||||
(defn- get-svg-content
|
||||
(defn- resolve-sobject-id
|
||||
[id]
|
||||
(let [fmobject (db/get *system* :file-media-object {:id id}
|
||||
{::db/check-deleted false
|
||||
::db/remove-deleted false
|
||||
::sql/columns [:media-id]})]
|
||||
(:media-id fmobject)))
|
||||
|
||||
(defn- get-sobject-content
|
||||
[id]
|
||||
(let [storage (::sto/storage *system*)
|
||||
conn (::db/conn *system*)
|
||||
fmobject (db/get conn :file-media-object {:id id})
|
||||
sobject (sto/get-object storage (:media-id fmobject))]
|
||||
|
||||
sobject (sto/get-object storage id)]
|
||||
(with-open [stream (sto/get-object-data storage sobject)]
|
||||
(slurp stream))))
|
||||
|
||||
(defn- create-shapes-for-svg
|
||||
[{:keys [id] :as mobj} file-id objects frame-id position]
|
||||
(let [svg-text (get-svg-content id)
|
||||
svg-text (svgo/optimize *system* svg-text)
|
||||
svg-data (-> (csvg/parse svg-text)
|
||||
(assoc :name (:name mobj))
|
||||
(collect-and-persist-images file-id))]
|
||||
(let [get-svg (fn [sid]
|
||||
(let [svg-text (get-sobject-content sid)
|
||||
svg-text (svgo/optimize *system* svg-text)]
|
||||
(-> (csvg/parse svg-text)
|
||||
(assoc :name (:name mobj)))))
|
||||
|
||||
sid (resolve-sobject-id id)
|
||||
|
||||
svg-data (if (cache/cache? *cache*)
|
||||
(cache/get *cache* sid get-svg)
|
||||
(get-svg sid))
|
||||
|
||||
svg-data (collect-and-persist-images file-id)]
|
||||
|
||||
(sbuilder/create-svg-shapes svg-data position objects frame-id frame-id #{} false)))
|
||||
|
||||
|
|
@ -717,42 +724,34 @@
|
|||
|
||||
(defn- create-media-grid
|
||||
[fdata page-id frame-id grid media-group]
|
||||
(let [process (fn [mobj position]
|
||||
(let [position (gpt/add position (gpt/point grid-gap grid-gap))
|
||||
tp1 (dt/tpoint)]
|
||||
(try
|
||||
(process-media-object fdata page-id frame-id mobj position)
|
||||
(catch Throwable cause
|
||||
(l/wrn :hint "unable to process file media object (skiping)"
|
||||
:file-id (str (:id fdata))
|
||||
:id (str (:id mobj))
|
||||
:cause cause)
|
||||
(if-not *skip-on-error*
|
||||
(throw cause)
|
||||
nil))
|
||||
(finally
|
||||
(l/trc :hint "graphic processed"
|
||||
:file-id (str (:id fdata))
|
||||
:media-id (str (:id mobj))
|
||||
:elapsed (dt/format-duration (tp1)))))))]
|
||||
(letfn [(process [fdata mobj position]
|
||||
(let [position (gpt/add position (gpt/point grid-gap grid-gap))
|
||||
tp (dt/tpoint)]
|
||||
(try
|
||||
(let [changes (process-media-object fdata page-id frame-id mobj position)]
|
||||
(cp/process-changes fdata changes false))
|
||||
(catch Throwable cause
|
||||
(if *skip-on-graphic-error*
|
||||
(l/wrn :hint "unable to process file media object (skiping)"
|
||||
:file-id (str (:id fdata))
|
||||
:id (str (:id mobj))
|
||||
:cause cause)
|
||||
(throw cause))
|
||||
nil)
|
||||
(finally
|
||||
(let [elapsed (tp)]
|
||||
(l/trc :hint "graphic processed"
|
||||
:file-id (str (:id fdata))
|
||||
:media-id (str (:id mobj))
|
||||
:elapsed (dt/format-duration elapsed)))))))]
|
||||
|
||||
(->> (d/zip media-group grid)
|
||||
(partition-all (or *max-procs* 1))
|
||||
(mapcat (fn [partition]
|
||||
(->> partition
|
||||
(map (fn [[mobj position]]
|
||||
(sse/tap {:type :migration-progress
|
||||
:section :graphics
|
||||
:name (:name mobj)})
|
||||
(p/vthread (process mobj position))))
|
||||
(doall)
|
||||
(map deref)
|
||||
(doall))))
|
||||
(filter some?)
|
||||
(reduce (fn [fdata changes]
|
||||
(-> (assoc-in fdata [:options :components-v2] true)
|
||||
(cp/process-changes changes false)))
|
||||
fdata))))
|
||||
(reduce (fn [fdata [mobj position]]
|
||||
(sse/tap {:type :migration-progress
|
||||
:section :graphics
|
||||
:name (:name mobj)})
|
||||
(or (process fdata mobj position) fdata))
|
||||
(assoc-in fdata [:options :components-v2] true)))))
|
||||
|
||||
(defn- migrate-graphics
|
||||
[fdata]
|
||||
|
|
@ -832,17 +831,12 @@
|
|||
(decode-row)))
|
||||
|
||||
(defn- validate-file!
|
||||
[file libs throw-on-validate?]
|
||||
(try
|
||||
(cfv/validate-file! file libs)
|
||||
(cfv/validate-file-schema! file)
|
||||
(catch Throwable cause
|
||||
(if throw-on-validate?
|
||||
(throw cause)
|
||||
(l/wrn :hint "migrate:file:validation-error" :cause cause)))))
|
||||
[file libs]
|
||||
(cfv/validate-file! file libs)
|
||||
(cfv/validate-file-schema! file))
|
||||
|
||||
(defn- process-file
|
||||
[{:keys [::db/conn] :as system} id & {:keys [validate? throw-on-validate?]}]
|
||||
[{:keys [::db/conn] :as system} id & {:keys [validate?]}]
|
||||
(let [file (get-file system id)
|
||||
|
||||
libs (->> (files/get-file-libraries conn id)
|
||||
|
|
@ -855,7 +849,7 @@
|
|||
(update :features conj "components/v2"))
|
||||
|
||||
_ (when validate?
|
||||
(validate-file! file libs throw-on-validate?))
|
||||
(validate-file! file libs))
|
||||
|
||||
file (if (contains? (:features file) "fdata/objects-map")
|
||||
(fdata/enable-objects-map file)
|
||||
|
|
@ -901,10 +895,10 @@
|
|||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defn migrate-file!
|
||||
[system file-id & {:keys [validate? throw-on-validate? max-procs]}]
|
||||
(let [tpoint (dt/tpoint)]
|
||||
[system file-id & {:keys [validate? skip-on-graphic-error?]}]
|
||||
(let [tpoint (dt/tpoint)]
|
||||
(binding [*file-stats* (atom {})
|
||||
*max-procs* max-procs]
|
||||
*skip-on-graphic-error* skip-on-graphic-error?]
|
||||
(try
|
||||
(l/dbg :hint "migrate:file:start" :file-id (str file-id))
|
||||
|
||||
|
|
@ -913,9 +907,7 @@
|
|||
(fn [system]
|
||||
(binding [*system* system]
|
||||
(fsnap/take-file-snapshot! system {:file-id file-id :label "migration/components-v2"})
|
||||
(process-file system file-id
|
||||
:validate? validate?
|
||||
:throw-on-validate? throw-on-validate?)))))
|
||||
(process-file system file-id :validate? validate?)))))
|
||||
(finally
|
||||
(let [elapsed (tpoint)
|
||||
components (get @*file-stats* :processed/components 0)
|
||||
|
|
@ -931,7 +923,7 @@
|
|||
(some-> *team-stats* (swap! update :processed/files (fnil inc 0)))))))))
|
||||
|
||||
(defn migrate-team!
|
||||
[system team-id & {:keys [validate? throw-on-validate? max-procs]}]
|
||||
[system team-id & {:keys [validate? skip-on-graphic-error?]}]
|
||||
|
||||
(l/dbg :hint "migrate:team:start"
|
||||
:team-id (dm/str team-id))
|
||||
|
|
@ -941,9 +933,8 @@
|
|||
migrate-file
|
||||
(fn [system file-id]
|
||||
(migrate-file! system file-id
|
||||
:max-procs max-procs
|
||||
:validate? validate?
|
||||
:throw-on-validate? throw-on-validate?))
|
||||
:skip-on-graphics-error? skip-on-graphic-error?))
|
||||
migrate-team
|
||||
(fn [{:keys [::db/conn] :as system} {:keys [id features] :as team}]
|
||||
(let [features (-> features
|
||||
|
|
|
|||
|
|
@ -411,8 +411,7 @@
|
|||
::migrations (ig/ref :app.migrations/migrations)}
|
||||
|
||||
::svgo/optimizer
|
||||
{::wrk/executor (ig/ref ::wrk/executor)
|
||||
::svgo/max-procs (cf/get :svgo-max-procs)}
|
||||
{}
|
||||
|
||||
::audit.tasks/archive
|
||||
{::props (ig/ref ::setup/props)
|
||||
|
|
|
|||
|
|
@ -664,9 +664,7 @@
|
|||
(case feature
|
||||
"components/v2"
|
||||
(feat.compv2/migrate-file! options file-id
|
||||
:max-procs 2
|
||||
:validate? validate?
|
||||
:throw-on-validate? true)
|
||||
:validate? validate?)
|
||||
|
||||
"fdata/shape-data-type"
|
||||
nil
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
[app.common.pprint :as pp]
|
||||
[app.db :as db]
|
||||
[app.features.components-v2 :as feat]
|
||||
[app.svgo :as svgo]
|
||||
[app.util.time :as dt]
|
||||
[cuerdas.core :as str]
|
||||
[promesa.core :as p]
|
||||
|
|
@ -35,14 +36,10 @@
|
|||
(fn [_ _ oldv newv]
|
||||
(when (not= (:processed/files oldv)
|
||||
(:processed/files newv))
|
||||
(let [total (:total/files newv)
|
||||
completed (:processed/files newv)
|
||||
progress (/ (* completed 100.0) total)
|
||||
(let [completed (:processed/files newv)
|
||||
elapsed (tpoint)]
|
||||
(l/dbg :hint "progress"
|
||||
:completed (:processed/files newv)
|
||||
:total (:total/files newv)
|
||||
:progress (str (int progress) "%")
|
||||
:elapsed (dt/format-duration elapsed))))))
|
||||
|
||||
(defn- report-progress-teams
|
||||
|
|
@ -50,21 +47,13 @@
|
|||
(fn [_ _ oldv newv]
|
||||
(when (not= (:processed/teams oldv)
|
||||
(:processed/teams newv))
|
||||
(let [total (:total/teams newv)
|
||||
completed (:processed/teams newv)
|
||||
progress (/ (* completed 100.0) total)
|
||||
progress (str (int progress) "%")
|
||||
(let [completed (:processed/teams newv)
|
||||
elapsed (dt/format-duration (tpoint))]
|
||||
|
||||
(when (fn? on-progress)
|
||||
(on-progress {:total total
|
||||
:elapsed elapsed
|
||||
:completed completed
|
||||
:progress progress}))
|
||||
|
||||
(on-progress {:elapsed elapsed
|
||||
:completed completed}))
|
||||
(l/dbg :hint "progress"
|
||||
:completed completed
|
||||
:progress progress
|
||||
:elapsed elapsed)))))
|
||||
|
||||
(defn- get-total-files
|
||||
|
|
@ -92,7 +81,6 @@
|
|||
res (db/exec-one! pool [sql])]
|
||||
(:count res)))
|
||||
|
||||
|
||||
(defn- mark-team-migration!
|
||||
[{:keys [::db/pool]} team-id]
|
||||
;; We execute this out of transaction because we want this
|
||||
|
|
@ -113,24 +101,68 @@
|
|||
" WHERE id = ?")]
|
||||
(db/exec-one! pool [sql team-id])))
|
||||
|
||||
;; (def ^:private sql:get-teams
|
||||
;; "SELECT id, features
|
||||
;; FROM team
|
||||
;; WHERE deleted_at IS NULL
|
||||
;; ORDER BY created_at DESC")
|
||||
|
||||
;; (def ^:private sql:get-teams
|
||||
;; "SELECT t.id, t.features,
|
||||
;; (SELECT count(*)
|
||||
;; FROM file_media_object AS fmo
|
||||
;; JOIN file AS f ON (f.id = fmo.file_id)
|
||||
;; JOIN project AS p ON (p.id = f.project_id)
|
||||
;; WHERE p.team_id = t.id
|
||||
;; AND fmo.mtype = 'image/svg+xml'
|
||||
;; AND fmo.is_local = false) AS graphics
|
||||
;; FROM team AS t
|
||||
;; ORDER BY t.created_at DESC")
|
||||
|
||||
|
||||
(def ^:private sql:get-teams
|
||||
"SELECT id, features
|
||||
FROM team
|
||||
WHERE deleted_at IS NULL
|
||||
ORDER BY created_at ASC")
|
||||
"WITH teams AS (
|
||||
SELECT t.id, t.features,
|
||||
(SELECT count(*)
|
||||
FROM file_media_object AS fmo
|
||||
JOIN file AS f ON (f.id = fmo.file_id)
|
||||
JOIN project AS p ON (p.id = f.project_id)
|
||||
WHERE p.team_id = t.id
|
||||
AND fmo.mtype = 'image/svg+xml'
|
||||
AND fmo.is_local = false) AS graphics
|
||||
FROM team AS t
|
||||
ORDER BY 3 ASC
|
||||
)
|
||||
SELECT * FROM teams ")
|
||||
|
||||
(defn- read-pred
|
||||
[[op val field]]
|
||||
(let [field (name field)]
|
||||
(case op
|
||||
:lt [(str/ffmt "WHERE % < ?" field) val]
|
||||
:lte [(str/ffmt "WHERE % <= ?" field) val]
|
||||
:gt [(str/ffmt "WHERE % > ?" field) val]
|
||||
:gte [(str/ffmt "WHERE % >= ?" field) val]
|
||||
:eq [(str/ffmt "WHERE % = ?" field) val]
|
||||
[""])))
|
||||
|
||||
(defn- get-teams
|
||||
[conn]
|
||||
(->> (db/cursor conn sql:get-teams)
|
||||
(map feat/decode-row)))
|
||||
[conn pred]
|
||||
(let [[sql & params] (read-pred pred)]
|
||||
(->> (db/cursor conn (apply vector (str sql:get-teams sql) params))
|
||||
(map feat/decode-row)
|
||||
(remove (fn [{:keys [features]}]
|
||||
(or (contains? features "ephimeral/v2-migration")
|
||||
(contains? features "components/v2"))))
|
||||
(map :id))))
|
||||
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; PUBLIC API
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defn migrate-file!
|
||||
[system file-id & {:keys [rollback? max-procs]
|
||||
:or {rollback? true}}]
|
||||
[system file-id & {:keys [rollback?] :or {rollback? true}}]
|
||||
|
||||
(l/dbg :hint "migrate:start" :rollback rollback?)
|
||||
(let [tpoint (dt/tpoint)
|
||||
|
|
@ -140,7 +172,7 @@
|
|||
(binding [feat/*stats* (atom {})]
|
||||
(try
|
||||
(-> (assoc system ::db/rollback rollback?)
|
||||
(feat/migrate-file! file-id :max-procs max-procs))
|
||||
(feat/migrate-file! file-id))
|
||||
|
||||
(-> (deref feat/*stats*)
|
||||
(assoc :elapsed (dt/format-duration (tpoint))))
|
||||
|
|
@ -153,11 +185,11 @@
|
|||
(l/dbg :hint "migrate:end" :rollback rollback? :elapsed elapsed)))))))
|
||||
|
||||
(defn migrate-team!
|
||||
[{:keys [::db/pool] :as system} team-id & {:keys [rollback? skip-on-error validate? max-procs]
|
||||
[{:keys [::db/pool] :as system} team-id & {:keys [rollback? skip-on-graphic-error? validate? skip-mark?]
|
||||
:or {rollback? true
|
||||
skip-on-error true
|
||||
validate? false
|
||||
max-procs 1}
|
||||
validate? true
|
||||
skip-on-graphic-error? false
|
||||
skip-mark? false}
|
||||
:as opts}]
|
||||
|
||||
(l/dbg :hint "migrate:start" :rollback rollback?)
|
||||
|
|
@ -165,34 +197,30 @@
|
|||
(let [team-id (if (string? team-id)
|
||||
(parse-uuid team-id)
|
||||
team-id)
|
||||
total (get-total-files pool :team-id team-id)
|
||||
stats (atom {:total/files total})
|
||||
stats (atom {})
|
||||
tpoint (dt/tpoint)]
|
||||
|
||||
(add-watch stats :progress-report (report-progress-files tpoint))
|
||||
|
||||
(binding [feat/*stats* stats
|
||||
feat/*skip-on-error* skip-on-error]
|
||||
|
||||
(binding [feat/*stats* stats]
|
||||
(try
|
||||
(mark-team-migration! system team-id)
|
||||
(when-not skip-mark?
|
||||
(mark-team-migration! system team-id))
|
||||
|
||||
(-> (assoc system ::db/rollback rollback?)
|
||||
(feat/migrate-team! team-id
|
||||
:max-procs max-procs
|
||||
:validate? validate?
|
||||
:throw-on-validate? (not skip-on-error)))
|
||||
|
||||
:skip-on-graphics-error? skip-on-graphic-error?))
|
||||
(print-stats!
|
||||
(-> (deref feat/*stats*)
|
||||
(dissoc :total/files)
|
||||
(assoc :elapsed (dt/format-duration (tpoint)))))
|
||||
|
||||
(catch Throwable cause
|
||||
(l/dbg :hint "migrate:error" :cause cause))
|
||||
|
||||
(finally
|
||||
(unmark-team-migration! system team-id)
|
||||
(when-not skip-mark?
|
||||
(unmark-team-migration! system team-id))
|
||||
|
||||
(let [elapsed (dt/format-duration (tpoint))]
|
||||
(l/dbg :hint "migrate:end" :rollback rollback? :elapsed elapsed)))))))
|
||||
|
|
@ -202,100 +230,78 @@
|
|||
|
||||
This function starts multiple concurrent team migration processes
|
||||
until thw maximum number of jobs is reached which by default has the
|
||||
value of `1`. This is controled with the `:max-jobs` option.
|
||||
value of `1`. This is controled with the `:max-jobs` option."
|
||||
|
||||
Each tram migration process also can start multiple procs for
|
||||
graphics migration, the total of that procs is controled with the
|
||||
`:max-procs` option.
|
||||
|
||||
Internally, the graphics migration process uses SVGO module which by
|
||||
default has a limited number of maximum concurent
|
||||
operations (globally), ensure setting up correct number with
|
||||
PENPOT_SVGO_MAX_PROCS environment variable."
|
||||
|
||||
[{:keys [::db/pool] :as system} & {:keys [max-jobs max-procs max-items
|
||||
[{:keys [::db/pool] :as system} & {:keys [max-jobs max-items max-time
|
||||
rollback? validate? preset
|
||||
skip-on-error max-time
|
||||
pred max-procs skip-mark?
|
||||
on-start on-progress on-error on-end]
|
||||
:or {validate? false
|
||||
:or {validate? true
|
||||
rollback? true
|
||||
skip-on-error true
|
||||
preset :shutdown-on-failure
|
||||
skip-mark? true
|
||||
max-jobs 1
|
||||
max-procs 10
|
||||
max-items Long/MAX_VALUE}
|
||||
:as opts}]
|
||||
|
||||
(let [total (get-total-teams pool)
|
||||
stats (atom {:total/teams (min total max-items)})
|
||||
(let [stats (atom {})
|
||||
tpoint (dt/tpoint)
|
||||
mtime (some-> max-time dt/duration)
|
||||
|
||||
tpoint (dt/tpoint)
|
||||
mtime (some-> max-time dt/duration)
|
||||
|
||||
scope (px/structured-task-scope :preset preset :factory :virtual)
|
||||
sjobs (ps/create :permits max-jobs)
|
||||
factory (px/thread-factory :virtual false :prefix "penpot/migration/compv2/")
|
||||
executor (px/cached-executor :factory factory)
|
||||
max-procs (or max-procs max-jobs)
|
||||
sjobs (ps/create :permits max-jobs)
|
||||
sprocs (ps/create :permits max-procs)
|
||||
|
||||
migrate-team
|
||||
(fn [{:keys [id features] :as team}]
|
||||
(fn [team-id]
|
||||
(ps/acquire! sjobs)
|
||||
(let [ts (tpoint)]
|
||||
(cond
|
||||
(and mtime (neg? (compare mtime ts)))
|
||||
(if (and mtime (neg? (compare mtime ts)))
|
||||
(do
|
||||
(l/inf :hint "max time constraint reached"
|
||||
:team-id (str id)
|
||||
:team-id (str team-id)
|
||||
:elapsed (dt/format-duration ts))
|
||||
(ps/release! sjobs)
|
||||
(reduced nil))
|
||||
|
||||
(or (contains? features "ephimeral/v2-migration")
|
||||
(contains? features "components/v2"))
|
||||
(do
|
||||
(l/dbg :hint "skip team" :team-id (str id))
|
||||
(ps/release! sjobs))
|
||||
|
||||
:else
|
||||
(px/submit! scope (fn []
|
||||
(px/run! executor (fn []
|
||||
(try
|
||||
(mark-team-migration! system id)
|
||||
(when-not skip-mark?
|
||||
(mark-team-migration! system team-id))
|
||||
(-> (assoc system ::db/rollback rollback?)
|
||||
(feat/migrate-team! id
|
||||
:max-procs max-procs
|
||||
:validate? validate?
|
||||
:throw-on-validate? (not skip-on-error)))
|
||||
(feat/migrate-team! team-id :validate? validate?))
|
||||
(catch Throwable cause
|
||||
(l/err :hint "unexpected error on processing team"
|
||||
:team-id (str id)
|
||||
(l/err :hint "unexpected error on processing team (skiping)"
|
||||
:team-id (str team-id)
|
||||
:cause cause))
|
||||
(finally
|
||||
(ps/release! sjobs)
|
||||
(unmark-team-migration! system id))))))))]
|
||||
(when-not skip-mark?
|
||||
(unmark-team-migration! system team-id)))))))))]
|
||||
|
||||
(l/dbg :hint "migrate:start"
|
||||
:rollback rollback?
|
||||
:total total
|
||||
:max-jobs max-jobs
|
||||
:max-procs max-procs
|
||||
:max-items max-items)
|
||||
|
||||
(add-watch stats :progress-report (report-progress-teams tpoint on-progress))
|
||||
|
||||
(binding [feat/*stats* stats
|
||||
feat/*skip-on-error* skip-on-error]
|
||||
svgo/*semaphore* sprocs]
|
||||
(try
|
||||
(when (fn? on-start)
|
||||
(on-start {:total total :rollback rollback?}))
|
||||
(on-start {:rollback rollback?}))
|
||||
|
||||
(db/tx-run! system
|
||||
(fn [{:keys [::db/conn]}]
|
||||
(run! (partial migrate-team)
|
||||
(->> (get-teams conn)
|
||||
(->> (get-teams conn pred)
|
||||
(take max-items)))))
|
||||
(try
|
||||
(p/await! scope)
|
||||
(finally
|
||||
(pu/close! scope)))
|
||||
|
||||
;; Close and await tasks
|
||||
(pu/close! executor)
|
||||
|
||||
(if (fn? on-end)
|
||||
(-> (deref stats)
|
||||
|
|
|
|||
|
|
@ -7,16 +7,10 @@
|
|||
(ns app.svgo
|
||||
"A SVG Optimizer service"
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.data.macros :as dm]
|
||||
[app.common.jsrt :as jsrt]
|
||||
[app.common.logging :as l]
|
||||
[app.common.spec :as us]
|
||||
[app.worker :as-alias wrk]
|
||||
[clojure.spec.alpha :as s]
|
||||
[integrant.core :as ig]
|
||||
[promesa.exec :as px]
|
||||
[promesa.exec.bulkhead :as bh]
|
||||
[promesa.exec.semaphore :as ps]
|
||||
[promesa.util :as pu]))
|
||||
|
||||
|
|
@ -26,40 +20,23 @@
|
|||
nil)
|
||||
|
||||
(defn optimize
|
||||
[system data]
|
||||
(dm/assert! "expect data to be a string" (string? data))
|
||||
|
||||
(letfn [(optimize-fn [pool]
|
||||
(jsrt/run! pool
|
||||
(fn [context]
|
||||
(jsrt/set! context "svgData" data)
|
||||
(jsrt/eval! context "penpotSvgo.optimize(svgData, {plugins: ['safeAndFastPreset']})"))))]
|
||||
(try
|
||||
(some-> *semaphore* ps/acquire!)
|
||||
(let [{:keys [::jsrt/pool ::wrk/executor]} (::optimizer system)]
|
||||
(dm/assert! "expect optimizer instance" (jsrt/pool? pool))
|
||||
(px/invoke! executor (partial optimize-fn pool)))
|
||||
(finally
|
||||
(some-> *semaphore* ps/release!)))))
|
||||
|
||||
(s/def ::max-procs (s/nilable ::us/integer))
|
||||
|
||||
(defmethod ig/pre-init-spec ::optimizer [_]
|
||||
(s/keys :req [::wrk/executor ::max-procs]))
|
||||
|
||||
(defmethod ig/prep-key ::optimizer
|
||||
[_ cfg]
|
||||
(merge {::max-procs 20} (d/without-nils cfg)))
|
||||
[{pool ::optimizer} data]
|
||||
(try
|
||||
(some-> *semaphore* ps/acquire!)
|
||||
(jsrt/run! pool
|
||||
(fn [context]
|
||||
(jsrt/set! context "svgData" data)
|
||||
(jsrt/eval! context "penpotSvgo.optimize(svgData, {plugins: ['safeAndFastPreset']})")))
|
||||
(finally
|
||||
(some-> *semaphore* ps/release!))))
|
||||
|
||||
(defmethod ig/init-key ::optimizer
|
||||
[_ {:keys [::wrk/executor ::max-procs]}]
|
||||
(l/inf :hint "initializing svg optimizer pool" :max-procs max-procs)
|
||||
(let [init (jsrt/resource->source "app/common/svg/optimizer.js")
|
||||
executor (bh/create :type :executor :executor executor :permits max-procs)]
|
||||
{::jsrt/pool (jsrt/pool :init init)
|
||||
::wrk/executor executor}))
|
||||
[_ _]
|
||||
(l/inf :hint "initializing svg optimizer pool")
|
||||
(let [init (jsrt/resource->source "app/common/svg/optimizer.js")]
|
||||
(jsrt/pool :init init)))
|
||||
|
||||
(defmethod ig/halt-key! ::optimizer
|
||||
[_ {:keys [::jsrt/pool]}]
|
||||
[_ pool]
|
||||
(l/info :hint "stopping svg optimizer pool")
|
||||
(pu/close! pool))
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue