From 41287d8fc579dc0aab0f95f5694ff39116c92bd0 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 3 Jan 2024 15:16:14 +0100 Subject: [PATCH] :zap: Improve migration script performance and api usability --- backend/dev/user.clj | 10 + backend/src/app/config.clj | 5 +- backend/src/app/features/components_v2.clj | 291 +++++++++------- backend/src/app/main.clj | 9 +- backend/src/app/rpc/commands/binfile.clj | 1 + backend/src/app/srepl/components_v2.clj | 369 +++++++++++---------- backend/src/app/svgo.clj | 65 ++++ backend/src/app/worker.clj | 4 +- common/src/app/common/svg.cljc | 36 +- common/src/app/common/svg/optimizer.js | 148 ++++++--- 10 files changed, 559 insertions(+), 379 deletions(-) create mode 100644 backend/src/app/svgo.clj diff --git a/backend/dev/user.clj b/backend/dev/user.clj index 902ba6212..5d0cb7e37 100644 --- a/backend/dev/user.clj +++ b/backend/dev/user.clj @@ -8,6 +8,7 @@ (:require [app.common.data :as d] [app.common.exceptions :as ex] + [app.common.files.helpers :as cfh] [app.common.fressian :as fres] [app.common.geom.matrix :as gmt] [app.common.logging :as l] @@ -136,3 +137,12 @@ (add-tap #(locking debug-tap (prn "tap debug:" %))) 1)) + + +(defn calculate-frames + [{:keys [data]}] + (->> (vals (:pages-index data)) + (mapcat (comp vals :objects)) + (filter cfh/is-direct-child-of-root?) + (filter cfh/frame-shape?) + (count))) diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index f5a1602af..ef021d7c1 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -207,6 +207,7 @@ (s/def ::telemetry-uri ::us/string) (s/def ::telemetry-with-taiga ::us/boolean) (s/def ::tenant ::us/string) +(s/def ::svgo-max-procs ::us/integer) (s/def ::config (s/keys :opt-un [::secret-key @@ -326,7 +327,9 @@ ::telemetry-uri ::telemetry-referer ::telemetry-with-taiga - ::tenant])) + ::tenant + + ::svgo-max-procs])) (def default-flags [:enable-backend-api-doc diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index 1dc9f8325..3c5d3e5b0 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -39,27 +39,54 @@ [app.rpc.commands.media :as cmd.media] [app.storage :as sto] [app.storage.tmp :as tmp] + [app.svgo :as svgo] [app.util.blob :as blob] [app.util.pointer-map :as pmap] [app.util.time :as dt] [buddy.core.codecs :as bc] [cuerdas.core :as str] [datoteka.io :as io] - [promesa.exec :as px] - [promesa.exec.semaphore :as ps] - [promesa.util :as pu])) + [promesa.core :as p])) -(def ^:dynamic *system* nil) -(def ^:dynamic *stats* nil) -(def ^:dynamic *file-stats* nil) -(def ^:dynamic *team-stats* nil) -(def ^:dynamic *semaphore* nil) -(def ^:dynamic *skip-on-error* true) +(def ^:dynamic *stats* + "A dynamic var for setting up state for collect stats globally." + nil) + +(def ^:dynamic *skip-on-error* + "A dynamic var for setting up the default error behavior." + true) + +(def ^:dynamic ^:private *system* + "An internal var for making the current `system` available to all + internal functions without the need to explicitly pass it top down." + nil) + +(def ^:dynamic ^:private *max-procs* + "A dynamic variable that can optionally indicates the maxumum number + of concurrent graphics migration processes." + nil) + +(def ^:dynamic ^:private *file-stats* + "An internal dynamic var for collect stats by file." + nil) + +(def ^:dynamic ^:private *team-stats* + "An internal dynamic var for collect stats by team." + nil) (def grid-gap 50) (def frame-gap 200) (def max-group-size 50) +(defn decode-row + [{:keys [features data] :as row}] + (cond-> row + (some? features) + (assoc :features (db/decode-pgarray features #{})) + + (some? data) + (assoc :data (blob/decode data)))) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; FILE PREPARATION BEFORE MIGRATION ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -220,19 +247,17 @@ (fn [file-data] ;; Transform component and copy heads to frames, and set the ;; frame-id of its childrens - (letfn [(fix-container - [container] + (letfn [(fix-container [container] (update container :objects update-vals fix-shape)) - (fix-shape - [shape] + (fix-shape [shape] (if (or (nil? (:parent-id shape)) (ctk/instance-head? shape)) (assoc shape - :type :frame ; Old groups must be converted - :fills (or (:fills shape) []) ; to frames and conform to spec - :hide-in-viewer (or (:hide-in-viewer shape) true) - :rx (or (:rx shape) 0) - :ry (or (:ry shape) 0)) + :type :frame ; Old groups must be converted + :fills (or (:fills shape) []) ; to frames and conform to spec + :hide-in-viewer (or (:hide-in-viewer shape) true) + :rx (or (:rx shape) 0) + :ry (or (:ry shape) 0)) shape))] (-> file-data (update :pages-index update-vals fix-container) @@ -310,10 +335,10 @@ (defn- get-asset-groups [assets generic-name] - (let [; Group by first element of the path. + (let [;; Group by first element of the path. groups (d/group-by #(first (cfh/split-path (:path %))) assets) - ; Split large groups in chunks of max-group-size elements + ;; Split large groups in chunks of max-group-size elements groups (loop [groups (seq groups) result {}] (if (empty? groups) @@ -334,15 +359,14 @@ result splits))))))) - ; Sort assets in each group by path + ;; Sort assets in each group by path groups (update-vals groups (fn [assets] (sort-by (fn [{:keys [path name]}] (str/lower (cfh/merge-path-item path name))) - assets))) + assets)))] - ; Sort groups by name - groups (into (sorted-map) groups)] - groups)) + ;; Sort groups by name + (into (sorted-map) groups))) (defn- create-frame [name position width height] @@ -612,14 +636,11 @@ (defn- create-shapes-for-svg [{:keys [id] :as mobj} file-id objects frame-id position] - (let [svg-text (get-svg-content id) - - optimizer (::csvg/optimizer *system*) - svg-text (csvg/optimize optimizer svg-text) - - svg-data (-> (csvg/parse svg-text) - (assoc :name (:name mobj)) - (collect-and-persist-images file-id))] + (let [svg-text (get-svg-content id) + svg-text (svgo/optimize *system* svg-text) + svg-data (-> (csvg/parse svg-text) + (assoc :name (:name mobj)) + (collect-and-persist-images file-id))] (sbuilder/create-svg-shapes svg-data position objects frame-id frame-id #{} false))) @@ -678,9 +699,7 @@ (defn- create-media-grid [fdata page-id frame-id grid media-group] - (let [factory (px/thread-factory :virtual true) - executor (px/fixed-executor :parallelism 10 :factory factory) - process (fn [mobj position] + (let [process (fn [mobj position] (let [position (gpt/add position (gpt/point grid-gap grid-gap)) tp1 (dt/tpoint)] (try @@ -690,7 +709,6 @@ :file-id (str (:id fdata)) :id (str (:id mobj)) :cause cause) - (if-not *skip-on-error* (throw cause) nil)) @@ -699,21 +717,24 @@ :file-id (str (:id fdata)) :media-id (str (:id mobj)) :elapsed (dt/format-duration (tp1)))))))] - (try - (->> (d/zip media-group grid) - (map (fn [[mobj position]] - (sse/tap {:type :migration-progress - :section :graphics - :name (:name mobj)}) - (px/submit! executor (partial process mobj position)))) - (reduce (fn [fdata promise] - (if-let [changes (deref promise)] - (-> (assoc-in fdata [:options :components-v2] true) - (cp/process-changes changes false)) - fdata)) - fdata)) - (finally - (pu/close! executor))))) + + (->> (d/zip media-group grid) + (partition-all (or *max-procs* 1)) + (mapcat (fn [partition] + (->> partition + (map (fn [[mobj position]] + (sse/tap {:type :migration-progress + :section :graphics + :name (:name mobj)}) + (p/vthread (process mobj position)))) + (doall) + (map deref) + (doall)))) + (filter some?) + (reduce (fn [fdata changes] + (-> (assoc-in fdata [:options :components-v2] true) + (cp/process-changes changes false))) + fdata)))) (defn- migrate-graphics [fdata] @@ -759,6 +780,11 @@ (create-media-grid fdata page-id (:id frame) grid assets) (gpt/add position (gpt/point 0 (+ height (* 2 grid-gap) frame-gap)))))))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; PRIVATE HELPERS +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + (defn- migrate-fdata [fdata libs] (let [migrated? (dm/get-in fdata [:options :components-v2])] @@ -771,11 +797,22 @@ (defn- get-file [system id] (binding [pmap/*load-fn* (partial fdata/load-pointer system id)] - (-> (files/get-file system id :migrate? false) + (-> (db/get system :file {:id id} + {::db/remove-deleted false + ::db/check-deleted false}) + (decode-row) (update :data assoc :id id) (update :data fdata/process-pointers deref) (fmg/migrate-file)))) + +(defn- get-team + [system team-id] + (-> (db/get system :team {:id team-id} + {::db/remove-deleted false + ::db/check-deleted false}) + (decode-row))) + (defn- validate-file! [file libs throw-on-validate?] (try @@ -791,7 +828,8 @@ (let [file (get-file system id) libs (->> (files/get-file-libraries conn id) - (into [file] (comp (map :id) (map (partial get-file system)))) + (into [file] (comp (map :id) + (map (partial get-file system)))) (d/index-by :id)) file (-> file @@ -820,13 +858,35 @@ (dissoc file :data))) + +(def ^:private sql:get-and-lock-team-files + "SELECT f.id + FROM file AS f + JOIN project AS p ON (p.id = f.project_id) + WHERE p.team_id = ? + FOR UPDATE") + +(defn- get-and-lock-files + [conn team-id] + (->> (db/cursor conn [sql:get-and-lock-team-files team-id]) + (map :id))) + +(defn- update-team-features! + [conn team-id features] + (let [features (db/create-array conn "text" features)] + (db/update! conn :team + {:features features} + {:id team-id}))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; PUBLIC API +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + (defn migrate-file! - [system file-id & {:keys [validate? throw-on-validate?]}] - (let [tpoint (dt/tpoint) - file-id (if (string? file-id) - (parse-uuid file-id) - file-id)] - (binding [*file-stats* (atom {})] + [system file-id & {:keys [validate? throw-on-validate? max-procs]}] + (let [tpoint (dt/tpoint)] + (binding [*file-stats* (atom {}) + *max-procs* max-procs] (try (l/dbg :hint "migrate:file:start" :file-id (str file-id)) @@ -838,7 +898,6 @@ (process-file system file-id :validate? validate? :throw-on-validate? throw-on-validate?))))) - (finally (let [elapsed (tpoint) components (get @*file-stats* :processed/components 0) @@ -854,75 +913,51 @@ (some-> *team-stats* (swap! update :processed/files (fnil inc 0))))))))) (defn migrate-team! - [system team-id & {:keys [validate? throw-on-validate?]}] - (let [tpoint (dt/tpoint) - team-id (if (string? team-id) - (parse-uuid team-id) - team-id)] - (l/dbg :hint "migrate:team:start" :team-id (dm/str team-id)) + [system team-id & {:keys [validate? throw-on-validate? max-procs]}] + + (l/dbg :hint "migrate:team:start" + :team-id (dm/str team-id)) + + (let [tpoint (dt/tpoint) + + migrate-file + (fn [system file-id] + (migrate-file! system file-id + :max-procs max-procs + :validate? validate? + :throw-on-validate? throw-on-validate?)) + migrate-team + (fn [{:keys [::db/conn] :as system} {:keys [id features] :as team}] + (let [features (-> features + (disj "ephimeral/v2-migration") + (conj "components/v2") + (conj "layout/grid") + (conj "styles/v2"))] + + (run! (partial migrate-file system) + (get-and-lock-files conn id)) + + (update-team-features! conn id features)))] + (binding [*team-stats* (atom {})] (try - ;; We execute this out of transaction because we want this - ;; change to be visible to all other sessions before starting - ;; the migration - (let [sql (str "UPDATE team SET features = " - " array_append(features, 'ephimeral/v2-migration') " - " WHERE id = ?")] - (db/exec-one! system [sql team-id])) - - (db/tx-run! system - (fn [{:keys [::db/conn] :as system}] - ;; Lock the team - (db/exec-one! conn ["SET idle_in_transaction_session_timeout = 0"]) - - (let [{:keys [features] :as team} (-> (db/get conn :team {:id team-id}) - (update :features db/decode-pgarray #{}))] - - (if (contains? features "components/v2") - (l/dbg :hint "team already migrated") - (let [sql (str/concat - "SELECT f.id FROM file AS f " - " JOIN project AS p ON (p.id = f.project_id) " - "WHERE p.team_id = ? AND f.deleted_at IS NULL AND p.deleted_at IS NULL " - "FOR UPDATE")] - - (doseq [file-id (->> (db/exec! conn [sql team-id]) - (map :id))] - (migrate-file! system file-id - :validate? validate? - :throw-on-validate? throw-on-validate?)) - - (let [features (-> features - (disj "ephimeral/v2-migration") - (conj "components/v2") - (conj "layout/grid") - (conj "styles/v2"))] - (db/update! conn :team - {:features (db/create-array conn "text" features)} - {:id team-id}) - - nil)))))) + (db/tx-run! system (fn [system] + (db/exec-one! system ["SET idle_in_transaction_session_timeout = 0"]) + (let [team (get-team system team-id)] + (if (contains? (:features team) "components/v2") + (l/inf :hint "team already migrated") + (migrate-team system team))))) (finally - (some-> *semaphore* ps/release!) - (let [elapsed (tpoint)] + (let [elapsed (tpoint) + components (get @*team-stats* :processed/components 0) + graphics (get @*team-stats* :processed/graphics 0) + files (get @*team-stats* :processed/files 0)] + (some-> *stats* (swap! update :processed/teams (fnil inc 0))) - ;; We execute this out of transaction because we want this - ;; change to be visible to all other sessions before starting - ;; the migration - (let [sql (str "UPDATE team SET features = " - " array_remove(features, 'ephimeral/v2-migration') " - " WHERE id = ?")] - (db/exec-one! system [sql team-id])) - - (let [components (get @*team-stats* :processed/components 0) - graphics (get @*team-stats* :processed/graphics 0) - files (get @*team-stats* :processed/files 0)] - (l/dbg :hint "migrate:team:end" - :team-id (dm/str team-id) - :files files - :components components - :graphics graphics - :elapsed (dt/format-duration elapsed))))))))) - - + (l/dbg :hint "migrate:team:end" + :team-id (dm/str team-id) + :files files + :components components + :graphics graphics + :elapsed (dt/format-duration elapsed)))))))) diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index dde88473b..c80210a06 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -10,7 +10,6 @@ [app.auth.oidc :as-alias oidc] [app.auth.oidc.providers :as-alias oidc.providers] [app.common.logging :as l] - [app.common.svg :as csvg] [app.config :as cf] [app.db :as-alias db] [app.email :as-alias email] @@ -37,6 +36,7 @@ [app.storage.gc-deleted :as-alias sto.gc-deleted] [app.storage.gc-touched :as-alias sto.gc-touched] [app.storage.s3 :as-alias sto.s3] + [app.svgo :as-alias svgo] [app.util.time :as dt] [app.worker :as-alias wrk] [cider.nrepl :refer [cider-nrepl-handler]] @@ -316,7 +316,7 @@ ::mtx/metrics (ig/ref ::mtx/metrics) ::mbus/msgbus (ig/ref ::mbus/msgbus) ::rds/redis (ig/ref ::rds/redis) - ::csvg/optimizer (ig/ref ::csvg/optimizer) + ::svgo/optimizer (ig/ref ::svgo/optimizer) ::rpc/climit (ig/ref ::rpc/climit) ::rpc/rlimit (ig/ref ::rpc/rlimit) @@ -409,8 +409,9 @@ ;; module requires the migrations to run before initialize. ::migrations (ig/ref :app.migrations/migrations)} - ::csvg/optimizer - {} + ::svgo/optimizer + {::wrk/executor (ig/ref ::wrk/executor) + ::svgo/max-procs (cf/get :svgo-max-procs)} ::audit.tasks/archive {::props (ig/ref ::setup/props) diff --git a/backend/src/app/rpc/commands/binfile.clj b/backend/src/app/rpc/commands/binfile.clj index 3ccb22a0f..c173ec3bb 100644 --- a/backend/src/app/rpc/commands/binfile.clj +++ b/backend/src/app/rpc/commands/binfile.clj @@ -664,6 +664,7 @@ (case feature "components/v2" (feat.compv2/migrate-file! options file-id + :max-procs 2 :validate? validate? :throw-on-validate? true) diff --git a/backend/src/app/srepl/components_v2.clj b/backend/src/app/srepl/components_v2.clj index 598c13e6d..049fc3574 100644 --- a/backend/src/app/srepl/components_v2.clj +++ b/backend/src/app/srepl/components_v2.clj @@ -6,8 +6,6 @@ (ns app.srepl.components-v2 (:require - [app.common.data :as d] - [app.common.data.macros :as dm] [app.common.logging :as l] [app.common.pprint :as pp] [app.db :as db] @@ -19,6 +17,13 @@ [promesa.exec.semaphore :as ps] [promesa.util :as pu])) +(def ^:dynamic *scope* nil) +(def ^:dynamic *semaphore* nil) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; PRIVATE HELPERS +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + (defn- print-stats! [stats] (->> stats @@ -87,210 +92,228 @@ res (db/exec-one! pool [sql])] (:count res))) -(defn migrate-file! - [system file-id & {:keys [rollback?] :or {rollback? true}}] - (l/dbg :hint "migrate:start") - (let [tpoint (dt/tpoint)] - (try - (binding [feat/*stats* (atom {})] +(defn- mark-team-migration! + [{:keys [::db/pool]} team-id] + ;; We execute this out of transaction because we want this + ;; change to be visible to all other sessions before starting + ;; the migration + (let [sql (str "UPDATE team SET features = " + " array_append(features, 'ephimeral/v2-migration') " + " WHERE id = ?")] + (db/exec-one! pool [sql team-id]))) + +(defn- unmark-team-migration! + [{:keys [::db/pool]} team-id] + ;; We execute this out of transaction because we want this + ;; change to be visible to all other sessions before starting + ;; the migration + (let [sql (str "UPDATE team SET features = " + " array_remove(features, 'ephimeral/v2-migration') " + " WHERE id = ?")] + (db/exec-one! pool [sql team-id]))) + +(def ^:private sql:get-teams + "SELECT id, features + FROM team + WHERE deleted_at IS NULL + ORDER BY created_at ASC") + +(defn- get-teams + [conn] + (->> (db/cursor conn sql:get-teams) + (map feat/decode-row))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; PUBLIC API +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn migrate-file! + [system file-id & {:keys [rollback? max-procs] + :or {rollback? true}}] + + (l/dbg :hint "migrate:start" :rollback rollback?) + (let [tpoint (dt/tpoint) + file-id (if (string? file-id) + (parse-uuid file-id) + file-id)] + (binding [feat/*stats* (atom {})] + (try (-> (assoc system ::db/rollback rollback?) - (feat/migrate-file! file-id)) + (feat/migrate-file! file-id :max-procs max-procs)) (-> (deref feat/*stats*) - (assoc :elapsed (dt/format-duration (tpoint))))) + (assoc :elapsed (dt/format-duration (tpoint)))) - (catch Throwable cause - (l/wrn :hint "migrate:error" :cause cause)) + (catch Throwable cause + (l/wrn :hint "migrate:error" :cause cause)) - (finally - (let [elapsed (dt/format-duration (tpoint))] - (l/dbg :hint "migrate:end" :elapsed elapsed)))))) - -(defn migrate-files! - [{:keys [::db/pool] :as system} - & {:keys [chunk-size max-jobs max-items start-at preset rollback? skip-on-error validate?] - :or {chunk-size 10 - skip-on-error true - max-jobs 10 - max-items Long/MAX_VALUE - preset :shutdown-on-failure - rollback? true - validate? false}}] - (letfn [(get-chunk [cursor] - (let [sql (str/concat - "SELECT id, created_at FROM file " - " WHERE created_at < ? AND deleted_at IS NULL " - " ORDER BY created_at desc LIMIT ?") - rows (db/exec! pool [sql cursor chunk-size])] - [(some->> rows peek :created-at) (seq rows)])) - - (get-candidates [] - (->> (d/iteration get-chunk - :vf second - :kf first - :initk (or start-at (dt/now))) - (take max-items) - (map :id)))] - - (l/dbg :hint "migrate:start") - (let [fsem (ps/create :permits max-jobs) - total (get-total-files pool) - stats (atom {:files/total total}) - tpoint (dt/tpoint)] - - (add-watch stats :progress-report (report-progress-files tpoint)) - - (binding [feat/*stats* stats - feat/*semaphore* fsem - feat/*skip-on-error* skip-on-error] - (try - (pu/with-open [scope (px/structured-task-scope :preset preset :factory :virtual)] - - (run! (fn [file-id] - (ps/acquire! feat/*semaphore*) - (px/submit! scope (fn [] - (-> (assoc system ::db/rollback rollback?) - (feat/migrate-file! file-id - :validate? validate? - :throw-on-validate? (not skip-on-error)))))) - (get-candidates)) - - (p/await! scope)) - - (-> (deref feat/*stats*) - (assoc :elapsed (dt/format-duration (tpoint)))) - - (catch Throwable cause - (l/dbg :hint "migrate:error" :cause cause)) - - (finally - (let [elapsed (dt/format-duration (tpoint))] - (l/dbg :hint "migrate:end" :elapsed elapsed)))))))) + (finally + (let [elapsed (dt/format-duration (tpoint))] + (l/dbg :hint "migrate:end" :rollback rollback? :elapsed elapsed))))))) (defn migrate-team! - [{:keys [::db/pool] :as system} team-id - & {:keys [rollback? skip-on-error validate?] - :or {rollback? true skip-on-error true validate? false}}] - (l/dbg :hint "migrate:start") + [{:keys [::db/pool] :as system} team-id & {:keys [rollback? skip-on-error validate? max-procs] + :or {rollback? true + skip-on-error true + validate? false + max-procs 1 } + :as opts}] - (let [total (get-total-files pool :team-id team-id) - stats (atom {:total/files total}) - tpoint (dt/tpoint)] + (l/dbg :hint "migrate:start" :rollback rollback?) + + (let [team-id (if (string? team-id) + (parse-uuid team-id) + team-id) + total (get-total-files pool :team-id team-id) + stats (atom {:total/files total}) + tpoint (dt/tpoint)] (add-watch stats :progress-report (report-progress-files tpoint)) - (try - (binding [feat/*stats* stats - feat/*skip-on-error* skip-on-error] + (binding [feat/*stats* stats + feat/*skip-on-error* skip-on-error] + + (try + (mark-team-migration! system team-id) + (-> (assoc system ::db/rollback rollback?) (feat/migrate-team! team-id + :max-procs max-procs :validate? validate? :throw-on-validate? (not skip-on-error))) (print-stats! (-> (deref feat/*stats*) (dissoc :total/files) - (assoc :elapsed (dt/format-duration (tpoint)))))) + (assoc :elapsed (dt/format-duration (tpoint))))) - (catch Throwable cause - (l/dbg :hint "migrate:error" :cause cause)) + (catch Throwable cause + (l/dbg :hint "migrate:error" :cause cause)) - (finally - (let [elapsed (dt/format-duration (tpoint))] - (l/dbg :hint "migrate:end" :elapsed elapsed)))))) + (finally + (unmark-team-migration! system team-id) -(defn default-on-end - [stats] - (print-stats! - (-> stats - (update :elapsed/total dt/format-duration) - (dissoc :total/teams)))) + (let [elapsed (dt/format-duration (tpoint))] + (l/dbg :hint "migrate:end" :rollback rollback? :elapsed elapsed))))))) (defn migrate-teams! - [{:keys [::db/pool] :as system} - & {:keys [chunk-size max-jobs max-items start-at - rollback? validate? preset skip-on-error - max-time on-start on-progress on-error on-end] - :or {chunk-size 10000 - validate? false - rollback? true - skip-on-error true - on-end default-on-end - preset :shutdown-on-failure - max-jobs Integer/MAX_VALUE - max-items Long/MAX_VALUE}}] + "A REPL helper for migrate all teams. - (letfn [(get-chunk [cursor] - (let [sql (str/concat - "SELECT id, created_at, features FROM team " - " WHERE created_at < ? AND deleted_at IS NULL " - " ORDER BY created_at desc LIMIT ?") - rows (db/exec! pool [sql cursor chunk-size])] - [(some->> rows peek :created-at) (seq rows)])) + This function starts multiple concurrent team migration processes + until thw maximum number of jobs is reached which by default has the + value of `1`. This is controled with the `:max-jobs` option. - (get-candidates [] - (->> (d/iteration get-chunk - :vf second - :kf first - :initk (or start-at (dt/now))) - (map #(update % :features db/decode-pgarray #{})) - (remove #(contains? (:features %) "ephimeral/v2-migration")) - (take max-items) - (map :id))) + Each tram migration process also can start multiple procs for + graphics migration, the total of that procs is controled with the + `:max-procs` option. - (migrate-team [team-id] - (try - (-> (assoc system ::db/rollback rollback?) - (feat/migrate-team! team-id - :validate? validate? - :throw-on-validate? (not skip-on-error))) - (catch Throwable cause - (l/err :hint "unexpected error on processing team" :team-id (dm/str team-id) :cause cause)))) + Internally, the graphics migration process uses SVGO module which by + default has a limited number of maximum concurent + operations (globally), ensure setting up correct number with + PENPOT_SVGO_MAX_PROCS environment variable." - (process-team [scope tpoint mtime team-id] - (ps/acquire! feat/*semaphore*) - (let [ts (tpoint)] - (if (and mtime (neg? (compare mtime ts))) - (l/inf :hint "max time constraint reached" :elapsed (dt/format-duration ts)) - (px/submit! scope (partial migrate-team team-id)))))] + [{:keys [::db/pool] :as system} & {:keys [max-jobs max-procs max-items + rollback? validate? preset + skip-on-error max-time + on-start on-progress on-error on-end] + :or {validate? false + rollback? true + skip-on-error true + preset :shutdown-on-failure + max-jobs 1 + max-procs 10 + max-items Long/MAX_VALUE} + :as opts}] - (l/dbg :hint "migrate:start") + (let [total (get-total-teams pool) + stats (atom {:total/teams (min total max-items)}) - (let [sem (ps/create :permits max-jobs) - total (get-total-teams pool) - stats (atom {:total/teams (min total max-items)}) - tpoint (dt/tpoint) - mtime (some-> max-time dt/duration)] + tpoint (dt/tpoint) + mtime (some-> max-time dt/duration) - (when (fn? on-start) - (on-start {:total total :rollback rollback?})) + scope (px/structured-task-scope :preset preset :factory :virtual) + sjobs (ps/create :permits max-jobs) - (add-watch stats :progress-report (report-progress-teams tpoint on-progress)) + migrate-team + (fn [{:keys [id features] :as team}] + (ps/acquire! sjobs) + (let [ts (tpoint)] + (cond + (and mtime (neg? (compare mtime ts))) + (do + (l/inf :hint "max time constraint reached" + :team-id (str id) + :elapsed (dt/format-duration ts)) + (ps/release! sjobs) + (reduced nil)) - (binding [feat/*stats* stats - feat/*semaphore* sem - feat/*skip-on-error* skip-on-error] + (or (contains? features "ephimeral/v2-migration") + (contains? features "components/v2")) + (do + (l/dbg :hint "skip team" :team-id (str id)) + (ps/release! sjobs)) + + :else + (px/submit! scope (fn [] + (try + (mark-team-migration! system id) + (-> (assoc system ::db/rollback rollback?) + (feat/migrate-team! id + :max-procs max-procs + :validate? validate? + :throw-on-validate? (not skip-on-error))) + (catch Throwable cause + (l/err :hint "unexpected error on processing team" + :team-id (str id) + :cause cause)) + (finally + (ps/release! sjobs) + (unmark-team-migration! system id))))))))] + + (l/dbg :hint "migrate:start" + :rollback rollback? + :total total + :max-jobs max-jobs + :max-procs max-procs + :max-items max-items) + + (add-watch stats :progress-report (report-progress-teams tpoint on-progress)) + + (binding [feat/*stats* stats + feat/*skip-on-error* skip-on-error] + (try + (when (fn? on-start) + (on-start {:total total :rollback rollback?})) + + (db/tx-run! system + (fn [{:keys [::db/conn]}] + (run! (partial migrate-team) + (->> (get-teams conn) + (take max-items))))) (try - (pu/with-open [scope (px/structured-task-scope :preset preset - :factory :virtual)] - (loop [candidates (get-candidates)] - (when-let [team-id (first candidates)] - (when (process-team scope tpoint mtime team-id) - (recur (rest candidates))))) - - (p/await! scope)) - - (when (fn? on-end) - (-> (deref stats) - (assoc :elapsed/total (tpoint)) - (on-end))) - - (catch Throwable cause - (l/dbg :hint "migrate:error" :cause cause) - (when (fn? on-error) - (on-error cause))) - + (p/await! scope) (finally - (let [elapsed (dt/format-duration (tpoint))] - (l/dbg :hint "migrate:end" :elapsed elapsed)))))))) + (pu/close! scope))) + + + (if (fn? on-end) + (-> (deref stats) + (assoc :elapsed/total (tpoint)) + (on-end)) + (-> (deref stats) + (assoc :elapsed/total (tpoint)) + (update :elapsed/total dt/format-duration) + (dissoc :total/teams) + (print-stats!))) + + (catch Throwable cause + (l/dbg :hint "migrate:error" :cause cause) + (when (fn? on-error) + (on-error cause))) + + (finally + (let [elapsed (dt/format-duration (tpoint))] + (l/dbg :hint "migrate:end" + :rollback rollback? + :elapsed elapsed))))))) diff --git a/backend/src/app/svgo.clj b/backend/src/app/svgo.clj new file mode 100644 index 000000000..70d7c6b2b --- /dev/null +++ b/backend/src/app/svgo.clj @@ -0,0 +1,65 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns app.svgo + "A SVG Optimizer service" + (:require + [app.common.data :as d] + [app.common.data.macros :as dm] + [app.common.jsrt :as jsrt] + [app.common.logging :as l] + [app.common.spec :as us] + [app.worker :as-alias wrk] + [clojure.spec.alpha :as s] + [integrant.core :as ig] + [promesa.exec :as px] + [promesa.exec.bulkhead :as bh] + [promesa.exec.semaphore :as ps] + [promesa.util :as pu])) + +(def ^:dynamic *semaphore* + "A dynamic variable that can optionally contain a traffic light to + appropriately delimit the use of resources, managed externally." + nil) + +(defn optimize + [system data] + (dm/assert! "expect data to be a string" (string? data)) + + (letfn [(optimize-fn [pool] + (jsrt/run! pool + (fn [context] + (jsrt/set! context "svgData" data) + (jsrt/eval! context "penpotSvgo.optimize(svgData, {plugins: ['safeAndFastPreset']})"))))] + (try + (some-> *semaphore* ps/acquire!) + (let [{:keys [::jsrt/pool ::wrk/executor]} (::optimizer system)] + (dm/assert! "expect optimizer instance" (jsrt/pool? pool)) + (px/invoke! executor (partial optimize-fn pool))) + (finally + (some-> *semaphore* ps/release!))))) + +(s/def ::max-procs (s/nilable ::us/integer)) + +(defmethod ig/pre-init-spec ::optimizer [_] + (s/keys :req [::wrk/executor ::max-procs])) + +(defmethod ig/prep-key ::optimizer + [_ cfg] + (merge {::max-procs 20} (d/without-nils cfg))) + +(defmethod ig/init-key ::optimizer + [_ {:keys [::wrk/executor ::max-procs]}] + (l/inf :hint "initializing svg optimizer pool" :max-procs max-procs) + (let [init (jsrt/resource->source "app/common/svg/optimizer.js") + executor (bh/create :type :executor :executor executor :permits max-procs)] + {::jsrt/pool (jsrt/pool :init init) + ::wrk/executor executor})) + +(defmethod ig/halt-key! ::optimizer + [_ {:keys [::jsrt/pool]}] + (l/info :hint "stopping svg optimizer pool") + (pu/close! pool)) diff --git a/backend/src/app/worker.clj b/backend/src/app/worker.clj index 0ab867971..0e830ac9a 100644 --- a/backend/src/app/worker.clj +++ b/backend/src/app/worker.clj @@ -42,8 +42,8 @@ (defmethod ig/init-key ::executor [_ _] - (let [factory (px/thread-factory :prefix "penpot/default/") - executor (px/cached-executor :factory factory :keepalive 30000)] + (let [factory (px/thread-factory :prefix "penpot/default/") + executor (px/cached-executor :factory factory :keepalive 60000)] (l/inf :hint "starting executor") (reify java.lang.AutoCloseable diff --git a/common/src/app/common/svg.cljc b/common/src/app/common/svg.cljc index 39379028f..bc4f79f19 100644 --- a/common/src/app/common/svg.cljc +++ b/common/src/app/common/svg.cljc @@ -9,9 +9,6 @@ #?(:cljs ["./svg/optimizer.js" :as svgo]) #?(:clj [clojure.xml :as xml] :cljs [tubax.core :as tubax]) - #?(:clj [integrant.core :as ig]) - #?(:clj [app.common.jsrt :as jsrt]) - #?(:clj [app.common.logging :as l]) [app.common.data :as d] [app.common.data.macros :as dm] [app.common.geom.matrix :as gmt] @@ -1053,21 +1050,6 @@ :height (d/parse-integer (:height attrs) 0)})))] (reduce-nodes redfn [] svg-data ))) -#?(:cljs - (defn optimize - ([input] (optimize input nil)) - ([input options] - (svgo/optimize input (clj->js options)))) - :clj - (defn optimize - [pool data] - (dm/assert! "expected a valid pool" (jsrt/pool? pool)) - (dm/assert! "expect data to be a string" (string? data)) - (jsrt/run! pool - (fn [context] - (jsrt/set! context "svgData" data) - (jsrt/eval! context "penpotSvgo.optimize(svgData, {})"))))) - #?(:clj (defn- secure-parser-factory [^InputStream input ^XMLHandler handler] @@ -1091,15 +1073,9 @@ (dm/with-open [istream (IOUtils/toInputStream text "UTF-8")] (xml/parse istream secure-parser-factory))))) -#?(:clj - (defmethod ig/init-key ::optimizer - [_ _] - (l/info :hint "initializing svg optimizer pool") - (let [init (jsrt/resource->source "app/common/svg/optimizer.js")] - (jsrt/pool :init init)))) - -#?(:clj - (defmethod ig/halt-key! ::optimizer - [_ pool] - (l/info :hint "stopping svg optimizer pool") - (.close ^java.lang.AutoCloseable pool))) +;; FIXME pass correct plugin set +#?(:cljs + (defn optimize + ([input] (optimize input nil)) + ([input options] + (svgo/optimize input (clj->js options))))) diff --git a/common/src/app/common/svg/optimizer.js b/common/src/app/common/svg/optimizer.js index 70826b8bd..ccd19b697 100644 --- a/common/src/app/common/svg/optimizer.js +++ b/common/src/app/common/svg/optimizer.js @@ -29431,12 +29431,13 @@ const optimize = (input, config) => { exports.optimize = optimize; -},{"./svgo/parser.js":322,"./svgo/plugins.js":324,"./svgo/stringifier.js":381,"./svgo/tools.js":383,"lodash/isPlainObject":308}],320:[function(require,module,exports){ +},{"./svgo/parser.js":322,"./svgo/plugins.js":324,"./svgo/stringifier.js":382,"./svgo/tools.js":384,"lodash/isPlainObject":308}],320:[function(require,module,exports){ 'use strict'; exports.builtin = [ require('./plugins/default.js'), require('./plugins/safe.js'), + require('./plugins/safeAndFast.js'), require('./plugins/addAttributesToSVGElement.js'), require('./plugins/addClassesToSVGElement.js'), require('./plugins/cleanupAttrs.js'), @@ -29489,7 +29490,7 @@ exports.builtin = [ require('./plugins/sortDefsChildren.js'), ]; -},{"./plugins/addAttributesToSVGElement.js":328,"./plugins/addClassesToSVGElement.js":329,"./plugins/cleanupAttrs.js":331,"./plugins/cleanupEnableBackground.js":332,"./plugins/cleanupIds.js":333,"./plugins/cleanupListOfValues.js":334,"./plugins/cleanupNumericValues.js":335,"./plugins/collapseGroups.js":336,"./plugins/convertColors.js":337,"./plugins/convertEllipseToCircle.js":338,"./plugins/convertPathData.js":339,"./plugins/convertShapeToPath.js":340,"./plugins/convertStyleToAttrs.js":341,"./plugins/convertTransform.js":342,"./plugins/default.js":343,"./plugins/inlineStyles.js":344,"./plugins/mergePaths.js":345,"./plugins/mergeStyles.js":346,"./plugins/minifyStyles.js":347,"./plugins/moveElemsAttrsToGroup.js":348,"./plugins/moveGroupAttrsToElems.js":349,"./plugins/prefixIds.js":350,"./plugins/removeAttributesBySelector.js":351,"./plugins/removeAttrs.js":352,"./plugins/removeComments.js":353,"./plugins/removeDesc.js":354,"./plugins/removeDimensions.js":355,"./plugins/removeDoctype.js":356,"./plugins/removeEditorsNSData.js":357,"./plugins/removeElementsByAttr.js":358,"./plugins/removeEmptyAttrs.js":359,"./plugins/removeEmptyContainers.js":360,"./plugins/removeEmptyText.js":361,"./plugins/removeHiddenElems.js":362,"./plugins/removeMetadata.js":363,"./plugins/removeNonInheritableGroupAttrs.js":364,"./plugins/removeOffCanvasPaths.js":365,"./plugins/removeRasterImages.js":366,"./plugins/removeScriptElement.js":367,"./plugins/removeStyleElement.js":368,"./plugins/removeTitle.js":369,"./plugins/removeUnknownsAndDefaults.js":370,"./plugins/removeUnusedNS.js":371,"./plugins/removeUselessDefs.js":372,"./plugins/removeUselessStrokeAndFill.js":373,"./plugins/removeViewBox.js":374,"./plugins/removeXMLNS.js":375,"./plugins/removeXMLProcInst.js":376,"./plugins/reusePaths.js":377,"./plugins/safe.js":378,"./plugins/sortAttrs.js":379,"./plugins/sortDefsChildren.js":380}],321:[function(require,module,exports){ +},{"./plugins/addAttributesToSVGElement.js":328,"./plugins/addClassesToSVGElement.js":329,"./plugins/cleanupAttrs.js":331,"./plugins/cleanupEnableBackground.js":332,"./plugins/cleanupIds.js":333,"./plugins/cleanupListOfValues.js":334,"./plugins/cleanupNumericValues.js":335,"./plugins/collapseGroups.js":336,"./plugins/convertColors.js":337,"./plugins/convertEllipseToCircle.js":338,"./plugins/convertPathData.js":339,"./plugins/convertShapeToPath.js":340,"./plugins/convertStyleToAttrs.js":341,"./plugins/convertTransform.js":342,"./plugins/default.js":343,"./plugins/inlineStyles.js":344,"./plugins/mergePaths.js":345,"./plugins/mergeStyles.js":346,"./plugins/minifyStyles.js":347,"./plugins/moveElemsAttrsToGroup.js":348,"./plugins/moveGroupAttrsToElems.js":349,"./plugins/prefixIds.js":350,"./plugins/removeAttributesBySelector.js":351,"./plugins/removeAttrs.js":352,"./plugins/removeComments.js":353,"./plugins/removeDesc.js":354,"./plugins/removeDimensions.js":355,"./plugins/removeDoctype.js":356,"./plugins/removeEditorsNSData.js":357,"./plugins/removeElementsByAttr.js":358,"./plugins/removeEmptyAttrs.js":359,"./plugins/removeEmptyContainers.js":360,"./plugins/removeEmptyText.js":361,"./plugins/removeHiddenElems.js":362,"./plugins/removeMetadata.js":363,"./plugins/removeNonInheritableGroupAttrs.js":364,"./plugins/removeOffCanvasPaths.js":365,"./plugins/removeRasterImages.js":366,"./plugins/removeScriptElement.js":367,"./plugins/removeStyleElement.js":368,"./plugins/removeTitle.js":369,"./plugins/removeUnknownsAndDefaults.js":370,"./plugins/removeUnusedNS.js":371,"./plugins/removeUselessDefs.js":372,"./plugins/removeUselessStrokeAndFill.js":373,"./plugins/removeViewBox.js":374,"./plugins/removeXMLNS.js":375,"./plugins/removeXMLProcInst.js":376,"./plugins/reusePaths.js":377,"./plugins/safe.js":378,"./plugins/safeAndFast.js":379,"./plugins/sortAttrs.js":380,"./plugins/sortDefsChildren.js":381}],321:[function(require,module,exports){ 'use strict'; const isTag = (node) => { @@ -30102,7 +30103,7 @@ const stringifyPathData = ({ pathData, precision, disableSpaceAfterFlags }) => { }; exports.stringifyPathData = stringifyPathData; -},{"./tools.js":383}],324:[function(require,module,exports){ +},{"./tools.js":384}],324:[function(require,module,exports){ 'use strict'; const { builtin } = require('./builtin.js'); @@ -33826,7 +33827,7 @@ const applyMatrixToPathData = (pathData, matrix) => { } }; -},{"../style.js":382,"../tools.js":383,"./_collections.js":325,"./_path.js":326,"./_transforms.js":327}],331:[function(require,module,exports){ +},{"../style.js":383,"../tools.js":384,"./_collections.js":325,"./_path.js":326,"./_transforms.js":327}],331:[function(require,module,exports){ 'use strict'; exports.name = 'cleanupAttrs'; @@ -33948,7 +33949,7 @@ exports.fn = (root) => { }; }; -},{"../xast.js":384}],333:[function(require,module,exports){ +},{"../xast.js":385}],333:[function(require,module,exports){ 'use strict'; const { visitSkip } = require('../xast.js'); @@ -34207,7 +34208,7 @@ exports.fn = (_root, params) => { }; }; -},{"../xast.js":384,"./_collections.js":325}],334:[function(require,module,exports){ +},{"../xast.js":385,"./_collections.js":325}],334:[function(require,module,exports){ 'use strict'; const { removeLeadingZero } = require('../tools.js'); @@ -34345,7 +34346,7 @@ exports.fn = (_root, params) => { }; }; -},{"../tools.js":383}],335:[function(require,module,exports){ +},{"../tools.js":384}],335:[function(require,module,exports){ 'use strict'; const { removeLeadingZero } = require('../tools.js'); @@ -34451,7 +34452,7 @@ exports.fn = (_root, params) => { }; }; -},{"../tools.js":383}],336:[function(require,module,exports){ +},{"../tools.js":384}],336:[function(require,module,exports){ 'use strict'; const { inheritableAttrs, elemsGroups } = require('./_collections.js'); @@ -35838,7 +35839,7 @@ function data2Path(params, pathData) { }, ''); } -},{"../style.js":382,"../tools.js":383,"../xast.js":384,"./_collections.js":325,"./_path.js":326,"./applyTransforms.js":330}],340:[function(require,module,exports){ +},{"../style.js":383,"../tools.js":384,"../xast.js":385,"./_collections.js":325,"./_path.js":326,"./applyTransforms.js":330}],340:[function(require,module,exports){ 'use strict'; const { stringifyPathData } = require('../path.js'); @@ -36004,7 +36005,7 @@ exports.fn = (root, params) => { }; }; -},{"../path.js":323,"../xast.js":384}],341:[function(require,module,exports){ +},{"../path.js":323,"../xast.js":385}],341:[function(require,module,exports){ 'use strict'; const { attrsGroups } = require('./_collections'); @@ -36506,7 +36507,7 @@ const smartRound = (precision, data) => { return data; }; -},{"../tools.js":383,"./_transforms.js":327}],343:[function(require,module,exports){ +},{"../tools.js":384,"./_transforms.js":327}],343:[function(require,module,exports){ 'use strict'; const { createPreset } = require('../tools.js'); @@ -36590,7 +36591,7 @@ const presetDefault = createPreset({ module.exports = presetDefault; -},{"../tools.js":383,"./cleanupAttrs.js":331,"./cleanupEnableBackground.js":332,"./cleanupIds.js":333,"./cleanupNumericValues.js":335,"./collapseGroups.js":336,"./convertColors.js":337,"./convertEllipseToCircle.js":338,"./convertPathData.js":339,"./convertShapeToPath.js":340,"./convertTransform.js":342,"./inlineStyles.js":344,"./mergePaths.js":345,"./mergeStyles.js":346,"./minifyStyles.js":347,"./moveElemsAttrsToGroup.js":348,"./moveGroupAttrsToElems.js":349,"./removeComments.js":353,"./removeDesc.js":354,"./removeDoctype.js":356,"./removeEditorsNSData.js":357,"./removeEmptyAttrs.js":359,"./removeEmptyContainers.js":360,"./removeEmptyText.js":361,"./removeHiddenElems.js":362,"./removeMetadata.js":363,"./removeNonInheritableGroupAttrs.js":364,"./removeTitle.js":369,"./removeUnknownsAndDefaults.js":370,"./removeUnusedNS.js":371,"./removeUselessDefs.js":372,"./removeUselessStrokeAndFill.js":373,"./removeViewBox.js":374,"./removeXMLProcInst.js":376,"./sortAttrs.js":379,"./sortDefsChildren.js":380}],344:[function(require,module,exports){ +},{"../tools.js":384,"./cleanupAttrs.js":331,"./cleanupEnableBackground.js":332,"./cleanupIds.js":333,"./cleanupNumericValues.js":335,"./collapseGroups.js":336,"./convertColors.js":337,"./convertEllipseToCircle.js":338,"./convertPathData.js":339,"./convertShapeToPath.js":340,"./convertTransform.js":342,"./inlineStyles.js":344,"./mergePaths.js":345,"./mergeStyles.js":346,"./minifyStyles.js":347,"./moveElemsAttrsToGroup.js":348,"./moveGroupAttrsToElems.js":349,"./removeComments.js":353,"./removeDesc.js":354,"./removeDoctype.js":356,"./removeEditorsNSData.js":357,"./removeEmptyAttrs.js":359,"./removeEmptyContainers.js":360,"./removeEmptyText.js":361,"./removeHiddenElems.js":362,"./removeMetadata.js":363,"./removeNonInheritableGroupAttrs.js":364,"./removeTitle.js":369,"./removeUnknownsAndDefaults.js":370,"./removeUnusedNS.js":371,"./removeUselessDefs.js":372,"./removeUselessStrokeAndFill.js":373,"./removeViewBox.js":374,"./removeXMLProcInst.js":376,"./sortAttrs.js":380,"./sortDefsChildren.js":381}],344:[function(require,module,exports){ 'use strict'; const csstree = require('css-tree'); @@ -36937,7 +36938,7 @@ exports.fn = (root, params) => { }; }; -},{"../xast.js":384,"css-tree":25,"csso":138}],345:[function(require,module,exports){ +},{"../xast.js":385,"css-tree":25,"csso":138}],345:[function(require,module,exports){ 'use strict'; const { detachNodeFromParent } = require('../xast.js'); @@ -37035,7 +37036,7 @@ exports.fn = (root, params) => { }; }; -},{"../style.js":382,"../xast.js":384,"./_path.js":326}],346:[function(require,module,exports){ +},{"../style.js":383,"../xast.js":385,"./_path.js":326}],346:[function(require,module,exports){ 'use strict'; const { visitSkip, detachNodeFromParent } = require('../xast.js'); @@ -37119,7 +37120,7 @@ exports.fn = () => { }; }; -},{"../xast.js":384}],347:[function(require,module,exports){ +},{"../xast.js":385}],347:[function(require,module,exports){ 'use strict'; const csso = require('csso'); @@ -37364,7 +37365,7 @@ exports.fn = (root) => { }; }; -},{"../xast.js":384,"./_collections.js":325}],349:[function(require,module,exports){ +},{"../xast.js":385,"./_collections.js":325}],349:[function(require,module,exports){ 'use strict'; const { pathElems, referencesProps } = require('./_collections.js'); @@ -37742,7 +37743,7 @@ exports.fn = (root, params) => { return {}; }; -},{"../xast.js":384}],352:[function(require,module,exports){ +},{"../xast.js":385}],352:[function(require,module,exports){ 'use strict'; exports.name = 'removeAttrs'; @@ -37924,7 +37925,7 @@ exports.fn = () => { }; }; -},{"../xast.js":384}],354:[function(require,module,exports){ +},{"../xast.js":385}],354:[function(require,module,exports){ 'use strict'; const { detachNodeFromParent } = require('../xast.js'); @@ -37963,7 +37964,7 @@ exports.fn = (root, params) => { }; }; -},{"../xast.js":384}],355:[function(require,module,exports){ +},{"../xast.js":385}],355:[function(require,module,exports){ 'use strict'; exports.name = 'removeDimensions'; @@ -38046,7 +38047,7 @@ exports.fn = () => { }; }; -},{"../xast.js":384}],357:[function(require,module,exports){ +},{"../xast.js":385}],357:[function(require,module,exports){ 'use strict'; const { detachNodeFromParent } = require('../xast.js'); @@ -38110,7 +38111,7 @@ exports.fn = (_root, params) => { }; }; -},{"../xast.js":384,"./_collections.js":325}],358:[function(require,module,exports){ +},{"../xast.js":385,"./_collections.js":325}],358:[function(require,module,exports){ 'use strict'; const { detachNodeFromParent } = require('../xast.js'); @@ -38183,7 +38184,7 @@ exports.fn = (root, params) => { }; }; -},{"../xast.js":384}],359:[function(require,module,exports){ +},{"../xast.js":385}],359:[function(require,module,exports){ 'use strict'; const { attrsGroups } = require('./_collections.js'); @@ -38270,7 +38271,7 @@ exports.fn = () => { }; }; -},{"../xast.js":384,"./_collections.js":325}],361:[function(require,module,exports){ +},{"../xast.js":385,"./_collections.js":325}],361:[function(require,module,exports){ 'use strict'; const { detachNodeFromParent } = require('../xast.js'); @@ -38321,7 +38322,7 @@ exports.fn = (root, params) => { }; }; -},{"../xast.js":384}],362:[function(require,module,exports){ +},{"../xast.js":385}],362:[function(require,module,exports){ 'use strict'; const { @@ -38631,7 +38632,7 @@ exports.fn = (root, params) => { }; }; -},{"../path.js":323,"../style.js":382,"../xast.js":384}],363:[function(require,module,exports){ +},{"../path.js":323,"../style.js":383,"../xast.js":385}],363:[function(require,module,exports){ 'use strict'; const { detachNodeFromParent } = require('../xast.js'); @@ -38658,7 +38659,7 @@ exports.fn = () => { }; }; -},{"../xast.js":384}],364:[function(require,module,exports){ +},{"../xast.js":385}],364:[function(require,module,exports){ 'use strict'; const { @@ -38815,7 +38816,7 @@ exports.fn = () => { }; }; -},{"../path.js":323,"../xast.js":384,"./_path.js":326}],366:[function(require,module,exports){ +},{"../path.js":323,"../xast.js":385,"./_path.js":326}],366:[function(require,module,exports){ 'use strict'; const { detachNodeFromParent } = require('../xast.js'); @@ -38846,7 +38847,7 @@ exports.fn = () => { }; }; -},{"../xast.js":384}],367:[function(require,module,exports){ +},{"../xast.js":385}],367:[function(require,module,exports){ 'use strict'; const { detachNodeFromParent } = require('../xast.js'); @@ -38873,7 +38874,7 @@ exports.fn = () => { }; }; -},{"../xast.js":384}],368:[function(require,module,exports){ +},{"../xast.js":385}],368:[function(require,module,exports){ 'use strict'; const { detachNodeFromParent } = require('../xast.js'); @@ -38900,7 +38901,7 @@ exports.fn = () => { }; }; -},{"../xast.js":384}],369:[function(require,module,exports){ +},{"../xast.js":385}],369:[function(require,module,exports){ 'use strict'; const { detachNodeFromParent } = require('../xast.js'); @@ -38927,7 +38928,7 @@ exports.fn = () => { }; }; -},{"../xast.js":384}],370:[function(require,module,exports){ +},{"../xast.js":385}],370:[function(require,module,exports){ 'use strict'; const { visitSkip, detachNodeFromParent } = require('../xast.js'); @@ -39135,7 +39136,7 @@ exports.fn = (root, params) => { }; }; -},{"../style.js":382,"../xast.js":384,"./_collections":325}],371:[function(require,module,exports){ +},{"../style.js":383,"../xast.js":385,"./_collections":325}],371:[function(require,module,exports){ 'use strict'; exports.name = 'removeUnusedNS'; @@ -39252,7 +39253,7 @@ const collectUsefulNodes = (node, usefulNodes) => { } }; -},{"../xast.js":384,"./_collections.js":325}],373:[function(require,module,exports){ +},{"../xast.js":385,"./_collections.js":325}],373:[function(require,module,exports){ 'use strict'; const { visit, visitSkip, detachNodeFromParent } = require('../xast.js'); @@ -39390,7 +39391,7 @@ exports.fn = (root, params) => { }; }; -},{"../style.js":382,"../xast.js":384,"./_collections.js":325}],374:[function(require,module,exports){ +},{"../style.js":383,"../xast.js":385,"./_collections.js":325}],374:[function(require,module,exports){ 'use strict'; exports.name = 'removeViewBox'; @@ -39497,7 +39498,7 @@ exports.fn = () => { }; }; -},{"../xast.js":384}],377:[function(require,module,exports){ +},{"../xast.js":385}],377:[function(require,module,exports){ 'use strict'; exports.name = 'reusePaths'; @@ -39678,7 +39679,72 @@ const presetSafe = createPreset({ module.exports = presetSafe; -},{"../tools.js":383,"./cleanupAttrs.js":331,"./cleanupEnableBackground.js":332,"./cleanupIds.js":333,"./cleanupNumericValues.js":335,"./collapseGroups.js":336,"./convertColors.js":337,"./convertEllipseToCircle.js":338,"./convertPathData.js":339,"./convertTransform.js":342,"./inlineStyles.js":344,"./mergePaths.js":345,"./mergeStyles.js":346,"./minifyStyles.js":347,"./removeComments.js":353,"./removeDesc.js":354,"./removeDoctype.js":356,"./removeEditorsNSData.js":357,"./removeEmptyAttrs.js":359,"./removeEmptyContainers.js":360,"./removeEmptyText.js":361,"./removeHiddenElems.js":362,"./removeMetadata.js":363,"./removeNonInheritableGroupAttrs.js":364,"./removeTitle.js":369,"./removeUnknownsAndDefaults.js":370,"./removeUnusedNS.js":371,"./removeUselessDefs.js":372,"./removeUselessStrokeAndFill.js":373,"./removeViewBox.js":374,"./removeXMLProcInst.js":376,"./sortDefsChildren.js":380}],379:[function(require,module,exports){ +},{"../tools.js":384,"./cleanupAttrs.js":331,"./cleanupEnableBackground.js":332,"./cleanupIds.js":333,"./cleanupNumericValues.js":335,"./collapseGroups.js":336,"./convertColors.js":337,"./convertEllipseToCircle.js":338,"./convertPathData.js":339,"./convertTransform.js":342,"./inlineStyles.js":344,"./mergePaths.js":345,"./mergeStyles.js":346,"./minifyStyles.js":347,"./removeComments.js":353,"./removeDesc.js":354,"./removeDoctype.js":356,"./removeEditorsNSData.js":357,"./removeEmptyAttrs.js":359,"./removeEmptyContainers.js":360,"./removeEmptyText.js":361,"./removeHiddenElems.js":362,"./removeMetadata.js":363,"./removeNonInheritableGroupAttrs.js":364,"./removeTitle.js":369,"./removeUnknownsAndDefaults.js":370,"./removeUnusedNS.js":371,"./removeUselessDefs.js":372,"./removeUselessStrokeAndFill.js":373,"./removeViewBox.js":374,"./removeXMLProcInst.js":376,"./sortDefsChildren.js":381}],379:[function(require,module,exports){ +'use strict'; + +const { createPreset } = require('../tools.js'); + +const removeDoctype = require('./removeDoctype.js'); +const removeXMLProcInst = require('./removeXMLProcInst.js'); +const removeComments = require('./removeComments.js'); +const removeMetadata = require('./removeMetadata.js'); +const removeEditorsNSData = require('./removeEditorsNSData.js'); +const cleanupAttrs = require('./cleanupAttrs.js'); +const mergeStyles = require('./mergeStyles.js'); +const minifyStyles = require('./minifyStyles.js'); +const cleanupIds = require('./cleanupIds.js'); +const removeUselessDefs = require('./removeUselessDefs.js'); +const cleanupNumericValues = require('./cleanupNumericValues.js'); +const convertColors = require('./convertColors.js'); +const removeUnknownsAndDefaults = require('./removeUnknownsAndDefaults.js'); +const removeNonInheritableGroupAttrs = require('./removeNonInheritableGroupAttrs.js'); +const removeUselessStrokeAndFill = require('./removeUselessStrokeAndFill.js'); +const removeViewBox = require('./removeViewBox.js'); +const cleanupEnableBackground = require('./cleanupEnableBackground.js'); +const removeHiddenElems = require('./removeHiddenElems.js'); +const removeEmptyText = require('./removeEmptyText.js'); +const collapseGroups = require('./collapseGroups.js'); +const removeEmptyAttrs = require('./removeEmptyAttrs.js'); +const removeEmptyContainers = require('./removeEmptyContainers.js'); +const mergePaths = require('./mergePaths.js'); +const removeUnusedNS = require('./removeUnusedNS.js'); +const sortDefsChildren = require('./sortDefsChildren.js'); +const removeTitle = require('./removeTitle.js'); +const removeDesc = require('./removeDesc.js'); + +const presetSafe = createPreset({ + name: 'safeAndFastPreset', + plugins: [ + removeDoctype, + removeXMLProcInst, + removeComments, + removeMetadata, + removeEditorsNSData, + cleanupAttrs, + mergeStyles, + cleanupIds, + removeUselessDefs, + cleanupNumericValues, + convertColors, + removeUnknownsAndDefaults, + removeNonInheritableGroupAttrs, + removeUselessStrokeAndFill, + removeViewBox, + cleanupEnableBackground, + removeHiddenElems, + removeEmptyText, + collapseGroups, + removeEmptyAttrs, + removeEmptyContainers, + removeUnusedNS, + removeTitle, + removeDesc, + ], +}); + +module.exports = presetSafe; + +},{"../tools.js":384,"./cleanupAttrs.js":331,"./cleanupEnableBackground.js":332,"./cleanupIds.js":333,"./cleanupNumericValues.js":335,"./collapseGroups.js":336,"./convertColors.js":337,"./mergePaths.js":345,"./mergeStyles.js":346,"./minifyStyles.js":347,"./removeComments.js":353,"./removeDesc.js":354,"./removeDoctype.js":356,"./removeEditorsNSData.js":357,"./removeEmptyAttrs.js":359,"./removeEmptyContainers.js":360,"./removeEmptyText.js":361,"./removeHiddenElems.js":362,"./removeMetadata.js":363,"./removeNonInheritableGroupAttrs.js":364,"./removeTitle.js":369,"./removeUnknownsAndDefaults.js":370,"./removeUnusedNS.js":371,"./removeUselessDefs.js":372,"./removeUselessStrokeAndFill.js":373,"./removeViewBox.js":374,"./removeXMLProcInst.js":376,"./sortDefsChildren.js":381}],380:[function(require,module,exports){ 'use strict'; exports.name = 'sortAttrs'; @@ -39778,7 +39844,7 @@ exports.fn = (_root, params) => { }; }; -},{}],380:[function(require,module,exports){ +},{}],381:[function(require,module,exports){ 'use strict'; exports.name = 'sortDefsChildren'; @@ -39836,7 +39902,7 @@ exports.fn = () => { }; }; -},{}],381:[function(require,module,exports){ +},{}],382:[function(require,module,exports){ 'use strict'; const { textElems } = require('./plugins/_collections.js'); @@ -40070,7 +40136,7 @@ const stringifyText = (node, config, state) => { ); }; -},{"./plugins/_collections.js":325}],382:[function(require,module,exports){ +},{"./plugins/_collections.js":325}],383:[function(require,module,exports){ 'use strict'; const csstree = require('css-tree'); @@ -40300,7 +40366,7 @@ const computeStyle = (stylesheet, node) => { }; exports.computeStyle = computeStyle; -},{"./plugins/_collections.js":325,"./xast.js":384,"css-tree":25,"csso":138}],383:[function(require,module,exports){ +},{"./plugins/_collections.js":325,"./xast.js":385,"css-tree":25,"csso":138}],384:[function(require,module,exports){ (function (Buffer){(function (){ 'use strict'; @@ -40455,7 +40521,7 @@ exports.invokePlugins = invokePlugins; exports.createPreset = createPreset; }).call(this)}).call(this,require("buffer").Buffer) -},{"./xast.js":384,"buffer":4}],384:[function(require,module,exports){ +},{"./xast.js":385,"buffer":4}],385:[function(require,module,exports){ 'use strict'; const { selectAll, selectOne, is } = require('css-select');