Penpot/penpot
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-01-21 22:22:43 -05:00
Code Issues Activity

♻️ Refactor srepl helpers

Browse source
This commit is contained in:
Andrey Antukh 2023-11-30 13:14:51 +01:00 • committed by Andrés Moya
parent 76a6f077a6
commit f3e9efa6fe
3 changed files with 135 additions and 541 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

402
backend/src/app/srepl/fixes.clj
View file

@ -1,402 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC
(ns app.srepl.fixes
"A collection of adhoc fixes scripts."
(:require
[app.common.data :as d]
[app.common.files.helpers :as cfh]
[app.common.files.validate :as cfv]
[app.common.geom.shapes :as gsh]
[app.common.logging :as l]
[app.common.pprint :refer [pprint]]
[app.common.types.component :as ctk]
[app.common.types.container :as ctn]
[app.common.types.file :as ctf]
[app.common.uuid :as uuid]
[app.db :as db]
[app.rpc.commands.files :as files]
[app.srepl.helpers :as h]
[app.util.blob :as blob]))
(defn validate-file
[file]
(let [libs (->> (files/get-file-libraries app.srepl.helpers/*conn* (:id file))
(cons file)
(map #(files/get-file app.srepl.helpers/*conn* (:id %) (:features file)))
(d/index-by :id))
update-page (fn [page]
(let [errors (cfv/validate-shape uuid/zero file page libs)]
(when (seq errors)
(println "******Errors in file " (:id file) " page " (:id page))
(pprint errors {:level 3}))))]
(update file :data h/update-pages update-page)))
(defn repair-orphaned-shapes
"There are some shapes whose parent has been deleted. This function
detects them and puts them as children of the root node."
([data]
(letfn [(is-orphan? [shape objects]
(and (some? (:parent-id shape))
(nil? (get objects (:parent-id shape)))))
(update-page [page]
(let [objects (:objects page)
orphans (into #{} (filter #(is-orphan? % objects)) (vals objects))]
(if (seq orphans)
(do
(l/info :hint "found a file with orphans" :file-id (:id data) :broken-shapes (count orphans))
(-> page
(h/update-shapes (fn [shape]
(if (contains? orphans shape)
(assoc shape :parent-id uuid/zero)
shape)))
(update-in [:objects uuid/zero :shapes] into (map :id) orphans)))
page)))]
(h/update-pages data update-page)))
;; special arity for to be called from h/analyze-files to search for
;; files with possible issues
([file state]
(repair-orphaned-shapes (:data file))
(update state :total (fnil inc 0))))
(defn rename-layout-attrs
([file]
(let [found? (volatile! false)]
(letfn [(update-shape
[shape]
(when (or (= (:layout-flex-dir shape) :reverse-row)
(= (:layout-flex-dir shape) :reverse-column)
(= (:layout-wrap-type shape) :no-wrap))
(vreset! found? true))
(cond-> shape
(= (:layout-flex-dir shape) :reverse-row)
(assoc :layout-flex-dir :row-reverse)
(= (:layout-flex-dir shape) :reverse-column)
(assoc :layout-flex-dir :column-reverse)
(= (:layout-wrap-type shape) :no-wrap)
(assoc :layout-wrap-type :nowrap)))
(update-page
[page]
(h/update-shapes page update-shape))]
(let [new-file (update file :data h/update-pages update-page)]
(when @found?
(l/info :hint "Found attrs to rename in file"
:id (:id file)
:name (:name file)))
new-file))))
([file state]
(rename-layout-attrs file)
(update state :total (fnil inc 0))))
(defn fix-components-shaperefs
([file]
(if-not (contains? (:features file) "components/v2")
(do
(println " This file is not v2")
file)
(let [libs (->> (files/get-file-libraries app.srepl.helpers/*conn* (:id file))
(cons file)
(map #(files/get-file app.srepl.helpers/*conn* (:id %) (:features file)))
(d/index-by :id))
fix-copy-item
(fn fix-copy-item [allow-head shapes-copy shapes-base copy-id base-id]
(let [copy (first (filter #(= (:id %) copy-id) shapes-copy))
;; do nothing if it is a copy inside of a copy. It will be treated later
stop? (and (not allow-head) (ctk/instance-head? copy))
base (first (filter #(= (:id %) base-id) shapes-base))
fci (partial fix-copy-item false shapes-copy shapes-base)
updates (if (and
(not stop?)
(not= (:shape-ref copy) base-id))
[[(:id copy) base-id]]
[])
child-updates (if (and
(not stop?)
;; If the base has the same number of childrens than the copy, we asume
;; that the shaperefs can be fixed ad pointed in the same order
(= (count (:shapes copy)) (count (:shapes base))))
(apply concat (map fci (:shapes copy) (:shapes base)))
[])]
(concat updates child-updates)))
fix-copy
(fn [objects updates copy]
(let [component (ctf/find-component libs (:component-id copy) {:include-deleted? true})
component-file (get libs (:component-file copy))
component-shapes (ctf/get-component-shapes (:data component-file) component)
copy-shapes (cfh/get-children-with-self objects (:id copy))
copy-updates (fix-copy-item true copy-shapes component-shapes (:id copy) (:main-instance-id component))]
(concat updates copy-updates)))
update-page
(fn [page]
(let [objects (:objects page)
fc (partial fix-copy objects)
copies (->> objects
vals
(filter #(and (ctk/instance-head? %) (not (ctk/main-instance? %)))))
updates (reduce fc [] copies)
updated-page (reduce (fn [p [id shape-ref]]
(assoc-in p [:objects id :shape-ref] shape-ref))
page
updates)]
(println "Page " (:name page) " - Fixing " (count updates))
updated-page))]
(println "Updating " (:name file) (:id file))
(-> file
(update :data h/update-pages update-page)
(assoc ::updated true)))))
([file save?]
(let [file (-> file
(update :data blob/decode)
(fix-components-shaperefs))]
(when (and save? (::updated file))
(let [data (blob/encode (:data file))]
(db/update! h/*conn* :file
{:data data
;; :revn (:revn file)
}
{:id (:id file)})
(files/persist-pointers! h/*conn* (:id file)))))))
(defn fix-component-root
([file]
(let [update-shape (fn [page shape]
(let [parent (get (:objects page) (:parent-id shape))]
(if (and parent
(:component-root shape)
(:shape-ref parent))
(do
(println " Shape " (:name shape) (:id shape))
(dissoc shape :component-root))
shape)))
update-page (fn [page]
(println "Page " (:name page))
(h/update-shapes page (partial update-shape page)))]
(println "Updating " (:name file) (:id file))
(update file :data h/update-pages update-page)))
([file save?]
(let [file (-> file
(update :data blob/decode)
(fix-component-root))]
(when save?
(let [data (blob/encode (:data file))]
(db/update! h/*conn* :file
{:data data
;; :revn (:revn file)
}
{:id (:id file)})
(files/persist-pointers! h/*conn* (:id file)))))))
(defn update-near-components
([file]
(println "Updating " (:name file) (:id file))
(if-not (contains? (:features file) "components/v2")
(do
(println " This file is not v2")
file)
(let [libs (->> (files/get-file-libraries h/*conn* (:id file))
(cons file)
(map #(files/get-file h/*conn* (:id %) (:features file)))
(d/index-by :id))
update-shape
(fn [page shape]
(if-not (:shape-ref shape)
shape
(do
;; Uncomment println's to debug
;; (println " -> Shape " (:name shape) (:id shape) " shape-ref " (:shape-ref shape))
(let [root-shape (ctn/get-copy-root (:objects page) shape)]
(if root-shape
(let [component (ctf/get-component libs (:component-file root-shape) (:component-id root-shape) {:include-deleted? true})
component-file (get libs (:component-file root-shape))
component-shapes (ctf/get-component-shapes (:data component-file) component)
ref-shape (d/seek #(= (:id %) (:shape-ref shape)) component-shapes)]
(if-not (and component component-file component-shapes)
(do
;; (println " -> Shape " (:name shape) (:id shape) " shape-ref " (:shape-ref shape))
;; (when-not component (println " (component not found)"))
;; (when-not component-file (println " (component-file not found)"))
;; (when-not component-shapes (println " (component-shapes not found)"))
shape)
(if ref-shape
shape ; This means that the copy is not nested, or this script already was run
(let [near-shape (d/seek #(= (:shape-ref %) (:shape-ref shape)) component-shapes)]
(if near-shape
(do
(println " -> Shape " (:name shape) (:id shape) " shape-ref " (:shape-ref shape))
(println " new ref-shape " (:id near-shape))
(assoc shape :shape-ref (:id near-shape)))
(do
;; We assume in this case that this is a fostered sub instance, so we do nothing
;; (println " -> Shape " (:name shape) (:id shape) " shape-ref " (:shape-ref shape))
;; (println (near-shape not found)")
shape))))))
(do
;; (println " -> Shape " (:name shape) (:id shape) " shape-ref " (:shape-ref shape))
;; (println " (root shape not found)")
shape))))))
update-page
(fn [page]
(println "Page " (:name page))
(h/update-shapes page (partial update-shape page)))]
(-> file
(update :data h/update-pages update-page)
(assoc ::updated true)))))
([file save?]
(let [file (-> file
(update :data blob/decode)
(update-near-components))]
(when (and save? (::updated file))
(let [data (blob/encode (:data file))]
(db/update! h/*conn* :file
{:data data
;; :revn (:revn file)
}
{:id (:id file)})
(files/persist-pointers! h/*conn* (:id file)))))))
(defn fix-main-shape-name
([file]
(println "Updating " (:name file) (:id file))
(if-not (contains? (:features file) "components/v2")
(do
(println " This file is not v2")
file)
(let [libs (->> (files/get-file-libraries h/*conn* (:id file))
(cons file)
(map #(files/get-file h/*conn* (:id %) (:features file)))
(d/index-by :id))
update-shape
(fn [shape]
(if-not (ctk/instance-head? shape)
shape
(let [component (ctf/get-component libs (:component-file shape) (:component-id shape) {:include-deleted? true})
[_path name] (cfh/parse-path-name (:name shape))
full-name (cfh/clean-path (str (:path component) "/" (:name component)))]
(if (= name (:name component))
(assoc shape :name full-name)
shape))))
update-page
(fn [page]
(println "Page " (:name page))
(h/update-shapes page update-shape))]
(-> file
(update :data h/update-pages update-page)
(assoc ::updated true)))))
([file save?]
(let [file (-> file
(update :data blob/decode)
(fix-main-shape-name))]
(when (and save? (::updated file))
(let [data (blob/encode (:data file))]
(db/update! h/*conn* :file
{:data data
;; :revn (:revn file)
}
{:id (:id file)})
(files/persist-pointers! h/*conn* (:id file)))))))
(defn fix-touched
"For all copies, compare all synced attributes with the main, and set the touched attribute if needed."
([file]
(let [libraries (->> (files/get-file-libraries app.srepl.helpers/*conn* (:id file))
(map #(files/get-file app.srepl.helpers/*conn* (:id %) (:features file)))
(d/index-by :id))
update-shape (fn [page shape]
(if (ctk/in-component-copy? shape)
(let [ref-shape (ctf/find-ref-shape file
(:objects page)
libraries
shape
:include-deleted? true)
fix-touched-attr
(fn [shape [attr group]]
(if (nil? ref-shape)
shape
(let [equal?
(if (= group :geometry-group)
(if (#{:width :height} attr)
(gsh/close-attrs? attr (get shape attr) (get ref-shape attr) 1)
true)
(gsh/close-attrs? attr (get shape attr) (get ref-shape attr)))]
(when (and (not equal?) (not (cfh/touched-group? shape group)))
(println " -> set touched " (:name shape) (:id shape) attr group))
(cond-> shape
(and (not equal?) (not (cfh/touched-group? shape group)))
(update :touched cfh/set-touched-group group)))))
fix-touched-children
(fn [shape]
(let [matches? (fn [[child-id ref-child-id]]
(if child-id
(let [child (ctn/get-shape page child-id)]
(= (:shape-ref child) ref-child-id))
false))
equal? (every? matches? (d/zip-all (:shapes shape) (:shapes ref-shape)))]
(when (and (not equal?) (not (cfh/touched-group? shape :shapes-group)))
(println " -> set touched " (:name shape) (:id shape) :shapes :shapes-group))
(cond-> shape
(and (not equal?) (not (cfh/touched-group? shape :shapes-group)))
(update :touched cfh/set-touched-group :shapes-group))))]
(as-> shape $
(reduce fix-touched-attr $ ctk/sync-attrs)
(fix-touched-children $)))
shape))
update-page (fn [page]
(println "Page " (:name page))
(h/update-shapes page (partial update-shape page)))]
(println "Updating " (:name file) (:id file))
(update file :data h/update-pages update-page)))
([file save?]
(let [file (-> file
(update :data blob/decode)
(fix-touched))]
(when save?
(let [data (blob/encode (:data file))]
(db/update! h/*conn* :file
{:data data
:revn (inc (:revn file))}
{:id (:id file)})
(files/persist-pointers! h/*conn* (:id file)))))))

259
backend/src/app/srepl/helpers.clj
View file

@ -9,7 +9,6 @@
(:refer-clojure :exclude [parse-uuid]) (:refer-clojure :exclude [parse-uuid])
#_:clj-kondo/ignore #_:clj-kondo/ignore
(:require (:require
[app.auth :refer [derive-password]]
[app.common.data :as d] [app.common.data :as d]
[app.common.exceptions :as ex] [app.common.exceptions :as ex]
[app.common.features :as cfeat] [app.common.features :as cfeat]
@ -41,6 +40,7 @@
[promesa.exec.csp :as sp])) [promesa.exec.csp :as sp]))
(def ^:dynamic *conn* nil) (def ^:dynamic *conn* nil)
(def ^:dynamic *system* nil)
(defn println! (defn println!
[& params] [& params]
@ -53,44 +53,22 @@
v v
(d/parse-uuid v))) (d/parse-uuid v)))
(defn resolve-connectable
[o]
(if (db/connection? o)
o
(if (db/pool? o)
o
(or (::db/conn o)
(::db/pool o)))))
(defn reset-password!
"Reset a password to a specific one for a concrete user or all users
if email is `:all` keyword."
[system & {:keys [email password] :or {password "123123"} :as params}]
(us/verify! (contains? params :email) "`email` parameter is mandatory")
(db/with-atomic [conn (:app.db/pool system)]
(let [password (derive-password password)]
(if (= email :all)
(db/exec! conn ["update profile set password=?" password])
(let [email (str/lower email)]
(db/exec! conn ["update profile set password=? where email=?" password email]))))))
(defn reset-file-data! (defn reset-file-data!
"Hardcode replace of the data of one file." "Hardcode replace of the data of one file."
[system id data] [system id data]
(db/with-atomic [conn (:app.db/pool system)] (db/tx-run! system (fn [system]
(db/update! conn :file (db/update! system :file
{:data data} {:data data}
{:id id}))) {:id id}))))
(defn get-file (defn get-file
"Get the migrated data of one file." "Get the migrated data of one file."
[system id] [system id]
(db/with-atomic [conn (:app.db/pool system)] (db/run! system
(binding [pmap/*load-fn* (partial files/load-pointer conn id)] (fn [{:keys [::db/conn]}]
(-> (db/get-by-id conn :file id) (binding [pmap/*load-fn* (partial files/load-pointer conn id)]
(update :data blob/decode) (-> (files/get-file conn id)
(update :data pmg/migrate-data) (files/process-pointers deref))))))
(files/process-pointers deref)))))
(defn validate (defn validate
"Validate structure, referencial integrity and semantic coherence of "Validate structure, referencial integrity and semantic coherence of
@ -145,95 +123,99 @@
(defn update-file! (defn update-file!
"Apply a function to the data of one file. Optionally save the changes or not. "Apply a function to the data of one file. Optionally save the changes or not.
The function receives the decoded and migrated file data." The function receives the decoded and migrated file data."
[system & {:keys [update-fn id save? migrate? inc-revn?] [system & {:keys [update-fn id rollback? migrate? inc-revn?]
:or {save? false migrate? true inc-revn? true}}] :or {rollback? true migrate? true inc-revn? true}}]
(db/with-atomic [conn (:app.db/pool system)] (letfn [(process-file [conn {:keys [features] :as file}]
(let [file (-> (db/get-by-id conn :file id {::db/for-update? true}) (binding [pmap/*tracked* (atom {})
(update :features db/decode-pgarray #{}))] pmap/*load-fn* (partial files/load-pointer conn id)
(binding [*conn* conn cfeat/*wrap-with-pointer-map-fn*
pmap/*tracked* (atom {}) (if (contains? features "fdata/pointer-map") pmap/wrap identity)
pmap/*load-fn* (partial files/load-pointer conn id) cfeat/*wrap-with-objects-map-fn*
cfeat/*wrap-with-pointer-map-fn* (if (contains? features "fdata/objectd-map") omap/wrap identity)]
(if (contains? (:features file) "fdata/pointer-map") pmap/wrap identity)
cfeat/*wrap-with-objects-map-fn* (let [file (cond-> (update-fn file)
(if (contains? (:features file) "fdata/objectd-map") omap/wrap identity)] inc-revn? (update :revn inc))
(let [file (-> file features (db/create-array conn "text" (:features file))
(update :data blob/decode) data (blob/encode (:data file))]
(cond-> migrate? (update :data pmg/migrate-data))
(update-fn) (db/update! conn :file
(cond-> inc-revn? (update :revn inc)))] {:data data
(when save? :revn (:revn file)
(let [features (db/create-array conn "text" (:features file)) :features features}
data (blob/encode (:data file))] {:id id}))
(db/update! conn :file
{:data data
:revn (:revn file)
:features features}
{:id id})
(when (contains? (:features file) "fdata/pointer-map") (when (contains? (:features file) "fdata/pointer-map")
(files/persist-pointers! conn id)))) (files/persist-pointers! conn id))
(dissoc file :data)))))) (dissoc file :data)))]
(def ^:private sql:retrieve-files-chunk (db/tx-run! system
"SELECT id, name, features, created_at, revn, data FROM file (fn [{:keys [::db/conn] :as system}]
WHERE created_at < ? AND deleted_at is NULL (binding [*conn* conn *system* system]
ORDER BY created_at desc LIMIT ?") (try
(->> (files/get-file conn id :migrate? migrate?)
(process-file conn))
(finally
(when rollback?
(db/rollback! conn)))))))))
(defn analyze-files (defn analyze-files
"Apply a function to all files in the database, reading them in "Apply a function to all files in the database, reading them in
batches. Do not change data. batches. Do not change data.
The `on-file` parameter should be a function that receives the file The `on-file` parameter should be a function that receives the file
and the previous state and returns the new state." and the previous state and returns the new state.
Emits rollback at the end of operation."
[system & {:keys [chunk-size max-items start-at on-file on-error on-end on-init with-libraries?] [system & {:keys [chunk-size max-items start-at on-file on-error on-end on-init with-libraries?]
:or {chunk-size 10 max-items Long/MAX_VALUE}}] :or {chunk-size 10 max-items Long/MAX_VALUE}}]
(letfn [(get-chunk [conn cursor] (letfn [(get-chunk [conn cursor]
(let [rows (db/exec! conn [sql:retrieve-files-chunk cursor chunk-size])] (let [sql (str "SELECT id, created_at FROM file "
[(some->> rows peek :created-at) (seq rows)])) " WHERE created_at < ? AND deleted_at is NULL "
" ORDER BY created_at desc LIMIT ?")
rows (db/exec! conn [sql cursor chunk-size])]
[(some->> rows peek :created-at) (map :id rows)]))
(get-candidates [conn] (get-candidates [conn]
(->> (d/iteration (partial get-chunk conn) (->> (d/iteration (partial get-chunk conn)
:vf second :vf second
:kf first :kf first
:initk (or start-at (dt/now))) :initk (or start-at (dt/now)))
(take max-items) (take max-items)))
(map #(-> %
(update :data blob/decode)
(update :features db/decode-pgarray #{})))))
(on-error* [cause file] (on-error* [cause file]
(println "unexpected exception happened on processing file: " (:id file)) (println "unexpected exception happened on processing file: " (:id file))
(strace/print-stack-trace cause))] (strace/print-stack-trace cause))
(when (fn? on-init) (on-init)) (process-file [conn file-id]
(let [file (binding [pmap/*load-fn* (partial files/load-pointer conn file-id)]
(-> (files/get-file conn file-id)
(files/process-pointers deref)))
(db/with-atomic [conn (:app.db/pool system)] libs (when with-libraries?
(doseq [file (get-candidates conn)] (->> (files/get-file-libraries conn file-id)
(binding [*conn* conn (into [file] (map (fn [{:keys [id]}]
pmap/*tracked* (atom {}) (binding [pmap/*load-fn* (partial files/load-pointer conn id)]
pmap/*load-fn* (partial files/load-pointer conn (:id file)) (-> (files/get-file conn id)
cfeat/*wrap-with-pointer-map-fn* (files/process-pointers deref))))))
(if (contains? (:features file) "fdata/pointer-map") pmap/wrap identity) (d/index-by :id)))]
cfeat/*wrap-with-objects-map-fn* (try
(if (contains? (:features file) "fdata/objects-map") omap/wrap identity)] (if with-libraries?
(let [libraries (when with-libraries? (on-file file libs)
(->> (files/get-file-libraries conn (:id file)) (on-file file))
(into [file] (map (fn [{:keys [id]}] (catch Throwable cause
(binding [pmap/*load-fn* (partial files/load-pointer conn id)] ((or on-error on-error*) cause file)))))]
(-> (files-update/get-file conn id)
(update :data blob/decode)
(files/process-pointers deref)))))) ; ensure all pointers resolved
(d/index-by :id)))]
(try
(if with-libraries?
(on-file file libraries)
(on-file file))
(catch Throwable cause
((or on-error on-error*) cause file)))))))
(when (fn? on-end) (on-end)))) (db/tx-run! system
(fn [{:keys [::db/conn] :as system}]
(try
(binding [*conn* conn *system* system]
(when (fn? on-init) (on-init))
(run! (partial process-file conn) (get-candidates conn)))
(finally
(when (fn? on-end)
(ex/ignoring (on-end)))
(db/rollback! conn)))))))
(defn process-files! (defn process-files!
"Apply a function to all files in the database, reading them in "Apply a function to all files in the database, reading them in
@ -246,15 +228,18 @@
on-file on-file
on-error on-error
on-end on-end
on-init] on-init
rollback?]
:or {chunk-size 10 :or {chunk-size 10
max-items Long/MAX_VALUE max-items Long/MAX_VALUE
workers 1}}] workers 1
rollback? true}}]
(letfn [(get-chunk [conn cursor] (letfn [(get-chunk [conn cursor]
(let [rows (db/exec! conn [sql:retrieve-files-chunk cursor chunk-size])] (let [sql (str "SELECT id, created_at FROM file "
[(some->> rows peek :created-at) " WHERE created_at < ? AND deleted_at is NULL "
(map #(update % :features db/decode-pgarray #{}) rows)])) " ORDER BY created_at desc LIMIT ?")
rows (db/exec! conn [sql cursor chunk-size])]
[(some->> rows peek :created-at) (map :id rows)]))
(get-candidates [conn] (get-candidates [conn]
(->> (d/iteration (partial get-chunk conn) (->> (d/iteration (partial get-chunk conn)
@ -267,38 +252,43 @@
(println! "unexpected exception happened on processing file: " (:id file)) (println! "unexpected exception happened on processing file: " (:id file))
(strace/print-stack-trace cause)) (strace/print-stack-trace cause))
(process-file [conn file] (process-file [conn file-id]
(try (try
(binding [*conn* conn (let [{:keys [features] :as file} (files/get-file conn file-id)]
pmap/*tracked* (atom {}) (binding [pmap/*tracked* (atom {})
pmap/*load-fn* (partial files/load-pointer conn (:id file)) pmap/*load-fn* (partial files/load-pointer conn file-id)
cfeat/*wrap-with-pointer-map-fn* cfeat/*wrap-with-pointer-map-fn*
(if (contains? (:features file) "fdata/pointer-map") pmap/wrap identity) (if (contains? features "fdata/pointer-map") pmap/wrap identity)
cfeat/*wrap-with-objects-map-fn* cfeat/*wrap-with-objects-map-fn*
(if (contains? (:features file) "fdata/objectd-map") omap/wrap identity)] (if (contains? features "fdata/objectd-map") omap/wrap identity)]
(on-file file))
(on-file file)
(when (contains? features "fdata/pointer-map")
(files/persist-pointers! conn file-id))))
(catch Throwable cause (catch Throwable cause
((or on-error on-error*) cause file)))) ((or on-error on-error*) cause file-id))))
(run-worker [in index] (run-worker [in index]
(db/with-atomic [conn pool] (db/tx-run! system
(loop [i 0] (fn [{:keys [::db/conn] :as system}]
(when-let [file (sp/take! in)] (binding [*conn* conn *system* system]
(println! "=> worker: index:" index "| loop:" i "| file:" (:id file) "|" (px/get-name)) (loop [i 0]
(process-file conn file) (when-let [file-id (sp/take! in)]
(recur (inc i)))))) (println! "=> worker: index:" index "| loop:" i "| file:" (str file-id) "|" (px/get-name))
(process-file conn file-id)
(recur (inc i)))))
(when rollback?
(db/rollback! conn)))))
(run-producer [input] (run-producer [input]
(db/with-atomic [conn pool] (db/with-atomic [conn pool]
(doseq [file (get-candidates conn)] (doseq [file-id (get-candidates conn)]
(println! "=> producer:" (:id file) "|" (px/get-name)) (println! "=> producer:" file-id "|" (px/get-name))
(sp/put! input file)) (sp/put! input file-id))
(sp/close! input))) (sp/close! input)))]
(start-worker [input index]
(px/thread
{:name (str "penpot/srepl/worker/" index)}
(run-worker input index)))]
(when (fn? on-init) (on-init)) (when (fn? on-init) (on-init))
@ -307,19 +297,12 @@
{:name "penpot/srepl/producer"} {:name "penpot/srepl/producer"}
(run-producer input)) (run-producer input))
threads (->> (range workers) threads (->> (range workers)
(map (partial start-worker input)) (map (fn [index]
(px/thread
{:name (str "penpot/srepl/worker/" index)}
(run-worker input index))))
(cons producer) (cons producer)
(doall))] (doall))]
(run! p/await! threads) (run! p/await! threads)
(when (fn? on-end) (on-end))))) (when (fn? on-end) (on-end)))))
(defn update-pages
"Apply a function to all pages of one file. The function receives a page and returns an updated page."
[data f]
(update data :pages-index update-vals f))
(defn update-shapes
"Apply a function to all shapes of one page The function receives a shape and returns an updated shape"
[page f]
(update page :objects update-vals f))

15
backend/src/app/srepl/main.clj
View file

@ -8,6 +8,7 @@
"A collection of adhoc fixes scripts." "A collection of adhoc fixes scripts."
#_:clj-kondo/ignore #_:clj-kondo/ignore
(:require (:require
[app.auth :refer [derive-password]]
[app.common.data :as d] [app.common.data :as d]
[app.common.data.macros :as dm] [app.common.data.macros :as dm]
[app.common.features :as cfeat] [app.common.features :as cfeat]
@ -18,12 +19,12 @@
[app.config :as cf] [app.config :as cf]
[app.db :as db] [app.db :as db]
[app.features.fdata :as features.fdata] [app.features.fdata :as features.fdata]
[app.main :as main]
[app.msgbus :as mbus] [app.msgbus :as mbus]
[app.rpc.commands.auth :as auth] [app.rpc.commands.auth :as auth]
[app.rpc.commands.files-snapshot :as fsnap] [app.rpc.commands.files-snapshot :as fsnap]
[app.rpc.commands.profile :as profile] [app.rpc.commands.profile :as profile]
[app.srepl.cli :as cli] [app.srepl.cli :as cli]
[app.srepl.fixes :as f]
[app.srepl.helpers :as h] [app.srepl.helpers :as h]
[app.storage :as sto] [app.storage :as sto]
[app.util.blob :as blob] [app.util.blob :as blob]
@ -112,6 +113,18 @@
(db/delete! conn :http-session {:profile-id (:id profile)}) (db/delete! conn :http-session {:profile-id (:id profile)})
:blocked)))) :blocked))))
(defn reset-password!
"Reset a password to a specific one for a concrete user or all users
if email is `:all` keyword."
[system & {:keys [email password] :or {password "123123"} :as params}]
(us/verify! (contains? params :email) "`email` parameter is mandatory")
(db/with-atomic [conn (:app.db/pool system)]
(let [password (derive-password password)]
(if (= email :all)
(db/exec! conn ["update profile set password=?" password])
(let [email (str/lower email)]
(db/exec! conn ["update profile set password=? where email=?" password email]))))))
(defn enable-objects-map-feature-on-file! (defn enable-objects-map-feature-on-file!
[system & {:keys [save? id]}] [system & {:keys [save? id]}]
(h/update-file! system (h/update-file! system