mirror of
https://github.com/penpot/penpot.git
synced 2025-01-21 22:22:43 -05:00
♻️ Refactor srepl helpers
This commit is contained in:
Andrey Antukh
Andrés Moya
parent
76a6f077a6
commit
f3e9efa6fe
3 changed files with 135 additions and 541 deletions
|
|
@ -1,402 +0,0 @@
|
|||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC
|
||||
|
||||
(ns app.srepl.fixes
|
||||
"A collection of adhoc fixes scripts."
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.files.helpers :as cfh]
|
||||
[app.common.files.validate :as cfv]
|
||||
[app.common.geom.shapes :as gsh]
|
||||
[app.common.logging :as l]
|
||||
[app.common.pprint :refer [pprint]]
|
||||
[app.common.types.component :as ctk]
|
||||
[app.common.types.container :as ctn]
|
||||
[app.common.types.file :as ctf]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.db :as db]
|
||||
[app.rpc.commands.files :as files]
|
||||
[app.srepl.helpers :as h]
|
||||
[app.util.blob :as blob]))
|
||||
|
||||
(defn validate-file
|
||||
[file]
|
||||
(let [libs (->> (files/get-file-libraries app.srepl.helpers/*conn* (:id file))
|
||||
(cons file)
|
||||
(map #(files/get-file app.srepl.helpers/*conn* (:id %) (:features file)))
|
||||
(d/index-by :id))
|
||||
|
||||
update-page (fn [page]
|
||||
(let [errors (cfv/validate-shape uuid/zero file page libs)]
|
||||
(when (seq errors)
|
||||
(println "******Errors in file " (:id file) " page " (:id page))
|
||||
(pprint errors {:level 3}))))]
|
||||
|
||||
(update file :data h/update-pages update-page)))
|
||||
|
||||
(defn repair-orphaned-shapes
|
||||
"There are some shapes whose parent has been deleted. This function
|
||||
detects them and puts them as children of the root node."
|
||||
([data]
|
||||
(letfn [(is-orphan? [shape objects]
|
||||
(and (some? (:parent-id shape))
|
||||
(nil? (get objects (:parent-id shape)))))
|
||||
|
||||
(update-page [page]
|
||||
(let [objects (:objects page)
|
||||
orphans (into #{} (filter #(is-orphan? % objects)) (vals objects))]
|
||||
(if (seq orphans)
|
||||
(do
|
||||
(l/info :hint "found a file with orphans" :file-id (:id data) :broken-shapes (count orphans))
|
||||
(-> page
|
||||
(h/update-shapes (fn [shape]
|
||||
(if (contains? orphans shape)
|
||||
(assoc shape :parent-id uuid/zero)
|
||||
shape)))
|
||||
(update-in [:objects uuid/zero :shapes] into (map :id) orphans)))
|
||||
page)))]
|
||||
|
||||
(h/update-pages data update-page)))
|
||||
|
||||
;; special arity for to be called from h/analyze-files to search for
|
||||
;; files with possible issues
|
||||
|
||||
([file state]
|
||||
(repair-orphaned-shapes (:data file))
|
||||
(update state :total (fnil inc 0))))
|
||||
|
||||
(defn rename-layout-attrs
|
||||
([file]
|
||||
(let [found? (volatile! false)]
|
||||
(letfn [(update-shape
|
||||
[shape]
|
||||
(when (or (= (:layout-flex-dir shape) :reverse-row)
|
||||
(= (:layout-flex-dir shape) :reverse-column)
|
||||
(= (:layout-wrap-type shape) :no-wrap))
|
||||
(vreset! found? true))
|
||||
(cond-> shape
|
||||
(= (:layout-flex-dir shape) :reverse-row)
|
||||
(assoc :layout-flex-dir :row-reverse)
|
||||
(= (:layout-flex-dir shape) :reverse-column)
|
||||
(assoc :layout-flex-dir :column-reverse)
|
||||
(= (:layout-wrap-type shape) :no-wrap)
|
||||
(assoc :layout-wrap-type :nowrap)))
|
||||
|
||||
(update-page
|
||||
[page]
|
||||
(h/update-shapes page update-shape))]
|
||||
|
||||
(let [new-file (update file :data h/update-pages update-page)]
|
||||
(when @found?
|
||||
(l/info :hint "Found attrs to rename in file"
|
||||
:id (:id file)
|
||||
:name (:name file)))
|
||||
new-file))))
|
||||
|
||||
([file state]
|
||||
(rename-layout-attrs file)
|
||||
(update state :total (fnil inc 0))))
|
||||
|
||||
(defn fix-components-shaperefs
|
||||
([file]
|
||||
(if-not (contains? (:features file) "components/v2")
|
||||
(do
|
||||
(println " This file is not v2")
|
||||
file)
|
||||
(let [libs (->> (files/get-file-libraries app.srepl.helpers/*conn* (:id file))
|
||||
(cons file)
|
||||
(map #(files/get-file app.srepl.helpers/*conn* (:id %) (:features file)))
|
||||
(d/index-by :id))
|
||||
|
||||
fix-copy-item
|
||||
(fn fix-copy-item [allow-head shapes-copy shapes-base copy-id base-id]
|
||||
(let [copy (first (filter #(= (:id %) copy-id) shapes-copy))
|
||||
;; do nothing if it is a copy inside of a copy. It will be treated later
|
||||
stop? (and (not allow-head) (ctk/instance-head? copy))
|
||||
base (first (filter #(= (:id %) base-id) shapes-base))
|
||||
fci (partial fix-copy-item false shapes-copy shapes-base)
|
||||
|
||||
updates (if (and
|
||||
(not stop?)
|
||||
(not= (:shape-ref copy) base-id))
|
||||
[[(:id copy) base-id]]
|
||||
[])
|
||||
|
||||
child-updates (if (and
|
||||
(not stop?)
|
||||
;; If the base has the same number of childrens than the copy, we asume
|
||||
;; that the shaperefs can be fixed ad pointed in the same order
|
||||
(= (count (:shapes copy)) (count (:shapes base))))
|
||||
(apply concat (map fci (:shapes copy) (:shapes base)))
|
||||
[])]
|
||||
(concat updates child-updates)))
|
||||
|
||||
fix-copy
|
||||
(fn [objects updates copy]
|
||||
(let [component (ctf/find-component libs (:component-id copy) {:include-deleted? true})
|
||||
component-file (get libs (:component-file copy))
|
||||
component-shapes (ctf/get-component-shapes (:data component-file) component)
|
||||
copy-shapes (cfh/get-children-with-self objects (:id copy))
|
||||
|
||||
copy-updates (fix-copy-item true copy-shapes component-shapes (:id copy) (:main-instance-id component))]
|
||||
(concat updates copy-updates)))
|
||||
|
||||
update-page
|
||||
(fn [page]
|
||||
(let [objects (:objects page)
|
||||
fc (partial fix-copy objects)
|
||||
copies (->> objects
|
||||
vals
|
||||
(filter #(and (ctk/instance-head? %) (not (ctk/main-instance? %)))))
|
||||
updates (reduce fc [] copies)
|
||||
updated-page (reduce (fn [p [id shape-ref]]
|
||||
(assoc-in p [:objects id :shape-ref] shape-ref))
|
||||
page
|
||||
updates)]
|
||||
(println "Page " (:name page) " - Fixing " (count updates))
|
||||
updated-page))]
|
||||
|
||||
(println "Updating " (:name file) (:id file))
|
||||
(-> file
|
||||
(update :data h/update-pages update-page)
|
||||
(assoc ::updated true)))))
|
||||
|
||||
([file save?]
|
||||
(let [file (-> file
|
||||
(update :data blob/decode)
|
||||
(fix-components-shaperefs))]
|
||||
(when (and save? (::updated file))
|
||||
(let [data (blob/encode (:data file))]
|
||||
(db/update! h/*conn* :file
|
||||
{:data data
|
||||
;; :revn (:revn file)
|
||||
}
|
||||
{:id (:id file)})
|
||||
|
||||
(files/persist-pointers! h/*conn* (:id file)))))))
|
||||
|
||||
(defn fix-component-root
|
||||
([file]
|
||||
(let [update-shape (fn [page shape]
|
||||
(let [parent (get (:objects page) (:parent-id shape))]
|
||||
(if (and parent
|
||||
(:component-root shape)
|
||||
(:shape-ref parent))
|
||||
(do
|
||||
(println " Shape " (:name shape) (:id shape))
|
||||
(dissoc shape :component-root))
|
||||
shape)))
|
||||
|
||||
update-page (fn [page]
|
||||
(println "Page " (:name page))
|
||||
(h/update-shapes page (partial update-shape page)))]
|
||||
|
||||
(println "Updating " (:name file) (:id file))
|
||||
(update file :data h/update-pages update-page)))
|
||||
|
||||
([file save?]
|
||||
(let [file (-> file
|
||||
(update :data blob/decode)
|
||||
(fix-component-root))]
|
||||
(when save?
|
||||
(let [data (blob/encode (:data file))]
|
||||
(db/update! h/*conn* :file
|
||||
{:data data
|
||||
;; :revn (:revn file)
|
||||
}
|
||||
{:id (:id file)})
|
||||
|
||||
(files/persist-pointers! h/*conn* (:id file)))))))
|
||||
|
||||
(defn update-near-components
|
||||
([file]
|
||||
(println "Updating " (:name file) (:id file))
|
||||
(if-not (contains? (:features file) "components/v2")
|
||||
(do
|
||||
(println " This file is not v2")
|
||||
file)
|
||||
(let [libs (->> (files/get-file-libraries h/*conn* (:id file))
|
||||
(cons file)
|
||||
(map #(files/get-file h/*conn* (:id %) (:features file)))
|
||||
(d/index-by :id))
|
||||
|
||||
update-shape
|
||||
(fn [page shape]
|
||||
(if-not (:shape-ref shape)
|
||||
shape
|
||||
(do
|
||||
;; Uncomment println's to debug
|
||||
;; (println " -> Shape " (:name shape) (:id shape) " shape-ref " (:shape-ref shape))
|
||||
(let [root-shape (ctn/get-copy-root (:objects page) shape)]
|
||||
(if root-shape
|
||||
(let [component (ctf/get-component libs (:component-file root-shape) (:component-id root-shape) {:include-deleted? true})
|
||||
component-file (get libs (:component-file root-shape))
|
||||
component-shapes (ctf/get-component-shapes (:data component-file) component)
|
||||
ref-shape (d/seek #(= (:id %) (:shape-ref shape)) component-shapes)]
|
||||
(if-not (and component component-file component-shapes)
|
||||
(do
|
||||
;; (println " -> Shape " (:name shape) (:id shape) " shape-ref " (:shape-ref shape))
|
||||
;; (when-not component (println " (component not found)"))
|
||||
;; (when-not component-file (println " (component-file not found)"))
|
||||
;; (when-not component-shapes (println " (component-shapes not found)"))
|
||||
shape)
|
||||
(if ref-shape
|
||||
shape ; This means that the copy is not nested, or this script already was run
|
||||
(let [near-shape (d/seek #(= (:shape-ref %) (:shape-ref shape)) component-shapes)]
|
||||
(if near-shape
|
||||
(do
|
||||
(println " -> Shape " (:name shape) (:id shape) " shape-ref " (:shape-ref shape))
|
||||
(println " new ref-shape " (:id near-shape))
|
||||
(assoc shape :shape-ref (:id near-shape)))
|
||||
(do
|
||||
;; We assume in this case that this is a fostered sub instance, so we do nothing
|
||||
;; (println " -> Shape " (:name shape) (:id shape) " shape-ref " (:shape-ref shape))
|
||||
;; (println (near-shape not found)")
|
||||
shape))))))
|
||||
(do
|
||||
;; (println " -> Shape " (:name shape) (:id shape) " shape-ref " (:shape-ref shape))
|
||||
;; (println " (root shape not found)")
|
||||
shape))))))
|
||||
|
||||
update-page
|
||||
(fn [page]
|
||||
(println "Page " (:name page))
|
||||
(h/update-shapes page (partial update-shape page)))]
|
||||
|
||||
(-> file
|
||||
(update :data h/update-pages update-page)
|
||||
(assoc ::updated true)))))
|
||||
|
||||
([file save?]
|
||||
(let [file (-> file
|
||||
(update :data blob/decode)
|
||||
(update-near-components))]
|
||||
(when (and save? (::updated file))
|
||||
(let [data (blob/encode (:data file))]
|
||||
(db/update! h/*conn* :file
|
||||
{:data data
|
||||
;; :revn (:revn file)
|
||||
}
|
||||
{:id (:id file)})
|
||||
|
||||
(files/persist-pointers! h/*conn* (:id file)))))))
|
||||
|
||||
(defn fix-main-shape-name
|
||||
([file]
|
||||
(println "Updating " (:name file) (:id file))
|
||||
(if-not (contains? (:features file) "components/v2")
|
||||
(do
|
||||
(println " This file is not v2")
|
||||
file)
|
||||
(let [libs (->> (files/get-file-libraries h/*conn* (:id file))
|
||||
(cons file)
|
||||
(map #(files/get-file h/*conn* (:id %) (:features file)))
|
||||
(d/index-by :id))
|
||||
|
||||
update-shape
|
||||
(fn [shape]
|
||||
(if-not (ctk/instance-head? shape)
|
||||
shape
|
||||
(let [component (ctf/get-component libs (:component-file shape) (:component-id shape) {:include-deleted? true})
|
||||
[_path name] (cfh/parse-path-name (:name shape))
|
||||
full-name (cfh/clean-path (str (:path component) "/" (:name component)))]
|
||||
(if (= name (:name component))
|
||||
(assoc shape :name full-name)
|
||||
shape))))
|
||||
|
||||
|
||||
update-page
|
||||
(fn [page]
|
||||
(println "Page " (:name page))
|
||||
(h/update-shapes page update-shape))]
|
||||
|
||||
(-> file
|
||||
(update :data h/update-pages update-page)
|
||||
(assoc ::updated true)))))
|
||||
|
||||
([file save?]
|
||||
(let [file (-> file
|
||||
(update :data blob/decode)
|
||||
(fix-main-shape-name))]
|
||||
(when (and save? (::updated file))
|
||||
(let [data (blob/encode (:data file))]
|
||||
(db/update! h/*conn* :file
|
||||
{:data data
|
||||
;; :revn (:revn file)
|
||||
}
|
||||
{:id (:id file)})
|
||||
|
||||
(files/persist-pointers! h/*conn* (:id file)))))))
|
||||
|
||||
(defn fix-touched
|
||||
"For all copies, compare all synced attributes with the main, and set the touched attribute if needed."
|
||||
([file]
|
||||
(let [libraries (->> (files/get-file-libraries app.srepl.helpers/*conn* (:id file))
|
||||
(map #(files/get-file app.srepl.helpers/*conn* (:id %) (:features file)))
|
||||
(d/index-by :id))
|
||||
|
||||
update-shape (fn [page shape]
|
||||
(if (ctk/in-component-copy? shape)
|
||||
(let [ref-shape (ctf/find-ref-shape file
|
||||
(:objects page)
|
||||
libraries
|
||||
shape
|
||||
:include-deleted? true)
|
||||
fix-touched-attr
|
||||
(fn [shape [attr group]]
|
||||
(if (nil? ref-shape)
|
||||
shape
|
||||
(let [equal?
|
||||
(if (= group :geometry-group)
|
||||
(if (#{:width :height} attr)
|
||||
(gsh/close-attrs? attr (get shape attr) (get ref-shape attr) 1)
|
||||
true)
|
||||
(gsh/close-attrs? attr (get shape attr) (get ref-shape attr)))]
|
||||
(when (and (not equal?) (not (cfh/touched-group? shape group)))
|
||||
(println " -> set touched " (:name shape) (:id shape) attr group))
|
||||
(cond-> shape
|
||||
(and (not equal?) (not (cfh/touched-group? shape group)))
|
||||
(update :touched cfh/set-touched-group group)))))
|
||||
|
||||
fix-touched-children
|
||||
(fn [shape]
|
||||
(let [matches? (fn [[child-id ref-child-id]]
|
||||
(if child-id
|
||||
(let [child (ctn/get-shape page child-id)]
|
||||
(= (:shape-ref child) ref-child-id))
|
||||
false))
|
||||
equal? (every? matches? (d/zip-all (:shapes shape) (:shapes ref-shape)))]
|
||||
(when (and (not equal?) (not (cfh/touched-group? shape :shapes-group)))
|
||||
(println " -> set touched " (:name shape) (:id shape) :shapes :shapes-group))
|
||||
(cond-> shape
|
||||
(and (not equal?) (not (cfh/touched-group? shape :shapes-group)))
|
||||
(update :touched cfh/set-touched-group :shapes-group))))]
|
||||
|
||||
(as-> shape $
|
||||
(reduce fix-touched-attr $ ctk/sync-attrs)
|
||||
(fix-touched-children $)))
|
||||
|
||||
shape))
|
||||
|
||||
update-page (fn [page]
|
||||
(println "Page " (:name page))
|
||||
(h/update-shapes page (partial update-shape page)))]
|
||||
|
||||
(println "Updating " (:name file) (:id file))
|
||||
(update file :data h/update-pages update-page)))
|
||||
|
||||
([file save?]
|
||||
(let [file (-> file
|
||||
(update :data blob/decode)
|
||||
(fix-touched))]
|
||||
(when save?
|
||||
(let [data (blob/encode (:data file))]
|
||||
(db/update! h/*conn* :file
|
||||
{:data data
|
||||
:revn (inc (:revn file))}
|
||||
{:id (:id file)})
|
||||
|
||||
(files/persist-pointers! h/*conn* (:id file)))))))
|
||||
|
|
@ -9,7 +9,6 @@
|
|||
(:refer-clojure :exclude [parse-uuid])
|
||||
#_:clj-kondo/ignore
|
||||
(:require
|
||||
[app.auth :refer [derive-password]]
|
||||
[app.common.data :as d]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.features :as cfeat]
|
||||
|
|
@ -41,6 +40,7 @@
|
|||
[promesa.exec.csp :as sp]))
|
||||
|
||||
(def ^:dynamic *conn* nil)
|
||||
(def ^:dynamic *system* nil)
|
||||
|
||||
(defn println!
|
||||
[& params]
|
||||
|
|
@ -53,44 +53,22 @@
|
|||
v
|
||||
(d/parse-uuid v)))
|
||||
|
||||
(defn resolve-connectable
|
||||
[o]
|
||||
(if (db/connection? o)
|
||||
o
|
||||
(if (db/pool? o)
|
||||
o
|
||||
(or (::db/conn o)
|
||||
(::db/pool o)))))
|
||||
|
||||
(defn reset-password!
|
||||
"Reset a password to a specific one for a concrete user or all users
|
||||
if email is `:all` keyword."
|
||||
[system & {:keys [email password] :or {password "123123"} :as params}]
|
||||
(us/verify! (contains? params :email) "`email` parameter is mandatory")
|
||||
(db/with-atomic [conn (:app.db/pool system)]
|
||||
(let [password (derive-password password)]
|
||||
(if (= email :all)
|
||||
(db/exec! conn ["update profile set password=?" password])
|
||||
(let [email (str/lower email)]
|
||||
(db/exec! conn ["update profile set password=? where email=?" password email]))))))
|
||||
|
||||
(defn reset-file-data!
|
||||
"Hardcode replace of the data of one file."
|
||||
[system id data]
|
||||
(db/with-atomic [conn (:app.db/pool system)]
|
||||
(db/update! conn :file
|
||||
{:data data}
|
||||
{:id id})))
|
||||
(db/tx-run! system (fn [system]
|
||||
(db/update! system :file
|
||||
{:data data}
|
||||
{:id id}))))
|
||||
|
||||
(defn get-file
|
||||
"Get the migrated data of one file."
|
||||
[system id]
|
||||
(db/with-atomic [conn (:app.db/pool system)]
|
||||
(binding [pmap/*load-fn* (partial files/load-pointer conn id)]
|
||||
(-> (db/get-by-id conn :file id)
|
||||
(update :data blob/decode)
|
||||
(update :data pmg/migrate-data)
|
||||
(files/process-pointers deref)))))
|
||||
(db/run! system
|
||||
(fn [{:keys [::db/conn]}]
|
||||
(binding [pmap/*load-fn* (partial files/load-pointer conn id)]
|
||||
(-> (files/get-file conn id)
|
||||
(files/process-pointers deref))))))
|
||||
|
||||
(defn validate
|
||||
"Validate structure, referencial integrity and semantic coherence of
|
||||
|
|
@ -145,95 +123,99 @@
|
|||
(defn update-file!
|
||||
"Apply a function to the data of one file. Optionally save the changes or not.
|
||||
The function receives the decoded and migrated file data."
|
||||
[system & {:keys [update-fn id save? migrate? inc-revn?]
|
||||
:or {save? false migrate? true inc-revn? true}}]
|
||||
(db/with-atomic [conn (:app.db/pool system)]
|
||||
(let [file (-> (db/get-by-id conn :file id {::db/for-update? true})
|
||||
(update :features db/decode-pgarray #{}))]
|
||||
(binding [*conn* conn
|
||||
pmap/*tracked* (atom {})
|
||||
pmap/*load-fn* (partial files/load-pointer conn id)
|
||||
cfeat/*wrap-with-pointer-map-fn*
|
||||
(if (contains? (:features file) "fdata/pointer-map") pmap/wrap identity)
|
||||
cfeat/*wrap-with-objects-map-fn*
|
||||
(if (contains? (:features file) "fdata/objectd-map") omap/wrap identity)]
|
||||
(let [file (-> file
|
||||
(update :data blob/decode)
|
||||
(cond-> migrate? (update :data pmg/migrate-data))
|
||||
(update-fn)
|
||||
(cond-> inc-revn? (update :revn inc)))]
|
||||
(when save?
|
||||
(let [features (db/create-array conn "text" (:features file))
|
||||
data (blob/encode (:data file))]
|
||||
(db/update! conn :file
|
||||
{:data data
|
||||
:revn (:revn file)
|
||||
:features features}
|
||||
{:id id})
|
||||
[system & {:keys [update-fn id rollback? migrate? inc-revn?]
|
||||
:or {rollback? true migrate? true inc-revn? true}}]
|
||||
(letfn [(process-file [conn {:keys [features] :as file}]
|
||||
(binding [pmap/*tracked* (atom {})
|
||||
pmap/*load-fn* (partial files/load-pointer conn id)
|
||||
cfeat/*wrap-with-pointer-map-fn*
|
||||
(if (contains? features "fdata/pointer-map") pmap/wrap identity)
|
||||
cfeat/*wrap-with-objects-map-fn*
|
||||
(if (contains? features "fdata/objectd-map") omap/wrap identity)]
|
||||
|
||||
(let [file (cond-> (update-fn file)
|
||||
inc-revn? (update :revn inc))
|
||||
features (db/create-array conn "text" (:features file))
|
||||
data (blob/encode (:data file))]
|
||||
|
||||
(db/update! conn :file
|
||||
{:data data
|
||||
:revn (:revn file)
|
||||
:features features}
|
||||
{:id id}))
|
||||
|
||||
(when (contains? (:features file) "fdata/pointer-map")
|
||||
(files/persist-pointers! conn id))))
|
||||
(files/persist-pointers! conn id))
|
||||
|
||||
(dissoc file :data))))))
|
||||
(dissoc file :data)))]
|
||||
|
||||
(def ^:private sql:retrieve-files-chunk
|
||||
"SELECT id, name, features, created_at, revn, data FROM file
|
||||
WHERE created_at < ? AND deleted_at is NULL
|
||||
ORDER BY created_at desc LIMIT ?")
|
||||
(db/tx-run! system
|
||||
(fn [{:keys [::db/conn] :as system}]
|
||||
(binding [*conn* conn *system* system]
|
||||
(try
|
||||
(->> (files/get-file conn id :migrate? migrate?)
|
||||
(process-file conn))
|
||||
(finally
|
||||
(when rollback?
|
||||
(db/rollback! conn)))))))))
|
||||
|
||||
(defn analyze-files
|
||||
"Apply a function to all files in the database, reading them in
|
||||
batches. Do not change data.
|
||||
|
||||
The `on-file` parameter should be a function that receives the file
|
||||
and the previous state and returns the new state."
|
||||
and the previous state and returns the new state.
|
||||
|
||||
Emits rollback at the end of operation."
|
||||
[system & {:keys [chunk-size max-items start-at on-file on-error on-end on-init with-libraries?]
|
||||
:or {chunk-size 10 max-items Long/MAX_VALUE}}]
|
||||
(letfn [(get-chunk [conn cursor]
|
||||
(let [rows (db/exec! conn [sql:retrieve-files-chunk cursor chunk-size])]
|
||||
[(some->> rows peek :created-at) (seq rows)]))
|
||||
(let [sql (str "SELECT id, created_at FROM file "
|
||||
" WHERE created_at < ? AND deleted_at is NULL "
|
||||
" ORDER BY created_at desc LIMIT ?")
|
||||
rows (db/exec! conn [sql cursor chunk-size])]
|
||||
[(some->> rows peek :created-at) (map :id rows)]))
|
||||
|
||||
(get-candidates [conn]
|
||||
(->> (d/iteration (partial get-chunk conn)
|
||||
:vf second
|
||||
:kf first
|
||||
:initk (or start-at (dt/now)))
|
||||
(take max-items)
|
||||
(map #(-> %
|
||||
(update :data blob/decode)
|
||||
(update :features db/decode-pgarray #{})))))
|
||||
(take max-items)))
|
||||
|
||||
(on-error* [cause file]
|
||||
(println "unexpected exception happened on processing file: " (:id file))
|
||||
(strace/print-stack-trace cause))]
|
||||
(strace/print-stack-trace cause))
|
||||
|
||||
(when (fn? on-init) (on-init))
|
||||
(process-file [conn file-id]
|
||||
(let [file (binding [pmap/*load-fn* (partial files/load-pointer conn file-id)]
|
||||
(-> (files/get-file conn file-id)
|
||||
(files/process-pointers deref)))
|
||||
|
||||
(db/with-atomic [conn (:app.db/pool system)]
|
||||
(doseq [file (get-candidates conn)]
|
||||
(binding [*conn* conn
|
||||
pmap/*tracked* (atom {})
|
||||
pmap/*load-fn* (partial files/load-pointer conn (:id file))
|
||||
cfeat/*wrap-with-pointer-map-fn*
|
||||
(if (contains? (:features file) "fdata/pointer-map") pmap/wrap identity)
|
||||
cfeat/*wrap-with-objects-map-fn*
|
||||
(if (contains? (:features file) "fdata/objects-map") omap/wrap identity)]
|
||||
(let [libraries (when with-libraries?
|
||||
(->> (files/get-file-libraries conn (:id file))
|
||||
(into [file] (map (fn [{:keys [id]}]
|
||||
(binding [pmap/*load-fn* (partial files/load-pointer conn id)]
|
||||
(-> (files-update/get-file conn id)
|
||||
(update :data blob/decode)
|
||||
(files/process-pointers deref)))))) ; ensure all pointers resolved
|
||||
(d/index-by :id)))]
|
||||
(try
|
||||
(if with-libraries?
|
||||
(on-file file libraries)
|
||||
(on-file file))
|
||||
(catch Throwable cause
|
||||
((or on-error on-error*) cause file)))))))
|
||||
libs (when with-libraries?
|
||||
(->> (files/get-file-libraries conn file-id)
|
||||
(into [file] (map (fn [{:keys [id]}]
|
||||
(binding [pmap/*load-fn* (partial files/load-pointer conn id)]
|
||||
(-> (files/get-file conn id)
|
||||
(files/process-pointers deref))))))
|
||||
(d/index-by :id)))]
|
||||
(try
|
||||
(if with-libraries?
|
||||
(on-file file libs)
|
||||
(on-file file))
|
||||
(catch Throwable cause
|
||||
((or on-error on-error*) cause file)))))]
|
||||
|
||||
(when (fn? on-end) (on-end))))
|
||||
(db/tx-run! system
|
||||
(fn [{:keys [::db/conn] :as system}]
|
||||
(try
|
||||
(binding [*conn* conn *system* system]
|
||||
(when (fn? on-init) (on-init))
|
||||
(run! (partial process-file conn) (get-candidates conn)))
|
||||
(finally
|
||||
(when (fn? on-end)
|
||||
(ex/ignoring (on-end)))
|
||||
(db/rollback! conn)))))))
|
||||
|
||||
(defn process-files!
|
||||
"Apply a function to all files in the database, reading them in
|
||||
|
|
@ -246,15 +228,18 @@
|
|||
on-file
|
||||
on-error
|
||||
on-end
|
||||
on-init]
|
||||
on-init
|
||||
rollback?]
|
||||
:or {chunk-size 10
|
||||
max-items Long/MAX_VALUE
|
||||
workers 1}}]
|
||||
|
||||
workers 1
|
||||
rollback? true}}]
|
||||
(letfn [(get-chunk [conn cursor]
|
||||
(let [rows (db/exec! conn [sql:retrieve-files-chunk cursor chunk-size])]
|
||||
[(some->> rows peek :created-at)
|
||||
(map #(update % :features db/decode-pgarray #{}) rows)]))
|
||||
(let [sql (str "SELECT id, created_at FROM file "
|
||||
" WHERE created_at < ? AND deleted_at is NULL "
|
||||
" ORDER BY created_at desc LIMIT ?")
|
||||
rows (db/exec! conn [sql cursor chunk-size])]
|
||||
[(some->> rows peek :created-at) (map :id rows)]))
|
||||
|
||||
(get-candidates [conn]
|
||||
(->> (d/iteration (partial get-chunk conn)
|
||||
|
|
@ -267,38 +252,43 @@
|
|||
(println! "unexpected exception happened on processing file: " (:id file))
|
||||
(strace/print-stack-trace cause))
|
||||
|
||||
(process-file [conn file]
|
||||
(process-file [conn file-id]
|
||||
(try
|
||||
(binding [*conn* conn
|
||||
pmap/*tracked* (atom {})
|
||||
pmap/*load-fn* (partial files/load-pointer conn (:id file))
|
||||
cfeat/*wrap-with-pointer-map-fn*
|
||||
(if (contains? (:features file) "fdata/pointer-map") pmap/wrap identity)
|
||||
cfeat/*wrap-with-objects-map-fn*
|
||||
(if (contains? (:features file) "fdata/objectd-map") omap/wrap identity)]
|
||||
(on-file file))
|
||||
(let [{:keys [features] :as file} (files/get-file conn file-id)]
|
||||
(binding [pmap/*tracked* (atom {})
|
||||
pmap/*load-fn* (partial files/load-pointer conn file-id)
|
||||
cfeat/*wrap-with-pointer-map-fn*
|
||||
(if (contains? features "fdata/pointer-map") pmap/wrap identity)
|
||||
cfeat/*wrap-with-objects-map-fn*
|
||||
(if (contains? features "fdata/objectd-map") omap/wrap identity)]
|
||||
|
||||
(on-file file)
|
||||
|
||||
(when (contains? features "fdata/pointer-map")
|
||||
(files/persist-pointers! conn file-id))))
|
||||
|
||||
(catch Throwable cause
|
||||
((or on-error on-error*) cause file))))
|
||||
((or on-error on-error*) cause file-id))))
|
||||
|
||||
(run-worker [in index]
|
||||
(db/with-atomic [conn pool]
|
||||
(loop [i 0]
|
||||
(when-let [file (sp/take! in)]
|
||||
(println! "=> worker: index:" index "| loop:" i "| file:" (:id file) "|" (px/get-name))
|
||||
(process-file conn file)
|
||||
(recur (inc i))))))
|
||||
(db/tx-run! system
|
||||
(fn [{:keys [::db/conn] :as system}]
|
||||
(binding [*conn* conn *system* system]
|
||||
(loop [i 0]
|
||||
(when-let [file-id (sp/take! in)]
|
||||
(println! "=> worker: index:" index "| loop:" i "| file:" (str file-id) "|" (px/get-name))
|
||||
(process-file conn file-id)
|
||||
(recur (inc i)))))
|
||||
|
||||
(when rollback?
|
||||
(db/rollback! conn)))))
|
||||
|
||||
(run-producer [input]
|
||||
(db/with-atomic [conn pool]
|
||||
(doseq [file (get-candidates conn)]
|
||||
(println! "=> producer:" (:id file) "|" (px/get-name))
|
||||
(sp/put! input file))
|
||||
(sp/close! input)))
|
||||
|
||||
(start-worker [input index]
|
||||
(px/thread
|
||||
{:name (str "penpot/srepl/worker/" index)}
|
||||
(run-worker input index)))]
|
||||
(doseq [file-id (get-candidates conn)]
|
||||
(println! "=> producer:" file-id "|" (px/get-name))
|
||||
(sp/put! input file-id))
|
||||
(sp/close! input)))]
|
||||
|
||||
(when (fn? on-init) (on-init))
|
||||
|
||||
|
|
@ -307,19 +297,12 @@
|
|||
{:name "penpot/srepl/producer"}
|
||||
(run-producer input))
|
||||
threads (->> (range workers)
|
||||
(map (partial start-worker input))
|
||||
(map (fn [index]
|
||||
(px/thread
|
||||
{:name (str "penpot/srepl/worker/" index)}
|
||||
(run-worker input index))))
|
||||
(cons producer)
|
||||
(doall))]
|
||||
|
||||
(run! p/await! threads)
|
||||
(when (fn? on-end) (on-end)))))
|
||||
|
||||
(defn update-pages
|
||||
"Apply a function to all pages of one file. The function receives a page and returns an updated page."
|
||||
[data f]
|
||||
(update data :pages-index update-vals f))
|
||||
|
||||
(defn update-shapes
|
||||
"Apply a function to all shapes of one page The function receives a shape and returns an updated shape"
|
||||
[page f]
|
||||
(update page :objects update-vals f))
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
"A collection of adhoc fixes scripts."
|
||||
#_:clj-kondo/ignore
|
||||
(:require
|
||||
[app.auth :refer [derive-password]]
|
||||
[app.common.data :as d]
|
||||
[app.common.data.macros :as dm]
|
||||
[app.common.features :as cfeat]
|
||||
|
|
@ -18,12 +19,12 @@
|
|||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.features.fdata :as features.fdata]
|
||||
[app.main :as main]
|
||||
[app.msgbus :as mbus]
|
||||
[app.rpc.commands.auth :as auth]
|
||||
[app.rpc.commands.files-snapshot :as fsnap]
|
||||
[app.rpc.commands.profile :as profile]
|
||||
[app.srepl.cli :as cli]
|
||||
[app.srepl.fixes :as f]
|
||||
[app.srepl.helpers :as h]
|
||||
[app.storage :as sto]
|
||||
[app.util.blob :as blob]
|
||||
|
|
@ -112,6 +113,18 @@
|
|||
(db/delete! conn :http-session {:profile-id (:id profile)})
|
||||
:blocked))))
|
||||
|
||||
(defn reset-password!
|
||||
"Reset a password to a specific one for a concrete user or all users
|
||||
if email is `:all` keyword."
|
||||
[system & {:keys [email password] :or {password "123123"} :as params}]
|
||||
(us/verify! (contains? params :email) "`email` parameter is mandatory")
|
||||
(db/with-atomic [conn (:app.db/pool system)]
|
||||
(let [password (derive-password password)]
|
||||
(if (= email :all)
|
||||
(db/exec! conn ["update profile set password=?" password])
|
||||
(let [email (str/lower email)]
|
||||
(db/exec! conn ["update profile set password=? where email=?" password email]))))))
|
||||
|
||||
(defn enable-objects-map-feature-on-file!
|
||||
[system & {:keys [save? id]}]
|
||||
(h/update-file! system
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue