Penpot/penpot
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-02-22 06:46:40 -05:00
Code Issues Activity

✨ Improve efficiency and logging on process-files! srepl helper

Browse source
This commit is contained in:
Andrey Antukh 2025-02-14 09:40:12 +01:00
parent af93325fd9
commit 90b63f6ed4
1 changed files with 37 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
backend/src/app/srepl/main.clj
View file

@ -431,8 +431,12 @@
process-file process-file
(fn [file-id idx tpoint] (fn [file-id idx tpoint]
(let [thread-id (px/get-thread-id)]
(try (try
(l/trc :hint "process:file:start" :file-id (str file-id) :index idx) (l/trc :hint "process:file:start"
:tid thread-id
:file-id (str file-id)
:index idx)
(let [system (assoc main/system ::db/rollback rollback?)] (let [system (assoc main/system ::db/rollback rollback?)]
(db/tx-run! system (fn [system] (db/tx-run! system (fn [system]
(binding [h/*system* system] (binding [h/*system* system]
@ -440,16 +444,27 @@
(catch Throwable cause (catch Throwable cause
(l/wrn :hint "unexpected error on processing file (skiping)" (l/wrn :hint "unexpected error on processing file (skiping)"
:tid thread-id
:file-id (str file-id) :file-id (str file-id)
:index idx :index idx
:cause cause)) :cause cause))
(finally (finally
(when-let [pause (:pause opts)]
(Thread/sleep (int pause)))
(ps/release! sjobs) (ps/release! sjobs)
(let [elapsed (dt/format-duration (tpoint))] (let [elapsed (dt/format-duration (tpoint))]
(l/trc :hint "process:file:end" (l/trc :hint "process:file:end"
:tid thread-id
:file-id (str file-id) :file-id (str file-id)
:index idx :index idx
:elapsed elapsed))))) :elapsed elapsed))))))
process-file*
(fn [idx file-id]
(ps/acquire! sjobs)
(px/run! executor (partial process-file file-id idx (dt/tpoint)))
(inc idx))
process-files process-files
(fn [{:keys [::db/conn] :as system}] (fn [{:keys [::db/conn] :as system}]
@ -457,14 +472,12 @@
(db/exec! conn ["SET idle_in_transaction_session_timeout = 0"]) (db/exec! conn ["SET idle_in_transaction_session_timeout = 0"])
(try (try
(reduce (fn [idx file-id] (->> (db/plan conn [query])
(ps/acquire! sjobs) (transduce (comp
(px/run! executor (partial process-file file-id idx (dt/tpoint)))
(inc idx))
0
(->> (db/cursor conn [query] {:chunk-size 1})
(take max-items) (take max-items)
(map :id))) (map :id))
(completing process-file*)
0))
(finally (finally
;; Close and await tasks ;; Close and await tasks
(pu/close! executor))))] (pu/close! executor))))]