Penpot/penpot
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-02-22 06:46:40 -05:00
Code Issues Activity

✨ Improve efficiency and logging on process-files! srepl helper

Browse source
This commit is contained in:
Andrey Antukh 2025-02-14 09:40:12 +01:00
parent af93325fd9
commit 90b63f6ed4
1 changed files with 37 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

61
backend/src/app/srepl/main.clj
View file

@ -431,25 +431,40 @@
process-file process-file
(fn [file-id idx tpoint] (fn [file-id idx tpoint]
(try (let [thread-id (px/get-thread-id)]
(l/trc :hint "process:file:start" :file-id (str file-id) :index idx) (try
(let [system (assoc main/system ::db/rollback rollback?)] (l/trc :hint "process:file:start"
(db/tx-run! system (fn [system] :tid thread-id
(binding [h/*system* system]
(h/process-file! system file-id update-fn opts)))))
(catch Throwable cause
(l/wrn :hint "unexpected error on processing file (skiping)"
:file-id (str file-id) :file-id (str file-id)
:index idx :index idx)
:cause cause)) (let [system (assoc main/system ::db/rollback rollback?)]
(finally (db/tx-run! system (fn [system]
(ps/release! sjobs) (binding [h/*system* system]
(let [elapsed (dt/format-duration (tpoint))] (h/process-file! system file-id update-fn opts)))))
(l/trc :hint "process:file:end"
(catch Throwable cause
(l/wrn :hint "unexpected error on processing file (skiping)"
:tid thread-id
:file-id (str file-id) :file-id (str file-id)
:index idx :index idx
:elapsed elapsed))))) :cause cause))
(finally
(when-let [pause (:pause opts)]
(Thread/sleep (int pause)))
(ps/release! sjobs)
(let [elapsed (dt/format-duration (tpoint))]
(l/trc :hint "process:file:end"
:tid thread-id
:file-id (str file-id)
:index idx
:elapsed elapsed))))))
process-file*
(fn [idx file-id]
(ps/acquire! sjobs)
(px/run! executor (partial process-file file-id idx (dt/tpoint)))
(inc idx))
process-files process-files
(fn [{:keys [::db/conn] :as system}] (fn [{:keys [::db/conn] :as system}]
@ -457,14 +472,12 @@
(db/exec! conn ["SET idle_in_transaction_session_timeout = 0"]) (db/exec! conn ["SET idle_in_transaction_session_timeout = 0"])
(try (try
(reduce (fn [idx file-id] (->> (db/plan conn [query])
(ps/acquire! sjobs) (transduce (comp
(px/run! executor (partial process-file file-id idx (dt/tpoint))) (take max-items)
(inc idx)) (map :id))
0 (completing process-file*)
(->> (db/cursor conn [query] {:chunk-size 1}) 0))
(take max-items)
(map :id)))
(finally (finally
;; Close and await tasks ;; Close and await tasks
(pu/close! executor))))] (pu/close! executor))))]