From 893a8992c3bbd6f22890c42fdf102ea74af27711 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 31 Jan 2024 12:55:40 +0100 Subject: [PATCH] :sparkles: Add progress reporting on process-files! srepl helper --- backend/src/app/srepl/helpers.clj | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/backend/src/app/srepl/helpers.clj b/backend/src/app/srepl/helpers.clj index c83bc6cee..4d5ac7371 100644 --- a/backend/src/app/srepl/helpers.clj +++ b/backend/src/app/srepl/helpers.clj @@ -275,9 +275,9 @@ sjobs (ps/create :permits max-jobs) process-file - (fn [file-id tpoint] + (fn [file-id idx tpoint] (try - (l/trc :hint "process:file:start" :file-id (str file-id)) + (l/trc :hint "process:file:start" :file-id (str file-id) :index idx) (db/tx-run! (assoc main/system ::db/rollback rollback?) (fn [{:keys [::db/conn] :as system}] (let [file' (get-file* system file-id) @@ -311,12 +311,14 @@ (catch Throwable cause (l/wrn :hint "unexpected error on processing file (skiping)" :file-id (str file-id) + :index idx :cause cause)) (finally (ps/release! sjobs) (let [elapsed (dt/format-duration (tpoint))] (l/trc :hint "process:file:end" :file-id (str file-id) + :index idx :elapsed elapsed)))))] (try @@ -325,15 +327,18 @@ (db/exec! conn ["SET statement_timeout = 0"]) (db/exec! conn ["SET idle_in_transaction_session_timeout = 0"]) - (run! (fn [file-id] - (ps/acquire! sjobs) - (px/run! executor (partial process-file file-id (dt/tpoint)))) - (->> (db/cursor conn [sql:get-file-ids (or start-at (dt/now))]) - (take max-items) - (map :id))) - - ;; Close and await tasks - (pu/close! executor))) + (try + (reduce (fn [idx file-id] + (ps/acquire! sjobs) + (px/run! executor (partial process-file file-id idx (dt/tpoint))) + (inc idx)) + 1 + (->> (db/cursor conn [sql:get-file-ids (or start-at (dt/now))]) + (take max-items) + (map :id))) + (finally + ;; Close and await tasks + (pu/close! executor))))) (catch Throwable cause (l/dbg :hint "process:error" :cause cause))