Penpot/penpot
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-01-24 07:29:08 -05:00
Code Issues Activity

✨ Add max-time constraint for migration

Browse source
This commit is contained in:
Andrey Antukh 2023-11-08 11:13:31 +01:00
parent 8193cea7e1
commit ec51e0c0d7
1 changed files with 22 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
backend/src/app/srepl/components_v2.clj
View file

@ -203,7 +203,7 @@
(defn migrate-teams! (defn migrate-teams!
[{:keys [::db/pool] :as system} [{:keys [::db/pool] :as system}
& {:keys [chunk-size max-jobs max-items start-at rollback preset skip-on-error] & {:keys [chunk-size max-jobs max-items start-at rollback preset skip-on-error max-time]
:or {chunk-size 10000 :or {chunk-size 10000
rollback true rollback true
skip-on-error true skip-on-error true
@ -227,14 +227,29 @@
(map #(update % :features db/decode-pgarray #{})) (map #(update % :features db/decode-pgarray #{}))
(remove #(contains? (:features %) "ephimeral/v2-migration")) (remove #(contains? (:features %) "ephimeral/v2-migration"))
(take max-items) (take max-items)
(map :id)))] (map :id)))
(migrate-team [team-id]
(try
(-> (assoc system ::db/rollback rollback)
(feat/migrate-team! team-id))
(catch Throwable cause
(l/err :hint "unexpected error on processing team" :team-id (dm/str team-id) :cause cause))))
(process-team [scope tpoint mtime team-id]
(ps/acquire! feat/*semaphore*)
(let [ts (tpoint)]
(if (and mtime (neg? (compare mtime ts)))
(l/trc :hint "max time constraint reached" :elapsed (dt/format-duration ts))
(px/submit! scope (partial migrate-team team-id)))))]
(l/dbg :hint "migrate:start") (l/dbg :hint "migrate:start")
(let [sem (ps/create :permits max-jobs) (let [sem (ps/create :permits max-jobs)
total (get-total-teams pool) total (get-total-teams pool)
stats (atom {:total/teams (min total max-items)}) stats (atom {:total/teams (min total max-items)})
tpoint (dt/tpoint)] tpoint (dt/tpoint)
mtime (some-> max-time dt/duration)]
(add-watch stats :progress-report (report-progress-teams tpoint)) (add-watch stats :progress-report (report-progress-teams tpoint))
@ -244,17 +259,11 @@
(try (try
(pu/with-open [scope (px/structured-task-scope :preset preset (pu/with-open [scope (px/structured-task-scope :preset preset
:factory :virtual)] :factory :virtual)]
(run! (fn [team-id] (loop [candidates (get-candidates)]
(l/trc :hint "scheduling task" :team-id (dm/str team-id)) (when-let [team-id (first candidates)]
(ps/acquire! sem) (when (process-team scope tpoint mtime team-id)
(px/submit! scope (fn [] (recur (rest candidates)))))
(try
(-> (assoc system ::db/rollback rollback)
(feat/migrate-team! team-id))
(catch Throwable cause
(l/err :hint "unexpected error on processing team" :team-id (dm/str team-id) :cause cause))))))
(get-candidates))
(p/await! scope)) (p/await! scope))
(print-stats! (print-stats!