Penpot/penpot
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-01-23 06:58:58 -05:00
Code Issues Activity

♻️ Make file-xlog-gc task more scalable

Browse source
This commit is contained in:
Andrey Antukh 2024-08-26 10:47:28 +02:00
parent 215148ca81
commit 8dea5d5158
1 changed files with 26 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
backend/src/app/tasks/file_xlog_gc.clj
View file

@ -16,29 +16,37 @@
(def ^:private (def ^:private
sql:delete-files-xlog sql:delete-files-xlog
"delete from file_change "DELETE FROM file_change
where created_at < now() - ?::interval WHERE id IN (SELECT id FROM file_change
and label is NULL") WHERE label IS NULL
AND created_at < ?
ORDER BY created_at LIMIT ?)")
(defn- delete-in-chunks
[{:keys [::chunk-size ::threshold] :as cfg}]
(loop [total 0]
(let [result (-> (db/exec-one! cfg [sql:delete-files-xlog threshold chunk-size])
(db/get-update-count))]
(if (pos? result)
(recur (+ total result))
total))))
(defmethod ig/pre-init-spec ::handler [_] (defmethod ig/pre-init-spec ::handler [_]
(s/keys :req [::db/pool])) (s/keys :req [::db/pool]))
(defmethod ig/prep-key ::handler
[_ cfg]
(assoc cfg ::min-age (dt/duration {:hours 72})))
(defmethod ig/init-key ::handler (defmethod ig/init-key ::handler
[_ {:keys [::db/pool] :as cfg}] [_ {:keys [::db/pool] :as cfg}]
(fn [{:keys [props] :as task}] (fn [{:keys [props] :as task}]
(let [min-age (or (:min-age props) (::min-age cfg))] (let [min-age (or (:min-age props)
(db/with-atomic [conn pool] (dt/duration "72h"))
(let [interval (db/interval min-age) chunk-size (:chunk-size props 5000)
result (db/exec-one! conn [sql:delete-files-xlog interval]) threshold (dt/minus (dt/now) min-age)]
result (db/get-update-count result)]
(l/info :hint "task finished" :min-age (dt/format-duration min-age) :total result) (-> cfg
(assoc ::db/rollback (:rollback? props false))
(when (:rollback? props) (assoc ::threshold threshold)
(db/rollback! conn)) (assoc ::chunk-size chunk-size)
(db/tx-run! (fn [cfg]
result))))) (let [total (delete-in-chunks cfg)]
(l/trc :hint "file xlog cleaned" :total total)
total)))))))