Penpot/penpot
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-03-18 10:41:29 -05:00
Code Issues Activity

🔥 Replace own scheduled executor with the one defined in promesa lib

Browse source
This commit is contained in:
Andrey Antukh 2023-02-20 13:12:19 +01:00
parent 012ead65b5
commit cad1851e95
4 changed files with 47 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
backend/src/app/main.clj
View file

@ -176,9 +176,6 @@
::wrk/executor
{::wrk/parallelism (cf/get :default-executor-parallelism 100)}
::wrk/scheduled-executor
{::wrk/parallelism (cf/get :scheduled-executor-parallelism 20)}
::wrk/monitor
{::mtx/metrics (ig/ref ::mtx/metrics)
::wrk/name "default"
@ -203,8 +200,7 @@
:redis (ig/ref ::rds/redis)}
:app.storage.tmp/cleaner
{::wrk/executor (ig/ref ::wrk/executor)
::wrk/scheduled-executor (ig/ref ::wrk/scheduled-executor)}
{::wrk/executor (ig/ref ::wrk/executor)}
::sto/gc-deleted-task
{::db/pool (ig/ref ::db/pool)
@ -319,8 +315,7 @@
::wrk/executor (ig/ref ::wrk/executor)}
:app.rpc/rlimit
{::wrk/executor (ig/ref ::wrk/executor)
::wrk/scheduled-executor (ig/ref ::wrk/scheduled-executor)}
{::wrk/executor (ig/ref ::wrk/executor)}
:app.rpc/methods
{::http.client/client (ig/ref ::http.client/client)
@ -466,8 +461,7 @@
(def worker-config
{::wrk/cron
{::wrk/scheduled-executor (ig/ref ::wrk/scheduled-executor)
::wrk/registry (ig/ref ::wrk/registry)
{::wrk/registry (ig/ref ::wrk/registry)
::db/pool (ig/ref ::db/pool)
::wrk/entries
[{:cron #app/cron "0 0 * * * ?" ;; hourly

8
backend/src/app/rpc/rlimit.clj
View file

@ -352,7 +352,7 @@
::limits limits}))))
(defn- refresh-config
[{:keys [::state ::path ::wrk/executor ::wrk/scheduled-executor] :as cfg}]
[{:keys [::state ::path ::wrk/executor] :as cfg}]
(letfn [(update-config [{:keys [::updated-at] :as state}]
(let [updated-at' (fs/last-modified-time path)]
(merge state
@ -367,8 +367,7 @@
state)))))
(schedule-next [state]
(px/schedule! scheduled-executor
(inst-ms (::refresh state))
(px/schedule! (inst-ms (::refresh state))
(partial refresh-config cfg))
state)]
@ -391,8 +390,7 @@
(and (fs/exists? path) (fs/regular-file? path) path)))
(defmethod ig/pre-init-spec :app.rpc/rlimit [_]
(s/keys :req [::wrk/executor
::wrk/scheduled-executor]))
(s/keys :req [::wrk/executor]))
(defmethod ig/init-key ::rpc/rlimit
[_ {:keys [::wrk/executor] :as cfg}]

64
backend/src/app/storage/tmp.clj
View file

@ -10,57 +10,59 @@
the operating system cleaning task should be responsible of
permanently delete these files (look at systemd-tempfiles)."
(:require
[app.common.data :as d]
[app.common.logging :as l]
[app.storage :as-alias sto]
[app.util.time :as dt]
[app.worker :as wrk]
[clojure.core.async :as a]
[clojure.spec.alpha :as s]
[datoteka.fs :as fs]
[integrant.core :as ig]
[promesa.exec :as px]))
[promesa.exec :as px]
[promesa.exec.csp :as sp]))
(declare remove-temp-file)
(defonce queue (a/chan 128))
(declare ^:private remove-temp-file)
(declare ^:private io-loop)
(defonce queue (sp/chan :buf 128))
(defmethod ig/pre-init-spec ::cleaner [_]
(s/keys :req [::sto/min-age ::wrk/scheduled-executor]))
(s/keys :req [::wrk/executor]))
(defmethod ig/prep-key ::cleaner
[_ cfg]
(merge {::sto/min-age (dt/duration "30m")}
(d/without-nils cfg)))
(assoc cfg ::min-age (dt/duration "30m")))
(defmethod ig/init-key ::cleaner
[_ {:keys [::sto/min-age ::wrk/scheduled-executor] :as cfg}]
(px/thread
{:name "penpot/storage-tmp-cleaner"}
(try
(l/info :hint "started tmp file cleaner")
(loop []
(when-let [path (a/<!! queue)]
(l/trace :hint "schedule tempfile deletion" :path path
:expires-at (dt/plus (dt/now) min-age))
(px/schedule! scheduled-executor
(inst-ms min-age)
(partial remove-temp-file path))
(recur)))
(catch InterruptedException _
(l/debug :hint "interrupted"))
(finally
(l/info :hint "terminated tmp file cleaner")))))
[_ cfg]
(px/fn->thread (partial io-loop cfg)
{:name "penpot/storage/tmp-cleaner" :virtual true}))
(defmethod ig/halt-key! ::cleaner
[_ thread]
(px/interrupt! thread))
(defn- io-loop
[{:keys [::min-age] :as cfg}]
(l/info :hint "started tmp file cleaner")
(try
(loop []
(when-let [path (sp/take! queue)]
(l/debug :hint "schedule tempfile deletion" :path path
:expires-at (dt/plus (dt/now) min-age))
(px/schedule! (inst-ms min-age) (partial remove-temp-file cfg path))
(recur)))
(catch InterruptedException _
(l/trace :hint "cleaner interrupted"))
(finally
(l/info :hint "cleaner terminated"))))
(defn- remove-temp-file
"Permanently delete tempfile"
[path]
(l/trace :hint "permanently delete tempfile" :path path)
[{:keys [::wrk/executor path]}]
(when (fs/exists? path)
(fs/delete path)))
(px/run! executor
(fn []
(l/debug :hint "permanently delete tempfile" :path path)
(fs/delete path)))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; API
@ -72,7 +74,7 @@
:or {prefix "penpot."
suffix ".tmp"}}]
(let [candidate (fs/tempfile :suffix suffix :prefix prefix)]
(a/offer! queue candidate)
(sp/offer! queue candidate)
candidate))
(defn create-tempfile
@ -80,5 +82,5 @@
:or {prefix "penpot."
suffix ".tmp"}}]
(let [path (fs/create-tempfile :suffix suffix :prefix prefix)]
(a/offer! queue path)
(sp/offer! queue path)
path))

35
backend/src/app/worker.clj
View file

@ -22,6 +22,7 @@
[clojure.spec.alpha :as s]
[cuerdas.core :as str]
[integrant.core :as ig]
[promesa.core :as p]
[promesa.exec :as px])
(:import
java.util.concurrent.ExecutorService
@ -32,7 +33,6 @@
(set! *warn-on-reflection* true)
(s/def ::executor #(instance? ExecutorService %))
(s/def ::scheduled-executor #(instance? ScheduledExecutorService %))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Executor
@ -48,28 +48,14 @@
(let [prefix (if (vector? skey) (-> skey first name) "default")
tname (str "penpot/" prefix "/%s")
factory (px/forkjoin-thread-factory :name tname)]
(px/forkjoin-executor
:factory factory
:parallelism parallelism
:async? true)))
(px/forkjoin-executor {:factory factory
:parallelism parallelism
:async true})))
(defmethod ig/halt-key! ::executor
[_ instance]
(px/shutdown! instance))
(defmethod ig/pre-init-spec ::scheduled-executor [_]
(s/keys :req [::parallelism]))
(defmethod ig/init-key ::scheduled-executor
[_ {:keys [::parallelism]}]
(px/scheduled-executor
:parallelism parallelism
:factory (px/thread-factory :name "penpot/scheduled-executor/%s")))
(defmethod ig/halt-key! ::scheduled-executor
[_ instance]
(px/shutdown! instance))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; TASKS REGISTRY
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@ -531,7 +517,7 @@
(s/def ::entries (s/coll-of (s/nilable ::cron-task)))
(defmethod ig/pre-init-spec ::cron [_]
(s/keys :req [::scheduled-executor ::db/pool ::entries ::registry]))
(s/keys :req [::db/pool ::entries ::registry]))
(defmethod ig/init-key ::cron
[_ {:keys [::entries ::registry ::db/pool] :as cfg}]
@ -622,16 +608,11 @@
next (dt/next-valid-instant-from cron now)]
(inst-ms (dt/diff now next))))
(def ^:private
xf-without-done
(remove #(.isDone ^Future %)))
(defn- schedule-cron-task
[{:keys [::scheduled-executor ::running] :as cfg} {:keys [cron] :as task}]
(let [ft (px/schedule! scheduled-executor
(ms-until-valid cron)
[{:keys [::running] :as cfg} {:keys [cron] :as task}]
(let [ft (px/schedule! (ms-until-valid cron)
(partial execute-cron-task cfg task))]
(swap! running #(into #{ft} xf-without-done %))))
(swap! running #(into #{ft} (filter p/pending?) %))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;