Penpot/penpot
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-02-12 18:18:24 -05:00
Code Issues Activity

✨ Add proper climit configuration for file-thumbnails

Browse source
This commit is contained in:
Andrey Antukh 2023-11-08 15:15:09 +01:00
parent 3b463f334c
commit da68eae7c6
3 changed files with 51 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
backend/resources/climit.edn
View file

@ -10,5 +10,8 @@
:process-font {:permits 4 :queue 32} :process-font {:permits 4 :queue 32}
:process-image {:permits 8 :queue 32} :process-image {:permits 8 :queue 32}
:file-thumbnail-ops
{:permits 2}
:submit-audit-events-by-profile :submit-audit-events-by-profile
{:permits 1 :queue 3}} {:permits 1 :queue 3}}

82
backend/src/app/rpc/climit.clj
View file

@ -35,7 +35,7 @@
[{:keys [::wrk/executor]} config] [{:keys [::wrk/executor]} config]
(letfn [(load-fn [key] (letfn [(load-fn [key]
(let [config (get config (nth key 0))] (let [config (get config (nth key 0))]
(l/trace :hint "insert into cache" :key key) (l/trc :hint "insert into cache" :key key)
(pbh/create :permits (or (:permits config) (:concurrency config)) (pbh/create :permits (or (:permits config) (:concurrency config))
:queue (or (:queue config) (:queue-size config)) :queue (or (:queue config) (:queue-size config))
:timeout (:timeout config) :timeout (:timeout config)
@ -43,7 +43,7 @@
:type (:type config :semaphore)))) :type (:type config :semaphore))))
(on-remove [_ _ cause] (on-remove [_ _ cause]
(l/trace :hint "evict from cache" :key key :reason (str cause)))] (l/trc :hint "evict from cache" :key key :reason (str cause)))]
(cache/create :executor :same-thread (cache/create :executor :same-thread
:on-remove on-remove :on-remove on-remove
@ -71,7 +71,7 @@
[_ {:keys [::path ::mtx/metrics ::wrk/executor] :as cfg}] [_ {:keys [::path ::mtx/metrics ::wrk/executor] :as cfg}]
(when (contains? cf/flags :rpc-climit) (when (contains? cf/flags :rpc-climit)
(when-let [params (some->> path slurp edn/read-string)] (when-let [params (some->> path slurp edn/read-string)]
(l/info :hint "initializing concurrency limit" :config (str path)) (l/inf :hint "initializing concurrency limit" :config (str path))
(us/verify! ::config params) (us/verify! ::config params)
{::cache (create-bulkhead-cache cfg params) {::cache (create-bulkhead-cache cfg params)
::config params ::config params
@ -99,15 +99,15 @@
(fn [] (fn []
(let [elapsed (tpoint) (let [elapsed (tpoint)
stats (pbh/get-stats limiter)] stats (pbh/get-stats limiter)]
(l/trace :hint "executed" (l/trc :hint "executed"
:id (name id) :id (name id)
:key key :key key
:fnh (hash f) :fnh (hash f)
:permits (:permits stats) :permits (:permits stats)
:queue (:queue stats) :queue (:queue stats)
:max-permits (:max-permits stats) :max-permits (:max-permits stats)
:max-queue (:max-queue stats) :max-queue (:max-queue stats)
:elapsed (dt/format-duration elapsed)) :elapsed (dt/format-duration elapsed))
(mtx/run! metrics (mtx/run! metrics
:id :rpc-climit-timing :id :rpc-climit-timing
:val (inst-ms elapsed) :val (inst-ms elapsed)
@ -116,15 +116,15 @@
(f) (f)
(finally (finally
(let [elapsed (tpoint)] (let [elapsed (tpoint)]
(l/trace :hint "finished" (l/trc :hint "finished"
:id (name id) :id (name id)
:key key :key key
:fnh (hash f) :fnh (hash f)
:permits (:permits stats) :permits (:permits stats)
:queue (:queue stats) :queue (:queue stats)
:max-permits (:max-permits stats) :max-permits (:max-permits stats)
:max-queue (:max-queue stats) :max-queue (:max-queue stats)
:elapsed (dt/format-duration elapsed))))))) :elapsed (dt/format-duration elapsed)))))))
measure! measure!
(fn [stats] (fn [stats]
(mtx/run! metrics (mtx/run! metrics
@ -139,14 +139,14 @@
(try (try
(let [stats (pbh/get-stats limiter)] (let [stats (pbh/get-stats limiter)]
(measure! stats) (measure! stats)
(l/trace :hint "enqueued" (l/trc :hint "enqueued"
:id (name id) :id (name id)
:key key :key key
:fnh (hash f) :fnh (hash f)
:permits (:permits stats) :permits (:permits stats)
:queue (:queue stats) :queue (:queue stats)
:max-permits (:max-permits stats) :max-permits (:max-permits stats)
:max-queue (:max-queue stats)) :max-queue (:max-queue stats))
(pbh/invoke! limiter wrapped)) (pbh/invoke! limiter wrapped))
(catch ExceptionInfo cause (catch ExceptionInfo cause
(let [{:keys [type code]} (ex-data cause)] (let [{:keys [type code]} (ex-data cause)]
@ -193,13 +193,6 @@
(app.rpc.climit/run! (^:once fn* [] ~@body))) (app.rpc.climit/run! (^:once fn* [] ~@body)))
`(run! ~instance (^:once fn* [] ~@body)))) `(run! ~instance (^:once fn* [] ~@body))))
(defmacro with-dispatch
"Dispatch blocking operation to a separated thread protected with
the specified semaphore.
DEPRECATED"
[& params]
`(with-dispatch! ~@params))
(def noop-fn (constantly nil)) (def noop-fn (constantly nil))
(defn wrap (defn wrap
@ -207,18 +200,19 @@
(if (and (some? climit) (some? id)) (if (and (some? climit) (some? id))
(if-let [config (get-in climit [::config id])] (if-let [config (get-in climit [::config id])]
(let [cache (::cache climit)] (let [cache (::cache climit)]
(l/debug :hint "wrap: instrumenting method" (l/dbg :hint "instrumenting method"
:limit (name id) :limit (name id)
:service-name (::sv/name mdata) :service-name (::sv/name mdata)
:timeout (:timeout config) :timeout (:timeout config)
:permits (:permits config) :permits (:permits config)
:queue (:queue config) :queue (:queue config)
:keyed? (some? key-fn)) :keyed? (not= key-fn noop-fn))
(fn [cfg params] (fn [cfg params]
(invoke! cache metrics id (key-fn params) (partial f cfg params)))) (invoke! cache metrics id (key-fn params) (partial f cfg params))))
(do (do
(l/warn :hint "no config found for specified queue" :id id) (l/wrn :hint "no config found for specified queue" :id id)
f)) f))
f)) f))

14
backend/src/app/rpc/commands/files_thumbnails.clj
View file

@ -20,6 +20,7 @@
[app.loggers.webhooks :as-alias webhooks] [app.loggers.webhooks :as-alias webhooks]
[app.media :as media] [app.media :as media]
[app.rpc :as-alias rpc] [app.rpc :as-alias rpc]
[app.rpc.climit :as-alias climit]
[app.rpc.commands.files :as files] [app.rpc.commands.files :as files]
[app.rpc.commands.teams :as teams] [app.rpc.commands.teams :as teams]
[app.rpc.cond :as-alias cond] [app.rpc.cond :as-alias cond]
@ -262,8 +263,10 @@
[:tag {:optional true} :string]]) [:tag {:optional true} :string]])
(sv/defmethod ::create-file-object-thumbnail (sv/defmethod ::create-file-object-thumbnail
{:doc/added "1.19" {::doc/added "1.19"
::doc/module :files ::doc/module :files
::climit/id :file-thumbnail-ops
::climit/key-fn ::rpc/profile-id
::audit/skip true ::audit/skip true
::sm/params schema:create-file-object-thumbnail} ::sm/params schema:create-file-object-thumbnail}
@ -288,9 +291,8 @@
{:file-id file-id {:file-id file-id
:object-id object-id} :object-id object-id}
{::db/for-update? true})] {::db/for-update? true})]
(when media-id
(sto/del-object! storage media-id))
(sto/del-object! storage media-id)
(db/delete! conn :file-tagged-object-thumbnail (db/delete! conn :file-tagged-object-thumbnail
{:file-id file-id {:file-id file-id
:object-id object-id}) :object-id object-id})
@ -301,8 +303,10 @@
:req-un [::file-id ::object-id])) :req-un [::file-id ::object-id]))
(sv/defmethod ::delete-file-object-thumbnail (sv/defmethod ::delete-file-object-thumbnail
{:doc/added "1.19" {::doc/added "1.19"
::doc/module :files ::doc/module :files
::climit/id :file-thumbnail-ops
::climit/key-fn ::rpc/profile-id
::audit/skip true} ::audit/skip true}
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id object-id]}] [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id object-id]}]
@ -351,6 +355,8 @@
{::doc/added "1.19" {::doc/added "1.19"
::doc/module :files ::doc/module :files
::audit/skip true ::audit/skip true
::climit/id :file-thumbnail-ops
::climit/key-fn ::rpc/profile-id
::sm/params [:map {:title "create-file-thumbnail"} ::sm/params [:map {:title "create-file-thumbnail"}
[:file-id ::sm/uuid] [:file-id ::sm/uuid]
[:revn :int] [:revn :int]