diff --git a/backend/src/app/loggers/audit.clj b/backend/src/app/loggers/audit.clj index d004fe5d4..55c3339fd 100644 --- a/backend/src/app/loggers/audit.clj +++ b/backend/src/app/loggers/audit.clj @@ -104,11 +104,18 @@ (s/def ::type ::us/string) (s/def ::props (s/map-of ::us/keyword any?)) (s/def ::ip-addr ::us/string) + (s/def ::webhooks/event? ::us/boolean) +(s/def ::webhooks/batch-timeout ::dt/duration) +(s/def ::webhooks/batch-key + (s/or :fn fn? :str string? :kw keyword?)) (s/def ::event (s/keys :req-un [::type ::name ::profile-id] - :opt-un [::ip-addr ::props ::webhooks/event?])) + :opt-un [::ip-addr ::props] + :opt [::webhooks/event? + ::webhooks/batch-timeout + ::webhooks/batch-key])) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; COLLECTOR @@ -153,13 +160,22 @@ (when (and (contains? cf/flags :webhooks) (::webhooks/event? event)) - (wrk/submit! ::wrk/conn pool - ::wrk/task :process-webhook-event - ::wrk/queue :webhooks - ::wrk/max-retries 0 - ::webhooks/event (-> params - (dissoc :ip-addr) - (dissoc :type)))))) + (let [batch-key (::webhooks/batch-key event) + batch-timeout (::webhooks/batch-timeout event)] + (wrk/submit! ::wrk/conn pool + ::wrk/task :process-webhook-event + ::wrk/queue :webhooks + ::wrk/max-retries 0 + ::wrk/delay (or batch-timeout 0) + ::wrk/label (cond + (fn? batch-key) (batch-key (:props event)) + (keyword? batch-key) (name batch-key) + (string? batch-key) batch-key + :else "default") + ::wrk/dedupe true + ::webhooks/event (-> params + (dissoc :ip-addr) + (dissoc :type))))))) (defn submit! "Submit audit event to the collector." diff --git a/backend/src/app/loggers/webhooks.clj b/backend/src/app/loggers/webhooks.clj index a5849c6da..b05b81558 100644 --- a/backend/src/app/loggers/webhooks.clj +++ b/backend/src/app/loggers/webhooks.clj @@ -11,6 +11,7 @@ [app.common.logging :as l] [app.common.transit :as t] [app.common.uri :as uri] + [app.config :as cf] [app.db :as db] [app.http.client :as http] [app.util.json :as json] @@ -56,6 +57,7 @@ [_ {:keys [::db/pool] :as cfg}] (fn [{:keys [props] :as task}] (let [event (::event props)] + (l/debug :hint "process webhook event" :name (:name event)) @@ -134,7 +136,8 @@ :webhook-mtype (:mtype whook)) (let [req {:uri (:uri whook) - :headers {"content-type" (:mtype whook)} + :headers {"content-type" (:mtype whook) + "user-agent" (str/ffmt "penpot/%" (:main cf/version))} :timeout (dt/duration "4s") :method :post :body body}] diff --git a/backend/src/app/rpc.clj b/backend/src/app/rpc.clj index df4fd4ea9..849f8370c 100644 --- a/backend/src/app/rpc.clj +++ b/backend/src/app/rpc.clj @@ -171,9 +171,18 @@ :profile-id profile-id :ip-addr (some-> request audit/parse-client-ip) :props props - ::webhooks/event? (or (::webhooks/event? mdata) - (::webhooks/event? resultm) - false)}] + ::webhooks/batch-key + (or (::webhooks/batch-key mdata) + (::webhooks/batch-key resultm)) + + ::webhooks/batch-timeout + (or (::webhooks/batch-timeout mdata) + (::webhooks/batch-timeout resultm)) + + ::webhooks/event? + (or (::webhooks/event? mdata) + (::webhooks/event? resultm) + false)}] (audit/submit! collector event))) diff --git a/backend/src/app/rpc/commands/files/update.clj b/backend/src/app/rpc/commands/files/update.clj index a590d63e8..520df2897 100644 --- a/backend/src/app/rpc/commands/files/update.clj +++ b/backend/src/app/rpc/commands/files/update.clj @@ -17,6 +17,7 @@ [app.config :as cf] [app.db :as db] [app.loggers.audit :as audit] + [app.loggers.webhooks :as-alias webhooks] [app.metrics :as mtx] [app.msgbus :as mbus] [app.rpc.climit :as-alias climit] @@ -122,12 +123,18 @@ ;; set is different than the persisted one, update it on the ;; database. +(defn webhook-batch-keyfn + [props] + (str "rpc:update-file:" (:id props))) + (sv/defmethod ::update-file {::climit/queue :update-file ::climit/key-fn :id + ::webhooks/event? true + ::webhooks/batch-timeout (dt/duration "2s") + ::webhooks/batch-key webhook-batch-keyfn ::doc/added "1.17"} [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}] - (db/with-atomic [conn pool] (files/check-edition-permissions! conn profile-id id) (db/xact-lock! conn id) @@ -173,8 +180,12 @@ {:id id}))) (-> (update-fn cfg params) - (vary-meta assoc ::audit/props {:project-id (:project-id file) - :team-id (:team-id file)})))))) + (vary-meta assoc ::audit/replace-props + {:id (:id file) + :name (:name file) + :features (:features file) + :project-id (:project-id file) + :team-id (:team-id file)})))))) (defn- update-file* [{:keys [conn] :as cfg} {:keys [file changes session-id profile-id] :as params}]