🎉 Add update-file (batched) to webhooks

backend/src/app/loggers/audit.clj

@@ -104,11 +104,18 @@
 (s/def ::type ::us/string)
 (s/def ::props (s/map-of ::us/keyword any?))
 (s/def ::ip-addr ::us/string)
+
 (s/def ::webhooks/event? ::us/boolean)
+(s/def ::webhooks/batch-timeout ::dt/duration)
+(s/def ::webhooks/batch-key
+  (s/or :fn fn? :str string? :kw keyword?))
 
 (s/def ::event
   (s/keys :req-un [::type ::name ::profile-id]
-          :opt-un [::ip-addr ::props ::webhooks/event?]))
+          :opt-un [::ip-addr ::props]
+          :opt [::webhooks/event?
+                ::webhooks/batch-timeout
+                ::webhooks/batch-key]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; COLLECTOR
@@ -153,13 +160,22 @@
 
     (when (and (contains? cf/flags :webhooks)
                (::webhooks/event? event))
-      (wrk/submit! ::wrk/conn pool
-                   ::wrk/task :process-webhook-event
-                   ::wrk/queue :webhooks
-                   ::wrk/max-retries 0
-                   ::webhooks/event (-> params
-                                        (dissoc :ip-addr)
-                                        (dissoc :type))))))
+      (let [batch-key     (::webhooks/batch-key event)
+            batch-timeout (::webhooks/batch-timeout event)]
+        (wrk/submit! ::wrk/conn pool
+                     ::wrk/task :process-webhook-event
+                     ::wrk/queue :webhooks
+                     ::wrk/max-retries 0
+                     ::wrk/delay (or batch-timeout 0)
+                     ::wrk/label (cond
+                                   (fn? batch-key)        (batch-key (:props event))
+                                   (keyword? batch-key)   (name batch-key)
+                                   (string? batch-key)    batch-key
+                                   :else                  "default")
+                     ::wrk/dedupe true
+                     ::webhooks/event (-> params
+                                          (dissoc :ip-addr)
+                                          (dissoc :type)))))))
 
 (defn submit!
   "Submit audit event to the collector."

backend/src/app/loggers/webhooks.clj

@@ -11,6 +11,7 @@
    [app.common.logging :as l]
    [app.common.transit :as t]
    [app.common.uri :as uri]
+   [app.config :as cf]
    [app.db :as db]
    [app.http.client :as http]
    [app.util.json :as json]
@@ -56,6 +57,7 @@
   [_ {:keys [::db/pool] :as cfg}]
   (fn [{:keys [props] :as task}]
     (let [event (::event props)]
+
       (l/debug :hint "process webhook event"
                :name (:name event))
 
@@ -134,7 +136,8 @@
                  :webhook-mtype (:mtype whook))
 
         (let [req {:uri (:uri whook)
-                   :headers {"content-type" (:mtype whook)}
+                   :headers {"content-type" (:mtype whook)
+                             "user-agent" (str/ffmt "penpot/%" (:main cf/version))}
                    :timeout (dt/duration "4s")
                    :method :post
                    :body body}]

backend/src/app/rpc.clj

@@ -171,9 +171,18 @@
                                 :profile-id profile-id
                                 :ip-addr (some-> request audit/parse-client-ip)
                                 :props props
-                                ::webhooks/event? (or (::webhooks/event? mdata)
-                                                      (::webhooks/event? resultm)
-                                                      false)}]
+                                ::webhooks/batch-key
+                                (or (::webhooks/batch-key mdata)
+                                    (::webhooks/batch-key resultm))
+
+                                ::webhooks/batch-timeout
+                                (or (::webhooks/batch-timeout mdata)
+                                    (::webhooks/batch-timeout resultm))
+
+                                ::webhooks/event?
+                                (or (::webhooks/event? mdata)
+                                    (::webhooks/event? resultm)
+                                    false)}]
 
                 (audit/submit! collector event)))
 

backend/src/app/rpc/commands/files/update.clj

@@ -17,6 +17,7 @@
    [app.config :as cf]
    [app.db :as db]
    [app.loggers.audit :as audit]
+   [app.loggers.webhooks :as-alias webhooks]
    [app.metrics :as mtx]
    [app.msgbus :as mbus]
    [app.rpc.climit :as-alias climit]
@@ -122,12 +123,18 @@
 ;; set is different than the persisted one, update it on the
 ;; database.
 
+(defn webhook-batch-keyfn
+  [props]
+  (str "rpc:update-file:" (:id props)))
+
 (sv/defmethod ::update-file
   {::climit/queue :update-file
    ::climit/key-fn :id
+   ::webhooks/event? true
+   ::webhooks/batch-timeout (dt/duration "2s")
+   ::webhooks/batch-key webhook-batch-keyfn
    ::doc/added "1.17"}
   [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}]
-
   (db/with-atomic [conn pool]
     (files/check-edition-permissions! conn profile-id id)
     (db/xact-lock! conn id)
@@ -173,8 +180,12 @@
                         {:id id})))
 
         (-> (update-fn cfg params)
-            (vary-meta assoc ::audit/props {:project-id (:project-id file)
-                                            :team-id    (:team-id file)}))))))
+            (vary-meta assoc ::audit/replace-props
+                       {:id (:id file)
+                        :name (:name file)
+                        :features (:features file)
+                        :project-id (:project-id file)
+                        :team-id    (:team-id file)}))))))
 
 (defn- update-file*
   [{:keys [conn] :as cfg} {:keys [file changes session-id profile-id] :as params}]