♻️ Enable receiving frontend audit log on backend.

backend/src/app/db.clj

@@ -300,7 +300,6 @@
       (.createArrayOf conn ^String type (into-array Object objects))
       (.createArrayOf conn ^String type objects))))
 
-
 (defn decode-pgpoint
   [^PGpoint v]
   (gpt/point (.-x v) (.-y v)))

backend/src/app/http.clj

@@ -114,9 +114,14 @@
 (s/def ::storage map?)
 (s/def ::assets map?)
 (s/def ::feedback fn?)
+(s/def ::error-report-handler fn?)
+(s/def ::audit-http-handler fn?)
 
 (defmethod ig/pre-init-spec ::router [_]
-  (s/keys :req-un [::rpc ::session ::mtx/metrics ::oauth ::storage ::assets ::feedback]))
+  (s/keys :req-un [::rpc ::session ::mtx/metrics
+                   ::oauth ::storage ::assets ::feedback
+                   ::error-report-handler
+                   ::audit-http-handler]))
 
 (defmethod ig/init-key ::router
   [_ {:keys [session rpc oauth metrics assets feedback] :as cfg}]
@@ -136,10 +141,7 @@
     ["/webhooks"
      ["/sns" {:post (:sns-webhook cfg)}]]
 
-    ["/api" {:middleware [
+    ["/api" {:middleware [[middleware/format-response-body]
-                          ;; Temporary disabled
-                          #_[middleware/etag]
-                          [middleware/format-response-body]
                           [middleware/params]
                           [middleware/multipart-params]
                           [middleware/keyword-params]
@@ -149,10 +151,12 @@
 
      ["/feedback" {:middleware [(:middleware session)]
                    :post feedback}]
-
      ["/auth/oauth/:provider" {:post (:handler oauth)}]
      ["/auth/oauth/:provider/callback" {:get (:callback-handler oauth)}]
 
+     ["/audit/events" {:middleware [(:middleware session)]
+                       :post (:audit-http-handler cfg)}]
+
      ["/rpc" {:middleware [(:middleware session)]}
       ["/query/:type" {:get (:query-handler rpc)
                        :post (:query-handler rpc)}]

backend/src/app/loggers/audit.clj

@@ -23,7 +23,8 @@
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
    [integrant.core :as ig]
-   [lambdaisland.uri :as u]))
+   [lambdaisland.uri :as u]
+   [promesa.exec :as px]))
 
 (defn parse-client-ip
   [{:keys [headers] :as request}]
@@ -67,6 +68,65 @@
 
     (update event :props #(-> % clean-common clean-profile-id clean-complex-data))))
 
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; HTTP Handler
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(declare persist-http-events)
+
+(s/def ::profile-id ::us/uuid)
+(s/def ::name ::us/string)
+(s/def ::type ::us/string)
+(s/def ::props (s/map-of ::us/keyword any?))
+(s/def ::timestamp dt/instant?)
+(s/def ::context (s/map-of ::us/keyword any?))
+
+(s/def ::event
+  (s/keys :req-un [::type ::name ::props ::timestamp ::profile-id]
+          :opt-un [::context]))
+
+(s/def ::events (s/every ::event))
+
+(defmethod ig/init-key ::http-handler
+  [_  {:keys [executor enabled] :as cfg}]
+  (fn [{:keys [params headers cookies profile-id] :as request}]
+    (when enabled
+      (let [events  (->> (:events params)
+                         (remove #(not= profile-id (:profile-id %)))
+                         (us/conform ::events))
+            ip-addr (parse-client-ip request)
+            cfg     (-> cfg
+                        (assoc :source "frontend")
+                        (assoc :events events)
+                        (assoc :ip-addr ip-addr))]
+        (px/run! executor #(persist-http-events cfg))))
+    {:status 204 :body ""}))
+
+(defn- persist-http-events
+  [{:keys [pool events ip-addr source] :as cfg}]
+  (try
+    (let [columns    [:id :name :source :type :tracked-at :profile-id :ip-addr :props :context]
+          prepare-xf (map (fn [event]
+                            [(uuid/next)
+                             (:name event)
+                             source
+                             (:type event)
+                             (:timestamp event)
+                             (:profile-id event)
+                             (db/inet ip-addr)
+                             (db/tjson (:props event))
+                             (db/tjson (d/without-nils (:context event)))]))
+          events     (us/conform ::events events)
+          rows       (into [] prepare-xf events)]
+      (db/insert-multi! pool :audit-log columns rows))
+    (catch Throwable e
+      (let [xdata (ex-data e)]
+        (if (= :spec-validation (:code xdata))
+          (l/error ::l/raw (str "spec validation on persist-events:\n"
+                                (:explain xdata)))
+          (l/error :hint "error on persist-events"
+                   :cause e))))))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Collector
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -103,7 +163,9 @@
           (recur)))
 
       (fn [& {:keys [cmd] :as params}]
-        (let [params (dissoc params :cmd)]
+        (let [params (-> params
+                         (dissoc :cmd)
+                         (assoc :tracked-at (dt/now)))]
           (case cmd
             :stop   (a/close! input)
             :submit (when-not (a/offer! input params)
@@ -117,13 +179,14 @@
              (:name event)
              (:type event)
              (:profile-id event)
+             (:tracked-at event)
              (some-> (:ip-addr event) db/inet)
-             (db/tjson (:props event))])]
+             (db/tjson (:props event))
-
+             "backend"])]
     (aa/with-thread executor
       (db/with-atomic [conn pool]
         (db/insert-multi! conn :audit-log
-                          [:id :name :type :profile-id :ip-addr :props]
+                          [:id :name :type :profile-id :tracked-at :ip-addr :props :source]
                           (sequence (map event->row) events))))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -144,16 +207,19 @@
 
 (defmethod ig/init-key ::archive-task
   [_ {:keys [uri enabled] :as cfg}]
-  (fn [_]
+  (fn [props]
-    (when (and enabled (not uri))
+    (let [enabled (or enabled (:enabled props false))
-      (ex/raise :type :internal
+          uri     (or uri (:uri props))]
-                :code :task-not-configured
+      (when (and enabled (not uri))
-                :hint "archive task not configured, missing uri"))
+        (ex/raise :type :internal
-    (loop []
+                  :code :task-not-configured
-      (let [res (archive-events cfg)]
+                  :hint "archive task not configured, missing uri"))
-        (when (= res :continue)
+      (when enabled
-          (aa/thread-sleep 200)
+        (loop []
-          (recur))))))
+          (let [res (archive-events cfg)]
+            (when (= res :continue)
+              (aa/thread-sleep 200)
+              (recur))))))))
 
 (def sql:retrieve-batch-of-audit-log
   "select * from audit_log
@@ -164,22 +230,27 @@
 
 (defn archive-events
   [{:keys [pool uri tokens] :as cfg}]
-  (letfn [(decode-row [{:keys [props ip-addr] :as row}]
+  (letfn [(decode-row [{:keys [props ip-addr context] :as row}]
             (cond-> row
               (db/pgobject? props)
               (assoc :props (db/decode-transit-pgobject props))
 
+              (db/pgobject? context)
+              (assoc :context (db/decode-transit-pgobject context))
+
               (db/pgobject? ip-addr "inet")
               (assoc :ip-addr (db/decode-inet ip-addr))))
 
-          (row->event [{:keys [name type created-at profile-id props ip-addr]}]
+          (row->event [row]
-            (cond-> {:type type
+            (select-keys row [:type
-                     :name name
+                              :name
-                     :timestamp created-at
+                              :source
-                     :profile-id profile-id
+                              :created-at
-                     :props props}
+                              :tracked-at
-              (some? ip-addr)
+                              :profile-id
-              (update :context assoc :source-ip ip-addr)))
+                              :ip-addr
+                              :props
+                              :context]))
 
           (send [events]
             (let [token   (tokens :generate {:iss "authentication"

backend/src/app/main.clj

@@ -95,6 +95,7 @@
     :storage     (ig/ref :app.storage/storage)
     :sns-webhook (ig/ref :app.http.awsns/handler)
     :feedback    (ig/ref :app.http.feedback/handler)
+    :audit-http-handler   (ig/ref :app.loggers.audit/http-handler)
     :error-report-handler (ig/ref :app.loggers.mattermost/handler)}
 
    :app.http.assets/handlers
@@ -289,6 +290,11 @@
    :app.loggers.zmq/receiver
    {:endpoint (cf/get :loggers-zmq-uri)}
 
+   :app.loggers.audit/http-handler
+   {:enabled  (cf/get :audit-enabled false)
+    :pool     (ig/ref :app.db/pool)
+    :executor (ig/ref :app.worker/executor)}
+
    :app.loggers.audit/collector
    {:enabled  (cf/get :audit-enabled false)
     :pool     (ig/ref :app.db/pool)

backend/src/app/migrations.clj

@@ -199,6 +199,9 @@
 
    {:name "0063-add-share-link-table"
     :fn (mg/resource "app/migrations/sql/0063-add-share-link-table.sql")}
+
+   {:name "0064-mod-audit-log-table"
+    :fn (mg/resource "app/migrations/sql/0064-mod-audit-log-table.sql")}
    ])
 
 

backend/src/app/migrations/sql/0064-mod-audit-log-table.sql

@@ -0,0 +1,13 @@
+ALTER TABLE audit_log
+  ADD COLUMN tracked_at timestamptz NULL DEFAULT clock_timestamp(),
+  ADD COLUMN source text NULL,
+  ADD COLUMN context jsonb NULL;
+
+ALTER TABLE audit_log
+  ALTER COLUMN source SET STORAGE external,
+  ALTER COLUMN context SET STORAGE external;
+
+UPDATE audit_log SET source = 'backend', tracked_at=created_at;
+
+-- ALTER TABLE audit_log ALTER COLUMN source SET NOT NULL;
+-- ALTER TABLE audit_log ALTER COLUMN tracked_at SET NOT NULL;

backend/src/app/rpc.clj

@@ -117,14 +117,14 @@
                 profile-id (or (:profile-id params')
                                (:profile-id result)
                                (::audit/profile-id resultm))
-                props      (d/merge params (::audit/props resultm))]
+                props      (d/merge params' (::audit/props resultm))]
             (audit :cmd :submit
                    :type (::type cfg)
                    :name (or (::audit/name resultm)
                              (::sv/name mdata))
                    :profile-id profile-id
                    :ip-addr (audit/parse-client-ip request)
-                   :props (audit/profile->props props))))
+                   :props props)))
 
         result))))
 

frontend/src/app/main/data/events.cljs

@@ -164,7 +164,7 @@
 (defn- persist-events
   [events]
   (if (seq events)
-    (let [uri    (u/join cf/public-uri "events")
+    (let [uri    (u/join cf/public-uri "api/audit/events")
           params {:events events}]
       (->> (http/send! {:uri uri
                         :method :post