mirror of
https://github.com/penpot/penpot.git
synced 2025-03-16 01:31:22 -05:00
♻️ Enable receiving frontend audit log on backend.
This commit is contained in:
Andrey Antukh
parent
3dffb9c8a0
commit
e768600df3
8 changed files with 130 additions and 34 deletions
|
|
@ -300,7 +300,6 @@
|
|||
(.createArrayOf conn ^String type (into-array Object objects))
|
||||
(.createArrayOf conn ^String type objects))))
|
||||
|
||||
|
||||
(defn decode-pgpoint
|
||||
[^PGpoint v]
|
||||
(gpt/point (.-x v) (.-y v)))
|
||||
|
|
|
|||
|
|
@ -114,9 +114,14 @@
|
|||
(s/def ::storage map?)
|
||||
(s/def ::assets map?)
|
||||
(s/def ::feedback fn?)
|
||||
(s/def ::error-report-handler fn?)
|
||||
(s/def ::audit-http-handler fn?)
|
||||
|
||||
(defmethod ig/pre-init-spec ::router [_]
|
||||
(s/keys :req-un [::rpc ::session ::mtx/metrics ::oauth ::storage ::assets ::feedback]))
|
||||
(s/keys :req-un [::rpc ::session ::mtx/metrics
|
||||
::oauth ::storage ::assets ::feedback
|
||||
::error-report-handler
|
||||
::audit-http-handler]))
|
||||
|
||||
(defmethod ig/init-key ::router
|
||||
[_ {:keys [session rpc oauth metrics assets feedback] :as cfg}]
|
||||
|
|
@ -136,10 +141,7 @@
|
|||
["/webhooks"
|
||||
["/sns" {:post (:sns-webhook cfg)}]]
|
||||
|
||||
["/api" {:middleware [
|
||||
;; Temporary disabled
|
||||
#_[middleware/etag]
|
||||
[middleware/format-response-body]
|
||||
["/api" {:middleware [[middleware/format-response-body]
|
||||
[middleware/params]
|
||||
[middleware/multipart-params]
|
||||
[middleware/keyword-params]
|
||||
|
|
@ -149,10 +151,12 @@
|
|||
|
||||
["/feedback" {:middleware [(:middleware session)]
|
||||
:post feedback}]
|
||||
|
||||
["/auth/oauth/:provider" {:post (:handler oauth)}]
|
||||
["/auth/oauth/:provider/callback" {:get (:callback-handler oauth)}]
|
||||
|
||||
["/audit/events" {:middleware [(:middleware session)]
|
||||
:post (:audit-http-handler cfg)}]
|
||||
|
||||
["/rpc" {:middleware [(:middleware session)]}
|
||||
["/query/:type" {:get (:query-handler rpc)
|
||||
:post (:query-handler rpc)}]
|
||||
|
|
|
|||
|
|
@ -23,7 +23,8 @@
|
|||
[clojure.spec.alpha :as s]
|
||||
[cuerdas.core :as str]
|
||||
[integrant.core :as ig]
|
||||
[lambdaisland.uri :as u]))
|
||||
[lambdaisland.uri :as u]
|
||||
[promesa.exec :as px]))
|
||||
|
||||
(defn parse-client-ip
|
||||
[{:keys [headers] :as request}]
|
||||
|
|
@ -67,6 +68,65 @@
|
|||
|
||||
(update event :props #(-> % clean-common clean-profile-id clean-complex-data))))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; HTTP Handler
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(declare persist-http-events)
|
||||
|
||||
(s/def ::profile-id ::us/uuid)
|
||||
(s/def ::name ::us/string)
|
||||
(s/def ::type ::us/string)
|
||||
(s/def ::props (s/map-of ::us/keyword any?))
|
||||
(s/def ::timestamp dt/instant?)
|
||||
(s/def ::context (s/map-of ::us/keyword any?))
|
||||
|
||||
(s/def ::event
|
||||
(s/keys :req-un [::type ::name ::props ::timestamp ::profile-id]
|
||||
:opt-un [::context]))
|
||||
|
||||
(s/def ::events (s/every ::event))
|
||||
|
||||
(defmethod ig/init-key ::http-handler
|
||||
[_ {:keys [executor enabled] :as cfg}]
|
||||
(fn [{:keys [params headers cookies profile-id] :as request}]
|
||||
(when enabled
|
||||
(let [events (->> (:events params)
|
||||
(remove #(not= profile-id (:profile-id %)))
|
||||
(us/conform ::events))
|
||||
ip-addr (parse-client-ip request)
|
||||
cfg (-> cfg
|
||||
(assoc :source "frontend")
|
||||
(assoc :events events)
|
||||
(assoc :ip-addr ip-addr))]
|
||||
(px/run! executor #(persist-http-events cfg))))
|
||||
{:status 204 :body ""}))
|
||||
|
||||
(defn- persist-http-events
|
||||
[{:keys [pool events ip-addr source] :as cfg}]
|
||||
(try
|
||||
(let [columns [:id :name :source :type :tracked-at :profile-id :ip-addr :props :context]
|
||||
prepare-xf (map (fn [event]
|
||||
[(uuid/next)
|
||||
(:name event)
|
||||
source
|
||||
(:type event)
|
||||
(:timestamp event)
|
||||
(:profile-id event)
|
||||
(db/inet ip-addr)
|
||||
(db/tjson (:props event))
|
||||
(db/tjson (d/without-nils (:context event)))]))
|
||||
events (us/conform ::events events)
|
||||
rows (into [] prepare-xf events)]
|
||||
(db/insert-multi! pool :audit-log columns rows))
|
||||
(catch Throwable e
|
||||
(let [xdata (ex-data e)]
|
||||
(if (= :spec-validation (:code xdata))
|
||||
(l/error ::l/raw (str "spec validation on persist-events:\n"
|
||||
(:explain xdata)))
|
||||
(l/error :hint "error on persist-events"
|
||||
:cause e))))))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; Collector
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
|
@ -103,7 +163,9 @@
|
|||
(recur)))
|
||||
|
||||
(fn [& {:keys [cmd] :as params}]
|
||||
(let [params (dissoc params :cmd)]
|
||||
(let [params (-> params
|
||||
(dissoc :cmd)
|
||||
(assoc :tracked-at (dt/now)))]
|
||||
(case cmd
|
||||
:stop (a/close! input)
|
||||
:submit (when-not (a/offer! input params)
|
||||
|
|
@ -117,13 +179,14 @@
|
|||
(:name event)
|
||||
(:type event)
|
||||
(:profile-id event)
|
||||
(:tracked-at event)
|
||||
(some-> (:ip-addr event) db/inet)
|
||||
(db/tjson (:props event))])]
|
||||
|
||||
(db/tjson (:props event))
|
||||
"backend"])]
|
||||
(aa/with-thread executor
|
||||
(db/with-atomic [conn pool]
|
||||
(db/insert-multi! conn :audit-log
|
||||
[:id :name :type :profile-id :ip-addr :props]
|
||||
[:id :name :type :profile-id :tracked-at :ip-addr :props :source]
|
||||
(sequence (map event->row) events))))))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
|
@ -144,16 +207,19 @@
|
|||
|
||||
(defmethod ig/init-key ::archive-task
|
||||
[_ {:keys [uri enabled] :as cfg}]
|
||||
(fn [_]
|
||||
(when (and enabled (not uri))
|
||||
(ex/raise :type :internal
|
||||
:code :task-not-configured
|
||||
:hint "archive task not configured, missing uri"))
|
||||
(loop []
|
||||
(let [res (archive-events cfg)]
|
||||
(when (= res :continue)
|
||||
(aa/thread-sleep 200)
|
||||
(recur))))))
|
||||
(fn [props]
|
||||
(let [enabled (or enabled (:enabled props false))
|
||||
uri (or uri (:uri props))]
|
||||
(when (and enabled (not uri))
|
||||
(ex/raise :type :internal
|
||||
:code :task-not-configured
|
||||
:hint "archive task not configured, missing uri"))
|
||||
(when enabled
|
||||
(loop []
|
||||
(let [res (archive-events cfg)]
|
||||
(when (= res :continue)
|
||||
(aa/thread-sleep 200)
|
||||
(recur))))))))
|
||||
|
||||
(def sql:retrieve-batch-of-audit-log
|
||||
"select * from audit_log
|
||||
|
|
@ -164,22 +230,27 @@
|
|||
|
||||
(defn archive-events
|
||||
[{:keys [pool uri tokens] :as cfg}]
|
||||
(letfn [(decode-row [{:keys [props ip-addr] :as row}]
|
||||
(letfn [(decode-row [{:keys [props ip-addr context] :as row}]
|
||||
(cond-> row
|
||||
(db/pgobject? props)
|
||||
(assoc :props (db/decode-transit-pgobject props))
|
||||
|
||||
(db/pgobject? context)
|
||||
(assoc :context (db/decode-transit-pgobject context))
|
||||
|
||||
(db/pgobject? ip-addr "inet")
|
||||
(assoc :ip-addr (db/decode-inet ip-addr))))
|
||||
|
||||
(row->event [{:keys [name type created-at profile-id props ip-addr]}]
|
||||
(cond-> {:type type
|
||||
:name name
|
||||
:timestamp created-at
|
||||
:profile-id profile-id
|
||||
:props props}
|
||||
(some? ip-addr)
|
||||
(update :context assoc :source-ip ip-addr)))
|
||||
(row->event [row]
|
||||
(select-keys row [:type
|
||||
:name
|
||||
:source
|
||||
:created-at
|
||||
:tracked-at
|
||||
:profile-id
|
||||
:ip-addr
|
||||
:props
|
||||
:context]))
|
||||
|
||||
(send [events]
|
||||
(let [token (tokens :generate {:iss "authentication"
|
||||
|
|
|
|||
|
|
@ -95,6 +95,7 @@
|
|||
:storage (ig/ref :app.storage/storage)
|
||||
:sns-webhook (ig/ref :app.http.awsns/handler)
|
||||
:feedback (ig/ref :app.http.feedback/handler)
|
||||
:audit-http-handler (ig/ref :app.loggers.audit/http-handler)
|
||||
:error-report-handler (ig/ref :app.loggers.mattermost/handler)}
|
||||
|
||||
:app.http.assets/handlers
|
||||
|
|
@ -289,6 +290,11 @@
|
|||
:app.loggers.zmq/receiver
|
||||
{:endpoint (cf/get :loggers-zmq-uri)}
|
||||
|
||||
:app.loggers.audit/http-handler
|
||||
{:enabled (cf/get :audit-enabled false)
|
||||
:pool (ig/ref :app.db/pool)
|
||||
:executor (ig/ref :app.worker/executor)}
|
||||
|
||||
:app.loggers.audit/collector
|
||||
{:enabled (cf/get :audit-enabled false)
|
||||
:pool (ig/ref :app.db/pool)
|
||||
|
|
|
|||
|
|
@ -199,6 +199,9 @@
|
|||
|
||||
{:name "0063-add-share-link-table"
|
||||
:fn (mg/resource "app/migrations/sql/0063-add-share-link-table.sql")}
|
||||
|
||||
{:name "0064-mod-audit-log-table"
|
||||
:fn (mg/resource "app/migrations/sql/0064-mod-audit-log-table.sql")}
|
||||
])
|
||||
|
||||
|
||||
|
|
|
|||
13
backend/src/app/migrations/sql/0064-mod-audit-log-table.sql
Normal file
13
backend/src/app/migrations/sql/0064-mod-audit-log-table.sql
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
ALTER TABLE audit_log
|
||||
ADD COLUMN tracked_at timestamptz NULL DEFAULT clock_timestamp(),
|
||||
ADD COLUMN source text NULL,
|
||||
ADD COLUMN context jsonb NULL;
|
||||
|
||||
ALTER TABLE audit_log
|
||||
ALTER COLUMN source SET STORAGE external,
|
||||
ALTER COLUMN context SET STORAGE external;
|
||||
|
||||
UPDATE audit_log SET source = 'backend', tracked_at=created_at;
|
||||
|
||||
-- ALTER TABLE audit_log ALTER COLUMN source SET NOT NULL;
|
||||
-- ALTER TABLE audit_log ALTER COLUMN tracked_at SET NOT NULL;
|
||||
|
|
@ -117,14 +117,14 @@
|
|||
profile-id (or (:profile-id params')
|
||||
(:profile-id result)
|
||||
(::audit/profile-id resultm))
|
||||
props (d/merge params (::audit/props resultm))]
|
||||
props (d/merge params' (::audit/props resultm))]
|
||||
(audit :cmd :submit
|
||||
:type (::type cfg)
|
||||
:name (or (::audit/name resultm)
|
||||
(::sv/name mdata))
|
||||
:profile-id profile-id
|
||||
:ip-addr (audit/parse-client-ip request)
|
||||
:props (audit/profile->props props))))
|
||||
:props props)))
|
||||
|
||||
result))))
|
||||
|
||||
|
|
|
|||
|
|
@ -164,7 +164,7 @@
|
|||
(defn- persist-events
|
||||
[events]
|
||||
(if (seq events)
|
||||
(let [uri (u/join cf/public-uri "events")
|
||||
(let [uri (u/join cf/public-uri "api/audit/events")
|
||||
params {:events events}]
|
||||
(->> (http/send! {:uri uri
|
||||
:method :post
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue