From fd24831c7122b97f57c40f18546ca475d24a9610 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 25 Mar 2024 10:46:15 +0100 Subject: [PATCH 1/4] :sparkles: Move audit tasks to separated namespace files --- backend/src/app/loggers/audit.clj | 141 ------------------ .../src/app/loggers/audit/archive_task.clj | 140 +++++++++++++++++ backend/src/app/loggers/audit/gc_task.clj | 31 ++++ backend/src/app/main.clj | 9 +- 4 files changed, 175 insertions(+), 146 deletions(-) create mode 100644 backend/src/app/loggers/audit/archive_task.clj create mode 100644 backend/src/app/loggers/audit/gc_task.clj diff --git a/backend/src/app/loggers/audit.clj b/backend/src/app/loggers/audit.clj index aead09110..211799d30 100644 --- a/backend/src/app/loggers/audit.clj +++ b/backend/src/app/loggers/audit.clj @@ -9,31 +9,24 @@ (:require [app.common.data :as d] [app.common.data.macros :as dm] - [app.common.exceptions :as ex] [app.common.logging :as l] [app.common.spec :as us] - [app.common.transit :as t] [app.common.uuid :as uuid] [app.config :as cf] [app.db :as db] [app.http :as-alias http] [app.http.access-token :as-alias actoken] - [app.http.client :as http.client] [app.loggers.audit.tasks :as-alias tasks] [app.loggers.webhooks :as-alias webhooks] - [app.main :as-alias main] [app.rpc :as-alias rpc] [app.rpc.retry :as rtry] [app.setup :as-alias setup] - [app.tokens :as tokens] [app.util.services :as-alias sv] [app.util.time :as dt] [app.worker :as wrk] [clojure.spec.alpha :as s] [cuerdas.core :as str] [integrant.core :as ig] - [lambdaisland.uri :as u] - [promesa.exec :as px] [ring.request :as rreq])) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -249,137 +242,3 @@ (rtry/invoke! cfg db/tx-run! handle-event! event)) (catch Throwable cause (l/error :hint "unexpected error processing event" :cause cause)))) - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; TASK: ARCHIVE -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -;; This is a task responsible to send the accumulated events to -;; external service for archival. - -(declare archive-events) - -(s/def ::tasks/uri ::us/string) - -(defmethod ig/pre-init-spec ::tasks/archive-task [_] - (s/keys :req [::db/pool ::setup/props ::http.client/client])) - -(defmethod ig/init-key ::tasks/archive - [_ cfg] - (fn [params] - ;; NOTE: this let allows overwrite default configured values from - ;; the repl, when manually invoking the task. - (let [enabled (or (contains? cf/flags :audit-log-archive) - (:enabled params false)) - uri (cf/get :audit-log-archive-uri) - uri (or uri (:uri params)) - cfg (assoc cfg ::uri uri)] - - (when (and enabled (not uri)) - (ex/raise :type :internal - :code :task-not-configured - :hint "archive task not configured, missing uri")) - - (when enabled - (loop [total 0] - (let [n (archive-events cfg)] - (if n - (do - (px/sleep 100) - (recur (+ total ^long n))) - (when (pos? total) - (l/dbg :hint "events archived" :total total))))))))) - -(def ^:private sql:retrieve-batch-of-audit-log - "select * - from audit_log - where archived_at is null - order by created_at asc - limit 128 - for update skip locked;") - -(defn archive-events - [{:keys [::db/pool ::uri] :as cfg}] - (letfn [(decode-row [{:keys [props ip-addr context] :as row}] - (cond-> row - (db/pgobject? props) - (assoc :props (db/decode-transit-pgobject props)) - - (db/pgobject? context) - (assoc :context (db/decode-transit-pgobject context)) - - (db/pgobject? ip-addr "inet") - (assoc :ip-addr (db/decode-inet ip-addr)))) - - (row->event [row] - (select-keys row [:type - :name - :source - :created-at - :tracked-at - :profile-id - :ip-addr - :props - :context])) - - (send [events] - (let [token (tokens/generate (::setup/props cfg) - {:iss "authentication" - :iat (dt/now) - :uid uuid/zero}) - body (t/encode {:events events}) - headers {"content-type" "application/transit+json" - "origin" (cf/get :public-uri) - "cookie" (u/map->query-string {:auth-token token})} - params {:uri uri - :timeout 12000 - :method :post - :headers headers - :body body} - resp (http.client/req! cfg params)] - (if (= (:status resp) 204) - true - (do - (l/error :hint "unable to archive events" - :resp-status (:status resp) - :resp-body (:body resp)) - false)))) - - (mark-as-archived [conn rows] - (db/exec-one! conn ["update audit_log set archived_at=now() where id = ANY(?)" - (->> (map :id rows) - (db/create-array conn "uuid"))]))] - - (db/with-atomic [conn pool] - (let [rows (db/exec! conn [sql:retrieve-batch-of-audit-log]) - xform (comp (map decode-row) - (map row->event)) - events (into [] xform rows)] - (when-not (empty? events) - (l/trc :hint "archive events chunk" :uri uri :events (count events)) - (when (send events) - (mark-as-archived conn rows) - (count events))))))) - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; GC Task -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -(def ^:private sql:clean-archived - "delete from audit_log - where archived_at is not null") - -(defn- clean-archived - [{:keys [::db/pool]}] - (let [result (db/exec-one! pool [sql:clean-archived]) - result (:next.jdbc/update-count result)] - (l/debug :hint "delete archived audit log entries" :deleted result) - result)) - -(defmethod ig/pre-init-spec ::tasks/gc [_] - (s/keys :req [::db/pool])) - -(defmethod ig/init-key ::tasks/gc - [_ cfg] - (fn [_] - (clean-archived cfg))) diff --git a/backend/src/app/loggers/audit/archive_task.clj b/backend/src/app/loggers/audit/archive_task.clj new file mode 100644 index 000000000..046fb8068 --- /dev/null +++ b/backend/src/app/loggers/audit/archive_task.clj @@ -0,0 +1,140 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns app.loggers.audit.archive-task + (:require + [app.common.exceptions :as ex] + [app.common.logging :as l] + [app.common.transit :as t] + [app.common.uuid :as uuid] + [app.config :as cf] + [app.db :as db] + [app.http.client :as http] + [app.setup :as-alias setup] + [app.tokens :as tokens] + [app.util.time :as dt] + [clojure.spec.alpha :as s] + [integrant.core :as ig] + [lambdaisland.uri :as u] + [promesa.exec :as px])) + +;; This is a task responsible to send the accumulated events to +;; external service for archival. + +(defn- decode-row + [{:keys [props ip-addr context] :as row}] + (cond-> row + (db/pgobject? props) + (assoc :props (db/decode-transit-pgobject props)) + + (db/pgobject? context) + (assoc :context (db/decode-transit-pgobject context)) + + (db/pgobject? ip-addr "inet") + (assoc :ip-addr (db/decode-inet ip-addr)))) + +(def ^:private event-keys + [:type + :name + :source + :created-at + :tracked-at + :profile-id + :ip-addr + :props + :context]) + +(defn- row->event + [row] + (select-keys row event-keys)) + +(defn- send! + [{:keys [::uri] :as cfg} events] + (let [token (tokens/generate (::setup/props cfg) + {:iss "authentication" + :iat (dt/now) + :uid uuid/zero}) + body (t/encode {:events events}) + headers {"content-type" "application/transit+json" + "origin" (cf/get :public-uri) + "cookie" (u/map->query-string {:auth-token token})} + params {:uri uri + :timeout 12000 + :method :post + :headers headers + :body body} + resp (http/req! cfg params)] + (if (= (:status resp) 204) + true + (do + (l/error :hint "unable to archive events" + :resp-status (:status resp) + :resp-body (:body resp)) + false)))) + +(defn- mark-archived! + [{:keys [::db/conn]} rows] + (let [ids (db/create-array conn "uuid" (map :id rows))] + (db/exec-one! conn ["update audit_log set archived_at=now() where id = ANY(?)" ids]))) + +(def ^:private xf:create-event + (comp (map decode-row) + (map row->event))) + +(def ^:private sql:get-audit-log-chunk + "SELECT * + FROM audit_log + WHERE archived_at is null + ORDER BY created_at ASC + LIMIT 128 + FOR UPDATE + SKIP LOCKED") + +(defn- get-event-rows + [{:keys [::db/conn] :as cfg}] + (->> (db/exec! conn [sql:get-audit-log-chunk]) + (not-empty))) + +(defn- archive-events! + [{:keys [::uri] :as cfg}] + (db/tx-run! cfg (fn [cfg] + (when-let [rows (get-event-rows cfg)] + (let [events (into [] xf:create-event rows)] + (l/trc :hint "archive events chunk" :uri uri :events (count events)) + (when (send! cfg events) + (mark-archived! cfg rows) + (count events))))))) + +(defmethod ig/pre-init-spec ::handler [_] + (s/keys :req [::db/pool ::setup/props ::http/client])) + +(defmethod ig/init-key ::handler + [_ cfg] + (fn [params] + ;; NOTE: this let allows overwrite default configured values from + ;; the repl, when manually invoking the task. + (let [enabled (or (contains? cf/flags :audit-log-archive) + (:enabled params false)) + + uri (cf/get :audit-log-archive-uri) + uri (or uri (:uri params)) + cfg (assoc cfg ::uri uri)] + + (when (and enabled (not uri)) + (ex/raise :type :internal + :code :task-not-configured + :hint "archive task not configured, missing uri")) + + (when enabled + (loop [total 0] + (if-let [n (archive-events! cfg)] + (do + (px/sleep 100) + (recur (+ total ^long n))) + + (when (pos? total) + (l/dbg :hint "events archived" :total total)))))))) + diff --git a/backend/src/app/loggers/audit/gc_task.clj b/backend/src/app/loggers/audit/gc_task.clj new file mode 100644 index 000000000..7f94217a4 --- /dev/null +++ b/backend/src/app/loggers/audit/gc_task.clj @@ -0,0 +1,31 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns app.loggers.audit.gc-task + (:require + [app.common.logging :as l] + [app.db :as db] + [clojure.spec.alpha :as s] + [integrant.core :as ig])) + +(def ^:private sql:clean-archived + "DELETE FROM audit_log + WHERE archived_at IS NOT NULL") + +(defn- clean-archived! + [{:keys [::db/pool]}] + (let [result (db/exec-one! pool [sql:clean-archived]) + result (db/get-update-count result)] + (l/debug :hint "delete archived audit log entries" :deleted result) + result)) + +(defmethod ig/pre-init-spec ::handler [_] + (s/keys :req [::db/pool])) + +(defmethod ig/init-key ::handler + [_ cfg] + (fn [_] + (clean-archived! cfg))) diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index e0177110f..056c99cc8 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -21,7 +21,6 @@ [app.http.session :as-alias session] [app.http.session.tasks :as-alias session.tasks] [app.http.websocket :as http.ws] - [app.loggers.audit.tasks :as-alias audit.tasks] [app.loggers.webhooks :as-alias webhooks] [app.metrics :as-alias mtx] [app.metrics.definition :as-alias mdef] @@ -346,8 +345,8 @@ :storage-gc-deleted (ig/ref ::sto.gc-deleted/handler) :storage-gc-touched (ig/ref ::sto.gc-touched/handler) :session-gc (ig/ref ::session.tasks/gc) - :audit-log-archive (ig/ref ::audit.tasks/archive) - :audit-log-gc (ig/ref ::audit.tasks/gc) + :audit-log-archive (ig/ref :app.loggers.audit.archive-task/handler) + :audit-log-gc (ig/ref :app.loggers.audit.gc-task/handler) :process-webhook-event (ig/ref ::webhooks/process-event-handler) @@ -411,12 +410,12 @@ ::svgo/optimizer {} - ::audit.tasks/archive + :app.loggers.audit.archive-task/handler {::setup/props (ig/ref ::setup/props) ::db/pool (ig/ref ::db/pool) ::http.client/client (ig/ref ::http.client/client)} - ::audit.tasks/gc + :app.loggers.audit.gc-task/handler {::db/pool (ig/ref ::db/pool)} ::webhooks/process-event-handler From eaf546ba5e04d5ec7fd587d9cc8f16271fa048d1 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 25 Mar 2024 11:28:24 +0100 Subject: [PATCH 2/4] :sparkles: Add improvements to telemetry task --- backend/src/app/loggers/audit.clj | 23 +- backend/src/app/tasks/telemetry.clj | 334 +++++++++++++++------------- 2 files changed, 200 insertions(+), 157 deletions(-) diff --git a/backend/src/app/loggers/audit.clj b/backend/src/app/loggers/audit.clj index 211799d30..d89809f37 100644 --- a/backend/src/app/loggers/audit.clj +++ b/backend/src/app/loggers/audit.clj @@ -188,14 +188,14 @@ :profile-id (::profile-id event) :ip-addr (::ip-addr event) :context (::context event) - :props (::props event)}] + :props (::props event)} + tnow (dt/now)] (when (contains? cf/flags :audit-log) ;; NOTE: this operation may cause primary key conflicts on inserts ;; because of the timestamp precission (two concurrent requests), in ;; this case we just retry the operation. - (let [tnow (dt/now) - params (-> params + (let [params (-> params (assoc :created-at tnow) (assoc :tracked-at tnow) (update :props db/tjson) @@ -204,6 +204,23 @@ (assoc :source "backend"))] (db/insert! cfg :audit-log params))) + (when (and (or (contains? cf/flags :telemetry) + (cf/get :telemetry-enabled)) + (not (contains? cf/flags :audit-log))) + ;; NOTE: this operation may cause primary key conflicts on inserts + ;; because of the timestamp precission (two concurrent requests), in + ;; this case we just retry the operation. + ;; + ;; NOTE: this is only executed when general audit log is disabled + (let [params (-> params + (assoc :created-at tnow) + (assoc :tracked-at tnow) + (assoc :props (db/tjson {})) + (assoc :context (db/tjson {})) + (assoc :ip-addr (db/inet "0.0.0.0")) + (assoc :source "backend"))] + (db/insert! cfg :audit-log params))) + (when (and (contains? cf/flags :webhooks) (::webhooks/event? event)) (let [batch-key (::webhooks/batch-key event) diff --git a/backend/src/app/tasks/telemetry.clj b/backend/src/app/tasks/telemetry.clj index 43c0b26f9..ec07c67b3 100644 --- a/backend/src/app/tasks/telemetry.clj +++ b/backend/src/app/tasks/telemetry.clj @@ -22,13 +22,182 @@ [promesa.exec :as px])) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; TASK ENTRY POINT +;; IMPL ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(declare get-stats) -(declare send!) -(declare get-subscriptions-newsletter-updates) -(declare get-subscriptions-newsletter-news) +(defn- send! + [cfg data] + (let [request {:method :post + :uri (cf/get :telemetry-uri) + :headers {"content-type" "application/json"} + :body (json/encode-str data)} + response (http/req! cfg request)] + (when (> (:status response) 206) + (ex/raise :type :internal + :code :invalid-response + :response-status (:status response) + :response-body (:body response))))) + +(defn- get-subscriptions-newsletter-updates + [conn] + (let [sql "SELECT email FROM profile where props->>'~:newsletter-updates' = 'true'"] + (->> (db/exec! conn [sql]) + (mapv :email)))) + +(defn- get-subscriptions-newsletter-news + [conn] + (let [sql "SELECT email FROM profile where props->>'~:newsletter-news' = 'true'"] + (->> (db/exec! conn [sql]) + (mapv :email)))) + +(defn- get-num-teams + [conn] + (-> (db/exec-one! conn ["SELECT count(*) AS count FROM team"]) :count)) + +(defn- get-num-projects + [conn] + (-> (db/exec-one! conn ["SELECT count(*) AS count FROM project"]) :count)) + +(defn- get-num-files + [conn] + (-> (db/exec-one! conn ["SELECT count(*) AS count FROM file"]) :count)) + +(defn- get-num-file-changes + [conn] + (let [sql (str "SELECT count(*) AS count " + " FROM file_change " + " where date_trunc('day', created_at) = date_trunc('day', now())")] + (-> (db/exec-one! conn [sql]) :count))) + +(defn- get-num-touched-files + [conn] + (let [sql (str "SELECT count(distinct file_id) AS count " + " FROM file_change " + " where date_trunc('day', created_at) = date_trunc('day', now())")] + (-> (db/exec-one! conn [sql]) :count))) + +(defn- get-num-users + [conn] + (-> (db/exec-one! conn ["SELECT count(*) AS count FROM profile"]) :count)) + +(defn- get-num-fonts + [conn] + (-> (db/exec-one! conn ["SELECT count(*) AS count FROM team_font_variant"]) :count)) + +(defn- get-num-comments + [conn] + (-> (db/exec-one! conn ["SELECT count(*) AS count FROM comment"]) :count)) + +(def sql:team-averages + "with projects_by_team AS ( + SELECT t.id, count(p.id) AS num_projects + FROM team AS t + LEFT JOIN project AS p ON (p.team_id = t.id) + GROUP BY 1 + ), files_by_project AS ( + SELECT p.id, count(f.id) AS num_files + FROM project AS p + LEFT JOIN file AS f ON (f.project_id = p.id) + GROUP BY 1 + ), comment_threads_by_file AS ( + SELECT f.id, count(ct.id) AS num_comment_threads + FROM file AS f + LEFT JOIN comment_thread AS ct ON (ct.file_id = f.id) + GROUP BY 1 + ), users_by_team AS ( + SELECT t.id, count(tp.profile_id) AS num_users + FROM team AS t + LEFT JOIN team_profile_rel AS tp ON(tp.team_id = t.id) + GROUP BY 1 + ) + SELECT (SELECT avg(num_projects)::integer FROM projects_by_team) AS avg_projects_on_team, + (SELECT max(num_projects)::integer FROM projects_by_team) AS max_projects_on_team, + (SELECT avg(num_files)::integer FROM files_by_project) AS avg_files_on_project, + (SELECT max(num_files)::integer FROM files_by_project) AS max_files_on_project, + (SELECT avg(num_comment_threads)::integer FROM comment_threads_by_file) AS avg_comment_threads_on_file, + (SELECT max(num_comment_threads)::integer FROM comment_threads_by_file) AS max_comment_threads_on_file, + (SELECT avg(num_users)::integer FROM users_by_team) AS avg_users_on_team, + (SELECT max(num_users)::integer FROM users_by_team) AS max_users_on_team") + +(defn- get-team-averages + [conn] + (->> [sql:team-averages] + (db/exec-one! conn))) + +(defn- get-enabled-auth-providers + [conn] + (let [sql (str "SELECT auth_backend AS backend, count(*) AS total " + " FROM profile GROUP BY 1") + rows (db/exec! conn [sql])] + (->> rows + (map (fn [{:keys [backend total]}] + (let [backend (or backend "penpot")] + [(keyword (str "auth-backend-" backend)) + total]))) + (into {})))) + +(defn- get-jvm-stats + [] + (let [^Runtime runtime (Runtime/getRuntime)] + {:jvm-heap-current (.totalMemory runtime) + :jvm-heap-max (.maxMemory runtime) + :jvm-cpus (.availableProcessors runtime) + :os-arch (System/getProperty "os.arch") + :os-name (System/getProperty "os.name") + :os-version (System/getProperty "os.version") + :user-tz (System/getProperty "user.timezone")})) + +(def ^:private sql:get-counters + "SELECT name, count(*) AS count + FROM audit_log + WHERE source = 'backend' + AND tracked_at >= date_trunc('day', now()) + GROUP BY 1 + ORDER BY 2 DESC") + +(defn- get-action-counters + [conn] + (let [counters (->> (db/exec! conn [sql:get-counters]) + (d/index-by (comp keyword :name) :count)) + total (reduce + 0 (vals counters))] + {:total-accomulated-events total + :event-counters counters})) + +(def ^:private sql:clean-counters + "DELETE FROM audit_log + WHERE ip_addr = '0.0.0.0'::inet -- we know this is from telemetry + AND tracked_at < (date_trunc('day', now()) - '1 day'::interval)") + +(defn- clean-counters-data! + [conn] + (when-not (contains? cf/flags :audit-log) + (db/exec-one! conn [sql:clean-counters]))) + +(defn- get-stats + [conn] + (let [referer (if (cf/get :telemetry-with-taiga) + "taiga" + (cf/get :telemetry-referer))] + (-> {:referer referer + :public-uri (cf/get :public-uri) + :total-teams (get-num-teams conn) + :total-projects (get-num-projects conn) + :total-files (get-num-files conn) + :total-users (get-num-users conn) + :total-fonts (get-num-fonts conn) + :total-comments (get-num-comments conn) + :total-file-changes (get-num-file-changes conn) + :total-touched-files (get-num-touched-files conn)} + (merge + (get-team-averages conn) + (get-jvm-stats) + (get-enabled-auth-providers conn) + (get-action-counters conn)) + (d/without-nils)))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; TASK ENTRY POINT +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (defmethod ig/pre-init-spec ::handler [_] (s/keys :req [::http/client @@ -48,6 +217,10 @@ data {:subscriptions subs :version (:full cf/version) :instance-id (:instance-id props)}] + + (when enabled? + (clean-counters-data! pool)) + (cond ;; If we have telemetry enabled, then proceed the normal ;; operation. @@ -63,7 +236,8 @@ ;; onboarding dialog or the profile section, then proceed to ;; send a limited telemetry data, that consists in the list of ;; subscribed emails and the running penpot version. - (seq subs) + (or (seq (:newsletter-updates subs)) + (seq (:newsletter-news subs))) (do (when send? (px/sleep (rand-int 10000)) @@ -72,151 +246,3 @@ :else data)))) - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; IMPL -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -(defn- send! - [cfg data] - (let [request {:method :post - :uri (cf/get :telemetry-uri) - :headers {"content-type" "application/json"} - :body (json/encode-str data)} - response (http/req! cfg request)] - (when (> (:status response) 206) - (ex/raise :type :internal - :code :invalid-response - :response-status (:status response) - :response-body (:body response))))) - -(defn- get-subscriptions-newsletter-updates - [conn] - (let [sql "select email from profile where props->>'~:newsletter-updates' = 'true'"] - (->> (db/exec! conn [sql]) - (mapv :email)))) - -(defn- get-subscriptions-newsletter-news - [conn] - (let [sql "select email from profile where props->>'~:newsletter-news' = 'true'"] - (->> (db/exec! conn [sql]) - (mapv :email)))) - -(defn- retrieve-num-teams - [conn] - (-> (db/exec-one! conn ["select count(*) as count from team;"]) :count)) - -(defn- retrieve-num-projects - [conn] - (-> (db/exec-one! conn ["select count(*) as count from project;"]) :count)) - -(defn- retrieve-num-files - [conn] - (-> (db/exec-one! conn ["select count(*) as count from file;"]) :count)) - -(defn- retrieve-num-file-changes - [conn] - (let [sql (str "select count(*) as count " - " from file_change " - " where date_trunc('day', created_at) = date_trunc('day', now())")] - (-> (db/exec-one! conn [sql]) :count))) - -(defn- retrieve-num-touched-files - [conn] - (let [sql (str "select count(distinct file_id) as count " - " from file_change " - " where date_trunc('day', created_at) = date_trunc('day', now())")] - (-> (db/exec-one! conn [sql]) :count))) - -(defn- retrieve-num-users - [conn] - (-> (db/exec-one! conn ["select count(*) as count from profile;"]) :count)) - -(defn- retrieve-num-fonts - [conn] - (-> (db/exec-one! conn ["select count(*) as count from team_font_variant;"]) :count)) - -(defn- retrieve-num-comments - [conn] - (-> (db/exec-one! conn ["select count(*) as count from comment;"]) :count)) - -(def sql:team-averages - "with projects_by_team as ( - select t.id, count(p.id) as num_projects - from team as t - left join project as p on (p.team_id = t.id) - group by 1 - ), files_by_project as ( - select p.id, count(f.id) as num_files - from project as p - left join file as f on (f.project_id = p.id) - group by 1 - ), comment_threads_by_file as ( - select f.id, count(ct.id) as num_comment_threads - from file as f - left join comment_thread as ct on (ct.file_id = f.id) - group by 1 - ), users_by_team as ( - select t.id, count(tp.profile_id) as num_users - from team as t - left join team_profile_rel as tp on(tp.team_id = t.id) - group by 1 - ) - select (select avg(num_projects)::integer from projects_by_team) as avg_projects_on_team, - (select max(num_projects)::integer from projects_by_team) as max_projects_on_team, - (select avg(num_files)::integer from files_by_project) as avg_files_on_project, - (select max(num_files)::integer from files_by_project) as max_files_on_project, - (select avg(num_comment_threads)::integer from comment_threads_by_file) as avg_comment_threads_on_file, - (select max(num_comment_threads)::integer from comment_threads_by_file) as max_comment_threads_on_file, - (select avg(num_users)::integer from users_by_team) as avg_users_on_team, - (select max(num_users)::integer from users_by_team) as max_users_on_team;") - -(defn- retrieve-team-averages - [conn] - (->> [sql:team-averages] - (db/exec-one! conn))) - -(defn- retrieve-enabled-auth-providers - [conn] - (let [sql (str "select auth_backend as backend, count(*) as total " - " from profile group by 1") - rows (db/exec! conn [sql])] - (->> rows - (map (fn [{:keys [backend total]}] - (let [backend (or backend "penpot")] - [(keyword (str "auth-backend-" backend)) - total]))) - (into {})))) - -(defn- retrieve-jvm-stats - [] - (let [^Runtime runtime (Runtime/getRuntime)] - {:jvm-heap-current (.totalMemory runtime) - :jvm-heap-max (.maxMemory runtime) - :jvm-cpus (.availableProcessors runtime) - :os-arch (System/getProperty "os.arch") - :os-name (System/getProperty "os.name") - :os-version (System/getProperty "os.version") - :user-tz (System/getProperty "user.timezone")})) - -(defn get-stats - [conn] - (let [referer (if (cf/get :telemetry-with-taiga) - "taiga" - (cf/get :telemetry-referer))] - (-> {:referer referer - :public-uri (cf/get :public-uri) - :total-teams (retrieve-num-teams conn) - :total-projects (retrieve-num-projects conn) - :total-files (retrieve-num-files conn) - :total-users (retrieve-num-users conn) - :total-fonts (retrieve-num-fonts conn) - :total-comments (retrieve-num-comments conn) - :total-file-changes (retrieve-num-file-changes conn) - :total-touched-files (retrieve-num-touched-files conn)} - (d/merge - (retrieve-team-averages conn) - (retrieve-jvm-stats) - (retrieve-enabled-auth-providers conn)) - (d/without-nils)))) - From 763fc3532e44d32e8838b289391c600547ec4b15 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 25 Mar 2024 15:39:01 +0100 Subject: [PATCH 3/4] :sparkles: Simplify local audit table Remove unnecessary partitioning --- backend/src/app/migrations.clj | 5 ++++- .../app/migrations/sql/0120-mod-audit-log-table.sql | 11 +++++++++++ backend/test/backend_tests/helpers.clj | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 backend/src/app/migrations/sql/0120-mod-audit-log-table.sql diff --git a/backend/src/app/migrations.clj b/backend/src/app/migrations.clj index 87f3d90b9..86f0fa6f5 100644 --- a/backend/src/app/migrations.clj +++ b/backend/src/app/migrations.clj @@ -376,7 +376,10 @@ :fn (mg/resource "app/migrations/sql/0118-mod-task-table.sql")} {:name "0119-mod-file-table" - :fn (mg/resource "app/migrations/sql/0119-mod-file-table.sql")}]) + :fn (mg/resource "app/migrations/sql/0119-mod-file-table.sql")} + + {:name "0120-mod-audit-log-table" + :fn (mg/resource "app/migrations/sql/0120-mod-audit-log-table.sql")}]) (defn apply-migrations! [pool name migrations] diff --git a/backend/src/app/migrations/sql/0120-mod-audit-log-table.sql b/backend/src/app/migrations/sql/0120-mod-audit-log-table.sql new file mode 100644 index 000000000..e9b4b83c5 --- /dev/null +++ b/backend/src/app/migrations/sql/0120-mod-audit-log-table.sql @@ -0,0 +1,11 @@ +CREATE TABLE new_audit_log (LIKE audit_log INCLUDING ALL); +INSERT INTO new_audit_log SELECT * FROM audit_log; +ALTER TABLE audit_log RENAME TO old_audit_log; +ALTER TABLE new_audit_log RENAME TO audit_log; +DROP TABLE old_audit_log; + +DROP INDEX new_audit_log_id_archived_at_idx; +ALTER TABLE audit_log DROP CONSTRAINT new_audit_log_pkey; +ALTER TABLE audit_log ADD PRIMARY KEY (id); +ALTER TABLE audit_log ALTER COLUMN created_at SET DEFAULT now(); +ALTER TABLE audit_log ALTER COLUMN tracked_at SET DEFAULT now(); diff --git a/backend/test/backend_tests/helpers.clj b/backend/test/backend_tests/helpers.clj index 987b55304..27544c4fa 100644 --- a/backend/test/backend_tests/helpers.clj +++ b/backend/test/backend_tests/helpers.clj @@ -112,7 +112,7 @@ ;; "alter table task set unlogged;\n" ;; "alter table task_default set unlogged;\n" ;; "alter table task_completed set unlogged;\n" - "alter table audit_log_default set unlogged ;\n" + "alter table audit_log set unlogged ;\n" "alter table storage_object set unlogged;\n" "alter table server_error_report set unlogged;\n" "alter table server_prop set unlogged;\n" From b85c3bec18c12b4a8f8f54e51e81413224d2a51b Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 25 Mar 2024 15:41:34 +0100 Subject: [PATCH 4/4] :sparkles: Add better timestamp control on audit handler --- backend/src/app/rpc/commands/audit.clj | 41 +++++++++++++++++++++----- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/backend/src/app/rpc/commands/audit.clj b/backend/src/app/rpc/commands/audit.clj index 76bd6e188..5db758b46 100644 --- a/backend/src/app/rpc/commands/audit.clj +++ b/backend/src/app/rpc/commands/audit.clj @@ -19,7 +19,20 @@ [app.rpc.climit :as-alias climit] [app.rpc.doc :as-alias doc] [app.rpc.helpers :as rph] - [app.util.services :as sv])) + [app.util.services :as sv] + [app.util.time :as dt])) + +(def ^:private event-columns + [:id + :name + :source + :type + :tracked-at + :created-at + :profile-id + :ip-addr + :props + :context]) (defn- event->row [event] [(uuid/next) @@ -27,24 +40,38 @@ (:source event) (:type event) (:timestamp event) + (:created-at event) (:profile-id event) (db/inet (:ip-addr event)) (db/tjson (:props event)) (db/tjson (d/without-nils (:context event)))]) -(def ^:private event-columns - [:id :name :source :type :tracked-at - :profile-id :ip-addr :props :context]) +(defn- adjust-timestamp + [{:keys [timestamp created-at] :as event}] + (let [margin (inst-ms (dt/diff timestamp created-at))] + (if (or (neg? margin) + (> margin 3600000)) + ;; If event is in future or lags more than 1 hour, we reasign + ;; timestamp to the server creation date + (-> event + (assoc :timestamp created-at) + (update :context assoc :original-timestamp timestamp)) + event))) (defn- handle-events [{:keys [::db/pool]} {:keys [::rpc/profile-id events] :as params}] (let [request (-> params meta ::http/request) ip-addr (audit/parse-client-ip request) + tnow (dt/now) xform (comp - (map #(assoc % :profile-id profile-id)) - (map #(assoc % :ip-addr ip-addr)) - (map #(assoc % :source "frontend")) + (map (fn [event] + (-> event + (assoc :created-at tnow) + (assoc :profile-id profile-id) + (assoc :ip-addr ip-addr) + (assoc :source "frontend")))) (filter :profile-id) + (map adjust-timestamp) (map event->row)) events (sequence xform events)] (when (seq events)