mirror of
https://github.com/penpot/penpot.git
synced 2025-01-23 23:18:48 -05:00
Merge pull request #4323 from penpot/niwinz-staging-bugfix-2
✨ Several improvements
This commit is contained in:
Alejandro
GitHub
commit
81a271961f
9 changed files with 425 additions and 312 deletions
|
|
@ -9,31 +9,24 @@
|
|||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.data.macros :as dm]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[app.common.spec :as us]
|
||||
[app.common.transit :as t]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.http :as-alias http]
|
||||
[app.http.access-token :as-alias actoken]
|
||||
[app.http.client :as http.client]
|
||||
[app.loggers.audit.tasks :as-alias tasks]
|
||||
[app.loggers.webhooks :as-alias webhooks]
|
||||
[app.main :as-alias main]
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.rpc.retry :as rtry]
|
||||
[app.setup :as-alias setup]
|
||||
[app.tokens :as tokens]
|
||||
[app.util.services :as-alias sv]
|
||||
[app.util.time :as dt]
|
||||
[app.worker :as wrk]
|
||||
[clojure.spec.alpha :as s]
|
||||
[cuerdas.core :as str]
|
||||
[integrant.core :as ig]
|
||||
[lambdaisland.uri :as u]
|
||||
[promesa.exec :as px]
|
||||
[ring.request :as rreq]))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
|
@ -195,14 +188,14 @@
|
|||
:profile-id (::profile-id event)
|
||||
:ip-addr (::ip-addr event)
|
||||
:context (::context event)
|
||||
:props (::props event)}]
|
||||
:props (::props event)}
|
||||
tnow (dt/now)]
|
||||
|
||||
(when (contains? cf/flags :audit-log)
|
||||
;; NOTE: this operation may cause primary key conflicts on inserts
|
||||
;; because of the timestamp precission (two concurrent requests), in
|
||||
;; this case we just retry the operation.
|
||||
(let [tnow (dt/now)
|
||||
params (-> params
|
||||
(let [params (-> params
|
||||
(assoc :created-at tnow)
|
||||
(assoc :tracked-at tnow)
|
||||
(update :props db/tjson)
|
||||
|
|
@ -211,6 +204,23 @@
|
|||
(assoc :source "backend"))]
|
||||
(db/insert! cfg :audit-log params)))
|
||||
|
||||
(when (and (or (contains? cf/flags :telemetry)
|
||||
(cf/get :telemetry-enabled))
|
||||
(not (contains? cf/flags :audit-log)))
|
||||
;; NOTE: this operation may cause primary key conflicts on inserts
|
||||
;; because of the timestamp precission (two concurrent requests), in
|
||||
;; this case we just retry the operation.
|
||||
;;
|
||||
;; NOTE: this is only executed when general audit log is disabled
|
||||
(let [params (-> params
|
||||
(assoc :created-at tnow)
|
||||
(assoc :tracked-at tnow)
|
||||
(assoc :props (db/tjson {}))
|
||||
(assoc :context (db/tjson {}))
|
||||
(assoc :ip-addr (db/inet "0.0.0.0"))
|
||||
(assoc :source "backend"))]
|
||||
(db/insert! cfg :audit-log params)))
|
||||
|
||||
(when (and (contains? cf/flags :webhooks)
|
||||
(::webhooks/event? event))
|
||||
(let [batch-key (::webhooks/batch-key event)
|
||||
|
|
@ -249,137 +259,3 @@
|
|||
(rtry/invoke! cfg db/tx-run! handle-event! event))
|
||||
(catch Throwable cause
|
||||
(l/error :hint "unexpected error processing event" :cause cause))))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; TASK: ARCHIVE
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
;; This is a task responsible to send the accumulated events to
|
||||
;; external service for archival.
|
||||
|
||||
(declare archive-events)
|
||||
|
||||
(s/def ::tasks/uri ::us/string)
|
||||
|
||||
(defmethod ig/pre-init-spec ::tasks/archive-task [_]
|
||||
(s/keys :req [::db/pool ::setup/props ::http.client/client]))
|
||||
|
||||
(defmethod ig/init-key ::tasks/archive
|
||||
[_ cfg]
|
||||
(fn [params]
|
||||
;; NOTE: this let allows overwrite default configured values from
|
||||
;; the repl, when manually invoking the task.
|
||||
(let [enabled (or (contains? cf/flags :audit-log-archive)
|
||||
(:enabled params false))
|
||||
uri (cf/get :audit-log-archive-uri)
|
||||
uri (or uri (:uri params))
|
||||
cfg (assoc cfg ::uri uri)]
|
||||
|
||||
(when (and enabled (not uri))
|
||||
(ex/raise :type :internal
|
||||
:code :task-not-configured
|
||||
:hint "archive task not configured, missing uri"))
|
||||
|
||||
(when enabled
|
||||
(loop [total 0]
|
||||
(let [n (archive-events cfg)]
|
||||
(if n
|
||||
(do
|
||||
(px/sleep 100)
|
||||
(recur (+ total ^long n)))
|
||||
(when (pos? total)
|
||||
(l/dbg :hint "events archived" :total total)))))))))
|
||||
|
||||
(def ^:private sql:retrieve-batch-of-audit-log
|
||||
"select *
|
||||
from audit_log
|
||||
where archived_at is null
|
||||
order by created_at asc
|
||||
limit 128
|
||||
for update skip locked;")
|
||||
|
||||
(defn archive-events
|
||||
[{:keys [::db/pool ::uri] :as cfg}]
|
||||
(letfn [(decode-row [{:keys [props ip-addr context] :as row}]
|
||||
(cond-> row
|
||||
(db/pgobject? props)
|
||||
(assoc :props (db/decode-transit-pgobject props))
|
||||
|
||||
(db/pgobject? context)
|
||||
(assoc :context (db/decode-transit-pgobject context))
|
||||
|
||||
(db/pgobject? ip-addr "inet")
|
||||
(assoc :ip-addr (db/decode-inet ip-addr))))
|
||||
|
||||
(row->event [row]
|
||||
(select-keys row [:type
|
||||
:name
|
||||
:source
|
||||
:created-at
|
||||
:tracked-at
|
||||
:profile-id
|
||||
:ip-addr
|
||||
:props
|
||||
:context]))
|
||||
|
||||
(send [events]
|
||||
(let [token (tokens/generate (::setup/props cfg)
|
||||
{:iss "authentication"
|
||||
:iat (dt/now)
|
||||
:uid uuid/zero})
|
||||
body (t/encode {:events events})
|
||||
headers {"content-type" "application/transit+json"
|
||||
"origin" (cf/get :public-uri)
|
||||
"cookie" (u/map->query-string {:auth-token token})}
|
||||
params {:uri uri
|
||||
:timeout 12000
|
||||
:method :post
|
||||
:headers headers
|
||||
:body body}
|
||||
resp (http.client/req! cfg params)]
|
||||
(if (= (:status resp) 204)
|
||||
true
|
||||
(do
|
||||
(l/error :hint "unable to archive events"
|
||||
:resp-status (:status resp)
|
||||
:resp-body (:body resp))
|
||||
false))))
|
||||
|
||||
(mark-as-archived [conn rows]
|
||||
(db/exec-one! conn ["update audit_log set archived_at=now() where id = ANY(?)"
|
||||
(->> (map :id rows)
|
||||
(db/create-array conn "uuid"))]))]
|
||||
|
||||
(db/with-atomic [conn pool]
|
||||
(let [rows (db/exec! conn [sql:retrieve-batch-of-audit-log])
|
||||
xform (comp (map decode-row)
|
||||
(map row->event))
|
||||
events (into [] xform rows)]
|
||||
(when-not (empty? events)
|
||||
(l/trc :hint "archive events chunk" :uri uri :events (count events))
|
||||
(when (send events)
|
||||
(mark-as-archived conn rows)
|
||||
(count events)))))))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; GC Task
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(def ^:private sql:clean-archived
|
||||
"delete from audit_log
|
||||
where archived_at is not null")
|
||||
|
||||
(defn- clean-archived
|
||||
[{:keys [::db/pool]}]
|
||||
(let [result (db/exec-one! pool [sql:clean-archived])
|
||||
result (:next.jdbc/update-count result)]
|
||||
(l/debug :hint "delete archived audit log entries" :deleted result)
|
||||
result))
|
||||
|
||||
(defmethod ig/pre-init-spec ::tasks/gc [_]
|
||||
(s/keys :req [::db/pool]))
|
||||
|
||||
(defmethod ig/init-key ::tasks/gc
|
||||
[_ cfg]
|
||||
(fn [_]
|
||||
(clean-archived cfg)))
|
||||
|
|
|
|||
140
backend/src/app/loggers/audit/archive_task.clj
Normal file
140
backend/src/app/loggers/audit/archive_task.clj
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC
|
||||
|
||||
(ns app.loggers.audit.archive-task
|
||||
(:require
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[app.common.transit :as t]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.http.client :as http]
|
||||
[app.setup :as-alias setup]
|
||||
[app.tokens :as tokens]
|
||||
[app.util.time :as dt]
|
||||
[clojure.spec.alpha :as s]
|
||||
[integrant.core :as ig]
|
||||
[lambdaisland.uri :as u]
|
||||
[promesa.exec :as px]))
|
||||
|
||||
;; This is a task responsible to send the accumulated events to
|
||||
;; external service for archival.
|
||||
|
||||
(defn- decode-row
|
||||
[{:keys [props ip-addr context] :as row}]
|
||||
(cond-> row
|
||||
(db/pgobject? props)
|
||||
(assoc :props (db/decode-transit-pgobject props))
|
||||
|
||||
(db/pgobject? context)
|
||||
(assoc :context (db/decode-transit-pgobject context))
|
||||
|
||||
(db/pgobject? ip-addr "inet")
|
||||
(assoc :ip-addr (db/decode-inet ip-addr))))
|
||||
|
||||
(def ^:private event-keys
|
||||
[:type
|
||||
:name
|
||||
:source
|
||||
:created-at
|
||||
:tracked-at
|
||||
:profile-id
|
||||
:ip-addr
|
||||
:props
|
||||
:context])
|
||||
|
||||
(defn- row->event
|
||||
[row]
|
||||
(select-keys row event-keys))
|
||||
|
||||
(defn- send!
|
||||
[{:keys [::uri] :as cfg} events]
|
||||
(let [token (tokens/generate (::setup/props cfg)
|
||||
{:iss "authentication"
|
||||
:iat (dt/now)
|
||||
:uid uuid/zero})
|
||||
body (t/encode {:events events})
|
||||
headers {"content-type" "application/transit+json"
|
||||
"origin" (cf/get :public-uri)
|
||||
"cookie" (u/map->query-string {:auth-token token})}
|
||||
params {:uri uri
|
||||
:timeout 12000
|
||||
:method :post
|
||||
:headers headers
|
||||
:body body}
|
||||
resp (http/req! cfg params)]
|
||||
(if (= (:status resp) 204)
|
||||
true
|
||||
(do
|
||||
(l/error :hint "unable to archive events"
|
||||
:resp-status (:status resp)
|
||||
:resp-body (:body resp))
|
||||
false))))
|
||||
|
||||
(defn- mark-archived!
|
||||
[{:keys [::db/conn]} rows]
|
||||
(let [ids (db/create-array conn "uuid" (map :id rows))]
|
||||
(db/exec-one! conn ["update audit_log set archived_at=now() where id = ANY(?)" ids])))
|
||||
|
||||
(def ^:private xf:create-event
|
||||
(comp (map decode-row)
|
||||
(map row->event)))
|
||||
|
||||
(def ^:private sql:get-audit-log-chunk
|
||||
"SELECT *
|
||||
FROM audit_log
|
||||
WHERE archived_at is null
|
||||
ORDER BY created_at ASC
|
||||
LIMIT 128
|
||||
FOR UPDATE
|
||||
SKIP LOCKED")
|
||||
|
||||
(defn- get-event-rows
|
||||
[{:keys [::db/conn] :as cfg}]
|
||||
(->> (db/exec! conn [sql:get-audit-log-chunk])
|
||||
(not-empty)))
|
||||
|
||||
(defn- archive-events!
|
||||
[{:keys [::uri] :as cfg}]
|
||||
(db/tx-run! cfg (fn [cfg]
|
||||
(when-let [rows (get-event-rows cfg)]
|
||||
(let [events (into [] xf:create-event rows)]
|
||||
(l/trc :hint "archive events chunk" :uri uri :events (count events))
|
||||
(when (send! cfg events)
|
||||
(mark-archived! cfg rows)
|
||||
(count events)))))))
|
||||
|
||||
(defmethod ig/pre-init-spec ::handler [_]
|
||||
(s/keys :req [::db/pool ::setup/props ::http/client]))
|
||||
|
||||
(defmethod ig/init-key ::handler
|
||||
[_ cfg]
|
||||
(fn [params]
|
||||
;; NOTE: this let allows overwrite default configured values from
|
||||
;; the repl, when manually invoking the task.
|
||||
(let [enabled (or (contains? cf/flags :audit-log-archive)
|
||||
(:enabled params false))
|
||||
|
||||
uri (cf/get :audit-log-archive-uri)
|
||||
uri (or uri (:uri params))
|
||||
cfg (assoc cfg ::uri uri)]
|
||||
|
||||
(when (and enabled (not uri))
|
||||
(ex/raise :type :internal
|
||||
:code :task-not-configured
|
||||
:hint "archive task not configured, missing uri"))
|
||||
|
||||
(when enabled
|
||||
(loop [total 0]
|
||||
(if-let [n (archive-events! cfg)]
|
||||
(do
|
||||
(px/sleep 100)
|
||||
(recur (+ total ^long n)))
|
||||
|
||||
(when (pos? total)
|
||||
(l/dbg :hint "events archived" :total total))))))))
|
||||
|
||||
31
backend/src/app/loggers/audit/gc_task.clj
Normal file
31
backend/src/app/loggers/audit/gc_task.clj
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC
|
||||
|
||||
(ns app.loggers.audit.gc-task
|
||||
(:require
|
||||
[app.common.logging :as l]
|
||||
[app.db :as db]
|
||||
[clojure.spec.alpha :as s]
|
||||
[integrant.core :as ig]))
|
||||
|
||||
(def ^:private sql:clean-archived
|
||||
"DELETE FROM audit_log
|
||||
WHERE archived_at IS NOT NULL")
|
||||
|
||||
(defn- clean-archived!
|
||||
[{:keys [::db/pool]}]
|
||||
(let [result (db/exec-one! pool [sql:clean-archived])
|
||||
result (db/get-update-count result)]
|
||||
(l/debug :hint "delete archived audit log entries" :deleted result)
|
||||
result))
|
||||
|
||||
(defmethod ig/pre-init-spec ::handler [_]
|
||||
(s/keys :req [::db/pool]))
|
||||
|
||||
(defmethod ig/init-key ::handler
|
||||
[_ cfg]
|
||||
(fn [_]
|
||||
(clean-archived! cfg)))
|
||||
|
|
@ -21,7 +21,6 @@
|
|||
[app.http.session :as-alias session]
|
||||
[app.http.session.tasks :as-alias session.tasks]
|
||||
[app.http.websocket :as http.ws]
|
||||
[app.loggers.audit.tasks :as-alias audit.tasks]
|
||||
[app.loggers.webhooks :as-alias webhooks]
|
||||
[app.metrics :as-alias mtx]
|
||||
[app.metrics.definition :as-alias mdef]
|
||||
|
|
@ -346,8 +345,8 @@
|
|||
:storage-gc-deleted (ig/ref ::sto.gc-deleted/handler)
|
||||
:storage-gc-touched (ig/ref ::sto.gc-touched/handler)
|
||||
:session-gc (ig/ref ::session.tasks/gc)
|
||||
:audit-log-archive (ig/ref ::audit.tasks/archive)
|
||||
:audit-log-gc (ig/ref ::audit.tasks/gc)
|
||||
:audit-log-archive (ig/ref :app.loggers.audit.archive-task/handler)
|
||||
:audit-log-gc (ig/ref :app.loggers.audit.gc-task/handler)
|
||||
|
||||
:process-webhook-event
|
||||
(ig/ref ::webhooks/process-event-handler)
|
||||
|
|
@ -411,12 +410,12 @@
|
|||
::svgo/optimizer
|
||||
{}
|
||||
|
||||
::audit.tasks/archive
|
||||
:app.loggers.audit.archive-task/handler
|
||||
{::setup/props (ig/ref ::setup/props)
|
||||
::db/pool (ig/ref ::db/pool)
|
||||
::http.client/client (ig/ref ::http.client/client)}
|
||||
|
||||
::audit.tasks/gc
|
||||
:app.loggers.audit.gc-task/handler
|
||||
{::db/pool (ig/ref ::db/pool)}
|
||||
|
||||
::webhooks/process-event-handler
|
||||
|
|
|
|||
|
|
@ -376,7 +376,10 @@
|
|||
:fn (mg/resource "app/migrations/sql/0118-mod-task-table.sql")}
|
||||
|
||||
{:name "0119-mod-file-table"
|
||||
:fn (mg/resource "app/migrations/sql/0119-mod-file-table.sql")}])
|
||||
:fn (mg/resource "app/migrations/sql/0119-mod-file-table.sql")}
|
||||
|
||||
{:name "0120-mod-audit-log-table"
|
||||
:fn (mg/resource "app/migrations/sql/0120-mod-audit-log-table.sql")}])
|
||||
|
||||
(defn apply-migrations!
|
||||
[pool name migrations]
|
||||
|
|
|
|||
11
backend/src/app/migrations/sql/0120-mod-audit-log-table.sql
Normal file
11
backend/src/app/migrations/sql/0120-mod-audit-log-table.sql
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
CREATE TABLE new_audit_log (LIKE audit_log INCLUDING ALL);
|
||||
INSERT INTO new_audit_log SELECT * FROM audit_log;
|
||||
ALTER TABLE audit_log RENAME TO old_audit_log;
|
||||
ALTER TABLE new_audit_log RENAME TO audit_log;
|
||||
DROP TABLE old_audit_log;
|
||||
|
||||
DROP INDEX new_audit_log_id_archived_at_idx;
|
||||
ALTER TABLE audit_log DROP CONSTRAINT new_audit_log_pkey;
|
||||
ALTER TABLE audit_log ADD PRIMARY KEY (id);
|
||||
ALTER TABLE audit_log ALTER COLUMN created_at SET DEFAULT now();
|
||||
ALTER TABLE audit_log ALTER COLUMN tracked_at SET DEFAULT now();
|
||||
|
|
@ -19,7 +19,20 @@
|
|||
[app.rpc.climit :as-alias climit]
|
||||
[app.rpc.doc :as-alias doc]
|
||||
[app.rpc.helpers :as rph]
|
||||
[app.util.services :as sv]))
|
||||
[app.util.services :as sv]
|
||||
[app.util.time :as dt]))
|
||||
|
||||
(def ^:private event-columns
|
||||
[:id
|
||||
:name
|
||||
:source
|
||||
:type
|
||||
:tracked-at
|
||||
:created-at
|
||||
:profile-id
|
||||
:ip-addr
|
||||
:props
|
||||
:context])
|
||||
|
||||
(defn- event->row [event]
|
||||
[(uuid/next)
|
||||
|
|
@ -27,24 +40,38 @@
|
|||
(:source event)
|
||||
(:type event)
|
||||
(:timestamp event)
|
||||
(:created-at event)
|
||||
(:profile-id event)
|
||||
(db/inet (:ip-addr event))
|
||||
(db/tjson (:props event))
|
||||
(db/tjson (d/without-nils (:context event)))])
|
||||
|
||||
(def ^:private event-columns
|
||||
[:id :name :source :type :tracked-at
|
||||
:profile-id :ip-addr :props :context])
|
||||
(defn- adjust-timestamp
|
||||
[{:keys [timestamp created-at] :as event}]
|
||||
(let [margin (inst-ms (dt/diff timestamp created-at))]
|
||||
(if (or (neg? margin)
|
||||
(> margin 3600000))
|
||||
;; If event is in future or lags more than 1 hour, we reasign
|
||||
;; timestamp to the server creation date
|
||||
(-> event
|
||||
(assoc :timestamp created-at)
|
||||
(update :context assoc :original-timestamp timestamp))
|
||||
event)))
|
||||
|
||||
(defn- handle-events
|
||||
[{:keys [::db/pool]} {:keys [::rpc/profile-id events] :as params}]
|
||||
(let [request (-> params meta ::http/request)
|
||||
ip-addr (audit/parse-client-ip request)
|
||||
tnow (dt/now)
|
||||
xform (comp
|
||||
(map #(assoc % :profile-id profile-id))
|
||||
(map #(assoc % :ip-addr ip-addr))
|
||||
(map #(assoc % :source "frontend"))
|
||||
(map (fn [event]
|
||||
(-> event
|
||||
(assoc :created-at tnow)
|
||||
(assoc :profile-id profile-id)
|
||||
(assoc :ip-addr ip-addr)
|
||||
(assoc :source "frontend"))))
|
||||
(filter :profile-id)
|
||||
(map adjust-timestamp)
|
||||
(map event->row))
|
||||
events (sequence xform events)]
|
||||
(when (seq events)
|
||||
|
|
|
|||
|
|
@ -22,13 +22,182 @@
|
|||
[promesa.exec :as px]))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; TASK ENTRY POINT
|
||||
;; IMPL
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(declare get-stats)
|
||||
(declare send!)
|
||||
(declare get-subscriptions-newsletter-updates)
|
||||
(declare get-subscriptions-newsletter-news)
|
||||
(defn- send!
|
||||
[cfg data]
|
||||
(let [request {:method :post
|
||||
:uri (cf/get :telemetry-uri)
|
||||
:headers {"content-type" "application/json"}
|
||||
:body (json/encode-str data)}
|
||||
response (http/req! cfg request)]
|
||||
(when (> (:status response) 206)
|
||||
(ex/raise :type :internal
|
||||
:code :invalid-response
|
||||
:response-status (:status response)
|
||||
:response-body (:body response)))))
|
||||
|
||||
(defn- get-subscriptions-newsletter-updates
|
||||
[conn]
|
||||
(let [sql "SELECT email FROM profile where props->>'~:newsletter-updates' = 'true'"]
|
||||
(->> (db/exec! conn [sql])
|
||||
(mapv :email))))
|
||||
|
||||
(defn- get-subscriptions-newsletter-news
|
||||
[conn]
|
||||
(let [sql "SELECT email FROM profile where props->>'~:newsletter-news' = 'true'"]
|
||||
(->> (db/exec! conn [sql])
|
||||
(mapv :email))))
|
||||
|
||||
(defn- get-num-teams
|
||||
[conn]
|
||||
(-> (db/exec-one! conn ["SELECT count(*) AS count FROM team"]) :count))
|
||||
|
||||
(defn- get-num-projects
|
||||
[conn]
|
||||
(-> (db/exec-one! conn ["SELECT count(*) AS count FROM project"]) :count))
|
||||
|
||||
(defn- get-num-files
|
||||
[conn]
|
||||
(-> (db/exec-one! conn ["SELECT count(*) AS count FROM file"]) :count))
|
||||
|
||||
(defn- get-num-file-changes
|
||||
[conn]
|
||||
(let [sql (str "SELECT count(*) AS count "
|
||||
" FROM file_change "
|
||||
" where date_trunc('day', created_at) = date_trunc('day', now())")]
|
||||
(-> (db/exec-one! conn [sql]) :count)))
|
||||
|
||||
(defn- get-num-touched-files
|
||||
[conn]
|
||||
(let [sql (str "SELECT count(distinct file_id) AS count "
|
||||
" FROM file_change "
|
||||
" where date_trunc('day', created_at) = date_trunc('day', now())")]
|
||||
(-> (db/exec-one! conn [sql]) :count)))
|
||||
|
||||
(defn- get-num-users
|
||||
[conn]
|
||||
(-> (db/exec-one! conn ["SELECT count(*) AS count FROM profile"]) :count))
|
||||
|
||||
(defn- get-num-fonts
|
||||
[conn]
|
||||
(-> (db/exec-one! conn ["SELECT count(*) AS count FROM team_font_variant"]) :count))
|
||||
|
||||
(defn- get-num-comments
|
||||
[conn]
|
||||
(-> (db/exec-one! conn ["SELECT count(*) AS count FROM comment"]) :count))
|
||||
|
||||
(def sql:team-averages
|
||||
"with projects_by_team AS (
|
||||
SELECT t.id, count(p.id) AS num_projects
|
||||
FROM team AS t
|
||||
LEFT JOIN project AS p ON (p.team_id = t.id)
|
||||
GROUP BY 1
|
||||
), files_by_project AS (
|
||||
SELECT p.id, count(f.id) AS num_files
|
||||
FROM project AS p
|
||||
LEFT JOIN file AS f ON (f.project_id = p.id)
|
||||
GROUP BY 1
|
||||
), comment_threads_by_file AS (
|
||||
SELECT f.id, count(ct.id) AS num_comment_threads
|
||||
FROM file AS f
|
||||
LEFT JOIN comment_thread AS ct ON (ct.file_id = f.id)
|
||||
GROUP BY 1
|
||||
), users_by_team AS (
|
||||
SELECT t.id, count(tp.profile_id) AS num_users
|
||||
FROM team AS t
|
||||
LEFT JOIN team_profile_rel AS tp ON(tp.team_id = t.id)
|
||||
GROUP BY 1
|
||||
)
|
||||
SELECT (SELECT avg(num_projects)::integer FROM projects_by_team) AS avg_projects_on_team,
|
||||
(SELECT max(num_projects)::integer FROM projects_by_team) AS max_projects_on_team,
|
||||
(SELECT avg(num_files)::integer FROM files_by_project) AS avg_files_on_project,
|
||||
(SELECT max(num_files)::integer FROM files_by_project) AS max_files_on_project,
|
||||
(SELECT avg(num_comment_threads)::integer FROM comment_threads_by_file) AS avg_comment_threads_on_file,
|
||||
(SELECT max(num_comment_threads)::integer FROM comment_threads_by_file) AS max_comment_threads_on_file,
|
||||
(SELECT avg(num_users)::integer FROM users_by_team) AS avg_users_on_team,
|
||||
(SELECT max(num_users)::integer FROM users_by_team) AS max_users_on_team")
|
||||
|
||||
(defn- get-team-averages
|
||||
[conn]
|
||||
(->> [sql:team-averages]
|
||||
(db/exec-one! conn)))
|
||||
|
||||
(defn- get-enabled-auth-providers
|
||||
[conn]
|
||||
(let [sql (str "SELECT auth_backend AS backend, count(*) AS total "
|
||||
" FROM profile GROUP BY 1")
|
||||
rows (db/exec! conn [sql])]
|
||||
(->> rows
|
||||
(map (fn [{:keys [backend total]}]
|
||||
(let [backend (or backend "penpot")]
|
||||
[(keyword (str "auth-backend-" backend))
|
||||
total])))
|
||||
(into {}))))
|
||||
|
||||
(defn- get-jvm-stats
|
||||
[]
|
||||
(let [^Runtime runtime (Runtime/getRuntime)]
|
||||
{:jvm-heap-current (.totalMemory runtime)
|
||||
:jvm-heap-max (.maxMemory runtime)
|
||||
:jvm-cpus (.availableProcessors runtime)
|
||||
:os-arch (System/getProperty "os.arch")
|
||||
:os-name (System/getProperty "os.name")
|
||||
:os-version (System/getProperty "os.version")
|
||||
:user-tz (System/getProperty "user.timezone")}))
|
||||
|
||||
(def ^:private sql:get-counters
|
||||
"SELECT name, count(*) AS count
|
||||
FROM audit_log
|
||||
WHERE source = 'backend'
|
||||
AND tracked_at >= date_trunc('day', now())
|
||||
GROUP BY 1
|
||||
ORDER BY 2 DESC")
|
||||
|
||||
(defn- get-action-counters
|
||||
[conn]
|
||||
(let [counters (->> (db/exec! conn [sql:get-counters])
|
||||
(d/index-by (comp keyword :name) :count))
|
||||
total (reduce + 0 (vals counters))]
|
||||
{:total-accomulated-events total
|
||||
:event-counters counters}))
|
||||
|
||||
(def ^:private sql:clean-counters
|
||||
"DELETE FROM audit_log
|
||||
WHERE ip_addr = '0.0.0.0'::inet -- we know this is from telemetry
|
||||
AND tracked_at < (date_trunc('day', now()) - '1 day'::interval)")
|
||||
|
||||
(defn- clean-counters-data!
|
||||
[conn]
|
||||
(when-not (contains? cf/flags :audit-log)
|
||||
(db/exec-one! conn [sql:clean-counters])))
|
||||
|
||||
(defn- get-stats
|
||||
[conn]
|
||||
(let [referer (if (cf/get :telemetry-with-taiga)
|
||||
"taiga"
|
||||
(cf/get :telemetry-referer))]
|
||||
(-> {:referer referer
|
||||
:public-uri (cf/get :public-uri)
|
||||
:total-teams (get-num-teams conn)
|
||||
:total-projects (get-num-projects conn)
|
||||
:total-files (get-num-files conn)
|
||||
:total-users (get-num-users conn)
|
||||
:total-fonts (get-num-fonts conn)
|
||||
:total-comments (get-num-comments conn)
|
||||
:total-file-changes (get-num-file-changes conn)
|
||||
:total-touched-files (get-num-touched-files conn)}
|
||||
(merge
|
||||
(get-team-averages conn)
|
||||
(get-jvm-stats)
|
||||
(get-enabled-auth-providers conn)
|
||||
(get-action-counters conn))
|
||||
(d/without-nils))))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; TASK ENTRY POINT
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defmethod ig/pre-init-spec ::handler [_]
|
||||
(s/keys :req [::http/client
|
||||
|
|
@ -48,6 +217,10 @@
|
|||
data {:subscriptions subs
|
||||
:version (:full cf/version)
|
||||
:instance-id (:instance-id props)}]
|
||||
|
||||
(when enabled?
|
||||
(clean-counters-data! pool))
|
||||
|
||||
(cond
|
||||
;; If we have telemetry enabled, then proceed the normal
|
||||
;; operation.
|
||||
|
|
@ -63,7 +236,8 @@
|
|||
;; onboarding dialog or the profile section, then proceed to
|
||||
;; send a limited telemetry data, that consists in the list of
|
||||
;; subscribed emails and the running penpot version.
|
||||
(seq subs)
|
||||
(or (seq (:newsletter-updates subs))
|
||||
(seq (:newsletter-news subs)))
|
||||
(do
|
||||
(when send?
|
||||
(px/sleep (rand-int 10000))
|
||||
|
|
@ -72,151 +246,3 @@
|
|||
|
||||
:else
|
||||
data))))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; IMPL
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defn- send!
|
||||
[cfg data]
|
||||
(let [request {:method :post
|
||||
:uri (cf/get :telemetry-uri)
|
||||
:headers {"content-type" "application/json"}
|
||||
:body (json/encode-str data)}
|
||||
response (http/req! cfg request)]
|
||||
(when (> (:status response) 206)
|
||||
(ex/raise :type :internal
|
||||
:code :invalid-response
|
||||
:response-status (:status response)
|
||||
:response-body (:body response)))))
|
||||
|
||||
(defn- get-subscriptions-newsletter-updates
|
||||
[conn]
|
||||
(let [sql "select email from profile where props->>'~:newsletter-updates' = 'true'"]
|
||||
(->> (db/exec! conn [sql])
|
||||
(mapv :email))))
|
||||
|
||||
(defn- get-subscriptions-newsletter-news
|
||||
[conn]
|
||||
(let [sql "select email from profile where props->>'~:newsletter-news' = 'true'"]
|
||||
(->> (db/exec! conn [sql])
|
||||
(mapv :email))))
|
||||
|
||||
(defn- retrieve-num-teams
|
||||
[conn]
|
||||
(-> (db/exec-one! conn ["select count(*) as count from team;"]) :count))
|
||||
|
||||
(defn- retrieve-num-projects
|
||||
[conn]
|
||||
(-> (db/exec-one! conn ["select count(*) as count from project;"]) :count))
|
||||
|
||||
(defn- retrieve-num-files
|
||||
[conn]
|
||||
(-> (db/exec-one! conn ["select count(*) as count from file;"]) :count))
|
||||
|
||||
(defn- retrieve-num-file-changes
|
||||
[conn]
|
||||
(let [sql (str "select count(*) as count "
|
||||
" from file_change "
|
||||
" where date_trunc('day', created_at) = date_trunc('day', now())")]
|
||||
(-> (db/exec-one! conn [sql]) :count)))
|
||||
|
||||
(defn- retrieve-num-touched-files
|
||||
[conn]
|
||||
(let [sql (str "select count(distinct file_id) as count "
|
||||
" from file_change "
|
||||
" where date_trunc('day', created_at) = date_trunc('day', now())")]
|
||||
(-> (db/exec-one! conn [sql]) :count)))
|
||||
|
||||
(defn- retrieve-num-users
|
||||
[conn]
|
||||
(-> (db/exec-one! conn ["select count(*) as count from profile;"]) :count))
|
||||
|
||||
(defn- retrieve-num-fonts
|
||||
[conn]
|
||||
(-> (db/exec-one! conn ["select count(*) as count from team_font_variant;"]) :count))
|
||||
|
||||
(defn- retrieve-num-comments
|
||||
[conn]
|
||||
(-> (db/exec-one! conn ["select count(*) as count from comment;"]) :count))
|
||||
|
||||
(def sql:team-averages
|
||||
"with projects_by_team as (
|
||||
select t.id, count(p.id) as num_projects
|
||||
from team as t
|
||||
left join project as p on (p.team_id = t.id)
|
||||
group by 1
|
||||
), files_by_project as (
|
||||
select p.id, count(f.id) as num_files
|
||||
from project as p
|
||||
left join file as f on (f.project_id = p.id)
|
||||
group by 1
|
||||
), comment_threads_by_file as (
|
||||
select f.id, count(ct.id) as num_comment_threads
|
||||
from file as f
|
||||
left join comment_thread as ct on (ct.file_id = f.id)
|
||||
group by 1
|
||||
), users_by_team as (
|
||||
select t.id, count(tp.profile_id) as num_users
|
||||
from team as t
|
||||
left join team_profile_rel as tp on(tp.team_id = t.id)
|
||||
group by 1
|
||||
)
|
||||
select (select avg(num_projects)::integer from projects_by_team) as avg_projects_on_team,
|
||||
(select max(num_projects)::integer from projects_by_team) as max_projects_on_team,
|
||||
(select avg(num_files)::integer from files_by_project) as avg_files_on_project,
|
||||
(select max(num_files)::integer from files_by_project) as max_files_on_project,
|
||||
(select avg(num_comment_threads)::integer from comment_threads_by_file) as avg_comment_threads_on_file,
|
||||
(select max(num_comment_threads)::integer from comment_threads_by_file) as max_comment_threads_on_file,
|
||||
(select avg(num_users)::integer from users_by_team) as avg_users_on_team,
|
||||
(select max(num_users)::integer from users_by_team) as max_users_on_team;")
|
||||
|
||||
(defn- retrieve-team-averages
|
||||
[conn]
|
||||
(->> [sql:team-averages]
|
||||
(db/exec-one! conn)))
|
||||
|
||||
(defn- retrieve-enabled-auth-providers
|
||||
[conn]
|
||||
(let [sql (str "select auth_backend as backend, count(*) as total "
|
||||
" from profile group by 1")
|
||||
rows (db/exec! conn [sql])]
|
||||
(->> rows
|
||||
(map (fn [{:keys [backend total]}]
|
||||
(let [backend (or backend "penpot")]
|
||||
[(keyword (str "auth-backend-" backend))
|
||||
total])))
|
||||
(into {}))))
|
||||
|
||||
(defn- retrieve-jvm-stats
|
||||
[]
|
||||
(let [^Runtime runtime (Runtime/getRuntime)]
|
||||
{:jvm-heap-current (.totalMemory runtime)
|
||||
:jvm-heap-max (.maxMemory runtime)
|
||||
:jvm-cpus (.availableProcessors runtime)
|
||||
:os-arch (System/getProperty "os.arch")
|
||||
:os-name (System/getProperty "os.name")
|
||||
:os-version (System/getProperty "os.version")
|
||||
:user-tz (System/getProperty "user.timezone")}))
|
||||
|
||||
(defn get-stats
|
||||
[conn]
|
||||
(let [referer (if (cf/get :telemetry-with-taiga)
|
||||
"taiga"
|
||||
(cf/get :telemetry-referer))]
|
||||
(-> {:referer referer
|
||||
:public-uri (cf/get :public-uri)
|
||||
:total-teams (retrieve-num-teams conn)
|
||||
:total-projects (retrieve-num-projects conn)
|
||||
:total-files (retrieve-num-files conn)
|
||||
:total-users (retrieve-num-users conn)
|
||||
:total-fonts (retrieve-num-fonts conn)
|
||||
:total-comments (retrieve-num-comments conn)
|
||||
:total-file-changes (retrieve-num-file-changes conn)
|
||||
:total-touched-files (retrieve-num-touched-files conn)}
|
||||
(d/merge
|
||||
(retrieve-team-averages conn)
|
||||
(retrieve-jvm-stats)
|
||||
(retrieve-enabled-auth-providers conn))
|
||||
(d/without-nils))))
|
||||
|
||||
|
|
|
|||
|
|
@ -112,7 +112,7 @@
|
|||
;; "alter table task set unlogged;\n"
|
||||
;; "alter table task_default set unlogged;\n"
|
||||
;; "alter table task_completed set unlogged;\n"
|
||||
"alter table audit_log_default set unlogged ;\n"
|
||||
"alter table audit_log set unlogged ;\n"
|
||||
"alter table storage_object set unlogged;\n"
|
||||
"alter table server_error_report set unlogged;\n"
|
||||
"alter table server_prop set unlogged;\n"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue