From ff6482fa29eaf421df89d0f0e0dfb6fe0a6edfe1 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 28 Dec 2020 13:03:12 +0100 Subject: [PATCH] :tada: Add telemetry client. --- backend/src/app/config.clj | 7 +- backend/src/app/main.clj | 34 ++++--- backend/src/app/tasks/telemetry.clj | 145 ++++++++++++++++++++++++++++ backend/src/app/telemetry.clj | 19 ++-- backend/src/app/worker.clj | 9 +- 5 files changed, 184 insertions(+), 30 deletions(-) create mode 100644 backend/src/app/tasks/telemetry.clj diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index 68fc961be..d90baa02b 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -44,6 +44,9 @@ :registration-enabled true :registration-domain-whitelist "" + :telemetry-enabled true + :telemetry-uri "http://localhost:6063/" + :debug true ;; This is the time should transcurr after the last page @@ -121,7 +124,7 @@ (s/def ::ldap-auth-avatar-attribute ::us/string) (s/def ::telemetry-enabled ::us/boolean) -(s/def ::telemetry-url ::us/string) +(s/def ::telemetry-uri ::us/string) (s/def ::telemetry-server-enabled ::us/boolean) (s/def ::telemetry-server-port ::us/integer) @@ -158,7 +161,7 @@ ::file-trimming-threshold ::telemetry-enabled ::telemetry-server-enabled - ::telemetry-url + ::telemetry-uri ::telemetry-server-port ::debug ::allow-demo-users diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index ca820d9ad..63158fa89 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -133,18 +133,24 @@ :app.worker/scheduler {:executor (ig/ref :app.worker/executor) :pool (ig/ref :app.db/pool) - :schedule [;; TODO: pending to refactor - ;; {:id "file-media-gc" - ;; :cron #app/cron "0 0 0 */1 * ? *" ;; daily - ;; :fn (ig/ref :app.tasks.file-media-gc/handler)} + :schedule + [;; TODO: pending to refactor + ;; {:id "file-media-gc" + ;; :cron #app/cron "0 0 0 */1 * ? *" ;; daily + ;; :fn (ig/ref :app.tasks.file-media-gc/handler)} - {:id "file-xlog-gc" - :cron #app/cron "0 0 0 */1 * ?" ;; daily - :fn (ig/ref :app.tasks.file-xlog-gc/handler)} + {:id "file-xlog-gc" + :cron #app/cron "0 0 0 */1 * ?" ;; daily + :fn (ig/ref :app.tasks.file-xlog-gc/handler)} - {:id "tasks-gc" - :cron #app/cron "0 0 0 */1 * ?" ;; daily - :fn (ig/ref :app.tasks.tasks-gc/handler)}]} + {:id "tasks-gc" + :cron #app/cron "0 0 0 */1 * ?" ;; daily + :fn (ig/ref :app.tasks.tasks-gc/handler)} + + (when (:telemetry-enabled cfg/config) + {:id "telemetry" + :cron #app/cron "0 0 */3 * * ?" ;; every 3h + :fn (ig/ref :app.tasks.telemetry/handler)})]} :app.tasks/all {"sendmail" (ig/ref :app.tasks.sendmail/handler) @@ -185,13 +191,15 @@ :max-age (dt/duration {:hours 12}) :metrics (ig/ref :app.metrics/metrics)} - ;; :app.tasks.telemetry/handler - ;; {:pool (ig/ref :app.db/pool)} + :app.tasks.telemetry/handler + {:pool (ig/ref :app.db/pool) + :version (:full cfg/version) + :uri (:telemetry-uri cfg/config)} :app.srepl/server {:port 6062}} - (when (:telemetry-server-enabled cfg/config true) + (when (:telemetry-server-enabled cfg/config) {:app.telemetry/handler {:pool (ig/ref :app.db/pool) :executor (ig/ref :app.worker/executor)} diff --git a/backend/src/app/tasks/telemetry.clj b/backend/src/app/tasks/telemetry.clj new file mode 100644 index 000000000..e278a7fe4 --- /dev/null +++ b/backend/src/app/tasks/telemetry.clj @@ -0,0 +1,145 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; This Source Code Form is "Incompatible With Secondary Licenses", as +;; defined by the Mozilla Public License, v. 2.0. +;; +;; Copyright (c) 2020 UXBOX Labs SL + +(ns app.tasks.telemetry + "A task that is reponsible to collect anonymous statistical + information about the current instance and send it to the telemetry + server." + (:require + [app.common.exceptions :as ex] + [app.common.spec :as us] + [app.common.uuid :as uuid] + [app.db :as db] + [app.util.http :as http] + [app.util.json :as json] + [clojure.spec.alpha :as s] + [clojure.tools.logging :as log] + [integrant.core :as ig])) + +(declare handler) +(declare acquire-lock) +(declare release-all-locks) +(declare retrieve-stats) + +(s/def ::version ::us/string) +(s/def ::uri ::us/string) + +(defmethod ig/pre-init-spec ::handler [_] + (s/keys :req-un [::db/pool ::version ::uri])) + +(defmethod ig/init-key ::handler + [_ {:keys [pool] :as cfg}] + (fn [_] + (db/with-atomic [conn pool] + (try + (acquire-lock conn) + (handler (assoc cfg :conn conn)) + (finally + (release-all-locks conn)))))) + +(defn- acquire-lock + [conn] + (db/exec-one! conn ["select pg_advisory_lock(87562985867332);"])) + +(defn- release-all-locks + [conn] + (db/exec-one! conn ["select pg_advisory_unlock_all();"])) + +(defn- get-or-create-instance-id + [{:keys [conn] :as cfg}] + (if-let [result (db/exec-one! conn ["select id from telemetry.instance"])] + (:id result) + (let [result (db/exec-one! conn ["insert into telemetry.instance (id) values (?) returning *" + (uuid/random)])] + (:id result)))) + +(defonce debug {}) + +(defn- handler + [cfg] + (let [instance-id (get-or-create-instance-id cfg) + data (retrieve-stats cfg) + data (assoc data :instance-id instance-id)] + (alter-var-root #'debug (constantly data)) + (let [response (http/send! {:method :post + :uri (:uri cfg) + :headers {"content-type" "application/json"} + :body (json/encode-str data)})] + (when (not= 200 (:status response)) + (ex/raise :type :internal + :code :invalid-response-from-google + :context {:status (:status response) + :body (:body response)}))))) + + +(defn retrieve-num-teams + [conn] + (-> (db/exec-one! conn ["select count(*) as count from team;"]) :count)) + +(defn retrieve-num-projects + [conn] + (-> (db/exec-one! conn ["select count(*) as count from project;"]) :count)) + +(defn retrieve-num-files + [conn] + (-> (db/exec-one! conn ["select count(*) as count from project;"]) :count)) + +(def sql:team-averages + "with projects_by_team as ( + select t.id, count(p.id) as num_projects + from team as t + left join project as p on (p.team_id = t.id) + group by 1 + ), files_by_project as ( + select p.id, count(f.id) as num_files + from project as p + left join file as f on (f.project_id = p.id) + group by 1 + ), comment_threads_by_file as ( + select f.id, count(ct.id) as num_comment_threads + from file as f + left join comment_thread as ct on (ct.file_id = f.id) + group by 1 + ), users_by_team as ( + select t.id, count(tp.profile_id) as num_users + from team as t + left join team_profile_rel as tp on(tp.team_id = t.id) + where t.is_default = false + group by 1 + ) + select (select avg(num_projects)::integer from projects_by_team) as avg_projects_on_team, + (select max(num_projects)::integer from projects_by_team) as max_projects_on_team, + (select avg(num_files)::integer from files_by_project) as avg_files_on_project, + (select max(num_files)::integer from files_by_project) as max_files_on_project, + (select avg(num_comment_threads)::integer from comment_threads_by_file) as avg_comment_threads_on_file, + (select max(num_comment_threads)::integer from comment_threads_by_file) as max_comment_threads_on_file, + (select avg(num_users)::integer from users_by_team) as avg_users_on_team, + (select max(num_users)::integer from users_by_team) as max_users_on_team;") + +(defn retrieve-team-averages + [conn] + (->> [sql:team-averages] + (db/exec-one! conn))) + +(defn retrieve-jvm-stats + [] + (let [^Runtime runtime (Runtime/getRuntime)] + {:jvm-heap-current (.totalMemory runtime) + :jvm-heap-max (.maxMemory runtime) + :jvm-cpus (.availableProcessors runtime)})) + +(defn- retrieve-stats + [{:keys [conn version]}] + (merge + {:version version + :total-teams (retrieve-num-teams conn) + :total-projects (retrieve-num-projects conn) + :total-files (retrieve-num-files conn)} + (retrieve-team-averages conn) + (retrieve-jvm-stats))) diff --git a/backend/src/app/telemetry.clj b/backend/src/app/telemetry.clj index c85fd3cdf..6251f53ae 100644 --- a/backend/src/app/telemetry.clj +++ b/backend/src/app/telemetry.clj @@ -24,6 +24,12 @@ ;; Migrations ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +(def sql:create-instance-table + "CREATE TABLE IF NOT EXISTS telemetry.instance ( + id uuid PRIMARY KEY, + created_at timestamptz NOT NULL DEFAULT now() + );") + (def sql:create-info-table "CREATE TABLE telemetry.info ( instance_id uuid, @@ -38,17 +44,6 @@ ALTER TABLE telemetry.info ATTACH PARTITION telemetry.info_default DEFAULT;") -;; Research on this -;; ALTER TABLE telemetry.instance_info -;; SET (autovacuum_freeze_min_age = 0, -;; autovacuum_freeze_max_age = 100000);") - -(def sql:create-instance-table - "CREATE TABLE IF NOT EXISTS telemetry.instance ( - id uuid PRIMARY KEY, - created_at timestamptz NOT NULL DEFAULT now() - );") - (def migrations [{:name "0001-add-telemetry-schema" :fn #(db/exec! % ["CREATE SCHEMA IF NOT EXISTS telemetry;"])} @@ -98,7 +93,7 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (def sql:insert-instance-info - "insert into telemetry.instance_info (instance_id, data, created_at) + "insert into telemetry.info (instance_id, data, created_at) values (?, ?, date_trunc('day', now())) on conflict (instance_id, created_at) do update set data = ?") diff --git a/backend/src/app/worker.clj b/backend/src/app/worker.clj index ff7a5c97a..a452e9e00 100644 --- a/backend/src/app/worker.clj +++ b/backend/src/app/worker.clj @@ -306,10 +306,15 @@ (defmethod ig/init-key ::scheduler [_ {:keys [executor schedule] :as cfg}] (let [scheduler (Executors/newScheduledThreadPool (int 1)) - cfg (assoc cfg :scheduler scheduler)] + schedule (filter some? schedule) + cfg (assoc cfg + :scheduler scheduler + :schedule schedule)] + (synchronize-schedule cfg) (run! (partial schedule-task cfg) (filter some? schedule)) + (reify java.lang.AutoCloseable (close [_] @@ -339,8 +344,6 @@ (def sql:lock-scheduled-task "select id from scheduled_task where id=? for update skip locked") -(declare schedule-task) - (defn exception->string [error] (with-out-str