From ce61b783fb4bb86a534d340b4f7f1b3c64b2ca24 Mon Sep 17 00:00:00 2001
From: Andrey Antukh <niwi@niwi.nz>
Date: Tue, 15 Feb 2022 11:06:13 +0100
Subject: [PATCH] :sparkles: Minor improvements on telemetry task

---
 backend/src/app/main.clj            |  4 ++--
 backend/src/app/tasks/telemetry.clj | 36 +++++++++++++++++++++++++----
 backend/src/app/util/time.clj       |  5 ++++
 3 files changed, 39 insertions(+), 6 deletions(-)

diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj
index 2264e96ba..90bf86016 100644
--- a/backend/src/app/main.clj
+++ b/backend/src/app/main.clj
@@ -176,7 +176,7 @@
         :task :file-offload})
 
      (when (contains? cf/flags :audit-log-archive)
-       {:cron #app/cron "0 */3 * * * ?" ;; every 3m
+       {:cron #app/cron "0 */5 * * * ?" ;; every 5m
         :task :audit-log-archive})
 
      (when (contains? cf/flags :audit-log-gc)
@@ -185,7 +185,7 @@
 
      (when (or (contains? cf/flags :telemetry)
                (cf/get :telemetry-enabled))
-       {:cron #app/cron "0 0 */6 * * ?" ;; every 6h
+       {:cron #app/cron "0 30 */3,23 * * ?"
         :task :telemetry})]}
 
    :app.worker/registry
diff --git a/backend/src/app/tasks/telemetry.clj b/backend/src/app/tasks/telemetry.clj
index 7f24fe7c0..812e36f97 100644
--- a/backend/src/app/tasks/telemetry.clj
+++ b/backend/src/app/tasks/telemetry.clj
@@ -14,12 +14,17 @@
    [app.common.spec :as us]
    [app.config :as cfg]
    [app.db :as db]
+   [app.util.async :refer [thread-sleep]]
    [app.util.http :as http]
    [app.util.json :as json]
    [clojure.spec.alpha :as s]
    [integrant.core :as ig]))
 
-(declare retrieve-stats)
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; TASK ENTRY POINT
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(declare get-stats)
 (declare send!)
 
 (s/def ::version ::us/string)
@@ -34,11 +39,18 @@
 (defmethod ig/init-key ::handler
   [_ {:keys [pool sprops version] :as cfg}]
   (fn [_]
+    ;; Sleep randomly between 0 to 10s
+    (thread-sleep (rand-int 10000))
+
     (let [instance-id (:instance-id sprops)]
-      (-> (retrieve-stats pool version)
+      (-> (get-stats pool version)
           (assoc :instance-id instance-id)
           (send! cfg)))))
 
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; IMPL
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
 (defn- send!
   [data cfg]
   (let [response (http/send! {:method :post
@@ -63,6 +75,20 @@
   [conn]
   (-> (db/exec-one! conn ["select count(*) as count from file;"]) :count))
 
+(defn- retrieve-num-file-changes
+  [conn]
+  (let [sql (str "select count(*) as count "
+                 "  from file_change "
+                 " where date_trunc('day', created_at) = date_trunc('day', now())")]
+    (-> (db/exec-one! conn [sql]) :count)))
+
+(defn- retrieve-num-touched-files
+  [conn]
+  (let [sql (str "select count(distinct file_id) as count "
+                 "  from file_change "
+                 " where date_trunc('day', created_at) = date_trunc('day', now())")]
+    (-> (db/exec-one! conn [sql]) :count)))
+
 (defn- retrieve-num-users
   [conn]
   (-> (db/exec-one! conn ["select count(*) as count from profile;"]) :count))
@@ -118,7 +144,7 @@
      :jvm-heap-max     (.maxMemory runtime)
      :jvm-cpus         (.availableProcessors runtime)}))
 
-(defn retrieve-stats
+(defn get-stats
   [conn version]
   (let [referer (if (cfg/get :telemetry-with-taiga)
                   "taiga"
@@ -130,7 +156,9 @@
          :total-files    (retrieve-num-files conn)
          :total-users    (retrieve-num-users conn)
          :total-fonts    (retrieve-num-fonts conn)
-         :total-comments (retrieve-num-comments conn)}
+         :total-comments (retrieve-num-comments conn)
+         :total-file-changes  (retrieve-num-file-changes conn)
+         :total-touched-files (retrieve-num-touched-files conn)}
         (d/merge
          (retrieve-team-averages conn)
          (retrieve-jvm-stats))
diff --git a/backend/src/app/util/time.clj b/backend/src/app/util/time.clj
index a3bd8c27b..7adeabf54 100644
--- a/backend/src/app/util/time.clj
+++ b/backend/src/app/util/time.clj
@@ -295,6 +295,11 @@
   (s/assert cron? cron)
   (.toInstant (.getNextValidTimeAfter cron (Date/from now))))
 
+(defn get-next
+  [cron tnow]
+  (let [nt (next-valid-instant-from cron tnow)]
+    (cons nt (lazy-seq (get-next cron nt)))))
+
 (defmethod print-method CronExpression
   [mv ^java.io.Writer writer]
   (.write writer (str "#app/cron \"" (.toString ^CronExpression mv) "\"")))