Merge remote-tracking branch 'origin/staging' into develop

backend/src/app/main.clj

@@ -40,13 +40,14 @@
    {:parallelism (cf/get :worker-executor-parallelism 10)
     :prefix :worker}
 
-   :app.worker/executors-monitor
+   :app.worker/executors
-   {:executors
+   {:default  (ig/ref [::default :app.worker/executor])
-    {:default  (ig/ref [::default :app.worker/executor])
+    :worker   (ig/ref [::worker :app.worker/executor])
-     :blocking (ig/ref [::blocking :app.worker/executor])
+    :blocking (ig/ref [::blocking :app.worker/executor])}
-     :worker   (ig/ref [::worker :app.worker/executor])}
 
-    :metrics   (ig/ref :app.metrics/metrics)}
+   :app.worker/executors-monitor
+   {:metrics   (ig/ref :app.metrics/metrics)
+    :executors (ig/ref :app.worker/executors)}
 
    :app.migrations/migrations
    {}
@@ -155,9 +156,7 @@
     :msgbus     (ig/ref :app.msgbus/msgbus)
     :public-uri (cf/get :public-uri)
     :audit      (ig/ref :app.loggers.audit/collector)
-    :executors
+    :executors  (ig/ref :app.worker/executors)}
-    {:default  (ig/ref [::default :app.worker/executor])
-     :blocking (ig/ref [::blocking :app.worker/executor])}}
 
    :app.worker/worker
    {:executor (ig/ref [::worker :app.worker/executor])

backend/src/app/metrics.clj

@@ -111,6 +111,12 @@
     :labels ["name"]
     :type :gauge}
 
+   :executors-completed-tasks
+   {:name "penpot_executors_completed_tasks_total"
+    :help "Aproximate number of completed tasks by the executor."
+    :labels ["name"]
+    :type :counter}
+
    :executors-running-threads
    {:name "penpot_executors_running_threads"
     :help "Current number of threads with state RUNNING."

backend/src/app/worker.clj

@@ -75,30 +75,40 @@
   (s/keys :req-un [::executors ::mtx/metrics]))
 
 (defmethod ig/init-key ::executors-monitor
-  [_ {:keys [executors metrics interval] :or {interval 2500}}]
+  [_ {:keys [executors metrics interval] :or {interval 3000}}]
-  (letfn [(log-stats [scheduler]
+  (letfn [(log-stats [scheduler state]
             (doseq [[key ^ForkJoinPool executor] executors]
-              (let [labels (into-array String [(name key)])]
+              (let [labels  (into-array String [(name key)])
-                (mtx/run! metrics {:id :executors-active-threads
+                    active  (.getActiveThreadCount executor)
-                                   :labels labels
+                    running (.getRunningThreadCount executor)
-                                   :val (.getPoolSize executor)})
+                    queued  (.getQueuedSubmissionCount executor)
-                (mtx/run! metrics {:id :executors-running-threads
+                    steals  (.getStealCount executor)
-                                   :labels labels
+                    steals-increment (- steals (or (get-in @state [key :steals]) 9))
-                                   :val (.getRunningThreadCount executor)})
+                    steals-increment (if (neg? steals-increment) 0 steals-increment)]
-                (mtx/run! metrics {:id :executors-queued-submissions
+
-                                   :labels labels
+                (mtx/run! metrics {:id :executors-active-threads :labels labels :val active})
-                                   :val (.getQueuedSubmissionCount executor)})))
+                (mtx/run! metrics {:id :executors-running-threads :labels labels :val running})
+                (mtx/run! metrics {:id :executors-queued-submissions :labels labels :val queued})
+                (mtx/run! metrics {:id :executors-completed-tasks :labels labels :inc steals-increment})
+
+                (swap! state update key assoc
+                       :running running
+                       :active active
+                       :queued queued
+                       :steals steals)))
 
             (when-not (.isShutdown scheduler)
-              (px/schedule! scheduler interval (partial log-stats scheduler))))]
+              (px/schedule! scheduler interval (partial log-stats scheduler state))))]
 
-    (let [scheduler (px/scheduled-pool 1)]
+    (let [scheduler (px/scheduled-pool 1)
-      (px/schedule! scheduler interval (partial log-stats scheduler))
+          state     (atom {})]
-      scheduler)))
+      (px/schedule! scheduler interval (partial log-stats scheduler state))
+      {::scheduler scheduler
+       ::state state})))
 
 (defmethod ig/halt-key! ::executors-monitor
-  [_ instance]
+  [_ {:keys [::scheduler]}]
-  (.shutdown ^ExecutorService instance))
+  (.shutdown ^ExecutorService scheduler))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Worker

frontend/src/app/util/snap_data.cljs

@@ -63,14 +63,14 @@
                                           :pt %)))
 
         grid-x-data (->> (gg/grid-snap-points frame :x)
-                         (mapv #(array-map :type :grid-x
+                         (mapv #(array-map :type :layout
                                            :id frame-id
                                            :pt %)))
 
         grid-y-data (->> (gg/grid-snap-points frame :y)
-                         (mapv #(array-map :type :grid-y
+                         (mapv #(array-map :type :layout
-                                          :id frame-id
+                                           :id frame-id
-                                          :pt %)))]
+                                           :pt %)))]
 
     (-> page-data
         ;; Update root frame information