From f84d0f34e69aa62ff784f594a3e8c6fdb6ebb7eb Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Fri, 11 Dec 2020 15:43:13 +0100 Subject: [PATCH] :recycle: Minor task naming and directory structure refactor. --- backend/src/app/tasks/clean_tasks_table.clj | 40 ++++++++++++ backend/src/app/tasks/file_media_gc.clj | 9 ++- backend/src/app/tasks/file_xlog_gc.clj | 38 +++++++++++ backend/src/app/tasks/maintenance.clj | 71 --------------------- backend/src/app/worker.clj | 15 ++--- 5 files changed, 93 insertions(+), 80 deletions(-) create mode 100644 backend/src/app/tasks/clean_tasks_table.clj create mode 100644 backend/src/app/tasks/file_xlog_gc.clj delete mode 100644 backend/src/app/tasks/maintenance.clj diff --git a/backend/src/app/tasks/clean_tasks_table.clj b/backend/src/app/tasks/clean_tasks_table.clj new file mode 100644 index 000000000..17e9bfb1e --- /dev/null +++ b/backend/src/app/tasks/clean_tasks_table.clj @@ -0,0 +1,40 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; This Source Code Form is "Incompatible With Secondary Licenses", as +;; defined by the Mozilla Public License, v. 2.0. +;; +;; Copyright (c) 2020 UXBOX Labs SL + +(ns app.tasks.clean-tasks-table + "A maintenance task that performs a cleanup of already executed tasks + from the database table." + (:require + [app.common.spec :as us] + [app.db :as db] + [app.metrics :as mtx] + [app.util.time :as dt] + [clojure.spec.alpha :as s] + [clojure.tools.logging :as log])) + +(def max-age (dt/duration {:hours 24})) + +(def sql:delete-completed-tasks + "delete from task_completed + where scheduled_at < now() - ?::interval") + +(defn handler + [task] + (db/with-atomic [conn db/pool] + (let [interval (db/interval max-age) + result (db/exec-one! conn [sql:delete-completed-tasks interval])] + (log/infof "removed %s rows from tasks_completed table." (:next.jdbc/update-count result)) + nil))) + +(mtx/instrument-with-summary! + {:var #'handler + :id "tasks__clean_tasks_table" + :help "Timing of task: clean_task_table"}) + + diff --git a/backend/src/app/tasks/file_media_gc.clj b/backend/src/app/tasks/file_media_gc.clj index ae8a2274b..23f478e36 100644 --- a/backend/src/app/tasks/file_media_gc.clj +++ b/backend/src/app/tasks/file_media_gc.clj @@ -15,6 +15,7 @@ [app.common.pages.migrations :as pmg] [app.config :as cfg] [app.db :as db] + [app.metrics :as mtx] [app.tasks :as tasks] [app.util.blob :as blob] [app.util.time :as dt] @@ -94,10 +95,16 @@ (defn handler [_task] - (log/debug "running 'file-media-gc' task.") (db/with-atomic [conn db/pool] (loop [] (let [files (retrieve-candidates conn)] (when (seq files) (run! (partial process-file conn) files) (recur)))))) + + +(mtx/instrument-with-summary! + {:var #'handler + :id "tasks__file_media_gc" + :help "Timing of task: file_media_gc"}) + diff --git a/backend/src/app/tasks/file_xlog_gc.clj b/backend/src/app/tasks/file_xlog_gc.clj new file mode 100644 index 000000000..1c689abeb --- /dev/null +++ b/backend/src/app/tasks/file_xlog_gc.clj @@ -0,0 +1,38 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; This Source Code Form is "Incompatible With Secondary Licenses", as +;; defined by the Mozilla Public License, v. 2.0. +;; +;; Copyright (c) 2020 UXBOX Labs SL + +(ns app.tasks.file-xlog-gc + "A maintenance task that performs a garbage collection of the file + change (transaction) log." + (:require + [app.common.spec :as us] + [app.db :as db] + [app.metrics :as mtx] + [app.util.time :as dt] + [clojure.spec.alpha :as s] + [clojure.tools.logging :as log])) + +(def max-age (dt/duration {:hours 12})) + +(def sql:delete-files-xlog + "delete from task_completed + where scheduled_at < now() - ?::interval") + +(defn handler + [{:keys [props] :as task}] + (db/with-atomic [conn db/pool] + (let [interval (db/interval max-age) + result (db/exec-one! conn [sql:delete-files-xlog interval])] + (log/infof "removed %s rows from file_changes table." (:next.jdbc/update-count result)) + nil))) + +(mtx/instrument-with-summary! + {:var #'handler + :id "tasks__file_xlog_gc" + :help "Timing of task: file_xlog_gc"}) diff --git a/backend/src/app/tasks/maintenance.clj b/backend/src/app/tasks/maintenance.clj deleted file mode 100644 index 786bdd6a6..000000000 --- a/backend/src/app/tasks/maintenance.clj +++ /dev/null @@ -1,71 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; This Source Code Form is "Incompatible With Secondary Licenses", as -;; defined by the Mozilla Public License, v. 2.0. -;; -;; Copyright (c) 2020 UXBOX Labs SL - -(ns app.tasks.maintenance - (:require - [app.common.spec :as us] - [app.db :as db] - [app.metrics :as mtx] - [app.util.time :as dt] - [clojure.spec.alpha :as s] - [clojure.tools.logging :as log])) - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; Task: Delete Executed Tasks -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -;; This tasks perform a cleanup of already executed tasks from the -;; database. - -(s/def ::max-age ::dt/duration) -(s/def ::delete-completed-tasks - (s/keys :req-un [::max-age])) - -(def sql:delete-completed-tasks - "delete from task_completed - where scheduled_at < now() - ?::interval") - -(defn delete-executed-tasks - [{:keys [props] :as task}] - (us/verify ::delete-completed-tasks props) - (db/with-atomic [conn db/pool] - (let [max-age (:max-age props) - result (db/exec-one! conn [sql:delete-completed-tasks (db/interval max-age)])] - (log/infof "Removed %s rows from tasks_completed table." (:next.jdbc/update-count result)) - nil))) - -(mtx/instrument-with-summary! - {:var #'delete-executed-tasks - :id "tasks__maintenance__delete_executed_tasks" - :help "Timing of mainentance task function: delete-remove-tasks."}) - - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; Task: Delete old files xlog -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -(s/def ::delete-old-file-xlog - (s/keys :req-un [::max-age])) - -(def sql:delete-files-xlog - "delete from task_completed - where scheduled_at < now() - ?::interval") - -(defn delete-old-files-xlog - [{:keys [props] :as task}] - (db/with-atomic [conn db/pool] - (let [max-age (:max-age props) - result (db/exec-one! conn [sql:delete-files-xlog (db/interval max-age)])] - (log/infof "Removed %s rows from file_changes table." (:next.jdbc/update-count result)) - nil))) - -(mtx/instrument-with-summary! - {:var #'delete-old-files-xlog - :id "tasks__maintenance__delete_old_files_xlog" - :help "Timing of mainentance task function: delete-old-files-xlog."}) diff --git a/backend/src/app/worker.clj b/backend/src/app/worker.clj index 7462ced1e..f90abf192 100644 --- a/backend/src/app/worker.clj +++ b/backend/src/app/worker.clj @@ -13,12 +13,13 @@ [app.common.spec :as us] [app.common.uuid :as uuid] [app.db :as db] + [app.tasks.clean-tasks-table] [app.tasks.delete-object] [app.tasks.delete-profile] - [app.tasks.maintenance] + [app.tasks.file-media-gc] + [app.tasks.file-xlog-gc] [app.tasks.remove-media] [app.tasks.sendmail] - [app.tasks.file-media-gc] [app.util.async :as aa] [app.util.time :as dt] [clojure.core.async :as a] @@ -56,15 +57,13 @@ :cron #app/cron "0 0 0 */1 * ? *" ;; daily :fn #'app.tasks.file-media-gc/handler} - {:id "maintenance/delete-executed-tasks" + {:id "file-xlog-gc" :cron #app/cron "0 0 0 */1 * ?" ;; daily - :fn #'app.tasks.maintenance/delete-executed-tasks - :props {:max-age #app/duration "24h"}} + :fn #'app.tasks.file-xlog-gc/handler} - {:id "maintenance/delete-old-files-xlog" + {:id "clean-tasks-table" :cron #app/cron "0 0 0 */1 * ?" ;; daily - :fn #'app.tasks.maintenance/delete-old-files-xlog - :props {:max-age #app/duration "12h"}} + :fn #'app.tasks.clean-tasks-table/handler} ]) (defstate executor