From 3dd22fd298db1e2b0fae53cb9c69bba2535de22d Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Sun, 31 Jan 2021 20:55:53 +0100 Subject: [PATCH] :tada: Add tests for file-media-gc task. --- backend/src/app/main.clj | 3 +- backend/src/app/tasks/file_media_gc.clj | 21 ++-- backend/tests/app/tests/helpers.clj | 4 +- .../tests/app/tests/test_services_files.clj | 109 +++++++++++++++++- 4 files changed, 121 insertions(+), 16 deletions(-) diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index 1b86832c5..4d16ac60c 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -255,8 +255,7 @@ :app.tasks.file-media-gc/handler {:pool (ig/ref :app.db/pool) :metrics (ig/ref :app.metrics/metrics) - :storage (ig/ref :app.storage/storage) - :max-age (dt/duration {:hours 72})} + :max-age (dt/duration {:hours 48})} :app.tasks.file-xlog-gc/handler {:pool (ig/ref :app.db/pool) diff --git a/backend/src/app/tasks/file_media_gc.clj b/backend/src/app/tasks/file_media_gc.clj index 2e77a6e54..dd9939e0f 100644 --- a/backend/src/app/tasks/file_media_gc.clj +++ b/backend/src/app/tasks/file_media_gc.clj @@ -25,10 +25,10 @@ (declare process-file) (declare retrieve-candidates) -(s/def ::storage some?) +(s/def ::max-age ::dt/duration) (defmethod ig/pre-init-spec ::handler [_] - (s/keys :req-un [::db/pool ::storage])) + (s/keys :req-un [::db/pool ::mtx/metrics ::max-age])) (defmethod ig/init-key ::handler [_ {:keys [metrics] :as cfg}] @@ -43,11 +43,15 @@ [{:keys [pool] :as cfg} _] (db/with-atomic [conn pool] (let [cfg (assoc cfg :conn conn)] - (loop [] + (loop [n 0] (let [files (retrieve-candidates cfg)] - (when files - (run! (partial process-file cfg) files) - (recur))))))) + (if (seq files) + (do + (run! (partial process-file cfg) files) + (recur (+ n (count files)))) + (do + (log/infof "finalized with total of %s processed files" n) + {:processed n}))))))) (def ^:private sql:retrieve-candidates-chunk @@ -65,9 +69,8 @@ [{:keys [conn max-age] :as cfg}] (let [interval (db/interval max-age)] (->> (db/exec! conn [sql:retrieve-candidates-chunk interval]) - (map (fn [{:keys [age] :as row}] - (assoc row :age (dt/duration {:seconds age})))) - (seq)))) + (mapv (fn [{:keys [age] :as row}] + (assoc row :age (dt/duration {:seconds age}))))))) (def ^:private collect-media-xf diff --git a/backend/tests/app/tests/helpers.clj b/backend/tests/app/tests/helpers.clj index e4381587d..1f3f78d4b 100644 --- a/backend/tests/app/tests/helpers.clj +++ b/backend/tests/app/tests/helpers.clj @@ -23,6 +23,7 @@ [app.rpc.mutations.projects :as projects] [app.rpc.mutations.teams :as teams] [app.util.blob :as blob] + [app.util.time :as dt] [clojure.java.io :as io] [clojure.spec.alpha :as s] [cuerdas.core :as str] @@ -48,7 +49,8 @@ :app.worker/scheduler :app.worker/worker) (d/deep-merge - {:app.storage/storage {:backend :tmp}})) + {:app.storage/storage {:backend :tmp} + :app.tasks.file-media-gc/handler {:max-age (dt/duration 300)}})) _ (ig/load-namespaces config) system (-> (ig/prep config) (ig/init))] diff --git a/backend/tests/app/tests/test_services_files.clj b/backend/tests/app/tests/test_services_files.clj index 2fa54bd67..bfd87f287 100644 --- a/backend/tests/app/tests/test_services_files.clj +++ b/backend/tests/app/tests/test_services_files.clj @@ -9,18 +9,19 @@ (ns app.tests.test-services-files (:require - [clojure.test :as t] - [datoteka.core :as fs] [app.common.uuid :as uuid] [app.db :as db] [app.http :as http] - [app.tests.helpers :as th])) + [app.storage :as sto] + [app.tests.helpers :as th] + [clojure.test :as t] + [datoteka.core :as fs])) (t/use-fixtures :once th/state-init) (t/use-fixtures :each th/database-reset) (t/deftest files-crud - (let [prof (th/create-profile th/*pool* 1) + (let [prof (th/create-profile* 1 {:is-active true}) team-id (:default-team-id prof) proj-id (:default-project-id prof) file-id (uuid/next) @@ -118,3 +119,103 @@ (let [result (:result out)] (t/is (= 0 (count result)))))) )) + +(defn- create-file-media-object + [{:keys [profile-id file-id]}] + (let [mfile {:filename "sample.jpg" + :tempfile (th/tempfile "app/tests/_files/sample.jpg") + :content-type "image/jpeg" + :size 312043} + params {::th/type :upload-file-media-object + :profile-id profile-id + :file-id file-id + :is-local true + :name "testfile" + :content mfile} + out (th/mutation! params)] + (t/is (nil? (:error out))) + (:result out))) + +(defn- update-file + [{:keys [profile-id file-id changes revn] :or {revn 0}}] + (let [params {::th/type :update-file + :id file-id + :session-id (uuid/random) + :profile-id profile-id + :revn revn + :changes changes} + out (th/mutation! params)] + (t/is (nil? (:error out))) + (:result out))) + +(t/deftest file-media-gc-task + (let [task (:app.tasks.file-media-gc/handler th/*system*) + storage (:app.storage/storage th/*system*) + + prof (th/create-profile* 1) + proj (th/create-project* 1 {:profile-id (:id prof) + :team-id (:default-team-id prof)}) + file (th/create-file* 1 {:profile-id (:id prof) + :project-id (:default-project-id prof) + :is-shared false}) + + fmo1 (create-file-media-object {:profile-id (:id prof) + :file-id (:id file)}) + fmo2 (create-file-media-object {:profile-id (:id prof) + :file-id (:id file)}) + shid (uuid/random) + + ures (update-file + {:file-id (:id file) + :profile-id (:id prof) + :revn 0 + :changes + [{:type :add-obj + :page-id (first (get-in file [:data :pages])) + :id shid + :parent-id uuid/zero + :frame-id uuid/zero + :obj {:id shid + :name "image" + :frame-id uuid/zero + :parent-id uuid/zero + :type :image + :metadata {:id (:id fmo1)}}}]})] + + ;; run the task inmediatelly + (let [res (task {})] + (t/is (= 0 (:processed res)))) + + ;; make the file ellegible for GC waiting 300ms + (th/sleep 300) + + ;; run the task again + (let [res (task {})] + (t/is (= 1 (:processed res)))) + + ;; Retrieve file and check trimmed attribute + (let [row (db/exec-one! th/*pool* ["select * from file where id = ?" (:id file)])] + (t/is (:has-media-trimmed row))) + + ;; check file media objects + (let [fmos (db/exec! th/*pool* ["select * from file_media_object where file_id = ?" (:id file)])] + (t/is (= 1 (count fmos)))) + + ;; The underlying storage objects are still available. + (t/is (some? (sto/get-object storage (:media-id fmo2)))) + (t/is (some? (sto/get-object storage (:thumbnail-id fmo2)))) + (t/is (some? (sto/get-object storage (:media-id fmo1)))) + (t/is (some? (sto/get-object storage (:thumbnail-id fmo1)))) + + ;; but if we pass the touched gc task two of them should disappear + (let [task (:app.storage/gc-touched-task th/*system*) + res (task {})] + (t/is (= 0 (:freeze res))) + (t/is (= 2 (:delete res))) + + (t/is (nil? (sto/get-object storage (:media-id fmo2)))) + (t/is (nil? (sto/get-object storage (:thumbnail-id fmo2)))) + (t/is (some? (sto/get-object storage (:media-id fmo1)))) + (t/is (some? (sto/get-object storage (:thumbnail-id fmo1))))) + + ))