🎉 Add tests for file-media-gc task.

backend/src/app/main.clj

@@ -255,8 +255,7 @@
     :app.tasks.file-media-gc/handler
     {:pool    (ig/ref :app.db/pool)
      :metrics (ig/ref :app.metrics/metrics)
-     :storage (ig/ref :app.storage/storage)
-     :max-age (dt/duration {:hours 72})}
+     :max-age (dt/duration {:hours 48})}
 
     :app.tasks.file-xlog-gc/handler
     {:pool    (ig/ref :app.db/pool)

backend/src/app/tasks/file_media_gc.clj

@@ -25,10 +25,10 @@
 (declare process-file)
 (declare retrieve-candidates)
 
-(s/def ::storage some?)
+(s/def ::max-age ::dt/duration)
 
 (defmethod ig/pre-init-spec ::handler [_]
-  (s/keys :req-un [::db/pool ::storage]))
+  (s/keys :req-un [::db/pool ::mtx/metrics ::max-age]))
 
 (defmethod ig/init-key ::handler
   [_ {:keys [metrics] :as cfg}]
@@ -43,11 +43,15 @@
   [{:keys [pool] :as cfg} _]
   (db/with-atomic [conn pool]
     (let [cfg (assoc cfg :conn conn)]
-      (loop []
+      (loop [n 0]
         (let [files (retrieve-candidates cfg)]
-          (when files
-            (run! (partial process-file cfg) files)
-            (recur)))))))
+          (if (seq files)
+            (do
+              (run! (partial process-file cfg) files)
+              (recur (+ n (count files))))
+            (do
+              (log/infof "finalized with total of %s processed files" n)
+              {:processed n})))))))
 
 (def ^:private
   sql:retrieve-candidates-chunk
@@ -65,9 +69,8 @@
   [{:keys [conn max-age] :as cfg}]
   (let [interval (db/interval max-age)]
     (->> (db/exec! conn [sql:retrieve-candidates-chunk interval])
-         (map (fn [{:keys [age] :as row}]
-                (assoc row :age (dt/duration {:seconds age}))))
-         (seq))))
+         (mapv (fn [{:keys [age] :as row}]
+                 (assoc row :age (dt/duration {:seconds age})))))))
 
 (def ^:private
   collect-media-xf

backend/tests/app/tests/helpers.clj

@@ -23,6 +23,7 @@
    [app.rpc.mutations.projects :as projects]
    [app.rpc.mutations.teams :as teams]
    [app.util.blob :as blob]
+   [app.util.time :as dt]
    [clojure.java.io :as io]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
@@ -48,7 +49,8 @@
                            :app.worker/scheduler
                            :app.worker/worker)
                    (d/deep-merge
-                    {:app.storage/storage {:backend :tmp}}))
+                    {:app.storage/storage {:backend :tmp}
+                     :app.tasks.file-media-gc/handler {:max-age (dt/duration 300)}}))
         _      (ig/load-namespaces config)
         system (-> (ig/prep config)
                    (ig/init))]

backend/tests/app/tests/test_services_files.clj

@@ -9,18 +9,19 @@
 
 (ns app.tests.test-services-files
   (:require
-   [clojure.test :as t]
-   [datoteka.core :as fs]
    [app.common.uuid :as uuid]
    [app.db :as db]
    [app.http :as http]
-   [app.tests.helpers :as th]))
+   [app.storage :as sto]
+   [app.tests.helpers :as th]
+   [clojure.test :as t]
+   [datoteka.core :as fs]))
 
 (t/use-fixtures :once th/state-init)
 (t/use-fixtures :each th/database-reset)
 
 (t/deftest files-crud
-  (let [prof (th/create-profile th/*pool* 1)
+  (let [prof    (th/create-profile* 1 {:is-active true})
         team-id (:default-team-id prof)
         proj-id (:default-project-id prof)
         file-id (uuid/next)
@@ -118,3 +119,103 @@
         (let [result (:result out)]
           (t/is (= 0 (count result))))))
     ))
+
+(defn- create-file-media-object
+  [{:keys [profile-id file-id]}]
+  (let [mfile  {:filename "sample.jpg"
+                :tempfile (th/tempfile "app/tests/_files/sample.jpg")
+                :content-type "image/jpeg"
+                :size 312043}
+        params {::th/type :upload-file-media-object
+                :profile-id profile-id
+                :file-id file-id
+                :is-local true
+                :name "testfile"
+                :content mfile}
+        out    (th/mutation! params)]
+    (t/is (nil? (:error out)))
+    (:result out)))
+
+(defn- update-file
+  [{:keys [profile-id file-id changes revn] :or {revn 0}}]
+  (let [params {::th/type :update-file
+                :id file-id
+                :session-id (uuid/random)
+                :profile-id profile-id
+                :revn revn
+                :changes changes}
+        out    (th/mutation! params)]
+    (t/is (nil? (:error out)))
+    (:result out)))
+
+(t/deftest file-media-gc-task
+  (let [task    (:app.tasks.file-media-gc/handler th/*system*)
+        storage (:app.storage/storage th/*system*)
+
+        prof    (th/create-profile* 1)
+        proj    (th/create-project* 1 {:profile-id (:id prof)
+                                       :team-id (:default-team-id prof)})
+        file    (th/create-file* 1 {:profile-id (:id prof)
+                                    :project-id (:default-project-id prof)
+                                    :is-shared false})
+
+        fmo1    (create-file-media-object {:profile-id (:id prof)
+                                           :file-id (:id file)})
+        fmo2    (create-file-media-object {:profile-id (:id prof)
+                                           :file-id (:id file)})
+        shid    (uuid/random)
+
+        ures    (update-file
+                 {:file-id (:id file)
+                  :profile-id (:id prof)
+                  :revn 0
+                  :changes
+                  [{:type :add-obj
+                    :page-id (first (get-in file [:data :pages]))
+                    :id shid
+                    :parent-id uuid/zero
+                    :frame-id uuid/zero
+                    :obj {:id shid
+                          :name "image"
+                          :frame-id uuid/zero
+                          :parent-id uuid/zero
+                          :type :image
+                          :metadata {:id (:id fmo1)}}}]})]
+
+    ;; run the task inmediatelly
+    (let [res (task {})]
+      (t/is (= 0 (:processed res))))
+
+    ;; make the file ellegible for GC waiting 300ms
+    (th/sleep 300)
+
+    ;; run the task again
+    (let [res (task {})]
+      (t/is (= 1 (:processed res))))
+
+    ;; Retrieve file and check trimmed attribute
+    (let [row (db/exec-one! th/*pool* ["select * from file where id = ?" (:id file)])]
+      (t/is (:has-media-trimmed row)))
+
+    ;; check file media objects
+    (let [fmos (db/exec! th/*pool* ["select * from file_media_object where file_id = ?" (:id file)])]
+      (t/is (= 1 (count fmos))))
+
+    ;; The underlying storage objects are still available.
+    (t/is (some? (sto/get-object storage (:media-id fmo2))))
+    (t/is (some? (sto/get-object storage (:thumbnail-id fmo2))))
+    (t/is (some? (sto/get-object storage (:media-id fmo1))))
+    (t/is (some? (sto/get-object storage (:thumbnail-id fmo1))))
+
+    ;; but if we pass the touched gc task two of them should disappear
+    (let [task (:app.storage/gc-touched-task th/*system*)
+          res  (task {})]
+      (t/is (= 0 (:freeze res)))
+      (t/is (= 2 (:delete res)))
+
+      (t/is (nil? (sto/get-object storage (:media-id fmo2))))
+      (t/is (nil? (sto/get-object storage (:thumbnail-id fmo2))))
+      (t/is (some? (sto/get-object storage (:media-id fmo1))))
+      (t/is (some? (sto/get-object storage (:thumbnail-id fmo1)))))
+
+    ))