✨ Fix all tasks related tests

backend/resources/log4j2-devenv.xml

@@ -25,7 +25,7 @@
     <Logger name="org.postgresql" level="error" />
 
     <Logger name="app.rpc.commands.binfile" level="debug" />
-    <Logger name="app.storage.tmp" level="trace" />
+    <Logger name="app.storage.tmp" level="info" />
     <Logger name="app.worker" level="info" />
     <Logger name="app.msgbus" level="info" />
     <Logger name="app.http.websocket" level="info" />

backend/src/app/main.clj

@@ -253,10 +253,10 @@
       :task :file-xlog-gc}
 
      {:cron #app/cron "0 0 0 * * ?"  ;; daily
-      :task :storage-deleted-gc}
+      :task :storage-gc-deleted}
 
      {:cron #app/cron "0 0 0 * * ?"  ;; daily
-      :task :storage-touched-gc}
+      :task :storage-gc-touched}
 
      {:cron #app/cron "0 0 0 * * ?"  ;; daily
       :task :session-gc}
@@ -285,8 +285,8 @@
      :objects-gc         (ig/ref :app.tasks.objects-gc/handler)
      :file-gc            (ig/ref :app.tasks.file-gc/handler)
      :file-xlog-gc       (ig/ref :app.tasks.file-xlog-gc/handler)
-     :storage-deleted-gc (ig/ref :app.storage/gc-deleted-task)
+     :storage-gc-deleted (ig/ref :app.storage/gc-deleted-task)
-     :storage-touched-gc (ig/ref :app.storage/gc-touched-task)
+     :storage-gc-touched (ig/ref :app.storage/gc-touched-task)
      :tasks-gc           (ig/ref :app.tasks.tasks-gc/handler)
      :telemetry          (ig/ref :app.tasks.telemetry/handler)
      :session-gc         (ig/ref :app.http.session/gc-task)

backend/src/app/tasks/file_gc.clj

@@ -43,9 +43,9 @@
 (defmethod ig/init-key ::handler
   [_ {:keys [pool] :as cfg}]
   (fn [params]
-    (let [cfg (merge cfg params)]
     (db/with-atomic [conn pool]
-        (let [cfg (assoc cfg :conn conn)]
+      (let [min-age (or (:min-age params) (:min-age cfg))
+            cfg     (assoc cfg :min-age min-age :conn conn)]
         (loop [total 0
                files (retrieve-candidates cfg)]
           (if-let [file (first files)]
@@ -54,13 +54,13 @@
               (recur (inc total)
                      (rest files)))
             (do
-                (l/info :hint "task finished" :total total)
+              (l/info :hint "task finished" :min-age (dt/format-duration min-age) :total total)
 
               ;; Allow optional rollback passed by params
               (when (:rollback? params)
                 (db/rollback! conn))
 
-                {:processed total}))))))))
+              {:processed total})))))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; IMPL

backend/test/app/services_files_test.clj

@@ -187,23 +187,18 @@
       ;; freeze because of the deduplication (we have uploaded 2 times
       ;; 2 two same files).
       (let [task (:app.storage/gc-touched-task th/*system*)
-            res  (task {})]
+            res  (task {:min-age (dt/duration 0)})]
-
         (t/is (= 2 (:freeze res)))
         (t/is (= 0 (:delete res))))
 
-      ;; run the task immediately
+      ;; run the file-gc task immediately without forced min-age
       (let [task  (:app.tasks.file-gc/handler th/*system*)
             res   (task {})]
         (t/is (= 0 (:processed res))))
 
-      ;; make the file eligible for GC waiting 300ms (configured
-      ;; timeout for testing)
-      (th/sleep 300)
-
       ;; run the task again
       (let [task  (:app.tasks.file-gc/handler th/*system*)
-            res   (task {})]
+            res   (task {:min-age (dt/duration 0)})]
         (t/is (= 1 (:processed res))))
 
       ;; retrieve file and check trimmed attribute
@@ -220,22 +215,36 @@
       (t/is (some? @(sto/get-object storage (:media-id fmo1))))
       (t/is (some? @(sto/get-object storage (:thumbnail-id fmo1))))
 
-      ;; now, we have deleted the unused file-media-object, if we
+      ;; proceed to remove usage of the file
-      ;; execute the touched-gc task, we should see that two of them
+      (update-file {:file-id (:id file)
-      ;; are marked to be deleted.
+                    :profile-id (:id profile)
+                    :revn 0
+                    :changes [{:type :del-obj
+                               :page-id (first (get-in file [:data :pages]))
+                               :id shid}]})
+
+      ;; Now, we have deleted the usag of pointers to the
+      ;; file-media-objects, if we pase file-gc, they should be marked
+      ;; as deleted.
+      (let [task  (:app.tasks.file-gc/handler th/*system*)
+            res   (task {:min-age (dt/duration 0)})]
+        (t/is (= 1 (:processed res))))
+
+      ;; Now that file-gc have deleted the file-media-object usage,
+      ;; lets execute the touched-gc task, we should see that two of
+      ;; them are marked to be deleted.
       (let [task (:app.storage/gc-touched-task th/*system*)
-            res  (task {})]
+            res  (task {:min-age (dt/duration 0)})]
-        (t/is (= 2 (:freeze res)))
+        (t/is (= 0 (:freeze res)))
-        (t/is (= 0 (:delete res))))
+        (t/is (= 2 (:delete res))))
 
       ;; Finally, check that some of the objects that are marked as
       ;; deleted we are unable to retrieve them using standard storage
       ;; public api.
-      (t/is (some? @(sto/get-object storage (:media-id fmo2))))
+      (t/is (nil? @(sto/get-object storage (:media-id fmo2))))
-      (t/is (some? @(sto/get-object storage (:thumbnail-id fmo2))))
+      (t/is (nil? @(sto/get-object storage (:thumbnail-id fmo2))))
-      (t/is (some? @(sto/get-object storage (:media-id fmo1))))
+      (t/is (nil? @(sto/get-object storage (:media-id fmo1))))
-      (t/is (some? @(sto/get-object storage (:thumbnail-id fmo1))))
+      (t/is (nil? @(sto/get-object storage (:thumbnail-id fmo1))))
-
       )))
 
 (t/deftest permissions-checks-creating-file
@@ -353,8 +362,8 @@
                                      :profile-id (:id profile1)})]
     ;; file is not deleted because it does not meet all
     ;; conditions to be deleted.
-    (let [result (task {:max-age (dt/duration 0)})]
+    (let [result (task {:min-age (dt/duration 0)})]
-      (t/is (nil? result)))
+      (t/is (= 0 (:processed result))))
 
     ;; query the list of files
     (let [data {::th/type :project-files
@@ -384,8 +393,8 @@
         (t/is (= 0 (count result)))))
 
     ;; run permanent deletion (should be noop)
-    (let [result (task {:max-age (dt/duration {:minutes 1})})]
+    (let [result (task {:min-age (dt/duration {:minutes 1})})]
-      (t/is (nil? result)))
+      (t/is (= 0 (:processed result))))
 
     ;; query the list of file libraries of a after hard deletion
     (let [data {::th/type :file-libraries
@@ -398,8 +407,8 @@
         (t/is (= 0 (count result)))))
 
     ;; run permanent deletion
-    (let [result (task {:max-age (dt/duration 0)})]
+    (let [result (task {:min-age (dt/duration 0)})]
-      (t/is (nil? result)))
+      (t/is (= 1 (:processed result))))
 
     ;; query the list of file libraries of a after hard deletion
     (let [data {::th/type :file-libraries
@@ -590,7 +599,7 @@
 
       ;; run the task again
       (let [task  (:app.tasks.file-gc/handler th/*system*)
-            res   (task {})]
+            res   (task {:min-age (dt/duration 0)})]
         (t/is (= 1 (:processed res))))
 
       ;; check that object thumbnails are still here
@@ -617,7 +626,7 @@
 
       ;; run the task again
       (let [task  (:app.tasks.file-gc/handler th/*system*)
-            res   (task {})]
+            res   (task {:min-age (dt/duration 0)})]
         (t/is (= 1 (:processed res))))
 
       ;; check that the unknown frame thumbnail is deleted
@@ -701,7 +710,7 @@
 
       ;; run the task again
       (let [task  (:app.tasks.file-gc/handler th/*system*)
-            res   (task {})]
+            res   (task {:min-age (dt/duration 0)})]
         (t/is (= 1 (:processed res))))
 
       ;; Then query the specific revn

backend/test/app/services_profile_test.clj

@@ -126,8 +126,8 @@
 
     ;; profile is not deleted because it does not meet all
     ;; conditions to be deleted.
-    (let [result (task {:max-age (dt/duration 0)})]
+    (let [result (task {:min-age (dt/duration 0)})]
-      (t/is (nil? result)))
+      (t/is (= 0 (:processed result))))
 
     ;; Request profile to be deleted
     (let [params {::th/type :delete-profile
@@ -145,8 +145,8 @@
       (t/is (= 1 (count (:result out)))))
 
     ;; execute permanent deletion task
-    (let [result (task {:max-age (dt/duration "-1m")})]
+    (let [result (task {:min-age (dt/duration "-1m")})]
-      (t/is (nil? result)))
+      (t/is (= 1 (:processed result))))
 
     ;; query profile after delete
     (let [params {::th/type :profile

backend/test/app/services_projects_test.clj

@@ -179,8 +179,8 @@
 
     ;; project is not deleted because it does not meet all
     ;; conditions to be deleted.
-    (let [result (task {:max-age (dt/duration 0)})]
+    (let [result (task {:min-age (dt/duration 0)})]
-      (t/is (nil? result)))
+      (t/is (= 0 (:processed result))))
 
     ;; query the list of projects
     (let [data {::th/type :projects
@@ -210,8 +210,8 @@
         (t/is (= 1 (count result)))))
 
     ;; run permanent deletion (should be noop)
-    (let [result (task {:max-age (dt/duration {:minutes 1})})]
+    (let [result (task {:min-age (dt/duration {:minutes 1})})]
-      (t/is (nil? result)))
+      (t/is (= 0 (:processed result))))
 
     ;; query the list of files of a after soft deletion
     (let [data {::th/type :project-files
@@ -224,8 +224,8 @@
         (t/is (= 0 (count result)))))
 
     ;; run permanent deletion
-    (let [result (task {:max-age (dt/duration 0)})]
+    (let [result (task {:min-age (dt/duration 0)})]
-      (t/is (nil? result)))
+      (t/is (= 1 (:processed result))))
 
     ;; query the list of files of a after hard deletion
     (let [data {::th/type :project-files

backend/test/app/services_teams_test.clj

@@ -99,8 +99,8 @@
 
     ;; team is not deleted because it does not meet all
     ;; conditions to be deleted.
-    (let [result (task {:max-age (dt/duration 0)})]
+    (let [result (task {:min-age (dt/duration 0)})]
-      (t/is (nil? result)))
+      (t/is (= 0 (:processed result))))
 
     ;; query the list of teams
     (let [data {::th/type :teams
@@ -132,8 +132,8 @@
         (t/is (= (:default-team-id profile1) (get-in result [0 :id])))))
 
     ;; run permanent deletion (should be noop)
-    (let [result (task {:max-age (dt/duration {:minutes 1})})]
+    (let [result (task {:min-age (dt/duration {:minutes 1})})]
-      (t/is (nil? result)))
+      (t/is (= 0 (:processed result))))
 
     ;; query the list of projects after hard deletion
     (let [data {::th/type :projects
@@ -147,8 +147,8 @@
         (t/is (= (:type error-data) :not-found))))
 
     ;; run permanent deletion
-    (let [result (task {:max-age (dt/duration 0)})]
+    (let [result (task {:min-age (dt/duration 0)})]
-      (t/is (nil? result)))
+      (t/is (= 1 (:processed result))))
 
     ;; query the list of projects of a after hard deletion
     (let [data {::th/type :projects