✨ Improve storage-gc-deleted task reliability

backend/src/app/storage.clj

@@ -251,53 +251,59 @@
 
 (defmethod ig/init-key ::gc-deleted-task
   [_ {:keys [::db/pool ::storage ::min-age]}]
-  (letfn [(retrieve-deleted-objects-chunk [conn min-age cursor]
+  (letfn [(get-to-delete-chunk [cursor]
-            (let [min-age (db/interval min-age)
+            (let [sql  (str "select s.* "
-                  rows    (db/exec! conn [sql:retrieve-deleted-objects-chunk min-age cursor])]
+                            "  from storage_object as s "
-              [(some-> rows peek :created-at)
+                            " where s.deleted_at is not null "
+                            "   and s.deleted_at < ? "
+                            " order by s.deleted_at desc "
+                            " limit 25")
+                  rows (db/exec! pool [sql cursor])]
+              [(some-> rows peek :deleted-at)
                (some->> (seq rows) (d/group-by #(-> % :backend keyword) :id #{}) seq)]))
 
-          (retrieve-deleted-objects [conn min-age]
+          (get-to-delete-chunks [min-age]
-            (d/iteration (partial retrieve-deleted-objects-chunk conn min-age)
+            (d/iteration get-to-delete-chunk
-                         :initk (dt/now)
+                         :initk (dt/minus (dt/now) min-age)
                          :vf second
                          :kf first))
 
-          (delete-in-bulk [backend-id ids]
+          (delete-in-bulk! [backend-id ids]
-            (let [backend (impl/resolve-backend storage backend-id)]
+            (try
+              (db/with-atomic [conn pool]
+                (let [sql   "delete from storage_object where id = ANY(?)"
+                      ids'  (db/create-array conn "uuid" ids)
 
-              (doseq [id ids]
+                      total (-> (db/exec-one! conn [sql ids'])
-                (l/debug :hint "gc-deleted: permanently delete storage object" :backend backend-id :id id))
+                                (db/get-update-count))]
 
-              (impl/del-objects-in-bulk backend ids)))]
+                  (-> (impl/resolve-backend storage backend-id)
+                      (impl/del-objects-in-bulk ids))
+
+                  (doseq [id ids]
+                    (l/dbg :hint "gc-deleted: permanently delete storage object" :backend backend-id :id id))
+
+                  total))
+
+              (catch Throwable cause
+                (l/err :hint "gc-deleted: unexpected error on bulk deletion"
+                       :ids (vec ids)
+                       :cause cause)
+                0)))]
 
     (fn [params]
-      (let [min-age (or (:min-age params) min-age)]
+      (let [min-age (or (some-> params :min-age dt/duration) min-age)]
-        (db/with-atomic [conn pool]
+        (loop [total  0
-          (loop [total  0
+               chunks (get-to-delete-chunks min-age)]
-                 groups (retrieve-deleted-objects conn min-age)]
+          (if-let [[backend-id ids] (first chunks)]
-            (if-let [[backend-id ids] (first groups)]
+            (let [deleted (delete-in-bulk! backend-id ids)]
-              (do
+              (recur (+ total deleted)
-                (delete-in-bulk backend-id ids)
+                     (rest chunks)))
-                (recur (+ total (count ids))
+            (do
-                       (rest groups)))
+              (l/inf :hint "gc-deleted: task finished"
-              (do
+                     :min-age (dt/format-duration min-age)
-                (l/info :hint "gc-deleted: task finished" :min-age (dt/format-duration min-age) :total total)
+                     :total total)
-                {:deleted total}))))))))
+              {:deleted total})))))))
-
-(def sql:retrieve-deleted-objects-chunk
-  "with items_part as (
-     select s.id
-       from storage_object as s
-      where s.deleted_at is not null
-        and s.deleted_at < (now() - ?::interval)
-        and s.created_at < ?
-      order by s.created_at desc
-      limit 25
-   )
-   delete from storage_object
-    where id in (select id from items_part)
-   returning *;")
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Garbage Collection: Analyze touched objects

backend/test/backend_tests/storage_test.clj

@@ -100,6 +100,7 @@
                     (configure-storage-backend))
         content1 (sto/content "content1")
         content2 (sto/content "content2")
+        content3 (sto/content "content3")
         object1  (sto/put-object! storage {::sto/content content1
                                            ::sto/expired-at (dt/now)
                                            :content-type "text/plain"
@@ -107,16 +108,20 @@
         object2  (sto/put-object! storage {::sto/content content2
                                            ::sto/expired-at (dt/in-past {:hours 2})
                                            :content-type "text/plain"
+                                           })
+        object3  (sto/put-object! storage {::sto/content content3
+                                           ::sto/expired-at (dt/in-past {:hours 1})
+                                           :content-type "text/plain"
                                            })]
 
+
     (th/sleep 200)
 
-    (let [task (:app.storage/gc-deleted-task th/*system*)
+    (let [res (th/run-task! :storage-gc-deleted {})]
-          res  (task {})]
       (t/is (= 1 (:deleted res))))
 
     (let [res (db/exec-one! th/*pool* ["select count(*) from storage_object;"])]
-      (t/is (= 1 (:count res))))))
+      (t/is (= 2 (:count res))))))
 
 (t/deftest test-touched-gc-task-1
   (let [storage (-> (:app.storage/storage th/*system*)