✨ Add proper fdata objects deletion.

backend/src/app/rpc/mutations/files.clj

@@ -11,15 +11,15 @@
    [app.common.pages.migrations :as pmg]
    [app.common.spec :as us]
    [app.common.uuid :as uuid]
-
+   [app.config :as cf]
    [app.db :as db]
    [app.rpc.permissions :as perms]
    [app.rpc.queries.files :as files]
    [app.rpc.queries.projects :as proj]
+   [app.storage.impl :as simpl]
    [app.util.blob :as blob]
    [app.util.services :as sv]
    [app.util.time :as dt]
-
    [clojure.spec.alpha :as s]))
 
 (declare create-file)
@@ -45,7 +45,6 @@
     (proj/check-edition-permissions! conn profile-id project-id)
     (create-file conn params)))
 
-
 (defn create-file-role
   [conn {:keys [file-id profile-id role]}]
   (let [params {:file-id file-id
@@ -112,7 +111,6 @@
               {:is-shared is-shared}
               {:id id}))
 
-
 ;; --- Mutation: Delete File
 
 (declare mark-file-deleted)
@@ -288,6 +286,11 @@
       (> (inst-ms (dt/diff modified-at (dt/now)))
          (inst-ms (dt/duration {:hours 3})))))
 
+(defn- delete-from-storage
+  [{:keys [storage] :as cfg} file]
+  (when-let [backend (simpl/resolve-backend storage (cf/get :fdata-storage-backend))]
+    (simpl/del-object backend file)))
+
 (defn- update-file
   [{:keys [conn] :as cfg} {:keys [file changes changes-with-metadata session-id profile-id] :as params}]
   (when (> (:revn params)
@@ -334,6 +337,10 @@
                  :has-media-trimmed false}
                 {:id (:id file)})
 
+    ;; We need to delete the data from external storage backend
+    (when-not (nil? (:data-backend file))
+      (delete-from-storage cfg file))
+
     (db/update! conn :project
                 {:modified-at ts}
                 {:id (:project-id file)})

backend/src/app/storage/db.clj

@@ -55,8 +55,15 @@
   [_ _]
   (throw (UnsupportedOperationException. "not supported")))
 
+(defmethod impl/del-object :db
+  [_ _]
+  ;; NOOP: because deleting the row already deletes the file data from
+  ;; the database.
+  nil)
+
 (defmethod impl/del-objects-in-bulk :db
   [_ _]
   ;; NOOP: because deleting the row already deletes the file data from
   ;; the database.
   nil)
+

backend/src/app/storage/fs.clj

@@ -91,6 +91,13 @@
               (str existing (impl/id->path id))
               (str existing "/" (impl/id->path id))))))
 
+(defmethod impl/del-object :fs
+  [backend {:keys [id] :as object}]
+  (let [base (fs/path (:directory backend))
+        path (fs/path (impl/id->path id))
+        path (fs/join base path)]
+    (Files/deleteIfExists ^Path path)))
+
 (defmethod impl/del-objects-in-bulk :fs
   [backend ids]
   (let [base (fs/path (:directory backend))]
@@ -98,3 +105,4 @@
       (let [path (fs/path (impl/id->path id))
             path (fs/join base path)]
         (Files/deleteIfExists ^Path path)))))
+

backend/src/app/storage/impl.clj

@@ -62,6 +62,14 @@
             :context cfg))
 
 
+(defmulti del-object (fn [cfg _] (:type cfg)))
+
+(defmethod del-object :default
+  [cfg _]
+  (ex/raise :type :internal
+            :code :invalid-storage-backend
+            :context cfg))
+
 (defmulti del-objects-in-bulk (fn [cfg _] (:type cfg)))
 
 (defmethod del-objects-in-bulk :default
@@ -70,7 +78,6 @@
             :code :invalid-storage-backend
             :context cfg))
 
-
 ;; --- HELPERS
 
 (defn uuid->hex

backend/src/app/storage/s3.clj

@@ -29,6 +29,7 @@
    software.amazon.awssdk.services.s3.model.CopyObjectRequest
    software.amazon.awssdk.services.s3.model.DeleteObjectsRequest
    software.amazon.awssdk.services.s3.model.DeleteObjectsResponse
+   software.amazon.awssdk.services.s3.model.DeleteObjectRequest
    software.amazon.awssdk.services.s3.model.GetObjectRequest
    software.amazon.awssdk.services.s3.model.ObjectIdentifier
    software.amazon.awssdk.services.s3.model.PutObjectRequest
@@ -44,6 +45,7 @@
 (declare get-object-bytes)
 (declare get-object-data)
 (declare get-object-url)
+(declare del-object)
 (declare del-object-in-bulk)
 (declare build-s3-client)
 (declare build-s3-presigner)
@@ -104,6 +106,10 @@
   [backend object options]
   (get-object-url backend object options))
 
+(defmethod impl/del-object :s3
+  [backend object]
+  (del-object backend object))
+
 (defmethod impl/del-objects-in-bulk :s3
   [backend ids]
   (del-object-in-bulk backend ids))
@@ -198,6 +204,15 @@
         pgor (.presignGetObject ^S3Presigner presigner ^GetObjectPresignRequest gopr)]
     (u/uri (str (.url ^PresignedGetObjectRequest pgor)))))
 
+(defn del-object
+  [{:keys [bucket client prefix]} {:keys [id] :as obj}]
+  (let [dor (.. (DeleteObjectRequest/builder)
+                (bucket bucket)
+                (key (str prefix (impl/id->path id)))
+                (build))]
+    (.deleteObject ^S3Client client
+                   ^DeleteObjectRequest dor)))
+
 (defn del-object-in-bulk
   [{:keys [bucket client prefix]} ids]
   (let [oids (map (fn [id]

backend/src/app/tasks/file_offload.clj

@@ -43,7 +43,6 @@
 (s/def ::max-age ::dt/duration)
 (s/def ::backend ::us/keyword)
 
-
 (defmethod ig/pre-init-spec ::handler [_]
   (s/keys :req-un [::db/pool ::max-age ::sto/storage ::backend]))
 

backend/src/app/tasks/objects_gc.clj

@@ -8,8 +8,10 @@
   "A maintenance task that performs a general purpose garbage collection
   of deleted objects."
   (:require
+   [app.config :as cf]
    [app.db :as db]
    [app.storage :as sto]
+   [app.storage.impl :as simpl]
    [app.util.logging :as l]
    [app.util.time :as dt]
    [clojure.spec.alpha :as s]
@@ -50,6 +52,23 @@
 
     (count result)))
 
+
+;; --- IMPL: file deletion
+
+(defmethod delete-objects "file"
+  [{:keys [conn max-age table storage] :as cfg}]
+  (let [sql     (str/fmt sql:delete-objects
+                         {:table table :limit 50})
+        result  (db/exec! conn [sql max-age])
+        backend (simpl/resolve-backend storage (cf/get :fdata-storage-backend))]
+
+    (doseq [{:keys [id] :as item} result]
+      (l/trace :action "delete object" :table table :id id)
+      (when backend
+        (simpl/del-object backend item)))
+
+    (count result)))
+
 ;; --- IMPL: team-font-variant deletion
 
 (defmethod delete-objects "team_font_variant"