diff --git a/backend/src/app/rpc/mutations/files.clj b/backend/src/app/rpc/mutations/files.clj index a08da44f8..0b6da5b51 100644 --- a/backend/src/app/rpc/mutations/files.clj +++ b/backend/src/app/rpc/mutations/files.clj @@ -11,15 +11,15 @@ [app.common.pages.migrations :as pmg] [app.common.spec :as us] [app.common.uuid :as uuid] - + [app.config :as cf] [app.db :as db] [app.rpc.permissions :as perms] [app.rpc.queries.files :as files] [app.rpc.queries.projects :as proj] + [app.storage.impl :as simpl] [app.util.blob :as blob] [app.util.services :as sv] [app.util.time :as dt] - [clojure.spec.alpha :as s])) (declare create-file) @@ -45,7 +45,6 @@ (proj/check-edition-permissions! conn profile-id project-id) (create-file conn params))) - (defn create-file-role [conn {:keys [file-id profile-id role]}] (let [params {:file-id file-id @@ -112,7 +111,6 @@ {:is-shared is-shared} {:id id})) - ;; --- Mutation: Delete File (declare mark-file-deleted) @@ -288,6 +286,11 @@ (> (inst-ms (dt/diff modified-at (dt/now))) (inst-ms (dt/duration {:hours 3}))))) +(defn- delete-from-storage + [{:keys [storage] :as cfg} file] + (when-let [backend (simpl/resolve-backend storage (cf/get :fdata-storage-backend))] + (simpl/del-object backend file))) + (defn- update-file [{:keys [conn] :as cfg} {:keys [file changes changes-with-metadata session-id profile-id] :as params}] (when (> (:revn params) @@ -334,6 +337,10 @@ :has-media-trimmed false} {:id (:id file)}) + ;; We need to delete the data from external storage backend + (when-not (nil? (:data-backend file)) + (delete-from-storage cfg file)) + (db/update! conn :project {:modified-at ts} {:id (:project-id file)}) diff --git a/backend/src/app/storage/db.clj b/backend/src/app/storage/db.clj index e5814c850..0890f7455 100644 --- a/backend/src/app/storage/db.clj +++ b/backend/src/app/storage/db.clj @@ -55,8 +55,15 @@ [_ _] (throw (UnsupportedOperationException. "not supported"))) +(defmethod impl/del-object :db + [_ _] + ;; NOOP: because deleting the row already deletes the file data from + ;; the database. + nil) + (defmethod impl/del-objects-in-bulk :db [_ _] ;; NOOP: because deleting the row already deletes the file data from ;; the database. nil) + diff --git a/backend/src/app/storage/fs.clj b/backend/src/app/storage/fs.clj index e467bd168..e15bb7b0e 100644 --- a/backend/src/app/storage/fs.clj +++ b/backend/src/app/storage/fs.clj @@ -91,6 +91,13 @@ (str existing (impl/id->path id)) (str existing "/" (impl/id->path id)))))) +(defmethod impl/del-object :fs + [backend {:keys [id] :as object}] + (let [base (fs/path (:directory backend)) + path (fs/path (impl/id->path id)) + path (fs/join base path)] + (Files/deleteIfExists ^Path path))) + (defmethod impl/del-objects-in-bulk :fs [backend ids] (let [base (fs/path (:directory backend))] @@ -98,3 +105,4 @@ (let [path (fs/path (impl/id->path id)) path (fs/join base path)] (Files/deleteIfExists ^Path path))))) + diff --git a/backend/src/app/storage/impl.clj b/backend/src/app/storage/impl.clj index e39aaa7f9..4c3a61900 100644 --- a/backend/src/app/storage/impl.clj +++ b/backend/src/app/storage/impl.clj @@ -62,6 +62,14 @@ :context cfg)) +(defmulti del-object (fn [cfg _] (:type cfg))) + +(defmethod del-object :default + [cfg _] + (ex/raise :type :internal + :code :invalid-storage-backend + :context cfg)) + (defmulti del-objects-in-bulk (fn [cfg _] (:type cfg))) (defmethod del-objects-in-bulk :default @@ -70,7 +78,6 @@ :code :invalid-storage-backend :context cfg)) - ;; --- HELPERS (defn uuid->hex diff --git a/backend/src/app/storage/s3.clj b/backend/src/app/storage/s3.clj index 335b284a7..2d42277a7 100644 --- a/backend/src/app/storage/s3.clj +++ b/backend/src/app/storage/s3.clj @@ -29,6 +29,7 @@ software.amazon.awssdk.services.s3.model.CopyObjectRequest software.amazon.awssdk.services.s3.model.DeleteObjectsRequest software.amazon.awssdk.services.s3.model.DeleteObjectsResponse + software.amazon.awssdk.services.s3.model.DeleteObjectRequest software.amazon.awssdk.services.s3.model.GetObjectRequest software.amazon.awssdk.services.s3.model.ObjectIdentifier software.amazon.awssdk.services.s3.model.PutObjectRequest @@ -44,6 +45,7 @@ (declare get-object-bytes) (declare get-object-data) (declare get-object-url) +(declare del-object) (declare del-object-in-bulk) (declare build-s3-client) (declare build-s3-presigner) @@ -104,6 +106,10 @@ [backend object options] (get-object-url backend object options)) +(defmethod impl/del-object :s3 + [backend object] + (del-object backend object)) + (defmethod impl/del-objects-in-bulk :s3 [backend ids] (del-object-in-bulk backend ids)) @@ -198,6 +204,15 @@ pgor (.presignGetObject ^S3Presigner presigner ^GetObjectPresignRequest gopr)] (u/uri (str (.url ^PresignedGetObjectRequest pgor))))) +(defn del-object + [{:keys [bucket client prefix]} {:keys [id] :as obj}] + (let [dor (.. (DeleteObjectRequest/builder) + (bucket bucket) + (key (str prefix (impl/id->path id))) + (build))] + (.deleteObject ^S3Client client + ^DeleteObjectRequest dor))) + (defn del-object-in-bulk [{:keys [bucket client prefix]} ids] (let [oids (map (fn [id] diff --git a/backend/src/app/tasks/file_offload.clj b/backend/src/app/tasks/file_offload.clj index 5c5abd7bf..ee784d02d 100644 --- a/backend/src/app/tasks/file_offload.clj +++ b/backend/src/app/tasks/file_offload.clj @@ -43,7 +43,6 @@ (s/def ::max-age ::dt/duration) (s/def ::backend ::us/keyword) - (defmethod ig/pre-init-spec ::handler [_] (s/keys :req-un [::db/pool ::max-age ::sto/storage ::backend])) diff --git a/backend/src/app/tasks/objects_gc.clj b/backend/src/app/tasks/objects_gc.clj index b146c85af..112524560 100644 --- a/backend/src/app/tasks/objects_gc.clj +++ b/backend/src/app/tasks/objects_gc.clj @@ -8,8 +8,10 @@ "A maintenance task that performs a general purpose garbage collection of deleted objects." (:require + [app.config :as cf] [app.db :as db] [app.storage :as sto] + [app.storage.impl :as simpl] [app.util.logging :as l] [app.util.time :as dt] [clojure.spec.alpha :as s] @@ -50,6 +52,23 @@ (count result))) + +;; --- IMPL: file deletion + +(defmethod delete-objects "file" + [{:keys [conn max-age table storage] :as cfg}] + (let [sql (str/fmt sql:delete-objects + {:table table :limit 50}) + result (db/exec! conn [sql max-age]) + backend (simpl/resolve-backend storage (cf/get :fdata-storage-backend))] + + (doseq [{:keys [id] :as item} result] + (l/trace :action "delete object" :table table :id id) + (when backend + (simpl/del-object backend item))) + + (count result))) + ;; --- IMPL: team-font-variant deletion (defmethod delete-objects "team_font_variant"