From 574c8d17890a89c5d2dea27b8435497241769d10 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 27 May 2024 10:13:59 +0200 Subject: [PATCH] :sparkles: Move library-absorb operation to async task And make it not mandatory in case of failure --- backend/src/app/rpc/commands/files.clj | 60 ++++++++++--------------- backend/src/app/srepl/main.clj | 18 +++++--- backend/src/app/tasks/delete_object.clj | 51 +++++++++++++-------- 3 files changed, 70 insertions(+), 59 deletions(-) diff --git a/backend/src/app/rpc/commands/files.clj b/backend/src/app/rpc/commands/files.clj index ea9653f1f..dc48abd6e 100644 --- a/backend/src/app/rpc/commands/files.clj +++ b/backend/src/app/rpc/commands/files.clj @@ -823,7 +823,7 @@ (feat.fdata/persist-pointers! cfg file-id)))) -(defn- absorb-library! +(defn- absorb-library "Find all files using a shared library, and absorb all library assets into the file local libraries" [cfg {:keys [id] :as library}] @@ -841,7 +841,26 @@ :library-id (str id) :files (str/join "," (map str ids))) - (run! (partial absorb-library-by-file! cfg ldata) ids))) + (run! (partial absorb-library-by-file! cfg ldata) ids) + library)) + +(defn absorb-library! + [{:keys [::db/conn] :as cfg} id] + (let [file (-> (get-file cfg id + :lock-for-update? true + :include-deleted? true) + (check-version!)) + + proj (db/get* conn :project {:id (:project-id file)} + {::db/remove-deleted false}) + team (-> (db/get* conn :team {:id (:team-id proj)} + {::db/remove-deleted false}) + (teams/decode-row))] + + (-> (cfeat/get-team-enabled-features cf/flags team) + (cfeat/check-file-features! (:features file))) + + (absorb-library cfg file))) (defn- set-file-shared [{:keys [::db/conn] :as cfg} {:keys [profile-id id] :as params}] @@ -854,25 +873,14 @@ ;; file, we need to perform more complex operation, ;; so in this case we retrieve the complete file and ;; perform all required validations. - (let [file (-> (get-file cfg id :lock-for-update? true) - (check-version!) - (assoc :is-shared false)) - team (teams/get-team conn - :profile-id profile-id - :project-id (:project-id file))] - - (-> (cfeat/get-team-enabled-features cf/flags team) - (cfeat/check-client-features! (:features params)) - (cfeat/check-file-features! (:features file))) - - (absorb-library! cfg file) - + (let [file (-> (absorb-library! cfg id) + (assoc :is-shared false))] (db/delete! conn :file-library-rel {:library-file-id id}) (db/update! conn :file {:is-shared false :modified-at (dt/now)} {:id id}) - file) + (select-keys file [:id :name :is-shared])) (and (false? (:is-shared file)) (true? (:is-shared params))) @@ -936,26 +944,6 @@ [{:keys [::db/conn] :as cfg} {:keys [profile-id id] :as params}] (check-edition-permissions! conn profile-id id) (let [file (mark-file-deleted conn id)] - - ;; NOTE: when a file is a shared library, then we proceed to load - ;; the whole file, proceed with feature checking and properly execute - ;; the absorb-library procedure - (when (:is-shared file) - (let [file (-> (get-file cfg id - :lock-for-update? true - :include-deleted? true) - (check-version!)) - - team (teams/get-team conn - :profile-id profile-id - :project-id (:project-id file))] - - (-> (cfeat/get-team-enabled-features cf/flags team) - (cfeat/check-client-features! (:features params)) - (cfeat/check-file-features! (:features file))) - - (absorb-library! cfg file))) - (rph/with-meta (rph/wrap) {::audit/props {:project-id (:project-id file) :name (:name file) diff --git a/backend/src/app/srepl/main.clj b/backend/src/app/srepl/main.clj index 87ff5cd81..193f72d1c 100644 --- a/backend/src/app/srepl/main.clj +++ b/backend/src/app/srepl/main.clj @@ -503,10 +503,14 @@ (db/update! conn :file-tagged-object-thumbnail {:deleted-at nil} - {:file-id file-id})) + {:file-id file-id}) + + :restored) + + (defn- restore-project* - [{:keys [::db/conn]} project-id] + [{:keys [::db/conn] :as cfg} project-id] (db/update! conn :project {:deleted-at nil} @@ -515,10 +519,12 @@ (doseq [{:keys [id]} (db/query conn :file {:project-id project-id} {::db/columns [:id]})] - (restore-file* conn id))) + (restore-file* cfg id)) + + :restored) (defn- restore-team* - [{:keys [::db/conn]} team-id] + [{:keys [::db/conn] :as cfg} team-id] (db/update! conn :team {:deleted-at nil} {:id team-id}) @@ -530,7 +536,9 @@ (doseq [{:keys [id]} (db/query conn :project {:team-id team-id} {::db/columns [:id]})] - (restore-project* conn id))) + (restore-project* cfg id)) + + :restored) (defn restore-deleted-team! "Mark a team and all related objects as not deleted" diff --git a/backend/src/app/tasks/delete_object.clj b/backend/src/app/tasks/delete_object.clj index bd954cd9d..557e35b59 100644 --- a/backend/src/app/tasks/delete_object.clj +++ b/backend/src/app/tasks/delete_object.clj @@ -9,28 +9,42 @@ (:require [app.common.logging :as l] [app.db :as db] + [app.rpc.commands.files :as files] [clojure.spec.alpha :as s] [integrant.core :as ig])) +(def ^:dynamic *team-deletion* false) + (defmulti delete-object (fn [_ props] (:object props))) (defmethod delete-object :file - [{:keys [::db/conn]} {:keys [id deleted-at]}] + [{:keys [::db/conn] :as cfg} {:keys [id deleted-at]}] (l/trc :hint "marking for deletion" :rel "file" :id (str id)) - ;; Mark file media objects to be deleted - (db/update! conn :file-media-object - {:deleted-at deleted-at} - {:file-id id}) + (when-let [file (db/get* conn :file {:id id} {::db/remove-deleted false})] + (when (and (:is-shared file) + (not *team-deletion*)) + ;; NOTE: we don't prevent file deletion on absorb operation failure + (try + (db/tx-run! cfg files/absorb-library! id) + (catch Throwable cause + (l/warn :hint "error on absorbing library" + :file-id id + :cause cause)))) - ;; Mark thumbnails to be deleted - (db/update! conn :file-thumbnail - {:deleted-at deleted-at} - {:file-id id}) + ;; Mark file media objects to be deleted + (db/update! conn :file-media-object + {:deleted-at deleted-at} + {:file-id id}) - (db/update! conn :file-tagged-object-thumbnail - {:deleted-at deleted-at} - {:file-id id})) + ;; Mark thumbnails to be deleted + (db/update! conn :file-thumbnail + {:deleted-at deleted-at} + {:file-id id}) + + (db/update! conn :file-tagged-object-thumbnail + {:deleted-at deleted-at} + {:file-id id}))) (defmethod delete-object :project [{:keys [::db/conn] :as cfg} {:keys [id deleted-at]}] @@ -49,12 +63,13 @@ {:deleted-at deleted-at} {:team-id id}) - (doseq [project (db/update! conn :project - {:deleted-at deleted-at} - {:team-id id} - {::db/return-keys [:id :deleted-at] - ::db/many true})] - (delete-object cfg (assoc project :object :project)))) + (binding [*team-deletion* true] + (doseq [project (db/update! conn :project + {:deleted-at deleted-at} + {:team-id id} + {::db/return-keys [:id :deleted-at] + ::db/many true})] + (delete-object cfg (assoc project :object :project))))) (defmethod delete-object :default [_cfg props]