diff --git a/backend/src/app/tasks/file_gc.clj b/backend/src/app/tasks/file_gc.clj index f4800a518..f40882f96 100644 --- a/backend/src/app/tasks/file_gc.clj +++ b/backend/src/app/tasks/file_gc.clj @@ -13,6 +13,7 @@ [app.common.data :as d] [app.common.logging :as l] [app.common.pages.migrations :as pmg] + [app.common.types.file :as ctf] [app.common.types.shape-tree :as ctt] [app.config :as cf] [app.db :as db] @@ -45,7 +46,8 @@ (fn [params] (db/with-atomic [conn pool] (let [min-age (or (:min-age params) (:min-age cfg)) - cfg (assoc cfg :min-age min-age :conn conn)] + id (:id params) + cfg (assoc cfg :min-age min-age :conn conn :id id)] (loop [total 0 files (retrieve-candidates cfg)] (if-let [file (first files)] @@ -162,7 +164,73 @@ (let [sql (str "delete from file_thumbnail " " where file_id=? and revn < ?") res (db/exec-one! conn [sql file-id revn])] - (l/debug :hint "delete file thumbnails" :file-id file-id :total (:next.jdbc/update-count res)))) + (when-not (zero? (:next.jdbc/update-count res)) + (l/debug :hint "delete file thumbnails" :file-id file-id :total (:next.jdbc/update-count res))))) + +(def ^:private + sql:retrieve-client-files + "select f.data, f.modified_at + from file as f + left join file_library_rel as fl on (fl.file_id = f.id) + where fl.library_file_id = ? + and f.modified_at < ? + and f.deleted_at is null + order by f.modified_at desc + limit 1") + +(defn- retrieve-client-files + "search al files that use the given library. + Returns a sequence of file-data (only reads database rows one by one)." + [conn library-id] + (let [get-chunk (fn [cursor] + (let [rows (db/exec! conn [sql:retrieve-client-files library-id cursor])] + [(some-> rows peek :modified-at) + (map #(->> % :data blob/decode) rows)]))] + + (d/iteration get-chunk + :vf second + :kf first + :initk (dt/now)))) + +(defn- clean-deleted-components! + "Performs the garbage collection of unreferenced deleted components." + [conn library-id library-data] + (let [find-used-components-file + (fn [components file-data] + ; Find what of the components are used in the file. + (d/filterm #(ctf/used-in? file-data library-id (second %) :component) + components)) + + find-used-components + (fn [components files-data] + ; Find what components are used in any of the files. + (loop [files-data files-data + components components + used-components {}] + (let [file-data (first files-data)] + (if (or (nil? file-data) (empty? components)) + used-components + (let [used-components-file (find-used-components-file components file-data)] + (recur (rest files-data) + (d/filterm #(not (contains? used-components-file (:id %))) components) + (into used-components used-components-file))))))) + + deleted-components (:deleted-components library-data) + saved-components (find-used-components deleted-components + (cons library-data + (retrieve-client-files conn library-id))) + + total (- (count deleted-components) + (count saved-components))] + + (when-not (zero? total) + (l/debug :hint "clean deleted components" :total total) + (let [new-data (-> library-data + (assoc :deleted-components saved-components) + (blob/encode))] + (db/update! conn :file + {:data new-data} + {:id library-id}))))) (defn- process-file [{:keys [conn] :as cfg} {:keys [id data revn modified-at] :as file}] @@ -175,6 +243,7 @@ (clean-file-media! conn id data) (clean-file-frame-thumbnails! conn id data) (clean-file-thumbnails! conn id revn) + (clean-deleted-components! conn id data) ;; Mark file as trimmed (db/update! conn :file diff --git a/common/src/app/common/types/file.cljc b/common/src/app/common/types/file.cljc index 2af87b2d5..74f11a20b 100644 --- a/common/src/app/common/types/file.cljc +++ b/common/src/app/common/types/file.cljc @@ -223,6 +223,18 @@ [[asset instances]]))) assets-seq))) +(defn used-in? + "Checks if a specific asset is used in a given file (by any shape in its pages or in + the components of the local library)." + [file-data library-id asset asset-type] + (letfn [(used-in-shape? [shape] + (uses-asset? asset-type shape library-id asset)) + + (used-in-container? [container] + (some used-in-shape? (ctn/shapes-seq container)))] + + (some used-in-container? (containers-seq file-data)))) + (defn get-or-add-library-page "If exists a page named 'Library backup', get the id and calculate the position to start adding new components. If not, create it and start at (0, 0)." diff --git a/frontend/src/debug.cljs b/frontend/src/debug.cljs index fd693882c..96c986604 100644 --- a/frontend/src/debug.cljs +++ b/frontend/src/debug.cljs @@ -158,6 +158,10 @@ (logjs "state" @st/state) nil) +(defn ^:export dump-data [] + (logjs "workspace-data" (get @st/state :workspace-data)) + nil) + (defn ^:export dump-buffer [] (logjs "last-events" @st/last-events) nil)