diff --git a/backend/src/app/tasks/file_gc.clj b/backend/src/app/tasks/file_gc.clj index 7e3c3ee27..facc7b60f 100644 --- a/backend/src/app/tasks/file_gc.clj +++ b/backend/src/app/tasks/file_gc.clj @@ -53,16 +53,27 @@ RETURNING id") (def ^:private xf:collect-used-media - (comp (map :data) (mapcat bfc/collect-used-media))) + (comp + (map :data) + (mapcat bfc/collect-used-media))) + +(def ^:private plan-opts + {:fetch-size 1 + :concurrency :read-only + :cursors :close + :result-type :forward-only}) (defn- clean-file-media! "Performs the garbage collection of file media objects." [{:keys [::db/conn] :as cfg} {:keys [id] :as file}] - (let [used (into #{} - xf:collect-used-media - (cons file - (->> (db/cursor conn [sql:get-snapshots id]) - (map (partial decode-file cfg))))) + (let [xform (comp + (map (partial decode-file cfg)) + xf:collect-used-media) + + used (->> (db/plan conn [sql:get-snapshots id] plan-opts) + (transduce xform conj #{})) + used (into used xf:collect-used-media [file]) + ids (db/create-array conn "uuid" used) unused (->> (db/exec! conn [sql:mark-file-media-object-deleted id ids]) (into #{} (map :id)))] diff --git a/backend/test/backend_tests/helpers.clj b/backend/test/backend_tests/helpers.clj index 3aa7d1589..1e1fb7abf 100644 --- a/backend/test/backend_tests/helpers.clj +++ b/backend/test/backend_tests/helpers.clj @@ -62,7 +62,7 @@ (def default {:database-uri "postgresql://postgres/penpot_test" :redis-uri "redis://redis/1" - :file-snapshot-every 1}) + :auto-file-snapshot-every 1}) (def config (cf/read-config :prefix "penpot-test" diff --git a/backend/test/backend_tests/rpc_file_test.clj b/backend/test/backend_tests/rpc_file_test.clj index 679d5221e..733c4a1fd 100644 --- a/backend/test/backend_tests/rpc_file_test.clj +++ b/backend/test/backend_tests/rpc_file_test.clj @@ -383,8 +383,19 @@ ;; as deleted. (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)}))) + ;; This only clears fragments, the file media objects still referenced because + ;; snapshots are preserved (let [res (th/run-task! :objects-gc {:min-age 0})] - (t/is (= 3 (:processed res)))) + (t/is (= 2 (:processed res)))) + + ;; Mark all snapshots to be a non-snapshot file change + (th/db-exec! ["update file_change set data = null where file_id = ?" (:id file)]) + (th/db-exec! ["update file set has_media_trimmed = false where id = ?" (:id file)]) + + ;; Rerun the file-gc and objects-gc + (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)}))) + (let [res (th/run-task! :objects-gc {:min-age 0})] + (t/is (= 2 (:processed res)))) ;; Now that file-gc have deleted the file-media-object usage, ;; lets execute the touched-gc task, we should see that two of @@ -417,20 +428,6 @@ ;; (th/print-result! out) (t/is (nil? (:error out))) - (:result out))) - - (update-file! [& {:keys [profile-id file-id changes revn] :or {revn 0}}] - (let [params {::th/type :update-file - ::rpc/profile-id profile-id - :id file-id - :session-id (uuid/random) - :revn revn - :vern 0 - :components-v2 true - :changes changes} - out (th/command! params)] - ;; (th/print-result! out) - (t/is (nil? (:error out))) (:result out)))] (let [storage (:app.storage/storage th/*system*) @@ -550,8 +547,20 @@ ;; as deleted. (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)}))) + ;; This only removes unused fragments, file media are still + ;; referenced on snapshots. (let [res (th/run-task! :objects-gc {:min-age 0})] - (t/is (= 7 (:processed res)))) + (t/is (= 2 (:processed res)))) + + ;; Mark all snapshots to be a non-snapshot file change + (th/db-exec! ["update file set has_media_trimmed = false where id = ?" (:id file)]) + (th/db-exec! ["update file_change set data = null where file_id = ?" (:id file)]) + + ;; Rerun file-gc and objects-gc task for the same file once all snapshots are + ;; "expired/deleted" + (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)}))) + (let [res (th/run-task! :objects-gc {:min-age 0})] + (t/is (= 6 (:processed res)))) (let [rows (th/db-query :file-data-fragment {:file-id (:id file) :deleted-at nil})] @@ -591,20 +600,7 @@ ;; (th/print-result! out) (t/is (nil? (:error out))) - (:result out))) - - #_(update-file! [& {:keys [profile-id file-id changes revn] :or {revn 0}}] - (let [params {::th/type :update-file - ::rpc/profile-id profile-id - :id file-id - :session-id (uuid/random) - :revn revn - :features cfeat/supported-features - :changes changes} - out (th/command! params)] - ;; (th/print-result! out) - (t/is (nil? (:error out))) - (:result out)))] + (:result out)))] (let [storage (:app.storage/storage th/*system*) profile (th/create-profile* 1)