From 9649878fd80fecd2267083e7b09854ced7902fc5 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 13 Feb 2024 19:56:12 +0100 Subject: [PATCH 1/2] :sparkles: Ensure id prop on :data on components-v2 migration --- backend/src/app/features/components_v2.clj | 47 +++++++++++++--------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index 72f416bdf..667e1bb60 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -1519,21 +1519,9 @@ (cfv/validate-file! file libs) (cfv/validate-file-schema! file)) -(defn- process-file - [{:keys [::db/conn] :as system} {:keys [id] :as file} & {:keys [validate?]}] - (let [libs (->> (files/get-file-libraries conn id) - (into [file] (comp (map :id) - (map (partial get-file system)))) - (d/index-by :id)) - - file (-> file - (update :data migrate-fdata libs) - (update :features conj "components/v2")) - - _ (when validate? - (validate-file! file libs)) - - file (if (contains? (:features file) "fdata/objects-map") +(defn- persist-file! + [{:keys [::db/conn] :as system} {:keys [id] :as file}] + (let [file (if (contains? (:features file) "fdata/objects-map") (fdata/enable-objects-map file) file) @@ -1542,15 +1530,32 @@ (let [file (fdata/enable-pointer-map file)] (fdata/persist-pointers! system id) file)) - file)] + file) + + ;; Ensure all files has :data with id + file (update file :data assoc :id id)] (db/update! conn :file {:data (blob/encode (:data file)) :features (db/create-array conn "text" (:features file)) :revn (:revn file)} - {:id (:id file)}) + {:id (:id file)}))) - (dissoc file :data))) +(defn- process-file! + [{:keys [::db/conn] :as system} {:keys [id] :as file} & {:keys [validate?]}] + (let [libs (->> (files/get-file-libraries conn id) + (into [file] (comp (map :id) + (map (partial get-file system)))) + (d/index-by :id)) + + file (-> file + (update :data migrate-fdata libs) + (update :features conj "components/v2"))] + + (when validate? + (validate-file! file libs)) + + file)) (def ^:private sql:get-and-lock-team-files "SELECT f.id @@ -1600,13 +1605,15 @@ (when (string? label) (fsnap/take-file-snapshot! system {:file-id file-id :label (str "migration/" label)})) - (let [file (get-file system file-id)] + (let [file (get-file system file-id) + file (process-file! system file :validate? validate?)] + (events/tap :progress {:op :migrate-file :name (:name file) :id (:id file)}) - (process-file system file :validate? validate?))))) + (persist-file! system file))))) (catch Throwable cause (vreset! err true) From a31be7e2ff7a5915eba58559817b1c576921e6b7 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 14 Feb 2024 09:31:59 +0100 Subject: [PATCH 2/2] :sparkles: Use a prefixed dir for storing temp files And mark them for deletion on JVM exit. --- backend/dev/user.clj | 2 +- backend/src/app/storage/tmp.clj | 19 +++++++++---------- backend/test/backend_tests/helpers.clj | 7 +++++-- .../test/backend_tests/rpc_comment_test.clj | 2 +- .../rpc_cond_middleware_test.clj | 2 +- .../rpc_file_thumbnails_test.clj | 2 +- .../backend_tests/rpc_management_test.clj | 2 +- backend/test/backend_tests/rpc_media_test.clj | 2 +- .../test/backend_tests/rpc_profile_test.clj | 2 +- .../test/backend_tests/rpc_quotes_test.clj | 2 +- backend/test/backend_tests/rpc_team_test.clj | 2 +- .../test/backend_tests/rpc_viewer_test.clj | 2 +- backend/test/backend_tests/storage_test.clj | 2 +- common/deps.edn | 11 +++++++---- 14 files changed, 32 insertions(+), 27 deletions(-) diff --git a/backend/dev/user.clj b/backend/dev/user.clj index efe2c935a..9fc59d5e1 100644 --- a/backend/dev/user.clj +++ b/backend/dev/user.clj @@ -44,7 +44,7 @@ [clojure.walk :refer [macroexpand-all]] [criterium.core :as crit] [cuerdas.core :as str] - [datoteka.core] + [datoteka.fs :as fs] [integrant.core :as ig] [malli.core :as m] [malli.dev.pretty :as mdp] diff --git a/backend/src/app/storage/tmp.clj b/backend/src/app/storage/tmp.clj index 43c9259d3..92cda29eb 100644 --- a/backend/src/app/storage/tmp.clj +++ b/backend/src/app/storage/tmp.clj @@ -19,6 +19,8 @@ [promesa.exec :as px] [promesa.exec.csp :as sp])) +(def default-tmp-dir "/tmp/penpot") + (declare ^:private remove-temp-file) (declare ^:private io-loop) @@ -33,6 +35,7 @@ (defmethod ig/init-key ::cleaner [_ cfg] + (fs/create-dir default-tmp-dir) (px/fn->thread (partial io-loop cfg) {:name "penpot/storage/tmp-cleaner" :virtual true})) @@ -70,18 +73,14 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (defn tempfile - "Returns a tmpfile candidate (without creating it)" [& {:keys [suffix prefix min-age] :or {prefix "penpot." suffix ".tmp"}}] - (let [path (fs/tempfile :suffix suffix :prefix prefix)] - (sp/offer! queue [path (some-> min-age dt/duration)]) - path)) - -(defn create-tempfile - [& {:keys [suffix prefix min-age] - :or {prefix "penpot." - suffix ".tmp"}}] - (let [path (fs/create-tempfile :suffix suffix :prefix prefix)] + (let [path (fs/create-tempfile + :perms "rw-r--r--" + :dir default-tmp-dir + :suffix suffix + :prefix prefix)] + (fs/delete-on-exit! path) (sp/offer! queue [path (some-> min-age dt/duration)]) path)) diff --git a/backend/test/backend_tests/helpers.clj b/backend/test/backend_tests/helpers.clj index ad08d5b62..987b55304 100644 --- a/backend/test/backend_tests/helpers.clj +++ b/backend/test/backend_tests/helpers.clj @@ -38,7 +38,7 @@ [clojure.spec.alpha :as s] [clojure.test :as t] [cuerdas.core :as str] - [datoteka.core :as fs] + [datoteka.fs :as fs] [environ.core :refer [env]] [expound.alpha :as expound] [integrant.core :as ig] @@ -127,6 +127,8 @@ app.auth/verify-password (fn [a b] {:valid (= a b)}) app.common.features/get-enabled-features (fn [& _] app.common.features/supported-features)] + (fs/create-dir "/tmp/penpot") + (let [templates [{:id "test" :name "test" :file-uri "test" @@ -191,6 +193,7 @@ (let [path (fs/path "/tmp/penpot")] (when (fs/exists? path) (fs/delete (fs/path "/tmp/penpot"))) + (fs/create-dir "/tmp/penpot") (next))) (defn serial @@ -496,7 +499,7 @@ (defn tempfile [source] (let [rsc (io/resource source) - tmp (fs/create-tempfile)] + tmp (fs/create-tempfile :dir "/tmp/penpot" :prefix "test-")] (io/copy (io/file rsc) (io/file tmp)) tmp)) diff --git a/backend/test/backend_tests/rpc_comment_test.clj b/backend/test/backend_tests/rpc_comment_test.clj index 774afb86a..9e0f86474 100644 --- a/backend/test/backend_tests/rpc_comment_test.clj +++ b/backend/test/backend_tests/rpc_comment_test.clj @@ -17,7 +17,7 @@ [app.util.time :as dt] [backend-tests.helpers :as th] [clojure.test :as t] - [datoteka.core :as fs] + [datoteka.fs :as fs] [mockery.core :refer [with-mocks]])) (t/use-fixtures :once th/state-init) diff --git a/backend/test/backend_tests/rpc_cond_middleware_test.clj b/backend/test/backend_tests/rpc_cond_middleware_test.clj index 20bc20059..e74a9c549 100644 --- a/backend/test/backend_tests/rpc_cond_middleware_test.clj +++ b/backend/test/backend_tests/rpc_cond_middleware_test.clj @@ -15,7 +15,7 @@ [backend-tests.helpers :as th] [backend-tests.storage-test :refer [configure-storage-backend]] [clojure.test :as t] - [datoteka.core :as fs])) + [datoteka.fs :as fs])) (t/use-fixtures :once th/state-init) (t/use-fixtures :each th/database-reset) diff --git a/backend/test/backend_tests/rpc_file_thumbnails_test.clj b/backend/test/backend_tests/rpc_file_thumbnails_test.clj index 1ad3c6e09..5ae85b8a1 100644 --- a/backend/test/backend_tests/rpc_file_thumbnails_test.clj +++ b/backend/test/backend_tests/rpc_file_thumbnails_test.clj @@ -21,7 +21,7 @@ [clojure.java.io :as io] [clojure.test :as t] [cuerdas.core :as str] - [datoteka.core :as fs] + [datoteka.fs :as fs] [mockery.core :refer [with-mocks]])) (t/use-fixtures :once th/state-init) diff --git a/backend/test/backend_tests/rpc_management_test.clj b/backend/test/backend_tests/rpc_management_test.clj index 325831480..63018af33 100644 --- a/backend/test/backend_tests/rpc_management_test.clj +++ b/backend/test/backend_tests/rpc_management_test.clj @@ -16,7 +16,7 @@ [backend-tests.storage-test :refer [configure-storage-backend]] [buddy.core.bytes :as b] [clojure.test :as t] - [datoteka.core :as fs])) + [datoteka.fs :as fs])) (t/use-fixtures :once th/state-init) (t/use-fixtures :each th/database-reset) diff --git a/backend/test/backend_tests/rpc_media_test.clj b/backend/test/backend_tests/rpc_media_test.clj index 552f3e033..5147b1e12 100644 --- a/backend/test/backend_tests/rpc_media_test.clj +++ b/backend/test/backend_tests/rpc_media_test.clj @@ -12,7 +12,7 @@ [app.storage :as sto] [backend-tests.helpers :as th] [clojure.test :as t] - [datoteka.core :as fs])) + [datoteka.fs :as fs])) (t/use-fixtures :once th/state-init) (t/use-fixtures :each th/database-reset) diff --git a/backend/test/backend_tests/rpc_profile_test.clj b/backend/test/backend_tests/rpc_profile_test.clj index 95737cad2..cbaff6038 100644 --- a/backend/test/backend_tests/rpc_profile_test.clj +++ b/backend/test/backend_tests/rpc_profile_test.clj @@ -18,7 +18,7 @@ [clojure.java.io :as io] [clojure.test :as t] [cuerdas.core :as str] - [datoteka.core :as fs] + [datoteka.fs :as fs] [mockery.core :refer [with-mocks]])) ;; TODO: profile deletion with teams diff --git a/backend/test/backend_tests/rpc_quotes_test.clj b/backend/test/backend_tests/rpc_quotes_test.clj index 232af4582..5907da58b 100644 --- a/backend/test/backend_tests/rpc_quotes_test.clj +++ b/backend/test/backend_tests/rpc_quotes_test.clj @@ -14,7 +14,7 @@ [app.rpc.quotes :as-alias quotes] [backend-tests.helpers :as th] [clojure.test :as t] - [datoteka.core :as fs] + [datoteka.fs :as fs] [mockery.core :refer [with-mocks]])) (t/use-fixtures :once th/state-init) diff --git a/backend/test/backend_tests/rpc_team_test.clj b/backend/test/backend_tests/rpc_team_test.clj index 60a42a8c5..65acef49d 100644 --- a/backend/test/backend_tests/rpc_team_test.clj +++ b/backend/test/backend_tests/rpc_team_test.clj @@ -16,7 +16,7 @@ [app.util.time :as dt] [backend-tests.helpers :as th] [clojure.test :as t] - [datoteka.core :as fs] + [datoteka.fs :as fs] [mockery.core :refer [with-mocks]])) (t/use-fixtures :once th/state-init) diff --git a/backend/test/backend_tests/rpc_viewer_test.clj b/backend/test/backend_tests/rpc_viewer_test.clj index b2ef4fa08..6c68c12e3 100644 --- a/backend/test/backend_tests/rpc_viewer_test.clj +++ b/backend/test/backend_tests/rpc_viewer_test.clj @@ -11,7 +11,7 @@ [app.rpc :as-alias rpc] [backend-tests.helpers :as th] [clojure.test :as t] - [datoteka.core :as fs])) + [datoteka.fs :as fs])) (t/use-fixtures :once th/state-init) (t/use-fixtures :each th/database-reset) diff --git a/backend/test/backend_tests/storage_test.clj b/backend/test/backend_tests/storage_test.clj index ccc0d0863..7e21ec970 100644 --- a/backend/test/backend_tests/storage_test.clj +++ b/backend/test/backend_tests/storage_test.clj @@ -15,7 +15,7 @@ [backend-tests.helpers :as th] [clojure.test :as t] [cuerdas.core :as str] - [datoteka.core :as fs] + [datoteka.fs :as fs] [datoteka.io :as io] [mockery.core :refer [with-mocks]])) diff --git a/common/deps.edn b/common/deps.edn index 61c52e909..32a9f3473 100644 --- a/common/deps.edn +++ b/common/deps.edn @@ -32,11 +32,14 @@ funcool/tubax {:mvn/version "2021.05.20-0"} funcool/cuerdas {:mvn/version "2023.11.09-407"} - funcool/promesa {:git/sha "0c5ed6ad033515a2df4b55addea044f60e9653d0" - :git/url "https://github.com/funcool/promesa"} + funcool/promesa + {:git/sha "0c5ed6ad033515a2df4b55addea044f60e9653d0" + :git/url "https://github.com/funcool/promesa"} - funcool/datoteka {:mvn/version "3.0.66" - :exclusions [funcool/promesa]} + funcool/datoteka + {:git/sha "5ac3781" + :git/tag "3.0.0" + :git/url "https://github.com/funcool/datoteka"} lambdaisland/uri {:mvn/version "1.16.134" :exclusions [org.clojure/data.json]}