Merge pull request #4145 from penpot/niwinz-staging-tmp

✨ Minor improvements on TMP storage API
2025-04-01 17:41:50 -05:00 · 2024-02-19 07:16:36 +01:00 · 2024-02-19 07:16:36 +01:00 · 7ac4b89a0e
commit 7ac4b89a0e
parent ff3c948056 a31be7e2ff
15 changed files with 59 additions and 47 deletions
--- a/backend/dev/user.clj
+++ b/backend/dev/user.clj
@ -44,7 +44,7 @@
   [clojure.walk :refer [macroexpand-all]]
   [criterium.core  :as crit]
   [cuerdas.core :as str]
-   [datoteka.core]
+   [datoteka.fs :as fs]
   [integrant.core :as ig]
   [malli.core :as m]
   [malli.dev.pretty :as mdp]
--- a/backend/src/app/features/components_v2.clj
+++ b/backend/src/app/features/components_v2.clj
@ -1524,21 +1524,9 @@
  (cfv/validate-file! file libs)
  (cfv/validate-file-schema! file))

-(defn- process-file
-  [{:keys [::db/conn] :as system} {:keys [id] :as file} & {:keys [validate?]}]
-  (let [libs  (->> (files/get-file-libraries conn id)
-                   (into [file] (comp (map :id)
-                                      (map (partial get-file system))))
-                   (d/index-by :id))
-
-        file  (-> file
-                  (update :data migrate-fdata libs)
-                  (update :features conj "components/v2"))
-
-        _     (when validate?
-                (validate-file! file libs))
-
-        file (if (contains? (:features file) "fdata/objects-map")
+(defn- persist-file!
+  [{:keys [::db/conn] :as system} {:keys [id] :as file}]
+  (let [file (if (contains? (:features file) "fdata/objects-map")
               (fdata/enable-objects-map file)
               file)

@ -1547,15 +1535,32 @@
                 (let [file (fdata/enable-pointer-map file)]
                   (fdata/persist-pointers! system id)
                   file))
-               file)]
+               file)
+
+        ;; Ensure all files has :data with id
+        file (update file :data assoc :id id)]

    (db/update! conn :file
                {:data (blob/encode (:data file))
                 :features (db/create-array conn "text" (:features file))
                 :revn (:revn file)}
-                {:id (:id file)})
+                {:id (:id file)})))

-    (dissoc file :data)))
+(defn- process-file!
+  [{:keys [::db/conn] :as system} {:keys [id] :as file} & {:keys [validate?]}]
+  (let [libs  (->> (files/get-file-libraries conn id)
+                   (into [file] (comp (map :id)
+                                      (map (partial get-file system))))
+                   (d/index-by :id))
+
+        file  (-> file
+                  (update :data migrate-fdata libs)
+                  (update :features conj "components/v2"))]
+
+    (when validate?
+      (validate-file! file libs))
+
+    file))

 (def ^:private sql:get-and-lock-team-files
  "SELECT f.id
@ -1605,13 +1610,15 @@
                        (when (string? label)
                          (fsnap/take-file-snapshot! system {:file-id file-id
                                                             :label (str "migration/" label)}))
-                        (let [file (get-file system file-id)]
+                        (let [file (get-file system file-id)
+                              file (process-file! system file :validate? validate?)]
+
                          (events/tap :progress
                                      {:op :migrate-file
                                       :name (:name file)
                                       :id (:id file)})

-                          (process-file system file :validate? validate?)))))
+                          (persist-file! system file)))))

        (catch Throwable cause
          (vreset! err true)
--- a/backend/src/app/storage/tmp.clj
+++ b/backend/src/app/storage/tmp.clj
@ -19,6 +19,8 @@
   [promesa.exec :as px]
   [promesa.exec.csp :as sp]))

+(def default-tmp-dir "/tmp/penpot")
+
 (declare ^:private remove-temp-file)
 (declare ^:private io-loop)

@ -33,6 +35,7 @@

 (defmethod ig/init-key ::cleaner
  [_ cfg]
+  (fs/create-dir default-tmp-dir)
  (px/fn->thread (partial io-loop cfg)
                 {:name "penpot/storage/tmp-cleaner" :virtual true}))

@ -70,18 +73,14 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

 (defn tempfile
-  "Returns a tmpfile candidate (without creating it)"
  [& {:keys [suffix prefix min-age]
      :or {prefix "penpot."
           suffix ".tmp"}}]
-  (let [path (fs/tempfile :suffix suffix :prefix prefix)]
-    (sp/offer! queue [path (some-> min-age dt/duration)])
-    path))
-
-(defn create-tempfile
-  [& {:keys [suffix prefix min-age]
-      :or {prefix "penpot."
-           suffix ".tmp"}}]
-  (let [path (fs/create-tempfile :suffix suffix :prefix prefix)]
+  (let [path (fs/create-tempfile
+              :perms "rw-r--r--"
+              :dir default-tmp-dir
+              :suffix suffix
+              :prefix prefix)]
+    (fs/delete-on-exit! path)
    (sp/offer! queue [path (some-> min-age dt/duration)])
    path))
--- a/backend/test/backend_tests/helpers.clj
+++ b/backend/test/backend_tests/helpers.clj
@ -38,7 +38,7 @@
   [clojure.spec.alpha :as s]
   [clojure.test :as t]
   [cuerdas.core :as str]
-   [datoteka.core :as fs]
+   [datoteka.fs :as fs]
   [environ.core :refer [env]]
   [expound.alpha :as expound]
   [integrant.core :as ig]
@ -127,6 +127,8 @@
                app.auth/verify-password (fn [a b] {:valid (= a b)})
                app.common.features/get-enabled-features (fn [& _] app.common.features/supported-features)]

+    (fs/create-dir "/tmp/penpot")
+
    (let [templates [{:id "test"
                      :name "test"
                      :file-uri "test"
@ -191,6 +193,7 @@
  (let [path (fs/path "/tmp/penpot")]
    (when (fs/exists? path)
      (fs/delete (fs/path "/tmp/penpot")))
+    (fs/create-dir "/tmp/penpot")
    (next)))

 (defn serial
@ -496,7 +499,7 @@
 (defn tempfile
  [source]
  (let [rsc (io/resource source)
-        tmp (fs/create-tempfile)]
+        tmp (fs/create-tempfile :dir "/tmp/penpot" :prefix "test-")]
    (io/copy (io/file rsc)
             (io/file tmp))
    tmp))
--- a/backend/test/backend_tests/rpc_comment_test.clj
+++ b/backend/test/backend_tests/rpc_comment_test.clj
@ -17,7 +17,7 @@
   [app.util.time :as dt]
   [backend-tests.helpers :as th]
   [clojure.test :as t]
-   [datoteka.core :as fs]
+   [datoteka.fs :as fs]
   [mockery.core :refer [with-mocks]]))

 (t/use-fixtures :once th/state-init)
--- a/backend/test/backend_tests/rpc_cond_middleware_test.clj
+++ b/backend/test/backend_tests/rpc_cond_middleware_test.clj
@ -15,7 +15,7 @@
   [backend-tests.helpers :as th]
   [backend-tests.storage-test :refer [configure-storage-backend]]
   [clojure.test :as t]
-   [datoteka.core :as fs]))
+   [datoteka.fs :as fs]))

 (t/use-fixtures :once th/state-init)
 (t/use-fixtures :each th/database-reset)
--- a/backend/test/backend_tests/rpc_file_thumbnails_test.clj
+++ b/backend/test/backend_tests/rpc_file_thumbnails_test.clj
@ -21,7 +21,7 @@
   [clojure.java.io :as io]
   [clojure.test :as t]
   [cuerdas.core :as str]
-   [datoteka.core :as fs]
+   [datoteka.fs :as fs]
   [mockery.core :refer [with-mocks]]))

 (t/use-fixtures :once th/state-init)
--- a/backend/test/backend_tests/rpc_management_test.clj
+++ b/backend/test/backend_tests/rpc_management_test.clj
@ -16,7 +16,7 @@
   [backend-tests.storage-test :refer [configure-storage-backend]]
   [buddy.core.bytes :as b]
   [clojure.test :as t]
-   [datoteka.core :as fs]))
+   [datoteka.fs :as fs]))

 (t/use-fixtures :once th/state-init)
 (t/use-fixtures :each th/database-reset)
--- a/backend/test/backend_tests/rpc_media_test.clj
+++ b/backend/test/backend_tests/rpc_media_test.clj
@ -12,7 +12,7 @@
   [app.storage :as sto]
   [backend-tests.helpers :as th]
   [clojure.test :as t]
-   [datoteka.core :as fs]))
+   [datoteka.fs :as fs]))

 (t/use-fixtures :once th/state-init)
 (t/use-fixtures :each th/database-reset)
--- a/backend/test/backend_tests/rpc_profile_test.clj
+++ b/backend/test/backend_tests/rpc_profile_test.clj
@ -18,7 +18,7 @@
   [clojure.java.io :as io]
   [clojure.test :as t]
   [cuerdas.core :as str]
-   [datoteka.core :as fs]
+   [datoteka.fs :as fs]
   [mockery.core :refer [with-mocks]]))

 ;; TODO: profile deletion with teams
--- a/backend/test/backend_tests/rpc_quotes_test.clj
+++ b/backend/test/backend_tests/rpc_quotes_test.clj
@ -14,7 +14,7 @@
   [app.rpc.quotes :as-alias quotes]
   [backend-tests.helpers :as th]
   [clojure.test :as t]
-   [datoteka.core :as fs]
+   [datoteka.fs :as fs]
   [mockery.core :refer [with-mocks]]))

 (t/use-fixtures :once th/state-init)
--- a/backend/test/backend_tests/rpc_team_test.clj
+++ b/backend/test/backend_tests/rpc_team_test.clj
@ -16,7 +16,7 @@
   [app.util.time :as dt]
   [backend-tests.helpers :as th]
   [clojure.test :as t]
-   [datoteka.core :as fs]
+   [datoteka.fs :as fs]
   [mockery.core :refer [with-mocks]]))

 (t/use-fixtures :once th/state-init)
--- a/backend/test/backend_tests/rpc_viewer_test.clj
+++ b/backend/test/backend_tests/rpc_viewer_test.clj
@ -11,7 +11,7 @@
   [app.rpc :as-alias rpc]
   [backend-tests.helpers :as th]
   [clojure.test :as t]
-   [datoteka.core :as fs]))
+   [datoteka.fs :as fs]))

 (t/use-fixtures :once th/state-init)
 (t/use-fixtures :each th/database-reset)
--- a/backend/test/backend_tests/storage_test.clj
+++ b/backend/test/backend_tests/storage_test.clj
@ -15,7 +15,7 @@
   [backend-tests.helpers :as th]
   [clojure.test :as t]
   [cuerdas.core :as str]
-   [datoteka.core :as fs]
+   [datoteka.fs :as fs]
   [datoteka.io :as io]
   [mockery.core :refer [with-mocks]]))

--- a/common/deps.edn
+++ b/common/deps.edn
@ -32,11 +32,14 @@

  funcool/tubax {:mvn/version "2021.05.20-0"}
  funcool/cuerdas {:mvn/version "2023.11.09-407"}
-  funcool/promesa {:git/sha "0c5ed6ad033515a2df4b55addea044f60e9653d0"
-                   :git/url "https://github.com/funcool/promesa"}
+  funcool/promesa
+  {:git/sha "0c5ed6ad033515a2df4b55addea044f60e9653d0"
+   :git/url "https://github.com/funcool/promesa"}

-  funcool/datoteka {:mvn/version "3.0.66"
-                    :exclusions [funcool/promesa]}
+  funcool/datoteka
+  {:git/sha "5ac3781"
+   :git/tag "3.0.0"
+   :git/url "https://github.com/funcool/datoteka"}

  lambdaisland/uri {:mvn/version "1.16.134"
                    :exclusions [org.clojure/data.json]}