From 8253ef90d0c601b8172e4ba6ca523e1cdf9b6a9e Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Sat, 30 Jan 2021 11:28:11 +0100 Subject: [PATCH] :sparkles: Improve handling of temporal files. Store temporal files outside of main fs backend. --- backend/src/app/main.clj | 23 +++++++------- backend/src/app/rpc/mutations/media.clj | 16 +++++----- backend/src/app/storage.clj | 41 +++++++++++++++---------- 3 files changed, 45 insertions(+), 35 deletions(-) diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index f9f1566c8..238366f22 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -287,24 +287,25 @@ :app.storage/storage {:pool (ig/ref :app.db/pool) :executor (ig/ref :app.worker/executor) - :backends {:s3 (ig/ref :app.storage.s3/backend) - :db (ig/ref :app.storage.db/backend) - :fs (ig/ref :app.storage.fs/backend)}} + :backend (:storage-backend cfg/config :fs) + :backends {:s3 (ig/ref [::main :app.storage.s3/backend]) + :db (ig/ref [::main :app.storage.db/backend]) + :fs (ig/ref [::main :app.storage.fs/backend]) + :tmp (ig/ref [::tmp :app.storage.fs/backend])}} - :app.storage.s3/backend + [::main :app.storage.s3/backend] {:region (:storage-s3-region cfg/config) :bucket (:storage-s3-bucket cfg/config)} - :app.storage.fs/backend - {:directory (:storage-fs-directory cfg/config) - :uri (:storage-fs-uri cfg/config)} + [::main :app.storage.fs/backend] + {:directory (:storage-fs-directory cfg/config)} - :app.storage.db/backend + [::tmp :app.storage.fs/backend] + {:directory "/tmp/penpot"} + + [::main :app.storage.db/backend] {:pool (ig/ref :app.db/pool)}} - (let [backend (:storage-backend cfg/config :fs)] - {:app.storage/storage {:backend backend}}) - (when (:telemetry-server-enabled cfg/config) {:app.telemetry/handler {:pool (ig/ref :app.db/pool) diff --git a/backend/src/app/rpc/mutations/media.clj b/backend/src/app/rpc/mutations/media.clj index 89dd6d3e9..a5bf2659b 100644 --- a/backend/src/app/rpc/mutations/media.clj +++ b/backend/src/app/rpc/mutations/media.clj @@ -19,7 +19,7 @@ [app.storage :as sto] [app.util.http :as http] [app.util.services :as sv] - [app.util.storage :as ust] + [app.util.time :as dt] [clojure.java.io :as io] [clojure.spec.alpha :as s] [datoteka.core :as fs])) @@ -73,12 +73,15 @@ data (:body result) mtype (get (:headers result) "content-type") format (cm/mtype->format mtype)] - (if (nil? format) + (when (nil? format) (ex/raise :type :validation :code :media-type-not-allowed - :hint "Seems like the url points to an invalid media object.") - (sto/put-tmp-object storage {:content (sto/content data) - :content-type mtype})))) + :hint "Seems like the url points to an invalid media object.")) + (-> (assoc storage :backend :tmp) + (sto/put-object {:content (sto/content data) + :content-type mtype + :expired-at (dt/in-future {:minutes 30})})))) + (defn create-file-media-object [{:keys [conn storage svgc] :as cfg} {:keys [id file-id is-local name content] :as params}] @@ -132,8 +135,7 @@ (let [mobj (download-media cfg url) content {:filename "tempfile" :size (:size mobj) - :tempfile (-> (sto/get-object-url storage mobj) - (sto/file-url->path)) + :tempfile (sto/get-object-path storage mobj) :content-type (:content-type (meta mobj))} params' (merge params {:content content :name (or name (:filename content))})] diff --git a/backend/src/app/storage.clj b/backend/src/app/storage.clj index 15c1f3330..5d4df1cc3 100644 --- a/backend/src/app/storage.clj +++ b/backend/src/app/storage.clj @@ -87,6 +87,8 @@ (defn- create-database-object [{:keys [conn backend]} {:keys [content] :as object}] (if (instance? StorageObject object) + ;; If we in this condition branch, this means we come from the + ;; clone-object, so we just need to clone it with a new backend. (let [id (uuid/random) mdata (meta object) result (insert-object conn @@ -151,6 +153,14 @@ (declare resolve-backend) +(defn object->relative-path + [{:keys [id] :as obj}] + (impl/id->path id)) + +(defn file-url->path + [url] + (fs/path (java.net.URI. (str url)))) + (defn content ([data] (impl/content data nil)) ([data size] (impl/content data size))) @@ -162,6 +172,7 @@ (retrieve-database-object id))) (defn put-object + "Creates a new object with the provided content." [{:keys [pool conn backend executor] :as storage} {:keys [content] :as params}] (us/assert ::storage storage) (us/assert impl/content? content) @@ -180,6 +191,8 @@ object)) (defn clone-object + "Creates a clone of the provided object using backend basded efficient + method. Always clones objects to the configured default." [{:keys [pool conn executor] :as storage} object] (us/assert ::storage storage) (let [storage (assoc storage :conn (or conn pool)) @@ -217,13 +230,17 @@ (resolve-backend (:backend object)) (impl/get-object-url object options)))) -(defn object->relative-path - [{:keys [id] :as obj}] - (impl/id->path id)) - -(defn file-url->path - [url] - (fs/path (java.net.URI. (str url)))) +(defn get-object-path + "Get the Path to the object. Only works with `:fs` type of + storages." + [{:keys [backend conn path] :as storage} object] + (let [backend (resolve-backend storage (:backend object))] + (when (not= :fs (:type backend)) + (ex/raise :type :internal + :code :operation-not-allowed + :hint "get-object-path only works with fs type backends")) + (-> (impl/get-object-url backend object nil) + (file-url->path)))) (defn del-object [{:keys [conn pool] :as storage} id-or-obj] @@ -231,16 +248,6 @@ (-> (assoc storage :conn (or conn pool)) (delete-database-object (if (uuid? id-or-obj) id-or-obj (:id id-or-obj))))) -(defn put-tmp-object - "A special function for create an object explicitly setting the TMP backend - and marking the object as deleted." - [storage params] - (let [storage (assoc storage :backend :fs) - params (assoc params - :expired-at (dt/in-future {:minutes 30}) - :temporal true)] - (put-object storage params))) - ;; --- impl (defn resolve-backend