0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-03-18 10:41:29 -05:00

Improve handling of temporal files.

Store temporal files outside of main fs backend.
This commit is contained in:
Andrey Antukh 2021-01-30 11:28:11 +01:00 committed by Alonso Torres
parent e54b443247
commit 8253ef90d0
3 changed files with 45 additions and 35 deletions

View file

@ -287,24 +287,25 @@
:app.storage/storage
{:pool (ig/ref :app.db/pool)
:executor (ig/ref :app.worker/executor)
:backends {:s3 (ig/ref :app.storage.s3/backend)
:db (ig/ref :app.storage.db/backend)
:fs (ig/ref :app.storage.fs/backend)}}
:backend (:storage-backend cfg/config :fs)
:backends {:s3 (ig/ref [::main :app.storage.s3/backend])
:db (ig/ref [::main :app.storage.db/backend])
:fs (ig/ref [::main :app.storage.fs/backend])
:tmp (ig/ref [::tmp :app.storage.fs/backend])}}
:app.storage.s3/backend
[::main :app.storage.s3/backend]
{:region (:storage-s3-region cfg/config)
:bucket (:storage-s3-bucket cfg/config)}
:app.storage.fs/backend
{:directory (:storage-fs-directory cfg/config)
:uri (:storage-fs-uri cfg/config)}
[::main :app.storage.fs/backend]
{:directory (:storage-fs-directory cfg/config)}
:app.storage.db/backend
[::tmp :app.storage.fs/backend]
{:directory "/tmp/penpot"}
[::main :app.storage.db/backend]
{:pool (ig/ref :app.db/pool)}}
(let [backend (:storage-backend cfg/config :fs)]
{:app.storage/storage {:backend backend}})
(when (:telemetry-server-enabled cfg/config)
{:app.telemetry/handler
{:pool (ig/ref :app.db/pool)

View file

@ -19,7 +19,7 @@
[app.storage :as sto]
[app.util.http :as http]
[app.util.services :as sv]
[app.util.storage :as ust]
[app.util.time :as dt]
[clojure.java.io :as io]
[clojure.spec.alpha :as s]
[datoteka.core :as fs]))
@ -73,12 +73,15 @@
data (:body result)
mtype (get (:headers result) "content-type")
format (cm/mtype->format mtype)]
(if (nil? format)
(when (nil? format)
(ex/raise :type :validation
:code :media-type-not-allowed
:hint "Seems like the url points to an invalid media object.")
(sto/put-tmp-object storage {:content (sto/content data)
:content-type mtype}))))
:hint "Seems like the url points to an invalid media object."))
(-> (assoc storage :backend :tmp)
(sto/put-object {:content (sto/content data)
:content-type mtype
:expired-at (dt/in-future {:minutes 30})}))))
(defn create-file-media-object
[{:keys [conn storage svgc] :as cfg} {:keys [id file-id is-local name content] :as params}]
@ -132,8 +135,7 @@
(let [mobj (download-media cfg url)
content {:filename "tempfile"
:size (:size mobj)
:tempfile (-> (sto/get-object-url storage mobj)
(sto/file-url->path))
:tempfile (sto/get-object-path storage mobj)
:content-type (:content-type (meta mobj))}
params' (merge params {:content content
:name (or name (:filename content))})]

View file

@ -87,6 +87,8 @@
(defn- create-database-object
[{:keys [conn backend]} {:keys [content] :as object}]
(if (instance? StorageObject object)
;; If we in this condition branch, this means we come from the
;; clone-object, so we just need to clone it with a new backend.
(let [id (uuid/random)
mdata (meta object)
result (insert-object conn
@ -151,6 +153,14 @@
(declare resolve-backend)
(defn object->relative-path
[{:keys [id] :as obj}]
(impl/id->path id))
(defn file-url->path
[url]
(fs/path (java.net.URI. (str url))))
(defn content
([data] (impl/content data nil))
([data size] (impl/content data size)))
@ -162,6 +172,7 @@
(retrieve-database-object id)))
(defn put-object
"Creates a new object with the provided content."
[{:keys [pool conn backend executor] :as storage} {:keys [content] :as params}]
(us/assert ::storage storage)
(us/assert impl/content? content)
@ -180,6 +191,8 @@
object))
(defn clone-object
"Creates a clone of the provided object using backend basded efficient
method. Always clones objects to the configured default."
[{:keys [pool conn executor] :as storage} object]
(us/assert ::storage storage)
(let [storage (assoc storage :conn (or conn pool))
@ -217,13 +230,17 @@
(resolve-backend (:backend object))
(impl/get-object-url object options))))
(defn object->relative-path
[{:keys [id] :as obj}]
(impl/id->path id))
(defn file-url->path
[url]
(fs/path (java.net.URI. (str url))))
(defn get-object-path
"Get the Path to the object. Only works with `:fs` type of
storages."
[{:keys [backend conn path] :as storage} object]
(let [backend (resolve-backend storage (:backend object))]
(when (not= :fs (:type backend))
(ex/raise :type :internal
:code :operation-not-allowed
:hint "get-object-path only works with fs type backends"))
(-> (impl/get-object-url backend object nil)
(file-url->path))))
(defn del-object
[{:keys [conn pool] :as storage} id-or-obj]
@ -231,16 +248,6 @@
(-> (assoc storage :conn (or conn pool))
(delete-database-object (if (uuid? id-or-obj) id-or-obj (:id id-or-obj)))))
(defn put-tmp-object
"A special function for create an object explicitly setting the TMP backend
and marking the object as deleted."
[storage params]
(let [storage (assoc storage :backend :fs)
params (assoc params
:expired-at (dt/in-future {:minutes 30})
:temporal true)]
(put-object storage params)))
;; --- impl
(defn resolve-backend