Penpot/penpot
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-01-26 00:19:07 -05:00
Code Issues Activity

♻️ Adapt media & fonts handling to new tmp service

Browse source 
And storage backend changes
This commit is contained in:
Andrey Antukh 2022-06-22 11:39:57 +02:00
parent ebcb385593
commit 46d075611d
8 changed files with 85 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

81
backend/src/app/media.clj
View file

@ -12,18 +12,16 @@
[app.common.media :as cm] [app.common.media :as cm]
[app.common.spec :as us] [app.common.spec :as us]
[app.config :as cf] [app.config :as cf]
[app.storage.tmp :as tmp]
[app.util.bytes :as bs]
[app.util.svg :as svg] [app.util.svg :as svg]
[buddy.core.bytes :as bb] [buddy.core.bytes :as bb]
[buddy.core.codecs :as bc] [buddy.core.codecs :as bc]
[clojure.java.io :as io]
[clojure.java.shell :as sh] [clojure.java.shell :as sh]
[clojure.spec.alpha :as s] [clojure.spec.alpha :as s]
[cuerdas.core :as str] [cuerdas.core :as str]
[datoteka.core :as fs]) [datoteka.core :as fs])
(:import (:import
java.io.ByteArrayInputStream
java.io.OutputStream
org.apache.commons.io.IOUtils
org.im4java.core.ConvertCmd org.im4java.core.ConvertCmd
org.im4java.core.IMOperation org.im4java.core.IMOperation
org.im4java.core.Info)) org.im4java.core.Info))
@ -93,18 +91,16 @@
(let [{:keys [path mtype]} input (let [{:keys [path mtype]} input
format (or (cm/mtype->format mtype) format) format (or (cm/mtype->format mtype) format)
ext (cm/format->extension format) ext (cm/format->extension format)
tmp (fs/create-tempfile :suffix ext)] tmp (tmp/tempfile :prefix "penpot.media." :suffix ext)]
(doto (ConvertCmd.) (doto (ConvertCmd.)
(.run operation (into-array (map str [path tmp])))) (.run operation (into-array (map str [path tmp]))))
(let [thumbnail-data (fs/slurp-bytes tmp)]
(fs/delete tmp)
(assoc params (assoc params
:format format :format format
:mtype (cm/format->mtype format) :mtype (cm/format->mtype format)
:size (alength ^bytes thumbnail-data) :size (fs/size tmp)
:data (ByteArrayInputStream. thumbnail-data))))) :data tmp)))
(defmethod process :generic-thumbnail (defmethod process :generic-thumbnail
[{:keys [quality width height] :as params}] [{:keys [quality width height] :as params}]
@ -201,58 +197,53 @@
(defmethod process :generate-fonts (defmethod process :generate-fonts
[{:keys [input] :as params}] [{:keys [input] :as params}]
(letfn [(ttf->otf [data] (letfn [(ttf->otf [data]
(let [input-file (fs/create-tempfile :prefix "penpot") (let [finput (tmp/tempfile :prefix "penpot.font." :suffix "")
output-file (fs/path (str input-file ".otf")) foutput (fs/path (str finput ".otf"))
_ (with-open [out (io/output-stream input-file)] _ (bs/write-to-file! data finput)
(IOUtils/writeChunked ^bytes data ^OutputStream out)
(.flush ^OutputStream out))
res (sh/sh "fontforge" "-lang=ff" "-c" res (sh/sh "fontforge" "-lang=ff" "-c"
(str/fmt "Open('%s'); Generate('%s')" (str/fmt "Open('%s'); Generate('%s')"
(str input-file) (str finput)
(str output-file)))] (str foutput)))]
(when (zero? (:exit res)) (when (zero? (:exit res))
(fs/slurp-bytes output-file)))) foutput)))
(otf->ttf [data] (otf->ttf [data]
(let [input-file (fs/create-tempfile :prefix "penpot") (let [finput (tmp/tempfile :prefix "penpot.font." :suffix "")
output-file (fs/path (str input-file ".ttf")) foutput (fs/path (str finput ".ttf"))
_ (with-open [out (io/output-stream input-file)] _ (bs/write-to-file! data finput)
(IOUtils/writeChunked ^bytes data ^OutputStream out)
(.flush ^OutputStream out))
res (sh/sh "fontforge" "-lang=ff" "-c" res (sh/sh "fontforge" "-lang=ff" "-c"
(str/fmt "Open('%s'); Generate('%s')" (str/fmt "Open('%s'); Generate('%s')"
(str input-file) (str finput)
(str output-file)))] (str foutput)))]
(when (zero? (:exit res)) (when (zero? (:exit res))
(fs/slurp-bytes output-file)))) foutput)))
(ttf-or-otf->woff [data] (ttf-or-otf->woff [data]
(let [input-file (fs/create-tempfile :prefix "penpot" :suffix "") ;; NOTE: foutput is not used directly, it represents the
output-file (fs/path (str input-file ".woff")) ;; default output of the exection of the underlying
_ (with-open [out (io/output-stream input-file)] ;; command.
(IOUtils/writeChunked ^bytes data ^OutputStream out) (let [finput (tmp/tempfile :prefix "penpot.font." :suffix "")
(.flush ^OutputStream out)) foutput (fs/path (str finput ".woff"))
res (sh/sh "sfnt2woff" (str input-file))] _ (bs/write-to-file! data finput)
res (sh/sh "sfnt2woff" (str finput))]
(when (zero? (:exit res)) (when (zero? (:exit res))
(fs/slurp-bytes output-file)))) foutput)))
(ttf-or-otf->woff2 [data] (ttf-or-otf->woff2 [data]
(let [input-file (fs/create-tempfile :prefix "penpot" :suffix "") ;; NOTE: foutput is not used directly, it represents the
output-file (fs/path (str input-file ".woff2")) ;; default output of the exection of the underlying
_ (with-open [out (io/output-stream input-file)] ;; command.
(IOUtils/writeChunked ^bytes data ^OutputStream out) (let [finput (tmp/tempfile :prefix "penpot.font." :suffix ".tmp")
(.flush ^OutputStream out)) foutput (fs/path (str (fs/base finput) ".woff2"))
res (sh/sh "woff2_compress" (str input-file))] _ (bs/write-to-file! data finput)
res (sh/sh "woff2_compress" (str finput))]
(when (zero? (:exit res)) (when (zero? (:exit res))
(fs/slurp-bytes output-file)))) foutput)))
(woff->sfnt [data] (woff->sfnt [data]
(let [input-file (fs/create-tempfile :prefix "penpot" :suffix "") (let [finput (tmp/tempfile :prefix "penpot" :suffix "")
_ (with-open [out (io/output-stream input-file)] _ (bs/write-to-file! data finput)
(IOUtils/writeChunked ^bytes data ^OutputStream out) res (sh/sh "woff2sfnt" (str finput)
(.flush ^OutputStream out))
res (sh/sh "woff2sfnt" (str input-file)
:out-enc :bytes)] :out-enc :bytes)]
(when (zero? (:exit res)) (when (zero? (:exit res))
(:out res)))) (:out res))))

6
backend/src/app/rpc/mutations/fonts.clj
View file

@ -71,9 +71,9 @@
data) data)
(persist-font-object [data mtype] (persist-font-object [data mtype]
(when-let [fdata (get data mtype)] (when-let [resource (get data mtype)]
(p/let [hash (calculate-hash fdata) (p/let [hash (calculate-hash resource)
content (-> (sto/content fdata) content (-> (sto/content resource)
(sto/wrap-with-hash hash))] (sto/wrap-with-hash hash))]
(sto/put-object! storage {::sto/content content (sto/put-object! storage {::sto/content content
::sto/touched-at (dt/now) ::sto/touched-at (dt/now)

37
backend/src/app/rpc/mutations/media.clj
View file

@ -17,6 +17,8 @@
[app.rpc.queries.teams :as teams] [app.rpc.queries.teams :as teams]
[app.rpc.rlimit :as rlimit] [app.rpc.rlimit :as rlimit]
[app.storage :as sto] [app.storage :as sto]
[app.storage.tmp :as tmp]
[app.util.bytes :as bs]
[app.util.services :as sv] [app.util.services :as sv]
[app.util.time :as dt] [app.util.time :as dt]
[clojure.spec.alpha :as s] [clojure.spec.alpha :as s]
@ -179,11 +181,12 @@
(* 1024 1024 100)) ; 100MiB (* 1024 1024 100)) ; 100MiB
(defn- create-file-media-object-from-url (defn- create-file-media-object-from-url
[{:keys [storage http-client] :as cfg} {:keys [url name] :as params}] [{:keys [http-client] :as cfg} {:keys [url name] :as params}]
(letfn [(parse-and-validate-size [headers] (letfn [(parse-and-validate-size [headers]
(let [size (some-> (get headers "content-length") d/parse-integer) (let [size (some-> (get headers "content-length") d/parse-integer)
mtype (get headers "content-type") mtype (get headers "content-type")
format (cm/mtype->format mtype)] format (cm/mtype->format mtype)]
(when-not size (when-not size
(ex/raise :type :validation (ex/raise :type :validation
:code :unknown-size :code :unknown-size
@ -203,24 +206,24 @@
:mtype mtype :mtype mtype
:format format})) :format format}))
(get-upload-object [sobj]
(p/let [path (sto/get-object-path storage sobj)
mdata (meta sobj)]
{:filename "tempfile"
:size (:size sobj)
:path path
:mtype (:content-type mdata)}))
(download-media [uri] (download-media [uri]
(p/let [{:keys [body headers]} (http-client {:method :get :uri uri} {:response-type :input-stream}) (-> (http-client {:method :get :uri uri} {:response-type :input-stream})
{:keys [size mtype]} (parse-and-validate-size headers)] (p/then process-response)))
(-> (assoc storage :backend :tmp) (process-response [{:keys [body headers] :as response}]
(sto/put-object! {::sto/content (sto/content body size) (let [{:keys [size mtype]} (parse-and-validate-size headers)
::sto/expired-at (dt/in-future {:minutes 30}) path (tmp/tempfile :prefix "penpot.media.download.")
:content-type mtype written (bs/write-to-file! body path :size size)]
:bucket "file-media-object"})
(p/then get-upload-object))))] (when (not= written size)
(ex/raise :type :internal
:code :mismatch-write-size
:hint "unexpected state: unable to write to file"))
{:filename "tempfile"
:size size
:path path
:mtype mtype}))]
(p/let [content (download-media url)] (p/let [content (download-media url)]
(->> (merge params {:content content :name (or name (:filename content))}) (->> (merge params {:content content :name (or name (:filename content))})

2
backend/src/app/tasks/file_gc.clj
View file

@ -82,7 +82,7 @@
:kf first :kf first
:initk (dt/now))))) :initk (dt/now)))))
(defn- collect-used-media (defn collect-used-media
[data] [data]
(let [xform (comp (let [xform (comp
(map :objects) (map :objects)

7
backend/test/app/services_fonts_test.clj
View file

@ -11,6 +11,7 @@
[app.http :as http] [app.http :as http]
[app.storage :as sto] [app.storage :as sto]
[app.test-helpers :as th] [app.test-helpers :as th]
[app.util.bytes :as bs]
[clojure.java.io :as io] [clojure.java.io :as io]
[clojure.test :as t] [clojure.test :as t]
[datoteka.core :as fs])) [datoteka.core :as fs]))
@ -25,7 +26,8 @@
font-id (uuid/custom 10 1) font-id (uuid/custom 10 1)
ttfdata (-> (io/resource "app/test_files/font-1.ttf") ttfdata (-> (io/resource "app/test_files/font-1.ttf")
(fs/slurp-bytes)) io/input-stream
bs/read-as-bytes)
params {::th/type :create-font-variant params {::th/type :create-font-variant
:profile-id (:id prof) :profile-id (:id prof)
@ -60,7 +62,8 @@
font-id (uuid/custom 10 1) font-id (uuid/custom 10 1)
data (-> (io/resource "app/test_files/font-1.woff") data (-> (io/resource "app/test_files/font-1.woff")
(fs/slurp-bytes)) io/input-stream
bs/read-as-bytes)
params {::th/type :create-font-variant params {::th/type :create-font-variant
:profile-id (:id prof) :profile-id (:id prof)

4
backend/test/app/storage_test.clj
View file

@ -12,6 +12,7 @@
[app.storage :as sto] [app.storage :as sto]
[app.test-helpers :as th] [app.test-helpers :as th]
[app.util.time :as dt] [app.util.time :as dt]
[app.util.bytes :as bs]
[clojure.java.io :as io] [clojure.java.io :as io]
[clojure.test :as t] [clojure.test :as t]
[cuerdas.core :as str] [cuerdas.core :as str]
@ -197,7 +198,8 @@
:is-shared false}) :is-shared false})
ttfdata (-> (io/resource "app/test_files/font-1.ttf") ttfdata (-> (io/resource "app/test_files/font-1.ttf")
(fs/slurp-bytes)) io/input-stream
bs/read-as-bytes)
mfile {:filename "sample.jpg" mfile {:filename "sample.jpg"
:path (th/tempfile "app/test_files/sample.jpg") :path (th/tempfile "app/test_files/sample.jpg")

6
common/src/app/common/exceptions.cljc
View file

@ -50,6 +50,12 @@
[& exprs] [& exprs]
`(try* (^:once fn* [] ~@exprs) identity)) `(try* (^:once fn* [] ~@exprs) identity))
(defn with-always
"A helper that evaluates an exptession independently if the body
raises exception or not."
[always-expr & body]
`(try ~@body (finally ~always-expr)))
(defn ex-info? (defn ex-info?
[v] [v]
(instance? #?(:clj clojure.lang.ExceptionInfo :cljs cljs.core.ExceptionInfo) v)) (instance? #?(:clj clojure.lang.ExceptionInfo :cljs cljs.core.ExceptionInfo) v))

3
common/src/app/common/uuid.cljc
View file

@ -48,3 +48,6 @@
#?(:clj #?(:clj
(dm/export impl/get-word-high)) (dm/export impl/get-word-high))
#?(:clj
(dm/export impl/get-word-low))