Penpot/penpot
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-01-09 08:20:45 -05:00
Code Issues Activity

♻️ Adapt media & fonts handling to new tmp service

Browse source 
And storage backend changes
This commit is contained in:
Andrey Antukh 2022-06-22 11:39:57 +02:00
parent ebcb385593
commit 46d075611d
8 changed files with 85 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

97
backend/src/app/media.clj
View file

@ -12,18 +12,16 @@
[app.common.media :as cm]
[app.common.spec :as us]
[app.config :as cf]
[app.storage.tmp :as tmp]
[app.util.bytes :as bs]
[app.util.svg :as svg]
[buddy.core.bytes :as bb]
[buddy.core.codecs :as bc]
[clojure.java.io :as io]
[clojure.java.shell :as sh]
[clojure.spec.alpha :as s]
[cuerdas.core :as str]
[datoteka.core :as fs])
(:import
java.io.ByteArrayInputStream
java.io.OutputStream
org.apache.commons.io.IOUtils
org.im4java.core.ConvertCmd
org.im4java.core.IMOperation
org.im4java.core.Info))
@ -93,18 +91,16 @@
(let [{:keys [path mtype]} input
format (or (cm/mtype->format mtype) format)
ext (cm/format->extension format)
tmp (fs/create-tempfile :suffix ext)]
tmp (tmp/tempfile :prefix "penpot.media." :suffix ext)]
(doto (ConvertCmd.)
(.run operation (into-array (map str [path tmp]))))
(let [thumbnail-data (fs/slurp-bytes tmp)]
(fs/delete tmp)
(assoc params
:format format
:mtype (cm/format->mtype format)
:size (alength ^bytes thumbnail-data)
:data (ByteArrayInputStream. thumbnail-data)))))
(assoc params
:format format
:mtype (cm/format->mtype format)
:size (fs/size tmp)
:data tmp)))
(defmethod process :generic-thumbnail
[{:keys [quality width height] :as params}]
@ -201,59 +197,54 @@
(defmethod process :generate-fonts
[{:keys [input] :as params}]
(letfn [(ttf->otf [data]
(let [input-file (fs/create-tempfile :prefix "penpot")
output-file (fs/path (str input-file ".otf"))
_ (with-open [out (io/output-stream input-file)]
(IOUtils/writeChunked ^bytes data ^OutputStream out)
(.flush ^OutputStream out))
res (sh/sh "fontforge" "-lang=ff" "-c"
(str/fmt "Open('%s'); Generate('%s')"
(str input-file)
(str output-file)))]
(let [finput (tmp/tempfile :prefix "penpot.font." :suffix "")
foutput (fs/path (str finput ".otf"))
_ (bs/write-to-file! data finput)
res (sh/sh "fontforge" "-lang=ff" "-c"
(str/fmt "Open('%s'); Generate('%s')"
(str finput)
(str foutput)))]
(when (zero? (:exit res))
(fs/slurp-bytes output-file))))
foutput)))
(otf->ttf [data]
(let [input-file (fs/create-tempfile :prefix "penpot")
output-file (fs/path (str input-file ".ttf"))
_ (with-open [out (io/output-stream input-file)]
(IOUtils/writeChunked ^bytes data ^OutputStream out)
(.flush ^OutputStream out))
res (sh/sh "fontforge" "-lang=ff" "-c"
(str/fmt "Open('%s'); Generate('%s')"
(str input-file)
(str output-file)))]
(let [finput (tmp/tempfile :prefix "penpot.font." :suffix "")
foutput (fs/path (str finput ".ttf"))
_ (bs/write-to-file! data finput)
res (sh/sh "fontforge" "-lang=ff" "-c"
(str/fmt "Open('%s'); Generate('%s')"
(str finput)
(str foutput)))]
(when (zero? (:exit res))
(fs/slurp-bytes output-file))))
foutput)))
(ttf-or-otf->woff [data]
(let [input-file (fs/create-tempfile :prefix "penpot" :suffix "")
output-file (fs/path (str input-file ".woff"))
_ (with-open [out (io/output-stream input-file)]
(IOUtils/writeChunked ^bytes data ^OutputStream out)
(.flush ^OutputStream out))
res (sh/sh "sfnt2woff" (str input-file))]
;; NOTE: foutput is not used directly, it represents the
;; default output of the exection of the underlying
;; command.
(let [finput (tmp/tempfile :prefix "penpot.font." :suffix "")
foutput (fs/path (str finput ".woff"))
_ (bs/write-to-file! data finput)
res (sh/sh "sfnt2woff" (str finput))]
(when (zero? (:exit res))
(fs/slurp-bytes output-file))))
foutput)))
(ttf-or-otf->woff2 [data]
(let [input-file (fs/create-tempfile :prefix "penpot" :suffix "")
output-file (fs/path (str input-file ".woff2"))
_ (with-open [out (io/output-stream input-file)]
(IOUtils/writeChunked ^bytes data ^OutputStream out)
(.flush ^OutputStream out))
res (sh/sh "woff2_compress" (str input-file))]
;; NOTE: foutput is not used directly, it represents the
;; default output of the exection of the underlying
;; command.
(let [finput (tmp/tempfile :prefix "penpot.font." :suffix ".tmp")
foutput (fs/path (str (fs/base finput) ".woff2"))
_ (bs/write-to-file! data finput)
res (sh/sh "woff2_compress" (str finput))]
(when (zero? (:exit res))
(fs/slurp-bytes output-file))))
foutput)))
(woff->sfnt [data]
(let [input-file (fs/create-tempfile :prefix "penpot" :suffix "")
_ (with-open [out (io/output-stream input-file)]
(IOUtils/writeChunked ^bytes data ^OutputStream out)
(.flush ^OutputStream out))
res (sh/sh "woff2sfnt" (str input-file)
:out-enc :bytes)]
(let [finput (tmp/tempfile :prefix "penpot" :suffix "")
_ (bs/write-to-file! data finput)
res (sh/sh "woff2sfnt" (str finput)
:out-enc :bytes)]
(when (zero? (:exit res))
(:out res))))

6
backend/src/app/rpc/mutations/fonts.clj
View file

@ -71,9 +71,9 @@
data)
(persist-font-object [data mtype]
(when-let [fdata (get data mtype)]
(p/let [hash (calculate-hash fdata)
content (-> (sto/content fdata)
(when-let [resource (get data mtype)]
(p/let [hash (calculate-hash resource)
content (-> (sto/content resource)
(sto/wrap-with-hash hash))]
(sto/put-object! storage {::sto/content content
::sto/touched-at (dt/now)

37
backend/src/app/rpc/mutations/media.clj
View file

@ -17,6 +17,8 @@
[app.rpc.queries.teams :as teams]
[app.rpc.rlimit :as rlimit]
[app.storage :as sto]
[app.storage.tmp :as tmp]
[app.util.bytes :as bs]
[app.util.services :as sv]
[app.util.time :as dt]
[clojure.spec.alpha :as s]
@ -179,11 +181,12 @@
(* 1024 1024 100)) ; 100MiB
(defn- create-file-media-object-from-url
[{:keys [storage http-client] :as cfg} {:keys [url name] :as params}]
[{:keys [http-client] :as cfg} {:keys [url name] :as params}]
(letfn [(parse-and-validate-size [headers]
(let [size (some-> (get headers "content-length") d/parse-integer)
mtype (get headers "content-type")
format (cm/mtype->format mtype)]
(when-not size
(ex/raise :type :validation
:code :unknown-size
@ -203,24 +206,24 @@
:mtype mtype
:format format}))
(get-upload-object [sobj]
(p/let [path (sto/get-object-path storage sobj)
mdata (meta sobj)]
{:filename "tempfile"
:size (:size sobj)
:path path
:mtype (:content-type mdata)}))
(download-media [uri]
(p/let [{:keys [body headers]} (http-client {:method :get :uri uri} {:response-type :input-stream})
{:keys [size mtype]} (parse-and-validate-size headers)]
(-> (http-client {:method :get :uri uri} {:response-type :input-stream})
(p/then process-response)))
(-> (assoc storage :backend :tmp)
(sto/put-object! {::sto/content (sto/content body size)
::sto/expired-at (dt/in-future {:minutes 30})
:content-type mtype
:bucket "file-media-object"})
(p/then get-upload-object))))]
(process-response [{:keys [body headers] :as response}]
(let [{:keys [size mtype]} (parse-and-validate-size headers)
path (tmp/tempfile :prefix "penpot.media.download.")
written (bs/write-to-file! body path :size size)]
(when (not= written size)
(ex/raise :type :internal
:code :mismatch-write-size
:hint "unexpected state: unable to write to file"))
{:filename "tempfile"
:size size
:path path
:mtype mtype}))]
(p/let [content (download-media url)]
(->> (merge params {:content content :name (or name (:filename content))})

2
backend/src/app/tasks/file_gc.clj
View file

@ -82,7 +82,7 @@
:kf first
:initk (dt/now)))))
(defn- collect-used-media
(defn collect-used-media
[data]
(let [xform (comp
(map :objects)

7
backend/test/app/services_fonts_test.clj
View file

@ -11,6 +11,7 @@
[app.http :as http]
[app.storage :as sto]
[app.test-helpers :as th]
[app.util.bytes :as bs]
[clojure.java.io :as io]
[clojure.test :as t]
[datoteka.core :as fs]))
@ -25,7 +26,8 @@
font-id (uuid/custom 10 1)
ttfdata (-> (io/resource "app/test_files/font-1.ttf")
(fs/slurp-bytes))
io/input-stream
bs/read-as-bytes)
params {::th/type :create-font-variant
:profile-id (:id prof)
@ -60,7 +62,8 @@
font-id (uuid/custom 10 1)
data (-> (io/resource "app/test_files/font-1.woff")
(fs/slurp-bytes))
io/input-stream
bs/read-as-bytes)
params {::th/type :create-font-variant
:profile-id (:id prof)

4
backend/test/app/storage_test.clj
View file

@ -12,6 +12,7 @@
[app.storage :as sto]
[app.test-helpers :as th]
[app.util.time :as dt]
[app.util.bytes :as bs]
[clojure.java.io :as io]
[clojure.test :as t]
[cuerdas.core :as str]
@ -197,7 +198,8 @@
:is-shared false})
ttfdata (-> (io/resource "app/test_files/font-1.ttf")
(fs/slurp-bytes))
io/input-stream
bs/read-as-bytes)
mfile {:filename "sample.jpg"
:path (th/tempfile "app/test_files/sample.jpg")

6
common/src/app/common/exceptions.cljc
View file

@ -50,6 +50,12 @@
[& exprs]
`(try* (^:once fn* [] ~@exprs) identity))
(defn with-always
"A helper that evaluates an exptession independently if the body
raises exception or not."
[always-expr & body]
`(try ~@body (finally ~always-expr)))
(defn ex-info?
[v]
(instance? #?(:clj clojure.lang.ExceptionInfo :cljs cljs.core.ExceptionInfo) v))

3
common/src/app/common/uuid.cljc
View file

@ -48,3 +48,6 @@
#?(:clj
(dm/export impl/get-word-high))
#?(:clj
(dm/export impl/get-word-low))