From 46d075611d85b83139a9156a032472c438249d79 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 22 Jun 2022 11:39:57 +0200 Subject: [PATCH] :recycle: Adapt media & fonts handling to new tmp service And storage backend changes --- backend/src/app/media.clj | 97 +++++++++++------------- backend/src/app/rpc/mutations/fonts.clj | 6 +- backend/src/app/rpc/mutations/media.clj | 37 ++++----- backend/src/app/tasks/file_gc.clj | 2 +- backend/test/app/services_fonts_test.clj | 7 +- backend/test/app/storage_test.clj | 4 +- common/src/app/common/exceptions.cljc | 6 ++ common/src/app/common/uuid.cljc | 3 + 8 files changed, 85 insertions(+), 77 deletions(-) diff --git a/backend/src/app/media.clj b/backend/src/app/media.clj index 3814d9473..99cbe15cd 100644 --- a/backend/src/app/media.clj +++ b/backend/src/app/media.clj @@ -12,18 +12,16 @@ [app.common.media :as cm] [app.common.spec :as us] [app.config :as cf] + [app.storage.tmp :as tmp] + [app.util.bytes :as bs] [app.util.svg :as svg] [buddy.core.bytes :as bb] [buddy.core.codecs :as bc] - [clojure.java.io :as io] [clojure.java.shell :as sh] [clojure.spec.alpha :as s] [cuerdas.core :as str] [datoteka.core :as fs]) (:import - java.io.ByteArrayInputStream - java.io.OutputStream - org.apache.commons.io.IOUtils org.im4java.core.ConvertCmd org.im4java.core.IMOperation org.im4java.core.Info)) @@ -93,18 +91,16 @@ (let [{:keys [path mtype]} input format (or (cm/mtype->format mtype) format) ext (cm/format->extension format) - tmp (fs/create-tempfile :suffix ext)] + tmp (tmp/tempfile :prefix "penpot.media." :suffix ext)] (doto (ConvertCmd.) (.run operation (into-array (map str [path tmp])))) - (let [thumbnail-data (fs/slurp-bytes tmp)] - (fs/delete tmp) - (assoc params - :format format - :mtype (cm/format->mtype format) - :size (alength ^bytes thumbnail-data) - :data (ByteArrayInputStream. thumbnail-data))))) + (assoc params + :format format + :mtype (cm/format->mtype format) + :size (fs/size tmp) + :data tmp))) (defmethod process :generic-thumbnail [{:keys [quality width height] :as params}] @@ -201,59 +197,54 @@ (defmethod process :generate-fonts [{:keys [input] :as params}] (letfn [(ttf->otf [data] - (let [input-file (fs/create-tempfile :prefix "penpot") - output-file (fs/path (str input-file ".otf")) - _ (with-open [out (io/output-stream input-file)] - (IOUtils/writeChunked ^bytes data ^OutputStream out) - (.flush ^OutputStream out)) - res (sh/sh "fontforge" "-lang=ff" "-c" - (str/fmt "Open('%s'); Generate('%s')" - (str input-file) - (str output-file)))] + (let [finput (tmp/tempfile :prefix "penpot.font." :suffix "") + foutput (fs/path (str finput ".otf")) + _ (bs/write-to-file! data finput) + res (sh/sh "fontforge" "-lang=ff" "-c" + (str/fmt "Open('%s'); Generate('%s')" + (str finput) + (str foutput)))] (when (zero? (:exit res)) - (fs/slurp-bytes output-file)))) - + foutput))) (otf->ttf [data] - (let [input-file (fs/create-tempfile :prefix "penpot") - output-file (fs/path (str input-file ".ttf")) - _ (with-open [out (io/output-stream input-file)] - (IOUtils/writeChunked ^bytes data ^OutputStream out) - (.flush ^OutputStream out)) - res (sh/sh "fontforge" "-lang=ff" "-c" - (str/fmt "Open('%s'); Generate('%s')" - (str input-file) - (str output-file)))] + (let [finput (tmp/tempfile :prefix "penpot.font." :suffix "") + foutput (fs/path (str finput ".ttf")) + _ (bs/write-to-file! data finput) + res (sh/sh "fontforge" "-lang=ff" "-c" + (str/fmt "Open('%s'); Generate('%s')" + (str finput) + (str foutput)))] (when (zero? (:exit res)) - (fs/slurp-bytes output-file)))) + foutput))) (ttf-or-otf->woff [data] - (let [input-file (fs/create-tempfile :prefix "penpot" :suffix "") - output-file (fs/path (str input-file ".woff")) - _ (with-open [out (io/output-stream input-file)] - (IOUtils/writeChunked ^bytes data ^OutputStream out) - (.flush ^OutputStream out)) - res (sh/sh "sfnt2woff" (str input-file))] + ;; NOTE: foutput is not used directly, it represents the + ;; default output of the exection of the underlying + ;; command. + (let [finput (tmp/tempfile :prefix "penpot.font." :suffix "") + foutput (fs/path (str finput ".woff")) + _ (bs/write-to-file! data finput) + res (sh/sh "sfnt2woff" (str finput))] (when (zero? (:exit res)) - (fs/slurp-bytes output-file)))) + foutput))) (ttf-or-otf->woff2 [data] - (let [input-file (fs/create-tempfile :prefix "penpot" :suffix "") - output-file (fs/path (str input-file ".woff2")) - _ (with-open [out (io/output-stream input-file)] - (IOUtils/writeChunked ^bytes data ^OutputStream out) - (.flush ^OutputStream out)) - res (sh/sh "woff2_compress" (str input-file))] + ;; NOTE: foutput is not used directly, it represents the + ;; default output of the exection of the underlying + ;; command. + (let [finput (tmp/tempfile :prefix "penpot.font." :suffix ".tmp") + foutput (fs/path (str (fs/base finput) ".woff2")) + _ (bs/write-to-file! data finput) + res (sh/sh "woff2_compress" (str finput))] (when (zero? (:exit res)) - (fs/slurp-bytes output-file)))) + foutput))) (woff->sfnt [data] - (let [input-file (fs/create-tempfile :prefix "penpot" :suffix "") - _ (with-open [out (io/output-stream input-file)] - (IOUtils/writeChunked ^bytes data ^OutputStream out) - (.flush ^OutputStream out)) - res (sh/sh "woff2sfnt" (str input-file) - :out-enc :bytes)] + (let [finput (tmp/tempfile :prefix "penpot" :suffix "") + _ (bs/write-to-file! data finput) + res (sh/sh "woff2sfnt" (str finput) + :out-enc :bytes)] (when (zero? (:exit res)) (:out res)))) diff --git a/backend/src/app/rpc/mutations/fonts.clj b/backend/src/app/rpc/mutations/fonts.clj index 91b8024d8..a405c279c 100644 --- a/backend/src/app/rpc/mutations/fonts.clj +++ b/backend/src/app/rpc/mutations/fonts.clj @@ -71,9 +71,9 @@ data) (persist-font-object [data mtype] - (when-let [fdata (get data mtype)] - (p/let [hash (calculate-hash fdata) - content (-> (sto/content fdata) + (when-let [resource (get data mtype)] + (p/let [hash (calculate-hash resource) + content (-> (sto/content resource) (sto/wrap-with-hash hash))] (sto/put-object! storage {::sto/content content ::sto/touched-at (dt/now) diff --git a/backend/src/app/rpc/mutations/media.clj b/backend/src/app/rpc/mutations/media.clj index 70c8c20ff..66ef41da6 100644 --- a/backend/src/app/rpc/mutations/media.clj +++ b/backend/src/app/rpc/mutations/media.clj @@ -17,6 +17,8 @@ [app.rpc.queries.teams :as teams] [app.rpc.rlimit :as rlimit] [app.storage :as sto] + [app.storage.tmp :as tmp] + [app.util.bytes :as bs] [app.util.services :as sv] [app.util.time :as dt] [clojure.spec.alpha :as s] @@ -179,11 +181,12 @@ (* 1024 1024 100)) ; 100MiB (defn- create-file-media-object-from-url - [{:keys [storage http-client] :as cfg} {:keys [url name] :as params}] + [{:keys [http-client] :as cfg} {:keys [url name] :as params}] (letfn [(parse-and-validate-size [headers] (let [size (some-> (get headers "content-length") d/parse-integer) mtype (get headers "content-type") format (cm/mtype->format mtype)] + (when-not size (ex/raise :type :validation :code :unknown-size @@ -203,24 +206,24 @@ :mtype mtype :format format})) - (get-upload-object [sobj] - (p/let [path (sto/get-object-path storage sobj) - mdata (meta sobj)] - {:filename "tempfile" - :size (:size sobj) - :path path - :mtype (:content-type mdata)})) - (download-media [uri] - (p/let [{:keys [body headers]} (http-client {:method :get :uri uri} {:response-type :input-stream}) - {:keys [size mtype]} (parse-and-validate-size headers)] + (-> (http-client {:method :get :uri uri} {:response-type :input-stream}) + (p/then process-response))) - (-> (assoc storage :backend :tmp) - (sto/put-object! {::sto/content (sto/content body size) - ::sto/expired-at (dt/in-future {:minutes 30}) - :content-type mtype - :bucket "file-media-object"}) - (p/then get-upload-object))))] + (process-response [{:keys [body headers] :as response}] + (let [{:keys [size mtype]} (parse-and-validate-size headers) + path (tmp/tempfile :prefix "penpot.media.download.") + written (bs/write-to-file! body path :size size)] + + (when (not= written size) + (ex/raise :type :internal + :code :mismatch-write-size + :hint "unexpected state: unable to write to file")) + + {:filename "tempfile" + :size size + :path path + :mtype mtype}))] (p/let [content (download-media url)] (->> (merge params {:content content :name (or name (:filename content))}) diff --git a/backend/src/app/tasks/file_gc.clj b/backend/src/app/tasks/file_gc.clj index 029f0b7fe..29ad2eeb8 100644 --- a/backend/src/app/tasks/file_gc.clj +++ b/backend/src/app/tasks/file_gc.clj @@ -82,7 +82,7 @@ :kf first :initk (dt/now))))) -(defn- collect-used-media +(defn collect-used-media [data] (let [xform (comp (map :objects) diff --git a/backend/test/app/services_fonts_test.clj b/backend/test/app/services_fonts_test.clj index 71f217b6a..dfe87e569 100644 --- a/backend/test/app/services_fonts_test.clj +++ b/backend/test/app/services_fonts_test.clj @@ -11,6 +11,7 @@ [app.http :as http] [app.storage :as sto] [app.test-helpers :as th] + [app.util.bytes :as bs] [clojure.java.io :as io] [clojure.test :as t] [datoteka.core :as fs])) @@ -25,7 +26,8 @@ font-id (uuid/custom 10 1) ttfdata (-> (io/resource "app/test_files/font-1.ttf") - (fs/slurp-bytes)) + io/input-stream + bs/read-as-bytes) params {::th/type :create-font-variant :profile-id (:id prof) @@ -60,7 +62,8 @@ font-id (uuid/custom 10 1) data (-> (io/resource "app/test_files/font-1.woff") - (fs/slurp-bytes)) + io/input-stream + bs/read-as-bytes) params {::th/type :create-font-variant :profile-id (:id prof) diff --git a/backend/test/app/storage_test.clj b/backend/test/app/storage_test.clj index cab9e01d8..832fbdc6f 100644 --- a/backend/test/app/storage_test.clj +++ b/backend/test/app/storage_test.clj @@ -12,6 +12,7 @@ [app.storage :as sto] [app.test-helpers :as th] [app.util.time :as dt] + [app.util.bytes :as bs] [clojure.java.io :as io] [clojure.test :as t] [cuerdas.core :as str] @@ -197,7 +198,8 @@ :is-shared false}) ttfdata (-> (io/resource "app/test_files/font-1.ttf") - (fs/slurp-bytes)) + io/input-stream + bs/read-as-bytes) mfile {:filename "sample.jpg" :path (th/tempfile "app/test_files/sample.jpg") diff --git a/common/src/app/common/exceptions.cljc b/common/src/app/common/exceptions.cljc index d76f943e5..424a594a8 100644 --- a/common/src/app/common/exceptions.cljc +++ b/common/src/app/common/exceptions.cljc @@ -50,6 +50,12 @@ [& exprs] `(try* (^:once fn* [] ~@exprs) identity)) +(defn with-always + "A helper that evaluates an exptession independently if the body + raises exception or not." + [always-expr & body] + `(try ~@body (finally ~always-expr))) + (defn ex-info? [v] (instance? #?(:clj clojure.lang.ExceptionInfo :cljs cljs.core.ExceptionInfo) v)) diff --git a/common/src/app/common/uuid.cljc b/common/src/app/common/uuid.cljc index 604502900..1a71256ef 100644 --- a/common/src/app/common/uuid.cljc +++ b/common/src/app/common/uuid.cljc @@ -48,3 +48,6 @@ #?(:clj (dm/export impl/get-word-high)) + +#?(:clj + (dm/export impl/get-word-low))