diff --git a/backend/src/app/http/debug.clj b/backend/src/app/http/debug.clj index 21dcb50f0..507c091d7 100644 --- a/backend/src/app/http/debug.clj +++ b/backend/src/app/http/debug.clj @@ -17,13 +17,12 @@ [app.rpc.mutations.files :refer [create-file]] [app.rpc.queries.profile :as profile] [app.util.blob :as blob] - [app.util.bytes :as bs] [app.util.template :as tmpl] [app.util.time :as dt] [app.worker :as wrk] - [clojure.java.io :as io] [clojure.spec.alpha :as s] [cuerdas.core :as str] + [datoteka.io :as io] [emoji.core :as emj] [integrant.core :as ig] [markdown.core :as md] @@ -126,7 +125,7 @@ (defn- upload-file-data [{:keys [pool]} {:keys [profile-id params] :as request}] (let [project-id (some-> (profile/retrieve-additional-data pool profile-id) :default-project-id) - data (some-> params :file :path bs/read-as-bytes blob/decode)] + data (some-> params :file :path io/read-as-bytes blob/decode)] (if (and data project-id) (let [fname (str "Imported file *: " (dt/now)) diff --git a/backend/src/app/media.clj b/backend/src/app/media.clj index 796047f10..c2755f425 100644 --- a/backend/src/app/media.clj +++ b/backend/src/app/media.clj @@ -13,14 +13,14 @@ [app.common.spec :as us] [app.config :as cf] [app.storage.tmp :as tmp] - [app.util.bytes :as bs] [app.util.svg :as svg] [buddy.core.bytes :as bb] [buddy.core.codecs :as bc] [clojure.java.shell :as sh] [clojure.spec.alpha :as s] [cuerdas.core :as str] - [datoteka.fs :as fs]) + [datoteka.fs :as fs] + [datoteka.io :as io]) (:import org.im4java.core.ConvertCmd org.im4java.core.IMOperation @@ -199,7 +199,7 @@ (letfn [(ttf->otf [data] (let [finput (tmp/tempfile :prefix "penpot.font." :suffix "") foutput (fs/path (str finput ".otf")) - _ (bs/write-to-file! data finput) + _ (io/write-to-file! data finput) res (sh/sh "fontforge" "-lang=ff" "-c" (str/fmt "Open('%s'); Generate('%s')" (str finput) @@ -210,7 +210,7 @@ (otf->ttf [data] (let [finput (tmp/tempfile :prefix "penpot.font." :suffix "") foutput (fs/path (str finput ".ttf")) - _ (bs/write-to-file! data finput) + _ (io/write-to-file! data finput) res (sh/sh "fontforge" "-lang=ff" "-c" (str/fmt "Open('%s'); Generate('%s')" (str finput) @@ -224,7 +224,7 @@ ;; command. (let [finput (tmp/tempfile :prefix "penpot.font." :suffix "") foutput (fs/path (str finput ".woff")) - _ (bs/write-to-file! data finput) + _ (io/write-to-file! data finput) res (sh/sh "sfnt2woff" (str finput))] (when (zero? (:exit res)) foutput))) @@ -235,14 +235,14 @@ ;; command. (let [finput (tmp/tempfile :prefix "penpot.font." :suffix ".tmp") foutput (fs/path (str (fs/base finput) ".woff2")) - _ (bs/write-to-file! data finput) + _ (io/write-to-file! data finput) res (sh/sh "woff2_compress" (str finput))] (when (zero? (:exit res)) foutput))) (woff->sfnt [data] (let [finput (tmp/tempfile :prefix "penpot" :suffix "") - _ (bs/write-to-file! data finput) + _ (io/write-to-file! data finput) res (sh/sh "woff2sfnt" (str finput) :out-enc :bytes)] (when (zero? (:exit res)) diff --git a/backend/src/app/rpc/commands/binfile.clj b/backend/src/app/rpc/commands/binfile.clj index 48b2b04c6..ce743cb90 100644 --- a/backend/src/app/rpc/commands/binfile.clj +++ b/backend/src/app/rpc/commands/binfile.clj @@ -23,16 +23,17 @@ [app.storage.tmp :as tmp] [app.tasks.file-gc] [app.util.blob :as blob] - [app.util.bytes :as bs] [app.util.fressian :as fres] [app.util.services :as sv] [app.util.time :as dt] - [clojure.java.io :as io] [clojure.spec.alpha :as s] [clojure.walk :as walk] [cuerdas.core :as str] + [datoteka.io :as io] [yetti.adapter :as yt]) (:import + com.github.luben.zstd.ZstdInputStream + com.github.luben.zstd.ZstdOutputStream java.io.DataInputStream java.io.DataOutputStream java.io.InputStream @@ -201,7 +202,7 @@ :position @*position* ::l/async false) (let [vers (-> version name (subs 1) parse-long) - output (bs/data-output-stream output)] + output (io/data-output-stream output)] (doto output (write-byte! (get-mark :header)) (write-long! penpot-magic-number) @@ -210,7 +211,7 @@ (defn read-header! [^InputStream input] (l/trace :fn "read-header!" :position @*position* ::l/async false) - (let [input (bs/data-input-stream input) + (let [input (io/data-input-stream input) mark (read-byte! input) mnum (read-long! input) vers (read-long! input)] @@ -225,7 +226,7 @@ (defn copy-stream! [^OutputStream output ^InputStream input ^long size] - (let [written (bs/copy! input output :size size)] + (let [written (io/copy! input output :size size)] (l/trace :fn "copy-stream!" :position @*position* :size size :written written ::l/async false) (swap! *position* + written) written)) @@ -254,11 +255,11 @@ (if (> s temp-file-threshold) (with-open [^OutputStream output (io/output-stream p)] - (let [readed (bs/copy! input output :offset 0 :size s)] + (let [readed (io/copy! input output :offset 0 :size s)] (l/trace :fn "read-stream*!" :expected s :readed readed :position @*position* ::l/async false) (swap! *position* + readed) [s p])) - [s (bs/read-as-bytes input :size s)]))) + [s (io/read-as-bytes input :size s)]))) (defmacro assert-read-label! [input expected-label] @@ -275,6 +276,16 @@ ;; --- HELPERS +(defn zstd-input-stream + ^InputStream + [input] + (ZstdInputStream. ^InputStream input)) + +(defn zstd-output-stream + ^OutputStream + [output & {:keys [level] :or {level 0}}] + (ZstdOutputStream. ^OutputStream output (int level))) + (defn- retrieve-file [pool file-id] (->> (db/query pool :file {:id file-id}) @@ -411,7 +422,7 @@ (defmulti write-export ::version) (defmulti write-section ::section) -(s/def ::output bs/output-stream?) +(s/def ::output io/output-stream?) (s/def ::file-ids (s/every ::us/uuid :kind vector? :min-count 1)) (s/def ::include-libraries? (s/nilable ::us/boolean)) (s/def ::embed-assets? (s/nilable ::us/boolean)) @@ -441,8 +452,8 @@ (defmethod write-export :default [{:keys [::output] :as options}] (write-header! output :v1) - (with-open [output (bs/zstd-output-stream output :level 12)] - (with-open [output (bs/data-output-stream output)] + (with-open [output (zstd-output-stream output :level 12)] + (with-open [output (io/data-output-stream output)] (binding [*state* (volatile! {})] (run! (fn [section] (l/debug :hint "write section" :section section ::l/async false) @@ -530,7 +541,7 @@ (defmulti read-section ::section) (s/def ::project-id ::us/uuid) -(s/def ::input bs/input-stream?) +(s/def ::input io/input-stream?) (s/def ::overwrite? (s/nilable ::us/boolean)) (s/def ::migrate? (s/nilable ::us/boolean)) (s/def ::ignore-index-errors? (s/nilable ::us/boolean)) @@ -562,8 +573,8 @@ (defmethod read-import :v1 [{:keys [pool ::input] :as options}] - (with-open [input (bs/zstd-input-stream input)] - (with-open [input (bs/data-input-stream input)] + (with-open [input (zstd-input-stream input)] + (with-open [input (io/data-input-stream input)] (db/with-atomic [conn pool] (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED;"]) (binding [*state* (volatile! {:media [] :index {}})] diff --git a/backend/src/app/rpc/mutations/media.clj b/backend/src/app/rpc/mutations/media.clj index 50ab06e0f..30df19922 100644 --- a/backend/src/app/rpc/mutations/media.clj +++ b/backend/src/app/rpc/mutations/media.clj @@ -18,11 +18,11 @@ [app.rpc.semaphore :as rsem] [app.storage :as sto] [app.storage.tmp :as tmp] - [app.util.bytes :as bs] [app.util.services :as sv] [app.util.time :as dt] [clojure.spec.alpha :as s] [cuerdas.core :as str] + [datoteka.io :as io] [promesa.core :as p] [promesa.exec :as px])) @@ -224,7 +224,7 @@ (process-response [{:keys [body headers] :as response}] (let [{:keys [size mtype]} (parse-and-validate-size headers) path (tmp/tempfile :prefix "penpot.media.download.") - written (bs/write-to-file! body path :size size)] + written (io/write-to-file! body path :size size)] (when (not= written size) (ex/raise :type :internal diff --git a/backend/src/app/storage/fs.clj b/backend/src/app/storage/fs.clj index fbfbc8369..40d6e09d6 100644 --- a/backend/src/app/storage/fs.clj +++ b/backend/src/app/storage/fs.clj @@ -10,11 +10,10 @@ [app.common.spec :as us] [app.common.uri :as u] [app.storage.impl :as impl] - [app.util.bytes :as bs] - [clojure.java.io :as io] [clojure.spec.alpha :as s] [cuerdas.core :as str] [datoteka.fs :as fs] + [datoteka.io :as io] [integrant.core :as ig] [promesa.core :as p] [promesa.exec :as px]) @@ -59,7 +58,7 @@ (fs/create-dir (fs/parent full))) (with-open [^InputStream src (io/input-stream content) ^OutputStream dst (io/output-stream full)] - (io/copy src dst))))) + (io/copy! src dst))))) (defmethod impl/get-object-data :fs [{:keys [executor] :as backend} {:keys [id] :as object}] @@ -76,8 +75,8 @@ (defmethod impl/get-object-bytes :fs [backend object] (p/let [input (impl/get-object-data backend object)] - (ex/with-always (bs/close! input) - (bs/read-as-bytes input)))) + (ex/with-always (io/close! input) + (io/read-as-bytes input)))) (defmethod impl/get-object-url :fs [{:keys [uri executor] :as backend} {:keys [id] :as object} _] diff --git a/backend/src/app/storage/impl.clj b/backend/src/app/storage/impl.clj index bca9b5e20..6d9e0336b 100644 --- a/backend/src/app/storage/impl.clj +++ b/backend/src/app/storage/impl.clj @@ -9,10 +9,10 @@ (:require [app.common.data.macros :as dm] [app.common.exceptions :as ex] - [app.util.bytes :as bs] [buddy.core.codecs :as bc] [buddy.core.hash :as bh] - [clojure.java.io :as io]) + [clojure.java.io :as jio] + [datoteka.io :as io]) (:import java.nio.ByteBuffer java.nio.file.Files @@ -110,14 +110,14 @@ IContentObject (get-size [_] size) - io/IOFactory + jio/IOFactory (make-reader [this opts] - (io/make-reader this opts)) + (jio/make-reader this opts)) (make-writer [_ _] (throw (UnsupportedOperationException. "not implemented"))) (make-input-stream [_ _] (-> (io/input-stream path) - (bs/bounded-input-stream size))) + (io/bounded-input-stream size))) (make-output-stream [_ _] (throw (UnsupportedOperationException. "not implemented"))))) @@ -127,14 +127,14 @@ IContentObject (get-size [_] size) - io/IOFactory + jio/IOFactory (make-reader [this opts] - (io/make-reader this opts)) + (jio/make-reader this opts)) (make-writer [_ _] (throw (UnsupportedOperationException. "not implemented"))) (make-input-stream [_ _] - (-> (bs/bytes-input-stream data) - (bs/bounded-input-stream size))) + (-> (io/bytes-input-stream data) + (io/bounded-input-stream size))) (make-output-stream [_ _] (throw (UnsupportedOperationException. "not implemented"))))) @@ -169,7 +169,7 @@ (when-not (satisfies? IContentObject content) (throw (UnsupportedOperationException. "`content` should be an instance of IContentObject"))) - (when-not (satisfies? io/IOFactory content) + (when-not (satisfies? jio/IOFactory content) (throw (UnsupportedOperationException. "`content` should be an instance of IOFactory"))) (reify @@ -179,15 +179,15 @@ IContentHash (get-hash [_] hash) - io/IOFactory + jio/IOFactory (make-reader [_ opts] - (io/make-reader content opts)) + (jio/make-reader content opts)) (make-writer [_ opts] - (io/make-writer content opts)) + (jio/make-writer content opts)) (make-input-stream [_ opts] - (io/make-input-stream content opts)) + (jio/make-input-stream content opts)) (make-output-stream [_ opts] - (io/make-output-stream content opts)))) + (jio/make-output-stream content opts)))) (defn content? [v] diff --git a/backend/src/app/util/bytes.clj b/backend/src/app/util/bytes.clj deleted file mode 100644 index 5e5c2dca4..000000000 --- a/backend/src/app/util/bytes.clj +++ /dev/null @@ -1,128 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) UXBOX Labs SL - -(ns app.util.bytes - "Bytes & Byte Streams helpers" - (:require - [clojure.java.io :as io] - [datoteka.fs :as fs] - [yetti.adapter :as yt]) - (:import - com.github.luben.zstd.ZstdInputStream - com.github.luben.zstd.ZstdOutputStream - java.io.ByteArrayInputStream - java.io.ByteArrayOutputStream - java.io.DataInputStream - java.io.DataOutputStream - java.io.OutputStream - java.io.InputStream - java.lang.AutoCloseable - org.apache.commons.io.IOUtils - org.apache.commons.io.input.BoundedInputStream)) - -;; TODO: migrate to datoteka.io - -(set! *warn-on-reflection* true) - -(def ^:const default-buffer-size - (:xnio/buffer-size yt/defaults)) - -(defn input-stream? - [s] - (instance? InputStream s)) - -(defn output-stream? - [s] - (instance? OutputStream s)) - -(defn data-input-stream? - [s] - (instance? DataInputStream s)) - -(defn data-output-stream? - [s] - (instance? DataOutputStream s)) - -(defn copy! - [src dst & {:keys [offset size buffer-size] - :or {offset 0 buffer-size default-buffer-size}}] - (let [^bytes buff (byte-array buffer-size)] - (if size - (IOUtils/copyLarge ^InputStream src ^OutputStream dst (long offset) (long size) buff) - (IOUtils/copyLarge ^InputStream src ^OutputStream dst buff)))) - -(defn write-to-file! - [src dst & {:keys [size]}] - (with-open [^OutputStream output (io/output-stream dst)] - (cond - (bytes? src) - (if size - (with-open [^InputStream input (ByteArrayInputStream. ^bytes src)] - (with-open [^InputStream input (BoundedInputStream. input (or size (alength ^bytes src)))] - (copy! input output :size size))) - - (do - (IOUtils/writeChunked ^bytes src output) - (.flush ^OutputStream output) - (alength ^bytes src))) - - (instance? InputStream src) - (copy! src output :size size) - - :else - (throw (IllegalArgumentException. "invalid arguments"))))) - -(defn read-as-bytes - "Read input stream as byte array." - [input & {:keys [size]}] - (cond - (instance? InputStream input) - (with-open [output (ByteArrayOutputStream. (or size (.available ^InputStream input)))] - (copy! input output :size size) - (.toByteArray output)) - - (fs/path? input) - (with-open [input (io/input-stream input) - output (ByteArrayOutputStream. (or size (.available input)))] - (copy! input output :size size) - (.toByteArray output)) - - :else - (throw (IllegalArgumentException. "invalid arguments")))) - -(defn bytes-input-stream - "Creates an instance of ByteArrayInputStream." - [^bytes data] - (ByteArrayInputStream. data)) - -(defn bounded-input-stream - [input size & {:keys [close?] :or {close? true}}] - (doto (BoundedInputStream. ^InputStream input ^long size) - (.setPropagateClose close?))) - -(defn zstd-input-stream - ^InputStream - [input] - (ZstdInputStream. ^InputStream input)) - -(defn zstd-output-stream - ^OutputStream - [output & {:keys [level] :or {level 0}}] - (ZstdOutputStream. ^OutputStream output (int level))) - -(defn data-input-stream - ^DataInputStream - [input] - (DataInputStream. ^InputStream input)) - -(defn data-output-stream - ^DataOutputStream - [output] - (DataOutputStream. ^OutputStream output)) - -(defn close! - [^AutoCloseable stream] - (.close stream)) diff --git a/backend/test/app/services_fonts_test.clj b/backend/test/app/services_fonts_test.clj index dfe87e569..ea892368b 100644 --- a/backend/test/app/services_fonts_test.clj +++ b/backend/test/app/services_fonts_test.clj @@ -11,10 +11,9 @@ [app.http :as http] [app.storage :as sto] [app.test-helpers :as th] - [app.util.bytes :as bs] - [clojure.java.io :as io] [clojure.test :as t] - [datoteka.core :as fs])) + [datoteka.fs :as fs] + [datoteka.io :as io])) (t/use-fixtures :once th/state-init) (t/use-fixtures :each th/database-reset) @@ -27,7 +26,7 @@ ttfdata (-> (io/resource "app/test_files/font-1.ttf") io/input-stream - bs/read-as-bytes) + io/read-as-bytes) params {::th/type :create-font-variant :profile-id (:id prof) @@ -63,7 +62,7 @@ data (-> (io/resource "app/test_files/font-1.woff") io/input-stream - bs/read-as-bytes) + io/read-as-bytes) params {::th/type :create-font-variant :profile-id (:id prof) diff --git a/backend/test/app/storage_test.clj b/backend/test/app/storage_test.clj index 3a921c8aa..bf2460a3a 100644 --- a/backend/test/app/storage_test.clj +++ b/backend/test/app/storage_test.clj @@ -12,11 +12,10 @@ [app.storage :as sto] [app.test-helpers :as th] [app.util.time :as dt] - [app.util.bytes :as bs] - [clojure.java.io :as io] [clojure.test :as t] [cuerdas.core :as str] [datoteka.core :as fs] + [datoteka.io :as io] [mockery.core :refer [with-mocks]])) (t/use-fixtures :once th/state-init) @@ -199,7 +198,7 @@ ttfdata (-> (io/resource "app/test_files/font-1.ttf") io/input-stream - bs/read-as-bytes) + io/read-as-bytes) mfile {:filename "sample.jpg" :path (th/tempfile "app/test_files/sample.jpg") diff --git a/common/deps.edn b/common/deps.edn index 64ad27bf0..0c1b1b271 100644 --- a/common/deps.edn +++ b/common/deps.edn @@ -29,7 +29,7 @@ frankiesardo/linked {:mvn/version "1.3.0"} - funcool/datoteka {:mvn/version "3.0.65"} + funcool/datoteka {:mvn/version "3.0.66"} com.sun.mail/jakarta.mail {:mvn/version "2.0.1"} ;; exception printing