✨ Start use datoteka.io ns

backend/src/app/http/debug.clj

@@ -17,13 +17,12 @@
    [app.rpc.mutations.files :refer [create-file]]
    [app.rpc.queries.profile :as profile]
    [app.util.blob :as blob]
-   [app.util.bytes :as bs]
    [app.util.template :as tmpl]
    [app.util.time :as dt]
    [app.worker :as wrk]
-   [clojure.java.io :as io]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
+   [datoteka.io :as io]
    [emoji.core :as emj]
    [integrant.core :as ig]
    [markdown.core :as md]
@@ -126,7 +125,7 @@
 (defn- upload-file-data
   [{:keys [pool]} {:keys [profile-id params] :as request}]
   (let [project-id (some-> (profile/retrieve-additional-data pool profile-id) :default-project-id)
-        data       (some-> params :file :path bs/read-as-bytes blob/decode)]
+        data       (some-> params :file :path io/read-as-bytes blob/decode)]
 
     (if (and data project-id)
       (let [fname      (str "Imported file *: " (dt/now))

backend/src/app/media.clj

@@ -13,14 +13,14 @@
    [app.common.spec :as us]
    [app.config :as cf]
    [app.storage.tmp :as tmp]
-   [app.util.bytes :as bs]
    [app.util.svg :as svg]
    [buddy.core.bytes :as bb]
    [buddy.core.codecs :as bc]
    [clojure.java.shell :as sh]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
-   [datoteka.fs :as fs])
+   [datoteka.fs :as fs]
+   [datoteka.io :as io])
   (:import
    org.im4java.core.ConvertCmd
    org.im4java.core.IMOperation
@@ -199,7 +199,7 @@
   (letfn [(ttf->otf [data]
             (let [finput  (tmp/tempfile :prefix "penpot.font." :suffix "")
                   foutput (fs/path (str finput ".otf"))
-                  _       (bs/write-to-file! data finput)
+                  _       (io/write-to-file! data finput)
                   res     (sh/sh "fontforge" "-lang=ff" "-c"
                                  (str/fmt "Open('%s'); Generate('%s')"
                                           (str finput)
@@ -210,7 +210,7 @@
           (otf->ttf [data]
             (let [finput  (tmp/tempfile :prefix "penpot.font." :suffix "")
                   foutput (fs/path (str finput ".ttf"))
-                  _       (bs/write-to-file! data finput)
+                  _       (io/write-to-file! data finput)
                   res     (sh/sh "fontforge" "-lang=ff" "-c"
                                  (str/fmt "Open('%s'); Generate('%s')"
                                           (str finput)
@@ -224,7 +224,7 @@
             ;; command.
             (let [finput  (tmp/tempfile :prefix "penpot.font." :suffix "")
                   foutput (fs/path (str finput ".woff"))
-                  _       (bs/write-to-file! data finput)
+                  _       (io/write-to-file! data finput)
                   res     (sh/sh "sfnt2woff" (str finput))]
               (when (zero? (:exit res))
                 foutput)))
@@ -235,14 +235,14 @@
             ;; command.
             (let [finput  (tmp/tempfile :prefix "penpot.font." :suffix ".tmp")
                   foutput (fs/path (str (fs/base finput) ".woff2"))
-                  _       (bs/write-to-file! data finput)
+                  _       (io/write-to-file! data finput)
                   res      (sh/sh "woff2_compress" (str finput))]
               (when (zero? (:exit res))
                 foutput)))
 
           (woff->sfnt [data]
             (let [finput  (tmp/tempfile :prefix "penpot" :suffix "")
-                  _       (bs/write-to-file! data finput)
+                  _       (io/write-to-file! data finput)
                   res     (sh/sh "woff2sfnt" (str finput)
                                  :out-enc :bytes)]
               (when (zero? (:exit res))

backend/src/app/rpc/commands/binfile.clj

@@ -23,16 +23,17 @@
    [app.storage.tmp :as tmp]
    [app.tasks.file-gc]
    [app.util.blob :as blob]
-   [app.util.bytes :as bs]
    [app.util.fressian :as fres]
    [app.util.services :as sv]
    [app.util.time :as dt]
-   [clojure.java.io :as io]
    [clojure.spec.alpha :as s]
    [clojure.walk :as walk]
    [cuerdas.core :as str]
+   [datoteka.io :as io]
    [yetti.adapter :as yt])
   (:import
+   com.github.luben.zstd.ZstdInputStream
+   com.github.luben.zstd.ZstdOutputStream
    java.io.DataInputStream
    java.io.DataOutputStream
    java.io.InputStream
@@ -201,7 +202,7 @@
            :position @*position*
            ::l/async false)
   (let [vers   (-> version name (subs 1) parse-long)
-        output (bs/data-output-stream output)]
+        output (io/data-output-stream output)]
     (doto output
       (write-byte! (get-mark :header))
       (write-long! penpot-magic-number)
@@ -210,7 +211,7 @@
 (defn read-header!
   [^InputStream input]
   (l/trace :fn "read-header!" :position @*position* ::l/async false)
-  (let [input (bs/data-input-stream input)
+  (let [input (io/data-input-stream input)
         mark  (read-byte! input)
         mnum  (read-long! input)
         vers  (read-long! input)]
@@ -225,7 +226,7 @@
 
 (defn copy-stream!
   [^OutputStream output ^InputStream input ^long size]
-  (let [written (bs/copy! input output :size size)]
+  (let [written (io/copy! input output :size size)]
     (l/trace :fn "copy-stream!" :position @*position* :size size :written written ::l/async false)
     (swap! *position* + written)
     written))
@@ -254,11 +255,11 @@
 
     (if (> s temp-file-threshold)
       (with-open [^OutputStream output (io/output-stream p)]
-        (let [readed (bs/copy! input output :offset 0 :size s)]
+        (let [readed (io/copy! input output :offset 0 :size s)]
           (l/trace :fn "read-stream*!" :expected s :readed readed :position @*position* ::l/async false)
           (swap! *position* + readed)
           [s p]))
-      [s (bs/read-as-bytes input :size s)])))
+      [s (io/read-as-bytes input :size s)])))
 
 (defmacro assert-read-label!
   [input expected-label]
@@ -275,6 +276,16 @@
 
 ;; --- HELPERS
 
+(defn zstd-input-stream
+  ^InputStream
+  [input]
+  (ZstdInputStream. ^InputStream input))
+
+(defn zstd-output-stream
+  ^OutputStream
+  [output & {:keys [level] :or {level 0}}]
+  (ZstdOutputStream. ^OutputStream output (int level)))
+
 (defn- retrieve-file
   [pool file-id]
   (->> (db/query pool :file {:id file-id})
@@ -411,7 +422,7 @@
 (defmulti write-export ::version)
 (defmulti write-section ::section)
 
-(s/def ::output bs/output-stream?)
+(s/def ::output io/output-stream?)
 (s/def ::file-ids (s/every ::us/uuid :kind vector? :min-count 1))
 (s/def ::include-libraries? (s/nilable ::us/boolean))
 (s/def ::embed-assets? (s/nilable ::us/boolean))
@@ -441,8 +452,8 @@
 (defmethod write-export :default
   [{:keys [::output] :as options}]
   (write-header! output :v1)
-  (with-open [output (bs/zstd-output-stream output :level 12)]
-    (with-open [output (bs/data-output-stream output)]
+  (with-open [output (zstd-output-stream output :level 12)]
+    (with-open [output (io/data-output-stream output)]
       (binding [*state* (volatile! {})]
         (run! (fn [section]
                 (l/debug :hint "write section" :section section ::l/async false)
@@ -530,7 +541,7 @@
 (defmulti read-section ::section)
 
 (s/def ::project-id ::us/uuid)
-(s/def ::input bs/input-stream?)
+(s/def ::input io/input-stream?)
 (s/def ::overwrite? (s/nilable ::us/boolean))
 (s/def ::migrate? (s/nilable ::us/boolean))
 (s/def ::ignore-index-errors? (s/nilable ::us/boolean))
@@ -562,8 +573,8 @@
 
 (defmethod read-import :v1
   [{:keys [pool ::input] :as options}]
-  (with-open [input (bs/zstd-input-stream input)]
-    (with-open [input (bs/data-input-stream input)]
+  (with-open [input (zstd-input-stream input)]
+    (with-open [input (io/data-input-stream input)]
       (db/with-atomic [conn pool]
         (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED;"])
         (binding [*state* (volatile! {:media [] :index {}})]

backend/src/app/rpc/mutations/media.clj

@@ -18,11 +18,11 @@
    [app.rpc.semaphore :as rsem]
    [app.storage :as sto]
    [app.storage.tmp :as tmp]
-   [app.util.bytes :as bs]
    [app.util.services :as sv]
    [app.util.time :as dt]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
+   [datoteka.io :as io]
    [promesa.core :as p]
    [promesa.exec :as px]))
 
@@ -224,7 +224,7 @@
           (process-response [{:keys [body headers] :as response}]
             (let [{:keys [size mtype]} (parse-and-validate-size headers)
                   path                 (tmp/tempfile :prefix "penpot.media.download.")
-                  written              (bs/write-to-file! body path :size size)]
+                  written              (io/write-to-file! body path :size size)]
 
               (when (not= written size)
                 (ex/raise :type :internal

backend/src/app/storage/fs.clj

@@ -10,11 +10,10 @@
    [app.common.spec :as us]
    [app.common.uri :as u]
    [app.storage.impl :as impl]
-   [app.util.bytes :as bs]
-   [clojure.java.io :as io]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
    [datoteka.fs :as fs]
+   [datoteka.io :as io]
    [integrant.core :as ig]
    [promesa.core :as p]
    [promesa.exec :as px])
@@ -59,7 +58,7 @@
         (fs/create-dir (fs/parent full)))
       (with-open [^InputStream src  (io/input-stream content)
                   ^OutputStream dst (io/output-stream full)]
-        (io/copy src dst)))))
+        (io/copy! src dst)))))
 
 (defmethod impl/get-object-data :fs
   [{:keys [executor] :as backend} {:keys [id] :as object}]
@@ -76,8 +75,8 @@
 (defmethod impl/get-object-bytes :fs
   [backend object]
   (p/let [input (impl/get-object-data backend object)]
-    (ex/with-always (bs/close! input)
-      (bs/read-as-bytes input))))
+    (ex/with-always (io/close! input)
+      (io/read-as-bytes input))))
 
 (defmethod impl/get-object-url :fs
   [{:keys [uri executor] :as backend} {:keys [id] :as object} _]

backend/src/app/storage/impl.clj

@@ -9,10 +9,10 @@
   (:require
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
-   [app.util.bytes :as bs]
    [buddy.core.codecs :as bc]
    [buddy.core.hash :as bh]
-   [clojure.java.io :as io])
+   [clojure.java.io :as jio]
+   [datoteka.io :as io])
   (:import
    java.nio.ByteBuffer
    java.nio.file.Files
@@ -110,14 +110,14 @@
     IContentObject
     (get-size [_] size)
 
-    io/IOFactory
+    jio/IOFactory
     (make-reader [this opts]
-      (io/make-reader this opts))
+      (jio/make-reader this opts))
     (make-writer [_ _]
       (throw (UnsupportedOperationException. "not implemented")))
     (make-input-stream [_ _]
       (-> (io/input-stream path)
-          (bs/bounded-input-stream size)))
+          (io/bounded-input-stream size)))
     (make-output-stream [_ _]
       (throw (UnsupportedOperationException. "not implemented")))))
 
@@ -127,14 +127,14 @@
     IContentObject
     (get-size [_] size)
 
-    io/IOFactory
+    jio/IOFactory
     (make-reader [this opts]
-      (io/make-reader this opts))
+      (jio/make-reader this opts))
     (make-writer [_ _]
       (throw (UnsupportedOperationException. "not implemented")))
     (make-input-stream [_ _]
-      (-> (bs/bytes-input-stream data)
-          (bs/bounded-input-stream size)))
+      (-> (io/bytes-input-stream data)
+          (io/bounded-input-stream size)))
     (make-output-stream [_ _]
       (throw (UnsupportedOperationException. "not implemented")))))
 
@@ -169,7 +169,7 @@
   (when-not (satisfies? IContentObject content)
     (throw (UnsupportedOperationException. "`content` should be an instance of IContentObject")))
 
-  (when-not (satisfies? io/IOFactory content)
+  (when-not (satisfies? jio/IOFactory content)
     (throw (UnsupportedOperationException. "`content` should be an instance of IOFactory")))
 
   (reify
@@ -179,15 +179,15 @@
     IContentHash
     (get-hash [_] hash)
 
-    io/IOFactory
+    jio/IOFactory
     (make-reader [_ opts]
-      (io/make-reader content opts))
+      (jio/make-reader content opts))
     (make-writer [_ opts]
-      (io/make-writer content opts))
+      (jio/make-writer content opts))
     (make-input-stream [_ opts]
-      (io/make-input-stream content opts))
+      (jio/make-input-stream content opts))
     (make-output-stream [_ opts]
-      (io/make-output-stream content opts))))
+      (jio/make-output-stream content opts))))
 
 (defn content?
   [v]

backend/src/app/util/bytes.clj

@@ -1,128 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) UXBOX Labs SL
-
-(ns app.util.bytes
-  "Bytes & Byte Streams helpers"
-  (:require
-   [clojure.java.io :as io]
-   [datoteka.fs :as fs]
-   [yetti.adapter :as yt])
-  (:import
-   com.github.luben.zstd.ZstdInputStream
-   com.github.luben.zstd.ZstdOutputStream
-   java.io.ByteArrayInputStream
-   java.io.ByteArrayOutputStream
-   java.io.DataInputStream
-   java.io.DataOutputStream
-   java.io.OutputStream
-   java.io.InputStream
-   java.lang.AutoCloseable
-   org.apache.commons.io.IOUtils
-   org.apache.commons.io.input.BoundedInputStream))
-
-;; TODO: migrate to datoteka.io
-
-(set! *warn-on-reflection* true)
-
-(def ^:const default-buffer-size
-  (:xnio/buffer-size yt/defaults))
-
-(defn input-stream?
-  [s]
-  (instance? InputStream s))
-
-(defn output-stream?
-  [s]
-  (instance? OutputStream s))
-
-(defn data-input-stream?
-  [s]
-  (instance? DataInputStream s))
-
-(defn data-output-stream?
-  [s]
-  (instance? DataOutputStream s))
-
-(defn copy!
-  [src dst & {:keys [offset size buffer-size]
-              :or {offset 0 buffer-size default-buffer-size}}]
-  (let [^bytes buff (byte-array buffer-size)]
-    (if size
-      (IOUtils/copyLarge ^InputStream src ^OutputStream dst (long offset) (long size) buff)
-      (IOUtils/copyLarge ^InputStream src ^OutputStream dst buff))))
-
-(defn write-to-file!
-  [src dst & {:keys [size]}]
-  (with-open [^OutputStream output (io/output-stream dst)]
-    (cond
-      (bytes? src)
-      (if size
-        (with-open [^InputStream input (ByteArrayInputStream. ^bytes src)]
-          (with-open [^InputStream input (BoundedInputStream. input (or size (alength ^bytes src)))]
-            (copy! input output :size size)))
-
-        (do
-          (IOUtils/writeChunked ^bytes src output)
-          (.flush ^OutputStream output)
-          (alength ^bytes src)))
-
-      (instance? InputStream src)
-      (copy! src output :size size)
-
-      :else
-      (throw (IllegalArgumentException. "invalid arguments")))))
-
-(defn read-as-bytes
-  "Read input stream as byte array."
-  [input & {:keys [size]}]
-  (cond
-    (instance? InputStream input)
-    (with-open [output (ByteArrayOutputStream. (or size (.available ^InputStream input)))]
-      (copy! input output :size size)
-      (.toByteArray output))
-
-    (fs/path? input)
-    (with-open [input  (io/input-stream input)
-                output (ByteArrayOutputStream. (or size (.available input)))]
-      (copy! input output :size size)
-      (.toByteArray output))
-
-    :else
-    (throw (IllegalArgumentException. "invalid arguments"))))
-
-(defn bytes-input-stream
-  "Creates an instance of ByteArrayInputStream."
-  [^bytes data]
-  (ByteArrayInputStream. data))
-
-(defn bounded-input-stream
-  [input size & {:keys [close?] :or {close? true}}]
-  (doto (BoundedInputStream. ^InputStream input ^long size)
-    (.setPropagateClose close?)))
-
-(defn zstd-input-stream
-  ^InputStream
-  [input]
-  (ZstdInputStream. ^InputStream input))
-
-(defn zstd-output-stream
-  ^OutputStream
-  [output & {:keys [level] :or {level 0}}]
-  (ZstdOutputStream. ^OutputStream output (int level)))
-
-(defn data-input-stream
-  ^DataInputStream
-  [input]
-  (DataInputStream. ^InputStream input))
-
-(defn data-output-stream
-  ^DataOutputStream
-  [output]
-  (DataOutputStream. ^OutputStream output))
-
-(defn close!
-  [^AutoCloseable stream]
-  (.close stream))

backend/test/app/services_fonts_test.clj

@@ -11,10 +11,9 @@
    [app.http :as http]
    [app.storage :as sto]
    [app.test-helpers :as th]
-   [app.util.bytes :as bs]
-   [clojure.java.io :as io]
    [clojure.test :as t]
-   [datoteka.core :as fs]))
+   [datoteka.fs :as fs]
+   [datoteka.io :as io]))
 
 (t/use-fixtures :once th/state-init)
 (t/use-fixtures :each th/database-reset)
@@ -27,7 +26,7 @@
 
         ttfdata (-> (io/resource "app/test_files/font-1.ttf")
                     io/input-stream
-                    bs/read-as-bytes)
+                    io/read-as-bytes)
 
         params  {::th/type :create-font-variant
                  :profile-id (:id prof)
@@ -63,7 +62,7 @@
 
         data    (-> (io/resource "app/test_files/font-1.woff")
                     io/input-stream
-                    bs/read-as-bytes)
+                    io/read-as-bytes)
 
         params  {::th/type :create-font-variant
                  :profile-id (:id prof)

backend/test/app/storage_test.clj

@@ -12,11 +12,10 @@
    [app.storage :as sto]
    [app.test-helpers :as th]
    [app.util.time :as dt]
-   [app.util.bytes :as bs]
-   [clojure.java.io :as io]
    [clojure.test :as t]
    [cuerdas.core :as str]
    [datoteka.core :as fs]
+   [datoteka.io :as io]
    [mockery.core :refer [with-mocks]]))
 
 (t/use-fixtures :once th/state-init)
@@ -199,7 +198,7 @@
 
         ttfdata (-> (io/resource "app/test_files/font-1.ttf")
                     io/input-stream
-                    bs/read-as-bytes)
+                    io/read-as-bytes)
 
         mfile   {:filename "sample.jpg"
                  :path (th/tempfile "app/test_files/sample.jpg")

common/deps.edn

@@ -29,7 +29,7 @@
 
   frankiesardo/linked {:mvn/version "1.3.0"}
 
-  funcool/datoteka {:mvn/version "3.0.65"}
+  funcool/datoteka {:mvn/version "3.0.66"}
   com.sun.mail/jakarta.mail {:mvn/version "2.0.1"}
 
   ;; exception printing