From 5fe3842d1e9db8ad69c36e27613be06622bba65e Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 6 Oct 2022 17:04:24 +0200 Subject: [PATCH] :tada: Add v5 blob format (lz4 framed, less gc) --- backend/src/app/config.clj | 2 +- backend/src/app/util/blob.clj | 48 ++++++++++++++++++++++++------- backend/src/app/util/fressian.clj | 37 ++++++++++++++++++++---- 3 files changed, 71 insertions(+), 16 deletions(-) diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index fdfcfc98b..9e65cd3d3 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -50,7 +50,7 @@ :database-username "penpot" :database-password "penpot" - :default-blob-version 4 + :default-blob-version 5 :loggers-zmq-uri "tcp://localhost:45556" :rpc-rlimit-config (fs/path "resources/rlimit.edn") diff --git a/backend/src/app/util/blob.clj b/backend/src/app/util/blob.clj index bd24ca3d2..9fe030e74 100644 --- a/backend/src/app/util/blob.clj +++ b/backend/src/app/util/blob.clj @@ -12,14 +12,18 @@ [app.config :as cf] [app.util.fressian :as fres]) (:import + com.github.luben.zstd.Zstd java.io.ByteArrayInputStream java.io.ByteArrayOutputStream java.io.DataInputStream java.io.DataOutputStream - com.github.luben.zstd.Zstd + java.io.InputStream + java.io.OutputStream + net.jpountz.lz4.LZ4Compressor net.jpountz.lz4.LZ4Factory net.jpountz.lz4.LZ4FastDecompressor - net.jpountz.lz4.LZ4Compressor)) + net.jpountz.lz4.LZ4FrameInputStream + net.jpountz.lz4.LZ4FrameOutputStream)) (set! *warn-on-reflection* true) @@ -28,18 +32,21 @@ (declare decode-v1) (declare decode-v3) (declare decode-v4) +(declare decode-v5) (declare encode-v1) (declare encode-v3) (declare encode-v4) +(declare encode-v5) (defn encode ([data] (encode data nil)) ([data {:keys [version]}] - (let [version (or version (cf/get :default-blob-version 4))] + (let [version (or version (cf/get :default-blob-version 5))] (case (long version) 1 (encode-v1 data) 3 (encode-v3 data) 4 (encode-v4 data) + 5 (encode-v5 data) (throw (ex-info "unsupported version" {:version version})))))) (defn decode @@ -53,6 +60,7 @@ 1 (decode-v1 data ulen) 3 (decode-v3 data ulen) 4 (decode-v4 data ulen) + 5 (decode-v5 data) (throw (ex-info "unsupported version" {:version version})))))) ;; --- IMPL @@ -108,15 +116,17 @@ dlen (alength ^bytes data) mlen (Zstd/compressBound dlen) cdata (byte-array mlen) - clen (Zstd/compressByteArray ^bytes cdata 0 mlen + cdlen (alength ^bytes cdata) + cmlen (Zstd/compressByteArray ^bytes cdata 0 mlen ^bytes data 0 dlen 0)] - (with-open [^ByteArrayOutputStream baos (ByteArrayOutputStream. (+ (alength cdata) 2 4)) - ^DataOutputStream dos (DataOutputStream. baos)] - (.writeShort dos (short 4)) ;; version number - (.writeInt dos (int dlen)) - (.write dos ^bytes cdata (int 0) clen) - (.toByteArray baos)))) + + (with-open [^ByteArrayOutputStream output (ByteArrayOutputStream. (+ cdlen 2 4))] + (with-open [^DataOutputStream output (DataOutputStream. output)] + (.writeShort output (short 4)) ;; version number + (.writeInt output (int dlen)) + (.write output ^bytes cdata (int 0) (int cmlen))) + (.toByteArray output)))) (defn- decode-v4 [^bytes cdata ^long ulen] @@ -124,3 +134,21 @@ (Zstd/decompressByteArray ^bytes udata 0 ulen ^bytes cdata 6 (- (alength cdata) 6)) (fres/decode udata))) + +(defn- encode-v5 + [data] + (with-open [^ByteArrayOutputStream output (ByteArrayOutputStream.)] + (with-open [^DataOutputStream output (DataOutputStream. output)] + (.writeShort output (short 5)) ;; version number + (.writeInt output (int -1)) + (with-open [^OutputStream output (LZ4FrameOutputStream. output)] + (-> (fres/writer output) + (fres/write! data)))) + (.toByteArray output))) + +(defn- decode-v5 + [^bytes cdata] + (with-open [^InputStream input (ByteArrayInputStream. cdata)] + (.skip input 6) + (with-open [^InputStream input (LZ4FrameInputStream. input)] + (-> input fres/reader fres/read!)))) diff --git a/backend/src/app/util/fressian.clj b/backend/src/app/util/fressian.clj index 1b4367889..66c23fe43 100644 --- a/backend/src/app/util/fressian.clj +++ b/backend/src/app/util/fressian.clj @@ -271,11 +271,38 @@ (defn encode [data] - (with-open [out (ByteArrayOutputStream.)] - (write! (writer out) data) - (.toByteArray out))) + (with-open [^ByteArrayOutputStream output (ByteArrayOutputStream.)] + (-> (writer output) + (write! data)) + (.toByteArray output))) (defn decode [data] - (with-open [input (ByteArrayInputStream. ^bytes data)] - (read! (reader input)))) + (with-open [^ByteArrayInputStream input (ByteArrayInputStream. ^bytes data)] + (-> input reader read!))) + +;; --- ADDITIONAL + +(add-handlers! + {:name "penpot/point" + :class app.common.geom.point.Point + :wfn (fn [n w ^Point o] + (write-tag! w n 1) + (write-list! w (List/of (.-x o) (.-y o)))) + :rfn (fn [^Reader rdr] + (let [^List x (read-object! rdr)] + (Point. (.get x 0) (.get x 1))))} + + {:name "penpot/matrix" + :class app.common.geom.matrix.Matrix + :wfn (fn [^String n ^Writer w o] + (write-tag! w n 1) + (write-list! w (List/of (.-a ^Matrix o) + (.-b ^Matrix o) + (.-c ^Matrix o) + (.-d ^Matrix o) + (.-e ^Matrix o) + (.-f ^Matrix o)))) + :rfn (fn [^Reader rdr] + (let [^List x (read-object! rdr)] + (Matrix. (.get x 0) (.get x 1) (.get x 2) (.get x 3) (.get x 4) (.get x 5))))})