diff --git a/backend/src/app/util/blob.clj b/backend/src/app/util/blob.clj index bd24ca3d2..d74fb4dae 100644 --- a/backend/src/app/util/blob.clj +++ b/backend/src/app/util/blob.clj @@ -12,14 +12,18 @@ [app.config :as cf] [app.util.fressian :as fres]) (:import + com.github.luben.zstd.Zstd java.io.ByteArrayInputStream java.io.ByteArrayOutputStream java.io.DataInputStream java.io.DataOutputStream - com.github.luben.zstd.Zstd + java.io.InputStream + java.io.OutputStream + net.jpountz.lz4.LZ4Compressor net.jpountz.lz4.LZ4Factory net.jpountz.lz4.LZ4FastDecompressor - net.jpountz.lz4.LZ4Compressor)) + net.jpountz.lz4.LZ4FrameInputStream + net.jpountz.lz4.LZ4FrameOutputStream)) (set! *warn-on-reflection* true) @@ -28,9 +32,11 @@ (declare decode-v1) (declare decode-v3) (declare decode-v4) +(declare decode-v5) (declare encode-v1) (declare encode-v3) (declare encode-v4) +(declare encode-v5) (defn encode ([data] (encode data nil)) @@ -40,6 +46,7 @@ 1 (encode-v1 data) 3 (encode-v3 data) 4 (encode-v4 data) + 5 (encode-v5 data) (throw (ex-info "unsupported version" {:version version})))))) (defn decode @@ -53,6 +60,7 @@ 1 (decode-v1 data ulen) 3 (decode-v3 data ulen) 4 (decode-v4 data ulen) + 5 (decode-v5 data) (throw (ex-info "unsupported version" {:version version})))))) ;; --- IMPL @@ -108,15 +116,17 @@ dlen (alength ^bytes data) mlen (Zstd/compressBound dlen) cdata (byte-array mlen) - clen (Zstd/compressByteArray ^bytes cdata 0 mlen + cdlen (alength ^bytes cdata) + cmlen (Zstd/compressByteArray ^bytes cdata 0 mlen ^bytes data 0 dlen 0)] - (with-open [^ByteArrayOutputStream baos (ByteArrayOutputStream. (+ (alength cdata) 2 4)) - ^DataOutputStream dos (DataOutputStream. baos)] - (.writeShort dos (short 4)) ;; version number - (.writeInt dos (int dlen)) - (.write dos ^bytes cdata (int 0) clen) - (.toByteArray baos)))) + + (with-open [^ByteArrayOutputStream output (ByteArrayOutputStream. (+ cdlen 2 4))] + (with-open [^DataOutputStream output (DataOutputStream. output)] + (.writeShort output (short 4)) ;; version number + (.writeInt output (int dlen)) + (.write output ^bytes cdata (int 0) (int cmlen))) + (.toByteArray output)))) (defn- decode-v4 [^bytes cdata ^long ulen] @@ -124,3 +134,21 @@ (Zstd/decompressByteArray ^bytes udata 0 ulen ^bytes cdata 6 (- (alength cdata) 6)) (fres/decode udata))) + +(defn- encode-v5 + [data] + (with-open [^ByteArrayOutputStream output (ByteArrayOutputStream.)] + (with-open [^DataOutputStream output (DataOutputStream. output)] + (.writeShort output (short 5)) ;; version number + (.writeInt output (int -1)) + (with-open [^OutputStream output (LZ4FrameOutputStream. output)] + (-> (fres/writer output) + (fres/write! data)))) + (.toByteArray output))) + +(defn- decode-v5 + [^bytes cdata] + (with-open [^InputStream input (ByteArrayInputStream. cdata)] + (.skip input 6) + (with-open [^InputStream input (LZ4FrameInputStream. input)] + (-> input fres/reader fres/read!))))