Penpot/penpot
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-02-15 03:28:25 -05:00
Code Issues Activity

🎉 Backport v5 blob format (lz4 framed, less gc)

Browse source
This commit is contained in:
Andrey Antukh 2022-10-19 23:22:25 +02:00 • committed by Andrés Moya
parent 99aea77355
commit 0548fdb43d
1 changed files with 37 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
backend/src/app/util/blob.clj
View file

@ -12,14 +12,18 @@
[app.config :as cf] [app.config :as cf]
[app.util.fressian :as fres]) [app.util.fressian :as fres])
(:import (:import
com.github.luben.zstd.Zstd
java.io.ByteArrayInputStream java.io.ByteArrayInputStream
java.io.ByteArrayOutputStream java.io.ByteArrayOutputStream
java.io.DataInputStream java.io.DataInputStream
java.io.DataOutputStream java.io.DataOutputStream
com.github.luben.zstd.Zstd java.io.InputStream
java.io.OutputStream
net.jpountz.lz4.LZ4Compressor
net.jpountz.lz4.LZ4Factory net.jpountz.lz4.LZ4Factory
net.jpountz.lz4.LZ4FastDecompressor net.jpountz.lz4.LZ4FastDecompressor
net.jpountz.lz4.LZ4Compressor)) net.jpountz.lz4.LZ4FrameInputStream
net.jpountz.lz4.LZ4FrameOutputStream))
(set! *warn-on-reflection* true) (set! *warn-on-reflection* true)
@ -28,9 +32,11 @@
(declare decode-v1) (declare decode-v1)
(declare decode-v3) (declare decode-v3)
(declare decode-v4) (declare decode-v4)
(declare decode-v5)
(declare encode-v1) (declare encode-v1)
(declare encode-v3) (declare encode-v3)
(declare encode-v4) (declare encode-v4)
(declare encode-v5)
(defn encode (defn encode
([data] (encode data nil)) ([data] (encode data nil))
@ -40,6 +46,7 @@
1 (encode-v1 data) 1 (encode-v1 data)
3 (encode-v3 data) 3 (encode-v3 data)
4 (encode-v4 data) 4 (encode-v4 data)
5 (encode-v5 data)
(throw (ex-info "unsupported version" {:version version})))))) (throw (ex-info "unsupported version" {:version version}))))))
(defn decode (defn decode
@ -53,6 +60,7 @@
1 (decode-v1 data ulen) 1 (decode-v1 data ulen)
3 (decode-v3 data ulen) 3 (decode-v3 data ulen)
4 (decode-v4 data ulen) 4 (decode-v4 data ulen)
5 (decode-v5 data)
(throw (ex-info "unsupported version" {:version version})))))) (throw (ex-info "unsupported version" {:version version}))))))
;; --- IMPL ;; --- IMPL
@ -108,15 +116,17 @@
dlen (alength ^bytes data) dlen (alength ^bytes data)
mlen (Zstd/compressBound dlen) mlen (Zstd/compressBound dlen)
cdata (byte-array mlen) cdata (byte-array mlen)
clen (Zstd/compressByteArray ^bytes cdata 0 mlen cdlen (alength ^bytes cdata)
cmlen (Zstd/compressByteArray ^bytes cdata 0 mlen
^bytes data 0 dlen ^bytes data 0 dlen
0)] 0)]
(with-open [^ByteArrayOutputStream baos (ByteArrayOutputStream. (+ (alength cdata) 2 4))
^DataOutputStream dos (DataOutputStream. baos)] (with-open [^ByteArrayOutputStream output (ByteArrayOutputStream. (+ cdlen 2 4))]
(.writeShort dos (short 4)) ;; version number (with-open [^DataOutputStream output (DataOutputStream. output)]
(.writeInt dos (int dlen)) (.writeShort output (short 4)) ;; version number
(.write dos ^bytes cdata (int 0) clen) (.writeInt output (int dlen))
(.toByteArray baos)))) (.write output ^bytes cdata (int 0) (int cmlen)))
(.toByteArray output))))
(defn- decode-v4 (defn- decode-v4
[^bytes cdata ^long ulen] [^bytes cdata ^long ulen]
@ -124,3 +134,21 @@
(Zstd/decompressByteArray ^bytes udata 0 ulen (Zstd/decompressByteArray ^bytes udata 0 ulen
^bytes cdata 6 (- (alength cdata) 6)) ^bytes cdata 6 (- (alength cdata) 6))
(fres/decode udata))) (fres/decode udata)))
(defn- encode-v5
[data]
(with-open [^ByteArrayOutputStream output (ByteArrayOutputStream.)]
(with-open [^DataOutputStream output (DataOutputStream. output)]
(.writeShort output (short 5)) ;; version number
(.writeInt output (int -1))
(with-open [^OutputStream output (LZ4FrameOutputStream. output)]
(-> (fres/writer output)
(fres/write! data))))
(.toByteArray output)))
(defn- decode-v5
[^bytes cdata]
(with-open [^InputStream input (ByteArrayInputStream. cdata)]
(.skip input 6)
(with-open [^InputStream input (LZ4FrameInputStream. input)]
(-> input fres/reader fres/read!))))