Penpot/penpot
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-02-13 02:28:18 -05:00
Code Issues Activity

🎉 Backport v5 blob format (lz4 framed, less gc)

Browse source
This commit is contained in:
Andrey Antukh 2022-10-19 23:22:25 +02:00 • committed by Andrés Moya
parent 99aea77355
commit 0548fdb43d
1 changed files with 37 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
backend/src/app/util/blob.clj
View file

@ -12,14 +12,18 @@
[app.config :as cf]
[app.util.fressian :as fres])
(:import
com.github.luben.zstd.Zstd
java.io.ByteArrayInputStream
java.io.ByteArrayOutputStream
java.io.DataInputStream
java.io.DataOutputStream
com.github.luben.zstd.Zstd
java.io.InputStream
java.io.OutputStream
net.jpountz.lz4.LZ4Compressor
net.jpountz.lz4.LZ4Factory
net.jpountz.lz4.LZ4FastDecompressor
net.jpountz.lz4.LZ4Compressor))
net.jpountz.lz4.LZ4FrameInputStream
net.jpountz.lz4.LZ4FrameOutputStream))
(set! *warn-on-reflection* true)
@ -28,9 +32,11 @@
(declare decode-v1)
(declare decode-v3)
(declare decode-v4)
(declare decode-v5)
(declare encode-v1)
(declare encode-v3)
(declare encode-v4)
(declare encode-v5)
(defn encode
([data] (encode data nil))
@ -40,6 +46,7 @@
1 (encode-v1 data)
3 (encode-v3 data)
4 (encode-v4 data)
5 (encode-v5 data)
(throw (ex-info "unsupported version" {:version version}))))))
(defn decode
@ -53,6 +60,7 @@
1 (decode-v1 data ulen)
3 (decode-v3 data ulen)
4 (decode-v4 data ulen)
5 (decode-v5 data)
(throw (ex-info "unsupported version" {:version version}))))))
;; --- IMPL
@ -108,15 +116,17 @@
dlen (alength ^bytes data)
mlen (Zstd/compressBound dlen)
cdata (byte-array mlen)
clen (Zstd/compressByteArray ^bytes cdata 0 mlen
cdlen (alength ^bytes cdata)
cmlen (Zstd/compressByteArray ^bytes cdata 0 mlen
^bytes data 0 dlen
0)]
(with-open [^ByteArrayOutputStream baos (ByteArrayOutputStream. (+ (alength cdata) 2 4))
^DataOutputStream dos (DataOutputStream. baos)]
(.writeShort dos (short 4)) ;; version number
(.writeInt dos (int dlen))
(.write dos ^bytes cdata (int 0) clen)
(.toByteArray baos))))
(with-open [^ByteArrayOutputStream output (ByteArrayOutputStream. (+ cdlen 2 4))]
(with-open [^DataOutputStream output (DataOutputStream. output)]
(.writeShort output (short 4)) ;; version number
(.writeInt output (int dlen))
(.write output ^bytes cdata (int 0) (int cmlen)))
(.toByteArray output))))
(defn- decode-v4
[^bytes cdata ^long ulen]
@ -124,3 +134,21 @@
(Zstd/decompressByteArray ^bytes udata 0 ulen
^bytes cdata 6 (- (alength cdata) 6))
(fres/decode udata)))
(defn- encode-v5
[data]
(with-open [^ByteArrayOutputStream output (ByteArrayOutputStream.)]
(with-open [^DataOutputStream output (DataOutputStream. output)]
(.writeShort output (short 5)) ;; version number
(.writeInt output (int -1))
(with-open [^OutputStream output (LZ4FrameOutputStream. output)]
(-> (fres/writer output)
(fres/write! data))))
(.toByteArray output)))
(defn- decode-v5
[^bytes cdata]
(with-open [^InputStream input (ByteArrayInputStream. cdata)]
(.skip input 6)
(with-open [^InputStream input (LZ4FrameInputStream. input)]
(-> input fres/reader fres/read!))))