🎉 Add v5 blob format (lz4 framed, less gc)

backend/src/app/config.clj

@@ -50,7 +50,7 @@
    :database-username "penpot"
    :database-password "penpot"
 
-   :default-blob-version 4
+   :default-blob-version 5
    :loggers-zmq-uri "tcp://localhost:45556"
    :rpc-rlimit-config (fs/path "resources/rlimit.edn")
 

backend/src/app/util/blob.clj

@@ -12,14 +12,18 @@
    [app.config :as cf]
    [app.util.fressian :as fres])
   (:import
+   com.github.luben.zstd.Zstd
    java.io.ByteArrayInputStream
    java.io.ByteArrayOutputStream
    java.io.DataInputStream
    java.io.DataOutputStream
-   com.github.luben.zstd.Zstd
+   java.io.InputStream
+   java.io.OutputStream
+   net.jpountz.lz4.LZ4Compressor
    net.jpountz.lz4.LZ4Factory
    net.jpountz.lz4.LZ4FastDecompressor
-   net.jpountz.lz4.LZ4Compressor))
+   net.jpountz.lz4.LZ4FrameInputStream
+   net.jpountz.lz4.LZ4FrameOutputStream))
 
 (set! *warn-on-reflection* true)
 
@@ -28,18 +32,21 @@
 (declare decode-v1)
 (declare decode-v3)
 (declare decode-v4)
+(declare decode-v5)
 (declare encode-v1)
 (declare encode-v3)
 (declare encode-v4)
+(declare encode-v5)
 
 (defn encode
   ([data] (encode data nil))
   ([data {:keys [version]}]
-   (let [version (or version (cf/get :default-blob-version 4))]
+   (let [version (or version (cf/get :default-blob-version 5))]
      (case (long version)
        1 (encode-v1 data)
        3 (encode-v3 data)
        4 (encode-v4 data)
+       5 (encode-v5 data)
        (throw (ex-info "unsupported version" {:version version}))))))
 
 (defn decode
@@ -53,6 +60,7 @@
         1 (decode-v1 data ulen)
         3 (decode-v3 data ulen)
         4 (decode-v4 data ulen)
+        5 (decode-v5 data)
         (throw (ex-info "unsupported version" {:version version}))))))
 
 ;; --- IMPL
@@ -108,15 +116,17 @@
         dlen  (alength ^bytes data)
         mlen  (Zstd/compressBound dlen)
         cdata (byte-array mlen)
-        clen  (Zstd/compressByteArray ^bytes cdata 0 mlen
+        cdlen (alength ^bytes cdata)
+        cmlen (Zstd/compressByteArray ^bytes cdata 0 mlen
                                       ^bytes data 0 dlen
                                       0)]
-    (with-open [^ByteArrayOutputStream baos (ByteArrayOutputStream. (+ (alength cdata) 2 4))
-                ^DataOutputStream dos (DataOutputStream. baos)]
-      (.writeShort dos (short 4)) ;; version number
-      (.writeInt dos (int dlen))
-      (.write dos ^bytes cdata (int 0) clen)
-      (.toByteArray baos))))
+
+    (with-open [^ByteArrayOutputStream output (ByteArrayOutputStream. (+ cdlen 2 4))]
+      (with-open [^DataOutputStream output (DataOutputStream. output)]
+        (.writeShort output (short 4)) ;; version number
+        (.writeInt output (int dlen))
+        (.write output ^bytes cdata (int 0) (int cmlen)))
+      (.toByteArray output))))
 
 (defn- decode-v4
   [^bytes cdata ^long ulen]
@@ -124,3 +134,21 @@
     (Zstd/decompressByteArray ^bytes udata 0 ulen
                               ^bytes cdata 6 (- (alength cdata) 6))
     (fres/decode udata)))
+
+(defn- encode-v5
+  [data]
+  (with-open [^ByteArrayOutputStream output (ByteArrayOutputStream.)]
+    (with-open [^DataOutputStream output (DataOutputStream. output)]
+      (.writeShort output (short 5)) ;; version number
+      (.writeInt output (int -1))
+      (with-open [^OutputStream output (LZ4FrameOutputStream. output)]
+        (-> (fres/writer output)
+            (fres/write! data))))
+    (.toByteArray output)))
+
+(defn- decode-v5
+  [^bytes cdata]
+  (with-open [^InputStream input (ByteArrayInputStream. cdata)]
+    (.skip input 6)
+    (with-open [^InputStream input (LZ4FrameInputStream. input)]
+      (-> input fres/reader fres/read!))))

backend/src/app/util/fressian.clj

@@ -271,11 +271,38 @@
 
 (defn encode
   [data]
-  (with-open [out (ByteArrayOutputStream.)]
-    (write! (writer out) data)
-    (.toByteArray out)))
+  (with-open [^ByteArrayOutputStream output (ByteArrayOutputStream.)]
+    (-> (writer output)
+        (write! data))
+    (.toByteArray output)))
 
 (defn decode
   [data]
-  (with-open [input (ByteArrayInputStream. ^bytes data)]
-    (read! (reader input))))
+  (with-open [^ByteArrayInputStream input (ByteArrayInputStream. ^bytes data)]
+    (-> input reader read!)))
+
+;; --- ADDITIONAL
+
+(add-handlers!
+ {:name "penpot/point"
+  :class app.common.geom.point.Point
+  :wfn (fn [n w ^Point o]
+         (write-tag! w n 1)
+         (write-list! w (List/of (.-x o) (.-y o))))
+  :rfn (fn [^Reader rdr]
+         (let [^List x (read-object! rdr)]
+           (Point. (.get x 0) (.get x 1))))}
+
+ {:name "penpot/matrix"
+  :class app.common.geom.matrix.Matrix
+  :wfn (fn [^String n ^Writer w o]
+         (write-tag! w n 1)
+         (write-list! w (List/of (.-a ^Matrix o)
+                                 (.-b ^Matrix o)
+                                 (.-c ^Matrix o)
+                                 (.-d ^Matrix o)
+                                 (.-e ^Matrix o)
+                                 (.-f ^Matrix o))))
+  :rfn (fn [^Reader rdr]
+         (let [^List x (read-object! rdr)]
+           (Matrix. (.get x 0) (.get x 1) (.get x 2) (.get x 3) (.get x 4) (.get x 5))))})