🎉 Add version 4 of blob encoding.

The version 4 starts using the hight performance fressian
binary encoding with very lightweight compression layer.

.gitignore

@@ -35,6 +35,7 @@ node_modules
 /common/target
 /common/coverage
 /.clj-kondo/.cache
+clj-profiler/
 /bundle*
 /media
 /deploy

backend/deps.edn

@@ -13,6 +13,7 @@
 
   com.taoensso/nippy {:mvn/version "3.1.1"}
   com.github.luben/zstd-jni {:mvn/version "1.5.0-4"}
+  org.clojure/data.fressian {:mvn/version "1.0.0"}
 
   ;; NOTE: don't upgrade to latest version, breaking change is
   ;; introduced on 0.10.0 that suffixes counters with _total if they
@@ -63,6 +64,7 @@
     org.clojure/test.check {:mvn/version "RELEASE"}
     org.clojure/data.csv {:mvn/version "1.0.0"}
     com.clojure-goes-fast/clj-async-profiler {:mvn/version "0.5.1"}
+    clojure-humanize/clojure-humanize {:mvn/version "0.2.2"}
 
     criterium/criterium {:mvn/version "RELEASE"}
     mockery/mockery {:mvn/version "RELEASE"}}

backend/dev/user.clj

@@ -6,15 +6,18 @@
 
 (ns user
   (:require
+   [datoteka.core]
    [app.common.exceptions :as ex]
    [app.config :as cfg]
    [app.main :as main]
    [app.util.blob :as blob]
    [app.util.json :as json]
+   [app.util.fressian :as fres]
    [app.util.time :as dt]
-   [app.util.transit :as t]
+   [app.common.transit :as t]
    [clojure.java.io :as io]
    [clojure.pprint :refer [pprint print-table]]
+   [clojure.contrib.humanize :as hum]
    [clojure.repl :refer :all]
    [clojure.spec.alpha :as s]
    [clojure.spec.gen.alpha :as sgen]
@@ -22,10 +25,12 @@
    [clojure.test :as test]
    [clojure.tools.namespace.repl :as repl]
    [clojure.walk :refer [macroexpand-all]]
+   [clj-async-profiler.core :as prof]
    [criterium.core :refer [quick-bench bench with-progress-reporting]]
    [integrant.core :as ig]))
 
 (repl/disable-reload! (find-ns 'integrant.core))
+(set! *warn-on-reflection* true)
 
 (defonce system nil)
 
@@ -91,7 +96,15 @@
 
 (defn compression-bench
   [data]
-  (print-table
-   [{:v1 (alength (blob/encode data {:version 1}))
-     :v2 (alength (blob/encode data {:version 2}))
-     :v3 (alength (blob/encode data {:version 3}))}]))
+  (let [humanize (fn [v] (hum/filesize v :binary true :format " %.4f "))]
+    (print-table
+     [{:v1 (humanize (alength (blob/encode data {:version 1})))
+       :v2 (humanize (alength (blob/encode data {:version 2})))
+       :v3 (humanize (alength (blob/encode data {:version 3})))
+       :v4 (humanize (alength (blob/encode data {:version 4})))
+       }])))
+
+
+;; ;; (def contents (read-string (slurp (io/resource "bool-contents-1.edn"))))
+;; (def pre-data (datoteka.core/slurp-bytes (io/resource "file-data-sample")))
+;; (def data (blob/decode pre-data))

backend/scripts/repl

@@ -9,7 +9,6 @@ export OPTIONS="-A:jmx-remote:dev \
                 -J-Dclojure.tools.logging.factory=clojure.tools.logging.impl/log4j2-factory \
                 -J-XX:+UseShenandoahGC -J-XX:-OmitStackTraceInFastThrow -J-Xms50m -J-Xmx512m";
 
-# export OPTIONS="$OPTIONS -J-XX:+UnlockDiagnosticVMOptions";
 # export OPTIONS="$OPTIONS -J-XX:-TieredCompilation -J-XX:CompileThreshold=10000";
 
 export OPTIONS_EVAL="nil"

backend/src/app/db.clj

@@ -27,14 +27,16 @@
    com.zaxxer.hikari.HikariConfig
    com.zaxxer.hikari.HikariDataSource
    com.zaxxer.hikari.metrics.prometheus.PrometheusMetricsTrackerFactory
+   java.io.InputStream
+   java.io.OutputStream
    java.lang.AutoCloseable
    java.sql.Connection
    java.sql.Savepoint
    org.postgresql.PGConnection
    org.postgresql.geometric.PGpoint
+   org.postgresql.jdbc.PgArray
    org.postgresql.largeobject.LargeObject
    org.postgresql.largeobject.LargeObjectManager
-   org.postgresql.jdbc.PgArray
    org.postgresql.util.PGInterval
    org.postgresql.util.PGobject))
 

backend/src/app/msgbus.clj

@@ -179,18 +179,18 @@
     ;; Add a unique listener to connection
     (.addListener sub-conn
                   (reify RedisPubSubListener
-                    (message [it pattern topic message])
-                    (message [it topic message]
+                    (message [_it _pattern _topic _message])
+                    (message [_it topic message]
                       ;; There are no back pressure, so we use a sliding
                       ;; buffer for cases when the pubsub broker sends
                       ;; more messages that we can process.
                       (let [val {:topic topic :message (blob/decode message)}]
                         (when-not (a/offer! rcv-ch val)
                           (l/warn :msg "dropping message on subscription loop"))))
-                    (psubscribed [it pattern count])
-                    (punsubscribed [it pattern count])
-                    (subscribed [it topic count])
-                    (unsubscribed [it topic count])))
+                    (psubscribed [_it _pattern _count])
+                    (punsubscribed [_it _pattern _count])
+                    (subscribed [_it _topic _count])
+                    (unsubscribed [_it _topic _count])))
 
     (letfn [(subscribe-to-single-topic [nsubs topic chan]
               (let [nsubs (if (nil? nsubs) #{chan} (conj nsubs chan))]

backend/src/app/rpc/mutations/files.clj

@@ -409,7 +409,6 @@
   [conn project-id]
   (:team-id (db/get-by-id conn :project project-id {:columns [:team-id]})))
 
-
 ;; TEMPORARY FILE CREATION
 
 (s/def ::create-temp-file ::create-file)

backend/src/app/storage/impl.clj

@@ -117,11 +117,11 @@
       io/IOFactory
       (make-reader [_ opts]
 	    (io/make-reader path opts))
-      (make-writer [_ opts]
+      (make-writer [_ _]
         (throw (UnsupportedOperationException. "not implemented")))
       (make-input-stream [_ opts]
         (io/make-input-stream path opts))
-      (make-output-stream [_ opts]
+      (make-output-stream [_ _]
         (throw (UnsupportedOperationException. "not implemented")))
       clojure.lang.Counted
       (count [_] size)
@@ -138,11 +138,11 @@
       io/IOFactory
       (make-reader [_ opts]
 	    (io/make-reader bais opts))
-      (make-writer [_ opts]
+      (make-writer [_ _]
         (throw (UnsupportedOperationException. "not implemented")))
       (make-input-stream [_ opts]
         (io/make-input-stream bais opts))
-      (make-output-stream [_ opts]
+      (make-output-stream [_ _]
         (throw (UnsupportedOperationException. "not implemented")))
 
       clojure.lang.Counted
@@ -159,11 +159,11 @@
     io/IOFactory
     (make-reader [_ opts]
       (io/make-reader is opts))
-    (make-writer [_ opts]
+    (make-writer [_ _]
       (throw (UnsupportedOperationException. "not implemented")))
     (make-input-stream [_ opts]
       (io/make-input-stream is opts))
-    (make-output-stream [_ opts]
+    (make-output-stream [_ _]
       (throw (UnsupportedOperationException. "not implemented")))
 
     clojure.lang.Counted

backend/src/app/util/blob.clj

@@ -10,6 +10,7 @@
   (:require
    [app.common.transit :as t]
    [app.config :as cf]
+   [app.util.fressian :as fres]
    [taoensso.nippy :as n])
   (:import
    java.io.ByteArrayInputStream
@@ -21,23 +22,28 @@
    net.jpountz.lz4.LZ4FastDecompressor
    net.jpountz.lz4.LZ4Compressor))
 
+(set! *warn-on-reflection* true)
+
 (def lz4-factory (LZ4Factory/fastestInstance))
 
 (declare decode-v1)
 (declare decode-v2)
 (declare decode-v3)
+(declare decode-v4)
 (declare encode-v1)
 (declare encode-v2)
 (declare encode-v3)
+(declare encode-v4)
 
 (defn encode
   ([data] (encode data nil))
   ([data {:keys [version]}]
-   (let [version (or version (cf/get :default-blob-version 1))]
+   (let [version (or version (cf/get :default-blob-version 3))]
      (case (long version)
        1 (encode-v1 data)
        2 (encode-v2 data)
        3 (encode-v3 data)
+       4 (encode-v4 data)
        (throw (ex-info "unsupported version" {:version version}))))))
 
 (defn decode
@@ -51,6 +57,7 @@
         1 (decode-v1 data ulen)
         2 (decode-v2 data ulen)
         3 (decode-v3 data ulen)
+        4 (decode-v4 data ulen)
         (throw (ex-info "unsupported version" {:version version}))))))
 
 ;; --- IMPL
@@ -122,3 +129,26 @@
     (Zstd/decompressByteArray ^bytes udata 0 ulen
                               ^bytes cdata 6 (- (alength cdata) 6))
     (t/decode udata {:type :json})))
+
+(defn- encode-v4
+  [data]
+  (let [data  (fres/encode data)
+        dlen  (alength ^bytes data)
+        mlen  (Zstd/compressBound dlen)
+        cdata (byte-array mlen)
+        clen  (Zstd/compressByteArray ^bytes cdata 0 mlen
+                                      ^bytes data 0 dlen
+                                      0)]
+    (with-open [^ByteArrayOutputStream baos (ByteArrayOutputStream. (+ (alength cdata) 2 4))
+                ^DataOutputStream dos (DataOutputStream. baos)]
+      (.writeShort dos (short 4)) ;; version number
+      (.writeInt dos (int dlen))
+      (.write dos ^bytes cdata (int 0) clen)
+      (.toByteArray baos))))
+
+(defn- decode-v4
+  [^bytes cdata ^long ulen]
+  (let [udata (byte-array ulen)]
+    (Zstd/decompressByteArray ^bytes udata 0 ulen
+                              ^bytes cdata 6 (- (alength cdata) 6))
+    (fres/decode udata)))

backend/src/app/util/fressian.clj

@@ -0,0 +1,259 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.util.fressian
+  (:require
+   [app.common.geom.matrix :as gmt]
+   [app.common.geom.point :as gpt]
+   [clojure.data.fressian :as fres])
+  (:import
+   clojure.lang.Ratio
+   org.fressian.handlers.WriteHandler
+   org.fressian.handlers.ReadHandler
+   org.fressian.Writer
+   org.fressian.Reader
+   org.fressian.StreamingWriter
+   app.common.geom.matrix.Matrix
+   app.common.geom.point.Point
+   java.io.ByteArrayInputStream
+   java.io.ByteArrayOutputStream))
+
+;; --- MISC
+
+(set! *warn-on-reflection* true)
+
+(defn str->bytes
+  ([^String s]
+   (str->bytes s "UTF-8"))
+  ([^String s, ^String encoding]
+   (.getBytes s encoding)))
+
+(defn write-named
+  [tag ^Writer w s]
+  (.writeTag w tag 2)
+  (.writeObject w (namespace s) true)
+  (.writeObject w (name s) true))
+
+(defn write-list-like
+  ([^Writer w tag o]
+   (.writeTag w tag 1)
+   (.writeList w o)))
+
+(defn read-list-like
+  [^Reader rdr build-fn]
+  (build-fn (.readObject rdr)))
+
+(defn write-map-like
+  "Writes a map as Fressian with the tag 'map' and all keys cached."
+  [^Writer w tag m]
+  (.writeTag w tag 1)
+  (.beginClosedList ^StreamingWriter w)
+  (loop [items (seq m)]
+    (when-let [^clojure.lang.MapEntry item (first items)]
+      (.writeObject w (.key item) true)
+      (.writeObject w (.val item))
+      (recur (rest items))))
+  (.endList ^StreamingWriter w))
+
+(defn read-map-like
+  [^Reader rdr]
+  (let [kvs ^java.util.List (.readObject rdr)]
+    (if (< (.size kvs) 16)
+      (clojure.lang.PersistentArrayMap. (.toArray kvs))
+      (clojure.lang.PersistentHashMap/create (seq kvs)))))
+
+(def write-handlers
+  { Character
+   {"char"
+    (reify WriteHandler
+      (write [_ w ch]
+        (.writeTag w "char" 1)
+        (.writeInt w (int ch))))}
+
+   app.common.geom.point.Point
+   {"penpot/point"
+    (reify WriteHandler
+      (write [_ w o]
+        (.writeTag ^Writer w "penpot/point" 1)
+        (.writeList ^Writer w (java.util.List/of (.-x ^Point o) (.-y ^Point o)))))}
+
+   app.common.geom.matrix.Matrix
+   {"penpot/matrix"
+    (reify WriteHandler
+      (write [_ w o]
+        (.writeTag ^Writer w "penpot/matrix" 1)
+        (.writeList ^Writer w (java.util.List/of (.-a ^Matrix o)
+                                                 (.-b ^Matrix o)
+                                                 (.-c ^Matrix o)
+                                                 (.-d ^Matrix o)
+                                                 (.-e ^Matrix o)
+                                                 (.-f ^Matrix o)))))}
+
+   Ratio
+   {"ratio"
+    (reify WriteHandler
+      (write [_ w n]
+        (.writeTag w "ratio" 2)
+        (.writeObject w (.numerator ^Ratio n))
+        (.writeObject w (.denominator ^Ratio n))))}
+
+   clojure.lang.IPersistentMap
+   {"clj/map"
+    (reify WriteHandler
+      (write [_ w d]
+        (write-map-like w "clj/map" d)))}
+
+   clojure.lang.Keyword
+   {"clj/keyword"
+    (reify WriteHandler
+      (write [_ w s]
+        (write-named "clj/keyword" w s)))}
+
+   clojure.lang.BigInt
+   {"bigint"
+    (reify WriteHandler
+      (write [_ w d]
+        (let [^BigInteger bi (if (instance? clojure.lang.BigInt d)
+                               (.toBigInteger ^clojure.lang.BigInt d)
+                               d)]
+          (.writeTag w "bigint" 1)
+          (.writeBytes w (.toByteArray bi)))))}
+
+   ;; Persistent set
+   clojure.lang.IPersistentSet
+   {"clj/set"
+    (reify WriteHandler
+      (write [_ w o]
+        (write-list-like w "clj/set" o)))}
+
+   ;; Persistent vector
+   clojure.lang.IPersistentVector
+   {"clj/vector"
+    (reify WriteHandler
+      (write [_ w o]
+        (write-list-like w "clj/vector" o)))}
+
+   ;; Persistent list
+   clojure.lang.IPersistentList
+   {"clj/list"
+    (reify WriteHandler
+      (write [_ w o]
+        (write-list-like w "clj/list" o)))}
+
+   ;; Persistent seq & lazy seqs
+   clojure.lang.ISeq
+   {"clj/seq"
+    (reify WriteHandler
+      (write [_ w o]
+        (write-list-like w "clj/seq" o)))}
+   })
+
+
+(def read-handlers
+  {"bigint"
+   (reify ReadHandler
+     (read [_ rdr _ _]
+       (let [^bytes bibytes (.readObject rdr)]
+         (bigint (BigInteger. bibytes)))))
+
+   "byte"
+   (reify ReadHandler
+     (read [_ rdr _ _]
+       (byte (.readObject rdr))))
+
+   "penpot/matrix"
+   (reify ReadHandler
+     (read [_ rdr _ _]
+       (let [^java.util.List x (.readObject rdr)]
+         (Matrix. (.get x 0) (.get x 1) (.get x 2) (.get x 3) (.get x 4) (.get x 5)))))
+
+   "penpot/point"
+   (reify ReadHandler
+     (read [_ rdr _ _]
+       (let [^java.util.List x (.readObject rdr)]
+         (Point. (.get x 0) (.get x 1)))))
+
+   "char"
+   (reify ReadHandler
+     (read [_ rdr _ _]
+       (char (.readObject rdr))))
+
+   "clj/ratio"
+   (reify ReadHandler
+     (read [_ rdr _ _]
+       (Ratio. (biginteger (.readObject rdr))
+               (biginteger (.readObject rdr)))))
+
+   "clj/keyword"
+   (reify ReadHandler
+     (read [_ rdr _ _]
+       (keyword (.readObject rdr) (.readObject rdr))))
+
+   "clj/map"
+   (reify ReadHandler
+     (read [_ rdr _ _]
+       (read-map-like rdr)))
+
+   "clj/set"
+   (reify ReadHandler
+     (read [_ rdr _ _]
+       (read-list-like rdr set)))
+
+   "clj/vector"
+   (reify ReadHandler
+     (read [_ rdr _ _]
+       (read-list-like rdr vec)))
+
+   "clj/list"
+   (reify ReadHandler
+     (read [_ rdr _ _]
+       (read-list-like rdr #(apply list %))))
+
+   "clj/seq"
+   (reify ReadHandler
+     (read [_ rdr _ _]
+       (read-list-like rdr sequence)))
+   })
+
+(def write-handler-lookup
+  (-> write-handlers
+      fres/associative-lookup
+      fres/inheritance-lookup))
+
+(def read-handler-lookup
+  (-> read-handlers
+      (fres/associative-lookup)))
+
+;; --- Low-Level Api
+
+(defn reader
+  [istream]
+  (fres/create-reader istream :handlers read-handler-lookup))
+
+(defn writer
+  [ostream]
+  (fres/create-writer ostream :handlers write-handler-lookup))
+
+(defn read!
+  [reader]
+  (fres/read-object reader))
+
+(defn write!
+  [writer data]
+  (fres/write-object writer data))
+
+;; --- High-Level Api
+
+(defn encode
+  [data]
+  (with-open [out (ByteArrayOutputStream.)]
+    (write! (writer out) data)
+    (.toByteArray out)))
+
+(defn decode
+  [data]
+  (with-open [input (ByteArrayInputStream. ^bytes data)]
+    (read! (reader input))))

common/src/app/common/data.cljc

@@ -6,7 +6,7 @@
 
 (ns app.common.data
   "Data manipulation and query helper functions."
-  (:refer-clojure :exclude [read-string hash-map merge name])
+  (:refer-clojure :exclude [read-string hash-map merge name parse-double])
   #?(:cljs
      (:require-macros [app.common.data]))
   (:require

common/src/app/common/transit.cljc

@@ -144,19 +144,28 @@
 
 ;; --- Low-Level Api
 
+#?(:clj
+   (def read-handlers
+     (t/read-handler-map +read-handlers+)))
+
+#?(:clj
+   (def write-handlers
+     (t/write-handler-map +write-handlers+)))
+
 #?(:clj
    (defn reader
      ([istream]
       (reader istream nil))
      ([istream {:keys [type] :or {type :json}}]
-      (t/reader istream type {:handlers +read-handlers+}))))
+      (t/reader istream type {:handlers read-handlers}))))
 
 #?(:clj
    (defn writer
      ([ostream]
       (writer ostream nil))
      ([ostream {:keys [type] :or {type :json}}]
-      (t/writer ostream type {:handlers +write-handlers+}))))
+      (t/writer ostream type {:handlers write-handlers}))))
+
 #?(:clj
    (defn read!
      [reader]