✨ Replace snappy with lz4 for blobs compression.

backend/deps.edn

@@ -35,7 +35,8 @@
   org.jsoup/jsoup {:mvn/version "1.12.1"}
   org.im4java/im4java {:mvn/version "1.4.0"}
 
-  org.xerial.snappy/snappy-java {:mvn/version "1.1.7.3"}
+  org.lz4/lz4-java {:mvn/version "1.7.1"}
+
   com.github.spullara.mustache.java/compiler {:mvn/version "0.9.6"}
   commons-io/commons-io {:mvn/version "2.6"}
   com.draines/postal {:mvn/version "2.0.3"
@@ -51,7 +52,9 @@
  {:dev
   {:extra-deps
    {com.bhauman/rebel-readline {:mvn/version "0.1.4"}
-    org.clojure/tools.namespace {:mvn/version "0.3.1"}
+    org.clojure/tools.namespace {:mvn/version "1.0.0"}
+    org.clojure/test.check {:mvn/version "1.0.0"}
+
     fipp/fipp {:mvn/version "0.6.21"}
     criterium/criterium {:mvn/version "0.4.5"}
     mockery/mockery {:mvn/version "0.1.4"}}

backend/src/uxbox/util/blob.clj

@@ -2,7 +2,10 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; Copyright (c) 2016 Andrey Antukh <niwi@niwi.nz>
+;; This Source Code Form is "Incompatible With Secondary Licenses", as
+;; defined by the Mozilla Public License, v. 2.0.
+;;
+;; Copyright (c) 2016-2020 Andrey Antukh <niwi@niwi.nz>
 
 (ns uxbox.util.blob
   "A generic blob storage encoding. Mainly used for
@@ -14,7 +17,9 @@
    java.io.ByteArrayOutputStream
    java.io.DataInputStream
    java.io.DataOutputStream
-   org.xerial.snappy.Snappy))
+   net.jpountz.lz4.LZ4Factory
+   net.jpountz.lz4.LZ4FastDecompressor
+   net.jpountz.lz4.LZ4Compressor))
 
 (defprotocol IDataToBytes
   (->bytes [data] "convert data to bytes"))
@@ -29,17 +34,21 @@
   String
   (->bytes [data] (.getBytes ^String data "UTF-8")))
 
+(def lz4-factory (LZ4Factory/fastestInstance))
+
 (defn encode
-  "A function used for encode data for persist in the database."
   [data]
   (let [data (t/encode data {:type :json})
         data-len (alength ^bytes data)
-        cdata (Snappy/compress ^bytes data)]
+        cp (.fastCompressor ^LZ4Factory lz4-factory)
+        max-len (.maxCompressedLength cp data-len)
+        cdata (byte-array max-len)
+        clen (.compress ^LZ4Compressor cp ^bytes data 0 data-len cdata 0 max-len)]
     (with-open [^ByteArrayOutputStream baos (ByteArrayOutputStream. (+ (alength cdata) 2 4))
                 ^DataOutputStream dos (DataOutputStream. baos)]
       (.writeShort dos (short 1)) ;; version number
       (.writeInt dos (int data-len))
-      (.write dos ^bytes cdata (int 0) (alength cdata))
+      (.write dos ^bytes cdata (int 0) clen)
       (-> (.toByteArray baos)
           (t/bytes->buffer)))))
 
@@ -53,12 +62,14 @@
                 dis  (DataInputStream. bais)]
       (let [version (.readShort dis)
             udata-len (.readInt dis)]
-        (when (not= version 1)
+        (case version
-          (throw (ex-info "unsupported version" {:version version})))
+          1 (decode-v1 data udata-len)
-        (decode-v1 data udata-len)))))
+          (throw (ex-info "unsupported version" {:version version})))))))
 
 (defn- decode-v1
-  [^bytes data udata-len]
+  [^bytes cdata ^long udata-len]
-  (let [^bytes output-ba (byte-array udata-len)]
+  (let [^LZ4FastDecompressor dcp (.fastDecompressor ^LZ4Factory lz4-factory)
-    (Snappy/uncompress data 6 (- (alength data) 6) output-ba 0)
+        ^bytes udata (byte-array udata-len)]
-    (t/decode output-ba {:type :json})))
+    (.decompress dcp cdata 6 udata 0 udata-len)
+    (t/decode udata {:type :json})))
+

backend/src/uxbox/util/snappy.clj

@@ -1,38 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) 2019 Andrey Antukh <niwi@niwi.nz>
-
-(ns uxbox.util.snappy
-  "A lightweight abstraction layer for snappy compression library."
-  (:import
-   java.io.ByteArrayInputStream
-   java.io.ByteArrayOutputStream
-   java.io.InputStream
-   java.io.OutputStream
-   org.xerial.snappy.Snappy
-   org.xerial.snappy.SnappyFramedInputStream
-   org.xerial.snappy.SnappyFramedOutputStream))
-
-(defn compress
-  "Compress data unsing snappy compression algorithm."
-  [^bytes data]
-  (Snappy/compress data))
-
-(defn uncompress
-  "Uncompress data using snappy compression algorithm."
-  [^bytes data]
-  (Snappy/uncompress data))
-
-(defn input-stream
-  "Create a Snappy framed input stream."
-  [^InputStream istream]
-  (SnappyFramedInputStream. istream))
-
-(defn output-stream
-  "Create a Snappy framed output stream."
-  ([ostream]
-   (output-stream ostream nil))
-  ([^OutputStream ostream {:keys [block-size] :or {block-size 65536}}]
-   (SnappyFramedOutputStream. ostream (int block-size) 1.0)))