diff --git a/backend/deps.edn b/backend/deps.edn index efdc4ed55..58abac566 100644 --- a/backend/deps.edn +++ b/backend/deps.edn @@ -35,7 +35,8 @@ org.jsoup/jsoup {:mvn/version "1.12.1"} org.im4java/im4java {:mvn/version "1.4.0"} - org.xerial.snappy/snappy-java {:mvn/version "1.1.7.3"} + org.lz4/lz4-java {:mvn/version "1.7.1"} + com.github.spullara.mustache.java/compiler {:mvn/version "0.9.6"} commons-io/commons-io {:mvn/version "2.6"} com.draines/postal {:mvn/version "2.0.3" @@ -51,7 +52,9 @@ {:dev {:extra-deps {com.bhauman/rebel-readline {:mvn/version "0.1.4"} - org.clojure/tools.namespace {:mvn/version "0.3.1"} + org.clojure/tools.namespace {:mvn/version "1.0.0"} + org.clojure/test.check {:mvn/version "1.0.0"} + fipp/fipp {:mvn/version "0.6.21"} criterium/criterium {:mvn/version "0.4.5"} mockery/mockery {:mvn/version "0.1.4"}} diff --git a/backend/src/uxbox/util/blob.clj b/backend/src/uxbox/util/blob.clj index e32dcfb9b..a1953555c 100644 --- a/backend/src/uxbox/util/blob.clj +++ b/backend/src/uxbox/util/blob.clj @@ -2,7 +2,10 @@ ;; License, v. 2.0. If a copy of the MPL was not distributed with this ;; file, You can obtain one at http://mozilla.org/MPL/2.0/. ;; -;; Copyright (c) 2016 Andrey Antukh +;; This Source Code Form is "Incompatible With Secondary Licenses", as +;; defined by the Mozilla Public License, v. 2.0. +;; +;; Copyright (c) 2016-2020 Andrey Antukh (ns uxbox.util.blob "A generic blob storage encoding. Mainly used for @@ -14,7 +17,9 @@ java.io.ByteArrayOutputStream java.io.DataInputStream java.io.DataOutputStream - org.xerial.snappy.Snappy)) + net.jpountz.lz4.LZ4Factory + net.jpountz.lz4.LZ4FastDecompressor + net.jpountz.lz4.LZ4Compressor)) (defprotocol IDataToBytes (->bytes [data] "convert data to bytes")) @@ -29,17 +34,21 @@ String (->bytes [data] (.getBytes ^String data "UTF-8"))) +(def lz4-factory (LZ4Factory/fastestInstance)) + (defn encode - "A function used for encode data for persist in the database." [data] (let [data (t/encode data {:type :json}) data-len (alength ^bytes data) - cdata (Snappy/compress ^bytes data)] + cp (.fastCompressor ^LZ4Factory lz4-factory) + max-len (.maxCompressedLength cp data-len) + cdata (byte-array max-len) + clen (.compress ^LZ4Compressor cp ^bytes data 0 data-len cdata 0 max-len)] (with-open [^ByteArrayOutputStream baos (ByteArrayOutputStream. (+ (alength cdata) 2 4)) ^DataOutputStream dos (DataOutputStream. baos)] (.writeShort dos (short 1)) ;; version number (.writeInt dos (int data-len)) - (.write dos ^bytes cdata (int 0) (alength cdata)) + (.write dos ^bytes cdata (int 0) clen) (-> (.toByteArray baos) (t/bytes->buffer))))) @@ -53,12 +62,14 @@ dis (DataInputStream. bais)] (let [version (.readShort dis) udata-len (.readInt dis)] - (when (not= version 1) - (throw (ex-info "unsupported version" {:version version}))) - (decode-v1 data udata-len))))) + (case version + 1 (decode-v1 data udata-len) + (throw (ex-info "unsupported version" {:version version}))))))) (defn- decode-v1 - [^bytes data udata-len] - (let [^bytes output-ba (byte-array udata-len)] - (Snappy/uncompress data 6 (- (alength data) 6) output-ba 0) - (t/decode output-ba {:type :json}))) + [^bytes cdata ^long udata-len] + (let [^LZ4FastDecompressor dcp (.fastDecompressor ^LZ4Factory lz4-factory) + ^bytes udata (byte-array udata-len)] + (.decompress dcp cdata 6 udata 0 udata-len) + (t/decode udata {:type :json}))) + diff --git a/backend/src/uxbox/util/snappy.clj b/backend/src/uxbox/util/snappy.clj deleted file mode 100644 index 4816bf9c6..000000000 --- a/backend/src/uxbox/util/snappy.clj +++ /dev/null @@ -1,38 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) 2019 Andrey Antukh - -(ns uxbox.util.snappy - "A lightweight abstraction layer for snappy compression library." - (:import - java.io.ByteArrayInputStream - java.io.ByteArrayOutputStream - java.io.InputStream - java.io.OutputStream - org.xerial.snappy.Snappy - org.xerial.snappy.SnappyFramedInputStream - org.xerial.snappy.SnappyFramedOutputStream)) - -(defn compress - "Compress data unsing snappy compression algorithm." - [^bytes data] - (Snappy/compress data)) - -(defn uncompress - "Uncompress data using snappy compression algorithm." - [^bytes data] - (Snappy/uncompress data)) - -(defn input-stream - "Create a Snappy framed input stream." - [^InputStream istream] - (SnappyFramedInputStream. istream)) - -(defn output-stream - "Create a Snappy framed output stream." - ([ostream] - (output-stream ostream nil)) - ([^OutputStream ostream {:keys [block-size] :or {block-size 65536}}] - (SnappyFramedOutputStream. ostream (int block-size) 1.0)))