From 68d8a49466269c3791281b459cf69cf65fa89182 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 8 Mar 2021 13:14:52 +0100 Subject: [PATCH] :tada: Add blob encoding v3. Safer approach (uses json instead of custom binary format) than v2. --- CHANGES.md | 1 + backend/dev/user.clj | 15 +++++++++++---- backend/src/app/util/blob.clj | 27 +++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4e755a340..50148ceb4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,7 @@ ### :sparkles: New features - Add more chinese translations [#726](https://github.com/penpot/penpot/pull/726) +- Add blob-encoding v3 (uses ZSTD+transit) [#738](https://github.com/penpot/penpot/pull/738) - Duplicate and move files and projects [Taiga #267](https://tree.taiga.io/project/penpot/us/267) - Import SVG will create Penpot's shapes - Improve french translations [#731](https://github.com/penpot/penpot/pull/731) diff --git a/backend/dev/user.clj b/backend/dev/user.clj index 5b861a9fa..d7ee62abf 100644 --- a/backend/dev/user.clj +++ b/backend/dev/user.clj @@ -12,11 +12,12 @@ [app.common.exceptions :as ex] [app.config :as cfg] [app.main :as main] + [app.util.blob :as blob] + [app.util.json :as json] [app.util.time :as dt] [app.util.transit :as t] - [app.util.json :as json] [clojure.java.io :as io] - [clojure.pprint :refer [pprint]] + [clojure.pprint :refer [pprint print-table]] [clojure.repl :refer :all] [clojure.spec.alpha :as s] [clojure.spec.gen.alpha :as sgen] @@ -25,8 +26,7 @@ [clojure.tools.namespace.repl :as repl] [clojure.walk :refer [macroexpand-all]] [criterium.core :refer [quick-bench bench with-progress-reporting]] - [integrant.core :as ig] - [taoensso.nippy :as nippy])) + [integrant.core :as ig])) (repl/disable-reload! (find-ns 'integrant.core)) @@ -91,3 +91,10 @@ [] (stop) (repl/refresh-all :after 'user/start)) + +(defn compression-bench + [data] + (print-table + [{:v1 (alength (blob/encode data {:version 1})) + :v2 (alength (blob/encode data {:version 2})) + :v3 (alength (blob/encode data {:version 3}))}])) diff --git a/backend/src/app/util/blob.clj b/backend/src/app/util/blob.clj index dd1624b1f..b478f9de2 100644 --- a/backend/src/app/util/blob.clj +++ b/backend/src/app/util/blob.clj @@ -28,8 +28,10 @@ (declare decode-v1) (declare decode-v2) +(declare decode-v3) (declare encode-v1) (declare encode-v2) +(declare encode-v3) (def default-version (:default-blob-version cfg/config 1)) @@ -40,6 +42,7 @@ (case (long version) 1 (encode-v1 data) 2 (encode-v2 data) + 3 (encode-v3 data) (throw (ex-info "unsupported version" {:version version}))))) (defn decode @@ -52,6 +55,7 @@ (case version 1 (decode-v1 data ulen) 2 (decode-v2 data ulen) + 3 (decode-v3 data ulen) (throw (ex-info "unsupported version" {:version version})))))) ;; --- IMPL @@ -100,3 +104,26 @@ (Zstd/decompressByteArray ^bytes udata 0 ulen ^bytes cdata 6 (- (alength cdata) 6)) (n/fast-thaw udata))) + +(defn- encode-v3 + [data] + (let [data (t/encode data {:type :json}) + dlen (alength ^bytes data) + mlen (Zstd/compressBound dlen) + cdata (byte-array mlen) + clen (Zstd/compressByteArray ^bytes cdata 0 mlen + ^bytes data 0 dlen + 4)] + (with-open [^ByteArrayOutputStream baos (ByteArrayOutputStream. (+ (alength cdata) 2 4)) + ^DataOutputStream dos (DataOutputStream. baos)] + (.writeShort dos (short 3)) ;; version number + (.writeInt dos (int dlen)) + (.write dos ^bytes cdata (int 0) clen) + (.toByteArray baos)))) + +(defn- decode-v3 + [^bytes cdata ^long ulen] + (let [udata (byte-array ulen)] + (Zstd/decompressByteArray ^bytes udata 0 ulen + ^bytes cdata 6 (- (alength cdata) 6)) + (t/decode udata {:type :json})))