mirror of
https://github.com/penpot/penpot.git
synced 2025-03-15 01:01:30 -05:00
🎉 Add version 4 of blob encoding.
The version 4 starts using the hight performance fressian binary encoding with very lightweight compression layer.
This commit is contained in:
parent
a9904c6ada
commit
219f9c478d
12 changed files with 338 additions and 24 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
@ -35,6 +35,7 @@ node_modules
|
||||||
/common/target
|
/common/target
|
||||||
/common/coverage
|
/common/coverage
|
||||||
/.clj-kondo/.cache
|
/.clj-kondo/.cache
|
||||||
|
clj-profiler/
|
||||||
/bundle*
|
/bundle*
|
||||||
/media
|
/media
|
||||||
/deploy
|
/deploy
|
||||||
|
|
|
@ -13,6 +13,7 @@
|
||||||
|
|
||||||
com.taoensso/nippy {:mvn/version "3.1.1"}
|
com.taoensso/nippy {:mvn/version "3.1.1"}
|
||||||
com.github.luben/zstd-jni {:mvn/version "1.5.0-4"}
|
com.github.luben/zstd-jni {:mvn/version "1.5.0-4"}
|
||||||
|
org.clojure/data.fressian {:mvn/version "1.0.0"}
|
||||||
|
|
||||||
;; NOTE: don't upgrade to latest version, breaking change is
|
;; NOTE: don't upgrade to latest version, breaking change is
|
||||||
;; introduced on 0.10.0 that suffixes counters with _total if they
|
;; introduced on 0.10.0 that suffixes counters with _total if they
|
||||||
|
@ -63,6 +64,7 @@
|
||||||
org.clojure/test.check {:mvn/version "RELEASE"}
|
org.clojure/test.check {:mvn/version "RELEASE"}
|
||||||
org.clojure/data.csv {:mvn/version "1.0.0"}
|
org.clojure/data.csv {:mvn/version "1.0.0"}
|
||||||
com.clojure-goes-fast/clj-async-profiler {:mvn/version "0.5.1"}
|
com.clojure-goes-fast/clj-async-profiler {:mvn/version "0.5.1"}
|
||||||
|
clojure-humanize/clojure-humanize {:mvn/version "0.2.2"}
|
||||||
|
|
||||||
criterium/criterium {:mvn/version "RELEASE"}
|
criterium/criterium {:mvn/version "RELEASE"}
|
||||||
mockery/mockery {:mvn/version "RELEASE"}}
|
mockery/mockery {:mvn/version "RELEASE"}}
|
||||||
|
|
|
@ -6,15 +6,18 @@
|
||||||
|
|
||||||
(ns user
|
(ns user
|
||||||
(:require
|
(:require
|
||||||
|
[datoteka.core]
|
||||||
[app.common.exceptions :as ex]
|
[app.common.exceptions :as ex]
|
||||||
[app.config :as cfg]
|
[app.config :as cfg]
|
||||||
[app.main :as main]
|
[app.main :as main]
|
||||||
[app.util.blob :as blob]
|
[app.util.blob :as blob]
|
||||||
[app.util.json :as json]
|
[app.util.json :as json]
|
||||||
|
[app.util.fressian :as fres]
|
||||||
[app.util.time :as dt]
|
[app.util.time :as dt]
|
||||||
[app.util.transit :as t]
|
[app.common.transit :as t]
|
||||||
[clojure.java.io :as io]
|
[clojure.java.io :as io]
|
||||||
[clojure.pprint :refer [pprint print-table]]
|
[clojure.pprint :refer [pprint print-table]]
|
||||||
|
[clojure.contrib.humanize :as hum]
|
||||||
[clojure.repl :refer :all]
|
[clojure.repl :refer :all]
|
||||||
[clojure.spec.alpha :as s]
|
[clojure.spec.alpha :as s]
|
||||||
[clojure.spec.gen.alpha :as sgen]
|
[clojure.spec.gen.alpha :as sgen]
|
||||||
|
@ -22,10 +25,12 @@
|
||||||
[clojure.test :as test]
|
[clojure.test :as test]
|
||||||
[clojure.tools.namespace.repl :as repl]
|
[clojure.tools.namespace.repl :as repl]
|
||||||
[clojure.walk :refer [macroexpand-all]]
|
[clojure.walk :refer [macroexpand-all]]
|
||||||
|
[clj-async-profiler.core :as prof]
|
||||||
[criterium.core :refer [quick-bench bench with-progress-reporting]]
|
[criterium.core :refer [quick-bench bench with-progress-reporting]]
|
||||||
[integrant.core :as ig]))
|
[integrant.core :as ig]))
|
||||||
|
|
||||||
(repl/disable-reload! (find-ns 'integrant.core))
|
(repl/disable-reload! (find-ns 'integrant.core))
|
||||||
|
(set! *warn-on-reflection* true)
|
||||||
|
|
||||||
(defonce system nil)
|
(defonce system nil)
|
||||||
|
|
||||||
|
@ -91,7 +96,15 @@
|
||||||
|
|
||||||
(defn compression-bench
|
(defn compression-bench
|
||||||
[data]
|
[data]
|
||||||
(print-table
|
(let [humanize (fn [v] (hum/filesize v :binary true :format " %.4f "))]
|
||||||
[{:v1 (alength (blob/encode data {:version 1}))
|
(print-table
|
||||||
:v2 (alength (blob/encode data {:version 2}))
|
[{:v1 (humanize (alength (blob/encode data {:version 1})))
|
||||||
:v3 (alength (blob/encode data {:version 3}))}]))
|
:v2 (humanize (alength (blob/encode data {:version 2})))
|
||||||
|
:v3 (humanize (alength (blob/encode data {:version 3})))
|
||||||
|
:v4 (humanize (alength (blob/encode data {:version 4})))
|
||||||
|
}])))
|
||||||
|
|
||||||
|
|
||||||
|
;; ;; (def contents (read-string (slurp (io/resource "bool-contents-1.edn"))))
|
||||||
|
;; (def pre-data (datoteka.core/slurp-bytes (io/resource "file-data-sample")))
|
||||||
|
;; (def data (blob/decode pre-data))
|
||||||
|
|
|
@ -9,7 +9,6 @@ export OPTIONS="-A:jmx-remote:dev \
|
||||||
-J-Dclojure.tools.logging.factory=clojure.tools.logging.impl/log4j2-factory \
|
-J-Dclojure.tools.logging.factory=clojure.tools.logging.impl/log4j2-factory \
|
||||||
-J-XX:+UseShenandoahGC -J-XX:-OmitStackTraceInFastThrow -J-Xms50m -J-Xmx512m";
|
-J-XX:+UseShenandoahGC -J-XX:-OmitStackTraceInFastThrow -J-Xms50m -J-Xmx512m";
|
||||||
|
|
||||||
# export OPTIONS="$OPTIONS -J-XX:+UnlockDiagnosticVMOptions";
|
|
||||||
# export OPTIONS="$OPTIONS -J-XX:-TieredCompilation -J-XX:CompileThreshold=10000";
|
# export OPTIONS="$OPTIONS -J-XX:-TieredCompilation -J-XX:CompileThreshold=10000";
|
||||||
|
|
||||||
export OPTIONS_EVAL="nil"
|
export OPTIONS_EVAL="nil"
|
||||||
|
|
|
@ -27,14 +27,16 @@
|
||||||
com.zaxxer.hikari.HikariConfig
|
com.zaxxer.hikari.HikariConfig
|
||||||
com.zaxxer.hikari.HikariDataSource
|
com.zaxxer.hikari.HikariDataSource
|
||||||
com.zaxxer.hikari.metrics.prometheus.PrometheusMetricsTrackerFactory
|
com.zaxxer.hikari.metrics.prometheus.PrometheusMetricsTrackerFactory
|
||||||
|
java.io.InputStream
|
||||||
|
java.io.OutputStream
|
||||||
java.lang.AutoCloseable
|
java.lang.AutoCloseable
|
||||||
java.sql.Connection
|
java.sql.Connection
|
||||||
java.sql.Savepoint
|
java.sql.Savepoint
|
||||||
org.postgresql.PGConnection
|
org.postgresql.PGConnection
|
||||||
org.postgresql.geometric.PGpoint
|
org.postgresql.geometric.PGpoint
|
||||||
|
org.postgresql.jdbc.PgArray
|
||||||
org.postgresql.largeobject.LargeObject
|
org.postgresql.largeobject.LargeObject
|
||||||
org.postgresql.largeobject.LargeObjectManager
|
org.postgresql.largeobject.LargeObjectManager
|
||||||
org.postgresql.jdbc.PgArray
|
|
||||||
org.postgresql.util.PGInterval
|
org.postgresql.util.PGInterval
|
||||||
org.postgresql.util.PGobject))
|
org.postgresql.util.PGobject))
|
||||||
|
|
||||||
|
|
|
@ -179,18 +179,18 @@
|
||||||
;; Add a unique listener to connection
|
;; Add a unique listener to connection
|
||||||
(.addListener sub-conn
|
(.addListener sub-conn
|
||||||
(reify RedisPubSubListener
|
(reify RedisPubSubListener
|
||||||
(message [it pattern topic message])
|
(message [_it _pattern _topic _message])
|
||||||
(message [it topic message]
|
(message [_it topic message]
|
||||||
;; There are no back pressure, so we use a sliding
|
;; There are no back pressure, so we use a sliding
|
||||||
;; buffer for cases when the pubsub broker sends
|
;; buffer for cases when the pubsub broker sends
|
||||||
;; more messages that we can process.
|
;; more messages that we can process.
|
||||||
(let [val {:topic topic :message (blob/decode message)}]
|
(let [val {:topic topic :message (blob/decode message)}]
|
||||||
(when-not (a/offer! rcv-ch val)
|
(when-not (a/offer! rcv-ch val)
|
||||||
(l/warn :msg "dropping message on subscription loop"))))
|
(l/warn :msg "dropping message on subscription loop"))))
|
||||||
(psubscribed [it pattern count])
|
(psubscribed [_it _pattern _count])
|
||||||
(punsubscribed [it pattern count])
|
(punsubscribed [_it _pattern _count])
|
||||||
(subscribed [it topic count])
|
(subscribed [_it _topic _count])
|
||||||
(unsubscribed [it topic count])))
|
(unsubscribed [_it _topic _count])))
|
||||||
|
|
||||||
(letfn [(subscribe-to-single-topic [nsubs topic chan]
|
(letfn [(subscribe-to-single-topic [nsubs topic chan]
|
||||||
(let [nsubs (if (nil? nsubs) #{chan} (conj nsubs chan))]
|
(let [nsubs (if (nil? nsubs) #{chan} (conj nsubs chan))]
|
||||||
|
|
|
@ -409,7 +409,6 @@
|
||||||
[conn project-id]
|
[conn project-id]
|
||||||
(:team-id (db/get-by-id conn :project project-id {:columns [:team-id]})))
|
(:team-id (db/get-by-id conn :project project-id {:columns [:team-id]})))
|
||||||
|
|
||||||
|
|
||||||
;; TEMPORARY FILE CREATION
|
;; TEMPORARY FILE CREATION
|
||||||
|
|
||||||
(s/def ::create-temp-file ::create-file)
|
(s/def ::create-temp-file ::create-file)
|
||||||
|
|
|
@ -117,11 +117,11 @@
|
||||||
io/IOFactory
|
io/IOFactory
|
||||||
(make-reader [_ opts]
|
(make-reader [_ opts]
|
||||||
(io/make-reader path opts))
|
(io/make-reader path opts))
|
||||||
(make-writer [_ opts]
|
(make-writer [_ _]
|
||||||
(throw (UnsupportedOperationException. "not implemented")))
|
(throw (UnsupportedOperationException. "not implemented")))
|
||||||
(make-input-stream [_ opts]
|
(make-input-stream [_ opts]
|
||||||
(io/make-input-stream path opts))
|
(io/make-input-stream path opts))
|
||||||
(make-output-stream [_ opts]
|
(make-output-stream [_ _]
|
||||||
(throw (UnsupportedOperationException. "not implemented")))
|
(throw (UnsupportedOperationException. "not implemented")))
|
||||||
clojure.lang.Counted
|
clojure.lang.Counted
|
||||||
(count [_] size)
|
(count [_] size)
|
||||||
|
@ -138,11 +138,11 @@
|
||||||
io/IOFactory
|
io/IOFactory
|
||||||
(make-reader [_ opts]
|
(make-reader [_ opts]
|
||||||
(io/make-reader bais opts))
|
(io/make-reader bais opts))
|
||||||
(make-writer [_ opts]
|
(make-writer [_ _]
|
||||||
(throw (UnsupportedOperationException. "not implemented")))
|
(throw (UnsupportedOperationException. "not implemented")))
|
||||||
(make-input-stream [_ opts]
|
(make-input-stream [_ opts]
|
||||||
(io/make-input-stream bais opts))
|
(io/make-input-stream bais opts))
|
||||||
(make-output-stream [_ opts]
|
(make-output-stream [_ _]
|
||||||
(throw (UnsupportedOperationException. "not implemented")))
|
(throw (UnsupportedOperationException. "not implemented")))
|
||||||
|
|
||||||
clojure.lang.Counted
|
clojure.lang.Counted
|
||||||
|
@ -159,11 +159,11 @@
|
||||||
io/IOFactory
|
io/IOFactory
|
||||||
(make-reader [_ opts]
|
(make-reader [_ opts]
|
||||||
(io/make-reader is opts))
|
(io/make-reader is opts))
|
||||||
(make-writer [_ opts]
|
(make-writer [_ _]
|
||||||
(throw (UnsupportedOperationException. "not implemented")))
|
(throw (UnsupportedOperationException. "not implemented")))
|
||||||
(make-input-stream [_ opts]
|
(make-input-stream [_ opts]
|
||||||
(io/make-input-stream is opts))
|
(io/make-input-stream is opts))
|
||||||
(make-output-stream [_ opts]
|
(make-output-stream [_ _]
|
||||||
(throw (UnsupportedOperationException. "not implemented")))
|
(throw (UnsupportedOperationException. "not implemented")))
|
||||||
|
|
||||||
clojure.lang.Counted
|
clojure.lang.Counted
|
||||||
|
|
|
@ -10,6 +10,7 @@
|
||||||
(:require
|
(:require
|
||||||
[app.common.transit :as t]
|
[app.common.transit :as t]
|
||||||
[app.config :as cf]
|
[app.config :as cf]
|
||||||
|
[app.util.fressian :as fres]
|
||||||
[taoensso.nippy :as n])
|
[taoensso.nippy :as n])
|
||||||
(:import
|
(:import
|
||||||
java.io.ByteArrayInputStream
|
java.io.ByteArrayInputStream
|
||||||
|
@ -21,23 +22,28 @@
|
||||||
net.jpountz.lz4.LZ4FastDecompressor
|
net.jpountz.lz4.LZ4FastDecompressor
|
||||||
net.jpountz.lz4.LZ4Compressor))
|
net.jpountz.lz4.LZ4Compressor))
|
||||||
|
|
||||||
|
(set! *warn-on-reflection* true)
|
||||||
|
|
||||||
(def lz4-factory (LZ4Factory/fastestInstance))
|
(def lz4-factory (LZ4Factory/fastestInstance))
|
||||||
|
|
||||||
(declare decode-v1)
|
(declare decode-v1)
|
||||||
(declare decode-v2)
|
(declare decode-v2)
|
||||||
(declare decode-v3)
|
(declare decode-v3)
|
||||||
|
(declare decode-v4)
|
||||||
(declare encode-v1)
|
(declare encode-v1)
|
||||||
(declare encode-v2)
|
(declare encode-v2)
|
||||||
(declare encode-v3)
|
(declare encode-v3)
|
||||||
|
(declare encode-v4)
|
||||||
|
|
||||||
(defn encode
|
(defn encode
|
||||||
([data] (encode data nil))
|
([data] (encode data nil))
|
||||||
([data {:keys [version]}]
|
([data {:keys [version]}]
|
||||||
(let [version (or version (cf/get :default-blob-version 1))]
|
(let [version (or version (cf/get :default-blob-version 3))]
|
||||||
(case (long version)
|
(case (long version)
|
||||||
1 (encode-v1 data)
|
1 (encode-v1 data)
|
||||||
2 (encode-v2 data)
|
2 (encode-v2 data)
|
||||||
3 (encode-v3 data)
|
3 (encode-v3 data)
|
||||||
|
4 (encode-v4 data)
|
||||||
(throw (ex-info "unsupported version" {:version version}))))))
|
(throw (ex-info "unsupported version" {:version version}))))))
|
||||||
|
|
||||||
(defn decode
|
(defn decode
|
||||||
|
@ -51,6 +57,7 @@
|
||||||
1 (decode-v1 data ulen)
|
1 (decode-v1 data ulen)
|
||||||
2 (decode-v2 data ulen)
|
2 (decode-v2 data ulen)
|
||||||
3 (decode-v3 data ulen)
|
3 (decode-v3 data ulen)
|
||||||
|
4 (decode-v4 data ulen)
|
||||||
(throw (ex-info "unsupported version" {:version version}))))))
|
(throw (ex-info "unsupported version" {:version version}))))))
|
||||||
|
|
||||||
;; --- IMPL
|
;; --- IMPL
|
||||||
|
@ -122,3 +129,26 @@
|
||||||
(Zstd/decompressByteArray ^bytes udata 0 ulen
|
(Zstd/decompressByteArray ^bytes udata 0 ulen
|
||||||
^bytes cdata 6 (- (alength cdata) 6))
|
^bytes cdata 6 (- (alength cdata) 6))
|
||||||
(t/decode udata {:type :json})))
|
(t/decode udata {:type :json})))
|
||||||
|
|
||||||
|
(defn- encode-v4
|
||||||
|
[data]
|
||||||
|
(let [data (fres/encode data)
|
||||||
|
dlen (alength ^bytes data)
|
||||||
|
mlen (Zstd/compressBound dlen)
|
||||||
|
cdata (byte-array mlen)
|
||||||
|
clen (Zstd/compressByteArray ^bytes cdata 0 mlen
|
||||||
|
^bytes data 0 dlen
|
||||||
|
0)]
|
||||||
|
(with-open [^ByteArrayOutputStream baos (ByteArrayOutputStream. (+ (alength cdata) 2 4))
|
||||||
|
^DataOutputStream dos (DataOutputStream. baos)]
|
||||||
|
(.writeShort dos (short 4)) ;; version number
|
||||||
|
(.writeInt dos (int dlen))
|
||||||
|
(.write dos ^bytes cdata (int 0) clen)
|
||||||
|
(.toByteArray baos))))
|
||||||
|
|
||||||
|
(defn- decode-v4
|
||||||
|
[^bytes cdata ^long ulen]
|
||||||
|
(let [udata (byte-array ulen)]
|
||||||
|
(Zstd/decompressByteArray ^bytes udata 0 ulen
|
||||||
|
^bytes cdata 6 (- (alength cdata) 6))
|
||||||
|
(fres/decode udata)))
|
||||||
|
|
259
backend/src/app/util/fressian.clj
Normal file
259
backend/src/app/util/fressian.clj
Normal file
|
@ -0,0 +1,259 @@
|
||||||
|
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
;;
|
||||||
|
;; Copyright (c) UXBOX Labs SL
|
||||||
|
|
||||||
|
(ns app.util.fressian
|
||||||
|
(:require
|
||||||
|
[app.common.geom.matrix :as gmt]
|
||||||
|
[app.common.geom.point :as gpt]
|
||||||
|
[clojure.data.fressian :as fres])
|
||||||
|
(:import
|
||||||
|
clojure.lang.Ratio
|
||||||
|
org.fressian.handlers.WriteHandler
|
||||||
|
org.fressian.handlers.ReadHandler
|
||||||
|
org.fressian.Writer
|
||||||
|
org.fressian.Reader
|
||||||
|
org.fressian.StreamingWriter
|
||||||
|
app.common.geom.matrix.Matrix
|
||||||
|
app.common.geom.point.Point
|
||||||
|
java.io.ByteArrayInputStream
|
||||||
|
java.io.ByteArrayOutputStream))
|
||||||
|
|
||||||
|
;; --- MISC
|
||||||
|
|
||||||
|
(set! *warn-on-reflection* true)
|
||||||
|
|
||||||
|
(defn str->bytes
|
||||||
|
([^String s]
|
||||||
|
(str->bytes s "UTF-8"))
|
||||||
|
([^String s, ^String encoding]
|
||||||
|
(.getBytes s encoding)))
|
||||||
|
|
||||||
|
(defn write-named
|
||||||
|
[tag ^Writer w s]
|
||||||
|
(.writeTag w tag 2)
|
||||||
|
(.writeObject w (namespace s) true)
|
||||||
|
(.writeObject w (name s) true))
|
||||||
|
|
||||||
|
(defn write-list-like
|
||||||
|
([^Writer w tag o]
|
||||||
|
(.writeTag w tag 1)
|
||||||
|
(.writeList w o)))
|
||||||
|
|
||||||
|
(defn read-list-like
|
||||||
|
[^Reader rdr build-fn]
|
||||||
|
(build-fn (.readObject rdr)))
|
||||||
|
|
||||||
|
(defn write-map-like
|
||||||
|
"Writes a map as Fressian with the tag 'map' and all keys cached."
|
||||||
|
[^Writer w tag m]
|
||||||
|
(.writeTag w tag 1)
|
||||||
|
(.beginClosedList ^StreamingWriter w)
|
||||||
|
(loop [items (seq m)]
|
||||||
|
(when-let [^clojure.lang.MapEntry item (first items)]
|
||||||
|
(.writeObject w (.key item) true)
|
||||||
|
(.writeObject w (.val item))
|
||||||
|
(recur (rest items))))
|
||||||
|
(.endList ^StreamingWriter w))
|
||||||
|
|
||||||
|
(defn read-map-like
|
||||||
|
[^Reader rdr]
|
||||||
|
(let [kvs ^java.util.List (.readObject rdr)]
|
||||||
|
(if (< (.size kvs) 16)
|
||||||
|
(clojure.lang.PersistentArrayMap. (.toArray kvs))
|
||||||
|
(clojure.lang.PersistentHashMap/create (seq kvs)))))
|
||||||
|
|
||||||
|
(def write-handlers
|
||||||
|
{ Character
|
||||||
|
{"char"
|
||||||
|
(reify WriteHandler
|
||||||
|
(write [_ w ch]
|
||||||
|
(.writeTag w "char" 1)
|
||||||
|
(.writeInt w (int ch))))}
|
||||||
|
|
||||||
|
app.common.geom.point.Point
|
||||||
|
{"penpot/point"
|
||||||
|
(reify WriteHandler
|
||||||
|
(write [_ w o]
|
||||||
|
(.writeTag ^Writer w "penpot/point" 1)
|
||||||
|
(.writeList ^Writer w (java.util.List/of (.-x ^Point o) (.-y ^Point o)))))}
|
||||||
|
|
||||||
|
app.common.geom.matrix.Matrix
|
||||||
|
{"penpot/matrix"
|
||||||
|
(reify WriteHandler
|
||||||
|
(write [_ w o]
|
||||||
|
(.writeTag ^Writer w "penpot/matrix" 1)
|
||||||
|
(.writeList ^Writer w (java.util.List/of (.-a ^Matrix o)
|
||||||
|
(.-b ^Matrix o)
|
||||||
|
(.-c ^Matrix o)
|
||||||
|
(.-d ^Matrix o)
|
||||||
|
(.-e ^Matrix o)
|
||||||
|
(.-f ^Matrix o)))))}
|
||||||
|
|
||||||
|
Ratio
|
||||||
|
{"ratio"
|
||||||
|
(reify WriteHandler
|
||||||
|
(write [_ w n]
|
||||||
|
(.writeTag w "ratio" 2)
|
||||||
|
(.writeObject w (.numerator ^Ratio n))
|
||||||
|
(.writeObject w (.denominator ^Ratio n))))}
|
||||||
|
|
||||||
|
clojure.lang.IPersistentMap
|
||||||
|
{"clj/map"
|
||||||
|
(reify WriteHandler
|
||||||
|
(write [_ w d]
|
||||||
|
(write-map-like w "clj/map" d)))}
|
||||||
|
|
||||||
|
clojure.lang.Keyword
|
||||||
|
{"clj/keyword"
|
||||||
|
(reify WriteHandler
|
||||||
|
(write [_ w s]
|
||||||
|
(write-named "clj/keyword" w s)))}
|
||||||
|
|
||||||
|
clojure.lang.BigInt
|
||||||
|
{"bigint"
|
||||||
|
(reify WriteHandler
|
||||||
|
(write [_ w d]
|
||||||
|
(let [^BigInteger bi (if (instance? clojure.lang.BigInt d)
|
||||||
|
(.toBigInteger ^clojure.lang.BigInt d)
|
||||||
|
d)]
|
||||||
|
(.writeTag w "bigint" 1)
|
||||||
|
(.writeBytes w (.toByteArray bi)))))}
|
||||||
|
|
||||||
|
;; Persistent set
|
||||||
|
clojure.lang.IPersistentSet
|
||||||
|
{"clj/set"
|
||||||
|
(reify WriteHandler
|
||||||
|
(write [_ w o]
|
||||||
|
(write-list-like w "clj/set" o)))}
|
||||||
|
|
||||||
|
;; Persistent vector
|
||||||
|
clojure.lang.IPersistentVector
|
||||||
|
{"clj/vector"
|
||||||
|
(reify WriteHandler
|
||||||
|
(write [_ w o]
|
||||||
|
(write-list-like w "clj/vector" o)))}
|
||||||
|
|
||||||
|
;; Persistent list
|
||||||
|
clojure.lang.IPersistentList
|
||||||
|
{"clj/list"
|
||||||
|
(reify WriteHandler
|
||||||
|
(write [_ w o]
|
||||||
|
(write-list-like w "clj/list" o)))}
|
||||||
|
|
||||||
|
;; Persistent seq & lazy seqs
|
||||||
|
clojure.lang.ISeq
|
||||||
|
{"clj/seq"
|
||||||
|
(reify WriteHandler
|
||||||
|
(write [_ w o]
|
||||||
|
(write-list-like w "clj/seq" o)))}
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
(def read-handlers
|
||||||
|
{"bigint"
|
||||||
|
(reify ReadHandler
|
||||||
|
(read [_ rdr _ _]
|
||||||
|
(let [^bytes bibytes (.readObject rdr)]
|
||||||
|
(bigint (BigInteger. bibytes)))))
|
||||||
|
|
||||||
|
"byte"
|
||||||
|
(reify ReadHandler
|
||||||
|
(read [_ rdr _ _]
|
||||||
|
(byte (.readObject rdr))))
|
||||||
|
|
||||||
|
"penpot/matrix"
|
||||||
|
(reify ReadHandler
|
||||||
|
(read [_ rdr _ _]
|
||||||
|
(let [^java.util.List x (.readObject rdr)]
|
||||||
|
(Matrix. (.get x 0) (.get x 1) (.get x 2) (.get x 3) (.get x 4) (.get x 5)))))
|
||||||
|
|
||||||
|
"penpot/point"
|
||||||
|
(reify ReadHandler
|
||||||
|
(read [_ rdr _ _]
|
||||||
|
(let [^java.util.List x (.readObject rdr)]
|
||||||
|
(Point. (.get x 0) (.get x 1)))))
|
||||||
|
|
||||||
|
"char"
|
||||||
|
(reify ReadHandler
|
||||||
|
(read [_ rdr _ _]
|
||||||
|
(char (.readObject rdr))))
|
||||||
|
|
||||||
|
"clj/ratio"
|
||||||
|
(reify ReadHandler
|
||||||
|
(read [_ rdr _ _]
|
||||||
|
(Ratio. (biginteger (.readObject rdr))
|
||||||
|
(biginteger (.readObject rdr)))))
|
||||||
|
|
||||||
|
"clj/keyword"
|
||||||
|
(reify ReadHandler
|
||||||
|
(read [_ rdr _ _]
|
||||||
|
(keyword (.readObject rdr) (.readObject rdr))))
|
||||||
|
|
||||||
|
"clj/map"
|
||||||
|
(reify ReadHandler
|
||||||
|
(read [_ rdr _ _]
|
||||||
|
(read-map-like rdr)))
|
||||||
|
|
||||||
|
"clj/set"
|
||||||
|
(reify ReadHandler
|
||||||
|
(read [_ rdr _ _]
|
||||||
|
(read-list-like rdr set)))
|
||||||
|
|
||||||
|
"clj/vector"
|
||||||
|
(reify ReadHandler
|
||||||
|
(read [_ rdr _ _]
|
||||||
|
(read-list-like rdr vec)))
|
||||||
|
|
||||||
|
"clj/list"
|
||||||
|
(reify ReadHandler
|
||||||
|
(read [_ rdr _ _]
|
||||||
|
(read-list-like rdr #(apply list %))))
|
||||||
|
|
||||||
|
"clj/seq"
|
||||||
|
(reify ReadHandler
|
||||||
|
(read [_ rdr _ _]
|
||||||
|
(read-list-like rdr sequence)))
|
||||||
|
})
|
||||||
|
|
||||||
|
(def write-handler-lookup
|
||||||
|
(-> write-handlers
|
||||||
|
fres/associative-lookup
|
||||||
|
fres/inheritance-lookup))
|
||||||
|
|
||||||
|
(def read-handler-lookup
|
||||||
|
(-> read-handlers
|
||||||
|
(fres/associative-lookup)))
|
||||||
|
|
||||||
|
;; --- Low-Level Api
|
||||||
|
|
||||||
|
(defn reader
|
||||||
|
[istream]
|
||||||
|
(fres/create-reader istream :handlers read-handler-lookup))
|
||||||
|
|
||||||
|
(defn writer
|
||||||
|
[ostream]
|
||||||
|
(fres/create-writer ostream :handlers write-handler-lookup))
|
||||||
|
|
||||||
|
(defn read!
|
||||||
|
[reader]
|
||||||
|
(fres/read-object reader))
|
||||||
|
|
||||||
|
(defn write!
|
||||||
|
[writer data]
|
||||||
|
(fres/write-object writer data))
|
||||||
|
|
||||||
|
;; --- High-Level Api
|
||||||
|
|
||||||
|
(defn encode
|
||||||
|
[data]
|
||||||
|
(with-open [out (ByteArrayOutputStream.)]
|
||||||
|
(write! (writer out) data)
|
||||||
|
(.toByteArray out)))
|
||||||
|
|
||||||
|
(defn decode
|
||||||
|
[data]
|
||||||
|
(with-open [input (ByteArrayInputStream. ^bytes data)]
|
||||||
|
(read! (reader input))))
|
|
@ -6,7 +6,7 @@
|
||||||
|
|
||||||
(ns app.common.data
|
(ns app.common.data
|
||||||
"Data manipulation and query helper functions."
|
"Data manipulation and query helper functions."
|
||||||
(:refer-clojure :exclude [read-string hash-map merge name])
|
(:refer-clojure :exclude [read-string hash-map merge name parse-double])
|
||||||
#?(:cljs
|
#?(:cljs
|
||||||
(:require-macros [app.common.data]))
|
(:require-macros [app.common.data]))
|
||||||
(:require
|
(:require
|
||||||
|
|
|
@ -144,19 +144,28 @@
|
||||||
|
|
||||||
;; --- Low-Level Api
|
;; --- Low-Level Api
|
||||||
|
|
||||||
|
#?(:clj
|
||||||
|
(def read-handlers
|
||||||
|
(t/read-handler-map +read-handlers+)))
|
||||||
|
|
||||||
|
#?(:clj
|
||||||
|
(def write-handlers
|
||||||
|
(t/write-handler-map +write-handlers+)))
|
||||||
|
|
||||||
#?(:clj
|
#?(:clj
|
||||||
(defn reader
|
(defn reader
|
||||||
([istream]
|
([istream]
|
||||||
(reader istream nil))
|
(reader istream nil))
|
||||||
([istream {:keys [type] :or {type :json}}]
|
([istream {:keys [type] :or {type :json}}]
|
||||||
(t/reader istream type {:handlers +read-handlers+}))))
|
(t/reader istream type {:handlers read-handlers}))))
|
||||||
|
|
||||||
#?(:clj
|
#?(:clj
|
||||||
(defn writer
|
(defn writer
|
||||||
([ostream]
|
([ostream]
|
||||||
(writer ostream nil))
|
(writer ostream nil))
|
||||||
([ostream {:keys [type] :or {type :json}}]
|
([ostream {:keys [type] :or {type :json}}]
|
||||||
(t/writer ostream type {:handlers +write-handlers+}))))
|
(t/writer ostream type {:handlers write-handlers}))))
|
||||||
|
|
||||||
#?(:clj
|
#?(:clj
|
||||||
(defn read!
|
(defn read!
|
||||||
[reader]
|
[reader]
|
||||||
|
|
Loading…
Add table
Reference in a new issue