From 5cd0079e7fbe6fb39540b4cc57c7093c957fda56 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 6 Sep 2022 16:52:51 +0200 Subject: [PATCH 1/3] :paperclip: Add some utilities on user ns --- backend/dev/user.clj | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/backend/dev/user.clj b/backend/dev/user.clj index 7da07ecf0..43ec7d4da 100644 --- a/backend/dev/user.clj +++ b/backend/dev/user.clj @@ -10,6 +10,7 @@ [app.common.exceptions :as ex] [app.common.geom.matrix :as gmt] [app.common.perf :as perf] + [app.common.pprint :as pp] [app.common.transit :as t] [app.config :as cfg] [app.main :as main] @@ -35,6 +36,24 @@ (defonce system nil) +;; --- Benchmarking Tools + +(defmacro run-quick-bench + [& exprs] + `(with-progress-reporting (quick-bench (do ~@exprs) :verbose))) + +(defmacro run-quick-bench' + [& exprs] + `(quick-bench (do ~@exprs))) + +(defmacro run-bench + [& exprs] + `(with-progress-reporting (bench (do ~@exprs) :verbose))) + +(defmacro run-bench' + [& exprs] + `(bench (do ~@exprs))) + ;; --- Development Stuff (defn- run-tests From e0ac583abad8df3e471c070831d705331652d82e Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 6 Sep 2022 16:53:28 +0200 Subject: [PATCH 2/3] :paperclip: Improve analyze-files helper on srepl ns --- backend/src/app/srepl/helpers.clj | 34 ++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/backend/src/app/srepl/helpers.clj b/backend/src/app/srepl/helpers.clj index 307ccbc54..ecb25981d 100644 --- a/backend/src/app/srepl/helpers.clj +++ b/backend/src/app/srepl/helpers.clj @@ -25,6 +25,7 @@ [app.util.blob :as blob] [app.util.time :as dt] [clojure.spec.alpha :as s] + [clojure.stacktrace :as strace] [clojure.walk :as walk] [cuerdas.core :as str] [expound.alpha :as expound])) @@ -78,7 +79,7 @@ (update file :data blob/decode)))) (def ^:private sql:retrieve-files-chunk - "SELECT id, name, modified_at, data FROM file + "SELECT id, name, created_at, data FROM file WHERE created_at < ? AND deleted_at is NULL ORDER BY created_at desc LIMIT ?") @@ -88,26 +89,39 @@ The `on-file` parameter should be a function that receives the file and the previous state and returns the new state." - [system & {:keys [chunk-size on-file] :or {chunk-size 10}}] + [system & {:keys [chunk-size max-chunks start-at on-file on-error on-end] + :or {chunk-size 10 max-chunks Long/MAX_VALUE}}] (letfn [(get-chunk [conn cursor] (let [rows (db/exec! conn [sql:retrieve-files-chunk cursor chunk-size])] [(some->> rows peek :created-at) (seq rows)])) (get-candidates [conn] (->> (d/iteration (partial get-chunk conn) - :vf second - :kf first - :initk (dt/now)) - (sequence cat) - (map #(update % :data blob/decode))))] + :vf second + :kf first + :initk (or start-at (dt/now))) + (take max-chunks) + (mapcat identity) + (map #(update % :data blob/decode)))) + + (on-error* [file cause] + (println "unexpected exception happened on processing file: " (:id file)) + (strace/print-stack-trace cause))] (db/with-atomic [conn (:app.db/pool system)] (loop [state {} files (get-candidates conn)] (if-let [file (first files)] - (let [state (on-file file state)] - (recur state (rest files))) - state))))) + (let [state' (try + (on-file file state) + (catch Throwable cause + (let [on-error (or on-error on-error*)] + (on-error file cause))))] + (recur (or state' state) (rest files))) + + (if (fn? on-end) + (on-end state) + state)))))) (defn analyze-file-data From 675a07bac6d12f7455581ac147055ca4aa9904d2 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 6 Sep 2022 17:01:52 +0200 Subject: [PATCH 3/3] :fire: Remove internal file data blob format v2 --- CHANGES.md | 6 ++++++ backend/deps.edn | 1 - backend/src/app/util/blob.clj | 32 ++------------------------------ 3 files changed, 8 insertions(+), 31 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ef0a0fe8e..4317c6d53 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,12 @@ ## :rocket: Next ### :boom: Breaking changes & Deprecations + +- Removed the support for v2 internal file data blob format. This + version has never been documented nor set as default value so + technicaly this is not a breaking change because we are removing + a "private API". + ### :sparkles: New features - Add team hero in projects dashboard [Taiga #3863](https://tree.taiga.io/project/penpot/us/3863) diff --git a/backend/deps.edn b/backend/deps.edn index b75718a73..d96922748 100644 --- a/backend/deps.edn +++ b/backend/deps.edn @@ -6,7 +6,6 @@ ;; Logging org.zeromq/jeromq {:mvn/version "0.5.2"} - com.taoensso/nippy {:mvn/version "3.1.1"} com.github.luben/zstd-jni {:mvn/version "1.5.2-3"} org.clojure/data.fressian {:mvn/version "1.0.0"} diff --git a/backend/src/app/util/blob.clj b/backend/src/app/util/blob.clj index f4daebc35..6c657532c 100644 --- a/backend/src/app/util/blob.clj +++ b/backend/src/app/util/blob.clj @@ -10,8 +10,7 @@ (:require [app.common.transit :as t] [app.config :as cf] - [app.util.fressian :as fres] - [taoensso.nippy :as n]) + [app.util.fressian :as fres]) (:import java.io.ByteArrayInputStream java.io.ByteArrayOutputStream @@ -27,21 +26,18 @@ (def lz4-factory (LZ4Factory/fastestInstance)) (declare decode-v1) -(declare decode-v2) (declare decode-v3) (declare decode-v4) (declare encode-v1) -(declare encode-v2) (declare encode-v3) (declare encode-v4) (defn encode ([data] (encode data nil)) ([data {:keys [version]}] - (let [version (or version (cf/get :default-blob-version 3))] + (let [version (or version (cf/get :default-blob-version 4))] (case (long version) 1 (encode-v1 data) - 2 (encode-v2 data) 3 (encode-v3 data) 4 (encode-v4 data) (throw (ex-info "unsupported version" {:version version})))))) @@ -55,7 +51,6 @@ ulen (.readInt dis)] (case version 1 (decode-v1 data ulen) - 2 (decode-v2 data ulen) 3 (decode-v3 data ulen) 4 (decode-v4 data ulen) (throw (ex-info "unsupported version" {:version version})))))) @@ -84,29 +79,6 @@ (.decompress ^LZ4FastDecompressor dcp cdata 6 ^bytes udata 0 ulen) (t/decode udata {:type :json}))) -(defn- encode-v2 - [data] - (let [data (n/fast-freeze data) - dlen (alength ^bytes data) - mlen (Zstd/compressBound dlen) - cdata (byte-array mlen) - clen (Zstd/compressByteArray ^bytes cdata 0 mlen - ^bytes data 0 dlen - 8)] - (with-open [^ByteArrayOutputStream baos (ByteArrayOutputStream. (+ (alength cdata) 2 4)) - ^DataOutputStream dos (DataOutputStream. baos)] - (.writeShort dos (short 2)) ;; version number - (.writeInt dos (int dlen)) - (.write dos ^bytes cdata (int 0) clen) - (.toByteArray baos)))) - -(defn- decode-v2 - [^bytes cdata ^long ulen] - (let [udata (byte-array ulen)] - (Zstd/decompressByteArray ^bytes udata 0 ulen - ^bytes cdata 6 (- (alength cdata) 6)) - (n/fast-thaw udata))) - (defn- encode-v3 [data] (let [data (t/encode data {:type :json})