From 886ab0e152ee7299bca6af0c1410e199a40864a7 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 28 Sep 2022 23:26:31 +0200 Subject: [PATCH 1/2] :sparkles: Improve iteration and add concat-all and fully lazy mapcat helper --- backend/dev/user.clj | 2 + backend/src/app/srepl/helpers.clj | 7 +- backend/src/app/storage.clj | 19 +++-- backend/src/app/tasks/file_gc.clj | 9 ++- common/src/app/common/data.cljc | 115 +++++++++++++----------------- frontend/deps.edn | 2 +- 6 files changed, 68 insertions(+), 86 deletions(-) diff --git a/backend/dev/user.clj b/backend/dev/user.clj index 1313e7f54..591ddac70 100644 --- a/backend/dev/user.clj +++ b/backend/dev/user.clj @@ -9,11 +9,13 @@ [app.common.data :as d] [app.common.exceptions :as ex] [app.common.geom.matrix :as gmt] + [app.common.logging :as l] [app.common.perf :as perf] [app.common.pprint :as pp] [app.common.transit :as t] [app.config :as cfg] [app.main :as main] + [app.srepl.main :as srepl] [app.util.blob :as blob] [app.util.fressian :as fres] [app.util.json :as json] diff --git a/backend/src/app/srepl/helpers.clj b/backend/src/app/srepl/helpers.clj index 0f1a083d9..87dbd8b68 100644 --- a/backend/src/app/srepl/helpers.clj +++ b/backend/src/app/srepl/helpers.clj @@ -89,8 +89,8 @@ The `on-file` parameter should be a function that receives the file and the previous state and returns the new state." - [system & {:keys [chunk-size max-chunks start-at on-file on-error on-end] - :or {chunk-size 10 max-chunks Long/MAX_VALUE}}] + [system & {:keys [chunk-size max-items start-at on-file on-error on-end] + :or {chunk-size 10 max-items Long/MAX_VALUE}}] (letfn [(get-chunk [conn cursor] (let [rows (db/exec! conn [sql:retrieve-files-chunk cursor chunk-size])] [(some->> rows peek :created-at) (seq rows)])) @@ -100,8 +100,7 @@ :vf second :kf first :initk (or start-at (dt/now))) - (take max-chunks) - (mapcat identity) + (take max-items) (map #(update % :data blob/decode)))) (on-error* [file cause] diff --git a/backend/src/app/storage.clj b/backend/src/app/storage.clj index 572ee303e..556a365e6 100644 --- a/backend/src/app/storage.clj +++ b/backend/src/app/storage.clj @@ -284,11 +284,10 @@ (some->> (seq rows) (d/group-by #(-> % :backend keyword) :id #{}) seq)])) (retrieve-deleted-objects [conn min-age] - (->> (d/iteration (partial retrieve-deleted-objects-chunk conn min-age) - :initk (dt/now) - :vf second - :kf first) - (sequence cat))) + (d/iteration (partial retrieve-deleted-objects-chunk conn min-age) + :initk (dt/now) + :vf second + :kf first)) (delete-in-bulk [conn backend-name ids] (let [backend (impl/resolve-backend storage backend-name) @@ -397,12 +396,10 @@ (d/group-by get-bucket :id #{} rows)]))) (retrieve-touched [conn] - (->> (d/iteration (fn [cursor] - (retrieve-touched-chunk conn cursor)) - :initk (dt/now) - :vf second - :kf first) - (sequence cat))) + (d/iteration (partial retrieve-touched-chunk conn) + :initk (dt/now) + :vf second + :kf first)) (process-objects! [conn get-fn ids bucket] (loop [to-freeze #{} diff --git a/backend/src/app/tasks/file_gc.clj b/backend/src/app/tasks/file_gc.clj index db34e6934..f4800a518 100644 --- a/backend/src/app/tasks/file_gc.clj +++ b/backend/src/app/tasks/file_gc.clj @@ -90,11 +90,10 @@ get-chunk (fn [cursor] (let [rows (db/exec! conn [sql:retrieve-candidates-chunk interval cursor])] [(some->> rows peek :modified-at) (seq rows)]))] - - (sequence cat (d/iteration get-chunk - :vf second - :kf first - :initk (dt/now)))))) + (d/iteration get-chunk + :vf second + :kf first + :initk (dt/now))))) (defn collect-used-media [data] diff --git a/common/src/app/common/data.cljc b/common/src/app/common/data.cljc index 6c0a195d2..a413bb694 100644 --- a/common/src/app/common/data.cljc +++ b/common/src/app/common/data.cljc @@ -7,7 +7,7 @@ (ns app.common.data "Data manipulation and query helper functions." (:refer-clojure :exclude [read-string hash-map merge name update-vals - parse-double group-by iteration]) + parse-double group-by iteration concat mapcat]) #?(:cljs (:require-macros [app.common.data])) @@ -17,8 +17,8 @@ [cuerdas.core :as str] #?(:cljs [cljs.reader :as r] :clj [clojure.edn :as r]) - #?(:cljs [cljs.core :as core] - :clj [clojure.core :as core]) + #?(:cljs [cljs.core :as c] + :clj [clojure.core :as c]) [linked.set :as lks]) #?(:clj @@ -60,7 +60,7 @@ (defn editable-collection? [m] #?(:clj (instance? clojure.lang.IEditableCollection m) - :cljs (implements? core/IEditableCollection m))) + :cljs (implements? c/IEditableCollection m))) (defn deep-merge ([a b] @@ -81,6 +81,24 @@ m) (dissoc m k))) +(defn concat-all + "A totally lazy implementation of concat with different call + signature. It works like a flatten with a single level of neesting." + [colls] + (lazy-seq + (let [c (seq colls) + o (first c) + r (rest c)] + (if-let [o (seq o)] + (cons (first o) (concat-all (cons (rest o) r))) + (some-> (seq r) concat-all))))) + +(defn mapcat + "A fully lazy version of mapcat." + ([f] (c/mapcat f)) + ([f & colls] + (concat-all (apply map f colls)))) + (defn- transient-concat [c1 colls] (loop [result (transient c1) @@ -214,21 +232,12 @@ ([mfn coll] (into {} (mapm mfn) coll))) -;; TEMPORARY COPY of clojure.core/update-vals until we migrate to clojure 1.11 - (defn update-vals "m f => {k (f v) ...} Given a map m and a function f of 1-argument, returns a new map where the keys of m are mapped to result of applying f to the corresponding values of m." [m f] - (with-meta - (persistent! - (reduce-kv (fn [acc k v] (assoc! acc k (f v))) - (if (editable-collection? m) - (transient m) - (transient {})) - m)) - (meta m))) + (c/update-vals m f)) (defn removev "Returns a vector of the items in coll for which (fn item) returns logical false" @@ -294,7 +303,7 @@ (empty? col2) acc :else (recur (rest col1) col2 join-fn (let [other (mapv (partial join-fn (first col1)) col2)] - (concat acc other)))))) + (c/concat acc other)))))) (def sentinel #?(:clj (Object.) @@ -478,7 +487,7 @@ ([maybe-keyword default-value] (cond (keyword? maybe-keyword) - (core/name maybe-keyword) + (c/name maybe-keyword) (string? maybe-keyword) maybe-keyword @@ -496,7 +505,7 @@ [coll] (map vector coll - (concat (rest coll) [nil]))) + (c/concat (rest coll) [nil]))) (defn with-prev "Given a collection will return a new collection where each element @@ -505,7 +514,7 @@ [coll] (map vector coll - (concat [nil] coll))) + (c/concat [nil] coll))) (defn with-prev-next "Given a collection will return a new collection where every item is paired @@ -514,8 +523,8 @@ [coll] (map vector coll - (concat [nil] coll) - (concat (rest coll) [nil]))) + (c/concat [nil] coll) + (c/concat (rest coll) [nil]))) (defn prefix-keyword "Given a keyword and a prefix will return a new keyword with the prefix attached @@ -653,52 +662,28 @@ {} coll)))) -;; TEMPORAL COPY of clojure-1.11 iteration function, should be -;; replaced with the builtin on when stable version is released. +(defn iteration + "Creates a toally lazy seqable via repeated calls to step, a + function of some (continuation token) 'k'. The first call to step + will be passed initk, returning 'ret'. If (somef ret) is true, (vf + ret) will be included in the iteration, else iteration will + terminate and vf/kf will not be called. If (kf ret) is non-nil it + will be passed to the next step call, else iteration will terminate. -#?(:clj - (defn iteration - "Creates a seqable/reducible via repeated calls to step, - a function of some (continuation token) 'k'. The first call to step - will be passed initk, returning 'ret'. Iff (somef ret) is true, - (vf ret) will be included in the iteration, else iteration will - terminate and vf/kf will not be called. If (kf ret) is non-nil it - will be passed to the next step call, else iteration will terminate. - This can be used e.g. to consume APIs that return paginated or batched data. - step - (possibly impure) fn of 'k' -> 'ret' - :somef - fn of 'ret' -> logical true/false, default 'some?' - :vf - fn of 'ret' -> 'v', a value produced by the iteration, default 'identity' - :kf - fn of 'ret' -> 'next-k' or nil (signaling 'do not continue'), default 'identity' - :initk - the first value passed to step, default 'nil' - It is presumed that step with non-initk is unreproducible/non-idempotent. - If step with initk is unreproducible it is on the consumer to not consume twice." - {:added "1.11"} - [step & {:keys [somef vf kf initk] - :or {vf identity - kf identity - somef some? - initk nil}}] - (reify - clojure.lang.Seqable - (seq [_] - ((fn next [ret] - (when (somef ret) - (cons (vf ret) - (when-some [k (kf ret)] - (lazy-seq (next (step k))))))) - (step initk))) - clojure.lang.IReduceInit - (reduce [_ rf init] - (loop [acc init - ret (step initk)] - (if (somef ret) - (let [acc (rf acc (vf ret))] - (if (reduced? acc) - @acc - (if-some [k (kf ret)] - (recur acc (step k)) - acc))) - acc)))))) + This can be used e.g. to consume APIs that return paginated or batched data. + + step - (possibly impure) fn of 'k' -> 'ret' + :somef - fn of 'ret' -> logical true/false, default 'some?' + :vf - fn of 'ret' -> 'v', a value produced by the iteration, default 'identity' + :kf - fn of 'ret' -> 'next-k' or nil (signaling 'do not continue'), default 'identity' + :initk - the first value passed to step, default 'nil' + + It is presumed that step with non-initk is + unreproducible/non-idempotent. If step with initk is unreproducible + it is on the consumer to not consume twice." + [& args] + (->> (apply c/iteration args) + (concat-all))) (defn toggle-selection ([set value] diff --git a/frontend/deps.edn b/frontend/deps.edn index 6040a678e..0464f4daa 100644 --- a/frontend/deps.edn +++ b/frontend/deps.edn @@ -3,7 +3,7 @@ {penpot/common {:local/root "../common"} - org.clojure/clojure {:mvn/version "1.10.3"} + org.clojure/clojure {:mvn/version "1.11.1"} binaryage/devtools {:mvn/version "RELEASE"} metosin/reitit-core {:mvn/version "0.5.18"} From b7206d734ba074d97c8335ec661bfc9c6ed8ec7a Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 28 Sep 2022 23:27:06 +0200 Subject: [PATCH 2/2] :paperclip: Minor devenv update --- docker/devenv/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/devenv/Dockerfile b/docker/devenv/Dockerfile index 11bc43733..c40615d8b 100644 --- a/docker/devenv/Dockerfile +++ b/docker/devenv/Dockerfile @@ -4,7 +4,7 @@ LABEL maintainer="Andrey Antukh " ARG DEBIAN_FRONTEND=noninteractive ENV NODE_VERSION=v16.17.0 \ - CLOJURE_VERSION=1.11.1.1149 \ + CLOJURE_VERSION=1.11.1.1165 \ CLJKONDO_VERSION=2022.09.08 \ BABASHKA_VERSION=0.9.162 \ LANG=en_US.UTF-8 \ @@ -31,6 +31,7 @@ RUN set -ex; \ fakeroot \ netcat \ file \ + less \ ; \ echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen; \ locale-gen; \