Penpot/penpot
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-03-13 16:21:57 -05:00
Code Issues Activity

✨ Improve iteration and add concat-all and fully lazy mapcat helper

Browse source
This commit is contained in:
Andrey Antukh 2022-09-28 23:26:31 +02:00 • committed by Andrés Moya
parent 058727a44b
commit 886ab0e152
6 changed files with 68 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
backend/dev/user.clj
View file

@ -9,11 +9,13 @@
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.geom.matrix :as gmt]
[app.common.logging :as l]
[app.common.perf :as perf]
[app.common.pprint :as pp]
[app.common.transit :as t]
[app.config :as cfg]
[app.main :as main]
[app.srepl.main :as srepl]
[app.util.blob :as blob]
[app.util.fressian :as fres]
[app.util.json :as json]

7
backend/src/app/srepl/helpers.clj
View file

@ -89,8 +89,8 @@
The `on-file` parameter should be a function that receives the file
and the previous state and returns the new state."
[system & {:keys [chunk-size max-chunks start-at on-file on-error on-end]
:or {chunk-size 10 max-chunks Long/MAX_VALUE}}]
[system & {:keys [chunk-size max-items start-at on-file on-error on-end]
:or {chunk-size 10 max-items Long/MAX_VALUE}}]
(letfn [(get-chunk [conn cursor]
(let [rows (db/exec! conn [sql:retrieve-files-chunk cursor chunk-size])]
[(some->> rows peek :created-at) (seq rows)]))
@ -100,8 +100,7 @@
:vf second
:kf first
:initk (or start-at (dt/now)))
(take max-chunks)
(mapcat identity)
(take max-items)
(map #(update % :data blob/decode))))
(on-error* [file cause]

19
backend/src/app/storage.clj
View file

@ -284,11 +284,10 @@
(some->> (seq rows) (d/group-by #(-> % :backend keyword) :id #{}) seq)]))
(retrieve-deleted-objects [conn min-age]
(->> (d/iteration (partial retrieve-deleted-objects-chunk conn min-age)
:initk (dt/now)
:vf second
:kf first)
(sequence cat)))
(d/iteration (partial retrieve-deleted-objects-chunk conn min-age)
:initk (dt/now)
:vf second
:kf first))
(delete-in-bulk [conn backend-name ids]
(let [backend (impl/resolve-backend storage backend-name)
@ -397,12 +396,10 @@
(d/group-by get-bucket :id #{} rows)])))
(retrieve-touched [conn]
(->> (d/iteration (fn [cursor]
(retrieve-touched-chunk conn cursor))
:initk (dt/now)
:vf second
:kf first)
(sequence cat)))
(d/iteration (partial retrieve-touched-chunk conn)
:initk (dt/now)
:vf second
:kf first))
(process-objects! [conn get-fn ids bucket]
(loop [to-freeze #{}

9
backend/src/app/tasks/file_gc.clj
View file

@ -90,11 +90,10 @@
get-chunk (fn [cursor]
(let [rows (db/exec! conn [sql:retrieve-candidates-chunk interval cursor])]
[(some->> rows peek :modified-at) (seq rows)]))]
(sequence cat (d/iteration get-chunk
:vf second
:kf first
:initk (dt/now))))))
(d/iteration get-chunk
:vf second
:kf first
:initk (dt/now)))))
(defn collect-used-media
[data]

115
common/src/app/common/data.cljc
View file

@ -7,7 +7,7 @@
(ns app.common.data
"Data manipulation and query helper functions."
(:refer-clojure :exclude [read-string hash-map merge name update-vals
parse-double group-by iteration])
parse-double group-by iteration concat mapcat])
#?(:cljs
(:require-macros [app.common.data]))
@ -17,8 +17,8 @@
[cuerdas.core :as str]
#?(:cljs [cljs.reader :as r]
:clj [clojure.edn :as r])
#?(:cljs [cljs.core :as core]
:clj [clojure.core :as core])
#?(:cljs [cljs.core :as c]
:clj [clojure.core :as c])
[linked.set :as lks])
#?(:clj
@ -60,7 +60,7 @@
(defn editable-collection?
[m]
#?(:clj (instance? clojure.lang.IEditableCollection m)
:cljs (implements? core/IEditableCollection m)))
:cljs (implements? c/IEditableCollection m)))
(defn deep-merge
([a b]
@ -81,6 +81,24 @@
m)
(dissoc m k)))
(defn concat-all
"A totally lazy implementation of concat with different call
signature. It works like a flatten with a single level of neesting."
[colls]
(lazy-seq
(let [c (seq colls)
o (first c)
r (rest c)]
(if-let [o (seq o)]
(cons (first o) (concat-all (cons (rest o) r)))
(some-> (seq r) concat-all)))))
(defn mapcat
"A fully lazy version of mapcat."
([f] (c/mapcat f))
([f & colls]
(concat-all (apply map f colls))))
(defn- transient-concat
[c1 colls]
(loop [result (transient c1)
@ -214,21 +232,12 @@
([mfn coll]
(into {} (mapm mfn) coll)))
;; TEMPORARY COPY of clojure.core/update-vals until we migrate to clojure 1.11
(defn update-vals
"m f => {k (f v) ...}
Given a map m and a function f of 1-argument, returns a new map where the keys of m
are mapped to result of applying f to the corresponding values of m."
[m f]
(with-meta
(persistent!
(reduce-kv (fn [acc k v] (assoc! acc k (f v)))
(if (editable-collection? m)
(transient m)
(transient {}))
m))
(meta m)))
(c/update-vals m f))
(defn removev
"Returns a vector of the items in coll for which (fn item) returns logical false"
@ -294,7 +303,7 @@
(empty? col2) acc
:else (recur (rest col1) col2 join-fn
(let [other (mapv (partial join-fn (first col1)) col2)]
(concat acc other))))))
(c/concat acc other))))))
(def sentinel
#?(:clj (Object.)
@ -478,7 +487,7 @@
([maybe-keyword default-value]
(cond
(keyword? maybe-keyword)
(core/name maybe-keyword)
(c/name maybe-keyword)
(string? maybe-keyword)
maybe-keyword
@ -496,7 +505,7 @@
[coll]
(map vector
coll
(concat (rest coll) [nil])))
(c/concat (rest coll) [nil])))
(defn with-prev
"Given a collection will return a new collection where each element
@ -505,7 +514,7 @@
[coll]
(map vector
coll
(concat [nil] coll)))
(c/concat [nil] coll)))
(defn with-prev-next
"Given a collection will return a new collection where every item is paired
@ -514,8 +523,8 @@
[coll]
(map vector
coll
(concat [nil] coll)
(concat (rest coll) [nil])))
(c/concat [nil] coll)
(c/concat (rest coll) [nil])))
(defn prefix-keyword
"Given a keyword and a prefix will return a new keyword with the prefix attached
@ -653,52 +662,28 @@
{}
coll))))
;; TEMPORAL COPY of clojure-1.11 iteration function, should be
;; replaced with the builtin on when stable version is released.
(defn iteration
"Creates a toally lazy seqable via repeated calls to step, a
function of some (continuation token) 'k'. The first call to step
will be passed initk, returning 'ret'. If (somef ret) is true, (vf
ret) will be included in the iteration, else iteration will
terminate and vf/kf will not be called. If (kf ret) is non-nil it
will be passed to the next step call, else iteration will terminate.
#?(:clj
(defn iteration
"Creates a seqable/reducible via repeated calls to step,
a function of some (continuation token) 'k'. The first call to step
will be passed initk, returning 'ret'. Iff (somef ret) is true,
(vf ret) will be included in the iteration, else iteration will
terminate and vf/kf will not be called. If (kf ret) is non-nil it
will be passed to the next step call, else iteration will terminate.
This can be used e.g. to consume APIs that return paginated or batched data.
step - (possibly impure) fn of 'k' -> 'ret'
:somef - fn of 'ret' -> logical true/false, default 'some?'
:vf - fn of 'ret' -> 'v', a value produced by the iteration, default 'identity'
:kf - fn of 'ret' -> 'next-k' or nil (signaling 'do not continue'), default 'identity'
:initk - the first value passed to step, default 'nil'
It is presumed that step with non-initk is unreproducible/non-idempotent.
If step with initk is unreproducible it is on the consumer to not consume twice."
{:added "1.11"}
[step & {:keys [somef vf kf initk]
:or {vf identity
kf identity
somef some?
initk nil}}]
(reify
clojure.lang.Seqable
(seq [_]
((fn next [ret]
(when (somef ret)
(cons (vf ret)
(when-some [k (kf ret)]
(lazy-seq (next (step k)))))))
(step initk)))
clojure.lang.IReduceInit
(reduce [_ rf init]
(loop [acc init
ret (step initk)]
(if (somef ret)
(let [acc (rf acc (vf ret))]
(if (reduced? acc)
@acc
(if-some [k (kf ret)]
(recur acc (step k))
acc)))
acc))))))
This can be used e.g. to consume APIs that return paginated or batched data.
step - (possibly impure) fn of 'k' -> 'ret'
:somef - fn of 'ret' -> logical true/false, default 'some?'
:vf - fn of 'ret' -> 'v', a value produced by the iteration, default 'identity'
:kf - fn of 'ret' -> 'next-k' or nil (signaling 'do not continue'), default 'identity'
:initk - the first value passed to step, default 'nil'
It is presumed that step with non-initk is
unreproducible/non-idempotent. If step with initk is unreproducible
it is on the consumer to not consume twice."
[& args]
(->> (apply c/iteration args)
(concat-all)))
(defn toggle-selection
([set value]

2
frontend/deps.edn
View file

@ -3,7 +3,7 @@
{penpot/common
{:local/root "../common"}
org.clojure/clojure {:mvn/version "1.10.3"}
org.clojure/clojure {:mvn/version "1.11.1"}
binaryage/devtools {:mvn/version "RELEASE"}
metosin/reitit-core {:mvn/version "0.5.18"}