Penpot/penpot
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-02-13 02:28:18 -05:00
Code Issues Activity

Merge remote-tracking branch 'origin/main' into develop

Browse source
This commit is contained in:
Andrey Antukh 2022-02-16 11:23:26 +01:00
commit 7eed8c5ee5
17 changed files with 149 additions and 84 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
CHANGES.md
View file

@ -41,6 +41,24 @@
- Cleanup unused static images (by @rhcarvalho) [#1561](https://github.com/penpot/penpot/pull/1561)
- Compress static images to save space (by @rhcarvalho) [#1562](https://github.com/penpot/penpot/pull/1562)
## 1.11.2-beta
### :bug: Bugs fixed
- Fix issue on handling empty content on boolean shapes
- Fix race condition issue on component renaming
- Handle EOF errors on writting streamed response
- Handle EOF errors on websocket send/ping methods
- Disable parallel upload of file media on import (causes too much
contention on the rlimit subsistem that does not works as expected
on high load).
### :sparkles: New features
- Add health check endpoint on API
- Increase default max connection pool size to 60
- Reduce resource usage of the error reporter.
## 1.11.1-beta
### :bug: Bugs fixed

4
backend/src/app/http/errors.clj
View file

@ -10,7 +10,6 @@
[app.common.exceptions :as ex]
[app.common.logging :as l]
[app.common.spec :as us]
[app.common.uuid :as uuid]
[clojure.spec.alpha :as s]
[cuerdas.core :as str]))
@ -24,8 +23,7 @@
[request error]
(let [data (ex-data error)]
(merge
{:id (uuid/next)
:path (:uri request)
{:path (:uri request)
:method (:request-method request)
:hint (ex-message error)
:params (:params request)

20
backend/src/app/http/session.clj
View file

@ -40,8 +40,12 @@
token (tokens :generate {:iss "authentication"
:iat (dt/now)
:uid profile-id})
now (dt/now)
params {:user-agent user-agent
:profile-id profile-id
:created-at now
:updated-at now
:id token}]
(db/insert! pool :http-session params)
token))
@ -146,8 +150,7 @@
(defmethod ig/prep-key ::session
[_ cfg]
(d/merge {:buffer-size 64}
(d/without-nils cfg)))
(d/merge {:buffer-size 128} (d/without-nils cfg)))
(defmethod ig/init-key ::session
[_ {:keys [pool tokens] :as cfg}]
@ -222,7 +225,7 @@
(= :size reason)
(l/debug :task "updater"
:action "update sessions"
:hint "update sessions"
:reason (name reason)
:count result))
(recur))))))
@ -251,17 +254,20 @@
(defmethod ig/init-key ::gc-task
[_ {:keys [pool max-age] :as cfg}]
(l/debug :hint "initializing session gc task" :max-age max-age)
(fn [_]
(db/with-atomic [conn pool]
(let [interval (db/interval max-age)
result (db/exec-one! conn [sql:delete-expired interval])
result (db/exec-one! conn [sql:delete-expired interval interval])
result (:next.jdbc/update-count result)]
(l/debug :task "gc"
:action "clean http sessions"
:count result)
:hint "clean http sessions"
:deleted result)
result))))
(def ^:private
sql:delete-expired
"delete from http_session
where updated_at < now() - ?::interval")
where updated_at < now() - ?::interval
or (updated_at is null and
created_at < now() - ?::interval)")

16
backend/src/app/loggers/database.clj
View file

@ -29,9 +29,7 @@
(defn- persist-on-database!
[{:keys [pool] :as cfg} {:keys [id] :as event}]
(when-not (db/read-only? pool)
(db/with-atomic [conn pool]
(db/insert! conn :server-error-report
{:id id :content (db/tjson event)}))))
(db/insert! pool :server-error-report {:id id :content (db/tjson event)})))
(defn- parse-event-data
[event]
@ -52,7 +50,7 @@
(assoc :host (cf/get :host))
(assoc :public-uri (cf/get :public-uri))
(assoc :version (:full cf/version))
(update :id (fn [id] (or id (uuid/next))))))
(assoc :id (uuid/next))))
(defn handle-event
[{:keys [executor] :as cfg} event]
@ -60,12 +58,13 @@
(try
(let [event (parse-event event)
uri (cf/get :public-uri)]
(l/debug :hint "registering error on database" :id (:id event)
:uri (str uri "/dbg/error/" (:id event)))
(persist-on-database! cfg event))
(catch Exception e
(l/warn :hint "unexpected exception on database error logger"
:cause e)))))
(catch Exception cause
(l/warn :hint "unexpected exception on database error logger" :cause cause)))))
(defmethod ig/pre-init-spec ::reporter [_]
(s/keys :req-un [::wrk/executor ::db/pool ::receiver]))
@ -77,8 +76,7 @@
(defmethod ig/init-key ::reporter
[_ {:keys [receiver] :as cfg}]
(l/info :msg "initializing database error persistence")
(let [output (a/chan (a/sliding-buffer 5)
(filter error-event?))]
(let [output (a/chan (a/sliding-buffer 5) (filter error-event?))]
(receiver :sub output)
(a/go-loop []
(let [msg (a/<! output)]

4
backend/src/app/main.clj
View file

@ -177,7 +177,7 @@
:task :file-offload})
(when (contains? cf/flags :audit-log-archive)
{:cron #app/cron "0 */3 * * * ?" ;; every 3m
{:cron #app/cron "0 */5 * * * ?" ;; every 5m
:task :audit-log-archive})
(when (contains? cf/flags :audit-log-gc)
@ -186,7 +186,7 @@
(when (or (contains? cf/flags :telemetry)
(cf/get :telemetry-enabled))
{:cron #app/cron "0 0 */6 * * ?" ;; every 6h
{:cron #app/cron "0 30 */3,23 * * ?"
:task :telemetry})]}
:app.worker/registry

8
backend/src/app/storage.clj
View file

@ -305,7 +305,7 @@
(recur (+ n ^long total)))
(do
(l/info :task "gc-deleted"
:action "permanently delete items"
:hint "permanently delete items"
:count n)
{:deleted n})))))))
@ -379,10 +379,10 @@
(+ cntd (count to-delete))))
(do
(l/info :task "gc-touched"
:action "mark freeze"
:hint "mark freeze"
:count cntf)
(l/info :task "gc-touched"
:action "mark for deletion"
:hint "mark for deletion"
:count cntd)
{:freeze cntf :delete cntd})))))))
@ -461,7 +461,7 @@
(+ d (count to-delete))))
(do
(l/info :task "recheck"
:action "recheck items"
:hint "recheck items"
:processed n
:deleted d)
{:processed n :deleted d})))))))

10
backend/src/app/storage/s3.clj
View file

@ -33,6 +33,7 @@
software.amazon.awssdk.services.s3.model.GetObjectRequest
software.amazon.awssdk.services.s3.model.ObjectIdentifier
software.amazon.awssdk.services.s3.model.PutObjectRequest
software.amazon.awssdk.services.s3.model.S3Error
;; software.amazon.awssdk.services.s3.model.GetObjectResponse
software.amazon.awssdk.services.s3.presigner.S3Presigner
software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest
@ -231,6 +232,9 @@
^DeleteObjectsRequest dor)]
(when (.hasErrors ^DeleteObjectsResponse dres)
(let [errors (seq (.errors ^DeleteObjectsResponse dres))]
(ex/raise :type :s3-error
:code :error-on-bulk-delete
:context errors)))))
(ex/raise :type :internal
:code :error-on-s3-bulk-delete
:s3-errors (mapv (fn [^S3Error error]
{:key (.key error)
:msg (.message error)})
errors))))))

4
backend/src/app/tasks/file_media_gc.clj
View file

@ -106,7 +106,7 @@
unused (->> (db/query conn :file-media-object {:file-id id})
(remove #(contains? used (:id %))))]
(l/debug :action "processing file"
(l/debug :hint "processing file"
:id id
:age age
:to-delete (count unused))
@ -117,7 +117,7 @@
{:id id})
(doseq [mobj unused]
(l/debug :action "deleting media object"
(l/debug :hint "deleting media object"
:id (:id mobj)
:media-id (:media-id mobj)
:thumbnail-id (:thumbnail-id mobj))

2
backend/src/app/tasks/file_offload.clj
View file

@ -29,7 +29,7 @@
(defn- offload-candidate
[{:keys [storage conn backend] :as cfg} {:keys [id data] :as file}]
(l/debug :action "offload file data" :id id)
(l/debug :hint "offload file data" :id id)
(let [backend (simpl/resolve-backend storage backend)]
(->> (simpl/content data)
(simpl/put-object backend file))

3
backend/src/app/tasks/file_xlog_gc.clj
View file

@ -28,7 +28,8 @@
(let [interval (db/interval max-age)
result (db/exec-one! conn [sql:delete-files-xlog interval])
result (:next.jdbc/update-count result)]
(l/debug :removed result :hint "remove old file changes")
(l/info :hint "remove old file changes"
:removed result)
result))))
(def ^:private

10
backend/src/app/tasks/objects_gc.clj
View file

@ -48,7 +48,7 @@
result (db/exec! conn [sql max-age])]
(doseq [{:keys [id] :as item} result]
(l/trace :action "delete object" :table table :id id))
(l/trace :hint "delete object" :table table :id id))
(count result)))
@ -63,7 +63,7 @@
backend (simpl/resolve-backend storage (cf/get :fdata-storage-backend))]
(doseq [{:keys [id] :as item} result]
(l/trace :action "delete object" :table table :id id)
(l/trace :hint "delete object" :table table :id id)
(when backend
(simpl/del-object backend item)))
@ -78,7 +78,7 @@
fonts (db/exec! conn [sql max-age])
storage (assoc storage :conn conn)]
(doseq [{:keys [id] :as font} fonts]
(l/trace :action "delete object" :table table :id id)
(l/trace :hint "delete object" :table table :id id)
(some->> (:woff1-file-id font) (sto/del-object storage))
(some->> (:woff2-file-id font) (sto/del-object storage))
(some->> (:otf-file-id font) (sto/del-object storage))
@ -95,7 +95,7 @@
storage (assoc storage :conn conn)]
(doseq [{:keys [id] :as team} teams]
(l/trace :action "delete object" :table table :id id)
(l/trace :hint "delete object" :table table :id id)
(some->> (:photo-id team) (sto/del-object storage)))
(count teams)))
@ -127,7 +127,7 @@
storage (assoc storage :conn conn)]
(doseq [{:keys [id] :as profile} profiles]
(l/trace :action "delete object" :table table :id id)
(l/trace :hint "delete object" :table table :id id)
;; Mark the owned teams as deleted; this enables them to be processed
;; in the same transaction in the "team" table step.

2
backend/src/app/tasks/tasks_gc.clj
View file

@ -28,7 +28,7 @@
(let [interval (db/interval max-age)
result (db/exec-one! conn [sql:delete-completed-tasks interval])
result (:next.jdbc/update-count result)]
(l/debug :action "trim completed tasks table" :removed result)
(l/debug :hint "trim completed tasks table" :removed result)
result))))
(def ^:private

36
backend/src/app/tasks/telemetry.clj
View file

@ -14,12 +14,17 @@
[app.common.spec :as us]
[app.config :as cfg]
[app.db :as db]
[app.util.async :refer [thread-sleep]]
[app.util.http :as http]
[app.util.json :as json]
[clojure.spec.alpha :as s]
[integrant.core :as ig]))
(declare retrieve-stats)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; TASK ENTRY POINT
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(declare get-stats)
(declare send!)
(s/def ::version ::us/string)
@ -34,11 +39,18 @@
(defmethod ig/init-key ::handler
[_ {:keys [pool sprops version] :as cfg}]
(fn [_]
;; Sleep randomly between 0 to 10s
(thread-sleep (rand-int 10000))
(let [instance-id (:instance-id sprops)]
(-> (retrieve-stats pool version)
(-> (get-stats pool version)
(assoc :instance-id instance-id)
(send! cfg)))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; IMPL
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn- send!
[data cfg]
(let [response (http/send! {:method :post
@ -63,6 +75,20 @@
[conn]
(-> (db/exec-one! conn ["select count(*) as count from file;"]) :count))
(defn- retrieve-num-file-changes
[conn]
(let [sql (str "select count(*) as count "
" from file_change "
" where date_trunc('day', created_at) = date_trunc('day', now())")]
(-> (db/exec-one! conn [sql]) :count)))
(defn- retrieve-num-touched-files
[conn]
(let [sql (str "select count(distinct file_id) as count "
" from file_change "
" where date_trunc('day', created_at) = date_trunc('day', now())")]
(-> (db/exec-one! conn [sql]) :count)))
(defn- retrieve-num-users
[conn]
(-> (db/exec-one! conn ["select count(*) as count from profile;"]) :count))
@ -118,7 +144,7 @@
:jvm-heap-max (.maxMemory runtime)
:jvm-cpus (.availableProcessors runtime)}))
(defn retrieve-stats
(defn get-stats
[conn version]
(let [referer (if (cfg/get :telemetry-with-taiga)
"taiga"
@ -130,7 +156,9 @@
:total-files (retrieve-num-files conn)
:total-users (retrieve-num-users conn)
:total-fonts (retrieve-num-fonts conn)
:total-comments (retrieve-num-comments conn)}
:total-comments (retrieve-num-comments conn)
:total-file-changes (retrieve-num-file-changes conn)
:total-touched-files (retrieve-num-touched-files conn)}
(d/merge
(retrieve-team-averages conn)
(retrieve-jvm-stats))

10
backend/src/app/util/time.clj
View file

@ -295,6 +295,11 @@
(s/assert cron? cron)
(.toInstant (.getNextValidTimeAfter cron (Date/from now))))
(defn get-next
[cron tnow]
(let [nt (next-valid-instant-from cron tnow)]
(cons nt (lazy-seq (get-next cron nt)))))
(defmethod print-method CronExpression
[mv ^java.io.Writer writer]
(.write writer (str "#app/cron \"" (.toString ^CronExpression mv) "\"")))
@ -302,3 +307,8 @@
(defmethod print-dup CronExpression
[o w]
(print-ctor o (fn [o w] (print-dup (.toString ^CronExpression o) w)) w))
(extend-protocol fez/IEdn
CronExpression
(-edn [o] (pr-str o)))

33
backend/src/app/worker.clj
View file

@ -106,7 +106,7 @@
(and (instance? java.sql.SQLException val)
(contains? #{"08003" "08006" "08001" "08004"} (.getSQLState ^java.sql.SQLException val)))
(do
(l/error :hint "connection error, trying resume in some instants")
(l/warn :hint "connection error, trying resume in some instants")
(a/<! (a/timeout poll-interval))
(recur))
@ -119,8 +119,8 @@
(instance? Exception val)
(do
(l/error :cause val
:hint "unexpected error ocurried on polling the database (will resume in some instants)")
(l/warn :cause val
:hint "unexpected error ocurried on polling the database (will resume in some instants)")
(a/<! (a/timeout poll-ms))
(recur))
@ -262,8 +262,7 @@
[error item]
(let [data (ex-data error)]
(merge
{:id (uuid/next)
:hint (ex-message error)
{:hint (ex-message error)
:spec-problems (some->> data ::s/problems (take 10) seq vec)
:spec-value (some->> data ::s/value)
:data (some-> data (dissoc ::s/problems ::s/value ::s/spec))
@ -429,21 +428,19 @@
(defn- execute-scheduled-task
[{:keys [executor pool] :as cfg} {:keys [id] :as task}]
(letfn [(run-task [conn]
(try
(when (db/exec-one! conn [sql:lock-scheduled-task (d/name id)])
(l/debug :action "execute scheduled task" :id id)
((:fn task) task))
(catch Throwable e
e)))
(when (db/exec-one! conn [sql:lock-scheduled-task (d/name id)])
(l/debug :action "execute scheduled task" :id id)
((:fn task) task)))
(handle-task []
(db/with-atomic [conn pool]
(let [result (run-task conn)]
(when (ex/exception? result)
(l/error :cause result
:hint "unhandled exception on scheduled task"
:id id)))))]
(try
(db/with-atomic [conn pool]
(run-task conn))
(catch Throwable cause
(l/error :hint "unhandled exception on scheduled task"
::l/context (get-error-context cause task)
:task-id id
:cause cause))))]
(try
(px/run! executor handle-task)
(finally

9
common/src/app/common/path/bool.cljc
View file

@ -326,6 +326,9 @@
[bool-type contents]
;; We apply the boolean operation in to each pair and the result to the next
;; element
(->> contents
(reduce (partial content-bool-pair bool-type))
(into [])))
(if (seq contents)
(->> contents
(reduce (partial content-bool-pair bool-type))
(into []))
[]))

44
frontend/src/app/main/data/workspace/libraries.cljs
View file

@ -317,29 +317,31 @@
(ptk/reify ::rename-component
ptk/WatchEvent
(watch [it state _]
(let [[path name] (cph/parse-path-name new-name)
component (get-in state [:workspace-data :components id])
objects (get component :objects)
; Give the same name to the root shape
new-objects (assoc-in objects
[(:id component) :name]
name)
;; NOTE: we need to ensure the component exists, because there
;; are small posibilities of race conditions with component
;; deletion.
(when-let [component (get-in state [:workspace-data :components id])]
(let [[path name] (cp/parse-path-name new-name)
objects (get component :objects)
;; Give the same name to the root shape
new-objects (assoc-in objects
[(:id component) :name]
name)
rchanges [{:type :mod-component
:id id
:name name
:path path
:objects new-objects}]
rchanges [{:type :mod-component
:id id
:name name
:path path
:objects new-objects}]
uchanges [{:type :mod-component
:id id
:name (:name component)
:path (:path component)
:objects objects}]]
(rx/of (dch/commit-changes {:redo-changes rchanges
:undo-changes uchanges
:origin it}))))))
uchanges [{:type :mod-component
:id id
:name (:name component)
:path (:path component)
:objects objects}]]
(rx/of (dch/commit-changes {:redo-changes rchanges
:undo-changes uchanges
:origin it})))))))
(defn duplicate-component
"Create a new component copied from the one with the given id."