diff --git a/CHANGES.md b/CHANGES.md index f6dfb0227..81d16cf7d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -41,6 +41,24 @@ - Cleanup unused static images (by @rhcarvalho) [#1561](https://github.com/penpot/penpot/pull/1561) - Compress static images to save space (by @rhcarvalho) [#1562](https://github.com/penpot/penpot/pull/1562) +## 1.11.2-beta + +### :bug: Bugs fixed + +- Fix issue on handling empty content on boolean shapes +- Fix race condition issue on component renaming +- Handle EOF errors on writting streamed response +- Handle EOF errors on websocket send/ping methods +- Disable parallel upload of file media on import (causes too much + contention on the rlimit subsistem that does not works as expected + on high load). + +### :sparkles: New features + +- Add health check endpoint on API +- Increase default max connection pool size to 60 +- Reduce resource usage of the error reporter. + ## 1.11.1-beta ### :bug: Bugs fixed diff --git a/backend/src/app/http/errors.clj b/backend/src/app/http/errors.clj index a53637849..0c24a3df7 100644 --- a/backend/src/app/http/errors.clj +++ b/backend/src/app/http/errors.clj @@ -10,7 +10,6 @@ [app.common.exceptions :as ex] [app.common.logging :as l] [app.common.spec :as us] - [app.common.uuid :as uuid] [clojure.spec.alpha :as s] [cuerdas.core :as str])) @@ -24,8 +23,7 @@ [request error] (let [data (ex-data error)] (merge - {:id (uuid/next) - :path (:uri request) + {:path (:uri request) :method (:request-method request) :hint (ex-message error) :params (:params request) diff --git a/backend/src/app/http/session.clj b/backend/src/app/http/session.clj index 03bfd4a44..002b1ab36 100644 --- a/backend/src/app/http/session.clj +++ b/backend/src/app/http/session.clj @@ -40,8 +40,12 @@ token (tokens :generate {:iss "authentication" :iat (dt/now) :uid profile-id}) + + now (dt/now) params {:user-agent user-agent :profile-id profile-id + :created-at now + :updated-at now :id token}] (db/insert! pool :http-session params) token)) @@ -146,8 +150,7 @@ (defmethod ig/prep-key ::session [_ cfg] - (d/merge {:buffer-size 64} - (d/without-nils cfg))) + (d/merge {:buffer-size 128} (d/without-nils cfg))) (defmethod ig/init-key ::session [_ {:keys [pool tokens] :as cfg}] @@ -222,7 +225,7 @@ (= :size reason) (l/debug :task "updater" - :action "update sessions" + :hint "update sessions" :reason (name reason) :count result)) (recur)))))) @@ -251,17 +254,20 @@ (defmethod ig/init-key ::gc-task [_ {:keys [pool max-age] :as cfg}] + (l/debug :hint "initializing session gc task" :max-age max-age) (fn [_] (db/with-atomic [conn pool] (let [interval (db/interval max-age) - result (db/exec-one! conn [sql:delete-expired interval]) + result (db/exec-one! conn [sql:delete-expired interval interval]) result (:next.jdbc/update-count result)] (l/debug :task "gc" - :action "clean http sessions" - :count result) + :hint "clean http sessions" + :deleted result) result)))) (def ^:private sql:delete-expired "delete from http_session - where updated_at < now() - ?::interval") + where updated_at < now() - ?::interval + or (updated_at is null and + created_at < now() - ?::interval)") diff --git a/backend/src/app/loggers/database.clj b/backend/src/app/loggers/database.clj index 22389d3a2..748fca12e 100644 --- a/backend/src/app/loggers/database.clj +++ b/backend/src/app/loggers/database.clj @@ -29,9 +29,7 @@ (defn- persist-on-database! [{:keys [pool] :as cfg} {:keys [id] :as event}] (when-not (db/read-only? pool) - (db/with-atomic [conn pool] - (db/insert! conn :server-error-report - {:id id :content (db/tjson event)})))) + (db/insert! pool :server-error-report {:id id :content (db/tjson event)}))) (defn- parse-event-data [event] @@ -52,7 +50,7 @@ (assoc :host (cf/get :host)) (assoc :public-uri (cf/get :public-uri)) (assoc :version (:full cf/version)) - (update :id (fn [id] (or id (uuid/next)))))) + (assoc :id (uuid/next)))) (defn handle-event [{:keys [executor] :as cfg} event] @@ -60,12 +58,13 @@ (try (let [event (parse-event event) uri (cf/get :public-uri)] + (l/debug :hint "registering error on database" :id (:id event) :uri (str uri "/dbg/error/" (:id event))) + (persist-on-database! cfg event)) - (catch Exception e - (l/warn :hint "unexpected exception on database error logger" - :cause e))))) + (catch Exception cause + (l/warn :hint "unexpected exception on database error logger" :cause cause))))) (defmethod ig/pre-init-spec ::reporter [_] (s/keys :req-un [::wrk/executor ::db/pool ::receiver])) @@ -77,8 +76,7 @@ (defmethod ig/init-key ::reporter [_ {:keys [receiver] :as cfg}] (l/info :msg "initializing database error persistence") - (let [output (a/chan (a/sliding-buffer 5) - (filter error-event?))] + (let [output (a/chan (a/sliding-buffer 5) (filter error-event?))] (receiver :sub output) (a/go-loop [] (let [msg (a/> (db/query conn :file-media-object {:file-id id}) (remove #(contains? used (:id %))))] - (l/debug :action "processing file" + (l/debug :hint "processing file" :id id :age age :to-delete (count unused)) @@ -117,7 +117,7 @@ {:id id}) (doseq [mobj unused] - (l/debug :action "deleting media object" + (l/debug :hint "deleting media object" :id (:id mobj) :media-id (:media-id mobj) :thumbnail-id (:thumbnail-id mobj)) diff --git a/backend/src/app/tasks/file_offload.clj b/backend/src/app/tasks/file_offload.clj index a43afb8a9..e429d3872 100644 --- a/backend/src/app/tasks/file_offload.clj +++ b/backend/src/app/tasks/file_offload.clj @@ -29,7 +29,7 @@ (defn- offload-candidate [{:keys [storage conn backend] :as cfg} {:keys [id data] :as file}] - (l/debug :action "offload file data" :id id) + (l/debug :hint "offload file data" :id id) (let [backend (simpl/resolve-backend storage backend)] (->> (simpl/content data) (simpl/put-object backend file)) diff --git a/backend/src/app/tasks/file_xlog_gc.clj b/backend/src/app/tasks/file_xlog_gc.clj index b8dce2fa5..7b4e21ad5 100644 --- a/backend/src/app/tasks/file_xlog_gc.clj +++ b/backend/src/app/tasks/file_xlog_gc.clj @@ -28,7 +28,8 @@ (let [interval (db/interval max-age) result (db/exec-one! conn [sql:delete-files-xlog interval]) result (:next.jdbc/update-count result)] - (l/debug :removed result :hint "remove old file changes") + (l/info :hint "remove old file changes" + :removed result) result)))) (def ^:private diff --git a/backend/src/app/tasks/objects_gc.clj b/backend/src/app/tasks/objects_gc.clj index 9d2e44146..dc66a9aff 100644 --- a/backend/src/app/tasks/objects_gc.clj +++ b/backend/src/app/tasks/objects_gc.clj @@ -48,7 +48,7 @@ result (db/exec! conn [sql max-age])] (doseq [{:keys [id] :as item} result] - (l/trace :action "delete object" :table table :id id)) + (l/trace :hint "delete object" :table table :id id)) (count result))) @@ -63,7 +63,7 @@ backend (simpl/resolve-backend storage (cf/get :fdata-storage-backend))] (doseq [{:keys [id] :as item} result] - (l/trace :action "delete object" :table table :id id) + (l/trace :hint "delete object" :table table :id id) (when backend (simpl/del-object backend item))) @@ -78,7 +78,7 @@ fonts (db/exec! conn [sql max-age]) storage (assoc storage :conn conn)] (doseq [{:keys [id] :as font} fonts] - (l/trace :action "delete object" :table table :id id) + (l/trace :hint "delete object" :table table :id id) (some->> (:woff1-file-id font) (sto/del-object storage)) (some->> (:woff2-file-id font) (sto/del-object storage)) (some->> (:otf-file-id font) (sto/del-object storage)) @@ -95,7 +95,7 @@ storage (assoc storage :conn conn)] (doseq [{:keys [id] :as team} teams] - (l/trace :action "delete object" :table table :id id) + (l/trace :hint "delete object" :table table :id id) (some->> (:photo-id team) (sto/del-object storage))) (count teams))) @@ -127,7 +127,7 @@ storage (assoc storage :conn conn)] (doseq [{:keys [id] :as profile} profiles] - (l/trace :action "delete object" :table table :id id) + (l/trace :hint "delete object" :table table :id id) ;; Mark the owned teams as deleted; this enables them to be processed ;; in the same transaction in the "team" table step. diff --git a/backend/src/app/tasks/tasks_gc.clj b/backend/src/app/tasks/tasks_gc.clj index 788e29269..1350c4abf 100644 --- a/backend/src/app/tasks/tasks_gc.clj +++ b/backend/src/app/tasks/tasks_gc.clj @@ -28,7 +28,7 @@ (let [interval (db/interval max-age) result (db/exec-one! conn [sql:delete-completed-tasks interval]) result (:next.jdbc/update-count result)] - (l/debug :action "trim completed tasks table" :removed result) + (l/debug :hint "trim completed tasks table" :removed result) result)))) (def ^:private diff --git a/backend/src/app/tasks/telemetry.clj b/backend/src/app/tasks/telemetry.clj index 7f24fe7c0..812e36f97 100644 --- a/backend/src/app/tasks/telemetry.clj +++ b/backend/src/app/tasks/telemetry.clj @@ -14,12 +14,17 @@ [app.common.spec :as us] [app.config :as cfg] [app.db :as db] + [app.util.async :refer [thread-sleep]] [app.util.http :as http] [app.util.json :as json] [clojure.spec.alpha :as s] [integrant.core :as ig])) -(declare retrieve-stats) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; TASK ENTRY POINT +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(declare get-stats) (declare send!) (s/def ::version ::us/string) @@ -34,11 +39,18 @@ (defmethod ig/init-key ::handler [_ {:keys [pool sprops version] :as cfg}] (fn [_] + ;; Sleep randomly between 0 to 10s + (thread-sleep (rand-int 10000)) + (let [instance-id (:instance-id sprops)] - (-> (retrieve-stats pool version) + (-> (get-stats pool version) (assoc :instance-id instance-id) (send! cfg))))) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; IMPL +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + (defn- send! [data cfg] (let [response (http/send! {:method :post @@ -63,6 +75,20 @@ [conn] (-> (db/exec-one! conn ["select count(*) as count from file;"]) :count)) +(defn- retrieve-num-file-changes + [conn] + (let [sql (str "select count(*) as count " + " from file_change " + " where date_trunc('day', created_at) = date_trunc('day', now())")] + (-> (db/exec-one! conn [sql]) :count))) + +(defn- retrieve-num-touched-files + [conn] + (let [sql (str "select count(distinct file_id) as count " + " from file_change " + " where date_trunc('day', created_at) = date_trunc('day', now())")] + (-> (db/exec-one! conn [sql]) :count))) + (defn- retrieve-num-users [conn] (-> (db/exec-one! conn ["select count(*) as count from profile;"]) :count)) @@ -118,7 +144,7 @@ :jvm-heap-max (.maxMemory runtime) :jvm-cpus (.availableProcessors runtime)})) -(defn retrieve-stats +(defn get-stats [conn version] (let [referer (if (cfg/get :telemetry-with-taiga) "taiga" @@ -130,7 +156,9 @@ :total-files (retrieve-num-files conn) :total-users (retrieve-num-users conn) :total-fonts (retrieve-num-fonts conn) - :total-comments (retrieve-num-comments conn)} + :total-comments (retrieve-num-comments conn) + :total-file-changes (retrieve-num-file-changes conn) + :total-touched-files (retrieve-num-touched-files conn)} (d/merge (retrieve-team-averages conn) (retrieve-jvm-stats)) diff --git a/backend/src/app/util/time.clj b/backend/src/app/util/time.clj index 42d6ca64b..7adeabf54 100644 --- a/backend/src/app/util/time.clj +++ b/backend/src/app/util/time.clj @@ -295,6 +295,11 @@ (s/assert cron? cron) (.toInstant (.getNextValidTimeAfter cron (Date/from now)))) +(defn get-next + [cron tnow] + (let [nt (next-valid-instant-from cron tnow)] + (cons nt (lazy-seq (get-next cron nt))))) + (defmethod print-method CronExpression [mv ^java.io.Writer writer] (.write writer (str "#app/cron \"" (.toString ^CronExpression mv) "\""))) @@ -302,3 +307,8 @@ (defmethod print-dup CronExpression [o w] (print-ctor o (fn [o w] (print-dup (.toString ^CronExpression o) w)) w)) + +(extend-protocol fez/IEdn + CronExpression + (-edn [o] (pr-str o))) + diff --git a/backend/src/app/worker.clj b/backend/src/app/worker.clj index 3fa2e0721..c24ee88d4 100644 --- a/backend/src/app/worker.clj +++ b/backend/src/app/worker.clj @@ -106,7 +106,7 @@ (and (instance? java.sql.SQLException val) (contains? #{"08003" "08006" "08001" "08004"} (.getSQLState ^java.sql.SQLException val))) (do - (l/error :hint "connection error, trying resume in some instants") + (l/warn :hint "connection error, trying resume in some instants") (a/> data ::s/problems (take 10) seq vec) :spec-value (some->> data ::s/value) :data (some-> data (dissoc ::s/problems ::s/value ::s/spec)) @@ -429,21 +428,19 @@ (defn- execute-scheduled-task [{:keys [executor pool] :as cfg} {:keys [id] :as task}] (letfn [(run-task [conn] - (try - (when (db/exec-one! conn [sql:lock-scheduled-task (d/name id)]) - (l/debug :action "execute scheduled task" :id id) - ((:fn task) task)) - (catch Throwable e - e))) + (when (db/exec-one! conn [sql:lock-scheduled-task (d/name id)]) + (l/debug :action "execute scheduled task" :id id) + ((:fn task) task))) (handle-task [] - (db/with-atomic [conn pool] - (let [result (run-task conn)] - (when (ex/exception? result) - (l/error :cause result - :hint "unhandled exception on scheduled task" - :id id)))))] - + (try + (db/with-atomic [conn pool] + (run-task conn)) + (catch Throwable cause + (l/error :hint "unhandled exception on scheduled task" + ::l/context (get-error-context cause task) + :task-id id + :cause cause))))] (try (px/run! executor handle-task) (finally diff --git a/common/src/app/common/path/bool.cljc b/common/src/app/common/path/bool.cljc index 8b6a66cf5..543a60b13 100644 --- a/common/src/app/common/path/bool.cljc +++ b/common/src/app/common/path/bool.cljc @@ -326,6 +326,9 @@ [bool-type contents] ;; We apply the boolean operation in to each pair and the result to the next ;; element - (->> contents - (reduce (partial content-bool-pair bool-type)) - (into []))) + (if (seq contents) + (->> contents + (reduce (partial content-bool-pair bool-type)) + (into [])) + [])) + diff --git a/frontend/src/app/main/data/workspace/libraries.cljs b/frontend/src/app/main/data/workspace/libraries.cljs index e93a03983..2408406f9 100644 --- a/frontend/src/app/main/data/workspace/libraries.cljs +++ b/frontend/src/app/main/data/workspace/libraries.cljs @@ -317,29 +317,31 @@ (ptk/reify ::rename-component ptk/WatchEvent (watch [it state _] - (let [[path name] (cph/parse-path-name new-name) - component (get-in state [:workspace-data :components id]) - objects (get component :objects) - ; Give the same name to the root shape - new-objects (assoc-in objects - [(:id component) :name] - name) + ;; NOTE: we need to ensure the component exists, because there + ;; are small posibilities of race conditions with component + ;; deletion. + (when-let [component (get-in state [:workspace-data :components id])] + (let [[path name] (cp/parse-path-name new-name) + objects (get component :objects) + ;; Give the same name to the root shape + new-objects (assoc-in objects + [(:id component) :name] + name) - rchanges [{:type :mod-component - :id id - :name name - :path path - :objects new-objects}] + rchanges [{:type :mod-component + :id id + :name name + :path path + :objects new-objects}] - uchanges [{:type :mod-component - :id id - :name (:name component) - :path (:path component) - :objects objects}]] - - (rx/of (dch/commit-changes {:redo-changes rchanges - :undo-changes uchanges - :origin it})))))) + uchanges [{:type :mod-component + :id id + :name (:name component) + :path (:path component) + :objects objects}]] + (rx/of (dch/commit-changes {:redo-changes rchanges + :undo-changes uchanges + :origin it}))))))) (defn duplicate-component "Create a new component copied from the one with the given id."