From e4fb802d7a4b75d105f1cc535fa04ecbc5824697 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 9 Feb 2021 14:24:46 +0100 Subject: [PATCH 1/6] :sparkles: Minor improvement on telemetry server error reporting. --- backend/src/app/config.clj | 1 - backend/src/app/tasks/telemetry.clj | 3 ++- backend/src/app/telemetry.clj | 8 +++++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index 513358204..6910d5a45 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -52,7 +52,6 @@ :registration-domain-whitelist "" :telemetry-enabled false - :telemetry-with-taiga true :telemetry-uri "https://telemetry.penpot.app/" ;; LDAP auth disabled by default. Set ldap-auth-host to enable diff --git a/backend/src/app/tasks/telemetry.clj b/backend/src/app/tasks/telemetry.clj index bd5f5a3b4..64ca03c8e 100644 --- a/backend/src/app/tasks/telemetry.clj +++ b/backend/src/app/tasks/telemetry.clj @@ -63,6 +63,7 @@ :uri (:uri cfg) :headers {"content-type" "application/json"} :body (json/encode-str data)})] + (when (not= 200 (:status response)) (ex/raise :type :internal :code :invalid-response-from-google @@ -129,7 +130,7 @@ [{:keys [conn version]}] (merge {:version version - :with-taiga (:telemetry-with-taiga cfg/config) + :with-taiga (:telemetry-with-taiga cfg/config false) :total-teams (retrieve-num-teams conn) :total-projects (retrieve-num-projects conn) :total-files (retrieve-num-files conn)} diff --git a/backend/src/app/telemetry.clj b/backend/src/app/telemetry.clj index b97c79360..a8e5edae7 100644 --- a/backend/src/app/telemetry.clj +++ b/backend/src/app/telemetry.clj @@ -12,6 +12,7 @@ [app.common.spec :as us] [app.db :as db] [app.http.middleware :refer [wrap-parse-request-body]] + [clojure.pprint :refer [pprint]] [clojure.spec.alpha :as s] [clojure.tools.logging :as log] [integrant.core :as ig] @@ -87,7 +88,12 @@ (catch Exception e ;; We don't want notify user of a error, just log it for posible ;; future investigation. - (log/warnf e "Unexpected error on telemetry."))) + (log/warn e (str "Unexpected error on telemetry:\n" + (when-let [edata (ex-data e)] + (str "ex-data: \n" + (with-out-str (pprint edata)))) + (str "params: \n" + (with-out-str (pprint params))))))) {:status 200 :body "OK\n"}) From bedfb9a1ee71d980003dbcd3b2fa229263012d62 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 9 Feb 2021 14:30:31 +0100 Subject: [PATCH 2/6] :sparkles: Increment default statement timeout. --- backend/src/app/db.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/app/db.clj b/backend/src/app/db.clj index 12bbbcf0d..1fcb18e47 100644 --- a/backend/src/app/db.clj +++ b/backend/src/app/db.clj @@ -72,7 +72,7 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (def initsql - (str "SET statement_timeout = 10000;\n" + (str "SET statement_timeout = 60000;\n" "SET idle_in_transaction_session_timeout = 120000;")) (defn- create-datasource-config From 69b23e4000032671a19dcb49ac6e0761572ca858 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 9 Feb 2021 16:15:19 +0100 Subject: [PATCH 3/6] :sparkles: Change background tasks schedule. --- backend/src/app/main.clj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index 02cc651b4..ae2ece00e 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -192,19 +192,19 @@ :fn (ig/ref :app.tasks.file-media-gc/handler)} {:id "file-xlog-gc" - :cron #app/cron "0 0 */6 * * ?" ;; every 2 hours + :cron #app/cron "0 0 0 */1 * ?" ;; daily :fn (ig/ref :app.tasks.file-xlog-gc/handler)} {:id "storage-deleted-gc" - :cron #app/cron "0 0 */6 * * ?" ;; every 6 hours + :cron #app/cron "0 0 1 */1 * ?" ;; daily (1 hour shift) :fn (ig/ref :app.storage/gc-deleted-task)} {:id "storage-touched-gc" - :cron #app/cron "0 30 */6 * * ?" ;; every 6 hours + :cron #app/cron "0 0 2 */1 * ?" ;; daily (2 hour shift) :fn (ig/ref :app.storage/gc-touched-task)} {:id "storage-recheck" - :cron #app/cron "0 0 */6 * * ?" ;; every 6 hours + :cron #app/cron "0 0 */2 * * ?" ;; every 2 hours :fn (ig/ref :app.storage/recheck-task)} {:id "tasks-gc" From f60ad9e5596d0d369190f1a3438e952566eb9254 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 9 Feb 2021 17:08:15 +0100 Subject: [PATCH 4/6] :bug: Fix unexpected 404 error on access shared link. --- frontend/src/app/main/data/viewer.cljs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/main/data/viewer.cljs b/frontend/src/app/main/data/viewer.cljs index 96d3f1bdd..550b5ddb9 100644 --- a/frontend/src/app/main/data/viewer.cljs +++ b/frontend/src/app/main/data/viewer.cljs @@ -137,7 +137,8 @@ (d/index-by :id) (assoc state :comment-threads))) (on-error [{:keys [type] :as err}] - (if (= :authentication type) + (if (or (= :authentication type) + (= :not-found type)) (rx/empty) (rx/throw err)))] From e5cb6ebec735efd88fa4ba1fdf015d1edad38e1a Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 9 Feb 2021 18:55:31 +0100 Subject: [PATCH 5/6] :sparkles: More improvements on background task scheduling. --- backend/src/app/main.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index ae2ece00e..e26d84a92 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -192,7 +192,7 @@ :fn (ig/ref :app.tasks.file-media-gc/handler)} {:id "file-xlog-gc" - :cron #app/cron "0 0 0 */1 * ?" ;; daily + :cron #app/cron "0 0 */1 * * ?" ;; hourly :fn (ig/ref :app.tasks.file-xlog-gc/handler)} {:id "storage-deleted-gc" @@ -204,7 +204,7 @@ :fn (ig/ref :app.storage/gc-touched-task)} {:id "storage-recheck" - :cron #app/cron "0 0 */2 * * ?" ;; every 2 hours + :cron #app/cron "0 0 */1 * * ?" ;; hourly :fn (ig/ref :app.storage/recheck-task)} {:id "tasks-gc" @@ -260,7 +260,7 @@ :app.tasks.file-xlog-gc/handler {:pool (ig/ref :app.db/pool) :metrics (ig/ref :app.metrics/metrics) - :max-age (dt/duration {:hours 24})} + :max-age (dt/duration {:hours 48})} :app.tasks.telemetry/handler {:pool (ig/ref :app.db/pool) From 6f3a08be0c07d31f24573182d17fefe45431dcd4 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 10 Feb 2021 11:46:17 +0100 Subject: [PATCH 6/6] :bug: Remove file lock contention on media upload. --- backend/src/app/rpc/mutations/media.clj | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/backend/src/app/rpc/mutations/media.clj b/backend/src/app/rpc/mutations/media.clj index b6b115d7c..0cc6b967e 100644 --- a/backend/src/app/rpc/mutations/media.clj +++ b/backend/src/app/rpc/mutations/media.clj @@ -38,7 +38,7 @@ ;; --- Create File Media object (upload) (declare create-file-media-object) -(declare select-file-for-update) +(declare select-file) (s/def ::content ::media/upload) (s/def ::is-local ::us/boolean) @@ -50,7 +50,7 @@ (sv/defmethod ::upload-file-media-object [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}] (db/with-atomic [conn pool] - (let [file (select-file-for-update conn file-id)] + (let [file (select-file conn file-id)] (teams/check-edition-permissions! conn profile-id (:team-id file)) (-> (assoc cfg :conn conn) (create-file-media-object params))))) @@ -129,7 +129,7 @@ (sv/defmethod ::create-file-media-object-from-url [{:keys [pool storage] :as cfg} {:keys [profile-id file-id url name] :as params}] (db/with-atomic [conn pool] - (let [file (select-file-for-update conn file-id)] + (let [file (select-file conn file-id)] (teams/check-edition-permissions! conn profile-id (:team-id file)) (let [mobj (download-media cfg url) content {:filename "tempfile" @@ -152,7 +152,7 @@ (sv/defmethod ::clone-file-media-object [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}] (db/with-atomic [conn pool] - (let [file (select-file-for-update conn file-id)] + (let [file (select-file conn file-id)] (teams/check-edition-permissions! conn profile-id (:team-id file)) (-> (assoc cfg :conn conn) @@ -175,17 +175,17 @@ ;; --- HELPERS -(def ^:private sql:select-file-for-update +(def ^:private + sql:select-file "select file.*, project.team_id as team_id from file inner join project on (project.id = file.project_id) - where file.id = ? - for update of file") + where file.id = ?") -(defn- select-file-for-update +(defn- select-file [conn id] - (let [row (db/exec-one! conn [sql:select-file-for-update id])] + (let [row (db/exec-one! conn [sql:select-file id])] (when-not row (ex/raise :type :not-found)) row))