From 827c2140b761fd91144f41f2ad52ae17d305a30b Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 9 Feb 2022 17:45:45 +0100 Subject: [PATCH 1/9] :recycle: Refactor error reporting and logging context formatting The prev approach uses clojure.pprint without any limit extensivelly for format error context data and all this is done on the calling thread. The clojure.pprint seems very inneficient in cpu and memory usage on pprinting large data structures. This is improved in the following way: - All formatting and pretty printing is moved to logging thread, reducing unnecesary blocking and load on jetty http threads. - Replace the clojure.pprint with fipp.edn that looks considerably faster than the first one. - Add some safe limits on pretty printer for strip printing some data when the data structure is very large, very deep or both. --- backend/src/app/cli/manage.clj | 1 - backend/src/app/http.clj | 10 ++++---- backend/src/app/http/debug.clj | 31 +++++++++++----------- backend/src/app/http/errors.clj | 41 +++++++++++------------------- backend/src/app/loggers/sentry.clj | 2 -- backend/src/app/srepl/main.clj | 2 +- backend/src/app/util/time.clj | 12 ++++++++- backend/src/app/worker.clj | 20 ++++++++++----- common/src/app/common/logging.cljc | 27 ++++++++------------ common/src/app/common/spec.cljc | 13 +++++++++- 10 files changed, 84 insertions(+), 75 deletions(-) diff --git a/backend/src/app/cli/manage.clj b/backend/src/app/cli/manage.clj index d20758196..dad68fc6b 100644 --- a/backend/src/app/cli/manage.clj +++ b/backend/src/app/cli/manage.clj @@ -140,7 +140,6 @@ indicating the action the program should take and the options provided." [args] (let [{:keys [options arguments errors summary] :as opts} (parse-opts args cli-options)] - ;; (pp/pprint opts) (cond (:help options) ; help => exit OK with usage summary {:exit-message (usage summary) :ok? true} diff --git a/backend/src/app/http.clj b/backend/src/app/http.clj index d55028199..d053786bb 100644 --- a/backend/src/app/http.clj +++ b/backend/src/app/http.clj @@ -76,11 +76,11 @@ (try (handler request) (catch Throwable e - (l/with-context (errors/get-error-context request e) - (l/error :hint "unexpected error processing request" - :query-string (:query-string request) - :cause e) - {:status 500 :body "internal server error"})))))) + (l/error :hint "unexpected error processing request" + ::l/context (errors/get-error-context request e) + :query-string (:query-string request) + :cause e) + {:status 500 :body "internal server error"}))))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Http Router diff --git a/backend/src/app/http/debug.clj b/backend/src/app/http/debug.clj index 2098cd084..8e021fcd8 100644 --- a/backend/src/app/http/debug.clj +++ b/backend/src/app/http/debug.clj @@ -18,7 +18,7 @@ [app.util.template :as tmpl] [app.util.time :as dt] [clojure.java.io :as io] - [clojure.pprint :as ppr] + [fipp.edn :as fpp] [cuerdas.core :as str] [datoteka.core :as fs] [integrant.core :as ig])) @@ -147,21 +147,20 @@ (some-> (db/get-by-id pool :server-error-report id) :content db/decode-transit-pgobject))) (render-template [report] - (binding [ppr/*print-right-margin* 300] - (let [context (dissoc report - :trace :cause :params :data :spec-problems - :spec-explain :spec-value :error :explain :hint) - params {:context (with-out-str (ppr/pprint context)) - :hint (:hint report) - :spec-explain (:spec-explain report) - :spec-problems (:spec-problems report) - :spec-value (:spec-value report) - :data (:data report) - :trace (or (:trace report) - (some-> report :error :trace)) - :params (:params report)}] - (-> (io/resource "templates/error-report.tmpl") - (tmpl/render params))))) + (let [context (dissoc report + :trace :cause :params :data :spec-problems + :spec-explain :spec-value :error :explain :hint) + params {:context (with-out-str (fpp/pprint context {:width 300})) + :hint (:hint report) + :spec-explain (:spec-explain report) + :spec-problems (:spec-problems report) + :spec-value (:spec-value report) + :data (:data report) + :trace (or (:trace report) + (some-> report :error :trace)) + :params (:params report)}] + (-> (io/resource "templates/error-report.tmpl") + (tmpl/render params)))) ] (when-not (authorized? pool request) diff --git a/backend/src/app/http/errors.clj b/backend/src/app/http/errors.clj index b49df8e46..a53637849 100644 --- a/backend/src/app/http/errors.clj +++ b/backend/src/app/http/errors.clj @@ -9,11 +9,10 @@ (:require [app.common.exceptions :as ex] [app.common.logging :as l] + [app.common.spec :as us] [app.common.uuid :as uuid] - [clojure.pprint] [clojure.spec.alpha :as s] - [cuerdas.core :as str] - [expound.alpha :as expound])) + [cuerdas.core :as str])) (defn- parse-client-ip [{:keys [headers] :as request}] @@ -36,14 +35,13 @@ :data (some-> data (dissoc ::s/problems ::s/value ::s/spec)) :ip-addr (parse-client-ip request) :profile-id (:profile-id request)} + (let [headers (:headers request)] {:user-agent (get headers "user-agent") :frontend-version (get headers "x-frontend-version" "unknown")}) (when (and data (::s/problems data)) - {:spec-explain (binding [s/*explain-out* expound/printer] - (with-out-str - (s/explain-out (update data ::s/problems #(take 10 %)))))})))) + {:spec-explain (us/pretty-explain data)})))) (defmulti handle-exception (fn [err & _rest] @@ -59,20 +57,10 @@ [err _] {:status 400 :body (ex-data err)}) -(defn- explain-spec-error-data - [data] - (when (and (::s/problems data) - (::s/value data) - (::s/spec data)) - (binding [s/*explain-out* expound/printer] - (with-out-str - (s/explain-out (update data ::s/problems #(take 10 %))))))) - - (defmethod handle-exception :validation [err _] (let [data (ex-data err) - explain (explain-spec-error-data data)] + explain (us/pretty-explain data)] {:status 400 :body (-> data (dissoc ::s/problems) @@ -82,8 +70,10 @@ (defmethod handle-exception :assertion [error request] (let [edata (ex-data error)] - (l/with-context (get-error-context request error) - (l/error ::l/raw (ex-message error) :cause error)) + (l/error ::l/raw (ex-message error) + ::l/context (get-error-context request error) + :cause error) + {:status 500 :body {:type :server-error :code :assertion @@ -105,9 +95,9 @@ (ex/exception? (:handling edata))) (handle-exception (:handling edata) request) (do - (l/with-context (get-error-context request error) - (l/error ::l/raw (ex-message error) :cause error)) - + (l/error ::l/raw (ex-message error) + ::l/context (get-error-context request error) + :cause error) {:status 500 :body {:type :server-error :code :unexpected @@ -117,10 +107,9 @@ (defmethod handle-exception org.postgresql.util.PSQLException [error request] (let [state (.getSQLState ^java.sql.SQLException error)] - - (l/with-context (get-error-context request error) - (l/error ::l/raw (ex-message error) :cause error)) - + (l/error ::l/raw (ex-message error) + ::l/context (get-error-context request error) + :cause error) (cond (= state "57014") {:status 504 diff --git a/backend/src/app/loggers/sentry.clj b/backend/src/app/loggers/sentry.clj index 78ac1ee39..5f4b93898 100644 --- a/backend/src/app/loggers/sentry.clj +++ b/backend/src/app/loggers/sentry.clj @@ -120,8 +120,6 @@ (.captureMessage ^IHub shub msg) )) ] - ;; (clojure.pprint/pprint event) - (when @enabled (.withScope ^IHub shub (reify ScopeCallback (run [_ scope] diff --git a/backend/src/app/srepl/main.clj b/backend/src/app/srepl/main.clj index 08ea82a9b..111f46e32 100644 --- a/backend/src/app/srepl/main.clj +++ b/backend/src/app/srepl/main.clj @@ -17,7 +17,7 @@ [app.srepl.dev :as dev] [app.util.blob :as blob] [app.util.time :as dt] - [clojure.pprint :refer [pprint]] + [fipp.edn :refer [pprint]] [clojure.spec.alpha :as s] [cuerdas.core :as str] [expound.alpha :as expound])) diff --git a/backend/src/app/util/time.clj b/backend/src/app/util/time.clj index ed8f809e0..42d6ca64b 100644 --- a/backend/src/app/util/time.clj +++ b/backend/src/app/util/time.clj @@ -8,7 +8,8 @@ (:require [app.common.exceptions :as ex] [clojure.spec.alpha :as s] - [cuerdas.core :as str]) + [cuerdas.core :as str] + [fipp.ednize :as fez]) (:import java.time.Duration java.time.Instant @@ -111,6 +112,11 @@ (defmethod print-dup Duration [o w] (print-method o w)) +(extend-protocol fez/IEdn + Duration + (-edn [o] (pr-str o))) + + ;; --- INSTANT (defn instant @@ -175,6 +181,10 @@ (defmethod print-dup Instant [o w] (print-method o w)) +(extend-protocol fez/IEdn + Instant + (-edn [o] (pr-str o))) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Cron Expression ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/backend/src/app/worker.clj b/backend/src/app/worker.clj index 7d532f5e0..3bd8461fc 100644 --- a/backend/src/app/worker.clj +++ b/backend/src/app/worker.clj @@ -249,10 +249,16 @@ (defn get-error-context [error item] - (let [edata (ex-data error)] - {:id (uuid/next) - :data edata - :params item})) + (let [data (ex-data error)] + (merge + {:id (uuid/next) + :hint (ex-message error) + :spec-problems (some->> data ::s/problems (take 10) seq vec) + :spec-value (some->> data ::s/value) + :data (some-> data (dissoc ::s/problems ::s/value ::s/spec)) + :params item} + (when (and data (::s/problems data)) + {:spec-explain (us/pretty-explain data)})))) (defn- handle-exception [error item] @@ -266,8 +272,10 @@ (= ::noop (:strategy edata)) (assoc :inc-by 0)) - (l/with-context (get-error-context error item) - (l/error :cause error :hint "unhandled exception on task") + (do + (l/error :hint "unhandled exception on task" + ::l/context (get-error-context error item) + :cause error) (if (>= (:retry-num item) (:max-retries item)) {:status :failed :task item :error error} {:status :retry :task item :error error}))))) diff --git a/common/src/app/common/logging.cljc b/common/src/app/common/logging.cljc index aea40b935..8cead52ed 100644 --- a/common/src/app/common/logging.cljc +++ b/common/src/app/common/logging.cljc @@ -9,6 +9,7 @@ [app.common.exceptions :as ex] [clojure.pprint :refer [pprint]] [cuerdas.core :as str] + [fipp.edn :as fpp] #?(:clj [io.aviso.exception :as ie]) #?(:cljs [goog.log :as glog])) #?(:cljs (:require-macros [app.common.logging]) @@ -52,22 +53,16 @@ (defn stringify-data [val] (cond - (instance? clojure.lang.Named val) - (name val) - - (instance? Throwable val) - (binding [ie/*app-frame-names* [#"app.*"] - ie/*fonts* nil - ie/*traditional* true] - (ie/format-exception val nil)) - (string? val) val + (instance? clojure.lang.Named val) + (name val) + (coll? val) - (binding [clojure.pprint/*print-right-margin* 200] - (-> (with-out-str (pprint val)) - (simple-prune (* 1024 1024 3)))) + (binding [*print-level* 5 + *print-length* 20] + (with-out-str (fpp/pprint val {:width 200}))) :else (str val)))) @@ -163,13 +158,13 @@ (.isEnabled ^Logger logger ^Level level))) (defmacro log - [& {:keys [level cause ::logger ::async ::raw] :or {async true} :as props}] + [& {:keys [level cause ::logger ::async ::raw ::context] :or {async true} :as props}] (if (:ns &env) ; CLJS `(write-log! ~(or logger (str *ns*)) ~level ~cause - (or ~raw ~(dissoc props :level :cause ::logger ::raw))) - (let [props (dissoc props :level :cause ::logger ::async ::raw) + (or ~raw ~(dissoc props :level :cause ::logger ::raw ::context))) + (let [props (dissoc props :level :cause ::logger ::async ::raw ::context) logger (or logger (str *ns*)) logger-sym (gensym "log") level-sym (gensym "log")] @@ -180,7 +175,7 @@ `(->> (ThreadContext/getImmutableContext) (send-off logging-agent (fn [_# cdata#] - (with-context (into {} cdata#) + (with-context (-> {} (into cdata#) (into ~context)) (->> (or ~raw (build-map-message ~props)) (write-log! ~logger-sym ~level-sym ~cause)))))) diff --git a/common/src/app/common/spec.cljc b/common/src/app/common/spec.cljc index 85c4a6912..1742b6f62 100644 --- a/common/src/app/common/spec.cljc +++ b/common/src/app/common/spec.cljc @@ -19,7 +19,7 @@ [app.common.geom.point :as gpt] [app.common.uuid :as uuid] [cuerdas.core :as str] - [expound.alpha])) + [expound.alpha :as expound])) (s/check-asserts true) @@ -270,3 +270,14 @@ (spec-assert* ~spec params# ~message mdata#) (apply origf# params#))))))) +(defn pretty-explain + ([data] (pretty-explain data nil)) + ([data {:keys [max-problems] :or {max-problems 10}}] + (when (and (::s/problems data) + (::s/value data) + (::s/spec data)) + (binding [s/*explain-out* expound/printer] + (with-out-str + (s/explain-out (update data ::s/problems #(take max-problems %)))))))) + + From 3dd94bd362dc5ae15e1e1dc7fd6340d4b5cc2415 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 8 Feb 2022 00:18:48 +0100 Subject: [PATCH 2/9] :bug: Log correct deleted number value on recheck task --- backend/src/app/storage.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/app/storage.clj b/backend/src/app/storage.clj index 947d69776..413b33882 100644 --- a/backend/src/app/storage.clj +++ b/backend/src/app/storage.clj @@ -463,7 +463,7 @@ (l/info :task "recheck" :action "recheck items" :processed n - :deleted n) + :deleted d) {:processed n :deleted d}))))))) (def sql:retrieve-pending-to-recheck From 3461ec228152985c4210cdbd628dfbd1c8303786 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 2 Feb 2022 23:50:43 +0100 Subject: [PATCH 3/9] :sparkles: Ignore EOF errors on writting streamed response --- backend/src/app/http/middleware.clj | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/src/app/http/middleware.clj b/backend/src/app/http/middleware.clj index 3845fe839..153ddc727 100644 --- a/backend/src/app/http/middleware.clj +++ b/backend/src/app/http/middleware.clj @@ -90,6 +90,9 @@ (with-open [bos (buffered-output-stream output-stream buffer-size)] (let [tw (t/writer bos opts)] (t/write! tw data))) + (catch org.eclipse.jetty.io.EofException _cause + ;; Do nothing, EOF means client closes connection abruptly + nil) (catch Throwable cause (l/warn :hint "unexpected error on encoding response" :cause cause)))))) From 8babb59f75c7a51ad23a1772ea98622ff42b1009 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 10 Feb 2022 00:01:14 +0100 Subject: [PATCH 4/9] :sparkles: Process audit log events only if profile-id is known --- backend/src/app/loggers/audit.clj | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/backend/src/app/loggers/audit.clj b/backend/src/app/loggers/audit.clj index d8c5d33f7..86e41d3f4 100644 --- a/backend/src/app/loggers/audit.clj +++ b/backend/src/app/loggers/audit.clj @@ -176,20 +176,21 @@ (defn- persist-events [{:keys [pool executor] :as cfg} events] (letfn [(event->row [event] - [(uuid/next) - (:name event) - (:type event) - (:profile-id event) - (:tracked-at event) - (some-> (:ip-addr event) db/inet) - (db/tjson (:props event)) - "backend"])] + (when (:profile-id event) + [(uuid/next) + (:name event) + (:type event) + (:profile-id event) + (:tracked-at event) + (some-> (:ip-addr event) db/inet) + (db/tjson (:props event)) + "backend"]))] (aa/with-thread executor (when (seq events) (db/with-atomic [conn pool] (db/insert-multi! conn :audit-log [:id :name :type :profile-id :tracked-at :ip-addr :props :source] - (sequence (map event->row) events))))))) + (sequence (keep event->row) events))))))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Archive Task From d358185a04f3cd8f4f45451963fbaba3a7a227d3 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 10 Feb 2022 00:02:12 +0100 Subject: [PATCH 5/9] :lipstick: Minor cosmetic change on database logger processor --- backend/src/app/loggers/database.clj | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/src/app/loggers/database.clj b/backend/src/app/loggers/database.clj index aea476bbb..ebece9af0 100644 --- a/backend/src/app/loggers/database.clj +++ b/backend/src/app/loggers/database.clj @@ -69,12 +69,15 @@ (defmethod ig/pre-init-spec ::reporter [_] (s/keys :req-un [::wrk/executor ::db/pool ::receiver])) +(defn error-event? + [event] + (= "error" (:logger/level event))) + (defmethod ig/init-key ::reporter [_ {:keys [receiver] :as cfg}] (l/info :msg "initializing database error persistence") - (let [output (a/chan (a/sliding-buffer 128) - (filter (fn [event] - (= (:logger/level event) "error"))))] + (let [output (a/chan (a/sliding-buffer 5) + (filter error-event?))] (receiver :sub output) (a/go-loop [] (let [msg (a/ Date: Thu, 10 Feb 2022 00:02:38 +0100 Subject: [PATCH 6/9] :sparkles: Handle EOF on websocket write/ping operations --- backend/src/app/util/websocket.clj | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/backend/src/app/util/websocket.clj b/backend/src/app/util/websocket.clj index 994070797..b82783e3c 100644 --- a/backend/src/app/util/websocket.clj +++ b/backend/src/app/util/websocket.clj @@ -15,7 +15,9 @@ [clojure.core.async :as a] [yetti.websocket :as yws]) (:import - java.nio.ByteBuffer)) + java.nio.ByteBuffer + org.eclipse.jetty.io.EofException)) + (declare decode-beat) (declare encode-beat) @@ -132,17 +134,25 @@ (defn- ws-send! [conn s] (let [ch (a/chan 1)] - (yws/send! conn s (fn [e] - (when e (a/offer! ch e)) - (a/close! ch))) + (try + (yws/send! conn s (fn [e] + (when e (a/offer! ch e)) + (a/close! ch))) + (catch EofException cause + (a/offer! ch cause) + (a/close! ch))) ch)) (defn- ws-ping! [conn s] (let [ch (a/chan 1)] - (yws/ping! conn s (fn [e] - (when e (a/offer! ch e)) - (a/close! ch))) + (try + (yws/ping! conn s (fn [e] + (when e (a/offer! ch e)) + (a/close! ch))) + (catch EofException cause + (a/offer! ch cause) + (a/close! ch))) ch)) (defn- encode-beat From b591fbecf043eb5eb6bf40c759e15d24639bc35a Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 10 Feb 2022 00:15:00 +0100 Subject: [PATCH 7/9] :tada: Add health check api endpoint --- backend/src/app/http.clj | 2 +- backend/src/app/http/debug.clj | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/backend/src/app/http.clj b/backend/src/app/http.clj index d053786bb..594916a06 100644 --- a/backend/src/app/http.clj +++ b/backend/src/app/http.clj @@ -150,8 +150,8 @@ [middleware/errors errors/handle] [middleware/cookies]]} + ["/health" {:get (:health-check debug)}] ["/_doc" {:get (doc/handler rpc)}] - ["/feedback" {:middleware [(:middleware session)] :post feedback}] ["/auth/oauth/:provider" {:post (:handler oauth)}] diff --git a/backend/src/app/http/debug.clj b/backend/src/app/http/debug.clj index 8e021fcd8..e31040618 100644 --- a/backend/src/app/http/debug.clj +++ b/backend/src/app/http/debug.clj @@ -18,9 +18,9 @@ [app.util.template :as tmpl] [app.util.time :as dt] [clojure.java.io :as io] - [fipp.edn :as fpp] [cuerdas.core :as str] [datoteka.core :as fs] + [fipp.edn :as fpp] [integrant.core :as ig])) ;; (selmer.parser/cache-off!) @@ -194,9 +194,17 @@ :body (-> (io/resource "templates/error-list.tmpl") (tmpl/render {:items items}))})) +(defn health-check + "Mainly a task that performs a health check." + [{:keys [pool]} _] + (db/with-atomic [conn pool] + (db/exec-one! conn ["select count(*) as count from server_prop;"]) + {:status 200 :body "Ok"})) + (defmethod ig/init-key ::handlers [_ cfg] {:index (partial index cfg) + :health-check (partial health-check cfg) :retrieve-file-data (partial retrieve-file-data cfg) :retrieve-file-changes (partial retrieve-file-changes cfg) :retrieve-error (partial retrieve-error cfg) From 231256104113f6f5215c927b6d2de43222647aaf Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 10 Feb 2022 01:09:53 +0100 Subject: [PATCH 8/9] :sparkles: Temporaly disable parallel uploading of files on import --- frontend/src/app/worker/import.cljs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/worker/import.cljs b/frontend/src/app/worker/import.cljs index b337f0e8b..4f269dcc4 100644 --- a/frontend/src/app/worker/import.cljs +++ b/frontend/src/app/worker/import.cljs @@ -342,7 +342,10 @@ pre-process-images (->> (rx/from nodes) (rx/filter media-node?) - (rx/merge-map + ;; TODO: this should be merge-map, but we disable the + ;; parallel upload until we resolve resource usage issues + ;; on backend. + (rx/mapcat (fn [node] (->> (resolve-media context file-id node) (rx/map (fn [result] [node result]))))) From dcbd89ff7ce38fe664559942f2277634202eea97 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 10 Feb 2022 09:27:54 +0100 Subject: [PATCH 9/9] :sparkles: Increase default max connection pool size to 60. --- backend/src/app/db.clj | 26 ++++++++++++++++++-------- backend/src/app/main.clj | 2 +- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/backend/src/app/db.clj b/backend/src/app/db.clj index a4976b47b..80fb8d5c9 100644 --- a/backend/src/app/db.clj +++ b/backend/src/app/db.clj @@ -50,16 +50,26 @@ (declare instrument-jdbc!) (declare apply-migrations!) -(s/def ::name keyword?) -(s/def ::uri ::us/not-empty-string) -(s/def ::min-pool-size ::us/integer) +(s/def ::connection-timeout ::us/integer) (s/def ::max-pool-size ::us/integer) (s/def ::migrations map?) +(s/def ::min-pool-size ::us/integer) +(s/def ::name keyword?) +(s/def ::password ::us/string) (s/def ::read-only ::us/boolean) +(s/def ::uri ::us/not-empty-string) +(s/def ::username ::us/string) +(s/def ::validation-timeout ::us/integer) (defmethod ig/pre-init-spec ::pool [_] - (s/keys :req-un [::uri ::name ::min-pool-size ::max-pool-size] - :opt-un [::migrations ::mtx/metrics ::read-only])) + (s/keys :req-un [::uri ::name ::username ::password] + :opt-un [::min-pool-size + ::max-pool-size + ::connection-timeout + ::validation-timeout + ::migrations + ::mtx/metrics + ::read-only])) (defmethod ig/init-key ::pool [_ {:keys [migrations metrics name] :as cfg}] @@ -110,11 +120,11 @@ (.setPoolName (d/name (:name cfg))) (.setAutoCommit true) (.setReadOnly read-only) - (.setConnectionTimeout 10000) ;; 10seg - (.setValidationTimeout 10000) ;; 10seg + (.setConnectionTimeout (:connection-timeout cfg 10000)) ;; 10seg + (.setValidationTimeout (:validation-timeout cfg 10000)) ;; 10seg (.setIdleTimeout 120000) ;; 2min (.setMaxLifetime 1800000) ;; 30min - (.setMinimumIdle (:min-pool-size cfg 0)) + (.setMinimumIdle (:min-pool-size cfg 0)) (.setMaximumPoolSize (:max-pool-size cfg 50)) (.setConnectionInitSql initsql) (.setInitializationFailTimeout -1)) diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index 27e47fb0e..2264e96ba 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -21,7 +21,7 @@ :migrations (ig/ref :app.migrations/all) :name :main :min-pool-size 0 - :max-pool-size 30} + :max-pool-size 60} :app.migrations/migrations {}