From 199360efa6c675a42a91c0b7729bdfd5c66c546f Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 15 Jun 2022 12:18:39 +0200 Subject: [PATCH 1/3] :paperclip: Update default repl script --- backend/scripts/repl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/scripts/repl b/backend/scripts/repl index 49e105305..d7d60368b 100755 --- a/backend/scripts/repl +++ b/backend/scripts/repl @@ -24,9 +24,8 @@ mc mb penpot-s3/penpot -p export AWS_ACCESS_KEY_ID=penpot-devenv export AWS_SECRET_ACCESS_KEY=penpot-devenv -export PENPOT_ASSETS_STORAGE_BACKEND=assets-fs +export PENPOT_ASSETS_STORAGE_BACKEND=assets-s3 export PENPOT_STORAGE_ASSETS_S3_ENDPOINT=http://minio:9000 -export PENPOT_STORAGE_ASSETS_S3_REGION=eu-central-1 export PENPOT_STORAGE_ASSETS_S3_BUCKET=penpot export OPTIONS=" From dce479bc4b52513941cecc7b3733fd8112045469 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 15 Jun 2022 12:19:16 +0200 Subject: [PATCH 2/3] :sparkles: Make the pool initialization process and defaults reusable And add the ability to skip pool initialization if no enough data is provided. Mainly for initialize pools based on configuration for not essential/dynamic services. --- backend/src/app/db.clj | 72 ++++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/backend/src/app/db.clj b/backend/src/app/db.clj index 9b923495b..6604df6b5 100644 --- a/backend/src/app/db.clj +++ b/backend/src/app/db.clj @@ -55,54 +55,66 @@ (s/def ::migrations map?) (s/def ::name keyword?) (s/def ::password ::us/string) -(s/def ::read-only ::us/boolean) (s/def ::uri ::us/not-empty-string) (s/def ::username ::us/string) (s/def ::validation-timeout ::us/integer) +(s/def ::read-only? ::us/boolean) -(defmethod ig/pre-init-spec ::pool [_] - (s/keys :req-un [::uri ::name +(s/def ::pool-options + (s/keys :opt-un [::uri ::name ::min-size ::max-size ::connection-timeout - ::validation-timeout] - :opt-un [::migrations + ::validation-timeout + ::migrations ::username ::password ::mtx/metrics - ::read-only])) + ::read-only?])) + +(def defaults + {:name :main + :min-size 0 + :max-size 30 + :connection-timeout 10000 + :validation-timeout 10000 + :idle-timeout 120000 ; 2min + :max-lifetime 1800000 ; 30m + :read-only? false}) (defmethod ig/prep-key ::pool [_ cfg] - (merge {:name :main - :min-size 0 - :max-size 30 - :connection-timeout 10000 - :validation-timeout 10000 - :idle-timeout 120000 ; 2min - :max-lifetime 1800000 ; 30m - :read-only false} - (d/without-nils cfg))) + (merge defaults (d/without-nils cfg))) + +;; Don't validate here, just validate that a map is received. +(defmethod ig/pre-init-spec ::pool [_] ::pool-options) (defmethod ig/init-key ::pool - [_ {:keys [migrations name read-only] :as cfg}] - (l/info :hint "initialize connection pool" - :name (d/name name) - :uri (:uri cfg) - :read-only read-only - :with-credentials (and (contains? cfg :username) - (contains? cfg :password)) - :min-size (:min-size cfg) - :max-size (:max-size cfg)) + [_ {:keys [migrations read-only? uri] :as cfg}] + (if uri + (let [pool (create-pool cfg)] + (l/info :hint "initialize connection pool" + :name (d/name (:name cfg)) + :uri uri + :read-only read-only? + :with-credentials (and (contains? cfg :username) + (contains? cfg :password)) + :min-size (:min-size cfg) + :max-size (:max-size cfg)) + (when-not read-only? + (some->> (seq migrations) (apply-migrations! pool))) + pool) - (let [pool (create-pool cfg)] - (when-not read-only - (some->> (seq migrations) (apply-migrations! pool))) - pool)) + (do + (l/warn :hint "unable to initialize pool, missing url" + :name (d/name (:name cfg)) + :read-only read-only?) + nil))) (defmethod ig/halt-key! ::pool [_ pool] - (.close ^HikariDataSource pool)) + (when pool + (.close ^HikariDataSource pool))) (defn- apply-migrations! [pool migrations] @@ -126,7 +138,7 @@ (.setJdbcUrl (str "jdbc:" uri)) (.setPoolName (d/name (:name cfg))) (.setAutoCommit true) - (.setReadOnly (:read-only cfg)) + (.setReadOnly (:read-only? cfg)) (.setConnectionTimeout (:connection-timeout cfg)) (.setValidationTimeout (:validation-timeout cfg)) (.setIdleTimeout (:idle-timeout cfg)) From adf2d82a52749bad970ed5103c57f86207c285e5 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 15 Jun 2022 12:21:23 +0200 Subject: [PATCH 3/3] :tada: Add proper logging reports on audit-log-archive task --- backend/src/app/loggers/audit.clj | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/backend/src/app/loggers/audit.clj b/backend/src/app/loggers/audit.clj index 8b5a8c2ec..8fb9ce6b9 100644 --- a/backend/src/app/loggers/audit.clj +++ b/backend/src/app/loggers/audit.clj @@ -257,12 +257,16 @@ (ex/raise :type :internal :code :task-not-configured :hint "archive task not configured, missing uri")) + (when enabled - (loop [] - (let [res (archive-events cfg)] - (when (= res :continue) - (aa/thread-sleep 200) - (recur)))))))) + (loop [total 0] + (let [n (archive-events cfg)] + (if n + (do + (aa/thread-sleep 200) + (recur (+ total n))) + (when (pos? total) + (l/trace :hint "events chunk archived" :num total))))))))) (def sql:retrieve-batch-of-audit-log "select * from audit_log @@ -332,7 +336,7 @@ (l/debug :action "archive-events" :uri uri :events (count events)) (when (send events) (mark-as-archived conn rows) - :continue)))))) + (count events))))))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; GC Task