From fd5b1c034157b9a28e0a88e3e2d4873bb5d0d076 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 8 Apr 2024 11:04:56 +0200 Subject: [PATCH 1/6] :sparkles: Enable by default components v2 feature --- backend/scripts/repl | 1 - backend/scripts/start-dev | 1 - common/src/app/common/features.cljc | 3 ++- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/scripts/repl b/backend/scripts/repl index 6336331e4..ddc2fba5f 100755 --- a/backend/scripts/repl +++ b/backend/scripts/repl @@ -26,7 +26,6 @@ export PENPOT_FLAGS="\ enable-soft-rpc-rlimit \ enable-webhooks \ enable-access-tokens \ - disable-feature-components-v2 \ enable-file-validation \ enable-file-schema-validation"; diff --git a/backend/scripts/start-dev b/backend/scripts/start-dev index 1a2fa2842..2497e801e 100755 --- a/backend/scripts/start-dev +++ b/backend/scripts/start-dev @@ -18,7 +18,6 @@ export PENPOT_FLAGS="\ enable-rpc-climit \ enable-smtp \ enable-access-tokens \ - disable-feature-components-v2 \ enable-file-validation \ enable-file-schema-validation"; diff --git a/common/src/app/common/features.cljc b/common/src/app/common/features.cljc index 90b27587f..fffa5b6a6 100644 --- a/common/src/app/common/features.cljc +++ b/common/src/app/common/features.cljc @@ -58,7 +58,8 @@ (def default-enabled-features #{"fdata/shape-data-type" "styles/v2" - "layout/grid"}) + "layout/grid" + "components/v2"}) ;; A set of features which only affects on frontend and can be enabled ;; and disabled freely by the user any time. This features does not From 06206f39b01cfc9c15a572b37c598a6c953c22cb Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 8 Apr 2024 11:07:32 +0200 Subject: [PATCH 2/6] :paperclip: Update devenv nginx configuration --- docker/devenv/files/nginx.conf | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docker/devenv/files/nginx.conf b/docker/devenv/files/nginx.conf index 24a0f2ddf..5c1ae44db 100644 --- a/docker/devenv/files/nginx.conf +++ b/docker/devenv/files/nginx.conf @@ -14,11 +14,20 @@ http { tcp_nodelay on; keepalive_timeout 65; types_hash_max_size 2048; - # server_tokens off; + server_tokens off; # server_names_hash_bucket_size 64; # server_name_in_redirect off; + reset_timedout_connection on; + client_body_timeout 20s; + client_header_timeout 20s; + + proxy_connect_timeout 300s; + proxy_send_timeout 300s; + proxy_read_timeout 300s; + send_timeout 300s; + include /etc/nginx/mime.types; default_type application/octet-stream; @@ -33,7 +42,7 @@ http { gzip_buffers 16 8k; gzip_http_version 1.1; - gzip_types text/plain text/css text/javascript application/javascript application/json application/transit+json; + gzip_types text/plain text/css text/javascript application/javascript application/json application/transit+json image/svg+xml; map $http_upgrade $connection_upgrade { default upgrade; From d435b17452d64275a1bf7ce4616e302cf5e22ae3 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 8 Apr 2024 11:18:26 +0200 Subject: [PATCH 3/6] :bug: Fix io exception incorrect reporting --- backend/src/app/http/errors.clj | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/backend/src/app/http/errors.clj b/backend/src/app/http/errors.clj index 18350d21d..47a9a4dde 100644 --- a/backend/src/app/http/errors.clj +++ b/backend/src/app/http/errors.clj @@ -218,6 +218,14 @@ :hint (ex-message error) :data edata}})))) +(defmethod handle-exception java.io.IOException + [cause _ _] + (l/wrn :hint "io exception" :cause cause) + {::rres/status 500 + ::rres/body {:type :server-error + :code :io-exception + :hint (ex-message cause)}}) + (defmethod handle-exception java.util.concurrent.CompletionException [cause request _] (let [cause' (ex-cause cause)] From 0965c71a08cdb9543c89a4502aab4366aad340a8 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 8 Apr 2024 11:42:01 +0200 Subject: [PATCH 4/6] :paperclip: Update logging on webhook task handler --- backend/src/app/loggers/webhooks.clj | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/backend/src/app/loggers/webhooks.clj b/backend/src/app/loggers/webhooks.clj index 00ebd3f38..d93ce0185 100644 --- a/backend/src/app/loggers/webhooks.clj +++ b/backend/src/app/loggers/webhooks.clj @@ -67,12 +67,10 @@ [_ {:keys [::db/pool] :as cfg}] (fn [{:keys [props] :as task}] (let [event (::event props)] - - (l/debug :hint "process webhook event" - :name (:name event)) + (l/dbg :hint "process webhook event" :name (:name event)) (when-let [items (lookup-webhooks cfg event)] - (l/trace :hint "webhooks found for event" :total (count items)) + (l/trc :hint "webhooks found for event" :total (count items)) (db/with-atomic [conn pool] (doseq [item items] From 37507c3697d96ee4e5ba3eb8c38153401fffe985 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 8 Apr 2024 11:53:33 +0200 Subject: [PATCH 5/6] :paperclip: Update logging on worker module --- backend/src/app/worker/dispatcher.clj | 2 +- backend/src/app/worker/runner.clj | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/src/app/worker/dispatcher.clj b/backend/src/app/worker/dispatcher.clj index dbdb06042..9b901747f 100644 --- a/backend/src/app/worker/dispatcher.clj +++ b/backend/src/app/worker/dispatcher.clj @@ -56,7 +56,7 @@ (db/create-array conn "uuid" ids)]] (db/exec-one! conn sql) - (l/trc :hist "queue tasks" + (l/trc :hist "enqueue tasks on redis" :queue queue :tasks (count ids) :queued res))) diff --git a/backend/src/app/worker/runner.clj b/backend/src/app/worker/runner.clj index 40332ab23..11ca3d972 100644 --- a/backend/src/app/worker/runner.clj +++ b/backend/src/app/worker/runner.clj @@ -140,20 +140,20 @@ :else (try (l/trc :hint "start task" - :queue queue - :runner-id id :name (:name task) :task-id (str task-id) + :queue queue + :runner-id id :retry (:retry-num task)) (let [tpoint (dt/tpoint) result (handle-task task) elapsed (dt/format-duration (tpoint))] (l/trc :hint "end task" - :queue queue - :runner-id id :name (:name task) :task-id (str task-id) + :queue queue + :runner-id id :retry (:retry-num task) :elapsed elapsed) From 5f8d66e7eb4aac2974bbcc7630b4452dde110c36 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 8 Apr 2024 12:49:02 +0200 Subject: [PATCH 6/6] :sparkles: Remove unnecesarry transaction wrapping on import-binfile --- backend/src/app/binfile/v1.clj | 2 +- backend/src/app/rpc/commands/binfile.clj | 22 ++++++++++------------ 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/backend/src/app/binfile/v1.clj b/backend/src/app/binfile/v1.clj index d2b7cdf7f..5bad01f6d 100644 --- a/backend/src/app/binfile/v1.clj +++ b/backend/src/app/binfile/v1.clj @@ -468,7 +468,7 @@ (defn- read-import-v1 [{:keys [::db/conn ::project-id ::profile-id ::input] :as cfg}] - (db/exec-one! conn ["SET idle_in_transaction_session_timeout = 0"]) + (db/exec-one! conn ["SET LOCAL idle_in_transaction_session_timeout = 0"]) (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"]) (pu/with-open [input (zstd-input-stream input) diff --git a/backend/src/app/rpc/commands/binfile.clj b/backend/src/app/rpc/commands/binfile.clj index 60148eb7e..d6759eb42 100644 --- a/backend/src/app/rpc/commands/binfile.clj +++ b/backend/src/app/rpc/commands/binfile.clj @@ -65,18 +65,16 @@ ;; --- Command: import-binfile (defn- import-binfile - [{:keys [::wrk/executor ::bf.v1/project-id] :as cfg} input] - (db/tx-run! cfg - (fn [{:keys [::db/conn] :as cfg}] - ;; NOTE: the importation process performs some operations that - ;; are not very friendly with virtual threads, and for avoid - ;; unexpected blocking of other concurrent operations we - ;; dispatch that operation to a dedicated executor. - (let [result (px/invoke! executor (partial bf.v1/import-files! cfg input))] - (db/update! conn :project - {:modified-at (dt/now)} - {:id project-id}) - result)))) + [{:keys [::wrk/executor ::bf.v1/project-id ::db/pool] :as cfg} input] + ;; NOTE: the importation process performs some operations that + ;; are not very friendly with virtual threads, and for avoid + ;; unexpected blocking of other concurrent operations we + ;; dispatch that operation to a dedicated executor. + (let [result (px/invoke! executor (partial bf.v1/import-files! cfg input))] + (db/update! pool :project + {:modified-at (dt/now)} + {:id project-id}) + result)) (def ^:private schema:import-binfile