From ab461ba5604e5930605b902dcbfbfb653de8d2f0 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 12 Aug 2024 11:03:38 +0200 Subject: [PATCH 1/4] :bug: Launch offload only if file-gc has processed the file --- backend/src/app/tasks/file_gc.clj | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/backend/src/app/tasks/file_gc.clj b/backend/src/app/tasks/file_gc.clj index e84e5a450..730dbe8ae 100644 --- a/backend/src/app/tasks/file_gc.clj +++ b/backend/src/app/tasks/file_gc.clj @@ -301,16 +301,15 @@ (try (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}] - (let [cfg (update cfg ::sto/storage sto/configure conn) - res (process-file! cfg)] - - (when (contains? cf/flags :tiered-file-data-storage) + (let [cfg (update cfg ::sto/storage sto/configure conn) + processed? (process-file! cfg)] + (when (and processed? (contains? cf/flags :tiered-file-data-storage)) (wrk/submit! (-> cfg (assoc ::wrk/task :offload-file-data) (assoc ::wrk/params props) (assoc ::wrk/priority 10) (assoc ::wrk/delay 1000)))) - res))) + processed?))) (catch Throwable cause (l/err :hint "error on cleaning file" From 7df68bb8bdf363698c88e0ad9071dc612173c151 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 12 Aug 2024 11:05:32 +0200 Subject: [PATCH 2/4] :sparkles: Add `:params` prop to `:not-found` exception --- backend/src/app/db.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/app/db.clj b/backend/src/app/db.clj index 8c03be660..2df9a53b1 100644 --- a/backend/src/app/db.clj +++ b/backend/src/app/db.clj @@ -407,6 +407,7 @@ (ex/raise :type :not-found :code :object-not-found :table table + :params params :hint "database object not found")) row)) From 77c45ed109195145a507b251b5e07aa7353b5b18 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 12 Aug 2024 11:15:28 +0200 Subject: [PATCH 3/4] :sparkles: Add better error reporting on offload-file-data task --- backend/src/app/tasks/offload_file_data.clj | 40 ++++++++++++--------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/backend/src/app/tasks/offload_file_data.clj b/backend/src/app/tasks/offload_file_data.clj index ec8739179..99788cb9f 100644 --- a/backend/src/app/tasks/offload_file_data.clj +++ b/backend/src/app/tasks/offload_file_data.clj @@ -8,6 +8,7 @@ "A maintenance task responsible of moving file data from hot storage (the database row) to a cold storage (fs or s3)." (:require + [app.common.exceptions :as ex] [app.common.logging :as l] [app.db :as db] [app.db.sql :as-alias sql] @@ -18,26 +19,31 @@ (defn- offload-file-data! [{:keys [::db/conn ::sto/storage ::file-id] :as cfg}] (let [file (db/get conn :file {:id file-id} - {::sql/for-update true}) + {::sql/for-update true})] + (when (nil? (:data file)) + (ex/raise :hint "file already offloaded" + :type :internal + :code :file-already-offloaded + :file-id file-id)) - data (sto/content (:data file)) - sobj (sto/put-object! storage - {::sto/content data - ::sto/touch true - :bucket "file-data" - :content-type "application/octet-stream" - :file-id file-id})] + (let [data (sto/content (:data file)) + sobj (sto/put-object! storage + {::sto/content data + ::sto/touch true + :bucket "file-data" + :content-type "application/octet-stream" + :file-id file-id})] - (l/trc :hint "offload file data" - :file-id (str file-id) - :storage-id (str (:id sobj))) + (l/trc :hint "offload file data" + :file-id (str file-id) + :storage-id (str (:id sobj))) - (db/update! conn :file - {:data-backend "objects-storage" - :data-ref-id (:id sobj) - :data nil} - {:id file-id} - {::db/return-keys false}))) + (db/update! conn :file + {:data-backend "objects-storage" + :data-ref-id (:id sobj) + :data nil} + {:id file-id} + {::db/return-keys false})))) (defn- offload-file-data-fragments! [{:keys [::db/conn ::sto/storage ::file-id] :as cfg}] From c0174ab501d2f2689bacf5d7dc33cc22f96fb47d Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 12 Aug 2024 11:45:01 +0200 Subject: [PATCH 4/4] :bug: Fix unhandled exception on try to reuse registration token --- backend/src/app/rpc/commands/auth.clj | 38 ++++++++++++++++----------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/backend/src/app/rpc/commands/auth.clj b/backend/src/app/rpc/commands/auth.clj index a8fc218ec..20a25e9f8 100644 --- a/backend/src/app/rpc/commands/auth.clj +++ b/backend/src/app/rpc/commands/auth.clj @@ -355,16 +355,22 @@ profile (if-let [profile-id (:profile-id claims)] (profile/get-profile conn profile-id) - (let [is-active (or (boolean (:is-active claims)) - (not (contains? cf/flags :email-verification))) - params (-> params - (assoc :is-active is-active) - (update :password #(profile/derive-password cfg %)))] - (->> (create-profile! conn params) - (create-profile-rels! conn)))) + ;; NOTE: we first try to match existing profile + ;; by email, that in normal circumstances will + ;; not return anything, but when a user tries to + ;; reuse the same token multiple times, we need + ;; to detect if the profile is already registered + (or (profile/get-profile-by-email conn (:email claims)) + (let [is-active (or (boolean (:is-active claims)) + (not (contains? cf/flags :email-verification))) + params (-> params + (assoc :is-active is-active) + (update :password #(profile/derive-password cfg %))) + profile (->> (create-profile! conn params) + (create-profile-rels! conn))] + (vary-meta profile assoc :created true)))) - ;; When no profile-id comes on claims means a new register - created? (not (:profile-id claims)) + created? (-> profile meta :created true?) invitation (when-let [token (:invitation-token params)] (tokens/verify (::setup/props cfg) {:token token :iss :team-invitation})) @@ -422,13 +428,13 @@ ::audit/profile-id (:id profile)}))) :else - (let [elapsed? (elapsed-verify-threshold? profile) - complaints? (eml/has-reports? conn (:email profile)) - action (if complaints? - "ignore-because-complaints" - (if elapsed? - "resend-email-verification" - "ignore"))] + (let [elapsed? (elapsed-verify-threshold? profile) + reports? (eml/has-reports? conn (:email profile)) + action (if reports? + "ignore-because-complaints" + (if elapsed? + "resend-email-verification" + "ignore"))] (l/wrn :hint "repeated registry detected" :profile-id (str (:id profile))