From 0dc7f4e07ecc095bc91b7a3870d7f8cd809fd27a Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Sun, 19 Mar 2023 11:20:32 +0100 Subject: [PATCH] :sparkles: Add test for orphaned teams deletion --- backend/src/app/tasks/objects_gc.clj | 163 ++++++++---------- backend/test/backend_tests/helpers.clj | 4 + .../test/backend_tests/rpc_profile_test.clj | 62 +++++++ 3 files changed, 141 insertions(+), 88 deletions(-) diff --git a/backend/src/app/tasks/objects_gc.clj b/backend/src/app/tasks/objects_gc.clj index a5b1b0195..2ca13664b 100644 --- a/backend/src/app/tasks/objects_gc.clj +++ b/backend/src/app/tasks/objects_gc.clj @@ -62,7 +62,9 @@ (when (:rollback? params) (db/rollback! conn)) - {:processed (+ stotal htotal)})))) + {:processed (+ stotal htotal) + :orphans stotal})))) + (def ^:private sql:get-profiles-chunk @@ -78,24 +80,22 @@ [{:keys [::conn ::min-age ::storage] :as cfg}] (letfn [(get-chunk [cursor] (let [rows (db/exec! conn [sql:get-profiles-chunk min-age cursor])] - [(some->> rows peek :created-at) rows]))] - (reduce - (fn [total {:keys [id photo-id]}] - (l/debug :hint "permanently delete profile" :id (str id)) + [(some->> rows peek :created-at) rows])) - ;; Mark as deleted the storage object related with the - ;; photo-id field. - (some->> photo-id (sto/touch-object! storage)) + (process-profile [total {:keys [id photo-id]}] + (l/debug :hint "permanently delete profile" :id (str id)) - ;; And finally, permanently delete the profile. - (db/delete! conn :profile {:id id}) + ;; Mark as deleted the storage object related with the + ;; photo-id field. + (some->> photo-id (sto/touch-object! storage)) - (inc total)) - 0 - (d/iteration get-chunk - :vf second - :kf first - :initk (dt/now))))) + ;; And finally, permanently delete the profile. + (db/delete! conn :profile {:id id}) + + (inc total))] + + (->> (d/iteration get-chunk :vf second :kf first :initk (dt/now)) + (reduce process-profile 0)))) (def ^:private sql:get-teams-chunk "select id, photo_id, created_at from team @@ -110,24 +110,22 @@ [{:keys [::conn ::min-age ::storage] :as cfg}] (letfn [(get-chunk [cursor] (let [rows (db/exec! conn [sql:get-teams-chunk min-age cursor])] - [(some->> rows peek :created-at) rows]))] - (reduce - (fn [total {:keys [id photo-id]}] - (l/debug :hint "permanently delete team" :id (str id)) + [(some->> rows peek :created-at) rows])) - ;; Mark as deleted the storage object related with the - ;; photo-id field. - (some->> photo-id (sto/touch-object! storage)) + (process-team [total {:keys [id photo-id]}] + (l/debug :hint "permanently delete team" :id (str id)) - ;; And finally, permanently delete the team. - (db/delete! conn :team {:id id}) + ;; Mark as deleted the storage object related with the + ;; photo-id field. + (some->> photo-id (sto/touch-object! storage)) - (inc total)) - 0 - (d/iteration get-chunk - :vf second - :kf first - :initk (dt/now))))) + ;; And finally, permanently delete the team. + (db/delete! conn :team {:id id}) + + (inc total))] + + (->> (d/iteration get-chunk :vf second :kf first :initk (dt/now)) + (reduce process-team 0)))) (def ^:private sql:get-orphan-teams-chunk "select t.id, t.created_at @@ -146,23 +144,21 @@ [{:keys [::conn] :as cfg}] (letfn [(get-chunk [cursor] (let [rows (db/exec! conn [sql:get-orphan-teams-chunk cursor])] - [(some->> rows peek :created-at) rows]))] - (reduce - (fn [total {:keys [id]}] - (let [result (db/update! conn :team - {:deleted-at (dt/now)} - {:id id :deleted-at nil} - {::db/return-keys? false}) - count (db/get-update-count result)] - (when (pos? count) - (l/debug :hint "mark team for deletion" :id (str id) )) + [(some->> rows peek :created-at) rows])) - (+ total count))) - 0 - (d/iteration get-chunk - :vf second - :kf first - :initk (dt/now))))) + (process-team [total {:keys [id]}] + (let [result (db/update! conn :team + {:deleted-at (dt/now)} + {:id id :deleted-at nil} + {::db/return-keys? false}) + count (db/get-update-count result)] + (when (pos? count) + (l/debug :hint "mark team for deletion" :id (str id) )) + + (+ total count)))] + + (->> (d/iteration get-chunk :vf second :kf first :initk (dt/now)) + (reduce process-team 0)))) (def ^:private sql:get-fonts-chunk "select id, created_at, woff1_file_id, woff2_file_id, otf_file_id, ttf_file_id @@ -178,26 +174,24 @@ [{:keys [::conn ::min-age ::storage] :as cfg}] (letfn [(get-chunk [cursor] (let [rows (db/exec! conn [sql:get-fonts-chunk min-age cursor])] - [(some->> rows peek :created-at) rows]))] - (reduce - (fn [total {:keys [id] :as font}] - (l/debug :hint "permanently delete font variant" :id (str id)) + [(some->> rows peek :created-at) rows])) - ;; Mark as deleted the all related storage objects - (some->> (:woff1-file-id font) (sto/touch-object! storage)) - (some->> (:woff2-file-id font) (sto/touch-object! storage)) - (some->> (:otf-file-id font) (sto/touch-object! storage)) - (some->> (:ttf-file-id font) (sto/touch-object! storage)) + (process-font [total {:keys [id] :as font}] + (l/debug :hint "permanently delete font variant" :id (str id)) - ;; And finally, permanently delete the team font variant - (db/delete! conn :team-font-variant {:id id}) + ;; Mark as deleted the all related storage objects + (some->> (:woff1-file-id font) (sto/touch-object! storage)) + (some->> (:woff2-file-id font) (sto/touch-object! storage)) + (some->> (:otf-file-id font) (sto/touch-object! storage)) + (some->> (:ttf-file-id font) (sto/touch-object! storage)) - (inc total)) - 0 - (d/iteration get-chunk - :vf second - :kf first - :initk (dt/now))))) + ;; And finally, permanently delete the team font variant + (db/delete! conn :team-font-variant {:id id}) + + (inc total))] + + (->> (d/iteration get-chunk :vf second :kf first :initk (dt/now)) + (reduce process-font 0)))) (def ^:private sql:get-projects-chunk "select id, created_at @@ -213,20 +207,17 @@ [{:keys [::conn ::min-age] :as cfg}] (letfn [(get-chunk [cursor] (let [rows (db/exec! conn [sql:get-projects-chunk min-age cursor])] - [(some->> rows peek :created-at) rows]))] - (reduce - (fn [total {:keys [id]}] - (l/debug :hint "permanently delete project" :id (str id)) + [(some->> rows peek :created-at) rows])) - ;; And finally, permanently delete the project. - (db/delete! conn :project {:id id}) + (process-project [total {:keys [id]}] + (l/debug :hint "permanently delete project" :id (str id)) + ;; And finally, permanently delete the project. + (db/delete! conn :project {:id id}) - (inc total)) - 0 - (d/iteration get-chunk - :vf second - :kf first - :initk (dt/now))))) + (inc total))] + + (->> (d/iteration get-chunk :vf second :kf first :initk (dt/now)) + (reduce process-project 0)))) (def ^:private sql:get-files-chunk "select id, created_at @@ -242,17 +233,13 @@ [{:keys [::conn ::min-age] :as cfg}] (letfn [(get-chunk [cursor] (let [rows (db/exec! conn [sql:get-files-chunk min-age cursor])] - [(some->> rows peek :created-at) rows]))] - (reduce - (fn [total {:keys [id]}] - (l/debug :hint "permanently delete file" :id (str id)) + [(some->> rows peek :created-at) rows])) - ;; And finally, permanently delete the file. - (db/delete! conn :file {:id id}) + (process-file [total {:keys [id]}] + (l/debug :hint "permanently delete file" :id (str id)) + ;; And finally, permanently delete the file. + (db/delete! conn :file {:id id}) + (inc total))] - (inc total)) - 0 - (d/iteration get-chunk - :vf second - :kf first - :initk (dt/now))))) + (->> (d/iteration get-chunk :vf second :kf first :initk (dt/now)) + (reduce process-file 0)))) diff --git a/backend/test/backend_tests/helpers.clj b/backend/test/backend_tests/helpers.clj index a6aa4e158..ac8b0ce45 100644 --- a/backend/test/backend_tests/helpers.clj +++ b/backend/test/backend_tests/helpers.clj @@ -498,6 +498,10 @@ [& params] (apply db/insert! *pool* params)) +(defn db-delete! + [& params] + (apply db/delete! *pool* params)) + (defn db-query [& params] (apply db/query *pool* params)) diff --git a/backend/test/backend_tests/rpc_profile_test.clj b/backend/test/backend_tests/rpc_profile_test.clj index afc13388c..00fa883c0 100644 --- a/backend/test/backend_tests/rpc_profile_test.clj +++ b/backend/test/backend_tests/rpc_profile_test.clj @@ -161,6 +161,68 @@ (let [result (:result out)] (t/is (= uuid/zero (:id result))))))) +(t/deftest profile-immediate-deletion + (let [prof1 (th/create-profile* 1) + prof2 (th/create-profile* 2) + file (th/create-file* 1 {:profile-id (:id prof1) + :project-id (:default-project-id prof1) + :is-shared false}) + + team (th/create-team* 1 {:profile-id (:id prof1)}) + _ (th/create-team-role* {:team-id (:id team) + :profile-id (:id prof2) + :role :admin})] + + ;; profile is not deleted because it does not meet all + ;; conditions to be deleted. + (let [result (th/run-task! :objects-gc {:min-age (dt/duration 0)})] + (t/is (= 0 (:orphans result))) + (t/is (= 0 (:processed result)))) + + ;; just delete the profile + (th/db-delete! :profile {:id (:id prof1)}) + + ;; query files after profile deletion, expecting not found + (let [params {::th/type :get-project-files + ::rpc/profile-id (:id prof1) + :project-id (:default-project-id prof1)} + out (th/command! params)] + ;; (th/print-result! out) + (t/is (not (th/success? out))) + (let [edata (-> out :error ex-data)] + (t/is (= :not-found (:type edata))))) + + ;; the files and projects still exists on the database + (let [files (th/db-query :file {:project-id (:default-project-id prof1)}) + projects (th/db-query :project {:team-id (:default-team-id prof1)})] + (t/is (= 1 (count files))) + (t/is (= 1 (count projects)))) + + ;; execute the gc task + (let [result (th/run-task! :objects-gc {:min-age (dt/duration "-1m")})] + (t/is (= 1 (:processed result))) + (t/is (= 1 (:orphans result)))) + + ;; Check the deletion flag on the default profile team + (let [row (th/db-get :team + {:id (:default-team-id prof1)} + {::db/remove-deleted? false})] + (t/is (dt/instant? (:deleted-at row)))) + + ;; Check the deletion flag on the shared team + (let [row (th/db-get :team + {:id (:id team)} + {::db/remove-deleted? false})] + (t/is (nil? (:deleted-at row)))) + + ;; Check the roles on the shared team + (let [rows (th/db-query :team-profile-rel {:team-id (:id team)})] + (t/is (= 1 (count rows))) + (t/is (= (:id prof2) (get-in rows [0 :profile-id]))) + (t/is (= false (get-in rows [0 :is-owner])))) + + )) + (t/deftest registration-domain-whitelist (let [whitelist #{"gmail.com" "hey.com" "ya.ru"}] (t/testing "allowed email domain"