From 860e0227af780736891f14d8d61098045d4e131e Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 7 Jun 2021 16:51:09 +0200 Subject: [PATCH] :recycle: Reimplement GC mechanism for penpot database objects. --- backend/src/app/main.clj | 15 +- backend/src/app/migrations.clj | 9 ++ .../0056-add-missing-index-on-deleted-at.sql | 15 ++ .../0057-del-profile-on-delete-trigger.sql | 2 + .../sql/0058-del-team-on-delete-trigger.sql | 2 + backend/src/app/rpc/mutations/demo.clj | 9 +- backend/src/app/rpc/mutations/files.clj | 10 -- backend/src/app/rpc/mutations/fonts.clj | 19 +-- backend/src/app/rpc/mutations/profile.clj | 17 +- backend/src/app/rpc/mutations/projects.clj | 10 -- backend/src/app/rpc/mutations/teams.clj | 9 -- backend/src/app/rpc/queries/profile.clj | 14 +- backend/src/app/tasks/delete_object.clj | 3 + backend/src/app/tasks/delete_profile.clj | 3 + backend/src/app/tasks/file_media_gc.clj | 1 + backend/src/app/tasks/objects_gc.clj | 152 ++++++++++++++++++ backend/test/app/services_files_test.clj | 67 ++++++++ backend/test/app/services_profile_test.clj | 37 ++--- backend/test/app/services_projects_test.clj | 72 ++++++++- backend/test/app/services_teams_test.clj | 75 +++++++++ 20 files changed, 437 insertions(+), 104 deletions(-) create mode 100644 backend/src/app/migrations/sql/0056-add-missing-index-on-deleted-at.sql create mode 100644 backend/src/app/migrations/sql/0057-del-profile-on-delete-trigger.sql create mode 100644 backend/src/app/migrations/sql/0058-del-team-on-delete-trigger.sql create mode 100644 backend/src/app/tasks/objects_gc.clj diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index 06765d5ff..6ba884197 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -172,18 +172,21 @@ {:cron #app/cron "0 0 * * * ?" ;; hourly :task :file-xlog-gc} - {:cron #app/cron "0 0 1 * * ?" ;; daily (1 hour shift) + {:cron #app/cron "0 0 0 * * ?" ;; daily :task :storage-deleted-gc} - {:cron #app/cron "0 0 2 * * ?" ;; daily (2 hour shift) + {:cron #app/cron "0 0 0 * * ?" ;; daily :task :storage-touched-gc} - {:cron #app/cron "0 0 3 * * ?" ;; daily (3 hour shift) + {:cron #app/cron "0 0 0 * * ?" ;; daily :task :session-gc} {:cron #app/cron "0 0 * * * ?" ;; hourly :task :storage-recheck} + {:cron #app/cron "0 0 0 * * ?" ;; daily + :task :objects-gc} + {:cron #app/cron "0 0 0 * * ?" ;; daily :task :tasks-gc} @@ -203,6 +206,7 @@ {:metrics (ig/ref :app.metrics/metrics) :tasks {:sendmail (ig/ref :app.emails/sendmail-handler) + :objects-gc (ig/ref :app.tasks.objects-gc/handler) :delete-object (ig/ref :app.tasks.delete-object/handler) :delete-profile (ig/ref :app.tasks.delete-profile/handler) :file-media-gc (ig/ref :app.tasks.file-media-gc/handler) @@ -236,6 +240,11 @@ {:pool (ig/ref :app.db/pool) :storage (ig/ref :app.storage/storage)} + :app.tasks.objects-gc/handler + {:pool (ig/ref :app.db/pool) + :storage (ig/ref :app.storage/storage) + :max-age cf/deletion-delay} + :app.tasks.delete-profile/handler {:pool (ig/ref :app.db/pool)} diff --git a/backend/src/app/migrations.clj b/backend/src/app/migrations.clj index 8e6350995..022abb69b 100644 --- a/backend/src/app/migrations.clj +++ b/backend/src/app/migrations.clj @@ -175,6 +175,15 @@ {:name "0055-mod-file-media-object-table" :fn (mg/resource "app/migrations/sql/0055-mod-file-media-object-table.sql")} + + {:name "0056-add-missing-index-on-deleted-at" + :fn (mg/resource "app/migrations/sql/0056-add-missing-index-on-deleted-at.sql")} + + {:name "0057-del-profile-on-delete-trigger" + :fn (mg/resource "app/migrations/sql/0057-del-profile-on-delete-trigger.sql")} + + {:name "0058-del-team-on-delete-trigger" + :fn (mg/resource "app/migrations/sql/0058-del-team-on-delete-trigger.sql")} ]) diff --git a/backend/src/app/migrations/sql/0056-add-missing-index-on-deleted-at.sql b/backend/src/app/migrations/sql/0056-add-missing-index-on-deleted-at.sql new file mode 100644 index 000000000..7a74deed2 --- /dev/null +++ b/backend/src/app/migrations/sql/0056-add-missing-index-on-deleted-at.sql @@ -0,0 +1,15 @@ +CREATE INDEX profile_deleted_at_idx + ON profile(deleted_at, id) + WHERE deleted_at IS NOT NULL; + +CREATE INDEX project_deleted_at_idx + ON project(deleted_at, id) + WHERE deleted_at IS NOT NULL; + +CREATE INDEX team_deleted_at_idx + ON team(deleted_at, id) + WHERE deleted_at IS NOT NULL; + +CREATE INDEX team_font_variant_deleted_at_idx + ON team_font_variant(deleted_at, id) + WHERE deleted_at IS NOT NULL; diff --git a/backend/src/app/migrations/sql/0057-del-profile-on-delete-trigger.sql b/backend/src/app/migrations/sql/0057-del-profile-on-delete-trigger.sql new file mode 100644 index 000000000..b748c3ce6 --- /dev/null +++ b/backend/src/app/migrations/sql/0057-del-profile-on-delete-trigger.sql @@ -0,0 +1,2 @@ +DROP TRIGGER profile__on_delete__tgr ON profile CASCADE; +DROP FUNCTION on_delete_profile (); diff --git a/backend/src/app/migrations/sql/0058-del-team-on-delete-trigger.sql b/backend/src/app/migrations/sql/0058-del-team-on-delete-trigger.sql new file mode 100644 index 000000000..d36c0e187 --- /dev/null +++ b/backend/src/app/migrations/sql/0058-del-team-on-delete-trigger.sql @@ -0,0 +1,2 @@ +DROP TRIGGER team__on_delete__tgr ON team CASCADE; +DROP FUNCTION on_delete_team (); diff --git a/backend/src/app/rpc/mutations/demo.clj b/backend/src/app/rpc/mutations/demo.clj index a74f8b4f8..fc7f184bf 100644 --- a/backend/src/app/rpc/mutations/demo.clj +++ b/backend/src/app/rpc/mutations/demo.clj @@ -15,7 +15,7 @@ [app.rpc.mutations.profile :as profile] [app.setup.initial-data :as sid] [app.util.services :as sv] - [app.worker :as wrk] + [app.util.time :as dt] [buddy.core.codecs :as bc] [buddy.core.nonce :as bn] [clojure.spec.alpha :as s])) @@ -35,6 +35,7 @@ :email email :fullname fullname :is-demo true + :deleted-at (dt/in-future cfg/deletion-delay) :password password :props {:onboarding-viewed true}}] @@ -48,12 +49,6 @@ (#'profile/create-profile-relations conn) (sid/load-initial-project! conn)) - ;; Schedule deletion of the demo profile - (wrk/submit! {::wrk/task :delete-profile - ::wrk/delay cfg/deletion-delay - ::wrk/conn conn - :profile-id id}) - (with-meta {:email email :password password} {::audit/profile-id id})))) diff --git a/backend/src/app/rpc/mutations/files.clj b/backend/src/app/rpc/mutations/files.clj index d90e453fb..02e8963a9 100644 --- a/backend/src/app/rpc/mutations/files.clj +++ b/backend/src/app/rpc/mutations/files.clj @@ -11,7 +11,6 @@ [app.common.pages.migrations :as pmg] [app.common.spec :as us] [app.common.uuid :as uuid] - [app.config :as cfg] [app.db :as db] [app.rpc.permissions :as perms] [app.rpc.queries.files :as files] @@ -19,7 +18,6 @@ [app.util.blob :as blob] [app.util.services :as sv] [app.util.time :as dt] - [app.worker :as wrk] [clojure.spec.alpha :as s])) ;; --- Helpers & Specs @@ -121,14 +119,6 @@ [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}] (db/with-atomic [conn pool] (files/check-edition-permissions! conn profile-id id) - - ;; Schedule object deletion - (wrk/submit! {::wrk/task :delete-object - ::wrk/delay cfg/deletion-delay - ::wrk/conn conn - :id id - :type :file}) - (mark-file-deleted conn params))) (defn mark-file-deleted diff --git a/backend/src/app/rpc/mutations/fonts.clj b/backend/src/app/rpc/mutations/fonts.clj index ca1d2263e..9037550f1 100644 --- a/backend/src/app/rpc/mutations/fonts.clj +++ b/backend/src/app/rpc/mutations/fonts.clj @@ -104,21 +104,10 @@ (db/with-atomic [conn pool] (teams/check-edition-permissions! conn profile-id team-id) - (let [items (db/query conn :team-font-variant - {:font-id id :team-id team-id} - {:for-update true})] - (doseq [item items] - ;; Schedule object deletion - (wrk/submit! {::wrk/task :delete-object - ::wrk/delay cf/deletion-delay - ::wrk/conn conn - :id (:id item) - :type :team-font-variant})) - - (db/update! conn :team-font-variant - {:deleted-at (dt/now)} - {:font-id id :team-id team-id}) - nil))) + (db/update! conn :team-font-variant + {:deleted-at (dt/now)} + {:font-id id :team-id team-id}) + nil)) ;; --- DELETE FONT VARIANT diff --git a/backend/src/app/rpc/mutations/profile.clj b/backend/src/app/rpc/mutations/profile.clj index 71c2a0d90..0cef24832 100644 --- a/backend/src/app/rpc/mutations/profile.clj +++ b/backend/src/app/rpc/mutations/profile.clj @@ -22,7 +22,6 @@ [app.storage :as sto] [app.util.services :as sv] [app.util.time :as dt] - [app.worker :as wrk] [buddy.hashers :as hashers] [clojure.spec.alpha :as s] [cuerdas.core :as str])) @@ -179,9 +178,9 @@ :valid false}))) (defn create-profile - "Create the profile entry on the database with limited input - filling all the other fields with defaults." - [conn {:keys [id fullname email password is-active is-muted is-demo opts] + "Create the profile entry on the database with limited input filling + all the other fields with defaults." + [conn {:keys [id fullname email password is-active is-muted is-demo opts deleted-at] :or {is-active false is-muted false is-demo false} :as params}] (let [id (or id (uuid/next)) @@ -193,6 +192,7 @@ :email (str/lower email) :auth-backend "penpot" :password password + :deleted-at deleted-at :props props :is-active is-active :is-muted is-muted @@ -264,7 +264,8 @@ (let [profile (->> (profile/retrieve-profile-data-by-email conn email) (validate-profile) (profile/strip-private-attrs) - (profile/populate-additional-data conn))] + (profile/populate-additional-data conn)) + profile (update profile :props db/decode-transit-pgobject)] (if-let [token (:invitation-token params)] ;; If the request comes with an invitation token, this means ;; that user wants to accept it with different user. A very @@ -619,12 +620,6 @@ (db/with-atomic [conn pool] (check-can-delete-profile! conn profile-id) - ;; Schedule a complete deletion of profile - (wrk/submit! {::wrk/task :delete-profile - ::wrk/delay cfg/deletion-delay - ::wrk/conn conn - :profile-id profile-id}) - (db/update! conn :profile {:deleted-at (dt/now)} {:id profile-id}) diff --git a/backend/src/app/rpc/mutations/projects.clj b/backend/src/app/rpc/mutations/projects.clj index bc25ef8ff..357b3841c 100644 --- a/backend/src/app/rpc/mutations/projects.clj +++ b/backend/src/app/rpc/mutations/projects.clj @@ -8,14 +8,12 @@ (:require [app.common.spec :as us] [app.common.uuid :as uuid] - [app.config :as cfg] [app.db :as db] [app.rpc.permissions :as perms] [app.rpc.queries.projects :as proj] [app.rpc.queries.teams :as teams] [app.util.services :as sv] [app.util.time :as dt] - [app.worker :as wrk] [clojure.spec.alpha :as s])) ;; --- Helpers & Specs @@ -123,14 +121,6 @@ [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}] (db/with-atomic [conn pool] (proj/check-edition-permissions! conn profile-id id) - - ;; Schedule object deletion - (wrk/submit! {::wrk/task :delete-object - ::wrk/delay cfg/deletion-delay - ::wrk/conn conn - :id id - :type :project}) - (db/update! conn :project {:deleted-at (dt/now)} {:id id}) diff --git a/backend/src/app/rpc/mutations/teams.clj b/backend/src/app/rpc/mutations/teams.clj index 662d2dc35..3b71c43c4 100644 --- a/backend/src/app/rpc/mutations/teams.clj +++ b/backend/src/app/rpc/mutations/teams.clj @@ -10,7 +10,6 @@ [app.common.exceptions :as ex] [app.common.spec :as us] [app.common.uuid :as uuid] - [app.config :as cfg] [app.db :as db] [app.emails :as eml] [app.media :as media] @@ -21,7 +20,6 @@ [app.storage :as sto] [app.util.services :as sv] [app.util.time :as dt] - [app.worker :as wrk] [clojure.spec.alpha :as s] [datoteka.core :as fs])) @@ -135,13 +133,6 @@ (ex/raise :type :validation :code :only-owner-can-delete-team)) - ;; Schedule object deletion - (wrk/submit! {::wrk/task :delete-object - ::wrk/delay cfg/deletion-delay - ::wrk/conn conn - :id id - :type :team}) - (db/update! conn :team {:deleted-at (dt/now)} {:id id}) diff --git a/backend/src/app/rpc/queries/profile.clj b/backend/src/app/rpc/queries/profile.clj index ee974c359..4508b740e 100644 --- a/backend/src/app/rpc/queries/profile.clj +++ b/backend/src/app/rpc/queries/profile.clj @@ -11,7 +11,8 @@ [app.common.uuid :as uuid] [app.db :as db] [app.util.services :as sv] - [clojure.spec.alpha :as s])) + [clojure.spec.alpha :as s] + [cuerdas.core :as str])) ;; --- Helpers & Specs @@ -90,16 +91,11 @@ profile)) -(def sql:retrieve-profile-by-email - "select p.* from profile as p - where p.email = lower(?) - and p.deleted_at is null") - (defn retrieve-profile-data-by-email [conn email] - (let [sql [sql:retrieve-profile-by-email email]] - (some-> (db/exec-one! conn sql) - (decode-profile-row)))) + (try + (db/get-by-params conn :profile {:email (str/lower email)}) + (catch Exception _e))) ;; --- Attrs Helpers diff --git a/backend/src/app/tasks/delete_object.clj b/backend/src/app/tasks/delete_object.clj index d2d9420e8..2a30a6324 100644 --- a/backend/src/app/tasks/delete_object.clj +++ b/backend/src/app/tasks/delete_object.clj @@ -4,6 +4,9 @@ ;; ;; Copyright (c) UXBOX Labs SL +;; TODO: DEPRECATED +;; Should be removed in the 1.8.x + (ns app.tasks.delete-object "Generic task for permanent deletion of objects." (:require diff --git a/backend/src/app/tasks/delete_profile.clj b/backend/src/app/tasks/delete_profile.clj index 17e2facb4..67a1733df 100644 --- a/backend/src/app/tasks/delete_profile.clj +++ b/backend/src/app/tasks/delete_profile.clj @@ -14,6 +14,9 @@ [clojure.spec.alpha :as s] [integrant.core :as ig])) +;; TODO: DEPRECATED +;; Should be removed in the 1.8.x + (declare delete-profile-data) ;; --- INIT diff --git a/backend/src/app/tasks/file_media_gc.clj b/backend/src/app/tasks/file_media_gc.clj index 8b8bc3d28..bc1675447 100644 --- a/backend/src/app/tasks/file_media_gc.clj +++ b/backend/src/app/tasks/file_media_gc.clj @@ -100,6 +100,7 @@ :id (:id mobj) :media-id (:media-id mobj) :thumbnail-id (:thumbnail-id mobj)) + ;; NOTE: deleting the file-media-object in the database ;; automatically marks as toched the referenced storage ;; objects. The touch mechanism is needed because many files can diff --git a/backend/src/app/tasks/objects_gc.clj b/backend/src/app/tasks/objects_gc.clj new file mode 100644 index 000000000..b146c85af --- /dev/null +++ b/backend/src/app/tasks/objects_gc.clj @@ -0,0 +1,152 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) UXBOX Labs SL + +(ns app.tasks.objects-gc + "A maintenance task that performs a general purpose garbage collection + of deleted objects." + (:require + [app.db :as db] + [app.storage :as sto] + [app.util.logging :as l] + [app.util.time :as dt] + [clojure.spec.alpha :as s] + [cuerdas.core :as str] + [integrant.core :as ig])) + +(def target-tables + ["profile" + "team" + "file" + "project" + "team_font_variant"]) + +(defmulti delete-objects :table) + +(def sql:delete-objects + "with deleted as ( + select id from %(table)s + where deleted_at is not null + and deleted_at < now() - ?::interval + order by deleted_at + limit %(limit)s + ) + delete from %(table)s + where id in (select id from deleted) + returning *") + +;; --- IMPL: generic object deletion + +(defmethod delete-objects :default + [{:keys [conn max-age table] :as cfg}] + (let [sql (str/fmt sql:delete-objects + {:table table :limit 50}) + result (db/exec! conn [sql max-age])] + + (doseq [{:keys [id] :as item} result] + (l/trace :action "delete object" :table table :id id)) + + (count result))) + +;; --- IMPL: team-font-variant deletion + +(defmethod delete-objects "team_font_variant" + [{:keys [conn max-age storage table] :as cfg}] + (let [sql (str/fmt sql:delete-objects + {:table table :limit 50}) + fonts (db/exec! conn [sql max-age]) + storage (assoc storage :conn conn)] + (doseq [{:keys [id] :as font} fonts] + (l/trace :action "delete object" :table table :id id) + (some->> (:woff1-file-id font) (sto/del-object storage)) + (some->> (:woff2-file-id font) (sto/del-object storage)) + (some->> (:otf-file-id font) (sto/del-object storage)) + (some->> (:ttf-file-id font) (sto/del-object storage))) + (count fonts))) + +;; --- IMPL: team deletion + +(defmethod delete-objects "team" + [{:keys [conn max-age storage table] :as cfg}] + (let [sql (str/fmt sql:delete-objects + {:table table :limit 50}) + teams (db/exec! conn [sql max-age]) + storage (assoc storage :conn conn)] + + (doseq [{:keys [id] :as team} teams] + (l/trace :action "delete object" :table table :id id) + (some->> (:photo-id team) (sto/del-object storage))) + + (count teams))) + +;; --- IMPL: profile deletion + +(def sql:retrieve-deleted-profiles + "select id, photo_id from profile + where deleted_at is not null + and deleted_at < now() - ?::interval + order by deleted_at + limit %(limit)s + for update") + +(def sql:mark-owned-teams-deleted + "with owned as ( + select tpr.team_id as id + from team_profile_rel as tpr + where tpr.is_owner is true + and tpr.profile_id = ? + ) + update team set deleted_at = now() - ?::interval + where id in (select id from owned)") + +(defmethod delete-objects "profile" + [{:keys [conn max-age storage table] :as cfg}] + (let [sql (str/fmt sql:retrieve-deleted-profiles {:limit 50}) + profiles (db/exec! conn [sql max-age]) + storage (assoc storage :conn conn)] + + (doseq [{:keys [id] :as profile} profiles] + (l/trace :action "delete object" :table table :id id) + + ;; Mark the owned teams as deleted; this enables them to be procesed + ;; in the same transaction in the "team" table step. + (db/exec-one! conn [sql:mark-owned-teams-deleted id max-age]) + + ;; Mark as deleted the storage object related with the photo-id + ;; field. + (some->> (:photo-id profile) (sto/del-object storage)) + + ;; And finally, permanently delete the profile. + (db/delete! conn :profile {:id id})) + + (count profiles))) + +;; --- INIT + +(defn- process-table + [{:keys [table] :as cfg}] + (loop [n 0] + (let [res (delete-objects cfg)] + (if (pos? res) + (recur (+ n res)) + (l/debug :hint "table gc summary" :table table :deleted n))))) + +(s/def ::max-age ::dt/duration) + +(defmethod ig/pre-init-spec ::handler [_] + (s/keys :req-un [::db/pool ::sto/storage ::max-age])) + +(defmethod ig/init-key ::handler + [_ {:keys [pool max-age] :as cfg}] + (fn [task] + ;; Checking first on task argument allows properly testing it. + (let [max-age (get task :max-age max-age)] + (db/with-atomic [conn pool] + (let [max-age (db/interval max-age) + cfg (-> cfg + (assoc :max-age max-age) + (assoc :conn conn))] + (doseq [table target-tables] + (process-table (assoc cfg :table table)))))))) diff --git a/backend/test/app/services_files_test.clj b/backend/test/app/services_files_test.clj index b9a1c4426..e22d8afc8 100644 --- a/backend/test/app/services_files_test.clj +++ b/backend/test/app/services_files_test.clj @@ -11,6 +11,7 @@ [app.http :as http] [app.storage :as sto] [app.test-helpers :as th] + [app.util.time :as dt] [clojure.test :as t] [datoteka.core :as fs])) @@ -337,3 +338,69 @@ (t/is (th/ex-info? error)) (t/is (th/ex-of-type? error :not-found)))) +(t/deftest deletion-test + (let [task (:app.tasks.objects-gc/handler th/*system*) + profile1 (th/create-profile* 1) + file (th/create-file* 1 {:project-id (:default-project-id profile1) + :profile-id (:id profile1)})] + ;; file is not deleted because it does not meet all + ;; conditions to be deleted. + (let [result (task {:max-age (dt/duration 0)})] + (t/is (nil? result))) + + ;; query the list of files + (let [data {::th/type :project-files + :project-id (:default-project-id profile1) + :profile-id (:id profile1)} + out (th/query! data)] + ;; (th/print-result! out) + (t/is (nil? (:error out))) + (let [result (:result out)] + (t/is (= 1 (count result))))) + + ;; Request file to be deleted + (let [params {::th/type :delete-file + :id (:id file) + :profile-id (:id profile1)} + out (th/mutation! params)] + (t/is (nil? (:error out)))) + + ;; query the list of files after soft deletion + (let [data {::th/type :project-files + :project-id (:default-project-id profile1) + :profile-id (:id profile1)} + out (th/query! data)] + ;; (th/print-result! out) + (t/is (nil? (:error out))) + (let [result (:result out)] + (t/is (= 0 (count result))))) + + ;; run permanent deletion (should be noop) + (let [result (task {:max-age (dt/duration {:minutes 1})})] + (t/is (nil? result))) + + ;; query the list of file libraries of a after hard deletion + (let [data {::th/type :file-libraries + :file-id (:id file) + :profile-id (:id profile1)} + out (th/query! data)] + ;; (th/print-result! out) + (t/is (nil? (:error out))) + (let [result (:result out)] + (t/is (= 0 (count result))))) + + ;; run permanent deletion + (let [result (task {:max-age (dt/duration 0)})] + (t/is (nil? result))) + + ;; query the list of file libraries of a after hard deletion + (let [data {::th/type :file-libraries + :file-id (:id file) + :profile-id (:id profile1)} + out (th/query! data)] + ;; (th/print-result! out) + (let [error (:error out) + error-data (ex-data error)] + (t/is (th/ex-info? error)) + (t/is (= (:type error-data) :not-found)))) + )) diff --git a/backend/test/app/services_profile_test.clj b/backend/test/app/services_profile_test.clj index 3fc1ab3a6..0f64da41d 100644 --- a/backend/test/app/services_profile_test.clj +++ b/backend/test/app/services_profile_test.clj @@ -9,6 +9,7 @@ [app.db :as db] [app.rpc.mutations.profile :as profile] [app.test-helpers :as th] + [app.util.time :as dt] [clojure.java.io :as io] [clojure.test :as t] [cuerdas.core :as str] @@ -117,7 +118,7 @@ )) (t/deftest profile-deletion-simple - (let [task (:app.tasks.delete-profile/handler th/*system*) + (let [task (:app.tasks.objects-gc/handler th/*system*) prof (th/create-profile* 1) file (th/create-file* 1 {:profile-id (:id prof) :project-id (:default-project-id prof) @@ -125,23 +126,14 @@ ;; profile is not deleted because it does not meet all ;; conditions to be deleted. - (let [result (task {:props {:profile-id (:id prof)}})] + (let [result (task {:max-age (dt/duration 0)})] (t/is (nil? result))) ;; Request profile to be deleted - (with-mocks [mock {:target 'app.worker/submit! :return nil}] - (let [params {::th/type :delete-profile - :profile-id (:id prof)} - out (th/mutation! params)] - (t/is (nil? (:error out))) - - ;; check the mock - (let [mock (deref mock) - mock-params (first (:call-args mock))] - (t/is (:called? mock)) - (t/is (= 1 (:call-count mock))) - (t/is (= :delete-profile (:app.worker/task mock-params))) - (t/is (= (:id prof) (:profile-id mock-params)))))) + (let [params {::th/type :delete-profile + :profile-id (:id prof)} + out (th/mutation! params)] + (t/is (nil? (:error out)))) ;; query files after profile soft deletion (let [params {::th/type :files @@ -153,8 +145,8 @@ (t/is (= 1 (count (:result out))))) ;; execute permanent deletion task - (let [result (task {:props {:profile-id (:id prof)}})] - (t/is (true? result))) + (let [result (task {:max-age (dt/duration "-1m")})] + (t/is (nil? result))) ;; query profile after delete (let [params {::th/type :profile @@ -165,17 +157,6 @@ error-data (ex-data error)] (t/is (th/ex-info? error)) (t/is (= (:type error-data) :not-found)))) - - ;; query files after profile soft deletion - (let [params {::th/type :files - :project-id (:default-project-id prof) - :profile-id (:id prof)} - out (th/query! params)] - ;; (th/print-result! out) - (let [error (:error out) - error-data (ex-data error)] - (t/is (th/ex-info? error)) - (t/is (= (:type error-data) :not-found)))) )) (t/deftest registration-domain-whitelist diff --git a/backend/test/app/services_projects_test.clj b/backend/test/app/services_projects_test.clj index 6137bb0ff..5f23577f3 100644 --- a/backend/test/app/services_projects_test.clj +++ b/backend/test/app/services_projects_test.clj @@ -10,8 +10,8 @@ [app.db :as db] [app.http :as http] [app.test-helpers :as th] - [clojure.test :as t] - [promesa.core :as p])) + [app.util.time :as dt] + [clojure.test :as t])) (t/use-fixtures :once th/state-init) (t/use-fixtures :each th/database-reset) @@ -170,3 +170,71 @@ (t/is (th/ex-info? error)) (t/is (th/ex-of-type? error :not-found)))) + +(t/deftest test-deletion + (let [task (:app.tasks.objects-gc/handler th/*system*) + profile1 (th/create-profile* 1) + project (th/create-project* 1 {:team-id (:default-team-id profile1) + :profile-id (:id profile1)})] + + ;; project is not deleted because it does not meet all + ;; conditions to be deleted. + (let [result (task {:max-age (dt/duration 0)})] + (t/is (nil? result))) + + ;; query the list of projects + (let [data {::th/type :projects + :team-id (:default-team-id profile1) + :profile-id (:id profile1)} + out (th/query! data)] + ;; (th/print-result! out) + (t/is (nil? (:error out))) + (let [result (:result out)] + (t/is (= 2 (count result))))) + + ;; Request project to be deleted + (let [params {::th/type :delete-project + :id (:id project) + :profile-id (:id profile1)} + out (th/mutation! params)] + (t/is (nil? (:error out)))) + + ;; query the list of projects after soft deletion + (let [data {::th/type :projects + :team-id (:default-team-id profile1) + :profile-id (:id profile1)} + out (th/query! data)] + ;; (th/print-result! out) + (t/is (nil? (:error out))) + (let [result (:result out)] + (t/is (= 1 (count result))))) + + ;; run permanent deletion (should be noop) + (let [result (task {:max-age (dt/duration {:minutes 1})})] + (t/is (nil? result))) + + ;; query the list of files of a after soft deletion + (let [data {::th/type :project-files + :project-id (:id project) + :profile-id (:id profile1)} + out (th/query! data)] + ;; (th/print-result! out) + (t/is (nil? (:error out))) + (let [result (:result out)] + (t/is (= 0 (count result))))) + + ;; run permanent deletion + (let [result (task {:max-age (dt/duration 0)})] + (t/is (nil? result))) + + ;; query the list of files of a after hard deletion + (let [data {::th/type :project-files + :project-id (:id project) + :profile-id (:id profile1)} + out (th/query! data)] + ;; (th/print-result! out) + (let [error (:error out) + error-data (ex-data error)] + (t/is (th/ex-info? error)) + (t/is (= (:type error-data) :not-found)))) + )) diff --git a/backend/test/app/services_teams_test.clj b/backend/test/app/services_teams_test.clj index 144d652f8..2831ea4ee 100644 --- a/backend/test/app/services_teams_test.clj +++ b/backend/test/app/services_teams_test.clj @@ -11,6 +11,7 @@ [app.http :as http] [app.storage :as sto] [app.test-helpers :as th] + [app.util.time :as dt] [clojure.test :as t] [datoteka.core :as fs] [mockery.core :refer [with-mocks]])) @@ -80,6 +81,80 @@ ))) +(t/deftest test-deletion + (let [task (:app.tasks.objects-gc/handler th/*system*) + profile1 (th/create-profile* 1 {:is-active true}) + team (th/create-team* 1 {:profile-id (:id profile1)}) + pool (:app.db/pool th/*system*) + data {::th/type :delete-team + :team-id (:id team) + :profile-id (:id profile1)}] + + ;; team is not deleted because it does not meet all + ;; conditions to be deleted. + (let [result (task {:max-age (dt/duration 0)})] + (t/is (nil? result))) + + ;; query the list of teams + (let [data {::th/type :teams + :profile-id (:id profile1)} + out (th/query! data)] + ;; (th/print-result! out) + (t/is (nil? (:error out))) + (let [result (:result out)] + (t/is (= 2 (count result))) + (t/is (= (:id team) (get-in result [1 :id]))) + (t/is (= (:default-team-id profile1) (get-in result [0 :id]))))) + + ;; Request team to be deleted + (let [params {::th/type :delete-team + :id (:id team) + :profile-id (:id profile1)} + out (th/mutation! params)] + (t/is (nil? (:error out)))) + + ;; query the list of teams after soft deletion + (let [data {::th/type :teams + :profile-id (:id profile1)} + out (th/query! data)] + ;; (th/print-result! out) + (t/is (nil? (:error out))) + (let [result (:result out)] + (t/is (= 1 (count result))) + (t/is (= (:default-team-id profile1) (get-in result [0 :id]))))) + + ;; run permanent deletion (should be noop) + (let [result (task {:max-age (dt/duration {:minutes 1})})] + (t/is (nil? result))) + + ;; query the list of projects of a after hard deletion + (let [data {::th/type :projects + :team-id (:id team) + :profile-id (:id profile1)} + out (th/query! data)] + ;; (th/print-result! out) + + (t/is (nil? (:error out))) + (let [result (:result out)] + (t/is (= 0 (count result))))) + + ;; run permanent deletion + (let [result (task {:max-age (dt/duration 0)})] + (t/is (nil? result))) + + ;; query the list of projects of a after hard deletion + (let [data {::th/type :projects + :team-id (:id team) + :profile-id (:id profile1)} + out (th/query! data)] + ;; (th/print-result! out) + (let [error (:error out) + error-data (ex-data error)] + (t/is (th/ex-info? error)) + (t/is (= (:type error-data) :not-found)))) + )) + +