Penpot/penpot
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-02-22 23:06:08 -05:00
Code Issues Activity

✨ Fix all tasks related tests

Browse source
This commit is contained in:
Andrey Antukh 2022-08-12 08:34:23 +02:00
parent 8d7baa75de
commit 95bb3f31af
7 changed files with 74 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
backend/resources/log4j2-devenv.xml
View file

@ -25,7 +25,7 @@
<Logger name="org.postgresql" level="error" />
<Logger name="app.rpc.commands.binfile" level="debug" />
<Logger name="app.storage.tmp" level="trace" />
<Logger name="app.storage.tmp" level="info" />
<Logger name="app.worker" level="info" />
<Logger name="app.msgbus" level="info" />
<Logger name="app.http.websocket" level="info" />

8
backend/src/app/main.clj
View file

@ -253,10 +253,10 @@
:task :file-xlog-gc}
{:cron #app/cron "0 0 0 * * ?" ;; daily
:task :storage-deleted-gc}
:task :storage-gc-deleted}
{:cron #app/cron "0 0 0 * * ?" ;; daily
:task :storage-touched-gc}
:task :storage-gc-touched}
{:cron #app/cron "0 0 0 * * ?" ;; daily
:task :session-gc}
@ -285,8 +285,8 @@
:objects-gc (ig/ref :app.tasks.objects-gc/handler)
:file-gc (ig/ref :app.tasks.file-gc/handler)
:file-xlog-gc (ig/ref :app.tasks.file-xlog-gc/handler)
:storage-deleted-gc (ig/ref :app.storage/gc-deleted-task)
:storage-touched-gc (ig/ref :app.storage/gc-touched-task)
:storage-gc-deleted (ig/ref :app.storage/gc-deleted-task)
:storage-gc-touched (ig/ref :app.storage/gc-touched-task)
:tasks-gc (ig/ref :app.tasks.tasks-gc/handler)
:telemetry (ig/ref :app.tasks.telemetry/handler)
:session-gc (ig/ref :app.http.session/gc-task)

32
backend/src/app/tasks/file_gc.clj
View file

@ -43,24 +43,24 @@
(defmethod ig/init-key ::handler
[_ {:keys [pool] :as cfg}]
(fn [params]
(let [cfg (merge cfg params)]
(db/with-atomic [conn pool]
(let [cfg (assoc cfg :conn conn)]
(loop [total 0
files (retrieve-candidates cfg)]
(if-let [file (first files)]
(do
(process-file cfg file)
(recur (inc total)
(rest files)))
(do
(l/info :hint "task finished" :total total)
(db/with-atomic [conn pool]
(let [min-age (or (:min-age params) (:min-age cfg))
cfg (assoc cfg :min-age min-age :conn conn)]
(loop [total 0
files (retrieve-candidates cfg)]
(if-let [file (first files)]
(do
(process-file cfg file)
(recur (inc total)
(rest files)))
(do
(l/info :hint "task finished" :min-age (dt/format-duration min-age) :total total)
;; Allow optional rollback passed by params
(when (:rollback? params)
(db/rollback! conn))
;; Allow optional rollback passed by params
(when (:rollback? params)
(db/rollback! conn))
{:processed total}))))))))
{:processed total})))))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; IMPL

65
backend/test/app/services_files_test.clj
View file

@ -187,23 +187,18 @@
;; freeze because of the deduplication (we have uploaded 2 times
;; 2 two same files).
(let [task (:app.storage/gc-touched-task th/*system*)
res (task {})]
res (task {:min-age (dt/duration 0)})]
(t/is (= 2 (:freeze res)))
(t/is (= 0 (:delete res))))
;; run the task immediately
;; run the file-gc task immediately without forced min-age
(let [task (:app.tasks.file-gc/handler th/*system*)
res (task {})]
(t/is (= 0 (:processed res))))
;; make the file eligible for GC waiting 300ms (configured
;; timeout for testing)
(th/sleep 300)
;; run the task again
(let [task (:app.tasks.file-gc/handler th/*system*)
res (task {})]
res (task {:min-age (dt/duration 0)})]
(t/is (= 1 (:processed res))))
;; retrieve file and check trimmed attribute
@ -220,22 +215,36 @@
(t/is (some? @(sto/get-object storage (:media-id fmo1))))
(t/is (some? @(sto/get-object storage (:thumbnail-id fmo1))))
;; now, we have deleted the unused file-media-object, if we
;; execute the touched-gc task, we should see that two of them
;; are marked to be deleted.
;; proceed to remove usage of the file
(update-file {:file-id (:id file)
:profile-id (:id profile)
:revn 0
:changes [{:type :del-obj
:page-id (first (get-in file [:data :pages]))
:id shid}]})
;; Now, we have deleted the usag of pointers to the
;; file-media-objects, if we pase file-gc, they should be marked
;; as deleted.
(let [task (:app.tasks.file-gc/handler th/*system*)
res (task {:min-age (dt/duration 0)})]
(t/is (= 1 (:processed res))))
;; Now that file-gc have deleted the file-media-object usage,
;; lets execute the touched-gc task, we should see that two of
;; them are marked to be deleted.
(let [task (:app.storage/gc-touched-task th/*system*)
res (task {})]
(t/is (= 2 (:freeze res)))
(t/is (= 0 (:delete res))))
res (task {:min-age (dt/duration 0)})]
(t/is (= 0 (:freeze res)))
(t/is (= 2 (:delete res))))
;; Finally, check that some of the objects that are marked as
;; deleted we are unable to retrieve them using standard storage
;; public api.
(t/is (some? @(sto/get-object storage (:media-id fmo2))))
(t/is (some? @(sto/get-object storage (:thumbnail-id fmo2))))
(t/is (some? @(sto/get-object storage (:media-id fmo1))))
(t/is (some? @(sto/get-object storage (:thumbnail-id fmo1))))
(t/is (nil? @(sto/get-object storage (:media-id fmo2))))
(t/is (nil? @(sto/get-object storage (:thumbnail-id fmo2))))
(t/is (nil? @(sto/get-object storage (:media-id fmo1))))
(t/is (nil? @(sto/get-object storage (:thumbnail-id fmo1))))
)))
(t/deftest permissions-checks-creating-file
@ -353,8 +362,8 @@
:profile-id (:id profile1)})]
;; file is not deleted because it does not meet all
;; conditions to be deleted.
(let [result (task {:max-age (dt/duration 0)})]
(t/is (nil? result)))
(let [result (task {:min-age (dt/duration 0)})]
(t/is (= 0 (:processed result))))
;; query the list of files
(let [data {::th/type :project-files
@ -384,8 +393,8 @@
(t/is (= 0 (count result)))))
;; run permanent deletion (should be noop)
(let [result (task {:max-age (dt/duration {:minutes 1})})]
(t/is (nil? result)))
(let [result (task {:min-age (dt/duration {:minutes 1})})]
(t/is (= 0 (:processed result))))
;; query the list of file libraries of a after hard deletion
(let [data {::th/type :file-libraries
@ -398,8 +407,8 @@
(t/is (= 0 (count result)))))
;; run permanent deletion
(let [result (task {:max-age (dt/duration 0)})]
(t/is (nil? result)))
(let [result (task {:min-age (dt/duration 0)})]
(t/is (= 1 (:processed result))))
;; query the list of file libraries of a after hard deletion
(let [data {::th/type :file-libraries
@ -590,7 +599,7 @@
;; run the task again
(let [task (:app.tasks.file-gc/handler th/*system*)
res (task {})]
res (task {:min-age (dt/duration 0)})]
(t/is (= 1 (:processed res))))
;; check that object thumbnails are still here
@ -617,7 +626,7 @@
;; run the task again
(let [task (:app.tasks.file-gc/handler th/*system*)
res (task {})]
res (task {:min-age (dt/duration 0)})]
(t/is (= 1 (:processed res))))
;; check that the unknown frame thumbnail is deleted
@ -701,7 +710,7 @@
;; run the task again
(let [task (:app.tasks.file-gc/handler th/*system*)
res (task {})]
res (task {:min-age (dt/duration 0)})]
(t/is (= 1 (:processed res))))
;; Then query the specific revn

8
backend/test/app/services_profile_test.clj
View file

@ -126,8 +126,8 @@
;; profile is not deleted because it does not meet all
;; conditions to be deleted.
(let [result (task {:max-age (dt/duration 0)})]
(t/is (nil? result)))
(let [result (task {:min-age (dt/duration 0)})]
(t/is (= 0 (:processed result))))
;; Request profile to be deleted
(let [params {::th/type :delete-profile
@ -145,8 +145,8 @@
(t/is (= 1 (count (:result out)))))
;; execute permanent deletion task
(let [result (task {:max-age (dt/duration "-1m")})]
(t/is (nil? result)))
(let [result (task {:min-age (dt/duration "-1m")})]
(t/is (= 1 (:processed result))))
;; query profile after delete
(let [params {::th/type :profile

12
backend/test/app/services_projects_test.clj
View file

@ -179,8 +179,8 @@
;; project is not deleted because it does not meet all
;; conditions to be deleted.
(let [result (task {:max-age (dt/duration 0)})]
(t/is (nil? result)))
(let [result (task {:min-age (dt/duration 0)})]
(t/is (= 0 (:processed result))))
;; query the list of projects
(let [data {::th/type :projects
@ -210,8 +210,8 @@
(t/is (= 1 (count result)))))
;; run permanent deletion (should be noop)
(let [result (task {:max-age (dt/duration {:minutes 1})})]
(t/is (nil? result)))
(let [result (task {:min-age (dt/duration {:minutes 1})})]
(t/is (= 0 (:processed result))))
;; query the list of files of a after soft deletion
(let [data {::th/type :project-files
@ -224,8 +224,8 @@
(t/is (= 0 (count result)))))
;; run permanent deletion
(let [result (task {:max-age (dt/duration 0)})]
(t/is (nil? result)))
(let [result (task {:min-age (dt/duration 0)})]
(t/is (= 1 (:processed result))))
;; query the list of files of a after hard deletion
(let [data {::th/type :project-files

12
backend/test/app/services_teams_test.clj
View file

@ -99,8 +99,8 @@
;; team is not deleted because it does not meet all
;; conditions to be deleted.
(let [result (task {:max-age (dt/duration 0)})]
(t/is (nil? result)))
(let [result (task {:min-age (dt/duration 0)})]
(t/is (= 0 (:processed result))))
;; query the list of teams
(let [data {::th/type :teams
@ -132,8 +132,8 @@
(t/is (= (:default-team-id profile1) (get-in result [0 :id])))))
;; run permanent deletion (should be noop)
(let [result (task {:max-age (dt/duration {:minutes 1})})]
(t/is (nil? result)))
(let [result (task {:min-age (dt/duration {:minutes 1})})]
(t/is (= 0 (:processed result))))
;; query the list of projects after hard deletion
(let [data {::th/type :projects
@ -147,8 +147,8 @@
(t/is (= (:type error-data) :not-found))))
;; run permanent deletion
(let [result (task {:max-age (dt/duration 0)})]
(t/is (nil? result)))
(let [result (task {:min-age (dt/duration 0)})]
(t/is (= 1 (:processed result))))
;; query the list of projects of a after hard deletion
(let [data {::th/type :projects