From a599835e1f6b866bcdb5dd473ef6411a2357c1ad Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Sun, 31 Jan 2021 17:02:38 +0100 Subject: [PATCH] :tada: Add tests for storage module. --- backend/tests/app/tests/test_storage.clj | 273 +++++++++++++++++++++++ 1 file changed, 273 insertions(+) create mode 100644 backend/tests/app/tests/test_storage.clj diff --git a/backend/tests/app/tests/test_storage.clj b/backend/tests/app/tests/test_storage.clj new file mode 100644 index 000000000..f967c491d --- /dev/null +++ b/backend/tests/app/tests/test_storage.clj @@ -0,0 +1,273 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; This Source Code Form is "Incompatible With Secondary Licenses", as +;; defined by the Mozilla Public License, v. 2.0. +;; +;; Copyright (c) 2020 UXBOX Labs SL + +(ns app.tests.test-storage + (:require + [app.common.exceptions :as ex] + [app.db :as db] + [app.storage :as sto] + [app.tests.helpers :as th] + [app.util.time :as dt] + [clojure.java.io :as io] + [clojure.test :as t] + [cuerdas.core :as str] + [datoteka.core :as fs] + [mockery.core :refer [with-mocks]])) + +(t/use-fixtures :once th/state-init) +(t/use-fixtures :each (th/serial + th/database-reset + th/clean-storage)) + +;; TODO: add specific tests for DB backend. + +(t/deftest put-and-retrieve-object + (let [storage (:app.storage/storage th/*system*) + content (sto/content "content") + object (sto/put-object storage {:content content + :content-type "text/plain" + :other "data"})] + (t/is (sto/storage-object? object)) + (t/is (fs/path? (sto/get-object-path storage object))) + (t/is (nil? (:expired-at object))) + (t/is (= :tmp (:backend object))) + (t/is (= "data" (:other (meta object)))) + (t/is (= "text/plain" (:content-type (meta object)))) + (t/is (= "content" (slurp (sto/get-object-data storage object)))) + (t/is (= "content" (slurp (sto/get-object-path storage object)))) + )) + + +(t/deftest put-and-retrieve-expired-object + (let [storage (:app.storage/storage th/*system*) + content (sto/content "content") + object (sto/put-object storage {:content content + :content-type "text/plain" + :expired-at (dt/in-future {:seconds 1})})] + (t/is (sto/storage-object? object)) + (t/is (dt/instant? (:expired-at object))) + (t/is (dt/is-after? (:expired-at object) (dt/now))) + (t/is (= object (sto/get-object storage (:id object)))) + + (th/sleep 1000) + (t/is (nil? (sto/get-object storage (:id object)))) + (t/is (nil? (sto/get-object-data storage object))) + (t/is (nil? (sto/get-object-url storage object))) + (t/is (nil? (sto/get-object-path storage object))) + )) + +(t/deftest put-and-delete-object + (let [storage (:app.storage/storage th/*system*) + content (sto/content "content") + object (sto/put-object storage {:content content + :content-type "text/plain" + :expired-at (dt/in-future {:seconds 1})})] + (t/is (sto/storage-object? object)) + (t/is (true? (sto/del-object storage object))) + + ;; retrieving the same object should be not nil because the + ;; deletion is not inmediate + (t/is (some? (sto/get-object-data storage object))) + (t/is (some? (sto/get-object-url storage object))) + (t/is (some? (sto/get-object-path storage object))) + + ;; But you can't retrieve the object again because in database is + ;; marked as deleted/expired. + (t/is (nil? (sto/get-object storage (:id object)))) + )) + +(t/deftest test-deleted-gc-task + (let [storage (:app.storage/storage th/*system*) + content (sto/content "content") + object1 (sto/put-object storage {:content content + :content-type "text/plain" + :expired-at (dt/now)}) + object2 (sto/put-object storage {:content content + :content-type "text/plain" + :expired-at (dt/in-past {:hours 2})})] + (th/sleep 200) + + (let [task (:app.storage/gc-deleted-task th/*system*) + res (task {})] + (t/is (= 1 (:deleted res)))) + + (let [res (db/exec-one! th/*pool* ["select count(*) from storage_object;"])] + (t/is (= 1 (:count res)))))) + +(t/deftest test-touched-gc-task + (let [storage (:app.storage/storage th/*system*) + prof (th/create-profile* 1) + proj (th/create-project* 1 {:profile-id (:id prof) + :team-id (:default-team-id prof)}) + file (th/create-file* 1 {:profile-id (:id prof) + :project-id (:default-project-id prof) + :is-shared false}) + mfile {:filename "sample.jpg" + :tempfile (th/tempfile "app/tests/_files/sample.jpg") + :content-type "image/jpeg" + :size 312043} + + params {::th/type :upload-file-media-object + :profile-id (:id prof) + :file-id (:id file) + :is-local true + :name "testfile" + :content mfile} + + out1 (th/mutation! params) + out2 (th/mutation! params)] + + (t/is (nil? (:error out1))) + (t/is (nil? (:error out2))) + + (let [result-1 (:result out1) + result-2 (:result out2)] + + (t/is (uuid? (:id result-1))) + (t/is (uuid? (:id result-2))) + + (t/is (uuid? (:media-id result-1))) + (t/is (uuid? (:media-id result-2))) + + ;; now we proceed to manually delete one file-media-object + (db/exec-one! th/*pool* ["delete from file_media_object where id = ?" (:id result-1)]) + + ;; check that we still have all the storage objects + (let [res (db/exec-one! th/*pool* ["select count(*) from storage_object"])] + (t/is (= 4 (:count res)))) + + ;; now check if the storage objects are touched + (let [res (db/exec-one! th/*pool* ["select count(*) from storage_object where touched_at is not null"])] + (t/is (= 2 (:count res)))) + + ;; run the touched gc task + (let [task (:app.storage/gc-touched-task th/*system*) + res (task {})] + (t/is (= 0 (:freeze res))) + (t/is (= 2 (:delete res)))) + + ;; now check that there are no touched objects + (let [res (db/exec-one! th/*pool* ["select count(*) from storage_object where touched_at is not null"])] + (t/is (= 0 (:count res)))) + + ;; now check that all objects are marked to be deleted + (let [res (db/exec-one! th/*pool* ["select count(*) from storage_object where deleted_at is not null"])] + (t/is (= 2 (:count res)))) + ))) + +(t/deftest test-touched-gc-task-without-delete + (let [storage (:app.storage/storage th/*system*) + prof (th/create-profile* 1) + proj (th/create-project* 1 {:profile-id (:id prof) + :team-id (:default-team-id prof)}) + file (th/create-file* 1 {:profile-id (:id prof) + :project-id (:default-project-id prof) + :is-shared false}) + mfile {:filename "sample.jpg" + :tempfile (th/tempfile "app/tests/_files/sample.jpg") + :content-type "image/jpeg" + :size 312043} + + params {::th/type :upload-file-media-object + :profile-id (:id prof) + :file-id (:id file) + :is-local true + :name "testfile" + :content mfile} + + out1 (th/mutation! params) + out2 (th/mutation! params)] + + (t/is (nil? (:error out1))) + (t/is (nil? (:error out2))) + + (let [result-1 (:result out1) + result-2 (:result out2)] + + ;; now we proceed to manually mark all storage objects touched + (db/exec-one! th/*pool* ["update storage_object set touched_at=now()"]) + + ;; run the touched gc task + (let [task (:app.storage/gc-touched-task th/*system*) + res (task {})] + (t/is (= 4 (:freeze res))) + (t/is (= 0 (:delete res)))) + + ;; check that we have all object in the db + (let [res (db/exec-one! th/*pool* ["select count(*) from storage_object where deleted_at is null"])] + (t/is (= 4 (:count res))))))) + + +;; Recheck is the mechanism for delete leaked resources on +;; transaction failure. + +(t/deftest test-recheck + (let [storage (:app.storage/storage th/*system*) + content (sto/content "content") + object (sto/put-object storage {:content content + :content-type "text/plain"})] + ;; Sleep fo 50ms + (th/sleep 50) + + (let [rows (db/exec! th/*pool* ["select * from storage_pending"])] + (t/is (= 1 (count rows))) + (t/is (= (:id object) (:id (first rows))))) + + ;; Artificially make all storage_pending object 1 hour older. + (db/exec-one! th/*pool* ["update storage_pending set created_at = created_at - '1 hour'::interval"]) + + ;; Sleep fo 50ms + (th/sleep 50) + + ;; Run recheck task + (let [task (:app.storage/recheck-task th/*system*) + res (task {})] + (t/is (= 1 (:processed res))) + (t/is (= 0 (:deleted res)))) + + ;; After recheck task, storage-pending table should be empty + (let [rows (db/exec! th/*pool* ["select * from storage_pending"])] + (t/is (= 0 (count rows)))))) + +(t/deftest test-recheck-with-rollback + (let [storage (:app.storage/storage th/*system*) + content (sto/content "content")] + + ;; check with aborted transaction + (ex/ignoring + (db/with-atomic [conn th/*pool*] + (let [storage (assoc storage :conn conn)] ; make participate storage in the transaction + (sto/put-object storage {:content content + :content-type "text/plain"}) + (throw (ex-info "expected" {}))))) + + ;; let a 200ms window for recheck registration thread + ;; completion before proceed. + (th/sleep 200) + + ;; storage_pending table should have the object + ;; registred independently of the aborted transaction. + (let [rows (db/exec! th/*pool* ["select * from storage_pending"])] + (t/is (= 1 (count rows)))) + + ;; Artificially make all storage_pending object 1 hour older. + (db/exec-one! th/*pool* ["update storage_pending set created_at = created_at - '1 hour'::interval"]) + + ;; Sleep fo 50ms + (th/sleep 50) + + ;; Run recheck task + (let [task (:app.storage/recheck-task th/*system*) + res (task {})] + (t/is (= 1 (:processed res))) + (t/is (= 1 (:deleted res)))) + + ;; After recheck task, storage-pending table should be empty + (let [rows (db/exec! th/*pool* ["select * from storage_pending"])] + (t/is (= 0 (count rows))))))