penpot/backend/test/backend_tests/rpc_file_thumbnails_test.clj

;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC

(ns backend-tests.rpc-file-thumbnails-test
  (:require
   [app.common.pprint :as pp]
   [app.common.thumbnails :as thc]
   [app.common.types.shape :as cts]
   [app.common.uuid :as uuid]
   [app.config :as cf]
   [app.db :as db]
   [app.rpc :as-alias rpc]
   [app.rpc.commands.auth :as cauth]
   [app.storage :as sto]
   [app.tokens :as tokens]
   [app.util.time :as dt]
   [backend-tests.helpers :as th]
   [clojure.java.io :as io]
   [clojure.test :as t]
   [cuerdas.core :as str]
   [datoteka.fs :as fs]
   [mockery.core :refer [with-mocks]]))

(t/use-fixtures :once th/state-init)
(t/use-fixtures :each th/database-reset)

(t/deftest upsert-file-object-thumbnail
  (let [storage (::sto/storage th/*system*)
        profile (th/create-profile* 1)
        file    (th/create-file* 1 {:profile-id (:id profile)
                                    :project-id (:default-project-id profile)
                                    :is-shared false})

        shid    (uuid/random)
        page-id (first (get-in file [:data :pages]))

        ;; Update file inserting a new frame object
        _       (th/update-file!
                 :file-id (:id file)
                 :profile-id (:id profile)
                 :revn 0
                 :changes
                 [{:type :add-obj
                   :page-id page-id
                   :id shid
                   :parent-id uuid/zero
                   :frame-id uuid/zero
                   :components-v2 true
                   :obj (cts/setup-shape
                         {:id shid
                          :name "Artboard"
                          :frame-id uuid/zero
                          :parent-id uuid/zero
                          :type :frame})}])

        data1   {::th/type :create-file-object-thumbnail
                 ::rpc/profile-id (:id profile)
                 :file-id (:id file)
                 :object-id "test-key-1"
                 :media {:filename "sample.jpg"
                         :size 312043
                         :path (th/tempfile "backend_tests/test_files/sample.jpg")
                         :mtype "image/jpeg"}}

        data2   {::th/type :create-file-object-thumbnail
                 ::rpc/profile-id (:id profile)
                 :file-id (:id file)
                 :object-id (thc/fmt-object-id (:id file) page-id shid "frame")
                 :media {:filename "sample.jpg"
                         :size 7923
                         :path (th/tempfile "backend_tests/test_files/sample2.jpg")
                         :mtype "image/jpeg"}}]

    (let [out (th/command! data1)]
      (t/is (nil? (:error out)))
      (t/is (map? (:result out))))

    (let [out (th/command! data2)]
      (t/is (nil? (:error out)))
      (t/is (map? (:result out))))

    ;; run the task again
    (let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
      (t/is (= 2 (:freeze res))))

    (let [[row1 row2 :as rows] (th/db-query :file-tagged-object-thumbnail
                                            {:file-id (:id file)}
                                            {:order-by [[:created-at :asc]]})]

      (t/is (= 2 (count rows)))
      (t/is (= (:file-id data1) (:file-id row1)))
      (t/is (= (:object-id data1) (:object-id row1)))
      (t/is (uuid? (:media-id row1)))
      (t/is (= (:file-id data2) (:file-id row2)))
      (t/is (= (:object-id data2) (:object-id row2)))
      (t/is (uuid? (:media-id row2)))

      (let [sobject (sto/get-object storage (:media-id row1))
            mobject (meta sobject)]
        (t/is (= "blake2b:4fdb63b8f3ffc81256ea79f13e53f366723b188554b5afed91b20897c14a1a8e" (:hash mobject)))
        (t/is (= "file-object-thumbnail" (:bucket mobject)))
        (t/is (= "image/jpeg" (:content-type mobject)))
        (t/is (= 312043 (:size sobject))))

      (let [sobject (sto/get-object storage (:media-id row2))
            mobject (meta sobject)]
        (t/is (= "blake2b:05870e3f8ee885841ee3799924d80805179ab57e6fde84a605d1068fd3138de9" (:hash mobject)))
        (t/is (= "file-object-thumbnail" (:bucket mobject)))
        (t/is (= "image/jpeg" (:content-type mobject)))
        (t/is (= 7923 (:size sobject))))

      ;; Run the File GC task that should remove unused file object
      ;; thumbnails
      (let [result (th/run-task! :file-gc {:min-age 0})]
        (t/is (= 1 (:processed result))))

      (let [result (th/run-task! :objects-gc {:min-age 0})]
        (t/is (= 2 (:processed result))))

      ;; check if row2 related thumbnail row still exists
      (let [[row :as rows] (th/db-query :file-tagged-object-thumbnail
                                        {:file-id (:id file)}
                                        {:order-by [[:created-at :asc]]})]
        (t/is (= 1 (count rows)))
        (t/is (= (:file-id data2) (:file-id row)))
        (t/is (= (:object-id data2) (:object-id row)))
        (t/is (uuid? (:media-id row2))))

      ;; Check if storage objects still exists after file-gc
      (t/is (some? (sto/get-object storage (:media-id row1))))
      (t/is (some? (sto/get-object storage (:media-id row2))))

      ;; run the task again
      (let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
        (t/is (= 1 (:delete res)))
        (t/is (= 0 (:freeze res))))

      ;; check that storage object is still exists but is marked as deleted
      (let [row (th/db-get :storage-object {:id (:media-id row1)} {::db/remove-deleted false})]
        (t/is (some? (:deleted-at row))))

      ;; Run the storage gc deleted task, it should permanently delete
      ;; all storage objects related to the deleted thumbnails
      (let [result (th/run-task! :storage-gc-deleted {:min-age 0})]
        (t/is (= 1 (:deleted result))))

      (t/is (nil? (sto/get-object storage (:media-id row1))))
      (t/is (some? (sto/get-object storage (:media-id row2))))

      ;; check that storage object is still exists but is marked as deleted
      (let [row (th/db-get :storage-object {:id (:media-id row1)} {::db/remove-deleted false})]
        (t/is (nil? row))))))

(t/deftest create-file-thumbnail
  (let [storage (::sto/storage th/*system*)
        profile (th/create-profile* 1)
        file    (th/create-file* 1 {:profile-id (:id profile)
                                    :project-id (:default-project-id profile)
                                    :is-shared false
                                    :revn 3})

        data1   {::th/type :create-file-thumbnail
                 ::rpc/profile-id (:id profile)
                 :file-id (:id file)
                 :revn 2
                 :media {:filename "sample.jpg"
                         :size 7923
                         :path (th/tempfile "backend_tests/test_files/sample2.jpg")
                         :mtype "image/jpeg"}}

        data2   {::th/type :create-file-thumbnail
                 ::rpc/profile-id (:id profile)
                 :file-id (:id file)
                 :revn 3
                 :media {:filename "sample.jpg"
                         :size 312043
                         :path (th/tempfile "backend_tests/test_files/sample.jpg")
                         :mtype "image/jpeg"}}]

    (let [out (th/command! data1)]
      ;; (th/print-result! out)
      (t/is (nil? (:error out)))
      (t/is (contains? (:result out) :uri)))

    (let [out (th/command! data2)]
      (t/is (nil? (:error out)))
      (t/is (contains? (:result out) :uri)))

    (let [[row1 row2 :as rows] (th/db-query :file-thumbnail
                                            {:file-id (:id file)}
                                            {:order-by [[:revn :asc]]})]
      (t/is (= 2 (count rows)))

      (t/is (= (:file-id data1) (:file-id row1)))
      (t/is (= (:revn data1) (:revn row1)))
      (t/is (uuid? (:media-id row1)))
      (t/is (= (:file-id data2) (:file-id row2)))
      (t/is (= (:revn data2) (:revn row2)))
      (t/is (uuid? (:media-id row2)))

      (let [sobject (sto/get-object storage (:media-id row1))
            mobject (meta sobject)]
        (t/is (= "blake2b:05870e3f8ee885841ee3799924d80805179ab57e6fde84a605d1068fd3138de9" (:hash mobject)))
        (t/is (= "file-thumbnail" (:bucket mobject)))
        (t/is (= "image/jpeg" (:content-type mobject)))
        (t/is (= 7923 (:size sobject))))

      (let [sobject (sto/get-object storage (:media-id row2))
            mobject (meta sobject)]
        (t/is (= "blake2b:4fdb63b8f3ffc81256ea79f13e53f366723b188554b5afed91b20897c14a1a8e" (:hash mobject)))
        (t/is (= "file-thumbnail" (:bucket mobject)))
        (t/is (= "image/jpeg" (:content-type mobject)))
        (t/is (= 312043 (:size sobject))))

      ;; Run the File GC task that should remove unused file object
      ;; thumbnails
      (let [result (th/run-task! :file-gc {:min-age 0})]
        (t/is (= 1 (:processed result))))

      (let [result (th/run-task! :objects-gc {:min-age 0})]
        (t/is (= 2 (:processed result))))

      ;; check if row1 related thumbnail row still exists
      (let [[row :as rows] (th/db-query :file-thumbnail
                                        {:file-id (:id file)}
                                        {:order-by [[:created-at :asc]]})]
        (t/is (= 1 (count rows)))
        (t/is (= (:file-id data1) (:file-id row)))
        (t/is (= (:object-id data1) (:object-id row)))
        (t/is (uuid? (:media-id row1))))

      (let [result (th/run-task! :storage-gc-touched {:min-age 0})]
        (t/is (= 1 (:delete result))))

      ;; Check if storage objects still exists after file-gc
      (t/is (nil? (sto/get-object storage (:media-id row1))))
      (t/is (some? (sto/get-object storage (:media-id row2))))

      (let [row (th/db-get :storage-object {:id (:media-id row1)} {::db/remove-deleted false})]
        (t/is (some? (:deleted-at row))))

      ;; Run the storage gc deleted task, it should permanently delete
      ;; all storage objects related to the deleted thumbnails
      (let [result (th/run-task! :storage-gc-deleted {:min-age 0})]
        (t/is (= 1 (:deleted result))))

      (t/is (some? (sto/get-object storage (:media-id row2)))))))

(t/deftest error-on-direct-storage-obj-deletion
  (let [storage (::sto/storage th/*system*)
        profile (th/create-profile* 1)
        file    (th/create-file* 1 {:profile-id (:id profile)
                                    :project-id (:default-project-id profile)
                                    :is-shared false
                                    :revn 3})

        data1   {::th/type :create-file-thumbnail
                 ::rpc/profile-id (:id profile)
                 :file-id (:id file)
                 :revn 2
                 :media {:filename "sample.jpg"
                         :size 7923
                         :path (th/tempfile "backend_tests/test_files/sample2.jpg")
                         :mtype "image/jpeg"}}]

    (let [out (th/command! data1)]
      ;; (th/print-result! out)
      (t/is (nil? (:error out)))
      (t/is (contains? (:result out) :uri)))

    (let [[row1 :as rows] (th/db-query :file-thumbnail {:file-id (:id file)})]
      (t/is (= 1 (count rows)))

      (t/is (thrown? org.postgresql.util.PSQLException
                     (th/db-delete! :storage-object {:id (:media-id row1)}))))))

(t/deftest get-file-object-thumbnail
  (let [storage (::sto/storage th/*system*)
        profile (th/create-profile* 1)
        file    (th/create-file* 1 {:profile-id (:id profile)
                                    :project-id (:default-project-id profile)
                                    :is-shared false})

        data   {::th/type :create-file-object-thumbnail
                ::rpc/profile-id (:id profile)
                :file-id (:id file)
                :object-id "test-key-2"
                :media {:filename "sample.jpg"
                        :size 7923
                        :path (th/tempfile "backend_tests/test_files/sample2.jpg")
                        :mtype "image/jpeg"}}]

    (let [out (th/command! data)]
      (t/is (nil? (:error out)))
      (t/is (map? (:result out))))

    (let [[row :as rows] (th/db-query :file-tagged-object-thumbnail
                                      {:file-id (:id file)}
                                      {:order-by [[:created-at :asc]]})]
      (t/is (= 1 (count rows)))

      (t/is (= (:file-id data) (:file-id row)))
      (t/is (= (:object-id data) (:object-id row)))
      (t/is (uuid? (:media-id row))))

    (let [params {::th/type :get-file-object-thumbnails
                  ::rpc/profile-id (:id profile)
                  :file-id (:id file)}
          out    (th/command! params)]

      ;; (th/print-result! out)

      (let [result (:result out)]
        (t/is (contains? result "test-key-2"))))))

(t/deftest create-file-object-thumbnail
  (th/db-delete! :task {:name "object-update"})
  (let [storage (::sto/storage th/*system*)
        profile (th/create-profile* 1)
        file    (th/create-file* 1 {:profile-id (:id profile)
                                    :project-id (:default-project-id profile)
                                    :is-shared false})
        data    {::th/type :create-file-object-thumbnail
                 ::rpc/profile-id (:id profile)
                 :file-id (:id file)
                 :object-id "test-key-2"
                 :media {:filename "sample.jpg"
                         :mtype "image/jpeg"}}]

    (let [data (update data :media
                       (fn [media]
                         (-> media
                             (assoc :path (th/tempfile "backend_tests/test_files/sample2.jpg"))
                             (assoc :size 7923))))
          out  (th/command! data)]
      (t/is (nil? (:error out)))
      (t/is (map? (:result out))))

    (let [data (update data :media
                       (fn [media]
                         (-> media
                             (assoc :path (th/tempfile "backend_tests/test_files/sample.jpg"))
                             (assoc :size 312043))))
          out  (th/command! data)]
      (t/is (nil? (:error out)))
      (t/is (map? (:result out))))

    (let [[row1 :as rows]
          (->> (th/db-query :task {:name "object-update"})
               (map #(update % :props db/decode-transit-pgobject)))]

      ;; (app.common.pprint/pprint rows)
      (t/is (= 1 (count rows)))
      (t/is (> (inst-ms (dt/diff (:created-at row1) (:scheduled-at row1)))
               (inst-ms (dt/duration "4m")))))))