mirror of
https://github.com/penpot/penpot.git
synced 2025-01-06 14:50:20 -05:00
🎉 Add support for multipart upload of thumbnails
and improve the thumbnails storage to offloading it to the storage subsystem
This commit is contained in:
Andrey Antukh
parent
c2b6b40554
commit
e8ffcbae69
14 changed files with 771 additions and 289 deletions
|
|
@ -381,7 +381,8 @@
|
|||
::sto/storage (ig/ref ::sto/storage)}
|
||||
|
||||
:app.tasks.file-gc/handler
|
||||
{::db/pool (ig/ref ::db/pool)}
|
||||
{::db/pool (ig/ref ::db/pool)
|
||||
::sto/storage (ig/ref ::sto/storage)}
|
||||
|
||||
:app.tasks.file-xlog-gc/handler
|
||||
{::db/pool (ig/ref ::db/pool)}
|
||||
|
|
|
|||
|
|
@ -29,6 +29,9 @@
|
|||
org.im4java.core.IMOperation
|
||||
org.im4java.core.Info))
|
||||
|
||||
(def default-max-file-size
|
||||
(* 1024 1024 30)) ; 30 MiB
|
||||
|
||||
(s/def ::path fs/path?)
|
||||
(s/def ::filename string?)
|
||||
(s/def ::size integer?)
|
||||
|
|
@ -54,6 +57,16 @@
|
|||
|
||||
upload))
|
||||
|
||||
(defn validate-media-size!
|
||||
[upload]
|
||||
(when (> (:size upload) (cf/get :media-max-file-size default-max-file-size))
|
||||
(ex/raise :type :restriction
|
||||
:code :media-max-file-size-reached
|
||||
:hint (str/ffmt "the uploaded file size % is greater than the maximum %"
|
||||
(:size upload)
|
||||
default-max-file-size)))
|
||||
upload)
|
||||
|
||||
(defmulti process :cmd)
|
||||
(defmulti process-error class)
|
||||
|
||||
|
|
|
|||
|
|
@ -316,7 +316,15 @@
|
|||
:fn (mg/resource "app/migrations/sql/0101-mod-server-error-report-table.sql")}
|
||||
|
||||
{:name "0102-mod-access-token-table"
|
||||
:fn (mg/resource "app/migrations/sql/0102-mod-access-token-table.sql")}])
|
||||
:fn (mg/resource "app/migrations/sql/0102-mod-access-token-table.sql")}
|
||||
|
||||
{:name "0103-mod-file-object-thumbnail-table"
|
||||
:fn (mg/resource "app/migrations/sql/0103-mod-file-object-thumbnail-table.sql")}
|
||||
|
||||
{:name "0104-mod-file-thumbnail-table"
|
||||
:fn (mg/resource "app/migrations/sql/0104-mod-file-thumbnail-table.sql")}
|
||||
|
||||
])
|
||||
|
||||
(defn apply-migrations!
|
||||
[pool name migrations]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
ALTER TABLE file_object_thumbnail
|
||||
ADD COLUMN media_id uuid NULL REFERENCES storage_object(id) ON DELETE CASCADE DEFERRABLE;
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
ALTER TABLE file_thumbnail
|
||||
ADD COLUMN media_id uuid NULL REFERENCES storage_object(id) ON DELETE CASCADE DEFERRABLE;
|
||||
|
|
@ -170,6 +170,7 @@
|
|||
'app.rpc.commands.files-share
|
||||
'app.rpc.commands.files-temp
|
||||
'app.rpc.commands.files-update
|
||||
'app.rpc.commands.files-thumbnails
|
||||
'app.rpc.commands.ldap
|
||||
'app.rpc.commands.management
|
||||
'app.rpc.commands.media
|
||||
|
|
|
|||
|
|
@ -9,16 +9,13 @@
|
|||
[app.common.data :as d]
|
||||
[app.common.data.macros :as dm]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.geom.shapes :as gsh]
|
||||
[app.common.pages.helpers :as cph]
|
||||
[app.common.pages.migrations :as pmg]
|
||||
[app.common.spec :as us]
|
||||
[app.common.types.components-list :as ctkl]
|
||||
[app.common.types.file :as ctf]
|
||||
[app.common.types.shape-tree :as ctt]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.db.sql :as sql]
|
||||
[app.loggers.audit :as-alias audit]
|
||||
[app.loggers.webhooks :as-alias webhooks]
|
||||
[app.rpc :as-alias rpc]
|
||||
|
|
@ -332,41 +329,6 @@
|
|||
(-> (get-file-fragment conn file-id fragment-id)
|
||||
(rph/with-http-cache long-cache-duration)))))
|
||||
|
||||
;; --- COMMAND QUERY: get-file-object-thumbnails
|
||||
|
||||
(defn get-object-thumbnails
|
||||
([conn file-id]
|
||||
(let [sql (str/concat
|
||||
"select object_id, data "
|
||||
" from file_object_thumbnail"
|
||||
" where file_id=?")]
|
||||
(->> (db/exec! conn [sql file-id])
|
||||
(d/index-by :object-id :data))))
|
||||
|
||||
([conn file-id object-ids]
|
||||
(let [sql (str/concat
|
||||
"select object_id, data "
|
||||
" from file_object_thumbnail"
|
||||
" where file_id=? and object_id = ANY(?)")
|
||||
ids (db/create-array conn "text" (seq object-ids))]
|
||||
(->> (db/exec! conn [sql file-id ids])
|
||||
(d/index-by :object-id :data)))))
|
||||
|
||||
(s/def ::get-file-object-thumbnails
|
||||
(s/keys :req [::rpc/profile-id] :req-un [::file-id]))
|
||||
|
||||
(sv/defmethod ::get-file-object-thumbnails
|
||||
"Retrieve a file object thumbnails."
|
||||
{::doc/added "1.17"
|
||||
::cond/get-object #(get-minimal-file %1 (:file-id %2))
|
||||
::cond/reuse-key? true
|
||||
::cond/key-fn get-file-etag}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
|
||||
(dm/with-open [conn (db/open pool)]
|
||||
(check-read-permissions! conn profile-id file-id)
|
||||
(get-object-thumbnails conn file-id)))
|
||||
|
||||
|
||||
;; --- COMMAND QUERY: get-project-files
|
||||
|
||||
(def ^:private sql:project-files
|
||||
|
|
@ -662,161 +624,6 @@
|
|||
(teams/check-read-permissions! conn profile-id team-id)
|
||||
(get-team-recent-files conn team-id)))
|
||||
|
||||
|
||||
;; --- COMMAND QUERY: get-file-thumbnail
|
||||
|
||||
(defn get-file-thumbnail
|
||||
[conn file-id revn]
|
||||
(let [sql (sql/select :file-thumbnail
|
||||
(cond-> {:file-id file-id}
|
||||
revn (assoc :revn revn))
|
||||
{:limit 1
|
||||
:order-by [[:revn :desc]]})
|
||||
row (db/exec-one! conn sql)]
|
||||
(when-not row
|
||||
(ex/raise :type :not-found
|
||||
:code :file-thumbnail-not-found))
|
||||
|
||||
{:data (:data row)
|
||||
:props (some-> (:props row) db/decode-transit-pgobject)
|
||||
:revn (:revn row)
|
||||
:file-id (:file-id row)}))
|
||||
|
||||
(s/def ::revn ::us/integer)
|
||||
|
||||
(s/def ::get-file-thumbnail
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::file-id]
|
||||
:opt-un [::revn]))
|
||||
|
||||
(sv/defmethod ::get-file-thumbnail
|
||||
{::doc/added "1.17"}
|
||||
[{:keys [::db/pool]} {:keys [::rpc/profile-id file-id revn]}]
|
||||
(dm/with-open [conn (db/open pool)]
|
||||
(check-read-permissions! conn profile-id file-id)
|
||||
(-> (get-file-thumbnail conn file-id revn)
|
||||
(rph/with-http-cache long-cache-duration))))
|
||||
|
||||
|
||||
;; --- COMMAND QUERY: get-file-data-for-thumbnail
|
||||
|
||||
;; FIXME: performance issue
|
||||
;;
|
||||
;; We need to improve how we set frame for thumbnail in order to avoid
|
||||
;; loading all pages into memory for find the frame set for thumbnail.
|
||||
|
||||
(defn get-file-data-for-thumbnail
|
||||
[conn {:keys [data id] :as file}]
|
||||
(letfn [;; function responsible on finding the frame marked to be
|
||||
;; used as thumbnail; the returned frame always have
|
||||
;; the :page-id set to the page that it belongs.
|
||||
|
||||
(get-thumbnail-frame [data]
|
||||
;; NOTE: this is a hack for avoid perform blocking
|
||||
;; operation inside the for loop, clojure lazy-seq uses
|
||||
;; synchronized blocks that does not plays well with
|
||||
;; virtual threads, so we need to perform the load
|
||||
;; operation first. This operation forces all pointer maps
|
||||
;; load into the memory.
|
||||
(->> (-> data :pages-index vals)
|
||||
(filter pmap/pointer-map?)
|
||||
(run! pmap/load!))
|
||||
|
||||
;; Then proceed to find the frame set for thumbnail
|
||||
|
||||
(d/seek :use-for-thumbnail?
|
||||
(for [page (-> data :pages-index vals)
|
||||
frame (-> page :objects ctt/get-frames)]
|
||||
(assoc frame :page-id (:id page)))))
|
||||
|
||||
;; function responsible to filter objects data structure of
|
||||
;; all unneeded shapes if a concrete frame is provided. If no
|
||||
;; frame, the objects is returned untouched.
|
||||
(filter-objects [objects frame-id]
|
||||
(d/index-by :id (cph/get-children-with-self objects frame-id)))
|
||||
|
||||
;; function responsible of assoc available thumbnails
|
||||
;; to frames and remove all children shapes from objects if
|
||||
;; thumbnails is available
|
||||
(assoc-thumbnails [objects page-id thumbnails]
|
||||
(loop [objects objects
|
||||
frames (filter cph/frame-shape? (vals objects))]
|
||||
|
||||
(if-let [frame (-> frames first)]
|
||||
(let [frame-id (:id frame)
|
||||
object-id (str page-id frame-id)
|
||||
frame (if-let [thumb (get thumbnails object-id)]
|
||||
(assoc frame :thumbnail thumb :shapes [])
|
||||
(dissoc frame :thumbnail))
|
||||
|
||||
children-ids
|
||||
(cph/get-children-ids objects frame-id)
|
||||
|
||||
bounds
|
||||
(when (:show-content frame)
|
||||
(gsh/selection-rect (concat [frame] (->> children-ids (map (d/getf objects))))))
|
||||
|
||||
frame
|
||||
(cond-> frame
|
||||
(some? bounds)
|
||||
(assoc :children-bounds bounds))]
|
||||
|
||||
(if (:thumbnail frame)
|
||||
(recur (-> objects
|
||||
(assoc frame-id frame)
|
||||
(d/without-keys children-ids))
|
||||
(rest frames))
|
||||
(recur (assoc objects frame-id frame)
|
||||
(rest frames))))
|
||||
|
||||
objects)))]
|
||||
|
||||
(binding [pmap/*load-fn* (partial load-pointer conn id)]
|
||||
(let [frame (get-thumbnail-frame data)
|
||||
frame-id (:id frame)
|
||||
page-id (or (:page-id frame)
|
||||
(-> data :pages first))
|
||||
|
||||
page (dm/get-in data [:pages-index page-id])
|
||||
page (cond-> page (pmap/pointer-map? page) deref)
|
||||
frame-ids (if (some? frame) (list frame-id) (map :id (ctt/get-frames (:objects page))))
|
||||
|
||||
obj-ids (map #(str page-id %) frame-ids)
|
||||
thumbs (get-object-thumbnails conn id obj-ids)]
|
||||
|
||||
(cond-> page
|
||||
;; If we have frame, we need to specify it on the page level
|
||||
;; and remove the all other unrelated objects.
|
||||
(some? frame-id)
|
||||
(-> (assoc :thumbnail-frame-id frame-id)
|
||||
(update :objects filter-objects frame-id))
|
||||
|
||||
;; Assoc the available thumbnails and prune not visible shapes
|
||||
;; for avoid transfer unnecessary data.
|
||||
:always
|
||||
(update :objects assoc-thumbnails page-id thumbs))))))
|
||||
|
||||
(s/def ::get-file-data-for-thumbnail
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::file-id]
|
||||
:opt-un [::features]))
|
||||
|
||||
(sv/defmethod ::get-file-data-for-thumbnail
|
||||
"Retrieves the data for generate the thumbnail of the file. Used
|
||||
mainly for render thumbnails on dashboard."
|
||||
{::doc/added "1.17"}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id features] :as props}]
|
||||
(dm/with-open [conn (db/open pool)]
|
||||
(check-read-permissions! conn profile-id file-id)
|
||||
;; NOTE: we force here the "storage/pointer-map" feature, because
|
||||
;; it used internally only and is independent if user supports it
|
||||
;; or not.
|
||||
(let [feat (into #{"storage/pointer-map"} features)
|
||||
file (get-file conn file-id feat)]
|
||||
{:file-id file-id
|
||||
:revn (:revn file)
|
||||
:page (get-file-data-for-thumbnail conn file)})))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; MUTATION COMMANDS
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
|
@ -1026,66 +833,3 @@
|
|||
(check-edition-permissions! conn profile-id file-id)
|
||||
(-> (ignore-sync conn params)
|
||||
(update :features db/decode-pgarray #{}))))
|
||||
|
||||
|
||||
;; --- MUTATION COMMAND: upsert-file-object-thumbnail
|
||||
|
||||
(def sql:upsert-object-thumbnail
|
||||
"insert into file_object_thumbnail(file_id, object_id, data)
|
||||
values (?, ?, ?)
|
||||
on conflict(file_id, object_id) do
|
||||
update set data = ?;")
|
||||
|
||||
(defn upsert-file-object-thumbnail!
|
||||
[conn {:keys [file-id object-id data]}]
|
||||
(if data
|
||||
(db/exec-one! conn [sql:upsert-object-thumbnail file-id object-id data data])
|
||||
(db/delete! conn :file-object-thumbnail {:file-id file-id :object-id object-id})))
|
||||
|
||||
(s/def ::data (s/nilable ::us/string))
|
||||
(s/def ::thumbs/object-id ::us/string)
|
||||
(s/def ::upsert-file-object-thumbnail
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::file-id ::thumbs/object-id]
|
||||
:opt-un [::data]))
|
||||
|
||||
(sv/defmethod ::upsert-file-object-thumbnail
|
||||
{::doc/added "1.17"
|
||||
::audit/skip true}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(check-edition-permissions! conn profile-id file-id)
|
||||
(upsert-file-object-thumbnail! conn params)
|
||||
nil))
|
||||
|
||||
;; --- MUTATION COMMAND: upsert-file-thumbnail
|
||||
|
||||
(def ^:private sql:upsert-file-thumbnail
|
||||
"insert into file_thumbnail (file_id, revn, data, props)
|
||||
values (?, ?, ?, ?::jsonb)
|
||||
on conflict(file_id, revn) do
|
||||
update set data = ?, props=?, updated_at=now();")
|
||||
|
||||
(defn- upsert-file-thumbnail!
|
||||
[conn {:keys [file-id revn data props]}]
|
||||
(let [props (db/tjson (or props {}))]
|
||||
(db/exec-one! conn [sql:upsert-file-thumbnail
|
||||
file-id revn data props data props])))
|
||||
|
||||
(s/def ::revn ::us/integer)
|
||||
(s/def ::props map?)
|
||||
(s/def ::upsert-file-thumbnail
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::file-id ::revn ::data ::props]))
|
||||
|
||||
(sv/defmethod ::upsert-file-thumbnail
|
||||
"Creates or updates the file thumbnail. Mainly used for paint the
|
||||
grid thumbnails."
|
||||
{::doc/added "1.17"
|
||||
::audit/skip true}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(check-edition-permissions! conn profile-id file-id)
|
||||
(when-not (db/read-only? conn)
|
||||
(upsert-file-thumbnail! conn params))
|
||||
nil))
|
||||
|
|
|
|||
368
backend/src/app/rpc/commands/files_thumbnails.clj
Normal file
368
backend/src/app/rpc/commands/files_thumbnails.clj
Normal file
|
|
@ -0,0 +1,368 @@
|
|||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC
|
||||
|
||||
(ns app.rpc.commands.files-thumbnails
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.data.macros :as dm]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.geom.shapes :as gsh]
|
||||
[app.common.pages.helpers :as cph]
|
||||
[app.common.spec :as us]
|
||||
[app.common.types.shape-tree :as ctt]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.db.sql :as sql]
|
||||
[app.loggers.audit :as-alias audit]
|
||||
[app.loggers.webhooks :as-alias webhooks]
|
||||
[app.media :as media]
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.rpc.commands.files :as files]
|
||||
[app.rpc.cond :as-alias cond]
|
||||
[app.rpc.doc :as-alias doc]
|
||||
[app.rpc.helpers :as rph]
|
||||
[app.storage :as sto]
|
||||
[app.util.pointer-map :as pmap]
|
||||
[app.util.services :as sv]
|
||||
[app.util.time :as dt]
|
||||
[clojure.spec.alpha :as s]
|
||||
[cuerdas.core :as str]))
|
||||
|
||||
;; --- FEATURES
|
||||
|
||||
(def long-cache-duration
|
||||
(dt/duration {:days 7}))
|
||||
|
||||
;; --- COMMAND QUERY: get-file-object-thumbnails
|
||||
|
||||
(defn- get-public-uri
|
||||
[media-id]
|
||||
(str (cf/get :public-uri) "/assets/by-id/" media-id))
|
||||
|
||||
(defn- get-object-thumbnails
|
||||
([conn file-id]
|
||||
(let [sql (str/concat
|
||||
"select object_id, data, media_id "
|
||||
" from file_object_thumbnail"
|
||||
" where file_id=?")]
|
||||
(->> (db/exec! conn [sql file-id])
|
||||
(d/index-by :object-id (fn [row]
|
||||
(or (some-> row :media-id get-public-uri)
|
||||
(:data row))))
|
||||
(d/without-nils))))
|
||||
|
||||
([conn file-id object-ids]
|
||||
(let [sql (str/concat
|
||||
"select object_id, data "
|
||||
" from file_object_thumbnail"
|
||||
" where file_id=? and object_id = ANY(?)")
|
||||
ids (db/create-array conn "text" (seq object-ids))]
|
||||
(->> (db/exec! conn [sql file-id ids])
|
||||
(d/index-by :object-id (fn [row]
|
||||
(or (some-> row :media-id get-public-uri)
|
||||
(:data row))))))))
|
||||
|
||||
(s/def ::file-id ::us/uuid)
|
||||
(s/def ::get-file-object-thumbnails
|
||||
(s/keys :req [::rpc/profile-id] :req-un [::file-id]))
|
||||
|
||||
(sv/defmethod ::get-file-object-thumbnails
|
||||
"Retrieve a file object thumbnails."
|
||||
{::doc/added "1.17"
|
||||
::cond/get-object #(files/get-minimal-file %1 (:file-id %2))
|
||||
::cond/reuse-key? true
|
||||
::cond/key-fn files/get-file-etag}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
|
||||
(dm/with-open [conn (db/open pool)]
|
||||
(files/check-read-permissions! conn profile-id file-id)
|
||||
(get-object-thumbnails conn file-id)))
|
||||
|
||||
;; --- COMMAND QUERY: get-file-thumbnail
|
||||
|
||||
;; FIXME: refactor to support uploading data to storage
|
||||
|
||||
(defn get-file-thumbnail
|
||||
[conn file-id revn]
|
||||
(let [sql (sql/select :file-thumbnail
|
||||
(cond-> {:file-id file-id}
|
||||
revn (assoc :revn revn))
|
||||
{:limit 1
|
||||
:order-by [[:revn :desc]]})
|
||||
row (db/exec-one! conn sql)]
|
||||
(when-not row
|
||||
(ex/raise :type :not-found
|
||||
:code :file-thumbnail-not-found))
|
||||
|
||||
{:data (:data row)
|
||||
:props (some-> (:props row) db/decode-transit-pgobject)
|
||||
:revn (:revn row)
|
||||
:file-id (:file-id row)}))
|
||||
|
||||
(s/def ::revn ::us/integer)
|
||||
|
||||
(s/def ::get-file-thumbnail
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::file-id]
|
||||
:opt-un [::revn]))
|
||||
|
||||
(sv/defmethod ::get-file-thumbnail
|
||||
"Method used in frontend for obtain the file thumbnail (used in the
|
||||
dashboard)."
|
||||
{::doc/added "1.17"}
|
||||
[{:keys [::db/pool]} {:keys [::rpc/profile-id file-id revn]}]
|
||||
(dm/with-open [conn (db/open pool)]
|
||||
(files/check-read-permissions! conn profile-id file-id)
|
||||
(-> (get-file-thumbnail conn file-id revn)
|
||||
(rph/with-http-cache long-cache-duration))))
|
||||
|
||||
|
||||
;; --- COMMAND QUERY: get-file-data-for-thumbnail
|
||||
|
||||
;; FIXME: performance issue, handle new media_id
|
||||
;;
|
||||
;; We need to improve how we set frame for thumbnail in order to avoid
|
||||
;; loading all pages into memory for find the frame set for thumbnail.
|
||||
|
||||
(defn get-file-data-for-thumbnail
|
||||
[conn {:keys [data id] :as file}]
|
||||
(letfn [;; function responsible on finding the frame marked to be
|
||||
;; used as thumbnail; the returned frame always have
|
||||
;; the :page-id set to the page that it belongs.
|
||||
|
||||
(get-thumbnail-frame [data]
|
||||
;; NOTE: this is a hack for avoid perform blocking
|
||||
;; operation inside the for loop, clojure lazy-seq uses
|
||||
;; synchronized blocks that does not plays well with
|
||||
;; virtual threads, so we need to perform the load
|
||||
;; operation first. This operation forces all pointer maps
|
||||
;; load into the memory.
|
||||
(->> (-> data :pages-index vals)
|
||||
(filter pmap/pointer-map?)
|
||||
(run! pmap/load!))
|
||||
|
||||
;; Then proceed to find the frame set for thumbnail
|
||||
|
||||
(d/seek :use-for-thumbnail?
|
||||
(for [page (-> data :pages-index vals)
|
||||
frame (-> page :objects ctt/get-frames)]
|
||||
(assoc frame :page-id (:id page)))))
|
||||
|
||||
;; function responsible to filter objects data structure of
|
||||
;; all unneeded shapes if a concrete frame is provided. If no
|
||||
;; frame, the objects is returned untouched.
|
||||
(filter-objects [objects frame-id]
|
||||
(d/index-by :id (cph/get-children-with-self objects frame-id)))
|
||||
|
||||
;; function responsible of assoc available thumbnails
|
||||
;; to frames and remove all children shapes from objects if
|
||||
;; thumbnails is available
|
||||
(assoc-thumbnails [objects page-id thumbnails]
|
||||
(loop [objects objects
|
||||
frames (filter cph/frame-shape? (vals objects))]
|
||||
|
||||
(if-let [frame (-> frames first)]
|
||||
(let [frame-id (:id frame)
|
||||
object-id (str page-id frame-id)
|
||||
frame (if-let [thumb (get thumbnails object-id)]
|
||||
(assoc frame :thumbnail thumb :shapes [])
|
||||
(dissoc frame :thumbnail))
|
||||
|
||||
children-ids
|
||||
(cph/get-children-ids objects frame-id)
|
||||
|
||||
bounds
|
||||
(when (:show-content frame)
|
||||
(gsh/selection-rect (concat [frame] (->> children-ids (map (d/getf objects))))))
|
||||
|
||||
frame
|
||||
(cond-> frame
|
||||
(some? bounds)
|
||||
(assoc :children-bounds bounds))]
|
||||
|
||||
(if (:thumbnail frame)
|
||||
(recur (-> objects
|
||||
(assoc frame-id frame)
|
||||
(d/without-keys children-ids))
|
||||
(rest frames))
|
||||
(recur (assoc objects frame-id frame)
|
||||
(rest frames))))
|
||||
|
||||
objects)))]
|
||||
|
||||
(binding [pmap/*load-fn* (partial files/load-pointer conn id)]
|
||||
(let [frame (get-thumbnail-frame data)
|
||||
frame-id (:id frame)
|
||||
page-id (or (:page-id frame)
|
||||
(-> data :pages first))
|
||||
|
||||
page (dm/get-in data [:pages-index page-id])
|
||||
page (cond-> page (pmap/pointer-map? page) deref)
|
||||
frame-ids (if (some? frame) (list frame-id) (map :id (ctt/get-frames (:objects page))))
|
||||
|
||||
obj-ids (map #(str page-id %) frame-ids)
|
||||
thumbs (get-object-thumbnails conn id obj-ids)]
|
||||
|
||||
(cond-> page
|
||||
;; If we have frame, we need to specify it on the page level
|
||||
;; and remove the all other unrelated objects.
|
||||
(some? frame-id)
|
||||
(-> (assoc :thumbnail-frame-id frame-id)
|
||||
(update :objects filter-objects frame-id))
|
||||
|
||||
;; Assoc the available thumbnails and prune not visible shapes
|
||||
;; for avoid transfer unnecessary data.
|
||||
:always
|
||||
(update :objects assoc-thumbnails page-id thumbs))))))
|
||||
|
||||
(s/def ::get-file-data-for-thumbnail
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::file-id]
|
||||
:opt-un [::features]))
|
||||
|
||||
(sv/defmethod ::get-file-data-for-thumbnail
|
||||
"Retrieves the data for generate the thumbnail of the file. Used
|
||||
mainly for render thumbnails on dashboard."
|
||||
{::doc/added "1.17"}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id features] :as props}]
|
||||
(dm/with-open [conn (db/open pool)]
|
||||
(files/check-read-permissions! conn profile-id file-id)
|
||||
;; NOTE: we force here the "storage/pointer-map" feature, because
|
||||
;; it used internally only and is independent if user supports it
|
||||
;; or not.
|
||||
(let [feat (into #{"storage/pointer-map"} features)
|
||||
file (files/get-file conn file-id feat)]
|
||||
{:file-id file-id
|
||||
:revn (:revn file)
|
||||
:page (get-file-data-for-thumbnail conn file)})))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; MUTATION COMMANDS
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
;; --- MUTATION COMMAND: upsert-file-object-thumbnail
|
||||
|
||||
(def sql:upsert-object-thumbnail-1
|
||||
"insert into file_object_thumbnail(file_id, object_id, data)
|
||||
values (?, ?, ?)
|
||||
on conflict(file_id, object_id) do
|
||||
update set data = ?;")
|
||||
|
||||
(def sql:upsert-object-thumbnail-2
|
||||
"insert into file_object_thumbnail(file_id, object_id, media_id)
|
||||
values (?, ?, ?)
|
||||
on conflict(file_id, object_id) do
|
||||
update set media_id = ?;")
|
||||
|
||||
(defn upsert-file-object-thumbnail!
|
||||
[{:keys [::db/conn ::sto/storage]} {:keys [file-id object-id] :as params}]
|
||||
|
||||
;; NOTE: params can come with data set but with `nil` value, so we
|
||||
;; need first check the existence of the key and then the value.
|
||||
(cond
|
||||
(contains? params :data)
|
||||
(if-let [data (:data params)]
|
||||
(db/exec-one! conn [sql:upsert-object-thumbnail-1 file-id object-id data data])
|
||||
(db/delete! conn :file-object-thumbnail {:file-id file-id :object-id object-id}))
|
||||
|
||||
(contains? params :media)
|
||||
(if-let [{:keys [path mtype] :as media} (:media params)]
|
||||
(let [_ (media/validate-media-type! media)
|
||||
_ (media/validate-media-size! media)
|
||||
hash (sto/calculate-hash path)
|
||||
data (-> (sto/content path)
|
||||
(sto/wrap-with-hash hash))
|
||||
media (sto/put-object! storage
|
||||
{::sto/content data
|
||||
::sto/deduplicate? false
|
||||
:content-type mtype
|
||||
:bucket "file-object-thumbnail"})]
|
||||
|
||||
(db/exec-one! conn [sql:upsert-object-thumbnail-2 file-id object-id (:id media) (:id media)]))
|
||||
(db/delete! conn :file-object-thumbnail {:file-id file-id :object-id object-id}))))
|
||||
|
||||
;; FIXME: change it on validation refactor
|
||||
(s/def ::data (s/nilable ::us/string))
|
||||
(s/def ::media (s/nilable ::media/upload))
|
||||
(s/def ::object-id ::us/string)
|
||||
|
||||
(s/def ::upsert-file-object-thumbnail
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::file-id ::object-id]
|
||||
:opt-un [::data ::media]))
|
||||
|
||||
(sv/defmethod ::upsert-file-object-thumbnail
|
||||
{::doc/added "1.17"
|
||||
::audit/skip true}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
|
||||
|
||||
(assert (or (contains? params :data)
|
||||
(contains? params :media)))
|
||||
|
||||
(db/with-atomic [conn pool]
|
||||
(files/check-edition-permissions! conn profile-id file-id)
|
||||
|
||||
(when-not (db/read-only? conn)
|
||||
(let [cfg (-> cfg
|
||||
(update ::sto/storage media/configure-assets-storage)
|
||||
(assoc ::db/conn conn))]
|
||||
(upsert-file-object-thumbnail! cfg params)
|
||||
nil))))
|
||||
|
||||
;; --- MUTATION COMMAND: upsert-file-thumbnail
|
||||
|
||||
(def ^:private sql:upsert-file-thumbnail
|
||||
"insert into file_thumbnail (file_id, revn, data, media_id, props)
|
||||
values (?, ?, ?, ?, ?::jsonb)
|
||||
on conflict(file_id, revn) do
|
||||
update set data=?, media_id=?, props=?, updated_at=now();")
|
||||
|
||||
(defn- upsert-file-thumbnail!
|
||||
[{:keys [::db/conn ::sto/storage]} {:keys [file-id revn props] :as params}]
|
||||
(let [props (db/tjson (or props {}))]
|
||||
(cond
|
||||
(contains? params :data)
|
||||
(when-let [data (:data params)]
|
||||
(db/exec-one! conn [sql:upsert-file-thumbnail
|
||||
file-id revn data nil props data nil props]))
|
||||
|
||||
(contains? params :media)
|
||||
(when-let [{:keys [path mtype] :as media} (:media params)]
|
||||
(let [_ (media/validate-media-type! media)
|
||||
_ (media/validate-media-size! media)
|
||||
hash (sto/calculate-hash path)
|
||||
data (-> (sto/content path)
|
||||
(sto/wrap-with-hash hash))
|
||||
media (sto/put-object! storage
|
||||
{::sto/content data
|
||||
::sto/deduplicate? false
|
||||
:content-type mtype
|
||||
:bucket "file-thumbnail"})]
|
||||
(db/exec-one! conn [sql:upsert-file-thumbnail
|
||||
file-id revn nil (:id media) props nil (:id media) props]))))))
|
||||
|
||||
(s/def ::revn ::us/integer)
|
||||
(s/def ::props map?)
|
||||
(s/def ::media ::media/upload)
|
||||
|
||||
(s/def ::upsert-file-thumbnail
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::file-id ::revn ::props]
|
||||
:opt-un [::data ::media]))
|
||||
|
||||
(sv/defmethod ::upsert-file-thumbnail
|
||||
"Creates or updates the file thumbnail. Mainly used for paint the
|
||||
grid thumbnails."
|
||||
{::doc/added "1.17"
|
||||
::audit/skip true}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(files/check-edition-permissions! conn profile-id file-id)
|
||||
(when-not (db/read-only? conn)
|
||||
(let [cfg (-> cfg
|
||||
(update ::sto/storage media/configure-assets-storage)
|
||||
(assoc ::db/conn conn))]
|
||||
(upsert-file-thumbnail! cfg params))
|
||||
nil)))
|
||||
|
|
@ -41,15 +41,6 @@
|
|||
(s/def ::file-id ::us/uuid)
|
||||
(s/def ::team-id ::us/uuid)
|
||||
|
||||
(defn validate-content-size!
|
||||
[content]
|
||||
(when (> (:size content) (cf/get :media-max-file-size default-max-file-size))
|
||||
(ex/raise :type :restriction
|
||||
:code :media-max-file-size-reached
|
||||
:hint (str/ffmt "the uploaded file size % is greater than the maximum %"
|
||||
(:size content)
|
||||
default-max-file-size))))
|
||||
|
||||
;; --- Create File Media object (upload)
|
||||
|
||||
(declare create-file-media-object)
|
||||
|
|
@ -68,7 +59,7 @@
|
|||
(let [cfg (update cfg ::sto/storage media/configure-assets-storage)]
|
||||
(files/check-edition-permissions! pool profile-id file-id)
|
||||
(media/validate-media-type! content)
|
||||
(validate-content-size! content)
|
||||
(media/validate-media-size! content)
|
||||
(let [object (create-file-media-object cfg params)
|
||||
props {:name (:name params)
|
||||
:file-id file-id
|
||||
|
|
|
|||
|
|
@ -18,7 +18,9 @@
|
|||
[app.common.types.shape-tree :as ctt]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.media :as media]
|
||||
[app.rpc.commands.files :as files]
|
||||
[app.storage :as sto]
|
||||
[app.util.blob :as blob]
|
||||
[app.util.pointer-map :as pmap]
|
||||
[app.util.time :as dt]
|
||||
|
|
@ -34,7 +36,7 @@
|
|||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defmethod ig/pre-init-spec ::handler [_]
|
||||
(s/keys :req [::db/pool]))
|
||||
(s/keys :req [::db/pool ::sto/storage]))
|
||||
|
||||
(defmethod ig/prep-key ::handler
|
||||
[_ cfg]
|
||||
|
|
@ -47,6 +49,7 @@
|
|||
(db/with-atomic [conn pool]
|
||||
(let [min-age (dt/duration (or (:min-age params) (::min-age cfg)))
|
||||
cfg (-> cfg
|
||||
(update ::sto/storage media/configure-assets-storage conn)
|
||||
(assoc ::db/conn conn)
|
||||
(assoc ::file-id file-id)
|
||||
(assoc ::min-age min-age))
|
||||
|
|
@ -141,36 +144,53 @@
|
|||
(db/delete! conn :file-media-object {:id (:id mobj)}))))
|
||||
|
||||
(defn- clean-file-object-thumbnails!
|
||||
[conn file-id data]
|
||||
[{:keys [::db/conn ::sto/storage]} file-id data]
|
||||
(let [stored (->> (db/query conn :file-object-thumbnail
|
||||
{:file-id file-id}
|
||||
{:columns [:object-id]})
|
||||
(into #{} (map :object-id)))
|
||||
|
||||
get-objects-ids
|
||||
(fn [{:keys [id objects]}]
|
||||
(->> (ctt/get-frames objects)
|
||||
(map #(str id (:id %)))))
|
||||
|
||||
using (into #{}
|
||||
(mapcat get-objects-ids)
|
||||
(vals (:pages-index data)))
|
||||
using (into #{}
|
||||
(mapcat (fn [{:keys [id objects]}]
|
||||
(->> (ctt/get-frames objects)
|
||||
(map #(str id (:id %))))))
|
||||
(vals (:pages-index data)))
|
||||
|
||||
unused (set/difference stored using)]
|
||||
|
||||
(when (seq unused)
|
||||
(let [sql (str "delete from file_object_thumbnail "
|
||||
" where file_id=? and object_id=ANY(?)")
|
||||
res (db/exec-one! conn [sql file-id (db/create-array conn "text" unused)])]
|
||||
(l/debug :hint "delete file object thumbnails" :file-id file-id :total (:next.jdbc/update-count res))))))
|
||||
" where file_id=? and object_id=ANY(?)"
|
||||
" returning media_id")
|
||||
res (db/exec! conn [sql file-id (db/create-array conn "text" unused)])]
|
||||
|
||||
(doseq [media-id (into #{} (keep :media-id) res)]
|
||||
;; Mark as deleted the storage object related with the
|
||||
;; photo-id field.
|
||||
(l/trace :hint "mark storage object as deleted" :id media-id)
|
||||
(sto/del-object! storage media-id))
|
||||
|
||||
(l/debug :hint "delete file object thumbnails"
|
||||
:file-id file-id
|
||||
:total (count res))))))
|
||||
|
||||
(defn- clean-file-thumbnails!
|
||||
[conn file-id revn]
|
||||
[{:keys [::db/conn ::sto/storage]} file-id revn]
|
||||
(let [sql (str "delete from file_thumbnail "
|
||||
" where file_id=? and revn < ?")
|
||||
res (db/exec-one! conn [sql file-id revn])]
|
||||
(when-not (zero? (:next.jdbc/update-count res))
|
||||
(l/debug :hint "delete file thumbnails" :file-id file-id :total (:next.jdbc/update-count res)))))
|
||||
" where file_id=? and revn < ? "
|
||||
" returning media_id")
|
||||
res (db/exec! conn [sql file-id revn])]
|
||||
|
||||
(when (seq res)
|
||||
(doseq [media-id (into #{} (keep :media-id) res)]
|
||||
;; Mark as deleted the storage object related with the
|
||||
;; photo-id field.
|
||||
(l/trace :hint "mark storage object as deleted" :id media-id)
|
||||
(sto/del-object! storage media-id))
|
||||
|
||||
(l/debug :hint "delete file thumbnails"
|
||||
:file-id file-id
|
||||
:total (count res)))))
|
||||
|
||||
(def ^:private
|
||||
sql:get-files-for-library
|
||||
|
|
@ -252,7 +272,7 @@
|
|||
(db/delete! conn :file-data-fragment {:id fragment-id :file-id file-id})))))
|
||||
|
||||
(defn- process-file
|
||||
[{:keys [::db/conn]} {:keys [id data revn modified-at features] :as file}]
|
||||
[{:keys [::db/conn] :as cfg} {:keys [id data revn modified-at features] :as file}]
|
||||
(l/debug :hint "processing file" :id id :modified-at modified-at)
|
||||
|
||||
(binding [pmap/*load-fn* (partial files/load-pointer conn id)]
|
||||
|
|
@ -261,8 +281,8 @@
|
|||
(pmg/migrate-data))]
|
||||
|
||||
(clean-file-media! conn id data)
|
||||
(clean-file-object-thumbnails! conn id data)
|
||||
(clean-file-thumbnails! conn id revn)
|
||||
(clean-file-object-thumbnails! cfg id data)
|
||||
(clean-file-thumbnails! cfg id revn)
|
||||
(clean-deleted-components! conn id data)
|
||||
|
||||
(when (contains? features "storage/pointer-map")
|
||||
|
|
|
|||
|
|
@ -335,6 +335,20 @@
|
|||
:session-id session-id
|
||||
:profile-id profile-id})))))
|
||||
|
||||
(declare command!)
|
||||
|
||||
(defn update-file! [& {:keys [profile-id file-id changes revn] :or {revn 0}}]
|
||||
(let [params {::type :update-file
|
||||
::rpc/profile-id profile-id
|
||||
:id file-id
|
||||
:session-id (uuid/random)
|
||||
:revn revn
|
||||
:components-v2 true
|
||||
:changes changes}
|
||||
out (command! params)]
|
||||
(t/is (nil? (:error out)))
|
||||
(:result out)))
|
||||
|
||||
(defn create-webhook*
|
||||
([params] (create-webhook* *pool* params))
|
||||
([pool {:keys [team-id id uri mtype is-active]
|
||||
|
|
|
|||
314
backend/test/backend_tests/rpc_file_thumbnails_test.clj
Normal file
314
backend/test/backend_tests/rpc_file_thumbnails_test.clj
Normal file
|
|
@ -0,0 +1,314 @@
|
|||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC
|
||||
|
||||
(ns backend-tests.rpc-file-thumbnails-test
|
||||
(:require
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.rpc.commands.auth :as cauth]
|
||||
[app.storage :as sto]
|
||||
[app.tokens :as tokens]
|
||||
[app.util.time :as dt]
|
||||
[backend-tests.helpers :as th]
|
||||
[clojure.java.io :as io]
|
||||
[clojure.test :as t]
|
||||
[cuerdas.core :as str]
|
||||
[datoteka.core :as fs]
|
||||
[mockery.core :refer [with-mocks]]))
|
||||
|
||||
(t/use-fixtures :once th/state-init)
|
||||
(t/use-fixtures :each th/database-reset)
|
||||
|
||||
(t/deftest upsert-file-object-thumbnail
|
||||
(let [storage (::sto/storage th/*system*)
|
||||
profile (th/create-profile* 1)
|
||||
file (th/create-file* 1 {:profile-id (:id profile)
|
||||
:project-id (:default-project-id profile)
|
||||
:is-shared false})
|
||||
|
||||
shid (uuid/random)
|
||||
page-id (first (get-in file [:data :pages]))
|
||||
|
||||
;; Update file inserting a new frame object
|
||||
_ (th/update-file!
|
||||
:file-id (:id file)
|
||||
:profile-id (:id profile)
|
||||
:revn 0
|
||||
:changes
|
||||
[{:type :add-obj
|
||||
:page-id page-id
|
||||
:id shid
|
||||
:parent-id uuid/zero
|
||||
:frame-id uuid/zero
|
||||
:components-v2 true
|
||||
:obj {:id shid
|
||||
:name "Artboard"
|
||||
:frame-id uuid/zero
|
||||
:parent-id uuid/zero
|
||||
:type :frame}}])
|
||||
|
||||
data1 {::th/type :upsert-file-object-thumbnail
|
||||
::rpc/profile-id (:id profile)
|
||||
:file-id (:id file)
|
||||
:object-id "test-key-1"
|
||||
:media {:filename "sample.jpg"
|
||||
:size 312043
|
||||
:path (th/tempfile "backend_tests/test_files/sample.jpg")
|
||||
:mtype "image/jpeg"}}
|
||||
|
||||
data2 {::th/type :upsert-file-object-thumbnail
|
||||
::rpc/profile-id (:id profile)
|
||||
:file-id (:id file)
|
||||
:object-id (str page-id shid)
|
||||
:media {:filename "sample.jpg"
|
||||
:size 7923
|
||||
:path (th/tempfile "backend_tests/test_files/sample2.jpg")
|
||||
:mtype "image/jpeg"}}]
|
||||
|
||||
(let [out (th/command! data1)]
|
||||
(t/is (nil? (:error out)))
|
||||
(t/is (nil? (:result out))))
|
||||
|
||||
(let [out (th/command! data2)]
|
||||
(t/is (nil? (:error out)))
|
||||
(t/is (nil? (:result out))))
|
||||
|
||||
(let [[row1 row2 :as rows] (th/db-query :file-object-thumbnail
|
||||
{:file-id (:id file)}
|
||||
{:order-by [[:created-at :asc]]})]
|
||||
|
||||
(t/is (= 2 (count rows)))
|
||||
(t/is (= (:file-id data1) (:file-id row1)))
|
||||
(t/is (= (:object-id data1) (:object-id row1)))
|
||||
(t/is (uuid? (:media-id row1)))
|
||||
(t/is (= (:file-id data2) (:file-id row2)))
|
||||
(t/is (= (:object-id data2) (:object-id row2)))
|
||||
(t/is (uuid? (:media-id row2)))
|
||||
|
||||
(let [sobject (sto/get-object storage (:media-id row1))
|
||||
mobject (meta sobject)]
|
||||
(t/is (= "blake2b:4fdb63b8f3ffc81256ea79f13e53f366723b188554b5afed91b20897c14a1a8e" (:hash mobject)))
|
||||
(t/is (= "file-object-thumbnail" (:bucket mobject)))
|
||||
(t/is (= "image/jpeg" (:content-type mobject)))
|
||||
(t/is (= 312043 (:size sobject))))
|
||||
|
||||
(let [sobject (sto/get-object storage (:media-id row2))
|
||||
mobject (meta sobject)]
|
||||
(t/is (= "blake2b:05870e3f8ee885841ee3799924d80805179ab57e6fde84a605d1068fd3138de9" (:hash mobject)))
|
||||
(t/is (= "file-object-thumbnail" (:bucket mobject)))
|
||||
(t/is (= "image/jpeg" (:content-type mobject)))
|
||||
(t/is (= 7923 (:size sobject))))
|
||||
|
||||
;; Run the File GC task that should remove unused file object
|
||||
;; thumbnails
|
||||
(let [result (th/run-task! :file-gc {:min-age (dt/duration 0)})]
|
||||
(t/is (= 1 (:processed result))))
|
||||
|
||||
;; check if row2 related thumbnail row still exists
|
||||
(let [[row :as rows] (th/db-query :file-object-thumbnail
|
||||
{:file-id (:id file)}
|
||||
{:order-by [[:created-at :asc]]})]
|
||||
(t/is (= 1 (count rows)))
|
||||
(t/is (= (:file-id data2) (:file-id row)))
|
||||
(t/is (= (:object-id data2) (:object-id row)))
|
||||
(t/is (uuid? (:media-id row2))))
|
||||
|
||||
;; Check if storage objects still exists after file-gc
|
||||
(t/is (nil? (sto/get-object storage (:media-id row1))))
|
||||
(t/is (some? (sto/get-object storage (:media-id row2))))
|
||||
|
||||
;; check that storage object is still exists but is marked as deleted
|
||||
(let [row (th/db-get :storage-object {:id (:media-id row1)} {::db/remove-deleted? false})]
|
||||
(t/is (some? (:deleted-at row))))
|
||||
|
||||
;; Run the storage gc deleted task, it should permanently delete
|
||||
;; all storage objects related to the deleted thumbnails
|
||||
(let [result (th/run-task! :storage-gc-deleted {:min-age (dt/duration 0)})]
|
||||
(t/is (= 1 (:deleted result))))
|
||||
|
||||
;; check that storage object is still exists but is marked as deleted
|
||||
(let [row (th/db-get :storage-object {:id (:media-id row1)} {::db/remove-deleted? false})]
|
||||
(t/is (nil? row)))
|
||||
|
||||
(t/is (some? (sto/get-object storage (:media-id row2))))
|
||||
|
||||
|
||||
)))
|
||||
|
||||
|
||||
(t/deftest upsert-file-thumbnail
|
||||
(let [storage (::sto/storage th/*system*)
|
||||
profile (th/create-profile* 1)
|
||||
file (th/create-file* 1 {:profile-id (:id profile)
|
||||
:project-id (:default-project-id profile)
|
||||
:is-shared false
|
||||
:revn 3})
|
||||
|
||||
data1 {::th/type :upsert-file-thumbnail
|
||||
::rpc/profile-id (:id profile)
|
||||
:file-id (:id file)
|
||||
:props {}
|
||||
:revn 1
|
||||
:data "data:base64,1234123124"}
|
||||
|
||||
data2 {::th/type :upsert-file-thumbnail
|
||||
::rpc/profile-id (:id profile)
|
||||
:file-id (:id file)
|
||||
:props {}
|
||||
:revn 2
|
||||
:media {:filename "sample.jpg"
|
||||
:size 7923
|
||||
:path (th/tempfile "backend_tests/test_files/sample2.jpg")
|
||||
:mtype "image/jpeg"}}
|
||||
|
||||
data3 {::th/type :upsert-file-thumbnail
|
||||
::rpc/profile-id (:id profile)
|
||||
:file-id (:id file)
|
||||
:props {}
|
||||
:revn 3
|
||||
:media {:filename "sample.jpg"
|
||||
:size 312043
|
||||
:path (th/tempfile "backend_tests/test_files/sample.jpg")
|
||||
:mtype "image/jpeg"}}]
|
||||
|
||||
(let [out (th/command! data1)]
|
||||
;; (th/print-result! out)
|
||||
(t/is (nil? (:error out)))
|
||||
(t/is (nil? (:result out))))
|
||||
|
||||
(let [out (th/command! data2)]
|
||||
(t/is (nil? (:error out)))
|
||||
(t/is (nil? (:result out))))
|
||||
|
||||
(let [out (th/command! data3)]
|
||||
(t/is (nil? (:error out)))
|
||||
(t/is (nil? (:result out))))
|
||||
|
||||
(let [[row1 row2 row3 :as rows] (th/db-query :file-thumbnail
|
||||
{:file-id (:id file)}
|
||||
{:order-by [[:created-at :asc]]})]
|
||||
(t/is (= 3 (count rows)))
|
||||
|
||||
(t/is (= (:file-id data1) (:file-id row1)))
|
||||
(t/is (= (:revn data1) (:revn row1)))
|
||||
(t/is (nil? (:media-id row1)))
|
||||
|
||||
(t/is (= (:file-id data2) (:file-id row2)))
|
||||
(t/is (= (:revn data2) (:revn row2)))
|
||||
(t/is (uuid? (:media-id row2)))
|
||||
|
||||
(t/is (= (:file-id data3) (:file-id row3)))
|
||||
(t/is (= (:revn data3) (:revn row3)))
|
||||
(t/is (uuid? (:media-id row3)))
|
||||
|
||||
(let [sobject (sto/get-object storage (:media-id row2))
|
||||
mobject (meta sobject)]
|
||||
(t/is (= "blake2b:05870e3f8ee885841ee3799924d80805179ab57e6fde84a605d1068fd3138de9" (:hash mobject)))
|
||||
(t/is (= "file-thumbnail" (:bucket mobject)))
|
||||
(t/is (= "image/jpeg" (:content-type mobject)))
|
||||
(t/is (= 7923 (:size sobject))))
|
||||
|
||||
(let [sobject (sto/get-object storage (:media-id row3))
|
||||
mobject (meta sobject)]
|
||||
(t/is (= "blake2b:4fdb63b8f3ffc81256ea79f13e53f366723b188554b5afed91b20897c14a1a8e" (:hash mobject)))
|
||||
(t/is (= "file-thumbnail" (:bucket mobject)))
|
||||
(t/is (= "image/jpeg" (:content-type mobject)))
|
||||
(t/is (= 312043 (:size sobject))))
|
||||
|
||||
;; Run the File GC task that should remove unused file object
|
||||
;; thumbnails
|
||||
(let [result (th/run-task! :file-gc {:min-age (dt/duration 0)})]
|
||||
(t/is (= 1 (:processed result))))
|
||||
|
||||
;; check if row2 related thumbnail row still exists
|
||||
(let [[row :as rows] (th/db-query :file-thumbnail
|
||||
{:file-id (:id file)}
|
||||
{:order-by [[:created-at :asc]]})]
|
||||
(t/is (= 1 (count rows)))
|
||||
(t/is (= (:file-id data2) (:file-id row)))
|
||||
(t/is (= (:object-id data2) (:object-id row)))
|
||||
(t/is (uuid? (:media-id row2))))
|
||||
|
||||
;; Check if storage objects still exists after file-gc
|
||||
(t/is (nil? (sto/get-object storage (:media-id row1))))
|
||||
(t/is (nil? (sto/get-object storage (:media-id row2))))
|
||||
(t/is (some? (sto/get-object storage (:media-id row3))))
|
||||
|
||||
(let [row (th/db-get :storage-object {:id (:media-id row2)} {::db/remove-deleted? false})]
|
||||
(t/is (some? (:deleted-at row))))
|
||||
|
||||
;; Run the storage gc deleted task, it should permanently delete
|
||||
;; all storage objects related to the deleted thumbnails
|
||||
(let [result (th/run-task! :storage-gc-deleted {:min-age (dt/duration 0)})]
|
||||
(t/is (= 1 (:deleted result))))
|
||||
|
||||
;; check that storage object is still exists but is marked as deleted
|
||||
(let [row (th/db-get :storage-object {:id (:media-id row1)} {::db/remove-deleted? false})]
|
||||
(t/is (nil? row)))
|
||||
|
||||
(t/is (some? (sto/get-object storage (:media-id row3))))
|
||||
|
||||
|
||||
)))
|
||||
|
||||
(t/deftest get-file-object-thumbnail
|
||||
(let [storage (::sto/storage th/*system*)
|
||||
profile (th/create-profile* 1)
|
||||
file (th/create-file* 1 {:profile-id (:id profile)
|
||||
:project-id (:default-project-id profile)
|
||||
:is-shared false})
|
||||
|
||||
data1 {::th/type :upsert-file-object-thumbnail
|
||||
::rpc/profile-id (:id profile)
|
||||
:file-id (:id file)
|
||||
:object-id "test-key-1"
|
||||
:data "data:base64,1234123124"}
|
||||
|
||||
data2 {::th/type :upsert-file-object-thumbnail
|
||||
::rpc/profile-id (:id profile)
|
||||
:file-id (:id file)
|
||||
:object-id "test-key-2"
|
||||
:media {:filename "sample.jpg"
|
||||
:size 7923
|
||||
:path (th/tempfile "backend_tests/test_files/sample2.jpg")
|
||||
:mtype "image/jpeg"}}]
|
||||
|
||||
(let [out (th/command! data1)]
|
||||
(t/is (nil? (:error out)))
|
||||
(t/is (nil? (:result out))))
|
||||
|
||||
(let [out (th/command! data2)]
|
||||
(t/is (nil? (:error out)))
|
||||
(t/is (nil? (:result out))))
|
||||
|
||||
(let [[row1 row2 :as rows] (th/db-query :file-object-thumbnail
|
||||
{:file-id (:id file)}
|
||||
{:order-by [[:created-at :asc]]})]
|
||||
(t/is (= 2 (count rows)))
|
||||
|
||||
(t/is (= (:file-id data1) (:file-id row1)))
|
||||
(t/is (= (:object-id data1) (:object-id row1)))
|
||||
(t/is (nil? (:media-id row1)))
|
||||
(t/is (string? (:data row1)))
|
||||
|
||||
(t/is (= (:file-id data2) (:file-id row2)))
|
||||
(t/is (= (:object-id data2) (:object-id row2)))
|
||||
(t/is (uuid? (:media-id row2))))
|
||||
|
||||
|
||||
(let [params {::th/type :get-file-object-thumbnails
|
||||
::rpc/profile-id (:id profile)
|
||||
:file-id (:id file)}
|
||||
out (th/command! params)]
|
||||
|
||||
(let [result (:result out)]
|
||||
(t/is (contains? result "test-key-1"))
|
||||
(t/is (contains? result "test-key-2"))))))
|
||||
|
||||
|
||||
|
||||
BIN
backend/test/backend_tests/test_files/sample2.jpg
Normal file
BIN
backend/test/backend_tests/test_files/sample2.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 7.7 KiB |
|
|
@ -150,6 +150,10 @@
|
|||
[id params]
|
||||
(send-command! id params {:forward-query-params [:file-id :object-id]}))
|
||||
|
||||
(defmethod command :get-file-object-thumbnails
|
||||
[id params]
|
||||
(send-command! id params {:forward-query-params [:file-id]}))
|
||||
|
||||
(defmethod command :export-binfile
|
||||
[id params]
|
||||
(send-command! id params {:response-type :blob}))
|
||||
|
|
|
|||
Loading…
Reference in a new issue