From e8ffcbae692e212e1319929297fcb0bdd3c30318 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Fri, 5 May 2023 11:42:45 +0200 Subject: [PATCH] :tada: Add support for multipart upload of thumbnails and improve the thumbnails storage to offloading it to the storage subsystem --- backend/src/app/main.clj | 3 +- backend/src/app/media.clj | 13 + backend/src/app/migrations.clj | 10 +- .../0103-mod-file-object-thumbnail-table.sql | 2 + .../sql/0104-mod-file-thumbnail-table.sql | 2 + backend/src/app/rpc.clj | 1 + backend/src/app/rpc/commands/files.clj | 256 ------------ .../src/app/rpc/commands/files_thumbnails.clj | 368 ++++++++++++++++++ backend/src/app/rpc/commands/media.clj | 11 +- backend/src/app/tasks/file_gc.clj | 62 ++- backend/test/backend_tests/helpers.clj | 14 + .../rpc_file_thumbnails_test.clj | 314 +++++++++++++++ .../test/backend_tests/test_files/sample2.jpg | Bin 0 -> 7923 bytes frontend/src/app/main/repo.cljs | 4 + 14 files changed, 771 insertions(+), 289 deletions(-) create mode 100644 backend/src/app/migrations/sql/0103-mod-file-object-thumbnail-table.sql create mode 100644 backend/src/app/migrations/sql/0104-mod-file-thumbnail-table.sql create mode 100644 backend/src/app/rpc/commands/files_thumbnails.clj create mode 100644 backend/test/backend_tests/rpc_file_thumbnails_test.clj create mode 100644 backend/test/backend_tests/test_files/sample2.jpg diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index d82b24a40..249adc019 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -381,7 +381,8 @@ ::sto/storage (ig/ref ::sto/storage)} :app.tasks.file-gc/handler - {::db/pool (ig/ref ::db/pool)} + {::db/pool (ig/ref ::db/pool) + ::sto/storage (ig/ref ::sto/storage)} :app.tasks.file-xlog-gc/handler {::db/pool (ig/ref ::db/pool)} diff --git a/backend/src/app/media.clj b/backend/src/app/media.clj index 192aa2cb9..72de03ddd 100644 --- a/backend/src/app/media.clj +++ b/backend/src/app/media.clj @@ -29,6 +29,9 @@ org.im4java.core.IMOperation org.im4java.core.Info)) +(def default-max-file-size + (* 1024 1024 30)) ; 30 MiB + (s/def ::path fs/path?) (s/def ::filename string?) (s/def ::size integer?) @@ -54,6 +57,16 @@ upload)) +(defn validate-media-size! + [upload] + (when (> (:size upload) (cf/get :media-max-file-size default-max-file-size)) + (ex/raise :type :restriction + :code :media-max-file-size-reached + :hint (str/ffmt "the uploaded file size % is greater than the maximum %" + (:size upload) + default-max-file-size))) + upload) + (defmulti process :cmd) (defmulti process-error class) diff --git a/backend/src/app/migrations.clj b/backend/src/app/migrations.clj index bd9c9e63a..787d34450 100644 --- a/backend/src/app/migrations.clj +++ b/backend/src/app/migrations.clj @@ -316,7 +316,15 @@ :fn (mg/resource "app/migrations/sql/0101-mod-server-error-report-table.sql")} {:name "0102-mod-access-token-table" - :fn (mg/resource "app/migrations/sql/0102-mod-access-token-table.sql")}]) + :fn (mg/resource "app/migrations/sql/0102-mod-access-token-table.sql")} + + {:name "0103-mod-file-object-thumbnail-table" + :fn (mg/resource "app/migrations/sql/0103-mod-file-object-thumbnail-table.sql")} + + {:name "0104-mod-file-thumbnail-table" + :fn (mg/resource "app/migrations/sql/0104-mod-file-thumbnail-table.sql")} + + ]) (defn apply-migrations! [pool name migrations] diff --git a/backend/src/app/migrations/sql/0103-mod-file-object-thumbnail-table.sql b/backend/src/app/migrations/sql/0103-mod-file-object-thumbnail-table.sql new file mode 100644 index 000000000..5aa7ef260 --- /dev/null +++ b/backend/src/app/migrations/sql/0103-mod-file-object-thumbnail-table.sql @@ -0,0 +1,2 @@ +ALTER TABLE file_object_thumbnail + ADD COLUMN media_id uuid NULL REFERENCES storage_object(id) ON DELETE CASCADE DEFERRABLE; diff --git a/backend/src/app/migrations/sql/0104-mod-file-thumbnail-table.sql b/backend/src/app/migrations/sql/0104-mod-file-thumbnail-table.sql new file mode 100644 index 000000000..790a30df8 --- /dev/null +++ b/backend/src/app/migrations/sql/0104-mod-file-thumbnail-table.sql @@ -0,0 +1,2 @@ +ALTER TABLE file_thumbnail + ADD COLUMN media_id uuid NULL REFERENCES storage_object(id) ON DELETE CASCADE DEFERRABLE; diff --git a/backend/src/app/rpc.clj b/backend/src/app/rpc.clj index 054beee6c..f260ddbf7 100644 --- a/backend/src/app/rpc.clj +++ b/backend/src/app/rpc.clj @@ -170,6 +170,7 @@ 'app.rpc.commands.files-share 'app.rpc.commands.files-temp 'app.rpc.commands.files-update + 'app.rpc.commands.files-thumbnails 'app.rpc.commands.ldap 'app.rpc.commands.management 'app.rpc.commands.media diff --git a/backend/src/app/rpc/commands/files.clj b/backend/src/app/rpc/commands/files.clj index 9f48652c3..6c41bde18 100644 --- a/backend/src/app/rpc/commands/files.clj +++ b/backend/src/app/rpc/commands/files.clj @@ -9,16 +9,13 @@ [app.common.data :as d] [app.common.data.macros :as dm] [app.common.exceptions :as ex] - [app.common.geom.shapes :as gsh] [app.common.pages.helpers :as cph] [app.common.pages.migrations :as pmg] [app.common.spec :as us] [app.common.types.components-list :as ctkl] [app.common.types.file :as ctf] - [app.common.types.shape-tree :as ctt] [app.config :as cf] [app.db :as db] - [app.db.sql :as sql] [app.loggers.audit :as-alias audit] [app.loggers.webhooks :as-alias webhooks] [app.rpc :as-alias rpc] @@ -332,41 +329,6 @@ (-> (get-file-fragment conn file-id fragment-id) (rph/with-http-cache long-cache-duration))))) -;; --- COMMAND QUERY: get-file-object-thumbnails - -(defn get-object-thumbnails - ([conn file-id] - (let [sql (str/concat - "select object_id, data " - " from file_object_thumbnail" - " where file_id=?")] - (->> (db/exec! conn [sql file-id]) - (d/index-by :object-id :data)))) - - ([conn file-id object-ids] - (let [sql (str/concat - "select object_id, data " - " from file_object_thumbnail" - " where file_id=? and object_id = ANY(?)") - ids (db/create-array conn "text" (seq object-ids))] - (->> (db/exec! conn [sql file-id ids]) - (d/index-by :object-id :data))))) - -(s/def ::get-file-object-thumbnails - (s/keys :req [::rpc/profile-id] :req-un [::file-id])) - -(sv/defmethod ::get-file-object-thumbnails - "Retrieve a file object thumbnails." - {::doc/added "1.17" - ::cond/get-object #(get-minimal-file %1 (:file-id %2)) - ::cond/reuse-key? true - ::cond/key-fn get-file-etag} - [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}] - (dm/with-open [conn (db/open pool)] - (check-read-permissions! conn profile-id file-id) - (get-object-thumbnails conn file-id))) - - ;; --- COMMAND QUERY: get-project-files (def ^:private sql:project-files @@ -662,161 +624,6 @@ (teams/check-read-permissions! conn profile-id team-id) (get-team-recent-files conn team-id))) - -;; --- COMMAND QUERY: get-file-thumbnail - -(defn get-file-thumbnail - [conn file-id revn] - (let [sql (sql/select :file-thumbnail - (cond-> {:file-id file-id} - revn (assoc :revn revn)) - {:limit 1 - :order-by [[:revn :desc]]}) - row (db/exec-one! conn sql)] - (when-not row - (ex/raise :type :not-found - :code :file-thumbnail-not-found)) - - {:data (:data row) - :props (some-> (:props row) db/decode-transit-pgobject) - :revn (:revn row) - :file-id (:file-id row)})) - -(s/def ::revn ::us/integer) - -(s/def ::get-file-thumbnail - (s/keys :req [::rpc/profile-id] - :req-un [::file-id] - :opt-un [::revn])) - -(sv/defmethod ::get-file-thumbnail - {::doc/added "1.17"} - [{:keys [::db/pool]} {:keys [::rpc/profile-id file-id revn]}] - (dm/with-open [conn (db/open pool)] - (check-read-permissions! conn profile-id file-id) - (-> (get-file-thumbnail conn file-id revn) - (rph/with-http-cache long-cache-duration)))) - - -;; --- COMMAND QUERY: get-file-data-for-thumbnail - -;; FIXME: performance issue -;; -;; We need to improve how we set frame for thumbnail in order to avoid -;; loading all pages into memory for find the frame set for thumbnail. - -(defn get-file-data-for-thumbnail - [conn {:keys [data id] :as file}] - (letfn [;; function responsible on finding the frame marked to be - ;; used as thumbnail; the returned frame always have - ;; the :page-id set to the page that it belongs. - - (get-thumbnail-frame [data] - ;; NOTE: this is a hack for avoid perform blocking - ;; operation inside the for loop, clojure lazy-seq uses - ;; synchronized blocks that does not plays well with - ;; virtual threads, so we need to perform the load - ;; operation first. This operation forces all pointer maps - ;; load into the memory. - (->> (-> data :pages-index vals) - (filter pmap/pointer-map?) - (run! pmap/load!)) - - ;; Then proceed to find the frame set for thumbnail - - (d/seek :use-for-thumbnail? - (for [page (-> data :pages-index vals) - frame (-> page :objects ctt/get-frames)] - (assoc frame :page-id (:id page))))) - - ;; function responsible to filter objects data structure of - ;; all unneeded shapes if a concrete frame is provided. If no - ;; frame, the objects is returned untouched. - (filter-objects [objects frame-id] - (d/index-by :id (cph/get-children-with-self objects frame-id))) - - ;; function responsible of assoc available thumbnails - ;; to frames and remove all children shapes from objects if - ;; thumbnails is available - (assoc-thumbnails [objects page-id thumbnails] - (loop [objects objects - frames (filter cph/frame-shape? (vals objects))] - - (if-let [frame (-> frames first)] - (let [frame-id (:id frame) - object-id (str page-id frame-id) - frame (if-let [thumb (get thumbnails object-id)] - (assoc frame :thumbnail thumb :shapes []) - (dissoc frame :thumbnail)) - - children-ids - (cph/get-children-ids objects frame-id) - - bounds - (when (:show-content frame) - (gsh/selection-rect (concat [frame] (->> children-ids (map (d/getf objects)))))) - - frame - (cond-> frame - (some? bounds) - (assoc :children-bounds bounds))] - - (if (:thumbnail frame) - (recur (-> objects - (assoc frame-id frame) - (d/without-keys children-ids)) - (rest frames)) - (recur (assoc objects frame-id frame) - (rest frames)))) - - objects)))] - - (binding [pmap/*load-fn* (partial load-pointer conn id)] - (let [frame (get-thumbnail-frame data) - frame-id (:id frame) - page-id (or (:page-id frame) - (-> data :pages first)) - - page (dm/get-in data [:pages-index page-id]) - page (cond-> page (pmap/pointer-map? page) deref) - frame-ids (if (some? frame) (list frame-id) (map :id (ctt/get-frames (:objects page)))) - - obj-ids (map #(str page-id %) frame-ids) - thumbs (get-object-thumbnails conn id obj-ids)] - - (cond-> page - ;; If we have frame, we need to specify it on the page level - ;; and remove the all other unrelated objects. - (some? frame-id) - (-> (assoc :thumbnail-frame-id frame-id) - (update :objects filter-objects frame-id)) - - ;; Assoc the available thumbnails and prune not visible shapes - ;; for avoid transfer unnecessary data. - :always - (update :objects assoc-thumbnails page-id thumbs)))))) - -(s/def ::get-file-data-for-thumbnail - (s/keys :req [::rpc/profile-id] - :req-un [::file-id] - :opt-un [::features])) - -(sv/defmethod ::get-file-data-for-thumbnail - "Retrieves the data for generate the thumbnail of the file. Used - mainly for render thumbnails on dashboard." - {::doc/added "1.17"} - [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id features] :as props}] - (dm/with-open [conn (db/open pool)] - (check-read-permissions! conn profile-id file-id) - ;; NOTE: we force here the "storage/pointer-map" feature, because - ;; it used internally only and is independent if user supports it - ;; or not. - (let [feat (into #{"storage/pointer-map"} features) - file (get-file conn file-id feat)] - {:file-id file-id - :revn (:revn file) - :page (get-file-data-for-thumbnail conn file)}))) - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; MUTATION COMMANDS ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1026,66 +833,3 @@ (check-edition-permissions! conn profile-id file-id) (-> (ignore-sync conn params) (update :features db/decode-pgarray #{})))) - - -;; --- MUTATION COMMAND: upsert-file-object-thumbnail - -(def sql:upsert-object-thumbnail - "insert into file_object_thumbnail(file_id, object_id, data) - values (?, ?, ?) - on conflict(file_id, object_id) do - update set data = ?;") - -(defn upsert-file-object-thumbnail! - [conn {:keys [file-id object-id data]}] - (if data - (db/exec-one! conn [sql:upsert-object-thumbnail file-id object-id data data]) - (db/delete! conn :file-object-thumbnail {:file-id file-id :object-id object-id}))) - -(s/def ::data (s/nilable ::us/string)) -(s/def ::thumbs/object-id ::us/string) -(s/def ::upsert-file-object-thumbnail - (s/keys :req [::rpc/profile-id] - :req-un [::file-id ::thumbs/object-id] - :opt-un [::data])) - -(sv/defmethod ::upsert-file-object-thumbnail - {::doc/added "1.17" - ::audit/skip true} - [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}] - (db/with-atomic [conn pool] - (check-edition-permissions! conn profile-id file-id) - (upsert-file-object-thumbnail! conn params) - nil)) - -;; --- MUTATION COMMAND: upsert-file-thumbnail - -(def ^:private sql:upsert-file-thumbnail - "insert into file_thumbnail (file_id, revn, data, props) - values (?, ?, ?, ?::jsonb) - on conflict(file_id, revn) do - update set data = ?, props=?, updated_at=now();") - -(defn- upsert-file-thumbnail! - [conn {:keys [file-id revn data props]}] - (let [props (db/tjson (or props {}))] - (db/exec-one! conn [sql:upsert-file-thumbnail - file-id revn data props data props]))) - -(s/def ::revn ::us/integer) -(s/def ::props map?) -(s/def ::upsert-file-thumbnail - (s/keys :req [::rpc/profile-id] - :req-un [::file-id ::revn ::data ::props])) - -(sv/defmethod ::upsert-file-thumbnail - "Creates or updates the file thumbnail. Mainly used for paint the - grid thumbnails." - {::doc/added "1.17" - ::audit/skip true} - [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}] - (db/with-atomic [conn pool] - (check-edition-permissions! conn profile-id file-id) - (when-not (db/read-only? conn) - (upsert-file-thumbnail! conn params)) - nil)) diff --git a/backend/src/app/rpc/commands/files_thumbnails.clj b/backend/src/app/rpc/commands/files_thumbnails.clj new file mode 100644 index 000000000..83a7afdab --- /dev/null +++ b/backend/src/app/rpc/commands/files_thumbnails.clj @@ -0,0 +1,368 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns app.rpc.commands.files-thumbnails + (:require + [app.common.data :as d] + [app.common.data.macros :as dm] + [app.common.exceptions :as ex] + [app.common.geom.shapes :as gsh] + [app.common.pages.helpers :as cph] + [app.common.spec :as us] + [app.common.types.shape-tree :as ctt] + [app.config :as cf] + [app.db :as db] + [app.db.sql :as sql] + [app.loggers.audit :as-alias audit] + [app.loggers.webhooks :as-alias webhooks] + [app.media :as media] + [app.rpc :as-alias rpc] + [app.rpc.commands.files :as files] + [app.rpc.cond :as-alias cond] + [app.rpc.doc :as-alias doc] + [app.rpc.helpers :as rph] + [app.storage :as sto] + [app.util.pointer-map :as pmap] + [app.util.services :as sv] + [app.util.time :as dt] + [clojure.spec.alpha :as s] + [cuerdas.core :as str])) + +;; --- FEATURES + +(def long-cache-duration + (dt/duration {:days 7})) + +;; --- COMMAND QUERY: get-file-object-thumbnails + +(defn- get-public-uri + [media-id] + (str (cf/get :public-uri) "/assets/by-id/" media-id)) + +(defn- get-object-thumbnails + ([conn file-id] + (let [sql (str/concat + "select object_id, data, media_id " + " from file_object_thumbnail" + " where file_id=?")] + (->> (db/exec! conn [sql file-id]) + (d/index-by :object-id (fn [row] + (or (some-> row :media-id get-public-uri) + (:data row)))) + (d/without-nils)))) + + ([conn file-id object-ids] + (let [sql (str/concat + "select object_id, data " + " from file_object_thumbnail" + " where file_id=? and object_id = ANY(?)") + ids (db/create-array conn "text" (seq object-ids))] + (->> (db/exec! conn [sql file-id ids]) + (d/index-by :object-id (fn [row] + (or (some-> row :media-id get-public-uri) + (:data row)))))))) + +(s/def ::file-id ::us/uuid) +(s/def ::get-file-object-thumbnails + (s/keys :req [::rpc/profile-id] :req-un [::file-id])) + +(sv/defmethod ::get-file-object-thumbnails + "Retrieve a file object thumbnails." + {::doc/added "1.17" + ::cond/get-object #(files/get-minimal-file %1 (:file-id %2)) + ::cond/reuse-key? true + ::cond/key-fn files/get-file-etag} + [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}] + (dm/with-open [conn (db/open pool)] + (files/check-read-permissions! conn profile-id file-id) + (get-object-thumbnails conn file-id))) + +;; --- COMMAND QUERY: get-file-thumbnail + +;; FIXME: refactor to support uploading data to storage + +(defn get-file-thumbnail + [conn file-id revn] + (let [sql (sql/select :file-thumbnail + (cond-> {:file-id file-id} + revn (assoc :revn revn)) + {:limit 1 + :order-by [[:revn :desc]]}) + row (db/exec-one! conn sql)] + (when-not row + (ex/raise :type :not-found + :code :file-thumbnail-not-found)) + + {:data (:data row) + :props (some-> (:props row) db/decode-transit-pgobject) + :revn (:revn row) + :file-id (:file-id row)})) + +(s/def ::revn ::us/integer) + +(s/def ::get-file-thumbnail + (s/keys :req [::rpc/profile-id] + :req-un [::file-id] + :opt-un [::revn])) + +(sv/defmethod ::get-file-thumbnail + "Method used in frontend for obtain the file thumbnail (used in the + dashboard)." + {::doc/added "1.17"} + [{:keys [::db/pool]} {:keys [::rpc/profile-id file-id revn]}] + (dm/with-open [conn (db/open pool)] + (files/check-read-permissions! conn profile-id file-id) + (-> (get-file-thumbnail conn file-id revn) + (rph/with-http-cache long-cache-duration)))) + + +;; --- COMMAND QUERY: get-file-data-for-thumbnail + +;; FIXME: performance issue, handle new media_id +;; +;; We need to improve how we set frame for thumbnail in order to avoid +;; loading all pages into memory for find the frame set for thumbnail. + +(defn get-file-data-for-thumbnail + [conn {:keys [data id] :as file}] + (letfn [;; function responsible on finding the frame marked to be + ;; used as thumbnail; the returned frame always have + ;; the :page-id set to the page that it belongs. + + (get-thumbnail-frame [data] + ;; NOTE: this is a hack for avoid perform blocking + ;; operation inside the for loop, clojure lazy-seq uses + ;; synchronized blocks that does not plays well with + ;; virtual threads, so we need to perform the load + ;; operation first. This operation forces all pointer maps + ;; load into the memory. + (->> (-> data :pages-index vals) + (filter pmap/pointer-map?) + (run! pmap/load!)) + + ;; Then proceed to find the frame set for thumbnail + + (d/seek :use-for-thumbnail? + (for [page (-> data :pages-index vals) + frame (-> page :objects ctt/get-frames)] + (assoc frame :page-id (:id page))))) + + ;; function responsible to filter objects data structure of + ;; all unneeded shapes if a concrete frame is provided. If no + ;; frame, the objects is returned untouched. + (filter-objects [objects frame-id] + (d/index-by :id (cph/get-children-with-self objects frame-id))) + + ;; function responsible of assoc available thumbnails + ;; to frames and remove all children shapes from objects if + ;; thumbnails is available + (assoc-thumbnails [objects page-id thumbnails] + (loop [objects objects + frames (filter cph/frame-shape? (vals objects))] + + (if-let [frame (-> frames first)] + (let [frame-id (:id frame) + object-id (str page-id frame-id) + frame (if-let [thumb (get thumbnails object-id)] + (assoc frame :thumbnail thumb :shapes []) + (dissoc frame :thumbnail)) + + children-ids + (cph/get-children-ids objects frame-id) + + bounds + (when (:show-content frame) + (gsh/selection-rect (concat [frame] (->> children-ids (map (d/getf objects)))))) + + frame + (cond-> frame + (some? bounds) + (assoc :children-bounds bounds))] + + (if (:thumbnail frame) + (recur (-> objects + (assoc frame-id frame) + (d/without-keys children-ids)) + (rest frames)) + (recur (assoc objects frame-id frame) + (rest frames)))) + + objects)))] + + (binding [pmap/*load-fn* (partial files/load-pointer conn id)] + (let [frame (get-thumbnail-frame data) + frame-id (:id frame) + page-id (or (:page-id frame) + (-> data :pages first)) + + page (dm/get-in data [:pages-index page-id]) + page (cond-> page (pmap/pointer-map? page) deref) + frame-ids (if (some? frame) (list frame-id) (map :id (ctt/get-frames (:objects page)))) + + obj-ids (map #(str page-id %) frame-ids) + thumbs (get-object-thumbnails conn id obj-ids)] + + (cond-> page + ;; If we have frame, we need to specify it on the page level + ;; and remove the all other unrelated objects. + (some? frame-id) + (-> (assoc :thumbnail-frame-id frame-id) + (update :objects filter-objects frame-id)) + + ;; Assoc the available thumbnails and prune not visible shapes + ;; for avoid transfer unnecessary data. + :always + (update :objects assoc-thumbnails page-id thumbs)))))) + +(s/def ::get-file-data-for-thumbnail + (s/keys :req [::rpc/profile-id] + :req-un [::file-id] + :opt-un [::features])) + +(sv/defmethod ::get-file-data-for-thumbnail + "Retrieves the data for generate the thumbnail of the file. Used + mainly for render thumbnails on dashboard." + {::doc/added "1.17"} + [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id features] :as props}] + (dm/with-open [conn (db/open pool)] + (files/check-read-permissions! conn profile-id file-id) + ;; NOTE: we force here the "storage/pointer-map" feature, because + ;; it used internally only and is independent if user supports it + ;; or not. + (let [feat (into #{"storage/pointer-map"} features) + file (files/get-file conn file-id feat)] + {:file-id file-id + :revn (:revn file) + :page (get-file-data-for-thumbnail conn file)}))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; MUTATION COMMANDS +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; --- MUTATION COMMAND: upsert-file-object-thumbnail + +(def sql:upsert-object-thumbnail-1 + "insert into file_object_thumbnail(file_id, object_id, data) + values (?, ?, ?) + on conflict(file_id, object_id) do + update set data = ?;") + +(def sql:upsert-object-thumbnail-2 + "insert into file_object_thumbnail(file_id, object_id, media_id) + values (?, ?, ?) + on conflict(file_id, object_id) do + update set media_id = ?;") + +(defn upsert-file-object-thumbnail! + [{:keys [::db/conn ::sto/storage]} {:keys [file-id object-id] :as params}] + + ;; NOTE: params can come with data set but with `nil` value, so we + ;; need first check the existence of the key and then the value. + (cond + (contains? params :data) + (if-let [data (:data params)] + (db/exec-one! conn [sql:upsert-object-thumbnail-1 file-id object-id data data]) + (db/delete! conn :file-object-thumbnail {:file-id file-id :object-id object-id})) + + (contains? params :media) + (if-let [{:keys [path mtype] :as media} (:media params)] + (let [_ (media/validate-media-type! media) + _ (media/validate-media-size! media) + hash (sto/calculate-hash path) + data (-> (sto/content path) + (sto/wrap-with-hash hash)) + media (sto/put-object! storage + {::sto/content data + ::sto/deduplicate? false + :content-type mtype + :bucket "file-object-thumbnail"})] + + (db/exec-one! conn [sql:upsert-object-thumbnail-2 file-id object-id (:id media) (:id media)])) + (db/delete! conn :file-object-thumbnail {:file-id file-id :object-id object-id})))) + +;; FIXME: change it on validation refactor +(s/def ::data (s/nilable ::us/string)) +(s/def ::media (s/nilable ::media/upload)) +(s/def ::object-id ::us/string) + +(s/def ::upsert-file-object-thumbnail + (s/keys :req [::rpc/profile-id] + :req-un [::file-id ::object-id] + :opt-un [::data ::media])) + +(sv/defmethod ::upsert-file-object-thumbnail + {::doc/added "1.17" + ::audit/skip true} + [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}] + + (assert (or (contains? params :data) + (contains? params :media))) + + (db/with-atomic [conn pool] + (files/check-edition-permissions! conn profile-id file-id) + + (when-not (db/read-only? conn) + (let [cfg (-> cfg + (update ::sto/storage media/configure-assets-storage) + (assoc ::db/conn conn))] + (upsert-file-object-thumbnail! cfg params) + nil)))) + +;; --- MUTATION COMMAND: upsert-file-thumbnail + +(def ^:private sql:upsert-file-thumbnail + "insert into file_thumbnail (file_id, revn, data, media_id, props) + values (?, ?, ?, ?, ?::jsonb) + on conflict(file_id, revn) do + update set data=?, media_id=?, props=?, updated_at=now();") + +(defn- upsert-file-thumbnail! + [{:keys [::db/conn ::sto/storage]} {:keys [file-id revn props] :as params}] + (let [props (db/tjson (or props {}))] + (cond + (contains? params :data) + (when-let [data (:data params)] + (db/exec-one! conn [sql:upsert-file-thumbnail + file-id revn data nil props data nil props])) + + (contains? params :media) + (when-let [{:keys [path mtype] :as media} (:media params)] + (let [_ (media/validate-media-type! media) + _ (media/validate-media-size! media) + hash (sto/calculate-hash path) + data (-> (sto/content path) + (sto/wrap-with-hash hash)) + media (sto/put-object! storage + {::sto/content data + ::sto/deduplicate? false + :content-type mtype + :bucket "file-thumbnail"})] + (db/exec-one! conn [sql:upsert-file-thumbnail + file-id revn nil (:id media) props nil (:id media) props])))))) + +(s/def ::revn ::us/integer) +(s/def ::props map?) +(s/def ::media ::media/upload) + +(s/def ::upsert-file-thumbnail + (s/keys :req [::rpc/profile-id] + :req-un [::file-id ::revn ::props] + :opt-un [::data ::media])) + +(sv/defmethod ::upsert-file-thumbnail + "Creates or updates the file thumbnail. Mainly used for paint the + grid thumbnails." + {::doc/added "1.17" + ::audit/skip true} + [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}] + (db/with-atomic [conn pool] + (files/check-edition-permissions! conn profile-id file-id) + (when-not (db/read-only? conn) + (let [cfg (-> cfg + (update ::sto/storage media/configure-assets-storage) + (assoc ::db/conn conn))] + (upsert-file-thumbnail! cfg params)) + nil))) diff --git a/backend/src/app/rpc/commands/media.clj b/backend/src/app/rpc/commands/media.clj index cd6ceeb86..9712f6035 100644 --- a/backend/src/app/rpc/commands/media.clj +++ b/backend/src/app/rpc/commands/media.clj @@ -41,15 +41,6 @@ (s/def ::file-id ::us/uuid) (s/def ::team-id ::us/uuid) -(defn validate-content-size! - [content] - (when (> (:size content) (cf/get :media-max-file-size default-max-file-size)) - (ex/raise :type :restriction - :code :media-max-file-size-reached - :hint (str/ffmt "the uploaded file size % is greater than the maximum %" - (:size content) - default-max-file-size)))) - ;; --- Create File Media object (upload) (declare create-file-media-object) @@ -68,7 +59,7 @@ (let [cfg (update cfg ::sto/storage media/configure-assets-storage)] (files/check-edition-permissions! pool profile-id file-id) (media/validate-media-type! content) - (validate-content-size! content) + (media/validate-media-size! content) (let [object (create-file-media-object cfg params) props {:name (:name params) :file-id file-id diff --git a/backend/src/app/tasks/file_gc.clj b/backend/src/app/tasks/file_gc.clj index 2ace1c827..34a394f45 100644 --- a/backend/src/app/tasks/file_gc.clj +++ b/backend/src/app/tasks/file_gc.clj @@ -18,7 +18,9 @@ [app.common.types.shape-tree :as ctt] [app.config :as cf] [app.db :as db] + [app.media :as media] [app.rpc.commands.files :as files] + [app.storage :as sto] [app.util.blob :as blob] [app.util.pointer-map :as pmap] [app.util.time :as dt] @@ -34,7 +36,7 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (defmethod ig/pre-init-spec ::handler [_] - (s/keys :req [::db/pool])) + (s/keys :req [::db/pool ::sto/storage])) (defmethod ig/prep-key ::handler [_ cfg] @@ -47,6 +49,7 @@ (db/with-atomic [conn pool] (let [min-age (dt/duration (or (:min-age params) (::min-age cfg))) cfg (-> cfg + (update ::sto/storage media/configure-assets-storage conn) (assoc ::db/conn conn) (assoc ::file-id file-id) (assoc ::min-age min-age)) @@ -141,36 +144,53 @@ (db/delete! conn :file-media-object {:id (:id mobj)})))) (defn- clean-file-object-thumbnails! - [conn file-id data] + [{:keys [::db/conn ::sto/storage]} file-id data] (let [stored (->> (db/query conn :file-object-thumbnail {:file-id file-id} {:columns [:object-id]}) (into #{} (map :object-id))) - get-objects-ids - (fn [{:keys [id objects]}] - (->> (ctt/get-frames objects) - (map #(str id (:id %))))) - - using (into #{} - (mapcat get-objects-ids) - (vals (:pages-index data))) + using (into #{} + (mapcat (fn [{:keys [id objects]}] + (->> (ctt/get-frames objects) + (map #(str id (:id %)))))) + (vals (:pages-index data))) unused (set/difference stored using)] (when (seq unused) (let [sql (str "delete from file_object_thumbnail " - " where file_id=? and object_id=ANY(?)") - res (db/exec-one! conn [sql file-id (db/create-array conn "text" unused)])] - (l/debug :hint "delete file object thumbnails" :file-id file-id :total (:next.jdbc/update-count res)))))) + " where file_id=? and object_id=ANY(?)" + " returning media_id") + res (db/exec! conn [sql file-id (db/create-array conn "text" unused)])] + + (doseq [media-id (into #{} (keep :media-id) res)] + ;; Mark as deleted the storage object related with the + ;; photo-id field. + (l/trace :hint "mark storage object as deleted" :id media-id) + (sto/del-object! storage media-id)) + + (l/debug :hint "delete file object thumbnails" + :file-id file-id + :total (count res)))))) (defn- clean-file-thumbnails! - [conn file-id revn] + [{:keys [::db/conn ::sto/storage]} file-id revn] (let [sql (str "delete from file_thumbnail " - " where file_id=? and revn < ?") - res (db/exec-one! conn [sql file-id revn])] - (when-not (zero? (:next.jdbc/update-count res)) - (l/debug :hint "delete file thumbnails" :file-id file-id :total (:next.jdbc/update-count res))))) + " where file_id=? and revn < ? " + " returning media_id") + res (db/exec! conn [sql file-id revn])] + + (when (seq res) + (doseq [media-id (into #{} (keep :media-id) res)] + ;; Mark as deleted the storage object related with the + ;; photo-id field. + (l/trace :hint "mark storage object as deleted" :id media-id) + (sto/del-object! storage media-id)) + + (l/debug :hint "delete file thumbnails" + :file-id file-id + :total (count res))))) (def ^:private sql:get-files-for-library @@ -252,7 +272,7 @@ (db/delete! conn :file-data-fragment {:id fragment-id :file-id file-id}))))) (defn- process-file - [{:keys [::db/conn]} {:keys [id data revn modified-at features] :as file}] + [{:keys [::db/conn] :as cfg} {:keys [id data revn modified-at features] :as file}] (l/debug :hint "processing file" :id id :modified-at modified-at) (binding [pmap/*load-fn* (partial files/load-pointer conn id)] @@ -261,8 +281,8 @@ (pmg/migrate-data))] (clean-file-media! conn id data) - (clean-file-object-thumbnails! conn id data) - (clean-file-thumbnails! conn id revn) + (clean-file-object-thumbnails! cfg id data) + (clean-file-thumbnails! cfg id revn) (clean-deleted-components! conn id data) (when (contains? features "storage/pointer-map") diff --git a/backend/test/backend_tests/helpers.clj b/backend/test/backend_tests/helpers.clj index 80f37853d..1d4e6f232 100644 --- a/backend/test/backend_tests/helpers.clj +++ b/backend/test/backend_tests/helpers.clj @@ -335,6 +335,20 @@ :session-id session-id :profile-id profile-id}))))) +(declare command!) + +(defn update-file! [& {:keys [profile-id file-id changes revn] :or {revn 0}}] + (let [params {::type :update-file + ::rpc/profile-id profile-id + :id file-id + :session-id (uuid/random) + :revn revn + :components-v2 true + :changes changes} + out (command! params)] + (t/is (nil? (:error out))) + (:result out))) + (defn create-webhook* ([params] (create-webhook* *pool* params)) ([pool {:keys [team-id id uri mtype is-active] diff --git a/backend/test/backend_tests/rpc_file_thumbnails_test.clj b/backend/test/backend_tests/rpc_file_thumbnails_test.clj new file mode 100644 index 000000000..468d8f08e --- /dev/null +++ b/backend/test/backend_tests/rpc_file_thumbnails_test.clj @@ -0,0 +1,314 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns backend-tests.rpc-file-thumbnails-test + (:require + [app.common.uuid :as uuid] + [app.config :as cf] + [app.db :as db] + [app.rpc :as-alias rpc] + [app.rpc.commands.auth :as cauth] + [app.storage :as sto] + [app.tokens :as tokens] + [app.util.time :as dt] + [backend-tests.helpers :as th] + [clojure.java.io :as io] + [clojure.test :as t] + [cuerdas.core :as str] + [datoteka.core :as fs] + [mockery.core :refer [with-mocks]])) + +(t/use-fixtures :once th/state-init) +(t/use-fixtures :each th/database-reset) + +(t/deftest upsert-file-object-thumbnail + (let [storage (::sto/storage th/*system*) + profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared false}) + + shid (uuid/random) + page-id (first (get-in file [:data :pages])) + + ;; Update file inserting a new frame object + _ (th/update-file! + :file-id (:id file) + :profile-id (:id profile) + :revn 0 + :changes + [{:type :add-obj + :page-id page-id + :id shid + :parent-id uuid/zero + :frame-id uuid/zero + :components-v2 true + :obj {:id shid + :name "Artboard" + :frame-id uuid/zero + :parent-id uuid/zero + :type :frame}}]) + + data1 {::th/type :upsert-file-object-thumbnail + ::rpc/profile-id (:id profile) + :file-id (:id file) + :object-id "test-key-1" + :media {:filename "sample.jpg" + :size 312043 + :path (th/tempfile "backend_tests/test_files/sample.jpg") + :mtype "image/jpeg"}} + + data2 {::th/type :upsert-file-object-thumbnail + ::rpc/profile-id (:id profile) + :file-id (:id file) + :object-id (str page-id shid) + :media {:filename "sample.jpg" + :size 7923 + :path (th/tempfile "backend_tests/test_files/sample2.jpg") + :mtype "image/jpeg"}}] + + (let [out (th/command! data1)] + (t/is (nil? (:error out))) + (t/is (nil? (:result out)))) + + (let [out (th/command! data2)] + (t/is (nil? (:error out))) + (t/is (nil? (:result out)))) + + (let [[row1 row2 :as rows] (th/db-query :file-object-thumbnail + {:file-id (:id file)} + {:order-by [[:created-at :asc]]})] + + (t/is (= 2 (count rows))) + (t/is (= (:file-id data1) (:file-id row1))) + (t/is (= (:object-id data1) (:object-id row1))) + (t/is (uuid? (:media-id row1))) + (t/is (= (:file-id data2) (:file-id row2))) + (t/is (= (:object-id data2) (:object-id row2))) + (t/is (uuid? (:media-id row2))) + + (let [sobject (sto/get-object storage (:media-id row1)) + mobject (meta sobject)] + (t/is (= "blake2b:4fdb63b8f3ffc81256ea79f13e53f366723b188554b5afed91b20897c14a1a8e" (:hash mobject))) + (t/is (= "file-object-thumbnail" (:bucket mobject))) + (t/is (= "image/jpeg" (:content-type mobject))) + (t/is (= 312043 (:size sobject)))) + + (let [sobject (sto/get-object storage (:media-id row2)) + mobject (meta sobject)] + (t/is (= "blake2b:05870e3f8ee885841ee3799924d80805179ab57e6fde84a605d1068fd3138de9" (:hash mobject))) + (t/is (= "file-object-thumbnail" (:bucket mobject))) + (t/is (= "image/jpeg" (:content-type mobject))) + (t/is (= 7923 (:size sobject)))) + + ;; Run the File GC task that should remove unused file object + ;; thumbnails + (let [result (th/run-task! :file-gc {:min-age (dt/duration 0)})] + (t/is (= 1 (:processed result)))) + + ;; check if row2 related thumbnail row still exists + (let [[row :as rows] (th/db-query :file-object-thumbnail + {:file-id (:id file)} + {:order-by [[:created-at :asc]]})] + (t/is (= 1 (count rows))) + (t/is (= (:file-id data2) (:file-id row))) + (t/is (= (:object-id data2) (:object-id row))) + (t/is (uuid? (:media-id row2)))) + + ;; Check if storage objects still exists after file-gc + (t/is (nil? (sto/get-object storage (:media-id row1)))) + (t/is (some? (sto/get-object storage (:media-id row2)))) + + ;; check that storage object is still exists but is marked as deleted + (let [row (th/db-get :storage-object {:id (:media-id row1)} {::db/remove-deleted? false})] + (t/is (some? (:deleted-at row)))) + + ;; Run the storage gc deleted task, it should permanently delete + ;; all storage objects related to the deleted thumbnails + (let [result (th/run-task! :storage-gc-deleted {:min-age (dt/duration 0)})] + (t/is (= 1 (:deleted result)))) + + ;; check that storage object is still exists but is marked as deleted + (let [row (th/db-get :storage-object {:id (:media-id row1)} {::db/remove-deleted? false})] + (t/is (nil? row))) + + (t/is (some? (sto/get-object storage (:media-id row2)))) + + + ))) + + +(t/deftest upsert-file-thumbnail + (let [storage (::sto/storage th/*system*) + profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared false + :revn 3}) + + data1 {::th/type :upsert-file-thumbnail + ::rpc/profile-id (:id profile) + :file-id (:id file) + :props {} + :revn 1 + :data "data:base64,1234123124"} + + data2 {::th/type :upsert-file-thumbnail + ::rpc/profile-id (:id profile) + :file-id (:id file) + :props {} + :revn 2 + :media {:filename "sample.jpg" + :size 7923 + :path (th/tempfile "backend_tests/test_files/sample2.jpg") + :mtype "image/jpeg"}} + + data3 {::th/type :upsert-file-thumbnail + ::rpc/profile-id (:id profile) + :file-id (:id file) + :props {} + :revn 3 + :media {:filename "sample.jpg" + :size 312043 + :path (th/tempfile "backend_tests/test_files/sample.jpg") + :mtype "image/jpeg"}}] + + (let [out (th/command! data1)] + ;; (th/print-result! out) + (t/is (nil? (:error out))) + (t/is (nil? (:result out)))) + + (let [out (th/command! data2)] + (t/is (nil? (:error out))) + (t/is (nil? (:result out)))) + + (let [out (th/command! data3)] + (t/is (nil? (:error out))) + (t/is (nil? (:result out)))) + + (let [[row1 row2 row3 :as rows] (th/db-query :file-thumbnail + {:file-id (:id file)} + {:order-by [[:created-at :asc]]})] + (t/is (= 3 (count rows))) + + (t/is (= (:file-id data1) (:file-id row1))) + (t/is (= (:revn data1) (:revn row1))) + (t/is (nil? (:media-id row1))) + + (t/is (= (:file-id data2) (:file-id row2))) + (t/is (= (:revn data2) (:revn row2))) + (t/is (uuid? (:media-id row2))) + + (t/is (= (:file-id data3) (:file-id row3))) + (t/is (= (:revn data3) (:revn row3))) + (t/is (uuid? (:media-id row3))) + + (let [sobject (sto/get-object storage (:media-id row2)) + mobject (meta sobject)] + (t/is (= "blake2b:05870e3f8ee885841ee3799924d80805179ab57e6fde84a605d1068fd3138de9" (:hash mobject))) + (t/is (= "file-thumbnail" (:bucket mobject))) + (t/is (= "image/jpeg" (:content-type mobject))) + (t/is (= 7923 (:size sobject)))) + + (let [sobject (sto/get-object storage (:media-id row3)) + mobject (meta sobject)] + (t/is (= "blake2b:4fdb63b8f3ffc81256ea79f13e53f366723b188554b5afed91b20897c14a1a8e" (:hash mobject))) + (t/is (= "file-thumbnail" (:bucket mobject))) + (t/is (= "image/jpeg" (:content-type mobject))) + (t/is (= 312043 (:size sobject)))) + + ;; Run the File GC task that should remove unused file object + ;; thumbnails + (let [result (th/run-task! :file-gc {:min-age (dt/duration 0)})] + (t/is (= 1 (:processed result)))) + + ;; check if row2 related thumbnail row still exists + (let [[row :as rows] (th/db-query :file-thumbnail + {:file-id (:id file)} + {:order-by [[:created-at :asc]]})] + (t/is (= 1 (count rows))) + (t/is (= (:file-id data2) (:file-id row))) + (t/is (= (:object-id data2) (:object-id row))) + (t/is (uuid? (:media-id row2)))) + + ;; Check if storage objects still exists after file-gc + (t/is (nil? (sto/get-object storage (:media-id row1)))) + (t/is (nil? (sto/get-object storage (:media-id row2)))) + (t/is (some? (sto/get-object storage (:media-id row3)))) + + (let [row (th/db-get :storage-object {:id (:media-id row2)} {::db/remove-deleted? false})] + (t/is (some? (:deleted-at row)))) + + ;; Run the storage gc deleted task, it should permanently delete + ;; all storage objects related to the deleted thumbnails + (let [result (th/run-task! :storage-gc-deleted {:min-age (dt/duration 0)})] + (t/is (= 1 (:deleted result)))) + + ;; check that storage object is still exists but is marked as deleted + (let [row (th/db-get :storage-object {:id (:media-id row1)} {::db/remove-deleted? false})] + (t/is (nil? row))) + + (t/is (some? (sto/get-object storage (:media-id row3)))) + + + ))) + +(t/deftest get-file-object-thumbnail + (let [storage (::sto/storage th/*system*) + profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared false}) + + data1 {::th/type :upsert-file-object-thumbnail + ::rpc/profile-id (:id profile) + :file-id (:id file) + :object-id "test-key-1" + :data "data:base64,1234123124"} + + data2 {::th/type :upsert-file-object-thumbnail + ::rpc/profile-id (:id profile) + :file-id (:id file) + :object-id "test-key-2" + :media {:filename "sample.jpg" + :size 7923 + :path (th/tempfile "backend_tests/test_files/sample2.jpg") + :mtype "image/jpeg"}}] + + (let [out (th/command! data1)] + (t/is (nil? (:error out))) + (t/is (nil? (:result out)))) + + (let [out (th/command! data2)] + (t/is (nil? (:error out))) + (t/is (nil? (:result out)))) + + (let [[row1 row2 :as rows] (th/db-query :file-object-thumbnail + {:file-id (:id file)} + {:order-by [[:created-at :asc]]})] + (t/is (= 2 (count rows))) + + (t/is (= (:file-id data1) (:file-id row1))) + (t/is (= (:object-id data1) (:object-id row1))) + (t/is (nil? (:media-id row1))) + (t/is (string? (:data row1))) + + (t/is (= (:file-id data2) (:file-id row2))) + (t/is (= (:object-id data2) (:object-id row2))) + (t/is (uuid? (:media-id row2)))) + + + (let [params {::th/type :get-file-object-thumbnails + ::rpc/profile-id (:id profile) + :file-id (:id file)} + out (th/command! params)] + + (let [result (:result out)] + (t/is (contains? result "test-key-1")) + (t/is (contains? result "test-key-2")))))) + + + diff --git a/backend/test/backend_tests/test_files/sample2.jpg b/backend/test/backend_tests/test_files/sample2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..037f5a51420c5be8e653cef69e49a3f723adca40 GIT binary patch literal 7923 zcmb7IWmFVSyj?n^yO!JqmQs+AmJ(Qc=@JkSSUMyY5ky)#mImqW?vO@Ax}}y5=@fbV zKfL$--8pCG{PNs$XFkl#Eq-c+Q00sa69UbF|?f(H23j-S+ z2Nw{UGV_*WXNU>S?amdJ7Wwb1CDFj?UQ?h|%VejH{ z*4?PsUkYjm#`jD-RnwyZo;v?0#sA`|8i4tvz*A7@i3Buv&GL}<@E>GK(W8ro6UuC`_WB?dN*ekqL_5htZ?g}gcf)860;TS zZ|ru~Qn$0C^7s}*JR%o=eSaozOL{+Qb?b}loK$q>(qeFlnJrU5NGil9t9mHAoE6>ih67f)M7q4Rsj>2JyDLm+v z8qPvp$i3vN2eW-G_>tegUize(7BW>DQUi;^u5&VR9CvFu7T!m|CjjUXz{z~3;9~eK zcT|E4$v=8<3)Jj%joxeD`9gJg!rDig9|QF<~J-c2fuPk<#D{L_3tydkCi5R zo_(V<*uG_GE&d27T^;pU-MsWK-oKaV{vAD_^J!Liv;NjpPbg{*bM6vi;+(_t9 zrGvin1=o1lT9NKes@S|8lGa>8f4#UwAu*&k{1&764l!G`;QRZ4QIf>8Q=ZS0AS6DY zIcp=nw&LNsN;GcV3i`s~b~-J@by zk&%&%WKb6w`lxQ;Crohfk`u(lupA5nM0&gM+ z$?da1gFv-|@?FRP4*>KI+Mlf~$rEf5q?uE_`wJQ!cFJ@OsJ4$5VVatG>n}p$OSsi# zfX5kEPSW_2*X-@Ldpl)5HI6aP(&>i?l1?$}h~H*Y4{=qpeIZD4+m;o_V)Lxo#dw_s z%y_pXl$&yHM^;V@z9W`B!+9qpDP~bDV>pg+m{%-P_*8b;jmDxe?9~yU7lRPrd#4nYr`yW*@a?Lh^*m{j%dX+ta*dM>SsHw<(q%g5MWnZ$p;QinXsl;Hw zbwwMPPRcP32BGLKbP|zTJeILIOL=jzr5pQf1zbjYxeXz|f;hO0VgSU5kUTAXwpIlT z9F8EO159Q-W!dE+{SZ9PH^ex1lNTa_?5)0CQhi~(6Zlg>6PEfB*oX@lU+{qkjq<$3XHovaO1~RunY$h0Gh2M8P_(|b7Xy=vvH8UTt zeJ|*t7(Gd6T_2Q#t4~3-Md3Wmwz!OK?w?%nGP>3lj+deIH?UMn45o8#ThMC__OKyJ)d~~PujSZz zRyXmkiGK%9GT5#SMW>*c$q2i2O!gFP_L|7pz`OuXVFr3Ta-mW&4JmSjj6nIA#Kwk3 zCbz8Z|=8q=3gZ};w zXQF$)5l2r)=%^!D#ha`IPX?xwAJeN7)pdn`RHg-XYbbwXN@VYGa$*WKOhkNrZ&H3I zT~_zrn9fb16HX7m+El%Mi8jMcSfP#&MFnR9Gnk%#t`!eu%Gkgjb_pr7Z5K?R*$07Ff^$$lGOI7L^gMM zMbf}BunJxs6>y0Yo!+$PUj_Pxplw>NuUcI!l(ty=b=uS^xK6(t9wa_?7~?rA^h^tS zi0W&>=*6@B{H3Y+Lh5pVue|aDhZzpxuM7v@TM9V$6u(XBI9k%rc1RUv^AIK&@)I4| zN%rzQ+oCz=eU8Fmj3Gs3GPvbJ)NdkL4wb(meEsp>}w}DZ%QxlDID_^?t zD)!-U!j1uwt_xd>N{k_~Nm%6!0jZ+s>mG|K&kLwZD(%B!%$*Qq6ird>Vs`dxj0(AP z+C1+l*`f#Wb2q(%fXtdenbjSk{%=r(0`;o2r?=7Uda0Xf&6R>IV|L4aXOLpUi-Wkz ztKdp;Yo;(!U~eP=qmsI`(bh>;9yGTld}*UXs${hOYmB;u-F=ms`MyPBY5-DRSk^U5(Edk-+>#J`2%lB9?$)Lqr0P-9*P97v`+6(%ZR)f#fBtpp1D z$zWkjiadd7fB-JsKO2`k{&dHDV#?W_(?tYts%&?toCFuPI_FUw&Q12|oo;qs>7hb( zyndo_!NNH#{_JwMyy*~fmX_9z8yQo`l)kaMUsiJ`TtNIBCR|~2^g#u`IO|288chaO zIFP`I)2D47=+3t2%f^0P>nGnya<-QD!>+San|NA1!SQGoLD%gYU*pX*&pX#wRJ1NZ zb2Yl~hJ6=xC+A;J7YWZ&)G$7Ha0kA?8JlY?aiqwW&xw56OfiZP`zeYA8uWK^wkK2L z=X4Y2^q(by+||vgr{tNP@L8~3wD`%&*k}_r1!ex)ZK5>&#G*!cfub(rqFOp$Mo`6l zeprjgkk$kppL%sdP7*!TkT()i0wd`=$Zx=e{_I*5WkgLHE+Tted~T^~X!YAH$D1k6@4>_Z1%Sls zo6lef-n!42AbWkKra$GuE?)nB;`^5M6u2Ws>2u*kd*!-*+`e{ z9WIOp6|Qr~T?kL^5=*DXZxf&AvPr8Z;9BmLkGJu=D*Z(zbc+fzS}Tz1?ubOW*xwWH z#fqK{;wwq+p8N`^Qsrit;jR@KcQ*mw299j03Rm9#4Kekw1WNmI;VGvRa7*PK(^pjb z6pINx0xaGoFl@S-`mo2AntbxCh?^u=wahF=c?6x)?7&5VETCDR#Z>-aKB3@v#dX@e zahX`85q_dZNvTyg-*-b=Ej*^_i`WCdfXIXP-(rlDTowf_ICA_>17Rvob0QOuJes+p8N%-C&aizP4-h)+)f@018BPN7-%}t3hwbJxMtiici zXVKe`miAkL1NUhnGv^l<+el<$<%b#(q2<`sun;zg1q9~4#jT&+B-*14A`z*taD%__ ztBcnxQ4rM_r+N144t+2$J*5l=7Ue?3w@hv+8CQ?u$aI$gs|zRc^nXn++p2HR zTf(L6sV=S#lMZvYs9tRMqv+Z-lWy5MU`x*)J=90se^OL$pvvCIYbuoUw-cVed4mzn zYpYT-sV)sLh7MmGgD~PDLc-o|+A3p|Q@R@pb0_{Kc6*+|4>7rK-(TGwQyZ`8q_>A^ zqGz~HAIIFmhya`MfS=f<^ycTIsM&Fk*;gi%(nS@*4_~zH(=C(bP1(8=hzOMN9MEay zfV&RG?u$S?wqJpMaKwk2VHMYlQB?+`Fgguc~?=oMkRV~vFc$Tu0 z%2h8L>7jeFIowU0lG%Fq)W{D-i_my+th&mmiQ>Hy3T%+XVQ+oMnR$`5|4nH!EwS&N zo#plDAaJ0Hk<@00J}?E-U?ku=H}ES}*lTw^d5w#5oq}1ONpD4S53wz$Y7e*Rqiw^l zt74X%DuXR^3kc}MXE zMzX~74GaO8J4gWv4ODP}#-q5#Oq5Loab^DsSSpUwLw-ShD>bL$&RGEZl7INcAbYH* zT+;n!E-4hMUo(y(tXUeviUzyIwM(q#x#zHb7K>9{SU$ZlZf0|`+u~vFw+?_tLa+1) z6~=K}!#coWTLVmZFx@P8P5fUwl18$G9;NmHyx29RdCrYi`#QH_>f?6YeEu#<)9xXH zS{jG!Z3%olXG;YyQVEgDxX&gOvfh)vZu`v|Oqn)=eGp=VN7en$(TEGlX(pPN#?g6q za+AD8P@sjopZ3;J0^}<^!=>!V5@%*PHlbFhNG<~89@oX%X{ko+`m~z+^I~N{~b-r{s5W9;(P+ivdE7%tQAz)5r=(OqVd9) zMfRM->tBzv@MGMSjTCKBJ0G3=$CTL(r81$@$G>!nJQeTD%1=IL9hNL7?&}oSE${4?E$1s%xK#}$i|d6h(;U~**2&#uoAocBtLQYQgLpSD)MB0iV3MIM7Fhne6nS$kT* zlm~ChKz#Cb6bn-C9|2nGwG0tfA<_jNdtGcE)a@JOH}tgEf{0MoTYT709d`vDdgf?t zOyLQ(Za%ac7)9J&AeXzLqFx$YK|cTB6|I@4rOBp5;u0$|a6 zhmWErk#DK7yWb<_?op8$sJh;(Z*LH}vWYxRSy zFE4$nmlqST^uw$~Z+xMkd>&2)irAMwB_W`*P^J01=LU9}&giOR{4UhEr=}+iK7_?= zZM3~QZzyjA?Gi(j_}pLHIHctp&~YcT*Jak|e`TLAXc5Hr9MfY>5-;;+Z)H_WGENgM z>wBZe$GJSS!3RZMoa@Sly)R13eUYWKozgouDEJ*sOr)H|28%SzH$q*er<~%%!++{f zq(MaM2-znPv>=$at!hLSeFkd3`(pbO>XwLE23aP{IcOTtO092?Se|BGO`SXBbTmCv zB#=uHqNNVAmv`5Sv(0z}T*}Nmh*}Zs6`RT>TQ?^$HJhf1fjbB7dZ2^eC*wUmm$$`A zwTc9$m!EPnW)?A$7i!H-wo(N&H)JtG#oN9_yar(<1iGkefX-iAx?M1p0j=SV*+q9^ z;3&e{VQtN=>acYb3s?mVhk)hLmFY9bJ9s!~KT*eJSTpJoH8ou-Dk(M9K3|Mcn?LUO zCXID)s7Mif7ofR5G-f35l8F49sd=tnrHA1J5VP^G4Q$iCr>8ewNjy**y8C5V6qnET zNiu~bm9~$krX(D!Med41Q)G@wU*kn2baVNjGd&VBngYxjS@5u4JLTmmHY-DxCov@w zj^wTI*MtHM5f-_q14*T2mJJcS1}x|Y439hsQ0!~1RocHNt`P)Db6j}|`JFLzE~@HK*X&Y!Hab6ie%i~5C+D}Igs0c%#UW_;t_D}n2pD*pV~A6 zdE~VBJG)NAwt?5xv}IL7r%*MYftQe_0gHko`VpW#NIA+ob|JPI)S6K-BicmiF>)vP zJGfS^R?_RRb)WvDz6sgAslKg~d@wKEp%q&vUSfD~U5{A6St?Xfc2M|qik`>VXr*SJ z@|_baux`wr+yBoC0S2XW%r$f!MK-^Kvoyi1xfy8x#+cDo_+Z%K#YdZ3m8P64Y890? zvm1?;D<9kc-6yKo!_$GAN_&0v=?>|eF?g_)h@FTjNDm_|OoI#LG)%XJP0F(6i*tZFSOUwcd3eN&>_H8j7+2JvD$}TJeM~x zv=$;i`$EpIae8eWmimDl5K5JJ(aNw~Ck6P%oRD{pNd)T3OI#O5?GjK~JAk-kg8c1j}OmnF(`IQx&1XjQT+p;l74eW>(8@|<6RPeMYgdaF%CHdZ2 zHQ4Sr;K?^b-hq}phlyH%-AVDUmpSwL#w)$HTY!$=zf`C}F1dw?i;mObM0xPWN-@7( z&Ad81Ew%=*qc_}$$^--oAd>Z@cW{BQ=^4wly-47itq{sT9NM>1qc7MJBXxZ@YU9I; zV@6DYL}50!mwkUWtY7%MJ{nROGLHRFZ@cL{jcjWzq-0-(lV~Q9=~eth{l(v{%=UGq zU99;$*5fng`G!NHP`@Y(ei|+!(Bi%}lS2keHEd>PI5H)D@^naMY?sGYzN4+V7S|=i zKIIu&rP#ypE|oXA(>53HW(@D5Z~?AG%iZLCZi;>+!MEwl!Q~w1A79_s{k5U{UtUJ#dAOpuDk|9>*EC{>(8lD+el5im(SoD zW6It24){*`V`{T{!dUz)oWn|h;w^b14cZd1QSR48eMArEzZm5qb#x~gaqHKaA!M#` z{ZJkEz=^IOIB)E3==A#fD+PYU;3j#GYNRSIpLcE}I|Kr|r>0vZqCmqx*JhHeOam7* zu5H+hzp-?62HtnYK3go~=m2+@ui$|s~CElrMewWz(bv>B%I2{%gkYt^Us z=)F63GmaF|eq$d>N4yd~v0?eB*F9ooW+gMkpx5n{{s>Umr%0%(A z32KC|xSTk+VpTLY-QZ4GI=$&}=h`k@g(TD2Y3Tv`7Pv|0N^4pM@4-i7#01V1x3Km+ z1S4&`EDi`onkGft{+c6u2Vo8H4j^L~TAnJ#--e{QsTsZCfJA^f(n`Zc_^puVX&wRF ze6de2pnSqL;y2?3!i4RlL)nzoj%%Z#r%!xki_Mmd(SFOD|JCbhieyp$v_jJuFY8`& zk=y$0xo@fP9YaaID}*d(QJ-`=JpOu&Eo|Rv=;P-x?x!_8)^&`pC-FyLRIvlS&!1%< zpPXv(p=9R!iFs#|&vTu&hZxnAJD!D_ls~YxY|?VWCD%0YKaiSqXE_TFV|cTb?@Xox zmWi4l0TeEdecVx}Pef#kv!9G@{Ludl%PapUrrSp9vs?H2xMR%!MjOjo*LK;6Ugc-a z?_IxQfl7sUZyA4V`Ww3RTq#MX%m2`ism3p*fwp$$IC;9XGaF~PP$}tN(-x9iFrrVI z03!PPYf&IB`KwC3cx1a{Fk#Lzg0Z50De_spXSyrh@=}_(OgrtMc$8RJv=576;UZxl z7j52WFH+wB7;?d*(VDyKPK~jb%?)ny0*?9l|8biy#YoNj>IttaJh-*nHX1A8x0&e9 z4=fO{v2|P$@4Xhi819uB3HJFkdle)o<|Q(^b}5;VgKWI$YH)Li;pgyy^V>lsT<{@-P7LtK04UiLpAxbJ_xf_z@x(%hl?M?KCXIGjxTEy*XOh0s2O|v$#s=7oA(KFPE~x*@$fia9Ev*i zds_@7YJQXK_tfY7$)ytFYDjI8Va6HG+guw<@^KLF)pj}rZ(3A1`71swj0nLonfu4^ zg2}QQltMh{+~>m4^lE;F&C2Q|*wwxSL=Q)Od!>_@_v%)#(J1$XroS^MfoCl7J%4X# zj>Po3@4xi|VivAgvRq>3wK>?nO3604!HPU$3if(RSaD~FxA{!_K5VNyNvy6GCRyue zpRHRGlGP*%6f+gP6y}t#x!6&I)>pSR9EM(EqBxF;J+*;=cx2pRT)dw$M?AXgt#pH5 nfu!G3{v99j)I;XCkcf&nQ(xpXF+b#-^DM8sisz5?