From c876534c859d410752e37360a2afa5d1a6ac6aa4 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 22 Mar 2022 17:22:53 +0100 Subject: [PATCH] :sparkles: Move the dashboard grid thumbnails to backend cache --- backend/src/app/migrations.clj | 3 + .../sql/0066-add-frame-thumbnail-table.sql | 3 + .../sql/0069-add-file-thumbnail-table.sql | 14 ++++ backend/src/app/rpc/helpers.clj | 16 +++++ backend/src/app/rpc/mutations/files.clj | 24 ++++++- backend/src/app/rpc/queries/files.clj | 58 +++++++++++---- frontend/src/app/main/ui/dashboard/grid.cljs | 60 +++------------- frontend/src/app/render.cljs | 1 - frontend/src/app/worker/thumbnails.cljs | 70 +++++++++++++++---- 9 files changed, 167 insertions(+), 82 deletions(-) create mode 100644 backend/src/app/migrations/sql/0069-add-file-thumbnail-table.sql create mode 100644 backend/src/app/rpc/helpers.clj diff --git a/backend/src/app/migrations.clj b/backend/src/app/migrations.clj index 76e6d0d68..7f552a532 100644 --- a/backend/src/app/migrations.clj +++ b/backend/src/app/migrations.clj @@ -214,6 +214,9 @@ {:name "0068-mod-storage-object-table" :fn (mg/resource "app/migrations/sql/0068-mod-storage-object-table.sql")} + + {:name "0069-add-file-thumbnail-table" + :fn (mg/resource "app/migrations/sql/0069-add-file-thumbnail-table.sql")} ]) diff --git a/backend/src/app/migrations/sql/0066-add-frame-thumbnail-table.sql b/backend/src/app/migrations/sql/0066-add-frame-thumbnail-table.sql index 3134cbe21..316a3ee6d 100644 --- a/backend/src/app/migrations/sql/0066-add-frame-thumbnail-table.sql +++ b/backend/src/app/migrations/sql/0066-add-frame-thumbnail-table.sql @@ -8,3 +8,6 @@ CREATE TABLE file_frame_thumbnail ( PRIMARY KEY(file_id, frame_id) ); + +ALTER TABLE file_frame_thumbnail + ALTER COLUMN data SET STORAGE external; diff --git a/backend/src/app/migrations/sql/0069-add-file-thumbnail-table.sql b/backend/src/app/migrations/sql/0069-add-file-thumbnail-table.sql new file mode 100644 index 000000000..d9a3fc2fe --- /dev/null +++ b/backend/src/app/migrations/sql/0069-add-file-thumbnail-table.sql @@ -0,0 +1,14 @@ +CREATE TABLE file_thumbnail ( + file_id uuid NOT NULL REFERENCES file(id) ON DELETE CASCADE, + revn bigint NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + deleted_at timestamptz NULL, + data text NULL, + props jsonb NULL, + PRIMARY KEY(file_id, revn) +); + +ALTER TABLE file_thumbnail + ALTER COLUMN data SET STORAGE external, + ALTER COLUMN props SET STORAGE external; diff --git a/backend/src/app/rpc/helpers.clj b/backend/src/app/rpc/helpers.clj new file mode 100644 index 000000000..f60879e95 --- /dev/null +++ b/backend/src/app/rpc/helpers.clj @@ -0,0 +1,16 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) UXBOX Labs SL + +(ns app.rpc.helpers + "General purpose RPC helpers." + (:require [app.common.data.macros :as dm])) + +(defn http-cache + [{:keys [max-age]}] + (fn [_ response] + (let [exp (if (integer? max-age) max-age (inst-ms max-age)) + val (dm/fmt "max-age=%" (int (/ exp 1000.0)))] + (update response :headers assoc "cache-control" val)))) diff --git a/backend/src/app/rpc/mutations/files.clj b/backend/src/app/rpc/mutations/files.clj index 19a00a404..cb6bd5aa3 100644 --- a/backend/src/app/rpc/mutations/files.clj +++ b/backend/src/app/rpc/mutations/files.clj @@ -487,14 +487,34 @@ update set data = ?;") (s/def ::data ::us/string) -(s/def ::upsert-frame-thumbnail +(s/def ::upsert-file-frame-thumbnail (s/keys :req-un [::profile-id ::file-id ::frame-id ::data])) -(sv/defmethod ::upsert-frame-thumbnail +(sv/defmethod ::upsert-file-frame-thumbnail [{:keys [pool] :as cfg} {:keys [profile-id file-id frame-id data]}] (db/with-atomic [conn pool] (files/check-edition-permissions! conn profile-id file-id) (db/exec-one! conn [sql:upsert-frame-thumbnail file-id frame-id data data]) nil)) +;; --- Mutation: Upsert file thumbnail +(def sql:upsert-file-thumbnail + "insert into file_thumbnail(file_id, revn, data, props) + values (?, ?, ?, ?) + on conflict(file_id, revn) do + update set data = ?, updated_at=now();") + +(s/def ::revn ::us/integer) +(s/def ::props (s/map-of ::us/keyword any?)) +(s/def ::upsert-file-thumbnail + (s/keys :req-un [::profile-id ::file-id ::revn ::data ::props])) + +(sv/defmethod ::upsert-file-thumbnail + [{:keys [pool] :as cfg} {:keys [profile-id file-id revn data props]}] + (db/with-atomic [conn pool] + (files/check-edition-permissions! conn profile-id file-id) + (let [props (db/tjson (or props {}))] + (db/exec-one! conn [sql:upsert-file-thumbnail + file-id revn data props data]) + nil))) diff --git a/backend/src/app/rpc/queries/files.clj b/backend/src/app/rpc/queries/files.clj index 9706c9e17..935b41c8e 100644 --- a/backend/src/app/rpc/queries/files.clj +++ b/backend/src/app/rpc/queries/files.clj @@ -7,11 +7,14 @@ (ns app.rpc.queries.files (:require [app.common.data :as d] + [app.common.exceptions :as ex] [app.common.pages.helpers :as cph] [app.common.pages.migrations :as pmg] [app.common.spec :as us] [app.common.uuid :as uuid] [app.db :as db] + [app.db.sql :as sql] + [app.rpc.helpers :as rpch] [app.rpc.permissions :as perms] [app.rpc.queries.projects :as projects] [app.rpc.queries.share-link :refer [retrieve-share-link]] @@ -267,7 +270,9 @@ [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as props}] (check-read-permissions! pool profile-id file-id) (let [file (retrieve-file cfg file-id)] - (get-thumbnail-data file props))) + {:data (get-thumbnail-data file props) + :file-id file-id + :revn (:revn file)})) (defn get-thumbnail-data [{:keys [data] :as file} props] @@ -325,7 +330,6 @@ (update data :objects update-objects))) - ;; --- Query: Shared Library Files (def ^:private sql:team-shared-files @@ -424,22 +428,48 @@ (teams/check-read-permissions! pool profile-id team-id) (db/exec! pool [sql:team-recent-files team-id])) +;; --- QUERY: get all file frame thumbnails -;; --- QUERY: get the thumbnail for an frame +(s/def ::file-frame-thumbnails + (s/keys :req-un [::profile-id ::file-id] + :opt-un [::frame-id])) -(def ^:private sql:file-frame-thumbnail - "select data - from file_frame_thumbnail - where file_id = ? - and frame_id = ?") - -(s/def ::file-frame-thumbnail - (s/keys :req-un [::profile-id ::file-id ::frame-id])) - -(sv/defmethod ::file-frame-thumbnail +(sv/defmethod ::file-frame-thumbnails [{:keys [pool]} {:keys [profile-id file-id frame-id]}] (check-read-permissions! pool profile-id file-id) - (db/exec-one! pool [sql:file-frame-thumbnail file-id frame-id])) + (let [params (cond-> {:file-id file-id} + frame-id (assoc :frame-id frame-id)) + rows (db/query pool :file-frame-thumbnail params)] + (d/group-by :frame-id :data rows))) + +;; --- QUERY: get file thumbnail + +(s/def ::revn ::us/integer) + +(s/def ::file-thumbnail + (s/keys :req-un [::profile-id ::file-id] + :opt-un [::revn])) + +(sv/defmethod ::file-thumbnail + [{:keys [pool]} {:keys [profile-id file-id revn]}] + (check-read-permissions! pool profile-id file-id) + (let [sql (sql/select :file-thumbnail + (cond-> {:file-id file-id} + revn (assoc :revn revn)) + {:limit 1 + :order-by [[:revn :desc]]}) + + row (db/exec-one! pool sql)] + + (when-not row + (ex/raise :type :not-found + :code :file-thumbnail-not-found)) + + (with-meta {:data (:data row) + :props (some-> (:props row) db/decode-transit-pgobject) + :revn (:revn row) + :file-id (:file-id row)} + {:transform-response (rpch/http-cache {:max-age (* 1000 60 60)})}))) ;; --- Helpers diff --git a/frontend/src/app/main/ui/dashboard/grid.cljs b/frontend/src/app/main/ui/dashboard/grid.cljs index 195fcee1d..33dd1159d 100644 --- a/frontend/src/app/main/ui/dashboard/grid.cljs +++ b/frontend/src/app/main/ui/dashboard/grid.cljs @@ -27,8 +27,6 @@ [app.util.timers :as ts] [app.util.webapi :as wapi] [beicon.core :as rx] - [cuerdas.core :as str] - [promesa.core :as p] [rumext.alpha :as mf])) (log/set-level! :warn) @@ -38,57 +36,15 @@ (def ^:const CACHE-NAME "penpot") (def ^:const CACHE-URL "https://penpot.app/cache/") - (defn use-thumbnail-cache "Creates some hooks to handle the files thumbnails cache" [file] - - (let [cache-url (str CACHE-URL (:id file) "/" (:revn file) ".svg") - get-thumbnail - (mf/use-callback - (mf/deps cache-url) - (fn [] - (p/let [response (.match js/caches cache-url)] - (when (some? response) - (p/let [blob (.blob response) - svg-content (.text blob) - headers (.-headers response) - fonts-header (or (.get headers "X-PENPOT-FONTS") "") - fonts (into #{} - (remove #(= "" %)) - (str/split fonts-header ","))] - {:svg svg-content - :fonts fonts}))))) - - cache-thumbnail - (mf/use-callback - (mf/deps cache-url) - (fn [{:keys [svg fonts]}] - (p/let [cache (.open js/caches CACHE-NAME) - blob (js/Blob. #js [svg] #js {:type "image/svg"}) - fonts (str/join "," fonts) - headers (js/Headers. #js {"X-PENPOT-FONTS" fonts}) - response (js/Response. blob #js {:headers headers})] - (.put cache cache-url response))))] - - (mf/use-callback - (mf/deps (:id file) (:revn file)) - (fn [] - (->> (rx/from (get-thumbnail)) - (rx/merge-map - (fn [thumb-data] - (log/debug :msg "retrieve thumbnail" :file (:id file) :revn (:revn file) - :cache (if (some? thumb-data) :hit :miss)) - - (if (some? thumb-data) - (rx/of thumb-data) - (->> (wrk/ask! {:cmd :thumbnails/generate - :file-id (:id file)}) - (rx/tap cache-thumbnail))))) - - ;; If we have a problem we delegate to the thumbnail generation - (rx/catch #(wrk/ask! {:cmd :thumbnails/generate - :file-id (:id file)}))))))) + (mf/use-fn + (mf/deps (:id file) (:revn file)) + (fn [] + (wrk/ask! {:cmd :thumbnails/generate + :revn (:revn file) + :file-id (:id file)})))) (mf/defc grid-item-thumbnail {::mf/wrap [mf/memo]} @@ -100,10 +56,10 @@ (mf/deps file) (fn [] (->> (generate) - (rx/subs (fn [{:keys [svg fonts]}] + (rx/subs (fn [{:keys [data fonts] :as params}] (run! fonts/ensure-loaded! fonts) (when-let [node (mf/ref-val container)] - (dom/set-html! node svg))))))) + (dom/set-html! node data))))))) [:div.grid-item-th {:style {:background-color (get-in file [:data :options :background])} :ref container} diff --git a/frontend/src/app/render.cljs b/frontend/src/app/render.cljs index d119796f8..b40802fb9 100644 --- a/frontend/src/app/render.cljs +++ b/frontend/src/app/render.cljs @@ -29,7 +29,6 @@ :version (:full @cf/version) :public-uri (str cf/public-uri)) - (defn- parse-params [loc] (let [href (unchecked-get loc "href")] diff --git a/frontend/src/app/worker/thumbnails.cljs b/frontend/src/app/worker/thumbnails.cljs index 5524c087d..ddd657643 100644 --- a/frontend/src/app/worker/thumbnails.cljs +++ b/frontend/src/app/worker/thumbnails.cljs @@ -16,6 +16,10 @@ [beicon.core :as rx] [rumext.alpha :as mf])) +(defn- not-found? + [{:keys [type]}] + (= :not-found type)) + (defn- handle-response [response] (cond @@ -29,30 +33,70 @@ (rx/throw {:type :unexpected :code (:error response)}))) -(defn- request-thumbnail - [file-id] - (let [uri (u/join (cfg/get-public-uri) "api/rpc/query/file-data-for-thumbnail") +(defn- request-data-for-thumbnail + [file-id revn] + (let [path "api/rpc/query/file-data-for-thumbnail" params {:file-id file-id + :revn revn :strip-frames-with-thumbnails true} request {:method :get - :uri uri + :uri (u/join (cfg/get-public-uri) path) :credentials "include" :query params}] (->> (http/send! request) (rx/map http/conditional-decode-transit) (rx/mapcat handle-response)))) -(defn render-frame - [data] +(defn- request-thumbnail + [file-id revn] + (let [path "api/rpc/query/file-thumbnail" + params {:file-id file-id + :revn revn} + request {:method :get + :uri (u/join (cfg/get-public-uri) path) + :credentials "include" + :query params}] + (->> (http/send! request) + (rx/map http/conditional-decode-transit) + (rx/mapcat handle-response)))) + +(defn- render-thumbnail + [{:keys [data file-id revn] :as params}] (let [elem (if-let [frame (:thumbnail-frame data)] (mf/element render/frame-svg #js {:objects (:objects data) :frame frame}) (mf/element render/page-svg #js {:data data :width "290" :height "150" :thumbnails? true}))] - (rds/renderToStaticMarkup elem))) + {:data (rds/renderToStaticMarkup elem) + :fonts @fonts/loaded + :file-id file-id + :revn revn})) + +(defn- persist-thumbnail + [{:keys [file-id data revn fonts]}] + (let [path "api/rpc/mutation/upsert-file-thumbnail" + params {:file-id file-id + :revn revn + :props {:fonts fonts} + :data data} + request {:method :post + :uri (u/join (cfg/get-public-uri) path) + :credentials "include" + :body (http/transit-data params)}] + (->> (http/send! request) + (rx/map http/conditional-decode-transit) + (rx/mapcat handle-response) + (rx/map (constantly params))))) (defmethod impl/handler :thumbnails/generate - [{:keys [file-id] :as message}] - (->> (request-thumbnail file-id) - (rx/map - (fn [data] - {:svg (render-frame data) - :fonts @fonts/loaded})))) + [{:keys [file-id revn] :as message}] + (letfn [(on-result [{:keys [data props]}] + {:data data + :fonts (:fonts props)}) + + (on-cache-miss [_] + (->> (request-data-for-thumbnail file-id revn) + (rx/map render-thumbnail) + (rx/mapcat persist-thumbnail)))] + + (->> (request-thumbnail file-id revn) + (rx/catch not-found? on-cache-miss) + (rx/map on-result))))