diff --git a/CHANGES.md b/CHANGES.md index d29a28d41..98ff1b78a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -43,6 +43,7 @@ ### :bug: Bugs fixed - Fix comments modal when changing pages [Taiga #2597](https://tree.taiga.io/project/penpot/issue/2508) +- Copy paste inside a text layer leaves pasted text transparent [Taiga #3096](https://tree.taiga.io/project/penpot/issue/3096) - On dashboard enter on empty search refresh the page [Taiga #2597](https://tree.taiga.io/project/penpot/issue/2597) - Pencil cursor changes when activated [Taiga #2276](https://tree.taiga.io/project/penpot/issue/2276) - Fix icon placement in Mixed message [Taiga #3037](https://tree.taiga.io/project/penpot/issue/3037) diff --git a/backend/deps.edn b/backend/deps.edn index 2f96d95ba..70251e7fa 100644 --- a/backend/deps.edn +++ b/backend/deps.edn @@ -19,7 +19,7 @@ io.lettuce/lettuce-core {:mvn/version "6.1.6.RELEASE"} java-http-clj/java-http-clj {:mvn/version "0.4.3"} - funcool/yetti {:git/tag "v8.0" :git/sha "ea7162d" + funcool/yetti {:git/tag "v9.0" :git/sha "e09e46c" :git/url "https://github.com/funcool/yetti.git" :exclusions [org.slf4j/slf4j-api]} diff --git a/backend/src/app/db.clj b/backend/src/app/db.clj index 704c7224a..c874fb9cc 100644 --- a/backend/src/app/db.clj +++ b/backend/src/app/db.clj @@ -233,14 +233,14 @@ ([ds table params opts] (exec-one! ds (sql/insert table params opts) - (assoc opts :return-keys true)))) + (merge {:return-keys true} opts)))) (defn insert-multi! ([ds table cols rows] (insert-multi! ds table cols rows nil)) ([ds table cols rows opts] (exec! ds (sql/insert-multi table cols rows opts) - (assoc opts :return-keys true)))) + (merge {:return-keys true} opts)))) (defn update! ([ds table params where] (update! ds table params where nil)) diff --git a/backend/src/app/http.clj b/backend/src/app/http.clj index 010bc941f..38c9340d0 100644 --- a/backend/src/app/http.clj +++ b/backend/src/app/http.clj @@ -8,6 +8,7 @@ (:require [app.common.data :as d] [app.common.logging :as l] + [app.common.transit :as t] [app.http.doc :as doc] [app.http.errors :as errors] [app.http.middleware :as middleware] @@ -67,7 +68,7 @@ :xnio/dispatch (:executor cfg) :ring/async true} handler (if (some? router) - (wrap-router cfg router) + (wrap-router router) handler) server (yt/server handler (d/without-nils options))] (assoc cfg :server (yt/start! server)))) @@ -81,30 +82,32 @@ [_ respond _] (respond (yrs/response 404))) -(defn- ring-handler - [router] - (fn [request respond raise] - (if-let [match (r/match-by-path router (yrq/path request))] - (let [params (:path-params match) - result (:result match) - handler (or (:handler result) not-found-handler) - request (-> request - (assoc :path-params params) - (update :params merge params))] - (handler request respond raise)) - (not-found-handler request respond raise)))) - (defn- wrap-router - [_ router] - (let [handler (ring-handler router)] + [router] + (letfn [(handler [request respond raise] + (if-let [match (r/match-by-path router (yrq/path request))] + (let [params (:path-params match) + result (:result match) + handler (or (:handler result) not-found-handler) + request (-> request + (assoc :path-params params) + (update :params merge params))] + (handler request respond raise)) + (not-found-handler request respond raise))) + + (on-error [cause request respond] + (let [{:keys [body] :as response} (errors/handle cause request)] + (respond + (cond-> response + (map? body) + (-> (update :headers assoc "content-type" "application/transit+json") + (assoc :body (t/encode-str body {:type :json-verbose})))))))] + (fn [request respond _] - (handler request respond - (fn [cause] - (l/error :hint "unexpected error processing request" - ::l/context (errors/get-context request) - :query-string (yrq/query request) - :cause cause) - (respond (yrs/response 500 "internal server error"))))))) + (try + (handler request respond #(on-error % request respond)) + (catch Throwable cause + (on-error cause request respond)))))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; HTTP ROUTER @@ -130,6 +133,8 @@ (rr/router [["" {:middleware [[middleware/server-timing] [middleware/format-response] + [middleware/params] + [middleware/parse-request] [middleware/errors errors/handle] [middleware/restrict-methods]]} ["/metrics" {:handler (:handler metrics)}] @@ -138,9 +143,7 @@ ["/by-file-media-id/:id" {:handler (:file-objects-handler assets)}] ["/by-file-media-id/:id/thumbnail" {:handler (:file-thumbnails-handler assets)}]] - ["/dbg" {:middleware [[middleware/params] - [middleware/parse-request] - (:middleware session)]} + ["/dbg" {:middleware [(:middleware session)]} ["" {:handler (:index debug)}] ["/error-by-id/:id" {:handler (:retrieve-error debug)}] ["/error/:id" {:handler (:retrieve-error debug)}] @@ -152,15 +155,11 @@ ["/sns" {:handler (:awsns-handler cfg) :allowed-methods #{:post}}]] - ["/ws/notifications" {:middleware [[middleware/params] - [middleware/parse-request] - (:middleware session)] + ["/ws/notifications" {:middleware [(:middleware session)] :handler ws :allowed-methods #{:get}}] ["/api" {:middleware [[middleware/cors] - [middleware/params] - [middleware/parse-request] (:middleware session)]} ["/health" {:handler (:health-check debug)}] ["/_doc" {:handler (doc/handler rpc) diff --git a/backend/src/app/http/errors.clj b/backend/src/app/http/errors.clj index 95d4f9819..bd40ded1d 100644 --- a/backend/src/app/http/errors.clj +++ b/backend/src/app/http/errors.clj @@ -24,8 +24,8 @@ (defn get-context [request] (merge - {:path (:uri request) - :method (:request-method request) + {:path (:path request) + :method (:method request) :params (:params request) :ip-addr (parse-client-ip request) :profile-id (:profile-id request)} @@ -51,9 +51,10 @@ [err _] (let [data (ex-data err) explain (us/pretty-explain data)] - (yrs/response 400 (-> data - (dissoc ::s/problems ::s/value) - (cond-> explain (assoc :explain explain)))))) + (yrs/response :status 400 + :body (-> data + (dissoc ::s/problems ::s/value) + (cond-> explain (assoc :explain explain)))))) (defmethod handle-exception :assertion [error request] @@ -73,26 +74,6 @@ [err _] (yrs/response 404 (ex-data err))) -(defmethod handle-exception :default - [error request] - (let [edata (ex-data error)] - ;; NOTE: this is a special case for the idle-in-transaction error; - ;; when it happens, the connection is automatically closed and - ;; next-jdbc combines the two errors in a single ex-info. We only - ;; need the :handling error, because the :rollback error will be - ;; always "connection closed". - (if (and (ex/exception? (:rollback edata)) - (ex/exception? (:handling edata))) - (handle-exception (:handling edata) request) - (do - (l/error ::l/raw (ex-message error) - ::l/context (get-context request) - :cause error) - (yrs/response 500 {:type :server-error - :code :unexpected - :hint (ex-message error) - :data edata}))))) - (defmethod handle-exception org.postgresql.util.PSQLException [error request] (let [state (.getSQLState ^java.sql.SQLException error)] @@ -101,24 +82,56 @@ :cause error) (cond (= state "57014") - (yrs/response 504 {:type :server-timeout + (yrs/response 504 {:type :server-error :code :statement-timeout :hint (ex-message error)}) (= state "25P03") - (yrs/response 504 {:type :server-timeout + (yrs/response 504 {:type :server-error :code :idle-in-transaction-timeout :hint (ex-message error)}) :else (yrs/response 500 {:type :server-error - :code :psql-exception + :code :unexpected :hint (ex-message error) :state state})))) +(defmethod handle-exception :default + [error request] + (let [edata (ex-data error)] + (cond + ;; This means that exception is not a controlled exception. + (nil? edata) + (do + (l/error ::l/raw (ex-message error) + ::l/context (get-context request) + :cause error) + (yrs/response 500 {:type :server-error + :code :unexpected + :hint (ex-message error)})) + + ;; This is a special case for the idle-in-transaction error; + ;; when it happens, the connection is automatically closed and + ;; next-jdbc combines the two errors in a single ex-info. We + ;; only need the :handling error, because the :rollback error + ;; will be always "connection closed". + (and (ex/exception? (:rollback edata)) + (ex/exception? (:handling edata))) + (handle-exception (:handling edata) request) + + :else + (do + (l/error ::l/raw (ex-message error) + ::l/context (get-context request) + :cause error) + (yrs/response 500 {:type :server-error + :code :unhandled + :hint (ex-message error) + :data edata}))))) (defn handle - [error req] + [error request] (if (or (instance? java.util.concurrent.CompletionException error) (instance? java.util.concurrent.ExecutionException error)) - (handle-exception (.getCause ^Throwable error) req) - (handle-exception error req))) + (handle-exception (.getCause ^Throwable error) request) + (handle-exception error request))) diff --git a/backend/src/app/http/middleware.clj b/backend/src/app/http/middleware.clj index df3d31c76..125d75e5e 100644 --- a/backend/src/app/http/middleware.clj +++ b/backend/src/app/http/middleware.clj @@ -6,6 +6,7 @@ (ns app.http.middleware (:require + [app.common.exceptions :as ex] [app.common.logging :as l] [app.common.transit :as t] [app.config :as cf] @@ -45,23 +46,17 @@ (update :params merge params)))) :else - request))) - - (handle-exception [cause] - (let [data {:type :validation - :code :unable-to-parse-request-body - :hint "malformed params"}] - (l/error :hint (ex-message cause) :cause cause) - (yrs/response :status 400 - :headers {"content-type" "application/transit+json"} - :body (t/encode-str data {:type :json-verbose}))))] + request)))] (fn [request respond raise] - (try - (let [request (process-request request)] - (handler request respond raise)) - (catch Exception cause - (respond (handle-exception cause))))))) + (when-let [request (try + (process-request request) + (catch Exception cause + (raise (ex/error :type :validation + :code :malformed-params + :hint (ex-message cause) + :cause cause))))] + (handler request respond raise))))) (def parse-request {:name ::parse-request diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index 4d785de3d..6a88a10c4 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -189,7 +189,7 @@ :pool (ig/ref :app.db/pool) :entries [{:cron #app/cron "0 0 0 * * ?" ;; daily - :task :file-media-gc} + :task :file-gc} {:cron #app/cron "0 0 * * * ?" ;; hourly :task :file-xlog-gc} @@ -231,7 +231,7 @@ :tasks {:sendmail (ig/ref :app.emails/sendmail-handler) :objects-gc (ig/ref :app.tasks.objects-gc/handler) - :file-media-gc (ig/ref :app.tasks.file-media-gc/handler) + :file-gc (ig/ref :app.tasks.file-gc/handler) :file-xlog-gc (ig/ref :app.tasks.file-xlog-gc/handler) :storage-deleted-gc (ig/ref :app.storage/gc-deleted-task) :storage-touched-gc (ig/ref :app.storage/gc-touched-task) @@ -262,7 +262,7 @@ :storage (ig/ref :app.storage/storage) :max-age cf/deletion-delay} - :app.tasks.file-media-gc/handler + :app.tasks.file-gc/handler {:pool (ig/ref :app.db/pool) :max-age cf/deletion-delay} diff --git a/backend/src/app/migrations.clj b/backend/src/app/migrations.clj index 76e6d0d68..7f552a532 100644 --- a/backend/src/app/migrations.clj +++ b/backend/src/app/migrations.clj @@ -214,6 +214,9 @@ {:name "0068-mod-storage-object-table" :fn (mg/resource "app/migrations/sql/0068-mod-storage-object-table.sql")} + + {:name "0069-add-file-thumbnail-table" + :fn (mg/resource "app/migrations/sql/0069-add-file-thumbnail-table.sql")} ]) diff --git a/backend/src/app/migrations/sql/0066-add-frame-thumbnail-table.sql b/backend/src/app/migrations/sql/0066-add-frame-thumbnail-table.sql index 3134cbe21..316a3ee6d 100644 --- a/backend/src/app/migrations/sql/0066-add-frame-thumbnail-table.sql +++ b/backend/src/app/migrations/sql/0066-add-frame-thumbnail-table.sql @@ -8,3 +8,6 @@ CREATE TABLE file_frame_thumbnail ( PRIMARY KEY(file_id, frame_id) ); + +ALTER TABLE file_frame_thumbnail + ALTER COLUMN data SET STORAGE external; diff --git a/backend/src/app/migrations/sql/0069-add-file-thumbnail-table.sql b/backend/src/app/migrations/sql/0069-add-file-thumbnail-table.sql new file mode 100644 index 000000000..d9a3fc2fe --- /dev/null +++ b/backend/src/app/migrations/sql/0069-add-file-thumbnail-table.sql @@ -0,0 +1,14 @@ +CREATE TABLE file_thumbnail ( + file_id uuid NOT NULL REFERENCES file(id) ON DELETE CASCADE, + revn bigint NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + deleted_at timestamptz NULL, + data text NULL, + props jsonb NULL, + PRIMARY KEY(file_id, revn) +); + +ALTER TABLE file_thumbnail + ALTER COLUMN data SET STORAGE external, + ALTER COLUMN props SET STORAGE external; diff --git a/backend/src/app/rpc/helpers.clj b/backend/src/app/rpc/helpers.clj new file mode 100644 index 000000000..f60879e95 --- /dev/null +++ b/backend/src/app/rpc/helpers.clj @@ -0,0 +1,16 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) UXBOX Labs SL + +(ns app.rpc.helpers + "General purpose RPC helpers." + (:require [app.common.data.macros :as dm])) + +(defn http-cache + [{:keys [max-age]}] + (fn [_ response] + (let [exp (if (integer? max-age) max-age (inst-ms max-age)) + val (dm/fmt "max-age=%" (int (/ exp 1000.0)))] + (update response :headers assoc "cache-control" val)))) diff --git a/backend/src/app/rpc/mutations/files.clj b/backend/src/app/rpc/mutations/files.clj index d3c605534..c3014bfc8 100644 --- a/backend/src/app/rpc/mutations/files.clj +++ b/backend/src/app/rpc/mutations/files.clj @@ -58,8 +58,9 @@ (db/insert! conn :file-profile-rel)))) (defn create-file - [conn {:keys [id name project-id is-shared data deleted-at] + [conn {:keys [id name project-id is-shared data deleted-at revn] :or {is-shared false + revn 0 deleted-at nil} :as params}] (let [id (or id (:id data) (uuid/next)) @@ -68,6 +69,7 @@ {:id id :project-id project-id :name name + :revn revn :is-shared is-shared :data (blob/encode data) :deleted-at deleted-at})] @@ -320,7 +322,7 @@ _ (mtx/run! metrics {:id :update-file-changes :inc (count changes)}) ts (dt/now) - file (-> (files/retrieve-data cfg file) + file (-> file (update :revn inc) (update :data (fn [data] ;; Trace the length of bytes of processed data @@ -487,14 +489,34 @@ update set data = ?;") (s/def ::data ::us/string) -(s/def ::upsert-frame-thumbnail +(s/def ::upsert-file-frame-thumbnail (s/keys :req-un [::profile-id ::file-id ::frame-id ::data])) -(sv/defmethod ::upsert-frame-thumbnail +(sv/defmethod ::upsert-file-frame-thumbnail [{:keys [pool] :as cfg} {:keys [profile-id file-id frame-id data]}] (db/with-atomic [conn pool] (files/check-edition-permissions! conn profile-id file-id) (db/exec-one! conn [sql:upsert-frame-thumbnail file-id frame-id data data]) nil)) +;; --- Mutation: Upsert file thumbnail +(def sql:upsert-file-thumbnail + "insert into file_thumbnail (file_id, revn, data, props) + values (?, ?, ?, ?::jsonb) + on conflict(file_id, revn) do + update set data = ?, props=?, updated_at=now();") + +(s/def ::revn ::us/integer) +(s/def ::props map?) +(s/def ::upsert-file-thumbnail + (s/keys :req-un [::profile-id ::file-id ::revn ::data ::props])) + +(sv/defmethod ::upsert-file-thumbnail + [{:keys [pool] :as cfg} {:keys [profile-id file-id revn data props]}] + (db/with-atomic [conn pool] + (files/check-edition-permissions! conn profile-id file-id) + (let [props (db/tjson (or props {}))] + (db/exec-one! conn [sql:upsert-file-thumbnail + file-id revn data props data props]) + nil))) diff --git a/backend/src/app/rpc/mutations/teams.clj b/backend/src/app/rpc/mutations/teams.clj index 494204ca8..ce5626d16 100644 --- a/backend/src/app/rpc/mutations/teams.clj +++ b/backend/src/app/rpc/mutations/teams.clj @@ -425,7 +425,6 @@ :email email :hint "looks like the email you invite has been repeatedly reported as spam or permanent bounce")) - (db/exec-one! conn [sql:upsert-team-invitation (:id team) (str/lower email) (name role) token-exp (name role) token-exp]) @@ -438,7 +437,6 @@ :token itoken :extra-data ptoken}))) - ;; --- Mutation: Create Team & Invite Members (s/def ::emails ::us/set-of-emails) @@ -463,7 +461,9 @@ :role role))) (with-meta team - {:before-complete + {::audit/props {:invitations (count emails)} + + :before-complete #(audit-fn :cmd :submit :type "mutation" :name "invite-team-member" diff --git a/backend/src/app/rpc/queries/files.clj b/backend/src/app/rpc/queries/files.clj index 30e349e8f..db6dfc45e 100644 --- a/backend/src/app/rpc/queries/files.clj +++ b/backend/src/app/rpc/queries/files.clj @@ -7,20 +7,21 @@ (ns app.rpc.queries.files (:require [app.common.data :as d] + [app.common.exceptions :as ex] [app.common.pages.helpers :as cph] [app.common.pages.migrations :as pmg] [app.common.spec :as us] [app.common.uuid :as uuid] [app.db :as db] + [app.db.sql :as sql] + [app.rpc.helpers :as rpch] [app.rpc.permissions :as perms] [app.rpc.queries.projects :as projects] [app.rpc.queries.share-link :refer [retrieve-share-link]] [app.rpc.queries.teams :as teams] - [app.storage.impl :as simpl] [app.util.blob :as blob] [app.util.services :as sv] - [clojure.spec.alpha :as s] - [promesa.core :as p])) + [clojure.spec.alpha :as s])) (declare decode-row) (declare decode-row-xf) @@ -186,25 +187,12 @@ ;; --- Query: File (By ID) -(defn- retrieve-data* - [{:keys [storage] :as cfg} file] - (p/do - (when-let [backend (simpl/resolve-backend storage (:data-backend file))] - (simpl/get-object-bytes backend file)))) - -(defn retrieve-data - [cfg file] - (if (bytes? (:data file)) - file - (p/->> (retrieve-data* cfg file) - (assoc file :data)))) - (defn retrieve-file [{:keys [pool] :as cfg} id] - (p/->> (db/get-by-id pool :file id) - (retrieve-data cfg) + (let [item (db/get-by-id pool :file id)] + (->> item (decode-row) - (pmg/migrate-file))) + (pmg/migrate-file)))) (s/def ::file (s/keys :req-un [::profile-id ::id])) @@ -214,8 +202,8 @@ [{:keys [pool] :as cfg} {:keys [profile-id id] :as params}] (let [perms (get-permissions pool profile-id id)] (check-read-permissions! perms) - (p/-> (retrieve-file cfg id) - (assoc :permissions perms)))) + (-> (retrieve-file cfg id) + (assoc :permissions perms)))) (declare trim-file-data) @@ -233,9 +221,9 @@ [{:keys [pool] :as cfg} {:keys [profile-id id] :as params}] (let [perms (get-permissions pool profile-id id)] (check-read-permissions! perms) - (p/-> (retrieve-file cfg id) - (trim-file-data params) - (assoc :permissions perms)))) + (-> (retrieve-file cfg id) + (trim-file-data params) + (assoc :permissions perms)))) (defn- trim-file-data [file {:keys [page-id object-id]}] @@ -248,9 +236,12 @@ (update :data assoc :pages-index {page-id page}) (update :data assoc :pages [page-id])))) +;; --- FILE THUMBNAIL + (declare strip-frames-with-thumbnails) (declare extract-file-thumbnail) (declare get-first-page-data) +(declare get-thumbnail-data) (s/def ::strip-frames-with-thumbnails ::us/boolean) @@ -258,6 +249,17 @@ (s/keys :req-un [::profile-id ::file-id] :opt-un [::strip-frames-with-thumbnails])) +(sv/defmethod ::page + "Retrieves the first page of the file. Used mainly for render + thumbnails on dashboard. + + DEPRECATED: still here for backward compatibility." + [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as props}] + (check-read-permissions! pool profile-id file-id) + (let [file (retrieve-file cfg file-id) + data (get-first-page-data file props)] + data)) + (s/def ::file-data-for-thumbnail (s/keys :req-un [::profile-id ::file-id] :opt-un [::strip-frames-with-thumbnails])) @@ -267,20 +269,31 @@ thumbnails on dashboard." [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as props}] (check-read-permissions! pool profile-id file-id) - (p/let [file (retrieve-file cfg file-id) - data (get-first-page-data file props) - file-thumbnail (extract-file-thumbnail (get-in file [:data :pages-index]))] + (let [file (retrieve-file cfg file-id)] + {:data (get-thumbnail-data file props) + :file-id file-id + :revn (:revn file)})) - (assoc data :file-thumbnail file-thumbnail))) +(defn get-thumbnail-data + [{:keys [data] :as file} props] + (if-let [[page frame] (first + (for [page (-> data :pages-index vals) + frame (-> page :objects cph/get-frames) + :when (:file-thumbnail frame)] + [page frame]))] + (let [objects (->> (cph/get-children-with-self (:objects page) (:id frame)) + (d/index-by :id))] + (cond-> (assoc page :objects objects) + (:strip-frames-with-thumbnails props) + (strip-frames-with-thumbnails) -(sv/defmethod ::page - "Retrieves the first page of the file. Used mainly for render - thumbnails on dashboard." - [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as props}] - (check-read-permissions! pool profile-id file-id) - (p/let [file (retrieve-file cfg file-id) - data (get-first-page-data file props)] - data)) + :always + (assoc :thumbnail-frame frame))) + + (let [page-id (-> data :pages first)] + (cond-> (get-in data [:pages-index page-id]) + (:strip-frames-with-thumbnails props) + (strip-frames-with-thumbnails))))) (defn get-first-page-data [file props] @@ -317,17 +330,6 @@ (update data :objects update-objects))) - -(defn extract-file-thumbnail - "Extract the frame marked as file-thumbnail" - [pages] - (->> pages - vals - (mapcat :objects) - vals - (filter :file-thumbnail) - first)) - ;; --- Query: Shared Library Files (def ^:private sql:team-shared-files @@ -384,7 +386,6 @@ [{:keys [pool] :as cfg} is-indirect file-id] (let [xform (comp (map #(assoc % :is-indirect is-indirect)) - (map #(retrieve-data cfg %)) (map decode-row))] (into #{} xform (db/exec! pool [sql:file-libraries file-id])))) @@ -427,22 +428,49 @@ (teams/check-read-permissions! pool profile-id team-id) (db/exec! pool [sql:team-recent-files team-id])) +;; --- QUERY: get all file frame thumbnails -;; --- QUERY: get the thumbnail for an frame +(s/def ::file-frame-thumbnails + (s/keys :req-un [::profile-id ::file-id] + :opt-un [::frame-id])) -(def ^:private sql:file-frame-thumbnail - "select data - from file_frame_thumbnail - where file_id = ? - and frame_id = ?") - -(s/def ::file-frame-thumbnail - (s/keys :req-un [::profile-id ::file-id ::frame-id])) - -(sv/defmethod ::file-frame-thumbnail +(sv/defmethod ::file-frame-thumbnails [{:keys [pool]} {:keys [profile-id file-id frame-id]}] (check-read-permissions! pool profile-id file-id) - (db/exec-one! pool [sql:file-frame-thumbnail file-id frame-id])) + (let [params (cond-> {:file-id file-id} + frame-id (assoc :frame-id frame-id)) + rows (db/query pool :file-frame-thumbnail params)] + (d/index-by :frame-id :data rows))) + +;; --- QUERY: get file thumbnail + +(s/def ::revn ::us/integer) + +(s/def ::file-thumbnail + (s/keys :req-un [::profile-id ::file-id] + :opt-un [::revn])) + +(sv/defmethod ::file-thumbnail + [{:keys [pool]} {:keys [profile-id file-id revn]}] + (check-read-permissions! pool profile-id file-id) + (let [sql (sql/select :file-thumbnail + (cond-> {:file-id file-id} + revn (assoc :revn revn)) + {:limit 1 + :order-by [[:revn :desc]]}) + + row (db/exec-one! pool sql)] + + (when-not row + (ex/raise :type :not-found + :code :file-thumbnail-not-found)) + + (with-meta + {:data (:data row) + :props (some-> (:props row) db/decode-transit-pgobject) + :revn (:revn row) + :file-id (:file-id row)} + {:transform-response (rpch/http-cache {:max-age (* 1000 60 60)})}))) ;; --- Helpers diff --git a/backend/src/app/storage.clj b/backend/src/app/storage.clj index 218835149..10e289b7e 100644 --- a/backend/src/app/storage.clj +++ b/backend/src/app/storage.clj @@ -274,7 +274,7 @@ (let [min-age (db/interval min-age) rows (db/exec! conn [sql:retrieve-deleted-objects-chunk min-age cursor])] [(some-> rows peek :created-at) - (some->> (seq rows) (d/group-by' #(-> % :backend keyword) :id) seq)])) + (some->> (seq rows) (d/group-by #(-> % :backend keyword) :id #{}) seq)])) (retrieve-deleted-objects [conn] (->> (d/iteration (fn [cursor] @@ -383,7 +383,7 @@ (mapv #(d/update-when % :metadata db/decode-transit-pgobject)))] (when (seq rows) [(-> rows peek :created-at) - (d/group-by' get-bucket :id rows)]))) + (d/group-by get-bucket :id #{} rows)]))) (retrieve-touched [conn] (->> (d/iteration (fn [cursor] diff --git a/backend/src/app/tasks/file_gc.clj b/backend/src/app/tasks/file_gc.clj new file mode 100644 index 000000000..b8669d96b --- /dev/null +++ b/backend/src/app/tasks/file_gc.clj @@ -0,0 +1,164 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) UXBOX Labs SL + +(ns app.tasks.file-gc + "A maintenance task that is responsible of: purge unused file media, + clean unused frame thumbnails and remove old file thumbnails. The + file is eligible to be garbage collected after some period of + inactivity (the default threshold is 72h)." + (:require + [app.common.data :as d] + [app.common.logging :as l] + [app.common.pages.helpers :as cph] + [app.common.pages.migrations :as pmg] + [app.db :as db] + [app.util.blob :as blob] + [app.util.time :as dt] + [clojure.spec.alpha :as s] + [integrant.core :as ig])) + +(declare ^:private retrieve-candidates) +(declare ^:private process-file) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; HANDLER +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(s/def ::max-age ::dt/duration) + +(defmethod ig/pre-init-spec ::handler [_] + (s/keys :req-un [::db/pool ::max-age])) + +(defmethod ig/init-key ::handler + [_ {:keys [pool] :as cfg}] + (fn [_] + (db/with-atomic [conn pool] + (let [cfg (assoc cfg :conn conn)] + (loop [total 0 + files (retrieve-candidates cfg)] + (if-let [file (first files)] + (do + (process-file cfg file) + (recur (inc total) + (rest files))) + (do + (l/debug :msg "finished processing files" :processed total) + {:processed total}))))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; IMPL +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(def ^:private + sql:retrieve-candidates-chunk + "select f.id, + f.data, + f.revn, + f.modified_at + from file as f + where f.has_media_trimmed is false + and f.modified_at < now() - ?::interval + and f.modified_at < ? + order by f.modified_at desc + limit 1 + for update skip locked") + +(defn- retrieve-candidates + [{:keys [conn max-age] :as cfg}] + (let [interval (db/interval max-age) + + get-chunk + (fn [cursor] + (let [rows (db/exec! conn [sql:retrieve-candidates-chunk interval cursor])] + [(some->> rows peek :modified-at) (seq rows)]))] + + (sequence cat (d/iteration get-chunk + :vf second + :kf first + :initk (dt/now))))) + +(defn- collect-used-media + [data] + (let [xform (comp + (map :objects) + (mapcat vals) + (keep (fn [{:keys [type] :as obj}] + (case type + :path (get-in obj [:fill-image :id]) + :image (get-in obj [:metadata :id]) + nil)))) + pages (concat + (vals (:pages-index data)) + (vals (:components data)))] + (-> #{} + (into xform pages) + (into (keys (:media data)))))) + +(defn- clean-file-media! + "Performs the garbage collection of file media objects." + [conn file-id data] + (let [used (collect-used-media data) + unused (->> (db/query conn :file-media-object {:file-id file-id}) + (remove #(contains? used (:id %))))] + + (doseq [mobj unused] + (l/debug :hint "delete file media object" + :id (:id mobj) + :media-id (:media-id mobj) + :thumbnail-id (:thumbnail-id mobj)) + + ;; NOTE: deleting the file-media-object in the database + ;; automatically marks as touched the referenced storage + ;; objects. The touch mechanism is needed because many files can + ;; point to the same storage objects and we can't just delete + ;; them. + (db/delete! conn :file-media-object {:id (:id mobj)})))) + +(defn- collect-frames + [data] + (let [xform (comp + (map :objects) + (mapcat vals) + (filter cph/frame-shape?) + (keep :id)) + pages (concat + (vals (:pages-index data)) + (vals (:components data)))] + (into #{} xform pages))) + +(defn- clean-file-frame-thumbnails! + [conn file-id data] + (let [sql (str "delete from file_frame_thumbnail " + " where file_id=? and not (frame_id=ANY(?))") + ids (->> (collect-frames data) + (db/create-array conn "uuid")) + res (db/exec-one! conn [sql file-id ids])] + (l/debug :hint "delete frame thumbnails" :total (:next.jdbc/update-count res)))) + +(defn- clean-file-thumbnails! + [conn file-id revn] + (let [sql (str "delete from file_thumbnail " + " where file_id=? and revn < ?") + res (db/exec-one! conn [sql file-id revn])] + (l/debug :hint "delete file thumbnails" :total (:next.jdbc/update-count res)))) + +(defn- process-file + [{:keys [conn] :as cfg} {:keys [id data revn modified-at] :as file}] + (l/debug :hint "processing file" :id id :modified-at modified-at) + + (let [data (-> (blob/decode data) + (assoc :id id) + (pmg/migrate-data))] + + (clean-file-media! conn id data) + (clean-file-frame-thumbnails! conn id data) + (clean-file-thumbnails! conn id revn) + + ;; Mark file as trimmed + (db/update! conn :file + {:has-media-trimmed true} + {:id id}) + nil)) diff --git a/backend/src/app/tasks/file_media_gc.clj b/backend/src/app/tasks/file_media_gc.clj deleted file mode 100644 index 40f55f53f..000000000 --- a/backend/src/app/tasks/file_media_gc.clj +++ /dev/null @@ -1,139 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) UXBOX Labs SL - -(ns app.tasks.file-media-gc - "A maintenance task that is responsible to purge the unused media - objects from files. A file is eligible to be garbage collected - after some period of inactivity (the default threshold is 72h)." - (:require - [app.common.logging :as l] - [app.common.pages.helpers :as cph] - [app.common.pages.migrations :as pmg] - [app.db :as db] - [app.util.blob :as blob] - [app.util.time :as dt] - [clojure.spec.alpha :as s] - [integrant.core :as ig])) - -(declare process-file) -(declare retrieve-candidates) - -(s/def ::max-age ::dt/duration) - -(defmethod ig/pre-init-spec ::handler [_] - (s/keys :req-un [::db/pool ::max-age])) - -(defmethod ig/init-key ::handler - [_ {:keys [pool] :as cfg}] - (fn [_] - (db/with-atomic [conn pool] - (let [cfg (assoc cfg :conn conn)] - (loop [n 0] - (let [files (retrieve-candidates cfg)] - (if (seq files) - (do - (run! (partial process-file cfg) files) - (recur (+ n (count files)))) - (do - (l/debug :msg "finished processing files" :processed n) - {:processed n})))))))) - -(def ^:private - sql:retrieve-candidates-chunk - "select f.id, - f.data, - extract(epoch from (now() - f.modified_at))::bigint as age - from file as f - where f.has_media_trimmed is false - and f.modified_at < now() - ?::interval - order by f.modified_at asc - limit 10 - for update skip locked") - - -(defn- retrieve-candidates - [{:keys [conn max-age] :as cfg}] - (let [interval (db/interval max-age)] - (->> (db/exec! conn [sql:retrieve-candidates-chunk interval]) - (mapv (fn [{:keys [age] :as row}] - (assoc row :age (dt/duration {:seconds age}))))))) - -(def ^:private - collect-media-xf - (comp - (map :objects) - (mapcat vals) - (keep (fn [{:keys [type] :as obj}] - (case type - :path (get-in obj [:fill-image :id]) - :image (get-in obj [:metadata :id]) - nil))))) - -(defn- collect-used-media - [data] - (let [pages (concat - (vals (:pages-index data)) - (vals (:components data)))] - (-> #{} - (into collect-media-xf pages) - (into (keys (:media data)))))) - -(def ^:private - collect-frames-xf - (comp - (map :objects) - (mapcat vals) - (filter cph/frame-shape?) - (keep :id))) - -(defn- collect-frames - [data] - (let [pages (concat - (vals (:pages-index data)) - (vals (:components data)))] - (into #{} collect-frames-xf pages))) - -(defn- process-file - [{:keys [conn] :as cfg} {:keys [id data age] :as file}] - (let [data (-> (blob/decode data) - (assoc :id id) - (pmg/migrate-data))] - - (let [used (collect-used-media data) - unused (->> (db/query conn :file-media-object {:file-id id}) - (remove #(contains? used (:id %))))] - - (l/debug :hint "processing file" - :id id - :age age - :to-delete (count unused)) - - ;; Mark file as trimmed - (db/update! conn :file - {:has-media-trimmed true} - {:id id}) - - (doseq [mobj unused] - (l/debug :hint "deleting media object" - :id (:id mobj) - :media-id (:media-id mobj) - :thumbnail-id (:thumbnail-id mobj)) - - ;; NOTE: deleting the file-media-object in the database - ;; automatically marks as touched the referenced storage - ;; objects. The touch mechanism is needed because many files can - ;; point to the same storage objects and we can't just delete - ;; them. - (db/delete! conn :file-media-object {:id (:id mobj)}))) - - (let [sql (str "delete from file_frame_thumbnail " - " where file_id = ? and not (frame_id = ANY(?))") - ids (->> (collect-frames data) - (db/create-array conn "uuid"))] - ;; delete the unused frame thumbnails - (db/exec! conn [sql (:id file) ids])) - - nil)) diff --git a/backend/test/app/services_files_test.clj b/backend/test/app/services_files_test.clj index 64c69feee..3977a1317 100644 --- a/backend/test/app/services_files_test.clj +++ b/backend/test/app/services_files_test.clj @@ -8,6 +8,7 @@ (:require [app.common.uuid :as uuid] [app.db :as db] + [app.db.sql :as sql] [app.http :as http] [app.storage :as sto] [app.test-helpers :as th] @@ -117,7 +118,7 @@ (t/is (= 0 (count result)))))) )) -(t/deftest file-media-gc-task +(t/deftest file-gc-task (letfn [(create-file-media-object [{:keys [profile-id file-id]}] (let [mfile {:filename "sample.jpg" :path (th/tempfile "app/test_files/sample.jpg") @@ -130,6 +131,9 @@ :name "testfile" :content mfile} out (th/mutation! params)] + + ;; (th/print-result! out) + (t/is (nil? (:error out))) (:result out))) @@ -189,7 +193,7 @@ (t/is (= 0 (:delete res)))) ;; run the task immediately - (let [task (:app.tasks.file-media-gc/handler th/*system*) + (let [task (:app.tasks.file-gc/handler th/*system*) res (task {})] (t/is (= 0 (:processed res)))) @@ -198,7 +202,7 @@ (th/sleep 300) ;; run the task again - (let [task (:app.tasks.file-media-gc/handler th/*system*) + (let [task (:app.tasks.file-gc/handler th/*system*) res (task {})] (t/is (= 1 (:processed res)))) @@ -342,7 +346,7 @@ (t/is (th/ex-info? error)) (t/is (th/ex-of-type? error :not-found)))) -(t/deftest deletion-test +(t/deftest deletion (let [task (:app.tasks.objects-gc/handler th/*system*) profile1 (th/create-profile* 1) file (th/create-file* 1 {:project-id (:default-project-id profile1) @@ -410,71 +414,158 @@ )) (t/deftest query-frame-thumbnails - (let [prof (th/create-profile* 1 {:is-active true}) - file (th/create-file* 1 {:profile-id (:id prof) - :project-id (:default-project-id prof) - :is-shared false}) - data {::th/type :file-frame-thumbnail - :profile-id (:id prof) - :file-id (:id file) - :frame-id (uuid/next)}] - - ;;insert an entry on the database with a test value for the thumbnail of this frame - (db/exec-one! th/*pool* - ["insert into file_frame_thumbnail(file_id, frame_id, data) values (?, ?, ?)" - (:file-id data) (:frame-id data) "testvalue"]) - - (let [out (th/query! data)] - (t/is (nil? (:error out))) - (let [result (:result out)] - (t/is (= 1 (count result))) - (t/is (= "testvalue" (:data result))))))) - -(t/deftest insert-frame-thumbnails - (let [prof (th/create-profile* 1 {:is-active true}) - file (th/create-file* 1 {:profile-id (:id prof) - :project-id (:default-project-id prof) - :is-shared false}) - data {::th/type :upsert-frame-thumbnail - :profile-id (:id prof) - :file-id (:id file) - :frame-id (uuid/next) - :data "test insert new value"} - out (th/mutation! data)] - - (t/is (nil? (:error out))) - (t/is (nil? (:result out))) - - ;;retrieve the value from the database and check its content - (let [result (db/exec-one! - th/*pool* - ["select data from file_frame_thumbnail where file_id = ? and frame_id = ?" - (:file-id data) (:frame-id data)])] - (t/is (= "test insert new value" (:data result)))))) - -(t/deftest frame-thumbnails (let [prof (th/create-profile* 1 {:is-active true}) file (th/create-file* 1 {:profile-id (:id prof) :project-id (:default-project-id prof) :is-shared false}) - data {::th/type :upsert-frame-thumbnail + data {::th/type :file-frame-thumbnails + :profile-id (:id prof) + :file-id (:id file) + :frame-id (uuid/next)}] + + ;; insert an entry on the database with a test value for the thumbnail of this frame + (th/db-insert! :file-frame-thumbnail + {:file-id (:file-id data) + :frame-id (:frame-id data) + :data "testvalue"}) + + (let [{:keys [result error] :as out} (th/query! data)] + ;; (th/print-result! out) + (t/is (nil? error)) + (t/is (= 1 (count result))) + (t/is (= "testvalue" (get result (:frame-id data))))))) + +(t/deftest insert-frame-thumbnails + (let [prof (th/create-profile* 1 {:is-active true}) + file (th/create-file* 1 {:profile-id (:id prof) + :project-id (:default-project-id prof) + :is-shared false}) + data {::th/type :upsert-file-frame-thumbnail + :profile-id (:id prof) + :file-id (:id file) + :frame-id (uuid/next) + :data "test insert new value"}] + + (let [out (th/mutation! data)] + (t/is (nil? (:error out))) + (t/is (nil? (:result out))) + (let [[result] (th/db-query :file-frame-thumbnail + {:file-id (:file-id data) + :frame-id (:frame-id data)})] + (t/is (= "test insert new value" (:data result))))))) + +(t/deftest upsert-frame-thumbnails + (let [prof (th/create-profile* 1 {:is-active true}) + file (th/create-file* 1 {:profile-id (:id prof) + :project-id (:default-project-id prof) + :is-shared false}) + data {::th/type :upsert-file-frame-thumbnail :profile-id (:id prof) :file-id (:id file) :frame-id (uuid/next) :data "updated value"}] - ;;insert an entry on the database with and old value for the thumbnail of this frame - (db/exec-one! th/*pool* - ["insert into file_frame_thumbnail(file_id, frame_id, data) values (?, ?, ?)" - (:file-id data) (:frame-id data) "old value"]) + ;; insert an entry on the database with and old value for the thumbnail of this frame + (th/db-insert! :file-frame-thumbnail + {:file-id (:file-id data) + :frame-id (:frame-id data) + :data "old value"}) (let [out (th/mutation! data)] + ;; (th/print-result! out) + (t/is (nil? (:error out))) (t/is (nil? (:result out))) - ;;retrieve the value from the database and check its content - (let [result (db/exec-one! - th/*pool* - ["select data from file_frame_thumbnail where file_id = ? and frame_id = ?" - (:file-id data) (:frame-id data)])] + ;; retrieve the value from the database and check its content + (let [[result] (th/db-query :file-frame-thumbnail + {:file-id (:file-id data) + :frame-id (:frame-id data)})] (t/is (= "updated value" (:data result))))))) + + +(t/deftest file-thumbnail-ops + (let [prof (th/create-profile* 1 {:is-active true}) + file (th/create-file* 1 {:profile-id (:id prof) + :project-id (:default-project-id prof) + :revn 2 + :is-shared false}) + data {::th/type :file-thumbnail + :profile-id (:id prof) + :file-id (:id file)}] + + (t/testing "query a thumbnail with single revn" + + ;; insert an entry on the database with a test value for the thumbnail of this frame + (th/db-insert! :file-thumbnail + {:file-id (:file-id data) + :revn 1 + :data "testvalue1"}) + + (let [{:keys [result error] :as out} (th/query! data)] + ;; (th/print-result! out) + (t/is (nil? error)) + (t/is (= 4 (count result))) + (t/is (= "testvalue1" (:data result))) + (t/is (= 1 (:revn result))))) + + (t/testing "query thumbnail with two revisions" + ;; insert an entry on the database with a test value for the thumbnail of this frame + (th/db-insert! :file-thumbnail + {:file-id (:file-id data) + :revn 2 + :data "testvalue2"}) + + (let [{:keys [result error] :as out} (th/query! data)] + ;; (th/print-result! out) + (t/is (nil? error)) + (t/is (= 4 (count result))) + (t/is (= "testvalue2" (:data result))) + (t/is (= 2 (:revn result)))) + + ;; Then query the specific revn + (let [{:keys [result error] :as out} (th/query! (assoc data :revn 1))] + ;; (th/print-result! out) + (t/is (nil? error)) + (t/is (= 4 (count result))) + (t/is (= "testvalue1" (:data result))) + (t/is (= 1 (:revn result))))) + + (t/testing "upsert file-thumbnail" + (let [data {::th/type :upsert-file-thumbnail + :profile-id (:id prof) + :file-id (:id file) + :data "foobar" + :props {:baz 1} + :revn 2} + {:keys [result error] :as out} (th/mutation! data)] + ;; (th/print-result! out) + (t/is (nil? error)) + (t/is (nil? result)))) + + (t/testing "query last result" + (let [{:keys [result error] :as out} (th/query! data)] + ;; (th/print-result! out) + (t/is (nil? error)) + (t/is (= 4 (count result))) + (t/is (= "foobar" (:data result))) + (t/is (= {:baz 1} (:props result))) + (t/is (= 2 (:revn result))))) + + (t/testing "gc task" + ;; make the file eligible for GC waiting 300ms (configured + ;; timeout for testing) + (th/sleep 300) + + ;; run the task again + (let [task (:app.tasks.file-gc/handler th/*system*) + res (task {})] + (t/is (= 1 (:processed res)))) + + ;; Then query the specific revn + (let [{:keys [result error] :as out} (th/query! (assoc data :revn 1))] + (t/is (= :not-found (th/ex-type error))) + (t/is (= :file-thumbnail-not-found (th/ex-code error))))) + )) + + diff --git a/backend/test/app/test_helpers.clj b/backend/test/app/test_helpers.clj index e626f4dd1..5699424a6 100644 --- a/backend/test/app/test_helpers.clj +++ b/backend/test/app/test_helpers.clj @@ -73,7 +73,7 @@ :app.worker/cron :app.worker/worker) (d/deep-merge - {:app.tasks.file-media-gc/handler {:max-age (dt/duration 300)}})) + {:app.tasks.file-gc/handler {:max-age (dt/duration 300)}})) _ (ig/load-namespaces config) system (-> (ig/prep config) (ig/init))] @@ -285,7 +285,8 @@ (let [data (ex-data error)] (cond (= :spec-validation (:code data)) - (expound/printer (:data data)) + (println + (us/pretty-explain data)) (= :service-error (:type data)) (print-error! (.getCause ^Throwable error)) @@ -302,7 +303,7 @@ (println "====> END ERROR")) (do (println "====> START RESPONSE") - (prn result) + (fipp.edn/pprint result) (println "====> END RESPONSE")))) (defn exception? @@ -374,3 +375,15 @@ (.readLine cnsl) nil)) +(defn db-exec! + [sql] + (db/exec! *pool* sql)) + +(defn db-insert! + [& params] + (apply db/insert! *pool* params)) + +(defn db-query + [& params] + (apply db/query *pool* params)) + diff --git a/common/src/app/common/data.cljc b/common/src/app/common/data.cljc index eb74bd3a4..77f39b694 100644 --- a/common/src/app/common/data.cljc +++ b/common/src/app/common/data.cljc @@ -128,9 +128,10 @@ (defn index-by "Return a indexed map of the collection keyed by the result of executing the getter over each element of the collection." - [getter coll] - (persistent! - (reduce #(assoc! %1 (getter %2) %2) (transient {}) coll))) + ([kf coll] (index-by kf identity coll)) + ([kf vf coll] + (persistent! + (reduce #(assoc! %1 (kf %2) (vf %2)) (transient {}) coll)))) (defn index-of-pred [coll pred] @@ -597,19 +598,10 @@ (defn group-by - ([kf coll] (group-by kf identity coll)) - ([kf vf coll] - (let [conj (fnil conj [])] - (reduce (fn [result item] - (update result (kf item) conj (vf item))) - {} - coll)))) - -(defn group-by' - "A variant of group-by that uses a set for collecting results." - ([kf coll] (group-by kf identity coll)) - ([kf vf coll] - (let [conj (fnil conj #{})] + ([kf coll] (group-by kf identity [] coll)) + ([kf vf coll] (group-by kf vf [] coll)) + ([kf vf iv coll] + (let [conj (fnil conj iv)] (reduce (fn [result item] (update result (kf item) conj (vf item))) {} diff --git a/common/src/app/common/spec/shape.cljc b/common/src/app/common/spec/shape.cljc index 422c95f1e..e2732a0f4 100644 --- a/common/src/app/common/spec/shape.cljc +++ b/common/src/app/common/spec/shape.cljc @@ -53,6 +53,7 @@ (s/def ::hide-fill-on-export boolean?) +(s/def ::file-thumbnail boolean?) (s/def ::masked-group? boolean?) (s/def ::font-family string?) (s/def ::font-size ::us/safe-integer) @@ -301,7 +302,8 @@ (defmethod shape-spec :frame [_] (s/and ::shape-attrs - (s/keys :opt-un [::hide-fill-on-export]))) + (s/keys :opt-un [::file-thumbnail + ::hide-fill-on-export]))) (s/def ::shape (s/and (s/multi-spec shape-spec :type) diff --git a/frontend/resources/styles/main/partials/modal.scss b/frontend/resources/styles/main/partials/modal.scss index 242b5c846..94c5a1e88 100644 --- a/frontend/resources/styles/main/partials/modal.scss +++ b/frontend/resources/styles/main/partials/modal.scss @@ -1464,7 +1464,8 @@ } } - & .unchecked { + & .intermediate, + .unchecked { svg { background-color: $color-gray-10; } diff --git a/frontend/src/app/main/data/workspace.cljs b/frontend/src/app/main/data/workspace.cljs index 25de91e04..8d85661df 100644 --- a/frontend/src/app/main/data/workspace.cljs +++ b/frontend/src/app/main/data/workspace.cljs @@ -957,35 +957,23 @@ (let [selected (wsh/lookup-selected state)] (rx/of (dch/update-shapes selected #(update % :blocked not))))))) -(defn extract-file-thumbnails-from-page - [state selected page] - (let [extract-frames (fn [page-id] - (let [objects (wsh/lookup-page-objects state page-id)] - (cph/get-frames objects))) - page-id (key page) - frames-with-thumbnail (->> (extract-frames page-id) - (filter (comp true? :file-thumbnail)) - (map :id) - (remove #(some #{%} selected)) - (map #(into {} {:id % :page-id page-id})))] - (when frames-with-thumbnail frames-with-thumbnail))) - - (defn toggle-file-thumbnail-selected [] (ptk/reify ::toggle-file-thumbnail-selected ptk/WatchEvent (watch [_ state _] (let [selected (wsh/lookup-selected state) - pages (get-in state [:workspace-data - :pages-index]) - file-thumbnails (->> pages - (mapcat #(extract-file-thumbnails-from-page state selected %)))] + pages (-> state :workspace-data :pages-index vals) + extract (fn [{:keys [objects id] :as page}] + (->> (cph/get-frames objects) + (filter :file-thumbnail) + (map :id) + (remove selected) + (map (fn [frame-id] [id frame-id]))))] (rx/concat - (rx/from - (for [ft file-thumbnails] - (dch/update-shapes [(:id ft)] #(update % :file-thumbnail not) (:page-id ft) nil))) - (rx/of (dch/update-shapes selected #(update % :file-thumbnail not)))))))) + (rx/from (for [[page-id frame-id] (mapcat extract pages)] + (dch/update-shapes [frame-id] #(dissoc % :file-thumbnail) page-id nil))) + (rx/of (dch/update-shapes selected #(assoc % :file-thumbnail true)))))))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Navigation @@ -1043,8 +1031,8 @@ [items element] (let [items (or items #{})] (if (contains? items element) - (disj set element) - (conj set element)))) + (disj items element) + (conj items element)))) (defn toggle-selected-assets [asset type] diff --git a/frontend/src/app/main/data/workspace/colors.cljs b/frontend/src/app/main/data/workspace/colors.cljs index f677aee8f..97deb5a06 100644 --- a/frontend/src/app/main/data/workspace/colors.cljs +++ b/frontend/src/app/main/data/workspace/colors.cljs @@ -259,7 +259,11 @@ (assoc :stroke-color-gradient (:gradient attrs)) (contains? attrs :opacity) - (assoc :stroke-opacity (:opacity attrs))) + (assoc :stroke-opacity (:opacity attrs)) + + :always + (d/without-nils)) + attrs (merge attrs color-attrs)] (rx/of (dch/update-shapes ids (fn [shape] diff --git a/frontend/src/app/main/errors.cljs b/frontend/src/app/main/errors.cljs index 12daf2712..d94e206b7 100644 --- a/frontend/src/app/main/errors.cljs +++ b/frontend/src/app/main/errors.cljs @@ -13,6 +13,7 @@ [app.main.data.messages :as msg] [app.main.data.users :as du] [app.main.store :as st] + [app.util.globals :as glob] [app.util.i18n :refer [tr]] [app.util.router :as rt] [app.util.timers :as ts] @@ -51,18 +52,6 @@ (st/emit! (du/logout {:capture-redirect true})) (ts/schedule 500 (st/emitf (msg/warn msg))))) - -;; That are special case server-errors that should be treated -;; differently. -(derive :not-found ::exceptional-state) -(derive :bad-gateway ::exceptional-state) -(derive :service-unavailable ::exceptional-state) - -(defmethod ptk/handle-error ::exceptional-state - [error] - (ts/schedule - (st/emitf (rt/assign-exception error)))) - ;; Error that happens on an active business model validation does not ;; passes an validation (example: profile can't leave a team). From ;; the user perspective a error flash message should be visualized but @@ -133,9 +122,22 @@ (js/console.error (with-out-str (expound/printer error))) (js/console.groupEnd message))) +;; That are special case server-errors that should be treated +;; differently. + +(derive :not-found ::exceptional-state) +(derive :bad-gateway ::exceptional-state) +(derive :service-unavailable ::exceptional-state) + +(defmethod ptk/handle-error ::exceptional-state + [error] + (ts/schedule + (st/emitf (rt/assign-exception error)))) + ;; This happens when the backed server fails to process the ;; request. This can be caused by an internal assertion or any other ;; uncontrolled error. + (defmethod ptk/handle-error :server-error [{:keys [data hint] :as error}] (let [hint (or hint (:hint data) (:message data)) @@ -145,8 +147,8 @@ (ts/schedule #(st/emit! (msg/show {:content "Something wrong has happened (on backend)." - :type :error - :timeout 3000}))) + :type :error + :timeout 3000}))) (js/console.group msg) (js/console.info info) @@ -174,6 +176,6 @@ (.preventDefault ^js event) (some-> (unchecked-get event "error") (on-unhandled-error)))] - (.addEventListener js/window "error" on-error) + (.addEventListener glob/window "error" on-error) (fn [] - (.removeEventListener js/window "error" on-error)))) + (.removeEventListener glob/window "error" on-error)))) diff --git a/frontend/src/app/main/render.cljs b/frontend/src/app/main/render.cljs index 7970bb173..34eebbabc 100644 --- a/frontend/src/app/main/render.cljs +++ b/frontend/src/app/main/render.cljs @@ -214,31 +214,6 @@ [:& shape-wrapper {:shape item :key (:id item)}])))]]])) -(mf/defc file-thumbnail-svg - {::mf/wrap [mf/memo]} - [{:keys [data embed? include-metadata?] :as props - :or {embed? false include-metadata? false}}] - (let [data (assoc data :x 0 :y 0) - vbox (format-viewbox {:width (:width data 0) :height (:height data 0)}) - background-color (get-in data [:options :background] default-color)] - - [:& (mf/provider embed/context) {:value embed?} - [:& (mf/provider export/include-metadata-ctx) {:value include-metadata?} - [:svg {:view-box vbox - :version "1.1" - :xmlns "http://www.w3.org/2000/svg" - :xmlnsXlink "http://www.w3.org/1999/xlink" - :xmlns:penpot (when include-metadata? "https://penpot.app/xmlns") - :style {:width "100%" - :height "100%" - :background background-color}} - - (when include-metadata? - [:& export/export-page {:options (:options data)}]) - - [:> shape-container {:shape data} - [:& frame/frame-thumbnail {:shape data}]]]]])) - (mf/defc frame-svg {::mf/wrap [mf/memo]} [{:keys [objects frame zoom show-thumbnails?] :or {zoom 1} :as props}] diff --git a/frontend/src/app/main/ui.cljs b/frontend/src/app/main/ui.cljs index 11a76dbbf..151e14b21 100644 --- a/frontend/src/app/main/ui.cljs +++ b/frontend/src/app/main/ui.cljs @@ -110,19 +110,7 @@ :index index :share-id share-id}])) - :render-object - (do - (let [file-id (uuid (get-in route [:path-params :file-id])) - page-id (uuid (get-in route [:path-params :page-id])) - object-id (uuid (get-in route [:path-params :object-id])) - embed? (= (get-in route [:query-params :embed]) "true") - render-texts (get-in route [:query-params :render-texts])] - [:& render/render-object {:file-id file-id - :page-id page-id - :object-id object-id - :embed? embed? - :render-texts? (and (some? render-texts) (= render-texts "true"))}])) - + ;; TODO: maybe move to `app.render` entrypoint (handled by render.html) :render-sprite (do (let [file-id (uuid (get-in route [:path-params :file-id])) diff --git a/frontend/src/app/main/ui/dashboard/grid.cljs b/frontend/src/app/main/ui/dashboard/grid.cljs index 195fcee1d..33dd1159d 100644 --- a/frontend/src/app/main/ui/dashboard/grid.cljs +++ b/frontend/src/app/main/ui/dashboard/grid.cljs @@ -27,8 +27,6 @@ [app.util.timers :as ts] [app.util.webapi :as wapi] [beicon.core :as rx] - [cuerdas.core :as str] - [promesa.core :as p] [rumext.alpha :as mf])) (log/set-level! :warn) @@ -38,57 +36,15 @@ (def ^:const CACHE-NAME "penpot") (def ^:const CACHE-URL "https://penpot.app/cache/") - (defn use-thumbnail-cache "Creates some hooks to handle the files thumbnails cache" [file] - - (let [cache-url (str CACHE-URL (:id file) "/" (:revn file) ".svg") - get-thumbnail - (mf/use-callback - (mf/deps cache-url) - (fn [] - (p/let [response (.match js/caches cache-url)] - (when (some? response) - (p/let [blob (.blob response) - svg-content (.text blob) - headers (.-headers response) - fonts-header (or (.get headers "X-PENPOT-FONTS") "") - fonts (into #{} - (remove #(= "" %)) - (str/split fonts-header ","))] - {:svg svg-content - :fonts fonts}))))) - - cache-thumbnail - (mf/use-callback - (mf/deps cache-url) - (fn [{:keys [svg fonts]}] - (p/let [cache (.open js/caches CACHE-NAME) - blob (js/Blob. #js [svg] #js {:type "image/svg"}) - fonts (str/join "," fonts) - headers (js/Headers. #js {"X-PENPOT-FONTS" fonts}) - response (js/Response. blob #js {:headers headers})] - (.put cache cache-url response))))] - - (mf/use-callback - (mf/deps (:id file) (:revn file)) - (fn [] - (->> (rx/from (get-thumbnail)) - (rx/merge-map - (fn [thumb-data] - (log/debug :msg "retrieve thumbnail" :file (:id file) :revn (:revn file) - :cache (if (some? thumb-data) :hit :miss)) - - (if (some? thumb-data) - (rx/of thumb-data) - (->> (wrk/ask! {:cmd :thumbnails/generate - :file-id (:id file)}) - (rx/tap cache-thumbnail))))) - - ;; If we have a problem we delegate to the thumbnail generation - (rx/catch #(wrk/ask! {:cmd :thumbnails/generate - :file-id (:id file)}))))))) + (mf/use-fn + (mf/deps (:id file) (:revn file)) + (fn [] + (wrk/ask! {:cmd :thumbnails/generate + :revn (:revn file) + :file-id (:id file)})))) (mf/defc grid-item-thumbnail {::mf/wrap [mf/memo]} @@ -100,10 +56,10 @@ (mf/deps file) (fn [] (->> (generate) - (rx/subs (fn [{:keys [svg fonts]}] + (rx/subs (fn [{:keys [data fonts] :as params}] (run! fonts/ensure-loaded! fonts) (when-let [node (mf/ref-val container)] - (dom/set-html! node svg))))))) + (dom/set-html! node data))))))) [:div.grid-item-th {:style {:background-color (get-in file [:data :options :background])} :ref container} diff --git a/frontend/src/app/main/ui/export.cljs b/frontend/src/app/main/ui/export.cljs index b96b8eab9..3b273c399 100644 --- a/frontend/src/app/main/ui/export.cljs +++ b/frontend/src/app/main/ui/export.cljs @@ -18,6 +18,7 @@ [app.main.ui.workspace.shapes :refer [shape-wrapper]] [app.util.dom :as dom] [app.util.i18n :as i18n :refer [tr c]] + [app.util.strings :as ust] [cuerdas.core :as str] [rumext.alpha :as mf])) @@ -76,7 +77,7 @@ (cond all-checked? [:span.checked i/checkbox-checked] all-unchecked? [:span.unchecked i/checkbox-unchecked] - :else [:span i/checkbox-intermediate])] + :else [:span.intermediate i/checkbox-intermediate])] [:div.field.title (tr "dashboard.export-multiple.selected" (c (count enabled-exports)) (c (count all-exports)))]] @@ -107,8 +108,8 @@ [:div.field.name (cond-> (:name shape) suffix (str suffix))] (when (:scale export) - [:div.field.scale (dm/str (* width (:scale export)) "x" - (* height (:scale export)) "px ")]) + [:div.field.scale (dm/str (ust/format-precision (* width (:scale export)) 2) "x" + (ust/format-precision (* height (:scale export)) 2) "px ")]) (when (:type export) [:div.field.extension (-> export :type d/name str/upper)])]))] @@ -175,6 +176,7 @@ progress (:progress state) exports (:exports state) total (count exports) + complete? (= progress total) circ (* 2 Math/PI 12) pct (- circ (* circ (/ progress total))) @@ -187,6 +189,7 @@ (not healthy?) clr/warning) title (cond error? (tr "workspace.options.exporting-object-error") + complete? (tr "workspace.options.exporting-complete") healthy? (tr "workspace.options.exporting-object") (not healthy?) (tr "workspace.options.exporting-object-slow")) diff --git a/frontend/src/app/main/worker.cljs b/frontend/src/app/main/worker.cljs index 8bd60284f..40e07456b 100644 --- a/frontend/src/app/main/worker.cljs +++ b/frontend/src/app/main/worker.cljs @@ -7,19 +7,16 @@ (ns app.main.worker (:require [app.config :as cfg] + [app.main.errors :as err] [app.util.worker :as uw])) -(defn on-error - [error] - (js/console.error "Error on worker" (pr-str error))) - (defonce instance (atom nil)) (defn init! [] (reset! instance - (uw/init cfg/worker-uri on-error))) + (uw/init cfg/worker-uri err/on-error))) (defn ask! [message] diff --git a/frontend/src/app/render.cljs b/frontend/src/app/render.cljs index d119796f8..b40802fb9 100644 --- a/frontend/src/app/render.cljs +++ b/frontend/src/app/render.cljs @@ -29,7 +29,6 @@ :version (:full @cf/version) :public-uri (str cf/public-uri)) - (defn- parse-params [loc] (let [href (unchecked-get loc "href")] diff --git a/frontend/src/app/util/globals.js b/frontend/src/app/util/globals.js index 502f70ddf..12841bec7 100644 --- a/frontend/src/app/util/globals.js +++ b/frontend/src/app/util/globals.js @@ -28,6 +28,10 @@ goog.scope(function() { addListener(...args) { }, removeListener(...args) { + }, + addEventListener(...args) { + }, + removeEventListener(...args) { } } } diff --git a/frontend/src/app/util/text_editor.cljs b/frontend/src/app/util/text_editor.cljs index 5cf866a9e..28da8d72c 100644 --- a/frontend/src/app/util/text_editor.cljs +++ b/frontend/src/app/util/text_editor.cljs @@ -15,7 +15,8 @@ (defn immutable-map->map [obj] - (into {} (map (fn [[k v]] [(keyword k) v])) (seq obj))) + (let [data (into {} (map (fn [[k v]] [(keyword k) v])) (seq obj))] + (assoc data :fills (js->clj (:fills data) :keywordize-keys true)))) ;; --- DRAFT-JS HELPERS diff --git a/frontend/src/app/worker.cljs b/frontend/src/app/worker.cljs index 52bb72451..a37bce958 100644 --- a/frontend/src/app/worker.cljs +++ b/frontend/src/app/worker.cljs @@ -52,10 +52,13 @@ (reply [result] (post {:payload result})) - (reply-error [err] - (.error js/console "error" (pr-str err)) - (post {:error {:data (ex-data err) - :message (ex-message err)}})) + (reply-error [cause] + (if (map? cause) + (post {:error cause}) + (post {:error {:type :unexpected + :code :unhandled-error-on-worker + :hint (ex-message cause) + :data (ex-data cause)}}))) (reply-completed ([] (reply-completed nil)) diff --git a/frontend/src/app/worker/thumbnails.cljs b/frontend/src/app/worker/thumbnails.cljs index f49e94b1e..ddd657643 100644 --- a/frontend/src/app/worker/thumbnails.cljs +++ b/frontend/src/app/worker/thumbnails.cljs @@ -16,6 +16,10 @@ [beicon.core :as rx] [rumext.alpha :as mf])) +(defn- not-found? + [{:keys [type]}] + (= :not-found type)) + (defn- handle-response [response] (cond @@ -29,38 +33,70 @@ (rx/throw {:type :unexpected :code (:error response)}))) -(defn- request-thumbnail - [file-id] - (let [uri (u/join (cfg/get-public-uri) "api/rpc/query/file-data-for-thumbnail") - params {:file-id file-id - :strip-frames-with-thumbnails true}] - (->> (http/send! - {:method :get - :uri uri - :credentials "include" - :query params}) +(defn- request-data-for-thumbnail + [file-id revn] + (let [path "api/rpc/query/file-data-for-thumbnail" + params {:file-id file-id + :revn revn + :strip-frames-with-thumbnails true} + request {:method :get + :uri (u/join (cfg/get-public-uri) path) + :credentials "include" + :query params}] + (->> (http/send! request) (rx/map http/conditional-decode-transit) (rx/mapcat handle-response)))) -(defonce cache (atom {})) +(defn- request-thumbnail + [file-id revn] + (let [path "api/rpc/query/file-thumbnail" + params {:file-id file-id + :revn revn} + request {:method :get + :uri (u/join (cfg/get-public-uri) path) + :credentials "include" + :query params}] + (->> (http/send! request) + (rx/map http/conditional-decode-transit) + (rx/mapcat handle-response)))) -(defn render-frame - [data ckey] - (let [prev (get @cache ckey)] - (if (= (:data prev) data) - (:result prev) - (let [file-thumbnail (:file-thumbnail data) - elem (if file-thumbnail - (mf/element render/file-thumbnail-svg #js {:data file-thumbnail :width "290" :height "150"}) - (mf/element render/page-svg #js {:data data :width "290" :height "150" :thumbnails? true})) - result (rds/renderToStaticMarkup elem)] - (swap! cache assoc ckey {:data data :result result}) - result)))) +(defn- render-thumbnail + [{:keys [data file-id revn] :as params}] + (let [elem (if-let [frame (:thumbnail-frame data)] + (mf/element render/frame-svg #js {:objects (:objects data) :frame frame}) + (mf/element render/page-svg #js {:data data :width "290" :height "150" :thumbnails? true}))] + {:data (rds/renderToStaticMarkup elem) + :fonts @fonts/loaded + :file-id file-id + :revn revn})) + +(defn- persist-thumbnail + [{:keys [file-id data revn fonts]}] + (let [path "api/rpc/mutation/upsert-file-thumbnail" + params {:file-id file-id + :revn revn + :props {:fonts fonts} + :data data} + request {:method :post + :uri (u/join (cfg/get-public-uri) path) + :credentials "include" + :body (http/transit-data params)}] + (->> (http/send! request) + (rx/map http/conditional-decode-transit) + (rx/mapcat handle-response) + (rx/map (constantly params))))) (defmethod impl/handler :thumbnails/generate - [{:keys [file-id] :as message}] - (->> (request-thumbnail file-id) - (rx/map - (fn [data] - {:svg (render-frame data #{file-id}) - :fonts @fonts/loaded})))) + [{:keys [file-id revn] :as message}] + (letfn [(on-result [{:keys [data props]}] + {:data data + :fonts (:fonts props)}) + + (on-cache-miss [_] + (->> (request-data-for-thumbnail file-id revn) + (rx/map render-thumbnail) + (rx/mapcat persist-thumbnail)))] + + (->> (request-thumbnail file-id revn) + (rx/catch not-found? on-cache-miss) + (rx/map on-result)))) diff --git a/frontend/src/debug.cljs b/frontend/src/debug.cljs index 21b6881a0..e7b48bbf0 100644 --- a/frontend/src/debug.cljs +++ b/frontend/src/debug.cljs @@ -10,8 +10,12 @@ [app.common.pages.helpers :as cph] [app.common.transit :as t] [app.common.uuid :as uuid] + [app.main.data.dashboard.shortcuts] + [app.main.data.viewer.shortcuts] [app.main.data.workspace :as dw] [app.main.data.workspace.changes :as dwc] + [app.main.data.workspace.path.shortcuts] + [app.main.data.workspace.shortcuts] [app.main.store :as st] [app.util.object :as obj] [app.util.timers :as timers] @@ -301,3 +305,31 @@ [] (st/emit! (dw/toggle-layout-flag :hide-ui))) + + +(defn ^:export shortcuts + [] + + (letfn [(print-shortcuts [shortcuts] + (.table js/console + (->> shortcuts + (map (fn [[key {:keys [command]}]] + [(d/name key) + (if (vector? command) + (str/join " | " command) + command)])) + (into {}) + (clj->js))))] + (let [style "font-weight: bold; font-size: 1.25rem;"] + (.log js/console "%c Dashboard" style) + (print-shortcuts app.main.data.dashboard.shortcuts/shortcuts) + + (.log js/console "%c Workspace" style) + (print-shortcuts app.main.data.workspace.shortcuts/shortcuts) + + (.log js/console "%c Path" style) + (print-shortcuts app.main.data.workspace.path.shortcuts/shortcuts) + + (.log js/console "%c Viewer" style) + (print-shortcuts app.main.data.viewer.shortcuts/shortcuts))) + nil) diff --git a/frontend/translations/en.po b/frontend/translations/en.po index 9ad21b2c0..ce9150436 100644 --- a/frontend/translations/en.po +++ b/frontend/translations/en.po @@ -2549,6 +2549,10 @@ msgstr "Suffix" msgid "workspace.options.exporting-object" msgstr "Exporting…" +#: src/app/main/ui/workspace/sidebar/options/menus/exports.cljs, src/app/main/ui/handoff/exports.cljs, src/app/main/ui/workspace/header.cljs +msgid "workspace.options.exporting-complete" +msgstr "Export complete" + #: src/app/main/ui/workspace/sidebar/options/menus/exports.cljs, src/app/main/ui/handoff/exports.cljs, src/app/main/ui/workspace/header.cljs msgid "workspace.options.exporting-object-error" msgstr "Export failed" diff --git a/frontend/translations/es.po b/frontend/translations/es.po index ccc402c43..c09e0e91b 100644 --- a/frontend/translations/es.po +++ b/frontend/translations/es.po @@ -2564,6 +2564,10 @@ msgstr "Sufijo" msgid "workspace.options.exporting-object" msgstr "Exportando..." +#: src/app/main/ui/workspace/sidebar/options/menus/exports.cljs, src/app/main/ui/handoff/exports.cljs, src/app/main/ui/workspace/header.cljs +msgid "workspace.options.exporting-complete" +msgstr "Exportación completa" + #: src/app/main/ui/workspace/sidebar/options/menus/exports.cljs, src/app/main/ui/handoff/exports.cljs, src/app/main/ui/workspace/header.cljs msgid "workspace.options.exporting-object-error" msgstr "Exportación fallida" @@ -3671,4 +3675,4 @@ msgid "workspace.updates.update" msgstr "Actualizar" msgid "workspace.viewport.click-to-close-path" -msgstr "Pulsar para cerrar la ruta" \ No newline at end of file +msgstr "Pulsar para cerrar la ruta"