0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-04-01 09:31:26 -05:00

Merge remote-tracking branch 'origin/staging' into develop

This commit is contained in:
Andrey Antukh 2022-03-28 20:27:19 +02:00
commit 4010fb7d1e
38 changed files with 758 additions and 544 deletions

View file

@ -43,6 +43,7 @@
### :bug: Bugs fixed
- Fix comments modal when changing pages [Taiga #2597](https://tree.taiga.io/project/penpot/issue/2508)
- Copy paste inside a text layer leaves pasted text transparent [Taiga #3096](https://tree.taiga.io/project/penpot/issue/3096)
- On dashboard enter on empty search refresh the page [Taiga #2597](https://tree.taiga.io/project/penpot/issue/2597)
- Pencil cursor changes when activated [Taiga #2276](https://tree.taiga.io/project/penpot/issue/2276)
- Fix icon placement in Mixed message [Taiga #3037](https://tree.taiga.io/project/penpot/issue/3037)

View file

@ -19,7 +19,7 @@
io.lettuce/lettuce-core {:mvn/version "6.1.6.RELEASE"}
java-http-clj/java-http-clj {:mvn/version "0.4.3"}
funcool/yetti {:git/tag "v8.0" :git/sha "ea7162d"
funcool/yetti {:git/tag "v9.0" :git/sha "e09e46c"
:git/url "https://github.com/funcool/yetti.git"
:exclusions [org.slf4j/slf4j-api]}

View file

@ -233,14 +233,14 @@
([ds table params opts]
(exec-one! ds
(sql/insert table params opts)
(assoc opts :return-keys true))))
(merge {:return-keys true} opts))))
(defn insert-multi!
([ds table cols rows] (insert-multi! ds table cols rows nil))
([ds table cols rows opts]
(exec! ds
(sql/insert-multi table cols rows opts)
(assoc opts :return-keys true))))
(merge {:return-keys true} opts))))
(defn update!
([ds table params where] (update! ds table params where nil))

View file

@ -8,6 +8,7 @@
(:require
[app.common.data :as d]
[app.common.logging :as l]
[app.common.transit :as t]
[app.http.doc :as doc]
[app.http.errors :as errors]
[app.http.middleware :as middleware]
@ -67,7 +68,7 @@
:xnio/dispatch (:executor cfg)
:ring/async true}
handler (if (some? router)
(wrap-router cfg router)
(wrap-router router)
handler)
server (yt/server handler (d/without-nils options))]
(assoc cfg :server (yt/start! server))))
@ -81,30 +82,32 @@
[_ respond _]
(respond (yrs/response 404)))
(defn- ring-handler
[router]
(fn [request respond raise]
(if-let [match (r/match-by-path router (yrq/path request))]
(let [params (:path-params match)
result (:result match)
handler (or (:handler result) not-found-handler)
request (-> request
(assoc :path-params params)
(update :params merge params))]
(handler request respond raise))
(not-found-handler request respond raise))))
(defn- wrap-router
[_ router]
(let [handler (ring-handler router)]
[router]
(letfn [(handler [request respond raise]
(if-let [match (r/match-by-path router (yrq/path request))]
(let [params (:path-params match)
result (:result match)
handler (or (:handler result) not-found-handler)
request (-> request
(assoc :path-params params)
(update :params merge params))]
(handler request respond raise))
(not-found-handler request respond raise)))
(on-error [cause request respond]
(let [{:keys [body] :as response} (errors/handle cause request)]
(respond
(cond-> response
(map? body)
(-> (update :headers assoc "content-type" "application/transit+json")
(assoc :body (t/encode-str body {:type :json-verbose})))))))]
(fn [request respond _]
(handler request respond
(fn [cause]
(l/error :hint "unexpected error processing request"
::l/context (errors/get-context request)
:query-string (yrq/query request)
:cause cause)
(respond (yrs/response 500 "internal server error")))))))
(try
(handler request respond #(on-error % request respond))
(catch Throwable cause
(on-error cause request respond))))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; HTTP ROUTER
@ -130,6 +133,8 @@
(rr/router
[["" {:middleware [[middleware/server-timing]
[middleware/format-response]
[middleware/params]
[middleware/parse-request]
[middleware/errors errors/handle]
[middleware/restrict-methods]]}
["/metrics" {:handler (:handler metrics)}]
@ -138,9 +143,7 @@
["/by-file-media-id/:id" {:handler (:file-objects-handler assets)}]
["/by-file-media-id/:id/thumbnail" {:handler (:file-thumbnails-handler assets)}]]
["/dbg" {:middleware [[middleware/params]
[middleware/parse-request]
(:middleware session)]}
["/dbg" {:middleware [(:middleware session)]}
["" {:handler (:index debug)}]
["/error-by-id/:id" {:handler (:retrieve-error debug)}]
["/error/:id" {:handler (:retrieve-error debug)}]
@ -152,15 +155,11 @@
["/sns" {:handler (:awsns-handler cfg)
:allowed-methods #{:post}}]]
["/ws/notifications" {:middleware [[middleware/params]
[middleware/parse-request]
(:middleware session)]
["/ws/notifications" {:middleware [(:middleware session)]
:handler ws
:allowed-methods #{:get}}]
["/api" {:middleware [[middleware/cors]
[middleware/params]
[middleware/parse-request]
(:middleware session)]}
["/health" {:handler (:health-check debug)}]
["/_doc" {:handler (doc/handler rpc)

View file

@ -24,8 +24,8 @@
(defn get-context
[request]
(merge
{:path (:uri request)
:method (:request-method request)
{:path (:path request)
:method (:method request)
:params (:params request)
:ip-addr (parse-client-ip request)
:profile-id (:profile-id request)}
@ -51,9 +51,10 @@
[err _]
(let [data (ex-data err)
explain (us/pretty-explain data)]
(yrs/response 400 (-> data
(dissoc ::s/problems ::s/value)
(cond-> explain (assoc :explain explain))))))
(yrs/response :status 400
:body (-> data
(dissoc ::s/problems ::s/value)
(cond-> explain (assoc :explain explain))))))
(defmethod handle-exception :assertion
[error request]
@ -73,26 +74,6 @@
[err _]
(yrs/response 404 (ex-data err)))
(defmethod handle-exception :default
[error request]
(let [edata (ex-data error)]
;; NOTE: this is a special case for the idle-in-transaction error;
;; when it happens, the connection is automatically closed and
;; next-jdbc combines the two errors in a single ex-info. We only
;; need the :handling error, because the :rollback error will be
;; always "connection closed".
(if (and (ex/exception? (:rollback edata))
(ex/exception? (:handling edata)))
(handle-exception (:handling edata) request)
(do
(l/error ::l/raw (ex-message error)
::l/context (get-context request)
:cause error)
(yrs/response 500 {:type :server-error
:code :unexpected
:hint (ex-message error)
:data edata})))))
(defmethod handle-exception org.postgresql.util.PSQLException
[error request]
(let [state (.getSQLState ^java.sql.SQLException error)]
@ -101,24 +82,56 @@
:cause error)
(cond
(= state "57014")
(yrs/response 504 {:type :server-timeout
(yrs/response 504 {:type :server-error
:code :statement-timeout
:hint (ex-message error)})
(= state "25P03")
(yrs/response 504 {:type :server-timeout
(yrs/response 504 {:type :server-error
:code :idle-in-transaction-timeout
:hint (ex-message error)})
:else
(yrs/response 500 {:type :server-error
:code :psql-exception
:code :unexpected
:hint (ex-message error)
:state state}))))
(defmethod handle-exception :default
[error request]
(let [edata (ex-data error)]
(cond
;; This means that exception is not a controlled exception.
(nil? edata)
(do
(l/error ::l/raw (ex-message error)
::l/context (get-context request)
:cause error)
(yrs/response 500 {:type :server-error
:code :unexpected
:hint (ex-message error)}))
;; This is a special case for the idle-in-transaction error;
;; when it happens, the connection is automatically closed and
;; next-jdbc combines the two errors in a single ex-info. We
;; only need the :handling error, because the :rollback error
;; will be always "connection closed".
(and (ex/exception? (:rollback edata))
(ex/exception? (:handling edata)))
(handle-exception (:handling edata) request)
:else
(do
(l/error ::l/raw (ex-message error)
::l/context (get-context request)
:cause error)
(yrs/response 500 {:type :server-error
:code :unhandled
:hint (ex-message error)
:data edata})))))
(defn handle
[error req]
[error request]
(if (or (instance? java.util.concurrent.CompletionException error)
(instance? java.util.concurrent.ExecutionException error))
(handle-exception (.getCause ^Throwable error) req)
(handle-exception error req)))
(handle-exception (.getCause ^Throwable error) request)
(handle-exception error request)))

View file

@ -6,6 +6,7 @@
(ns app.http.middleware
(:require
[app.common.exceptions :as ex]
[app.common.logging :as l]
[app.common.transit :as t]
[app.config :as cf]
@ -45,23 +46,17 @@
(update :params merge params))))
:else
request)))
(handle-exception [cause]
(let [data {:type :validation
:code :unable-to-parse-request-body
:hint "malformed params"}]
(l/error :hint (ex-message cause) :cause cause)
(yrs/response :status 400
:headers {"content-type" "application/transit+json"}
:body (t/encode-str data {:type :json-verbose}))))]
request)))]
(fn [request respond raise]
(try
(let [request (process-request request)]
(handler request respond raise))
(catch Exception cause
(respond (handle-exception cause)))))))
(when-let [request (try
(process-request request)
(catch Exception cause
(raise (ex/error :type :validation
:code :malformed-params
:hint (ex-message cause)
:cause cause))))]
(handler request respond raise)))))
(def parse-request
{:name ::parse-request

View file

@ -189,7 +189,7 @@
:pool (ig/ref :app.db/pool)
:entries
[{:cron #app/cron "0 0 0 * * ?" ;; daily
:task :file-media-gc}
:task :file-gc}
{:cron #app/cron "0 0 * * * ?" ;; hourly
:task :file-xlog-gc}
@ -231,7 +231,7 @@
:tasks
{:sendmail (ig/ref :app.emails/sendmail-handler)
:objects-gc (ig/ref :app.tasks.objects-gc/handler)
:file-media-gc (ig/ref :app.tasks.file-media-gc/handler)
:file-gc (ig/ref :app.tasks.file-gc/handler)
:file-xlog-gc (ig/ref :app.tasks.file-xlog-gc/handler)
:storage-deleted-gc (ig/ref :app.storage/gc-deleted-task)
:storage-touched-gc (ig/ref :app.storage/gc-touched-task)
@ -262,7 +262,7 @@
:storage (ig/ref :app.storage/storage)
:max-age cf/deletion-delay}
:app.tasks.file-media-gc/handler
:app.tasks.file-gc/handler
{:pool (ig/ref :app.db/pool)
:max-age cf/deletion-delay}

View file

@ -214,6 +214,9 @@
{:name "0068-mod-storage-object-table"
:fn (mg/resource "app/migrations/sql/0068-mod-storage-object-table.sql")}
{:name "0069-add-file-thumbnail-table"
:fn (mg/resource "app/migrations/sql/0069-add-file-thumbnail-table.sql")}
])

View file

@ -8,3 +8,6 @@ CREATE TABLE file_frame_thumbnail (
PRIMARY KEY(file_id, frame_id)
);
ALTER TABLE file_frame_thumbnail
ALTER COLUMN data SET STORAGE external;

View file

@ -0,0 +1,14 @@
CREATE TABLE file_thumbnail (
file_id uuid NOT NULL REFERENCES file(id) ON DELETE CASCADE,
revn bigint NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
deleted_at timestamptz NULL,
data text NULL,
props jsonb NULL,
PRIMARY KEY(file_id, revn)
);
ALTER TABLE file_thumbnail
ALTER COLUMN data SET STORAGE external,
ALTER COLUMN props SET STORAGE external;

View file

@ -0,0 +1,16 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) UXBOX Labs SL
(ns app.rpc.helpers
"General purpose RPC helpers."
(:require [app.common.data.macros :as dm]))
(defn http-cache
[{:keys [max-age]}]
(fn [_ response]
(let [exp (if (integer? max-age) max-age (inst-ms max-age))
val (dm/fmt "max-age=%" (int (/ exp 1000.0)))]
(update response :headers assoc "cache-control" val))))

View file

@ -58,8 +58,9 @@
(db/insert! conn :file-profile-rel))))
(defn create-file
[conn {:keys [id name project-id is-shared data deleted-at]
[conn {:keys [id name project-id is-shared data deleted-at revn]
:or {is-shared false
revn 0
deleted-at nil}
:as params}]
(let [id (or id (:id data) (uuid/next))
@ -68,6 +69,7 @@
{:id id
:project-id project-id
:name name
:revn revn
:is-shared is-shared
:data (blob/encode data)
:deleted-at deleted-at})]
@ -320,7 +322,7 @@
_ (mtx/run! metrics {:id :update-file-changes :inc (count changes)})
ts (dt/now)
file (-> (files/retrieve-data cfg file)
file (-> file
(update :revn inc)
(update :data (fn [data]
;; Trace the length of bytes of processed data
@ -487,14 +489,34 @@
update set data = ?;")
(s/def ::data ::us/string)
(s/def ::upsert-frame-thumbnail
(s/def ::upsert-file-frame-thumbnail
(s/keys :req-un [::profile-id ::file-id ::frame-id ::data]))
(sv/defmethod ::upsert-frame-thumbnail
(sv/defmethod ::upsert-file-frame-thumbnail
[{:keys [pool] :as cfg} {:keys [profile-id file-id frame-id data]}]
(db/with-atomic [conn pool]
(files/check-edition-permissions! conn profile-id file-id)
(db/exec-one! conn [sql:upsert-frame-thumbnail file-id frame-id data data])
nil))
;; --- Mutation: Upsert file thumbnail
(def sql:upsert-file-thumbnail
"insert into file_thumbnail (file_id, revn, data, props)
values (?, ?, ?, ?::jsonb)
on conflict(file_id, revn) do
update set data = ?, props=?, updated_at=now();")
(s/def ::revn ::us/integer)
(s/def ::props map?)
(s/def ::upsert-file-thumbnail
(s/keys :req-un [::profile-id ::file-id ::revn ::data ::props]))
(sv/defmethod ::upsert-file-thumbnail
[{:keys [pool] :as cfg} {:keys [profile-id file-id revn data props]}]
(db/with-atomic [conn pool]
(files/check-edition-permissions! conn profile-id file-id)
(let [props (db/tjson (or props {}))]
(db/exec-one! conn [sql:upsert-file-thumbnail
file-id revn data props data props])
nil)))

View file

@ -425,7 +425,6 @@
:email email
:hint "looks like the email you invite has been repeatedly reported as spam or permanent bounce"))
(db/exec-one! conn [sql:upsert-team-invitation
(:id team) (str/lower email) (name role) token-exp (name role) token-exp])
@ -438,7 +437,6 @@
:token itoken
:extra-data ptoken})))
;; --- Mutation: Create Team & Invite Members
(s/def ::emails ::us/set-of-emails)
@ -463,7 +461,9 @@
:role role)))
(with-meta team
{:before-complete
{::audit/props {:invitations (count emails)}
:before-complete
#(audit-fn :cmd :submit
:type "mutation"
:name "invite-team-member"

View file

@ -7,20 +7,21 @@
(ns app.rpc.queries.files
(:require
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.pages.helpers :as cph]
[app.common.pages.migrations :as pmg]
[app.common.spec :as us]
[app.common.uuid :as uuid]
[app.db :as db]
[app.db.sql :as sql]
[app.rpc.helpers :as rpch]
[app.rpc.permissions :as perms]
[app.rpc.queries.projects :as projects]
[app.rpc.queries.share-link :refer [retrieve-share-link]]
[app.rpc.queries.teams :as teams]
[app.storage.impl :as simpl]
[app.util.blob :as blob]
[app.util.services :as sv]
[clojure.spec.alpha :as s]
[promesa.core :as p]))
[clojure.spec.alpha :as s]))
(declare decode-row)
(declare decode-row-xf)
@ -186,25 +187,12 @@
;; --- Query: File (By ID)
(defn- retrieve-data*
[{:keys [storage] :as cfg} file]
(p/do
(when-let [backend (simpl/resolve-backend storage (:data-backend file))]
(simpl/get-object-bytes backend file))))
(defn retrieve-data
[cfg file]
(if (bytes? (:data file))
file
(p/->> (retrieve-data* cfg file)
(assoc file :data))))
(defn retrieve-file
[{:keys [pool] :as cfg} id]
(p/->> (db/get-by-id pool :file id)
(retrieve-data cfg)
(let [item (db/get-by-id pool :file id)]
(->> item
(decode-row)
(pmg/migrate-file)))
(pmg/migrate-file))))
(s/def ::file
(s/keys :req-un [::profile-id ::id]))
@ -214,8 +202,8 @@
[{:keys [pool] :as cfg} {:keys [profile-id id] :as params}]
(let [perms (get-permissions pool profile-id id)]
(check-read-permissions! perms)
(p/-> (retrieve-file cfg id)
(assoc :permissions perms))))
(-> (retrieve-file cfg id)
(assoc :permissions perms))))
(declare trim-file-data)
@ -233,9 +221,9 @@
[{:keys [pool] :as cfg} {:keys [profile-id id] :as params}]
(let [perms (get-permissions pool profile-id id)]
(check-read-permissions! perms)
(p/-> (retrieve-file cfg id)
(trim-file-data params)
(assoc :permissions perms))))
(-> (retrieve-file cfg id)
(trim-file-data params)
(assoc :permissions perms))))
(defn- trim-file-data
[file {:keys [page-id object-id]}]
@ -248,9 +236,12 @@
(update :data assoc :pages-index {page-id page})
(update :data assoc :pages [page-id]))))
;; --- FILE THUMBNAIL
(declare strip-frames-with-thumbnails)
(declare extract-file-thumbnail)
(declare get-first-page-data)
(declare get-thumbnail-data)
(s/def ::strip-frames-with-thumbnails ::us/boolean)
@ -258,6 +249,17 @@
(s/keys :req-un [::profile-id ::file-id]
:opt-un [::strip-frames-with-thumbnails]))
(sv/defmethod ::page
"Retrieves the first page of the file. Used mainly for render
thumbnails on dashboard.
DEPRECATED: still here for backward compatibility."
[{:keys [pool] :as cfg} {:keys [profile-id file-id] :as props}]
(check-read-permissions! pool profile-id file-id)
(let [file (retrieve-file cfg file-id)
data (get-first-page-data file props)]
data))
(s/def ::file-data-for-thumbnail
(s/keys :req-un [::profile-id ::file-id]
:opt-un [::strip-frames-with-thumbnails]))
@ -267,20 +269,31 @@
thumbnails on dashboard."
[{:keys [pool] :as cfg} {:keys [profile-id file-id] :as props}]
(check-read-permissions! pool profile-id file-id)
(p/let [file (retrieve-file cfg file-id)
data (get-first-page-data file props)
file-thumbnail (extract-file-thumbnail (get-in file [:data :pages-index]))]
(let [file (retrieve-file cfg file-id)]
{:data (get-thumbnail-data file props)
:file-id file-id
:revn (:revn file)}))
(assoc data :file-thumbnail file-thumbnail)))
(defn get-thumbnail-data
[{:keys [data] :as file} props]
(if-let [[page frame] (first
(for [page (-> data :pages-index vals)
frame (-> page :objects cph/get-frames)
:when (:file-thumbnail frame)]
[page frame]))]
(let [objects (->> (cph/get-children-with-self (:objects page) (:id frame))
(d/index-by :id))]
(cond-> (assoc page :objects objects)
(:strip-frames-with-thumbnails props)
(strip-frames-with-thumbnails)
(sv/defmethod ::page
"Retrieves the first page of the file. Used mainly for render
thumbnails on dashboard."
[{:keys [pool] :as cfg} {:keys [profile-id file-id] :as props}]
(check-read-permissions! pool profile-id file-id)
(p/let [file (retrieve-file cfg file-id)
data (get-first-page-data file props)]
data))
:always
(assoc :thumbnail-frame frame)))
(let [page-id (-> data :pages first)]
(cond-> (get-in data [:pages-index page-id])
(:strip-frames-with-thumbnails props)
(strip-frames-with-thumbnails)))))
(defn get-first-page-data
[file props]
@ -317,17 +330,6 @@
(update data :objects update-objects)))
(defn extract-file-thumbnail
"Extract the frame marked as file-thumbnail"
[pages]
(->> pages
vals
(mapcat :objects)
vals
(filter :file-thumbnail)
first))
;; --- Query: Shared Library Files
(def ^:private sql:team-shared-files
@ -384,7 +386,6 @@
[{:keys [pool] :as cfg} is-indirect file-id]
(let [xform (comp
(map #(assoc % :is-indirect is-indirect))
(map #(retrieve-data cfg %))
(map decode-row))]
(into #{} xform (db/exec! pool [sql:file-libraries file-id]))))
@ -427,22 +428,49 @@
(teams/check-read-permissions! pool profile-id team-id)
(db/exec! pool [sql:team-recent-files team-id]))
;; --- QUERY: get all file frame thumbnails
;; --- QUERY: get the thumbnail for an frame
(s/def ::file-frame-thumbnails
(s/keys :req-un [::profile-id ::file-id]
:opt-un [::frame-id]))
(def ^:private sql:file-frame-thumbnail
"select data
from file_frame_thumbnail
where file_id = ?
and frame_id = ?")
(s/def ::file-frame-thumbnail
(s/keys :req-un [::profile-id ::file-id ::frame-id]))
(sv/defmethod ::file-frame-thumbnail
(sv/defmethod ::file-frame-thumbnails
[{:keys [pool]} {:keys [profile-id file-id frame-id]}]
(check-read-permissions! pool profile-id file-id)
(db/exec-one! pool [sql:file-frame-thumbnail file-id frame-id]))
(let [params (cond-> {:file-id file-id}
frame-id (assoc :frame-id frame-id))
rows (db/query pool :file-frame-thumbnail params)]
(d/index-by :frame-id :data rows)))
;; --- QUERY: get file thumbnail
(s/def ::revn ::us/integer)
(s/def ::file-thumbnail
(s/keys :req-un [::profile-id ::file-id]
:opt-un [::revn]))
(sv/defmethod ::file-thumbnail
[{:keys [pool]} {:keys [profile-id file-id revn]}]
(check-read-permissions! pool profile-id file-id)
(let [sql (sql/select :file-thumbnail
(cond-> {:file-id file-id}
revn (assoc :revn revn))
{:limit 1
:order-by [[:revn :desc]]})
row (db/exec-one! pool sql)]
(when-not row
(ex/raise :type :not-found
:code :file-thumbnail-not-found))
(with-meta
{:data (:data row)
:props (some-> (:props row) db/decode-transit-pgobject)
:revn (:revn row)
:file-id (:file-id row)}
{:transform-response (rpch/http-cache {:max-age (* 1000 60 60)})})))
;; --- Helpers

View file

@ -274,7 +274,7 @@
(let [min-age (db/interval min-age)
rows (db/exec! conn [sql:retrieve-deleted-objects-chunk min-age cursor])]
[(some-> rows peek :created-at)
(some->> (seq rows) (d/group-by' #(-> % :backend keyword) :id) seq)]))
(some->> (seq rows) (d/group-by #(-> % :backend keyword) :id #{}) seq)]))
(retrieve-deleted-objects [conn]
(->> (d/iteration (fn [cursor]
@ -383,7 +383,7 @@
(mapv #(d/update-when % :metadata db/decode-transit-pgobject)))]
(when (seq rows)
[(-> rows peek :created-at)
(d/group-by' get-bucket :id rows)])))
(d/group-by get-bucket :id #{} rows)])))
(retrieve-touched [conn]
(->> (d/iteration (fn [cursor]

View file

@ -0,0 +1,164 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) UXBOX Labs SL
(ns app.tasks.file-gc
"A maintenance task that is responsible of: purge unused file media,
clean unused frame thumbnails and remove old file thumbnails. The
file is eligible to be garbage collected after some period of
inactivity (the default threshold is 72h)."
(:require
[app.common.data :as d]
[app.common.logging :as l]
[app.common.pages.helpers :as cph]
[app.common.pages.migrations :as pmg]
[app.db :as db]
[app.util.blob :as blob]
[app.util.time :as dt]
[clojure.spec.alpha :as s]
[integrant.core :as ig]))
(declare ^:private retrieve-candidates)
(declare ^:private process-file)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; HANDLER
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(s/def ::max-age ::dt/duration)
(defmethod ig/pre-init-spec ::handler [_]
(s/keys :req-un [::db/pool ::max-age]))
(defmethod ig/init-key ::handler
[_ {:keys [pool] :as cfg}]
(fn [_]
(db/with-atomic [conn pool]
(let [cfg (assoc cfg :conn conn)]
(loop [total 0
files (retrieve-candidates cfg)]
(if-let [file (first files)]
(do
(process-file cfg file)
(recur (inc total)
(rest files)))
(do
(l/debug :msg "finished processing files" :processed total)
{:processed total})))))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; IMPL
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(def ^:private
sql:retrieve-candidates-chunk
"select f.id,
f.data,
f.revn,
f.modified_at
from file as f
where f.has_media_trimmed is false
and f.modified_at < now() - ?::interval
and f.modified_at < ?
order by f.modified_at desc
limit 1
for update skip locked")
(defn- retrieve-candidates
[{:keys [conn max-age] :as cfg}]
(let [interval (db/interval max-age)
get-chunk
(fn [cursor]
(let [rows (db/exec! conn [sql:retrieve-candidates-chunk interval cursor])]
[(some->> rows peek :modified-at) (seq rows)]))]
(sequence cat (d/iteration get-chunk
:vf second
:kf first
:initk (dt/now)))))
(defn- collect-used-media
[data]
(let [xform (comp
(map :objects)
(mapcat vals)
(keep (fn [{:keys [type] :as obj}]
(case type
:path (get-in obj [:fill-image :id])
:image (get-in obj [:metadata :id])
nil))))
pages (concat
(vals (:pages-index data))
(vals (:components data)))]
(-> #{}
(into xform pages)
(into (keys (:media data))))))
(defn- clean-file-media!
"Performs the garbage collection of file media objects."
[conn file-id data]
(let [used (collect-used-media data)
unused (->> (db/query conn :file-media-object {:file-id file-id})
(remove #(contains? used (:id %))))]
(doseq [mobj unused]
(l/debug :hint "delete file media object"
:id (:id mobj)
:media-id (:media-id mobj)
:thumbnail-id (:thumbnail-id mobj))
;; NOTE: deleting the file-media-object in the database
;; automatically marks as touched the referenced storage
;; objects. The touch mechanism is needed because many files can
;; point to the same storage objects and we can't just delete
;; them.
(db/delete! conn :file-media-object {:id (:id mobj)}))))
(defn- collect-frames
[data]
(let [xform (comp
(map :objects)
(mapcat vals)
(filter cph/frame-shape?)
(keep :id))
pages (concat
(vals (:pages-index data))
(vals (:components data)))]
(into #{} xform pages)))
(defn- clean-file-frame-thumbnails!
[conn file-id data]
(let [sql (str "delete from file_frame_thumbnail "
" where file_id=? and not (frame_id=ANY(?))")
ids (->> (collect-frames data)
(db/create-array conn "uuid"))
res (db/exec-one! conn [sql file-id ids])]
(l/debug :hint "delete frame thumbnails" :total (:next.jdbc/update-count res))))
(defn- clean-file-thumbnails!
[conn file-id revn]
(let [sql (str "delete from file_thumbnail "
" where file_id=? and revn < ?")
res (db/exec-one! conn [sql file-id revn])]
(l/debug :hint "delete file thumbnails" :total (:next.jdbc/update-count res))))
(defn- process-file
[{:keys [conn] :as cfg} {:keys [id data revn modified-at] :as file}]
(l/debug :hint "processing file" :id id :modified-at modified-at)
(let [data (-> (blob/decode data)
(assoc :id id)
(pmg/migrate-data))]
(clean-file-media! conn id data)
(clean-file-frame-thumbnails! conn id data)
(clean-file-thumbnails! conn id revn)
;; Mark file as trimmed
(db/update! conn :file
{:has-media-trimmed true}
{:id id})
nil))

View file

@ -1,139 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) UXBOX Labs SL
(ns app.tasks.file-media-gc
"A maintenance task that is responsible to purge the unused media
objects from files. A file is eligible to be garbage collected
after some period of inactivity (the default threshold is 72h)."
(:require
[app.common.logging :as l]
[app.common.pages.helpers :as cph]
[app.common.pages.migrations :as pmg]
[app.db :as db]
[app.util.blob :as blob]
[app.util.time :as dt]
[clojure.spec.alpha :as s]
[integrant.core :as ig]))
(declare process-file)
(declare retrieve-candidates)
(s/def ::max-age ::dt/duration)
(defmethod ig/pre-init-spec ::handler [_]
(s/keys :req-un [::db/pool ::max-age]))
(defmethod ig/init-key ::handler
[_ {:keys [pool] :as cfg}]
(fn [_]
(db/with-atomic [conn pool]
(let [cfg (assoc cfg :conn conn)]
(loop [n 0]
(let [files (retrieve-candidates cfg)]
(if (seq files)
(do
(run! (partial process-file cfg) files)
(recur (+ n (count files))))
(do
(l/debug :msg "finished processing files" :processed n)
{:processed n}))))))))
(def ^:private
sql:retrieve-candidates-chunk
"select f.id,
f.data,
extract(epoch from (now() - f.modified_at))::bigint as age
from file as f
where f.has_media_trimmed is false
and f.modified_at < now() - ?::interval
order by f.modified_at asc
limit 10
for update skip locked")
(defn- retrieve-candidates
[{:keys [conn max-age] :as cfg}]
(let [interval (db/interval max-age)]
(->> (db/exec! conn [sql:retrieve-candidates-chunk interval])
(mapv (fn [{:keys [age] :as row}]
(assoc row :age (dt/duration {:seconds age})))))))
(def ^:private
collect-media-xf
(comp
(map :objects)
(mapcat vals)
(keep (fn [{:keys [type] :as obj}]
(case type
:path (get-in obj [:fill-image :id])
:image (get-in obj [:metadata :id])
nil)))))
(defn- collect-used-media
[data]
(let [pages (concat
(vals (:pages-index data))
(vals (:components data)))]
(-> #{}
(into collect-media-xf pages)
(into (keys (:media data))))))
(def ^:private
collect-frames-xf
(comp
(map :objects)
(mapcat vals)
(filter cph/frame-shape?)
(keep :id)))
(defn- collect-frames
[data]
(let [pages (concat
(vals (:pages-index data))
(vals (:components data)))]
(into #{} collect-frames-xf pages)))
(defn- process-file
[{:keys [conn] :as cfg} {:keys [id data age] :as file}]
(let [data (-> (blob/decode data)
(assoc :id id)
(pmg/migrate-data))]
(let [used (collect-used-media data)
unused (->> (db/query conn :file-media-object {:file-id id})
(remove #(contains? used (:id %))))]
(l/debug :hint "processing file"
:id id
:age age
:to-delete (count unused))
;; Mark file as trimmed
(db/update! conn :file
{:has-media-trimmed true}
{:id id})
(doseq [mobj unused]
(l/debug :hint "deleting media object"
:id (:id mobj)
:media-id (:media-id mobj)
:thumbnail-id (:thumbnail-id mobj))
;; NOTE: deleting the file-media-object in the database
;; automatically marks as touched the referenced storage
;; objects. The touch mechanism is needed because many files can
;; point to the same storage objects and we can't just delete
;; them.
(db/delete! conn :file-media-object {:id (:id mobj)})))
(let [sql (str "delete from file_frame_thumbnail "
" where file_id = ? and not (frame_id = ANY(?))")
ids (->> (collect-frames data)
(db/create-array conn "uuid"))]
;; delete the unused frame thumbnails
(db/exec! conn [sql (:id file) ids]))
nil))

View file

@ -8,6 +8,7 @@
(:require
[app.common.uuid :as uuid]
[app.db :as db]
[app.db.sql :as sql]
[app.http :as http]
[app.storage :as sto]
[app.test-helpers :as th]
@ -117,7 +118,7 @@
(t/is (= 0 (count result))))))
))
(t/deftest file-media-gc-task
(t/deftest file-gc-task
(letfn [(create-file-media-object [{:keys [profile-id file-id]}]
(let [mfile {:filename "sample.jpg"
:path (th/tempfile "app/test_files/sample.jpg")
@ -130,6 +131,9 @@
:name "testfile"
:content mfile}
out (th/mutation! params)]
;; (th/print-result! out)
(t/is (nil? (:error out)))
(:result out)))
@ -189,7 +193,7 @@
(t/is (= 0 (:delete res))))
;; run the task immediately
(let [task (:app.tasks.file-media-gc/handler th/*system*)
(let [task (:app.tasks.file-gc/handler th/*system*)
res (task {})]
(t/is (= 0 (:processed res))))
@ -198,7 +202,7 @@
(th/sleep 300)
;; run the task again
(let [task (:app.tasks.file-media-gc/handler th/*system*)
(let [task (:app.tasks.file-gc/handler th/*system*)
res (task {})]
(t/is (= 1 (:processed res))))
@ -342,7 +346,7 @@
(t/is (th/ex-info? error))
(t/is (th/ex-of-type? error :not-found))))
(t/deftest deletion-test
(t/deftest deletion
(let [task (:app.tasks.objects-gc/handler th/*system*)
profile1 (th/create-profile* 1)
file (th/create-file* 1 {:project-id (:default-project-id profile1)
@ -410,71 +414,158 @@
))
(t/deftest query-frame-thumbnails
(let [prof (th/create-profile* 1 {:is-active true})
file (th/create-file* 1 {:profile-id (:id prof)
:project-id (:default-project-id prof)
:is-shared false})
data {::th/type :file-frame-thumbnail
:profile-id (:id prof)
:file-id (:id file)
:frame-id (uuid/next)}]
;;insert an entry on the database with a test value for the thumbnail of this frame
(db/exec-one! th/*pool*
["insert into file_frame_thumbnail(file_id, frame_id, data) values (?, ?, ?)"
(:file-id data) (:frame-id data) "testvalue"])
(let [out (th/query! data)]
(t/is (nil? (:error out)))
(let [result (:result out)]
(t/is (= 1 (count result)))
(t/is (= "testvalue" (:data result)))))))
(t/deftest insert-frame-thumbnails
(let [prof (th/create-profile* 1 {:is-active true})
file (th/create-file* 1 {:profile-id (:id prof)
:project-id (:default-project-id prof)
:is-shared false})
data {::th/type :upsert-frame-thumbnail
:profile-id (:id prof)
:file-id (:id file)
:frame-id (uuid/next)
:data "test insert new value"}
out (th/mutation! data)]
(t/is (nil? (:error out)))
(t/is (nil? (:result out)))
;;retrieve the value from the database and check its content
(let [result (db/exec-one!
th/*pool*
["select data from file_frame_thumbnail where file_id = ? and frame_id = ?"
(:file-id data) (:frame-id data)])]
(t/is (= "test insert new value" (:data result))))))
(t/deftest frame-thumbnails
(let [prof (th/create-profile* 1 {:is-active true})
file (th/create-file* 1 {:profile-id (:id prof)
:project-id (:default-project-id prof)
:is-shared false})
data {::th/type :upsert-frame-thumbnail
data {::th/type :file-frame-thumbnails
:profile-id (:id prof)
:file-id (:id file)
:frame-id (uuid/next)}]
;; insert an entry on the database with a test value for the thumbnail of this frame
(th/db-insert! :file-frame-thumbnail
{:file-id (:file-id data)
:frame-id (:frame-id data)
:data "testvalue"})
(let [{:keys [result error] :as out} (th/query! data)]
;; (th/print-result! out)
(t/is (nil? error))
(t/is (= 1 (count result)))
(t/is (= "testvalue" (get result (:frame-id data)))))))
(t/deftest insert-frame-thumbnails
(let [prof (th/create-profile* 1 {:is-active true})
file (th/create-file* 1 {:profile-id (:id prof)
:project-id (:default-project-id prof)
:is-shared false})
data {::th/type :upsert-file-frame-thumbnail
:profile-id (:id prof)
:file-id (:id file)
:frame-id (uuid/next)
:data "test insert new value"}]
(let [out (th/mutation! data)]
(t/is (nil? (:error out)))
(t/is (nil? (:result out)))
(let [[result] (th/db-query :file-frame-thumbnail
{:file-id (:file-id data)
:frame-id (:frame-id data)})]
(t/is (= "test insert new value" (:data result)))))))
(t/deftest upsert-frame-thumbnails
(let [prof (th/create-profile* 1 {:is-active true})
file (th/create-file* 1 {:profile-id (:id prof)
:project-id (:default-project-id prof)
:is-shared false})
data {::th/type :upsert-file-frame-thumbnail
:profile-id (:id prof)
:file-id (:id file)
:frame-id (uuid/next)
:data "updated value"}]
;;insert an entry on the database with and old value for the thumbnail of this frame
(db/exec-one! th/*pool*
["insert into file_frame_thumbnail(file_id, frame_id, data) values (?, ?, ?)"
(:file-id data) (:frame-id data) "old value"])
;; insert an entry on the database with and old value for the thumbnail of this frame
(th/db-insert! :file-frame-thumbnail
{:file-id (:file-id data)
:frame-id (:frame-id data)
:data "old value"})
(let [out (th/mutation! data)]
;; (th/print-result! out)
(t/is (nil? (:error out)))
(t/is (nil? (:result out)))
;;retrieve the value from the database and check its content
(let [result (db/exec-one!
th/*pool*
["select data from file_frame_thumbnail where file_id = ? and frame_id = ?"
(:file-id data) (:frame-id data)])]
;; retrieve the value from the database and check its content
(let [[result] (th/db-query :file-frame-thumbnail
{:file-id (:file-id data)
:frame-id (:frame-id data)})]
(t/is (= "updated value" (:data result)))))))
(t/deftest file-thumbnail-ops
(let [prof (th/create-profile* 1 {:is-active true})
file (th/create-file* 1 {:profile-id (:id prof)
:project-id (:default-project-id prof)
:revn 2
:is-shared false})
data {::th/type :file-thumbnail
:profile-id (:id prof)
:file-id (:id file)}]
(t/testing "query a thumbnail with single revn"
;; insert an entry on the database with a test value for the thumbnail of this frame
(th/db-insert! :file-thumbnail
{:file-id (:file-id data)
:revn 1
:data "testvalue1"})
(let [{:keys [result error] :as out} (th/query! data)]
;; (th/print-result! out)
(t/is (nil? error))
(t/is (= 4 (count result)))
(t/is (= "testvalue1" (:data result)))
(t/is (= 1 (:revn result)))))
(t/testing "query thumbnail with two revisions"
;; insert an entry on the database with a test value for the thumbnail of this frame
(th/db-insert! :file-thumbnail
{:file-id (:file-id data)
:revn 2
:data "testvalue2"})
(let [{:keys [result error] :as out} (th/query! data)]
;; (th/print-result! out)
(t/is (nil? error))
(t/is (= 4 (count result)))
(t/is (= "testvalue2" (:data result)))
(t/is (= 2 (:revn result))))
;; Then query the specific revn
(let [{:keys [result error] :as out} (th/query! (assoc data :revn 1))]
;; (th/print-result! out)
(t/is (nil? error))
(t/is (= 4 (count result)))
(t/is (= "testvalue1" (:data result)))
(t/is (= 1 (:revn result)))))
(t/testing "upsert file-thumbnail"
(let [data {::th/type :upsert-file-thumbnail
:profile-id (:id prof)
:file-id (:id file)
:data "foobar"
:props {:baz 1}
:revn 2}
{:keys [result error] :as out} (th/mutation! data)]
;; (th/print-result! out)
(t/is (nil? error))
(t/is (nil? result))))
(t/testing "query last result"
(let [{:keys [result error] :as out} (th/query! data)]
;; (th/print-result! out)
(t/is (nil? error))
(t/is (= 4 (count result)))
(t/is (= "foobar" (:data result)))
(t/is (= {:baz 1} (:props result)))
(t/is (= 2 (:revn result)))))
(t/testing "gc task"
;; make the file eligible for GC waiting 300ms (configured
;; timeout for testing)
(th/sleep 300)
;; run the task again
(let [task (:app.tasks.file-gc/handler th/*system*)
res (task {})]
(t/is (= 1 (:processed res))))
;; Then query the specific revn
(let [{:keys [result error] :as out} (th/query! (assoc data :revn 1))]
(t/is (= :not-found (th/ex-type error)))
(t/is (= :file-thumbnail-not-found (th/ex-code error)))))
))

View file

@ -73,7 +73,7 @@
:app.worker/cron
:app.worker/worker)
(d/deep-merge
{:app.tasks.file-media-gc/handler {:max-age (dt/duration 300)}}))
{:app.tasks.file-gc/handler {:max-age (dt/duration 300)}}))
_ (ig/load-namespaces config)
system (-> (ig/prep config)
(ig/init))]
@ -285,7 +285,8 @@
(let [data (ex-data error)]
(cond
(= :spec-validation (:code data))
(expound/printer (:data data))
(println
(us/pretty-explain data))
(= :service-error (:type data))
(print-error! (.getCause ^Throwable error))
@ -302,7 +303,7 @@
(println "====> END ERROR"))
(do
(println "====> START RESPONSE")
(prn result)
(fipp.edn/pprint result)
(println "====> END RESPONSE"))))
(defn exception?
@ -374,3 +375,15 @@
(.readLine cnsl)
nil))
(defn db-exec!
[sql]
(db/exec! *pool* sql))
(defn db-insert!
[& params]
(apply db/insert! *pool* params))
(defn db-query
[& params]
(apply db/query *pool* params))

View file

@ -128,9 +128,10 @@
(defn index-by
"Return a indexed map of the collection keyed by the result of
executing the getter over each element of the collection."
[getter coll]
(persistent!
(reduce #(assoc! %1 (getter %2) %2) (transient {}) coll)))
([kf coll] (index-by kf identity coll))
([kf vf coll]
(persistent!
(reduce #(assoc! %1 (kf %2) (vf %2)) (transient {}) coll))))
(defn index-of-pred
[coll pred]
@ -597,19 +598,10 @@
(defn group-by
([kf coll] (group-by kf identity coll))
([kf vf coll]
(let [conj (fnil conj [])]
(reduce (fn [result item]
(update result (kf item) conj (vf item)))
{}
coll))))
(defn group-by'
"A variant of group-by that uses a set for collecting results."
([kf coll] (group-by kf identity coll))
([kf vf coll]
(let [conj (fnil conj #{})]
([kf coll] (group-by kf identity [] coll))
([kf vf coll] (group-by kf vf [] coll))
([kf vf iv coll]
(let [conj (fnil conj iv)]
(reduce (fn [result item]
(update result (kf item) conj (vf item)))
{}

View file

@ -53,6 +53,7 @@
(s/def ::hide-fill-on-export boolean?)
(s/def ::file-thumbnail boolean?)
(s/def ::masked-group? boolean?)
(s/def ::font-family string?)
(s/def ::font-size ::us/safe-integer)
@ -301,7 +302,8 @@
(defmethod shape-spec :frame [_]
(s/and ::shape-attrs
(s/keys :opt-un [::hide-fill-on-export])))
(s/keys :opt-un [::file-thumbnail
::hide-fill-on-export])))
(s/def ::shape
(s/and (s/multi-spec shape-spec :type)

View file

@ -1464,7 +1464,8 @@
}
}
& .unchecked {
& .intermediate,
.unchecked {
svg {
background-color: $color-gray-10;
}

View file

@ -957,35 +957,23 @@
(let [selected (wsh/lookup-selected state)]
(rx/of (dch/update-shapes selected #(update % :blocked not)))))))
(defn extract-file-thumbnails-from-page
[state selected page]
(let [extract-frames (fn [page-id]
(let [objects (wsh/lookup-page-objects state page-id)]
(cph/get-frames objects)))
page-id (key page)
frames-with-thumbnail (->> (extract-frames page-id)
(filter (comp true? :file-thumbnail))
(map :id)
(remove #(some #{%} selected))
(map #(into {} {:id % :page-id page-id})))]
(when frames-with-thumbnail frames-with-thumbnail)))
(defn toggle-file-thumbnail-selected
[]
(ptk/reify ::toggle-file-thumbnail-selected
ptk/WatchEvent
(watch [_ state _]
(let [selected (wsh/lookup-selected state)
pages (get-in state [:workspace-data
:pages-index])
file-thumbnails (->> pages
(mapcat #(extract-file-thumbnails-from-page state selected %)))]
pages (-> state :workspace-data :pages-index vals)
extract (fn [{:keys [objects id] :as page}]
(->> (cph/get-frames objects)
(filter :file-thumbnail)
(map :id)
(remove selected)
(map (fn [frame-id] [id frame-id]))))]
(rx/concat
(rx/from
(for [ft file-thumbnails]
(dch/update-shapes [(:id ft)] #(update % :file-thumbnail not) (:page-id ft) nil)))
(rx/of (dch/update-shapes selected #(update % :file-thumbnail not))))))))
(rx/from (for [[page-id frame-id] (mapcat extract pages)]
(dch/update-shapes [frame-id] #(dissoc % :file-thumbnail) page-id nil)))
(rx/of (dch/update-shapes selected #(assoc % :file-thumbnail true))))))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Navigation
@ -1043,8 +1031,8 @@
[items element]
(let [items (or items #{})]
(if (contains? items element)
(disj set element)
(conj set element))))
(disj items element)
(conj items element))))
(defn toggle-selected-assets
[asset type]

View file

@ -259,7 +259,11 @@
(assoc :stroke-color-gradient (:gradient attrs))
(contains? attrs :opacity)
(assoc :stroke-opacity (:opacity attrs)))
(assoc :stroke-opacity (:opacity attrs))
:always
(d/without-nils))
attrs (merge attrs color-attrs)]
(rx/of (dch/update-shapes ids (fn [shape]

View file

@ -13,6 +13,7 @@
[app.main.data.messages :as msg]
[app.main.data.users :as du]
[app.main.store :as st]
[app.util.globals :as glob]
[app.util.i18n :refer [tr]]
[app.util.router :as rt]
[app.util.timers :as ts]
@ -51,18 +52,6 @@
(st/emit! (du/logout {:capture-redirect true}))
(ts/schedule 500 (st/emitf (msg/warn msg)))))
;; That are special case server-errors that should be treated
;; differently.
(derive :not-found ::exceptional-state)
(derive :bad-gateway ::exceptional-state)
(derive :service-unavailable ::exceptional-state)
(defmethod ptk/handle-error ::exceptional-state
[error]
(ts/schedule
(st/emitf (rt/assign-exception error))))
;; Error that happens on an active business model validation does not
;; passes an validation (example: profile can't leave a team). From
;; the user perspective a error flash message should be visualized but
@ -133,9 +122,22 @@
(js/console.error (with-out-str (expound/printer error)))
(js/console.groupEnd message)))
;; That are special case server-errors that should be treated
;; differently.
(derive :not-found ::exceptional-state)
(derive :bad-gateway ::exceptional-state)
(derive :service-unavailable ::exceptional-state)
(defmethod ptk/handle-error ::exceptional-state
[error]
(ts/schedule
(st/emitf (rt/assign-exception error))))
;; This happens when the backed server fails to process the
;; request. This can be caused by an internal assertion or any other
;; uncontrolled error.
(defmethod ptk/handle-error :server-error
[{:keys [data hint] :as error}]
(let [hint (or hint (:hint data) (:message data))
@ -145,8 +147,8 @@
(ts/schedule
#(st/emit!
(msg/show {:content "Something wrong has happened (on backend)."
:type :error
:timeout 3000})))
:type :error
:timeout 3000})))
(js/console.group msg)
(js/console.info info)
@ -174,6 +176,6 @@
(.preventDefault ^js event)
(some-> (unchecked-get event "error")
(on-unhandled-error)))]
(.addEventListener js/window "error" on-error)
(.addEventListener glob/window "error" on-error)
(fn []
(.removeEventListener js/window "error" on-error))))
(.removeEventListener glob/window "error" on-error))))

View file

@ -214,31 +214,6 @@
[:& shape-wrapper {:shape item
:key (:id item)}])))]]]))
(mf/defc file-thumbnail-svg
{::mf/wrap [mf/memo]}
[{:keys [data embed? include-metadata?] :as props
:or {embed? false include-metadata? false}}]
(let [data (assoc data :x 0 :y 0)
vbox (format-viewbox {:width (:width data 0) :height (:height data 0)})
background-color (get-in data [:options :background] default-color)]
[:& (mf/provider embed/context) {:value embed?}
[:& (mf/provider export/include-metadata-ctx) {:value include-metadata?}
[:svg {:view-box vbox
:version "1.1"
:xmlns "http://www.w3.org/2000/svg"
:xmlnsXlink "http://www.w3.org/1999/xlink"
:xmlns:penpot (when include-metadata? "https://penpot.app/xmlns")
:style {:width "100%"
:height "100%"
:background background-color}}
(when include-metadata?
[:& export/export-page {:options (:options data)}])
[:> shape-container {:shape data}
[:& frame/frame-thumbnail {:shape data}]]]]]))
(mf/defc frame-svg
{::mf/wrap [mf/memo]}
[{:keys [objects frame zoom show-thumbnails?] :or {zoom 1} :as props}]

View file

@ -110,19 +110,7 @@
:index index
:share-id share-id}]))
:render-object
(do
(let [file-id (uuid (get-in route [:path-params :file-id]))
page-id (uuid (get-in route [:path-params :page-id]))
object-id (uuid (get-in route [:path-params :object-id]))
embed? (= (get-in route [:query-params :embed]) "true")
render-texts (get-in route [:query-params :render-texts])]
[:& render/render-object {:file-id file-id
:page-id page-id
:object-id object-id
:embed? embed?
:render-texts? (and (some? render-texts) (= render-texts "true"))}]))
;; TODO: maybe move to `app.render` entrypoint (handled by render.html)
:render-sprite
(do
(let [file-id (uuid (get-in route [:path-params :file-id]))

View file

@ -27,8 +27,6 @@
[app.util.timers :as ts]
[app.util.webapi :as wapi]
[beicon.core :as rx]
[cuerdas.core :as str]
[promesa.core :as p]
[rumext.alpha :as mf]))
(log/set-level! :warn)
@ -38,57 +36,15 @@
(def ^:const CACHE-NAME "penpot")
(def ^:const CACHE-URL "https://penpot.app/cache/")
(defn use-thumbnail-cache
"Creates some hooks to handle the files thumbnails cache"
[file]
(let [cache-url (str CACHE-URL (:id file) "/" (:revn file) ".svg")
get-thumbnail
(mf/use-callback
(mf/deps cache-url)
(fn []
(p/let [response (.match js/caches cache-url)]
(when (some? response)
(p/let [blob (.blob response)
svg-content (.text blob)
headers (.-headers response)
fonts-header (or (.get headers "X-PENPOT-FONTS") "")
fonts (into #{}
(remove #(= "" %))
(str/split fonts-header ","))]
{:svg svg-content
:fonts fonts})))))
cache-thumbnail
(mf/use-callback
(mf/deps cache-url)
(fn [{:keys [svg fonts]}]
(p/let [cache (.open js/caches CACHE-NAME)
blob (js/Blob. #js [svg] #js {:type "image/svg"})
fonts (str/join "," fonts)
headers (js/Headers. #js {"X-PENPOT-FONTS" fonts})
response (js/Response. blob #js {:headers headers})]
(.put cache cache-url response))))]
(mf/use-callback
(mf/deps (:id file) (:revn file))
(fn []
(->> (rx/from (get-thumbnail))
(rx/merge-map
(fn [thumb-data]
(log/debug :msg "retrieve thumbnail" :file (:id file) :revn (:revn file)
:cache (if (some? thumb-data) :hit :miss))
(if (some? thumb-data)
(rx/of thumb-data)
(->> (wrk/ask! {:cmd :thumbnails/generate
:file-id (:id file)})
(rx/tap cache-thumbnail)))))
;; If we have a problem we delegate to the thumbnail generation
(rx/catch #(wrk/ask! {:cmd :thumbnails/generate
:file-id (:id file)})))))))
(mf/use-fn
(mf/deps (:id file) (:revn file))
(fn []
(wrk/ask! {:cmd :thumbnails/generate
:revn (:revn file)
:file-id (:id file)}))))
(mf/defc grid-item-thumbnail
{::mf/wrap [mf/memo]}
@ -100,10 +56,10 @@
(mf/deps file)
(fn []
(->> (generate)
(rx/subs (fn [{:keys [svg fonts]}]
(rx/subs (fn [{:keys [data fonts] :as params}]
(run! fonts/ensure-loaded! fonts)
(when-let [node (mf/ref-val container)]
(dom/set-html! node svg)))))))
(dom/set-html! node data)))))))
[:div.grid-item-th {:style {:background-color (get-in file [:data :options :background])}
:ref container}

View file

@ -18,6 +18,7 @@
[app.main.ui.workspace.shapes :refer [shape-wrapper]]
[app.util.dom :as dom]
[app.util.i18n :as i18n :refer [tr c]]
[app.util.strings :as ust]
[cuerdas.core :as str]
[rumext.alpha :as mf]))
@ -76,7 +77,7 @@
(cond
all-checked? [:span.checked i/checkbox-checked]
all-unchecked? [:span.unchecked i/checkbox-unchecked]
:else [:span i/checkbox-intermediate])]
:else [:span.intermediate i/checkbox-intermediate])]
[:div.field.title (tr "dashboard.export-multiple.selected"
(c (count enabled-exports))
(c (count all-exports)))]]
@ -107,8 +108,8 @@
[:div.field.name (cond-> (:name shape) suffix (str suffix))]
(when (:scale export)
[:div.field.scale (dm/str (* width (:scale export)) "x"
(* height (:scale export)) "px ")])
[:div.field.scale (dm/str (ust/format-precision (* width (:scale export)) 2) "x"
(ust/format-precision (* height (:scale export)) 2) "px ")])
(when (:type export)
[:div.field.extension (-> export :type d/name str/upper)])]))]
@ -175,6 +176,7 @@
progress (:progress state)
exports (:exports state)
total (count exports)
complete? (= progress total)
circ (* 2 Math/PI 12)
pct (- circ (* circ (/ progress total)))
@ -187,6 +189,7 @@
(not healthy?) clr/warning)
title (cond
error? (tr "workspace.options.exporting-object-error")
complete? (tr "workspace.options.exporting-complete")
healthy? (tr "workspace.options.exporting-object")
(not healthy?) (tr "workspace.options.exporting-object-slow"))

View file

@ -7,19 +7,16 @@
(ns app.main.worker
(:require
[app.config :as cfg]
[app.main.errors :as err]
[app.util.worker :as uw]))
(defn on-error
[error]
(js/console.error "Error on worker" (pr-str error)))
(defonce instance (atom nil))
(defn init!
[]
(reset!
instance
(uw/init cfg/worker-uri on-error)))
(uw/init cfg/worker-uri err/on-error)))
(defn ask!
[message]

View file

@ -29,7 +29,6 @@
:version (:full @cf/version)
:public-uri (str cf/public-uri))
(defn- parse-params
[loc]
(let [href (unchecked-get loc "href")]

View file

@ -28,6 +28,10 @@ goog.scope(function() {
addListener(...args) {
},
removeListener(...args) {
},
addEventListener(...args) {
},
removeEventListener(...args) {
}
}
}

View file

@ -15,7 +15,8 @@
(defn immutable-map->map
[obj]
(into {} (map (fn [[k v]] [(keyword k) v])) (seq obj)))
(let [data (into {} (map (fn [[k v]] [(keyword k) v])) (seq obj))]
(assoc data :fills (js->clj (:fills data) :keywordize-keys true))))
;; --- DRAFT-JS HELPERS

View file

@ -52,10 +52,13 @@
(reply [result]
(post {:payload result}))
(reply-error [err]
(.error js/console "error" (pr-str err))
(post {:error {:data (ex-data err)
:message (ex-message err)}}))
(reply-error [cause]
(if (map? cause)
(post {:error cause})
(post {:error {:type :unexpected
:code :unhandled-error-on-worker
:hint (ex-message cause)
:data (ex-data cause)}})))
(reply-completed
([] (reply-completed nil))

View file

@ -16,6 +16,10 @@
[beicon.core :as rx]
[rumext.alpha :as mf]))
(defn- not-found?
[{:keys [type]}]
(= :not-found type))
(defn- handle-response
[response]
(cond
@ -29,38 +33,70 @@
(rx/throw {:type :unexpected
:code (:error response)})))
(defn- request-thumbnail
[file-id]
(let [uri (u/join (cfg/get-public-uri) "api/rpc/query/file-data-for-thumbnail")
params {:file-id file-id
:strip-frames-with-thumbnails true}]
(->> (http/send!
{:method :get
:uri uri
:credentials "include"
:query params})
(defn- request-data-for-thumbnail
[file-id revn]
(let [path "api/rpc/query/file-data-for-thumbnail"
params {:file-id file-id
:revn revn
:strip-frames-with-thumbnails true}
request {:method :get
:uri (u/join (cfg/get-public-uri) path)
:credentials "include"
:query params}]
(->> (http/send! request)
(rx/map http/conditional-decode-transit)
(rx/mapcat handle-response))))
(defonce cache (atom {}))
(defn- request-thumbnail
[file-id revn]
(let [path "api/rpc/query/file-thumbnail"
params {:file-id file-id
:revn revn}
request {:method :get
:uri (u/join (cfg/get-public-uri) path)
:credentials "include"
:query params}]
(->> (http/send! request)
(rx/map http/conditional-decode-transit)
(rx/mapcat handle-response))))
(defn render-frame
[data ckey]
(let [prev (get @cache ckey)]
(if (= (:data prev) data)
(:result prev)
(let [file-thumbnail (:file-thumbnail data)
elem (if file-thumbnail
(mf/element render/file-thumbnail-svg #js {:data file-thumbnail :width "290" :height "150"})
(mf/element render/page-svg #js {:data data :width "290" :height "150" :thumbnails? true}))
result (rds/renderToStaticMarkup elem)]
(swap! cache assoc ckey {:data data :result result})
result))))
(defn- render-thumbnail
[{:keys [data file-id revn] :as params}]
(let [elem (if-let [frame (:thumbnail-frame data)]
(mf/element render/frame-svg #js {:objects (:objects data) :frame frame})
(mf/element render/page-svg #js {:data data :width "290" :height "150" :thumbnails? true}))]
{:data (rds/renderToStaticMarkup elem)
:fonts @fonts/loaded
:file-id file-id
:revn revn}))
(defn- persist-thumbnail
[{:keys [file-id data revn fonts]}]
(let [path "api/rpc/mutation/upsert-file-thumbnail"
params {:file-id file-id
:revn revn
:props {:fonts fonts}
:data data}
request {:method :post
:uri (u/join (cfg/get-public-uri) path)
:credentials "include"
:body (http/transit-data params)}]
(->> (http/send! request)
(rx/map http/conditional-decode-transit)
(rx/mapcat handle-response)
(rx/map (constantly params)))))
(defmethod impl/handler :thumbnails/generate
[{:keys [file-id] :as message}]
(->> (request-thumbnail file-id)
(rx/map
(fn [data]
{:svg (render-frame data #{file-id})
:fonts @fonts/loaded}))))
[{:keys [file-id revn] :as message}]
(letfn [(on-result [{:keys [data props]}]
{:data data
:fonts (:fonts props)})
(on-cache-miss [_]
(->> (request-data-for-thumbnail file-id revn)
(rx/map render-thumbnail)
(rx/mapcat persist-thumbnail)))]
(->> (request-thumbnail file-id revn)
(rx/catch not-found? on-cache-miss)
(rx/map on-result))))

View file

@ -10,8 +10,12 @@
[app.common.pages.helpers :as cph]
[app.common.transit :as t]
[app.common.uuid :as uuid]
[app.main.data.dashboard.shortcuts]
[app.main.data.viewer.shortcuts]
[app.main.data.workspace :as dw]
[app.main.data.workspace.changes :as dwc]
[app.main.data.workspace.path.shortcuts]
[app.main.data.workspace.shortcuts]
[app.main.store :as st]
[app.util.object :as obj]
[app.util.timers :as timers]
@ -301,3 +305,31 @@
[]
(st/emit!
(dw/toggle-layout-flag :hide-ui)))
(defn ^:export shortcuts
[]
(letfn [(print-shortcuts [shortcuts]
(.table js/console
(->> shortcuts
(map (fn [[key {:keys [command]}]]
[(d/name key)
(if (vector? command)
(str/join " | " command)
command)]))
(into {})
(clj->js))))]
(let [style "font-weight: bold; font-size: 1.25rem;"]
(.log js/console "%c Dashboard" style)
(print-shortcuts app.main.data.dashboard.shortcuts/shortcuts)
(.log js/console "%c Workspace" style)
(print-shortcuts app.main.data.workspace.shortcuts/shortcuts)
(.log js/console "%c Path" style)
(print-shortcuts app.main.data.workspace.path.shortcuts/shortcuts)
(.log js/console "%c Viewer" style)
(print-shortcuts app.main.data.viewer.shortcuts/shortcuts)))
nil)

View file

@ -2549,6 +2549,10 @@ msgstr "Suffix"
msgid "workspace.options.exporting-object"
msgstr "Exporting…"
#: src/app/main/ui/workspace/sidebar/options/menus/exports.cljs, src/app/main/ui/handoff/exports.cljs, src/app/main/ui/workspace/header.cljs
msgid "workspace.options.exporting-complete"
msgstr "Export complete"
#: src/app/main/ui/workspace/sidebar/options/menus/exports.cljs, src/app/main/ui/handoff/exports.cljs, src/app/main/ui/workspace/header.cljs
msgid "workspace.options.exporting-object-error"
msgstr "Export failed"

View file

@ -2564,6 +2564,10 @@ msgstr "Sufijo"
msgid "workspace.options.exporting-object"
msgstr "Exportando..."
#: src/app/main/ui/workspace/sidebar/options/menus/exports.cljs, src/app/main/ui/handoff/exports.cljs, src/app/main/ui/workspace/header.cljs
msgid "workspace.options.exporting-complete"
msgstr "Exportación completa"
#: src/app/main/ui/workspace/sidebar/options/menus/exports.cljs, src/app/main/ui/handoff/exports.cljs, src/app/main/ui/workspace/header.cljs
msgid "workspace.options.exporting-object-error"
msgstr "Exportación fallida"
@ -3671,4 +3675,4 @@ msgid "workspace.updates.update"
msgstr "Actualizar"
msgid "workspace.viewport.click-to-close-path"
msgstr "Pulsar para cerrar la ruta"
msgstr "Pulsar para cerrar la ruta"