From b9b53258c145da0340c66fb7da5e5bd9a6701fef Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 10 Aug 2022 12:08:31 +0200 Subject: [PATCH] :sparkles: Make the upload media size configurable --- CHANGES.md | 1 + backend/src/app/config.clj | 11 ++---- backend/src/app/rpc/mutations/media.clj | 37 ++++++++++++------- common/src/app/common/media.cljc | 2 - docker/images/files/nginx.conf | 10 ++--- frontend/src/app/main/data/media.cljs | 7 +--- .../src/app/main/data/workspace/media.cljs | 2 +- frontend/src/app/main/errors.cljs | 24 +++++++----- frontend/translations/ar.po | 2 +- frontend/translations/ca.po | 2 +- frontend/translations/de.po | 2 +- frontend/translations/el.po | 2 +- frontend/translations/en.po | 2 +- frontend/translations/es.po | 2 +- frontend/translations/fa.po | 2 +- frontend/translations/fr.po | 2 +- frontend/translations/he.po | 2 +- frontend/translations/pl.po | 2 +- frontend/translations/pt_BR.po | 2 +- frontend/translations/ro.po | 2 +- frontend/translations/ru.po | 2 +- frontend/translations/tr.po | 2 +- frontend/translations/zh_CN.po | 2 +- frontend/translations/zh_Hant.po | 2 +- 24 files changed, 66 insertions(+), 60 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 18053ea6e..17114bc6a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -31,6 +31,7 @@ - Support for import/export binary format [Taiga #2991](https://tree.taiga.io/project/penpot/us/2991) - Comments positioning [Taiga #2007](https://tree.taiga.io/project/penpot/us/2007) - Select all inside a group select only the objects at this group level [Taiga #2382](https://tree.taiga.io/project/penpot/issue/2382) +- Make the media maximum upload size configurable ### :bug: Bugs fixed diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index aa5e65d2f..a09efb92e 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -83,11 +83,11 @@ ;; a server prop key where initial project is stored. :initial-project-skey "initial-project"}) -(s/def ::flags ::us/vec-of-valid-keywords) -;; DEPRECATED PROPERTIES +(s/def ::media-max-file-size ::us/integer) + +(s/def ::flags ::us/vec-of-valid-keywords) (s/def ::telemetry-enabled ::us/boolean) -;; END DEPRECATED (s/def ::audit-log-archive-uri ::us/string) (s/def ::audit-log-gc-max-age ::dt/duration) @@ -143,8 +143,6 @@ (s/def ::http-server-max-multipart-body-size ::us/integer) (s/def ::http-server-io-threads ::us/integer) (s/def ::http-server-worker-threads ::us/integer) -(s/def ::http-session-updater-batch-max-age ::dt/duration) -(s/def ::http-session-updater-batch-max-size ::us/integer) (s/def ::initial-project-skey ::us/string) (s/def ::ldap-attrs-email ::us/string) (s/def ::ldap-attrs-fullname ::us/string) @@ -251,8 +249,6 @@ ::http-server-max-multipart-body-size ::http-server-io-threads ::http-server-worker-threads - ::http-session-updater-batch-max-age - ::http-session-updater-batch-max-size ::initial-project-skey ::ldap-attrs-email ::ldap-attrs-fullname @@ -268,6 +264,7 @@ ::local-assets-uri ::loggers-loki-uri ::loggers-zmq-uri + ::media-max-file-size ::profile-bounce-max-age ::profile-bounce-threshold ::profile-complaint-max-age diff --git a/backend/src/app/rpc/mutations/media.clj b/backend/src/app/rpc/mutations/media.clj index 1a344289d..25d894d77 100644 --- a/backend/src/app/rpc/mutations/media.clj +++ b/backend/src/app/rpc/mutations/media.clj @@ -22,9 +22,12 @@ [app.util.services :as sv] [app.util.time :as dt] [clojure.spec.alpha :as s] + [cuerdas.core :as str] [promesa.core :as p] [promesa.exec :as px])) +(def default-max-file-size (* 1024 1024 10)) ; 10 MiB + (def thumbnail-options {:width 100 :height 100 @@ -51,10 +54,20 @@ (sv/defmethod ::upload-file-media-object {::rlimit/permits (cf/get :rlimit-image)} - [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}] + [{:keys [pool] :as cfg} {:keys [profile-id file-id content] :as params}] (let [file (select-file pool file-id) cfg (update cfg :storage media/configure-assets-storage)] + (teams/check-edition-permissions! pool profile-id (:team-id file)) + (media/validate-media-type! content) + + (when (> (:size content) (cf/get :media-max-file-size default-max-file-size)) + (ex/raise :type :restriction + :code :media-max-file-size-reached + :hint (str/ffmt "the uploaded file size % is greater than the maximum %" + (:size content) + default-max-file-size))) + (create-file-media-object cfg params))) (defn- big-enough-for-thumbnail? @@ -94,8 +107,6 @@ (defn create-file-media-object [{:keys [storage pool executors] :as cfg} {:keys [id file-id is-local name content] :as params}] - (media/validate-media-type! content) - (letfn [;; Function responsible to retrieve the file information, as ;; it is synchronous operation it should be wrapped into ;; with-dispatch macro. @@ -177,30 +188,30 @@ (teams/check-edition-permissions! pool profile-id (:team-id file)) (create-file-media-object-from-url cfg params))) -(def max-download-file-size - (* 1024 1024 100)) ; 100MiB - (defn- create-file-media-object-from-url [{:keys [http-client] :as cfg} {:keys [url name] :as params}] (letfn [(parse-and-validate-size [headers] - (let [size (some-> (get headers "content-length") d/parse-integer) - mtype (get headers "content-type") - format (cm/mtype->format mtype)] + (let [size (some-> (get headers "content-length") d/parse-integer) + mtype (get headers "content-type") + format (cm/mtype->format mtype) + max-size (cf/get :media-max-file-size default-max-file-size)] (when-not size (ex/raise :type :validation :code :unknown-size - :hint "Seems like the url points to resource with unknown size")) + :hint "seems like the url points to resource with unknown size")) - (when (> size max-download-file-size) + (when (> size max-size) (ex/raise :type :validation :code :file-too-large - :hint "Seems like the url points to resource with size greater than 100MiB")) + :hint (str/ffmt "the file size % is greater than the maximum %" + size + default-max-file-size))) (when (nil? format) (ex/raise :type :validation :code :media-type-not-allowed - :hint "Seems like the url points to an invalid media object")) + :hint "seems like the url points to an invalid media object")) {:size size :mtype mtype diff --git a/common/src/app/common/media.cljc b/common/src/app/common/media.cljc index a024d02b4..23cbe7101 100644 --- a/common/src/app/common/media.cljc +++ b/common/src/app/common/media.cljc @@ -59,8 +59,6 @@ "application/pdf" ".pdf" nil)) -(def max-file-size (* 5 1024 1024)) - (s/def ::id uuid?) (s/def ::name string?) (s/def ::width number?) diff --git a/docker/images/files/nginx.conf b/docker/images/files/nginx.conf index 35863895b..0302a2a83 100644 --- a/docker/images/files/nginx.conf +++ b/docker/images/files/nginx.conf @@ -19,8 +19,8 @@ http { server_tokens off; reset_timedout_connection on; - client_body_timeout 20s; - client_header_timeout 20s; + client_body_timeout 30s; + client_header_timeout 30s; include /etc/nginx/mime.types; default_type application/octet-stream; @@ -49,7 +49,7 @@ http { listen 80 default_server; server_name _; - client_max_body_size 50M; + client_max_body_size 100M; charset utf-8; proxy_http_version 1.1; @@ -78,10 +78,6 @@ http { proxy_pass http://penpot-backend:6060/api; } - location /dbg { - proxy_pass http://penpot-backend:6060/dbg; - } - location /ws/notifications { proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection 'upgrade'; diff --git a/frontend/src/app/main/data/media.cljs b/frontend/src/app/main/data/media.cljs index d4b28a805..d101dccd5 100644 --- a/frontend/src/app/main/data/media.cljs +++ b/frontend/src/app/main/data/media.cljs @@ -35,12 +35,9 @@ ;; --- Utility functions -(defn validate-file ;; Check that a file obtained with the file javascript API is valid. +(defn validate-file + "Check that a file obtained with the file javascript API is valid." [file] - (when (> (.-size file) cm/max-file-size) - (ex/raise :type :validation - :code :media-too-large - :hint (str/fmt "media size is large than 5mb (size: %s)" (.-size file)))) (when-not (contains? cm/valid-image-types (.-type file)) (ex/raise :type :validation :code :media-type-not-allowed diff --git a/frontend/src/app/main/data/workspace/media.cljs b/frontend/src/app/main/data/workspace/media.cljs index ce0959c33..428d5ac44 100644 --- a/frontend/src/app/main/data/workspace/media.cljs +++ b/frontend/src/app/main/data/workspace/media.cljs @@ -200,7 +200,7 @@ (= (:code error) :invalid-image) (rx/of (dm/error (tr "errors.media-type-not-allowed"))) - (= (:code error) :media-too-large) + (= (:code error) :media-max-file-size-reached) (rx/of (dm/error (tr "errors.media-too-large"))) (= (:code error) :media-type-mismatch) diff --git a/frontend/src/app/main/errors.cljs b/frontend/src/app/main/errors.cljs index 7a8edbbbe..abfde9001 100644 --- a/frontend/src/app/main/errors.cljs +++ b/frontend/src/app/main/errors.cljs @@ -46,6 +46,14 @@ ;; Set the main potok error handler (reset! st/on-error on-error) +(defmethod ptk/handle-error :default + [error] + (let [hint (str/concat "Unexpected error: " (:hint error))] + (ts/schedule #(st/emit! (rt/assign-exception error))) + (js/console.group hint) + (ex/ignoring (js/console.error (pr-str error))) + (js/console.groupEnd hint))) + ;; We receive a explicit authentication error; this explicitly clears ;; all profile data and redirect the user to the login page. This is ;; here and not in app.main.errors because of circular dependency. @@ -188,9 +196,10 @@ (defn on-unhandled-error [error] (letfn [(is-ignorable-exception? [cause] - (condp = (ex-message cause) - "Possible side-effect in debug-evaluate" true - false))] + (let [message (ex-message cause)] + (or (= message "Possible side-effect in debug-evaluate") + (= message "Unexpected end of input") true + (str/starts-with? message "Unexpected token "))))] (if (instance? ExceptionInfo error) (-> error ex-data ptk/handle-error) (when-not (is-ignorable-exception? error) @@ -203,12 +212,9 @@ (defonce uncaught-error-handler (letfn [(on-error [event] - ;; EvalError is a debug error that happens for unknown reason - (when-not (str/includes? (.-message event) "EvalError") - (.error js/console event) - (.preventDefault ^js event) - (some-> (unchecked-get event "error") - (on-unhandled-error))))] + (.preventDefault ^js event) + (some-> (unchecked-get event "error") + (on-unhandled-error)))] (.addEventListener glob/window "error" on-error) (fn [] (.removeEventListener glob/window "error" on-error)))) diff --git a/frontend/translations/ar.po b/frontend/translations/ar.po index 9b64bb5ad..19f2b752d 100644 --- a/frontend/translations/ar.po +++ b/frontend/translations/ar.po @@ -499,7 +499,7 @@ msgstr "تنسيق الصورة غير مدعوم (يجب أن يكون svg أو #: src/app/main/data/workspace/persistence.cljs msgid "errors.media-too-large" -msgstr "الصورة كبيرة جدا بحيث لا يمكن إدراجها (يجب أن تكون أقل من 5mb)." +msgstr "الصورة كبيرة جدا بحيث لا يمكن إدراجها." #: src/app/main/data/workspace/persistence.cljs, src/app/main/data/media.cljs msgid "errors.media-type-mismatch" diff --git a/frontend/translations/ca.po b/frontend/translations/ca.po index f39231241..57a3cd2a5 100644 --- a/frontend/translations/ca.po +++ b/frontend/translations/ca.po @@ -678,7 +678,7 @@ msgstr "El format d'imatge no està suportat (ha de ser SVG, JPG o PNG)." #: src/app/main/data/workspace/persistence.cljs msgid "errors.media-too-large" -msgstr "La imatge és massa gran (ha de ser inferior a 5 MB)." +msgstr "La imatge és massa gran." #: src/app/main/data/workspace/persistence.cljs, src/app/main/data/media.cljs msgid "errors.media-type-mismatch" diff --git a/frontend/translations/de.po b/frontend/translations/de.po index 1884b139f..bc2859935 100644 --- a/frontend/translations/de.po +++ b/frontend/translations/de.po @@ -693,7 +693,7 @@ msgstr "Das Bildformat wird nicht unterstützt (es muss ein SVG, JPG oder PNG se #: src/app/main/data/workspace/persistence.cljs msgid "errors.media-too-large" -msgstr "Das Bild ist zu groß, um eingefügt zu werden (es muss unter 5MB sein)." +msgstr "Das Bild ist zu groß, um eingefügt zu werden." #: src/app/main/data/workspace/persistence.cljs, src/app/main/data/media.cljs msgid "errors.media-type-mismatch" diff --git a/frontend/translations/el.po b/frontend/translations/el.po index 389f8cb21..70ccaf123 100644 --- a/frontend/translations/el.po +++ b/frontend/translations/el.po @@ -414,7 +414,7 @@ msgstr "Η μορφή εικόνας δεν αναγνωρίζεται (πρέπ #: src/app/main/data/workspace/persistence.cljs msgid "errors.media-too-large" -msgstr "Η εικόνα είναι πολύ μεγάλη (πρέπει να είναι μικρότερη από 5mb)." +msgstr "Η εικόνα είναι πολύ μεγάλη." #: src/app/main/data/workspace/persistence.cljs, src/app/main/data/media.cljs msgid "errors.media-type-mismatch" diff --git a/frontend/translations/en.po b/frontend/translations/en.po index ad5bec27f..691cdf557 100644 --- a/frontend/translations/en.po +++ b/frontend/translations/en.po @@ -720,7 +720,7 @@ msgstr "The image format is not supported (must be svg, jpg or png)." #: src/app/main/data/workspace/persistence.cljs msgid "errors.media-too-large" -msgstr "The image is too large to be inserted (must be under 5mb)." +msgstr "The image is too large to be inserted." #: src/app/main/data/workspace/persistence.cljs, src/app/main/data/media.cljs msgid "errors.media-type-mismatch" diff --git a/frontend/translations/es.po b/frontend/translations/es.po index f7c366b6a..e39a59fa6 100644 --- a/frontend/translations/es.po +++ b/frontend/translations/es.po @@ -738,7 +738,7 @@ msgstr "No se reconoce el formato de imagen (debe ser svg, jpg o png)." #: src/app/main/data/workspace/persistence.cljs msgid "errors.media-too-large" -msgstr "La imagen es demasiado grande (debe tener menos de 5mb)." +msgstr "La imagen es demasiado grande." #: src/app/main/data/workspace/persistence.cljs, src/app/main/data/media.cljs msgid "errors.media-type-mismatch" diff --git a/frontend/translations/fa.po b/frontend/translations/fa.po index 0b102042d..2e1a92081 100644 --- a/frontend/translations/fa.po +++ b/frontend/translations/fa.po @@ -670,7 +670,7 @@ msgstr "فرمت تصویر پشتیبانی نمی‌شود (باید svg، jpg #: src/app/main/data/workspace/persistence.cljs msgid "errors.media-too-large" -msgstr "تصویر برای درج خیلی بزرگ است (باید کمتر از 5 مگابایت باشد)." +msgstr "تصویر برای درج خیلی بزرگ است." #: src/app/main/data/workspace/persistence.cljs, src/app/main/data/media.cljs msgid "errors.media-type-mismatch" diff --git a/frontend/translations/fr.po b/frontend/translations/fr.po index f08b6c145..57128e888 100644 --- a/frontend/translations/fr.po +++ b/frontend/translations/fr.po @@ -703,7 +703,7 @@ msgstr "Le format d’image n’est pas supporté (doit être svg, jpg ou png)." #: src/app/main/data/workspace/persistence.cljs msgid "errors.media-too-large" -msgstr "L’image est trop grande (doit être inférieure à 5 Mo)." +msgstr "L’image est trop grande." #: src/app/main/data/workspace/persistence.cljs, src/app/main/data/media.cljs msgid "errors.media-type-mismatch" diff --git a/frontend/translations/he.po b/frontend/translations/he.po index 0a6a46101..66bb26c28 100644 --- a/frontend/translations/he.po +++ b/frontend/translations/he.po @@ -675,7 +675,7 @@ msgstr "סוג התמונה אינו נתמך (חייב להיות svg,‏ jpg #: src/app/main/data/workspace/persistence.cljs msgid "errors.media-too-large" -msgstr "התמונה גדולה מכדי להוסיף אותה (חייבת להיות פחות מ־5 מ״ב)." +msgstr "התמונה גדולה מכדי להוסיף אותה." #: src/app/main/data/workspace/persistence.cljs, src/app/main/data/media.cljs msgid "errors.media-type-mismatch" diff --git a/frontend/translations/pl.po b/frontend/translations/pl.po index a55ea73a5..214b12be8 100644 --- a/frontend/translations/pl.po +++ b/frontend/translations/pl.po @@ -680,7 +680,7 @@ msgstr "Format obrazu nie jest obsługiwany (musi to być svg, jpg lub png)." #: src/app/main/data/workspace/persistence.cljs msgid "errors.media-too-large" -msgstr "Obraz jest zbyt duży (musi mieć mniej niż 5 MB)." +msgstr "Obraz jest zbyt duży." #: src/app/main/data/workspace/persistence.cljs, src/app/main/data/media.cljs msgid "errors.media-type-mismatch" diff --git a/frontend/translations/pt_BR.po b/frontend/translations/pt_BR.po index 29fca3f72..d22c1f4dd 100644 --- a/frontend/translations/pt_BR.po +++ b/frontend/translations/pt_BR.po @@ -591,7 +591,7 @@ msgstr "O formato da imagem não é compatível (deve ser svg, jpg ou png)." #: src/app/main/data/workspace/persistence.cljs msgid "errors.media-too-large" -msgstr "A imagem é muito grande para ser inserida (deve ter menos de 5mb)." +msgstr "A imagem é muito grande para ser inserida." #: src/app/main/data/workspace/persistence.cljs, src/app/main/data/media.cljs msgid "errors.media-type-mismatch" diff --git a/frontend/translations/ro.po b/frontend/translations/ro.po index ab6d47f7a..44bfa44b7 100644 --- a/frontend/translations/ro.po +++ b/frontend/translations/ro.po @@ -469,7 +469,7 @@ msgstr "Formatul imaginii nu este acceptat (poate fi svg, jpg sau png)." #: src/app/main/data/workspace/persistence.cljs msgid "errors.media-too-large" -msgstr "Imaginea este prea mare pentru a fi inserată (trebuie să fie sub 5mb)." +msgstr "Imaginea este prea mare pentru a fi inserată." #: src/app/main/data/workspace/persistence.cljs, src/app/main/data/media.cljs msgid "errors.media-type-mismatch" diff --git a/frontend/translations/ru.po b/frontend/translations/ru.po index 8dbd53626..5fbf98192 100644 --- a/frontend/translations/ru.po +++ b/frontend/translations/ru.po @@ -632,7 +632,7 @@ msgstr "Формат изображения не поддерживается ( #: src/app/main/data/workspace/persistence.cljs msgid "errors.media-too-large" -msgstr "Изображение слишком большое для вставки (должно быть меньше 5mb)." +msgstr "Изображение слишком большое для вставки." #: src/app/main/data/workspace/persistence.cljs, src/app/main/data/media.cljs msgid "errors.media-type-mismatch" diff --git a/frontend/translations/tr.po b/frontend/translations/tr.po index 49d903753..2bc75fcc4 100644 --- a/frontend/translations/tr.po +++ b/frontend/translations/tr.po @@ -692,7 +692,7 @@ msgstr "Görsel biçimi desteklenmiyor (svg, jpg veya png olmalı)." #: src/app/main/data/workspace/persistence.cljs msgid "errors.media-too-large" -msgstr "Bu görsel eklemek için çok büyük (5MB altında olmalı)." +msgstr "Bu görsel eklemek için çok büyük." #: src/app/main/data/workspace/persistence.cljs, src/app/main/data/media.cljs msgid "errors.media-type-mismatch" diff --git a/frontend/translations/zh_CN.po b/frontend/translations/zh_CN.po index 2106ec75c..26a4b2830 100644 --- a/frontend/translations/zh_CN.po +++ b/frontend/translations/zh_CN.po @@ -655,7 +655,7 @@ msgstr "不支持该图片格式（只能是svg、jpg或png）。" #: src/app/main/data/workspace/persistence.cljs msgid "errors.media-too-large" -msgstr "图片尺寸过大，故无法插入（不能超过5MB）。" +msgstr "图片尺寸过大，故无法插入。" #: src/app/main/data/workspace/persistence.cljs, src/app/main/data/media.cljs msgid "errors.media-type-mismatch" diff --git a/frontend/translations/zh_Hant.po b/frontend/translations/zh_Hant.po index 104ee5422..672a86430 100644 --- a/frontend/translations/zh_Hant.po +++ b/frontend/translations/zh_Hant.po @@ -541,7 +541,7 @@ msgstr "不支援此影像格式（必須是 svg，jpg 或者 png）" #: src/app/main/data/workspace/persistence.cljs msgid "errors.media-too-large" -msgstr "影像檔案過大，無法插入（須小於 5MB）" +msgstr "影像檔案過大，無法插入" #: src/app/main/data/workspace/persistence.cljs, src/app/main/data/workspace/persistence.cljs, src/app/main/data/workspace/persistence.cljs, src/app/main/data/workspace/persistence.cljs, src/app/main/data/media.cljs msgid "errors.media-type-not-allowed"