From efa5120fac6d775f41af5b21680b95ccdf336c8c Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 29 Jul 2021 12:34:17 +0200 Subject: [PATCH] :sparkles: Fix inconsistencies on storage backend usage. --- backend/src/app/config.clj | 2 +- backend/src/app/main.clj | 5 +---- backend/src/app/media.clj | 19 +++++++++++++++---- backend/src/app/rpc/mutations/files.clj | 3 +-- backend/src/app/rpc/mutations/media.clj | 4 +--- backend/src/app/rpc/mutations/profile.clj | 19 +++++++++++-------- backend/src/app/rpc/mutations/teams.clj | 10 ++++++---- backend/src/app/rpc/queries/files.clj | 3 +-- backend/src/app/storage.clj | 6 ++---- docker/images/config.env | 4 ++-- 10 files changed, 41 insertions(+), 34 deletions(-) diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index f7c587d48..27df0e778 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -58,7 +58,7 @@ :srepl-host "127.0.0.1" :srepl-port 6062 - :assets-storage-backend :fs + :assets-storage-backend :assets-fs :storage-assets-fs-directory "assets" :feedback-destination "info@example.com" diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index dc4355910..15275370f 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -322,13 +322,10 @@ :app.storage/storage {:pool (ig/ref :app.db/pool) :executor (ig/ref :app.worker/executor) - :backend (cf/get :assets-storage-backend :assets-fs) + :backends {:assets-s3 (ig/ref [::assets :app.storage.s3/backend]) :assets-db (ig/ref [::assets :app.storage.db/backend]) :assets-fs (ig/ref [::assets :app.storage.fs/backend]) - :s3 (ig/ref [::assets :app.storage.s3/backend]) - :db (ig/ref [::assets :app.storage.db/backend]) - :fs (ig/ref [::assets :app.storage.fs/backend]) :tmp (ig/ref [::tmp :app.storage.fs/backend]) :fdata-s3 (ig/ref [::fdata :app.storage.s3/backend])}} diff --git a/backend/src/app/media.clj b/backend/src/app/media.clj index b2aeefa18..59019558d 100644 --- a/backend/src/app/media.clj +++ b/backend/src/app/media.clj @@ -11,6 +11,7 @@ [app.common.exceptions :as ex] [app.common.media :as cm] [app.common.spec :as us] + [app.config :as cf] [app.rlimits :as rlm] [app.rpc.queries.svg :as svg] [buddy.core.bytes :as bb] @@ -28,10 +29,6 @@ org.im4java.core.IMOperation org.im4java.core.Info)) -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; --- Utility functions -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - (s/def ::image-content-type cm/valid-image-types) (s/def ::font-content-type cm/valid-font-types) @@ -330,3 +327,17 @@ (= stype :ttf) (-> (assoc "font/otf" (ttf->otf sfnt)) (assoc "font/ttf" sfnt))))))))) + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Utility functions +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn configure-assets-storage + "Given storage map, returns a storage configured with the apropriate + backend for assets." + [storage conn] + (-> storage + (assoc :conn conn) + (assoc :backend (cf/get :assets-storage-backend :assets-fs)))) + diff --git a/backend/src/app/rpc/mutations/files.clj b/backend/src/app/rpc/mutations/files.clj index 3c266268d..6436f0fc9 100644 --- a/backend/src/app/rpc/mutations/files.clj +++ b/backend/src/app/rpc/mutations/files.clj @@ -11,7 +11,6 @@ [app.common.pages.migrations :as pmg] [app.common.spec :as us] [app.common.uuid :as uuid] - [app.config :as cf] [app.db :as db] [app.rpc.permissions :as perms] [app.rpc.queries.files :as files] @@ -288,7 +287,7 @@ (defn- delete-from-storage [{:keys [storage] :as cfg} file] - (when-let [backend (simpl/resolve-backend storage (cf/get :fdata-storage-backend))] + (when-let [backend (simpl/resolve-backend storage (:data-backend file))] (simpl/del-object backend file))) (defn- update-file diff --git a/backend/src/app/rpc/mutations/media.clj b/backend/src/app/rpc/mutations/media.clj index 3467522f1..bfe876df1 100644 --- a/backend/src/app/rpc/mutations/media.clj +++ b/backend/src/app/rpc/mutations/media.clj @@ -32,7 +32,6 @@ (s/def ::file-id ::us/uuid) (s/def ::team-id ::us/uuid) - ;; --- Create File Media object (upload) (declare create-file-media-object) @@ -94,10 +93,9 @@ (defn create-file-media-object [{:keys [conn storage] :as cfg} {:keys [id file-id is-local name content] :as params}] (media/validate-media-type (:content-type content)) - (let [storage (assoc storage :conn conn) + (let [storage (media/configure-assets-storage storage conn) source-path (fs/path (:tempfile content)) source-mtype (:content-type content) - source-info (media/run cfg {:cmd :info :input {:path source-path :mtype source-mtype}}) thumb (when (and (not (svg-image? source-info)) diff --git a/backend/src/app/rpc/mutations/profile.clj b/backend/src/app/rpc/mutations/profile.clj index 17c52869a..cc1dce079 100644 --- a/backend/src/app/rpc/mutations/profile.clj +++ b/backend/src/app/rpc/mutations/profile.clj @@ -9,7 +9,7 @@ [app.common.exceptions :as ex] [app.common.spec :as us] [app.common.uuid :as uuid] - [app.config :as cfg] + [app.config :as cf] [app.db :as db] [app.emails :as eml] [app.http.oauth :refer [extract-props]] @@ -99,11 +99,11 @@ (sv/defmethod ::prepare-register-profile {:auth false} [{:keys [pool tokens] :as cfg} params] - (when-not (cfg/get :registration-enabled) + (when-not (cf/get :registration-enabled) (ex/raise :type :restriction :code :registration-disabled)) - (when-let [domains (cfg/get :registration-domain-whitelist)] + (when-let [domains (cf/get :registration-domain-whitelist)] (when-not (email-domain-in-whitelist? domains (:email params)) (ex/raise :type :validation :code :email-domain-is-not-allowed))) @@ -402,6 +402,7 @@ {:password (derive-password password)} {:id id})) + ;; --- MUTATION: Update Photo (declare update-profile-photo) @@ -416,11 +417,13 @@ [{:keys [pool storage] :as cfg} {:keys [profile-id file] :as params}] (db/with-atomic [conn pool] (media/validate-media-type (:content-type file) #{"image/jpeg" "image/png" "image/webp"}) + (media/run cfg {:cmd :info :input {:path (:tempfile file) + :mtype (:content-type file)}}) + (let [profile (db/get-by-id conn :profile profile-id) - _ (media/run cfg {:cmd :info :input {:path (:tempfile file) - :mtype (:content-type file)}}) - photo (teams/upload-photo cfg params) - storage (assoc storage :conn conn)] + storage (media/configure-assets-storage storage conn) + cfg (assoc cfg :storage storage) + photo (teams/upload-photo cfg params)] ;; Schedule deletion of old photo (when-let [id (:photo-id profile)] @@ -453,7 +456,7 @@ params (assoc params :profile profile :email (str/lower email))] - (if (cfg/get :smtp-enabled) + (if (cf/get :smtp-enabled) (request-email-change cfg params) (change-email-inmediatelly cfg params))))) diff --git a/backend/src/app/rpc/mutations/teams.clj b/backend/src/app/rpc/mutations/teams.clj index 3b71c43c4..d0eae6dee 100644 --- a/backend/src/app/rpc/mutations/teams.clj +++ b/backend/src/app/rpc/mutations/teams.clj @@ -251,10 +251,12 @@ (db/with-atomic [conn pool] (teams/check-edition-permissions! conn profile-id team-id) (media/validate-media-type (:content-type file) #{"image/jpeg" "image/png" "image/webp"}) + (media/run cfg {:cmd :info :input {:path (:tempfile file) + :mtype (:content-type file)}}) (let [team (teams/retrieve-team conn profile-id team-id) - _ (media/run cfg {:cmd :info :input {:path (:tempfile file) - :mtype (:content-type file)}}) + storage (media/configure-assets-storage storage conn) + cfg (assoc cfg :storage storage) photo (upload-photo cfg params)] ;; Schedule deletion of old photo @@ -263,8 +265,8 @@ ;; Save new photo (db/update! conn :team - {:photo-id (:id photo)} - {:id team-id}) + {:photo-id (:id photo)} + {:id team-id}) (assoc team :photo-id (:id photo))))) diff --git a/backend/src/app/rpc/queries/files.clj b/backend/src/app/rpc/queries/files.clj index 74938feb2..55f536d5f 100644 --- a/backend/src/app/rpc/queries/files.clj +++ b/backend/src/app/rpc/queries/files.clj @@ -9,7 +9,6 @@ [app.common.pages.migrations :as pmg] [app.common.spec :as us] [app.common.uuid :as uuid] - [app.config :as cf] [app.db :as db] [app.rpc.permissions :as perms] [app.rpc.queries.projects :as projects] @@ -175,7 +174,7 @@ (defn- retrieve-data* [{:keys [storage] :as cfg} file] - (when-let [backend (simpl/resolve-backend storage (cf/get :fdata-storage-backend))] + (when-let [backend (simpl/resolve-backend storage (:data-backend file))] (simpl/get-object-bytes backend file))) (defn retrieve-data diff --git a/backend/src/app/storage.clj b/backend/src/app/storage.clj index f8a923a2d..e6acb0776 100644 --- a/backend/src/app/storage.clj +++ b/backend/src/app/storage.clj @@ -28,8 +28,6 @@ ;; Storage Module State ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(s/def ::backend ::us/keyword) - (s/def ::s3 ::ss3/backend) (s/def ::fs ::sfs/backend) (s/def ::db ::sdb/backend) @@ -42,7 +40,7 @@ :db ::sdb/backend)))) (defmethod ig/pre-init-spec ::storage [_] - (s/keys :req-un [::backend ::wrk/executor ::db/pool ::backends])) + (s/keys :req-un [::wrk/executor ::db/pool ::backends])) (defmethod ig/prep-key ::storage [_ {:keys [backends] :as cfg}] @@ -55,7 +53,7 @@ (assoc :backends (d/without-nils backends)))) (s/def ::storage - (s/keys :req-un [::backends ::wrk/executor ::db/pool ::backend])) + (s/keys :req-un [::backends ::wrk/executor ::db/pool])) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Database Objects diff --git a/docker/images/config.env b/docker/images/config.env index 6d1f76bd2..157d2be43 100644 --- a/docker/images/config.env +++ b/docker/images/config.env @@ -13,8 +13,8 @@ PENPOT_REDIS_URI=redis://penpot-redis/0 # can be configured to store in AWS S3 or completely in de the database. # Storing in the database makes the backups more easy but will make access to # media less performant. -PENPOT_STORAGE_BACKEND=fs -PENPOT_STORAGE_FS_DIRECTORY=/opt/data/assets +ASSETS_STORAGE_BACKEND=assets-fs +PENPOT_STORAGE_ASSETS_FS_DIRECTORY=/opt/data/assets # Telemetry. When enabled, a periodical process will send anonymous data about # this instance. Telemetry data will enable us to learn on how the application