From 9174bb140bc109409684af775e8d4e570a6761d6 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 20 Jun 2024 14:36:34 +0200 Subject: [PATCH] :recycle: Refactor configuration validation Replace spec with schema --- backend/src/app/auth/ldap.clj | 24 +- backend/src/app/auth/oidc.clj | 22 +- backend/src/app/config.clj | 421 +++++++------------ backend/src/app/email.clj | 42 +- backend/src/app/main.clj | 1 + backend/src/app/media.clj | 13 - backend/src/app/rpc/commands/demo.clj | 2 +- backend/src/app/tasks/file_gc.clj | 2 +- backend/src/app/tasks/objects_gc.clj | 2 +- backend/src/app/tasks/tasks_gc.clj | 2 +- backend/src/app/util/overrides.clj | 41 ++ backend/test/backend_tests/helpers.clj | 10 +- backend/test/backend_tests/rpc_file_test.clj | 22 +- common/src/app/common/schema.cljc | 7 +- 14 files changed, 267 insertions(+), 344 deletions(-) create mode 100644 backend/src/app/util/overrides.clj diff --git a/backend/src/app/auth/ldap.clj b/backend/src/app/auth/ldap.clj index 5100abff9..c430a794d 100644 --- a/backend/src/app/auth/ldap.clj +++ b/backend/src/app/auth/ldap.clj @@ -9,7 +9,6 @@ [app.common.exceptions :as ex] [app.common.logging :as l] [app.common.spec :as us] - [app.config :as cf] [clj-ldap.client :as ldap] [clojure.spec.alpha :as s] [clojure.string] @@ -104,17 +103,17 @@ nil)))) (s/def ::enabled? ::us/boolean) -(s/def ::host ::cf/ldap-host) -(s/def ::port ::cf/ldap-port) -(s/def ::ssl ::cf/ldap-ssl) -(s/def ::tls ::cf/ldap-starttls) -(s/def ::query ::cf/ldap-user-query) -(s/def ::base-dn ::cf/ldap-base-dn) -(s/def ::bind-dn ::cf/ldap-bind-dn) -(s/def ::bind-password ::cf/ldap-bind-password) -(s/def ::attrs-email ::cf/ldap-attrs-email) -(s/def ::attrs-fullname ::cf/ldap-attrs-fullname) -(s/def ::attrs-username ::cf/ldap-attrs-username) +(s/def ::host ::us/string) +(s/def ::port ::us/integer) +(s/def ::ssl ::us/boolean) +(s/def ::tls ::us/boolean) +(s/def ::query ::us/string) +(s/def ::base-dn ::us/string) +(s/def ::bind-dn ::us/string) +(s/def ::bind-password ::us/string) +(s/def ::attrs-email ::us/string) +(s/def ::attrs-fullname ::us/string) +(s/def ::attrs-username ::us/string) (s/def ::provider-params (s/keys :opt-un [::host ::port @@ -126,6 +125,7 @@ ::attrs-email ::attrs-username ::attrs-fullname])) + (s/def ::provider (s/nilable ::provider-params)) diff --git a/backend/src/app/auth/oidc.clj b/backend/src/app/auth/oidc.clj index 69f7eb7d0..986e24cbb 100644 --- a/backend/src/app/auth/oidc.clj +++ b/backend/src/app/auth/oidc.clj @@ -629,17 +629,17 @@ :provider provider :hint "provider not configured"))))))}) -(s/def ::client-id ::cf/oidc-client-id) -(s/def ::client-secret ::cf/oidc-client-secret) -(s/def ::base-uri ::cf/oidc-base-uri) -(s/def ::token-uri ::cf/oidc-token-uri) -(s/def ::auth-uri ::cf/oidc-auth-uri) -(s/def ::user-uri ::cf/oidc-user-uri) -(s/def ::scopes ::cf/oidc-scopes) -(s/def ::roles ::cf/oidc-roles) -(s/def ::roles-attr ::cf/oidc-roles-attr) -(s/def ::email-attr ::cf/oidc-email-attr) -(s/def ::name-attr ::cf/oidc-name-attr) +(s/def ::client-id ::us/string) +(s/def ::client-secret ::us/string) +(s/def ::base-uri ::us/string) +(s/def ::token-uri ::us/string) +(s/def ::auth-uri ::us/string) +(s/def ::user-uri ::us/string) +(s/def ::scopes ::us/set-of-strings) +(s/def ::roles ::us/set-of-strings) +(s/def ::roles-attr ::us/string) +(s/def ::email-attr ::us/string) +(s/def ::name-attr ::us/string) (s/def ::provider (s/keys :req-un [::client-id diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index aed780694..602da9d24 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -11,30 +11,17 @@ [app.common.data :as d] [app.common.exceptions :as ex] [app.common.flags :as flags] - [app.common.spec :as us] + [app.common.schema :as sm] [app.common.version :as v] + [app.util.overrides] [app.util.time :as dt] [clojure.core :as c] [clojure.java.io :as io] - [clojure.pprint :as pprint] - [clojure.spec.alpha :as s] [cuerdas.core :as str] [datoteka.fs :as fs] [environ.core :refer [env]] [integrant.core :as ig])) -(prefer-method print-method - clojure.lang.IRecord - clojure.lang.IDeref) - -(prefer-method print-method - clojure.lang.IPersistentMap - clojure.lang.IDeref) - -(prefer-method pprint/simple-dispatch - clojure.lang.IPersistentMap - clojure.lang.IDeref) - (defmethod ig/init-key :default [_ data] (d/without-nils data)) @@ -45,15 +32,15 @@ (d/without-nils data) data)) -(def defaults +(def default {:database-uri "postgresql://postgres/penpot" :database-username "penpot" :database-password "penpot" - :default-blob-version 5 + :default-blob-version 4 - :rpc-rlimit-config (fs/path "resources/rlimit.edn") - :rpc-climit-config (fs/path "resources/climit.edn") + :rpc-rlimit-config "resources/rlimit.edn" + :rpc-climit-config "resources/climit.edn" :file-change-snapshot-every 5 :file-change-snapshot-timeout "3h" @@ -92,254 +79,142 @@ ;; time to avoid email sending after profile modification :email-verify-threshold "15m"}) -(s/def ::default-rpc-rlimit ::us/vector-of-strings) -(s/def ::rpc-rlimit-config ::fs/path) -(s/def ::rpc-climit-config ::fs/path) +(def schema:config + (do #_sm/optional-keys + [:map {:title "config"} + [:flags {:optional true} [::sm/set :string]] + [:admins {:optional true} [::sm/set ::sm/email]] + [:secret-key {:optional true} :string] -(s/def ::media-max-file-size ::us/integer) + [:tenant {:optional false} :string] + [:public-uri {:optional false} :string] + [:host {:optional false} :string] -(s/def ::flags ::us/vector-of-keywords) -(s/def ::telemetry-enabled ::us/boolean) + [:http-server-port {:optional true} :int] + [:http-server-host {:optional true} :string] + [:http-server-max-body-size {:optional true} :int] + [:http-server-max-multipart-body-size {:optional true} :int] + [:http-server-io-threads {:optional true} :int] + [:http-server-worker-threads {:optional true} :int] -(s/def ::audit-log-archive-uri ::us/string) -(s/def ::audit-log-http-handler-concurrency ::us/integer) + [:telemetry-uri {:optional true} :string] + [:telemetry-with-taiga {:optional true} :boolean] ;; DELETE -(s/def ::email-domain-blacklist ::fs/path) -(s/def ::email-domain-whitelist ::fs/path) + [:file-change-snapshot-every {:optional true} :int] + [:file-change-snapshot-timeout {:optional true} ::dt/duration] -(s/def ::deletion-delay ::dt/duration) + [:media-max-file-size {:optional true} :int] + [:deletion-delay {:optional true} ::dt/duration] ;; REVIEW + [:telemetry-enabled {:optional true} :boolean] + [:default-blob-version {:optional true} :int] + [:allow-demo-users {:optional true} :boolean] + [:error-report-webhook {:optional true} :string] + [:user-feedback-destination {:optional true} :string] -(s/def ::admins ::us/set-of-valid-emails) -(s/def ::file-change-snapshot-every ::us/integer) -(s/def ::file-change-snapshot-timeout ::dt/duration) + [:default-rpc-rlimit {:optional true} [::sm/vec :string]] + [:rpc-rlimit-config {:optional true} ::fs/path] + [:rpc-climit-config {:optional true} ::fs/path] -(s/def ::default-executor-parallelism ::us/integer) -(s/def ::scheduled-executor-parallelism ::us/integer) + [:audit-log-archive-uri {:optional true} :string] + [:audit-log-http-handler-concurrency {:optional true} :int] -(s/def ::worker-default-parallelism ::us/integer) -(s/def ::worker-webhook-parallelism ::us/integer) + [:default-executor-parallelism {:optional true} :int] ;; REVIEW + [:scheduled-executor-parallelism {:optional true} :int] ;; REVIEW + [:worker-default-parallelism {:optional true} :int] + [:worker-webhook-parallelism {:optional true} :int] -(s/def ::auth-data-cookie-domain ::us/string) -(s/def ::auth-token-cookie-name ::us/string) -(s/def ::auth-token-cookie-max-age ::dt/duration) + [:database-password {:optional true} [:maybe :string]] + [:database-uri {:optional true} :string] + [:database-username {:optional true} [:maybe :string]] + [:database-readonly {:optional true} :boolean] + [:database-min-pool-size {:optional true} :int] + [:database-max-pool-size {:optional true} :int] -(s/def ::secret-key ::us/string) -(s/def ::allow-demo-users ::us/boolean) -(s/def ::assets-path ::us/string) -(s/def ::database-password (s/nilable ::us/string)) -(s/def ::database-uri ::us/string) -(s/def ::database-username (s/nilable ::us/string)) -(s/def ::database-readonly ::us/boolean) -(s/def ::database-min-pool-size ::us/integer) -(s/def ::database-max-pool-size ::us/integer) + [:quotes-teams-per-profile {:optional true} :int] + [:quotes-access-tokens-per-profile {:optional true} :int] + [:quotes-projects-per-team {:optional true} :int] + [:quotes-invitations-per-team {:optional true} :int] + [:quotes-profiles-per-team {:optional true} :int] + [:quotes-files-per-project {:optional true} :int] + [:quotes-files-per-team {:optional true} :int] + [:quotes-font-variants-per-team {:optional true} :int] + [:quotes-comment-threads-per-file {:optional true} :int] + [:quotes-comments-per-file {:optional true} :int] -(s/def ::quotes-teams-per-profile ::us/integer) -(s/def ::quotes-access-tokens-per-profile ::us/integer) -(s/def ::quotes-projects-per-team ::us/integer) -(s/def ::quotes-invitations-per-team ::us/integer) -(s/def ::quotes-profiles-per-team ::us/integer) -(s/def ::quotes-files-per-project ::us/integer) -(s/def ::quotes-files-per-team ::us/integer) -(s/def ::quotes-font-variants-per-team ::us/integer) -(s/def ::quotes-comment-threads-per-file ::us/integer) -(s/def ::quotes-comments-per-file ::us/integer) + [:auth-data-cookie-domain {:optional true} :string] + [:auth-token-cookie-name {:optional true} :string] + [:auth-token-cookie-max-age {:optional true} ::dt/duration] -(s/def ::default-blob-version ::us/integer) -(s/def ::error-report-webhook ::us/string) -(s/def ::user-feedback-destination ::us/string) -(s/def ::github-client-id ::us/string) -(s/def ::github-client-secret ::us/string) -(s/def ::gitlab-base-uri ::us/string) -(s/def ::gitlab-client-id ::us/string) -(s/def ::gitlab-client-secret ::us/string) -(s/def ::google-client-id ::us/string) -(s/def ::google-client-secret ::us/string) -(s/def ::oidc-client-id ::us/string) -(s/def ::oidc-user-info-source ::us/keyword) -(s/def ::oidc-client-secret ::us/string) -(s/def ::oidc-base-uri ::us/string) -(s/def ::oidc-token-uri ::us/string) -(s/def ::oidc-auth-uri ::us/string) -(s/def ::oidc-user-uri ::us/string) -(s/def ::oidc-jwks-uri ::us/string) -(s/def ::oidc-scopes ::us/set-of-strings) -(s/def ::oidc-roles ::us/set-of-strings) -(s/def ::oidc-roles-attr ::us/string) -(s/def ::oidc-email-attr ::us/string) -(s/def ::oidc-name-attr ::us/string) -(s/def ::host ::us/string) -(s/def ::http-server-port ::us/integer) -(s/def ::http-server-host ::us/string) -(s/def ::http-server-max-body-size ::us/integer) -(s/def ::http-server-max-multipart-body-size ::us/integer) -(s/def ::http-server-io-threads ::us/integer) -(s/def ::http-server-worker-threads ::us/integer) -(s/def ::ldap-attrs-email ::us/string) -(s/def ::ldap-attrs-fullname ::us/string) -(s/def ::ldap-attrs-username ::us/string) -(s/def ::ldap-base-dn ::us/string) -(s/def ::ldap-bind-dn ::us/string) -(s/def ::ldap-bind-password ::us/string) -(s/def ::ldap-host ::us/string) -(s/def ::ldap-port ::us/integer) -(s/def ::ldap-ssl ::us/boolean) -(s/def ::ldap-starttls ::us/boolean) -(s/def ::ldap-user-query ::us/string) -(s/def ::media-directory ::us/string) -(s/def ::media-uri ::us/string) -(s/def ::profile-bounce-max-age ::dt/duration) -(s/def ::profile-bounce-threshold ::us/integer) -(s/def ::profile-complaint-max-age ::dt/duration) -(s/def ::profile-complaint-threshold ::us/integer) -(s/def ::public-uri ::us/string) -(s/def ::redis-uri ::us/string) -(s/def ::registration-domain-whitelist ::us/set-of-strings) + [:registration-domain-whitelist {:optional true} [::sm/set :string]] + [:email-verify-threshold {:optional true} ::dt/duration] -(s/def ::smtp-default-from ::us/string) -(s/def ::smtp-default-reply-to ::us/string) -(s/def ::smtp-host ::us/string) -(s/def ::smtp-password (s/nilable ::us/string)) -(s/def ::smtp-port ::us/integer) -(s/def ::smtp-ssl ::us/boolean) -(s/def ::smtp-tls ::us/boolean) -(s/def ::smtp-username (s/nilable ::us/string)) -(s/def ::urepl-host ::us/string) -(s/def ::urepl-port ::us/integer) -(s/def ::prepl-host ::us/string) -(s/def ::prepl-port ::us/integer) -(s/def ::assets-storage-backend ::us/keyword) -(s/def ::storage-assets-fs-directory ::us/string) -(s/def ::storage-assets-s3-bucket ::us/string) -(s/def ::storage-assets-s3-region ::us/keyword) -(s/def ::storage-assets-s3-endpoint ::us/string) -(s/def ::storage-assets-s3-io-threads ::us/integer) -(s/def ::telemetry-uri ::us/string) -(s/def ::telemetry-with-taiga ::us/boolean) -(s/def ::tenant ::us/string) -(s/def ::email-verify-threshold ::dt/duration) + [:github-client-id {:optional true} :string] + [:github-client-secret {:optional true} :string] + [:gitlab-base-uri {:optional true} :string] + [:gitlab-client-id {:optional true} :string] + [:gitlab-client-secret {:optional true} :string] + [:google-client-id {:optional true} :string] + [:google-client-secret {:optional true} :string] + [:oidc-client-id {:optional true} :string] + [:oidc-user-info-source {:optional true} :keyword] + [:oidc-client-secret {:optional true} :string] + [:oidc-base-uri {:optional true} :string] + [:oidc-token-uri {:optional true} :string] + [:oidc-auth-uri {:optional true} :string] + [:oidc-user-uri {:optional true} :string] + [:oidc-jwks-uri {:optional true} :string] + [:oidc-scopes {:optional true} [::sm/set :string]] + [:oidc-roles {:optional true} [::sm/set :string]] + [:oidc-roles-attr {:optional true} :string] + [:oidc-email-attr {:optional true} :string] + [:oidc-name-attr {:optional true} :string] -(s/def ::config - (s/keys :opt-un [::secret-key - ::flags - ::admins - ::deletion-delay - ::allow-demo-users - ::audit-log-archive-uri - ::audit-log-http-handler-concurrency - ::auth-token-cookie-name - ::auth-token-cookie-max-age - ::authenticated-cookie-domain - ::database-password - ::database-uri - ::database-username - ::database-readonly - ::database-min-pool-size - ::database-max-pool-size - ::default-blob-version - ::default-rpc-rlimit - ::email-domain-blacklist - ::email-domain-whitelist - ::error-report-webhook - ::default-executor-parallelism - ::scheduled-executor-parallelism - ::worker-default-parallelism - ::worker-webhook-parallelism - ::file-change-snapshot-every - ::file-change-snapshot-timeout - ::user-feedback-destination - ::github-client-id - ::github-client-secret - ::gitlab-base-uri - ::gitlab-client-id - ::gitlab-client-secret - ::google-client-id - ::google-client-secret - ::oidc-client-id - ::oidc-client-secret - ::oidc-user-info-source - ::oidc-base-uri - ::oidc-token-uri - ::oidc-auth-uri - ::oidc-user-uri - ::oidc-jwks-uri - ::oidc-scopes - ::oidc-roles-attr - ::oidc-email-attr - ::oidc-name-attr - ::oidc-roles - ::host - ::http-server-host - ::http-server-port - ::http-server-max-body-size - ::http-server-max-multipart-body-size - ::http-server-io-threads - ::http-server-worker-threads - ::ldap-attrs-email - ::ldap-attrs-fullname - ::ldap-attrs-username - ::ldap-base-dn - ::ldap-bind-dn - ::ldap-bind-password - ::ldap-host - ::ldap-port - ::ldap-ssl - ::ldap-starttls - ::ldap-user-query - ::local-assets-uri - ::media-max-file-size - ::profile-bounce-max-age - ::profile-bounce-threshold - ::profile-complaint-max-age - ::profile-complaint-threshold - ::public-uri + [:ldap-attrs-email {:optional true} :string] + [:ldap-attrs-fullname {:optional true} :string] + [:ldap-attrs-username {:optional true} :string] + [:ldap-base-dn {:optional true} :string] + [:ldap-bind-dn {:optional true} :string] + [:ldap-bind-password {:optional true} :string] + [:ldap-host {:optional true} :string] + [:ldap-port {:optional true} :int] + [:ldap-ssl {:optional true} :boolean] + [:ldap-starttls {:optional true} :boolean] + [:ldap-user-query {:optional true} :string] - ::quotes-teams-per-profile - ::quotes-access-tokens-per-profile - ::quotes-projects-per-team - ::quotes-invitations-per-team - ::quotes-profiles-per-team - ::quotes-files-per-project - ::quotes-files-per-team - ::quotes-font-variants-per-team - ::quotes-comment-threads-per-file - ::quotes-comments-per-file + [:profile-bounce-max-age {:optional true} ::dt/duration] + [:profile-bounce-threshold {:optional true} :int] + [:profile-complaint-max-age {:optional true} ::dt/duration] + [:profile-complaint-threshold {:optional true} :int] - ::redis-uri - ::registration-domain-whitelist - ::rpc-rlimit-config - ::rpc-climit-config + [:redis-uri {:optional true} :string] - ::semaphore-process-font - ::semaphore-process-image - ::semaphore-update-file - ::semaphore-auth + [:email-domain-blacklist {:optional true} ::fs/path] + [:email-domain-whitelist {:optional true} ::fs/path] - ::smtp-default-from - ::smtp-default-reply-to - ::smtp-host - ::smtp-password - ::smtp-port - ::smtp-ssl - ::smtp-tls - ::smtp-username + [:smtp-default-from {:optional true} :string] + [:smtp-default-reply-to {:optional true} :string] + [:smtp-host {:optional true} :string] + [:smtp-password {:optional true} [:maybe :string]] + [:smtp-port {:optional true} :int] + [:smtp-ssl {:optional true} :boolean] + [:smtp-tls {:optional true} :boolean] + [:smtp-username {:optional true} [:maybe :string]] - ::urepl-host - ::urepl-port - ::prepl-host - ::prepl-port + [:urepl-host {:optional true} :string] + [:urepl-port {:optional true} :int] + [:prepl-host {:optional true} :string] + [:prepl-port {:optional true} :int] - ::assets-storage-backend - ::storage-assets-fs-directory - ::storage-assets-s3-bucket - ::storage-assets-s3-region - ::storage-assets-s3-endpoint - ::storage-assets-s3-io-threads - ::telemetry-enabled - ::telemetry-uri - ::telemetry-referer - ::telemetry-with-taiga - ::tenant - ::email-verify-threshold])) + [:assets-storage-backend {:optional true} :keyword] + [:media-directory {:optional true} :string] ;; REVIEW + [:media-uri {:optional true} :string] + [:assets-path {:optional true} :string] + + [:storage-assets-fs-directory {:optional true} :string] + [:storage-assets-s3-bucket {:optional true} :string] + [:storage-assets-s3-region {:optional true} :keyword] + [:storage-assets-s3-endpoint {:optional true} :string] + [:storage-assets-s3-io-threads {:optional true} :int]])) (def default-flags [:enable-backend-api-doc @@ -367,20 +242,22 @@ {} env))) -(defn- read-config - [] - (try - (->> (read-env "penpot") - (merge defaults) - (us/conform ::config)) - (catch Throwable e - (when (ex/error? e) - (println ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;") - (println "Error on validating configuration:") - (println (some-> e ex-data ex/explain)) - (println (ex/explain (ex-data e))) - (println ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;")) - (throw e)))) +(def decode-config + (sm/decoder schema:config sm/default-transformer)) + +(def validate-config + (sm/validator schema:config)) + +(def explain-config + (sm/explainer schema:config)) + +(defn read-config + "Reads the configuration from enviroment variables and decodes all + known values." + [& {:keys [prefix default] :or {prefix "penpot"}}] + (->> (read-env prefix) + (merge default) + (decode-config))) (def version (v/parse (or (some-> (io/resource "version.txt") @@ -388,10 +265,28 @@ (str/trim)) "%version%"))) -(defonce ^:dynamic config (read-config)) +(defonce ^:dynamic config (read-config :default default)) (defonce ^:dynamic flags (parse-flags config)) -(def deletion-delay +(defn validate! + "Validate the currently loaded configuration data." + [& {:keys [exit-on-error?] :or {exit-on-error? true}}] + (if (validate-config config) + true + (let [explain (explain-config config)] + (println "Error on validating configuration:") + (sm/pretty-explain explain + :variant ::sm/schemaless-explain + :message "Configuration Validation Error") + (flush) + (if exit-on-error? + (System/exit -1) + (ex/raise :type :validation + :code :config-validaton + ::sm/explain explain))))) + +(defn get-deletion-delay + [] (or (c/get config :deletion-delay) (dt/duration {:days 7}))) diff --git a/backend/src/app/email.clj b/backend/src/app/email.clj index 0f7e356b6..8339e889b 100644 --- a/backend/src/app/email.clj +++ b/backend/src/app/email.clj @@ -7,9 +7,11 @@ (ns app.email "Main api for send emails." (:require + [app.common.data.macros :as dm] [app.common.exceptions :as ex] [app.common.logging :as l] [app.common.pprint :as pp] + [app.common.schema :as sm] [app.common.spec :as us] [app.config :as cf] [app.db :as db] @@ -149,9 +151,27 @@ "mail.smtp.timeout" timeout "mail.smtp.connectiontimeout" timeout})) +(def ^:private schema:smtp-config + [:map + [::username {:optional true} :string] + [::password {:optional true} :string] + [::tls {:optional true} :boolean] + [::ssl {:optional true} :boolean] + [::host {:optional true} :string] + [::port {:optional true} :int] + [::default-from {:optional true} :string] + [::default-reply-to {:optional true} :string]]) + +(def valid-smtp-config? + (sm/check-fn schema:smtp-config)) + (defn- create-smtp-session ^Session [cfg] + (dm/assert! + "expected valid smtp config" + (valid-smtp-config? cfg)) + (let [props (opts->props cfg)] (Session/getInstance props))) @@ -273,32 +293,10 @@ ;; SENDMAIL FN / TASK HANDLER ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(s/def ::username ::cf/smtp-username) -(s/def ::password ::cf/smtp-password) -(s/def ::tls ::cf/smtp-tls) -(s/def ::ssl ::cf/smtp-ssl) -(s/def ::host ::cf/smtp-host) -(s/def ::port ::cf/smtp-port) -(s/def ::default-reply-to ::cf/smtp-default-reply-to) -(s/def ::default-from ::cf/smtp-default-from) - -(s/def ::smtp-config - (s/keys :opt [::username - ::password - ::tls - ::ssl - ::host - ::port - ::default-from - ::default-reply-to])) - (declare send-to-logger!) (s/def ::sendmail fn?) -(defmethod ig/pre-init-spec ::sendmail [_] - (s/spec ::smtp-config)) - (defmethod ig/init-key ::sendmail [_ cfg] (fn [params] diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index f9797521d..171efdc17 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -524,6 +524,7 @@ (defn start [] + (cf/validate!) (ig/load-namespaces (merge system-config worker-config)) (alter-var-root #'system (fn [sys] (when sys (ig/halt! sys)) diff --git a/backend/src/app/media.clj b/backend/src/app/media.clj index 018295546..9e1a120fe 100644 --- a/backend/src/app/media.clj +++ b/backend/src/app/media.clj @@ -11,7 +11,6 @@ [app.common.exceptions :as ex] [app.common.media :as cm] [app.common.schema :as sm] - [app.common.schema.generators :as sg] [app.common.schema.openapi :as-alias oapi] [app.common.spec :as us] [app.common.svg :as csvg] @@ -47,18 +46,6 @@ (s/keys :req-un [::path] :opt-un [::mtype])) -(sm/register! ::fs/path - {:type ::fs/path - :pred fs/path? - :type-properties - {:title "path" - :description "filesystem path" - :error/message "expected a valid fs path instance" - :gen/gen (sg/generator :string) - ::oapi/type "string" - ::oapi/format "unix-path" - ::oapi/decode fs/path}}) - (sm/register! ::upload [:map {:title "Upload"} [:filename :string] diff --git a/backend/src/app/rpc/commands/demo.clj b/backend/src/app/rpc/commands/demo.clj index 213f6abec..059548f22 100644 --- a/backend/src/app/rpc/commands/demo.clj +++ b/backend/src/app/rpc/commands/demo.clj @@ -45,7 +45,7 @@ params {:email email :fullname fullname :is-active true - :deleted-at (dt/in-future cf/deletion-delay) + :deleted-at (dt/in-future (cf/get-deletion-delay)) :password (profile/derive-password cfg password) :props {}}] diff --git a/backend/src/app/tasks/file_gc.clj b/backend/src/app/tasks/file_gc.clj index 8a2db9c57..5430b0c98 100644 --- a/backend/src/app/tasks/file_gc.clj +++ b/backend/src/app/tasks/file_gc.clj @@ -295,7 +295,7 @@ (defmethod ig/prep-key ::handler [_ cfg] - (assoc cfg ::min-age cf/deletion-delay)) + (assoc cfg ::min-age (cf/get-deletion-delay))) (defmethod ig/init-key ::handler [_ cfg] diff --git a/backend/src/app/tasks/objects_gc.clj b/backend/src/app/tasks/objects_gc.clj index 0cfd66a61..9858585cc 100644 --- a/backend/src/app/tasks/objects_gc.clj +++ b/backend/src/app/tasks/objects_gc.clj @@ -292,7 +292,7 @@ (defmethod ig/prep-key ::handler [_ cfg] (assoc cfg - ::min-age cf/deletion-delay + ::min-age (cf/get-deletion-delay) ::chunk-size 10)) (defmethod ig/init-key ::handler diff --git a/backend/src/app/tasks/tasks_gc.clj b/backend/src/app/tasks/tasks_gc.clj index cd6982799..0e93ea0d0 100644 --- a/backend/src/app/tasks/tasks_gc.clj +++ b/backend/src/app/tasks/tasks_gc.clj @@ -23,7 +23,7 @@ (defmethod ig/prep-key ::handler [_ cfg] - (assoc cfg ::min-age cf/deletion-delay)) + (assoc cfg ::min-age (cf/get-deletion-delay))) (defmethod ig/init-key ::handler [_ {:keys [::db/pool ::min-age] :as cfg}] diff --git a/backend/src/app/util/overrides.clj b/backend/src/app/util/overrides.clj new file mode 100644 index 000000000..8f8842718 --- /dev/null +++ b/backend/src/app/util/overrides.clj @@ -0,0 +1,41 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns app.util.overrides + "A utility ns for declare default overrides over clojure runtime" + (:require + [app.common.schema :as sm] + [app.common.schema.generators :as sg] + [app.common.schema.openapi :as-alias oapi] + [clojure.pprint :as pprint] + [datoteka.fs :as fs])) + + +(prefer-method print-method + clojure.lang.IRecord + clojure.lang.IDeref) + +(prefer-method print-method + clojure.lang.IPersistentMap + clojure.lang.IDeref) + +(prefer-method pprint/simple-dispatch + clojure.lang.IPersistentMap + clojure.lang.IDeref) + + +(sm/register! ::fs/path + {:type ::fs/path + :pred fs/path? + :type-properties + {:title "path" + :description "filesystem path" + :error/message "expected a valid fs path instance" + :error/code "errors.invalid-path" + :gen/gen (sg/generator :string) + ::oapi/type "string" + ::oapi/format "unix-path" + ::oapi/decode fs/path}}) diff --git a/backend/test/backend_tests/helpers.clj b/backend/test/backend_tests/helpers.clj index 5dc4f24d7..3b862d6dd 100644 --- a/backend/test/backend_tests/helpers.clj +++ b/backend/test/backend_tests/helpers.clj @@ -58,15 +58,14 @@ (def ^:dynamic *system* nil) (def ^:dynamic *pool* nil) -(def defaults +(def default {:database-uri "postgresql://postgres/penpot_test" :redis-uri "redis://redis/1" :file-change-snapshot-every 1}) (def config - (->> (cf/read-env "penpot-test") - (merge cf/defaults defaults) - (us/conform ::cf/config))) + (cf/read-config :prefix "penpot-test" + :default (merge cf/default default))) (def default-flags [:enable-secure-session-cookies @@ -88,6 +87,8 @@ app.auth/verify-password (fn [a b] {:valid (= a b)}) app.common.features/get-enabled-features (fn [& _] app.common.features/supported-features)] + (cf/validate! :exit-on-error? false) + (fs/create-dir "/tmp/penpot") (let [templates [{:id "test" @@ -524,7 +525,6 @@ ([key default] (get data key (get cf/config key default))))) - (defn reset-mock! [m] (swap! m (fn [m] diff --git a/backend/test/backend_tests/rpc_file_test.clj b/backend/test/backend_tests/rpc_file_test.clj index 35d76231f..d28213879 100644 --- a/backend/test/backend_tests/rpc_file_test.clj +++ b/backend/test/backend_tests/rpc_file_test.clj @@ -1127,9 +1127,9 @@ (t/is (= 1 (:processed res)))) ;; check that object thumbnails are still here - (let [res (th/db-exec! ["select * from file_tagged_object_thumbnail"])] - ;; (th/print-result! res) - (t/is (= 1 (count res)))) + (let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})] + ;; (app.common.pprint/pprint rows) + (t/is (= 1 (count rows)))) ;; insert object snapshot for for unknown frame (let [data {::th/type :create-file-object-thumbnail @@ -1148,13 +1148,20 @@ (th/db-exec! ["update file set has_media_trimmed=false where id=?" (:id file)]) ;; check that we have all object thumbnails - (let [res (th/db-exec! ["select * from file_tagged_object_thumbnail"])] - (t/is (= 2 (count res)))) + (let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})] + ;; (app.common.pprint/pprint rows) + (t/is (= 2 (count rows)))) ;; run the task again (let [res (th/run-task! :file-gc {:min-age 0})] (t/is (= 1 (:processed res)))) + ;; check that we have all object thumbnails + (let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})] + ;; (app.common.pprint/pprint rows) + (t/is (= 2 (count rows)))) + + ;; check that the unknown frame thumbnail is deleted (let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})] (t/is (= 2 (count rows))) @@ -1164,6 +1171,7 @@ (t/is (= 3 (:processed res)))) (let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})] + ;; (app.common.pprint/pprint rows) (t/is (= 1 (count rows))))))) (t/deftest file-thumbnail-ops @@ -1220,7 +1228,3 @@ (let [rows (th/db-query :file-thumbnail {:file-id (:id file)})] (t/is (= 1 (count rows))))))) - - - - diff --git a/common/src/app/common/schema.cljc b/common/src/app/common/schema.cljc index fc41156ca..570cfa062 100644 --- a/common/src/app/common/schema.cljc +++ b/common/src/app/common/schema.cljc @@ -219,12 +219,10 @@ :length (d/nilv length 12)}))))) (defmethod v/-format ::schemaless-explain - [_ {:keys [schema] :as explanation} printer] + [_ explanation printer] {:body [:group (v/-block "Value" (v/-visit (me/error-value explanation printer) printer) printer) :break :break - (v/-block "Errors" (v/-visit (me/humanize (me/with-spell-checking explanation)) printer) printer) :break :break - (v/-block "Schema" (v/-visit schema printer) printer)]}) - + (v/-block "Errors" (v/-visit (me/humanize (me/with-spell-checking explanation)) printer) printer)]}) (defmethod v/-format ::explain [_ {:keys [schema] :as explanation} printer] @@ -233,7 +231,6 @@ (v/-block "Errors" (v/-visit (me/humanize (me/with-spell-checking explanation)) printer) printer) :break :break (v/-block "Schema" (v/-visit schema printer) printer)]}) - (defn pretty-explain [explain & {:keys [variant message] :or {variant ::explain