diff --git a/backend/scripts/repl b/backend/scripts/repl index cee9d8c5b..2467d3f11 100755 --- a/backend/scripts/repl +++ b/backend/scripts/repl @@ -1,6 +1,6 @@ #!/usr/bin/env bash -export PENPOT_ASSERTS_ENABLED=true +export PENPOT_FLAGS="enable-asserts $PENPOT_FLAGS" export OPTIONS="-A:jmx-remote:dev -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -J-Dlog4j2.configurationFile=log4j2-devenv.xml -J-Djdk.attach.allowAttachSelf -J-XX:+UseZGC -J-XX:ConcGCThreads=1 -J-XX:-OmitStackTraceInFastThrow -J-Xms50m -J-Xmx512m"; # export OPTIONS="$OPTIONS -J-XX:+UnlockDiagnosticVMOptions"; diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index 27df0e778..e43ca8aad 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -10,6 +10,7 @@ (:require [app.common.data :as d] [app.common.exceptions :as ex] + [app.common.flags :as flags] [app.common.spec :as us] [app.common.version :as v] [app.util.time :as dt] @@ -50,8 +51,6 @@ :default-blob-version 3 :loggers-zmq-uri "tcp://localhost:45556" - :asserts-enabled false - :public-uri "http://localhost:3449" :redis-uri "redis://redis/0" @@ -61,15 +60,11 @@ :assets-storage-backend :assets-fs :storage-assets-fs-directory "assets" - :feedback-destination "info@example.com" - :feedback-enabled false - :assets-path "/internal/assets/" :rlimits-password 10 :rlimits-image 2 - :smtp-enabled false :smtp-default-reply-to "Penpot " :smtp-default-from "Penpot " @@ -79,10 +74,6 @@ :profile-bounce-max-age (dt/duration {:days 7}) :profile-bounce-threshold 10 - :allow-demo-users true - :registration-enabled true - - :telemetry-enabled false :telemetry-uri "https://telemetry.penpot.app/" :ldap-user-query "(|(uid=:username)(mail=:username))" @@ -95,24 +86,27 @@ :initial-project-skey "initial-project" }) -(s/def ::audit-enabled ::us/boolean) -(s/def ::audit-archive-enabled ::us/boolean) -(s/def ::audit-archive-uri ::us/string) -(s/def ::audit-archive-gc-enabled ::us/boolean) -(s/def ::audit-archive-gc-max-age ::dt/duration) +(s/def ::flags ::us/words) + +;; DEPRECATED PROPERTIES: should be removed in 1.10 +(s/def ::registration-enabled ::us/boolean) +(s/def ::smtp-enabled ::us/boolean) +(s/def ::telemetry-enabled ::us/boolean) +(s/def ::asserts-enabled ::us/boolean) +;; END DEPRECATED + +(s/def ::audit-log-archive-uri ::us/string) +(s/def ::audit-log-gc-max-age ::dt/duration) (s/def ::secret-key ::us/string) (s/def ::allow-demo-users ::us/boolean) -(s/def ::asserts-enabled ::us/boolean) (s/def ::assets-path ::us/string) (s/def ::database-password (s/nilable ::us/string)) (s/def ::database-uri ::us/string) (s/def ::database-username (s/nilable ::us/string)) (s/def ::default-blob-version ::us/integer) (s/def ::error-report-webhook ::us/string) -(s/def ::feedback-destination ::us/string) -(s/def ::feedback-enabled ::us/boolean) -(s/def ::feedback-token ::us/string) +(s/def ::user-feedback-destination ::us/string) (s/def ::github-client-id ::us/string) (s/def ::github-client-secret ::us/string) (s/def ::gitlab-base-uri ::us/string) @@ -158,12 +152,10 @@ (s/def ::public-uri ::us/string) (s/def ::redis-uri ::us/string) (s/def ::registration-domain-whitelist ::us/set-of-str) -(s/def ::registration-enabled ::us/boolean) (s/def ::rlimits-image ::us/integer) (s/def ::rlimits-password ::us/integer) (s/def ::smtp-default-from ::us/string) (s/def ::smtp-default-reply-to ::us/string) -(s/def ::smtp-enabled ::us/boolean) (s/def ::smtp-host ::us/string) (s/def ::smtp-password (s/nilable ::us/string)) (s/def ::smtp-port ::us/integer) @@ -180,28 +172,22 @@ (s/def ::storage-fdata-s3-bucket ::us/string) (s/def ::storage-fdata-s3-region ::us/keyword) (s/def ::storage-fdata-s3-prefix ::us/string) -(s/def ::telemetry-enabled ::us/boolean) (s/def ::telemetry-uri ::us/string) (s/def ::telemetry-with-taiga ::us/boolean) (s/def ::tenant ::us/string) (s/def ::config (s/keys :opt-un [::secret-key + ::flags ::allow-demo-users - ::audit-enabled - ::audit-archive-enabled - ::audit-archive-uri - ::audit-archive-gc-enabled - ::audit-archive-gc-max-age - ::asserts-enabled + ::audit-log-archive-uri + ::audit-log-gc-max-age ::database-password ::database-uri ::database-username ::default-blob-version ::error-report-webhook - ::feedback-destination - ::feedback-enabled - ::feedback-token + ::user-feedback-destination ::github-client-id ::github-client-secret ::gitlab-base-uri @@ -258,26 +244,26 @@ ::smtp-ssl ::smtp-tls ::smtp-username - ::srepl-host ::srepl-port - ::assets-storage-backend ::storage-assets-fs-directory ::storage-assets-s3-bucket ::storage-assets-s3-region - ::fdata-storage-backend ::storage-fdata-s3-bucket ::storage-fdata-s3-region ::storage-fdata-s3-prefix - ::telemetry-enabled ::telemetry-uri ::telemetry-referer ::telemetry-with-taiga ::tenant])) +(defn- parse-flags + [{:keys [flags]}] + (flags/parse flags flags/default)) + (defn read-env [prefix] (let [prefix (str prefix "-") @@ -304,11 +290,14 @@ (println ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;"))) (throw e)))) -(def version (v/parse (or (some-> (io/resource "version.txt") - (slurp) - (str/trim)) - "%version%"))) -(def config (atom (read-config))) +(def version + (v/parse (or (some-> (io/resource "version.txt") + (slurp) + (str/trim)) + "%version%"))) + +(def ^:dynamic config (read-config)) +(def ^:dynamic flags (parse-flags config)) (def deletion-delay (dt/duration {:days 7})) @@ -316,9 +305,9 @@ (defn get "A configuration getter. Helps code be more testable." ([key] - (c/get @config key)) + (c/get config key)) ([key default] - (c/get @config key default))) + (c/get config key default))) ;; Set value for all new threads bindings. -(alter-var-root #'*assert* (constantly (get :asserts-enabled))) +(alter-var-root #'*assert* (constantly (contains? flags :backend-asserts))) diff --git a/backend/src/app/emails.clj b/backend/src/app/emails.clj index ac55b688b..0487feaed 100644 --- a/backend/src/app/emails.clj +++ b/backend/src/app/emails.clj @@ -8,7 +8,7 @@ "Main api for send emails." (:require [app.common.spec :as us] - [app.config :as cfg] + [app.config :as cf] [app.db :as db] [app.db.sql :as sql] [app.util.emails :as emails] @@ -54,10 +54,10 @@ (defn allow-send-emails? [conn profile] (when-not (:is-muted profile false) - (let [complaint-threshold (cfg/get :profile-complaint-threshold) - complaint-max-age (cfg/get :profile-complaint-max-age) - bounce-threshold (cfg/get :profile-bounce-threshold) - bounce-max-age (cfg/get :profile-bounce-max-age) + (let [complaint-threshold (cf/get :profile-complaint-threshold) + complaint-max-age (cf/get :profile-complaint-max-age) + bounce-threshold (cf/get :profile-bounce-threshold) + bounce-max-age (cf/get :profile-bounce-max-age) {:keys [complaints bounces] :as result} (db/exec-one! conn [sql:profile-complaint-report @@ -140,19 +140,17 @@ (declare send-console!) -(s/def ::username ::cfg/smtp-username) -(s/def ::password ::cfg/smtp-password) -(s/def ::tls ::cfg/smtp-tls) -(s/def ::ssl ::cfg/smtp-ssl) -(s/def ::host ::cfg/smtp-host) -(s/def ::port ::cfg/smtp-port) -(s/def ::default-reply-to ::cfg/smtp-default-reply-to) -(s/def ::default-from ::cfg/smtp-default-from) -(s/def ::enabled ::cfg/smtp-enabled) +(s/def ::username ::cf/smtp-username) +(s/def ::password ::cf/smtp-password) +(s/def ::tls ::cf/smtp-tls) +(s/def ::ssl ::cf/smtp-ssl) +(s/def ::host ::cf/smtp-host) +(s/def ::port ::cf/smtp-port) +(s/def ::default-reply-to ::cf/smtp-default-reply-to) +(s/def ::default-from ::cf/smtp-default-from) (defmethod ig/pre-init-spec ::sendmail-handler [_] - (s/keys :req-un [::enabled] - :opt-un [::username + (s/keys :opt-un [::username ::password ::tls ::ssl @@ -164,9 +162,12 @@ (defmethod ig/init-key ::sendmail-handler [_ cfg] (fn [{:keys [props] :as task}] - (if (:enabled cfg) - (emails/send! cfg props) - (send-console! cfg props)))) + (let [enabled? (or (contains? cf/flags :smtp) + (cf/get :smtp-enabled) + (:enabled task))] + (if enabled? + (emails/send! cfg props) + (send-console! cfg props))))) (defn- send-console! [cfg email] diff --git a/backend/src/app/http/feedback.clj b/backend/src/app/http/feedback.clj index dd78cfd57..e82a93a4e 100644 --- a/backend/src/app/http/feedback.clj +++ b/backend/src/app/http/feedback.clj @@ -10,7 +10,7 @@ [app.common.data :as d] [app.common.exceptions :as ex] [app.common.spec :as us] - [app.config :as cfg] + [app.config :as cf] [app.db :as db] [app.emails :as eml] [app.rpc.queries.profile :as profile] @@ -24,8 +24,8 @@ (defmethod ig/init-key ::handler [_ {:keys [pool] :as scfg}] - (let [ftoken (cfg/get :feedback-token ::no-token) - enabled (cfg/get :feedback-enabled)] + (let [ftoken (cf/get :feedback-token ::no-token) + enabled (contains? cf/flags :user-feedback)] (fn [{:keys [profile-id] :as request}] (let [token (get-in request [:headers "x-feedback-token"]) params (d/merge (:params request) @@ -58,7 +58,7 @@ (defn send-feedback [pool profile params] (let [params (us/conform ::feedback params) - destination (cfg/get :feedback-destination)] + destination (cf/get :feedback-destination)] (eml/send! {::eml/conn pool ::eml/factory eml/feedback :to destination diff --git a/backend/src/app/loggers/audit.clj b/backend/src/app/loggers/audit.clj index 854884634..34b06ec81 100644 --- a/backend/src/app/loggers/audit.clj +++ b/backend/src/app/loggers/audit.clj @@ -88,9 +88,9 @@ (s/def ::events (s/every ::event)) (defmethod ig/init-key ::http-handler - [_ {:keys [executor enabled] :as cfg}] - (fn [{:keys [params _headers _cookies profile-id] :as request}] - (when enabled + [_ {:keys [executor] :as cfg}] + (fn [{:keys [params profile-id] :as request}] + (when (contains? cf/flags :audit-log) (let [events (->> (:events params) (remove #(not= profile-id (:profile-id %))) (us/conform ::events)) @@ -136,10 +136,9 @@ ;; an external storage and data cleared. (declare persist-events) -(s/def ::enabled ::us/boolean) (defmethod ig/pre-init-spec ::collector [_] - (s/keys :req-un [::db/pool ::wrk/executor ::enabled])) + (s/keys :req-un [::db/pool ::wrk/executor])) (def event-xform (comp @@ -147,9 +146,9 @@ (map clean-props))) (defmethod ig/init-key ::collector - [_ {:keys [enabled] :as cfg}] - (when enabled - (l/info :msg "intializing audit collector") + [_ cfg] + (when (contains? cf/flags :audit-log) + (l/info :msg "intializing audit log collector") (let [input (a/chan 512 event-xform) buffer (aa/batch input {:max-batch-size 100 :max-batch-age (* 10 1000) ; 10s @@ -202,15 +201,16 @@ (s/def ::tokens fn?) (defmethod ig/pre-init-spec ::archive-task [_] - (s/keys :req-un [::db/pool ::tokens ::enabled] + (s/keys :req-un [::db/pool ::tokens] :opt-un [::uri])) (defmethod ig/init-key ::archive-task - [_ {:keys [uri enabled] :as cfg}] + [_ {:keys [uri] :as cfg}] (fn [props] ;; NOTE: this let allows overwrite default configured values from ;; the repl, when manually invoking the task. - (let [enabled (or enabled (:enabled props false)) + (let [enabled (or (contains? cf/flags :audit-log-archive) + (:enabled props false)) uri (or uri (:uri props)) cfg (assoc cfg :uri uri)] (when (and enabled (not uri)) @@ -298,18 +298,6 @@ ;; GC Task ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(declare clean-archived) - -(s/def ::max-age ::cf/audit-archive-gc-max-age) - -(defmethod ig/pre-init-spec ::archive-gc-task [_] - (s/keys :req-un [::db/pool ::enabled ::max-age])) - -(defmethod ig/init-key ::archive-gc-task - [_ cfg] - (fn [_] - (clean-archived cfg))) - (def sql:clean-archived "delete from audit_log where archived_at is not null @@ -322,3 +310,13 @@ result (:next.jdbc/update-count result)] (l/debug :action "clean archived audit log" :removed result) result)) + +(s/def ::max-age ::cf/audit-log-gc-max-age) + +(defmethod ig/pre-init-spec ::gc-task [_] + (s/keys :req-un [::db/pool ::max-age])) + +(defmethod ig/init-key ::gc-task + [_ cfg] + (fn [_] + (clean-archived cfg))) diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index bfa65c4d4..bd178a1ea 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -210,15 +210,16 @@ {:cron #app/cron "0 0 * * * ?" ;; hourly :task :file-offload}) - (when (cf/get :audit-archive-enabled) + (when (contains? cf/flags :audit-log-archive) {:cron #app/cron "0 0 * * * ?" ;; every 1h - :task :audit-archive}) + :task :audit-log-archive}) - (when (cf/get :audit-archive-gc-enabled) + (when (contains? cf/flags :audit-log-gc) {:cron #app/cron "0 0 * * * ?" ;; every 1h - :task :audit-archive-gc}) + :task :audit-log-gc}) - (when (cf/get :telemetry-enabled) + (when (or (contains? cf/flags :telemetry) + (cf/get :telemetry-enabled)) {:cron #app/cron "0 0 */6 * * ?" ;; every 6h :task :telemetry})]} @@ -238,15 +239,14 @@ :telemetry (ig/ref :app.tasks.telemetry/handler) :session-gc (ig/ref :app.http.session/gc-task) :file-offload (ig/ref :app.tasks.file-offload/handler) - :audit-archive (ig/ref :app.loggers.audit/archive-task) - :audit-archive-gc (ig/ref :app.loggers.audit/archive-gc-task)}} + :audit-log-archive (ig/ref :app.loggers.audit/archive-task) + :audit-log-gc (ig/ref :app.loggers.audit/gc-task)}} :app.emails/sendmail-handler {:host (cf/get :smtp-host) :port (cf/get :smtp-port) :ssl (cf/get :smtp-ssl) :tls (cf/get :smtp-tls) - :enabled (cf/get :smtp-enabled) :username (cf/get :smtp-username) :password (cf/get :smtp-password) :metrics (ig/ref :app.metrics/metrics) @@ -304,24 +304,20 @@ {:endpoint (cf/get :loggers-zmq-uri)} :app.loggers.audit/http-handler - {:enabled (cf/get :audit-enabled false) - :pool (ig/ref :app.db/pool) + {:pool (ig/ref :app.db/pool) :executor (ig/ref :app.worker/executor)} :app.loggers.audit/collector - {:enabled (cf/get :audit-enabled false) - :pool (ig/ref :app.db/pool) + {:pool (ig/ref :app.db/pool) :executor (ig/ref :app.worker/executor)} :app.loggers.audit/archive-task - {:uri (cf/get :audit-archive-uri) - :enabled (cf/get :audit-archive-enabled false) + {:uri (cf/get :audit-log-archive-uri) :tokens (ig/ref :app.tokens/tokens) :pool (ig/ref :app.db/pool)} - :app.loggers.audit/archive-gc-task - {:enabled (cf/get :audit-archive-gc-enabled false) - :max-age (cf/get :audit-archive-gc-max-age cf/deletion-delay) + :app.loggers.audit/gc-task + {:max-age (cf/get :audit-log-gc-max-age cf/deletion-delay) :pool (ig/ref :app.db/pool)} :app.loggers.loki/reporter diff --git a/backend/src/app/rpc/mutations/demo.clj b/backend/src/app/rpc/mutations/demo.clj index fc7f184bf..cca26dbb4 100644 --- a/backend/src/app/rpc/mutations/demo.clj +++ b/backend/src/app/rpc/mutations/demo.clj @@ -9,7 +9,7 @@ (:require [app.common.exceptions :as ex] [app.common.uuid :as uuid] - [app.config :as cfg] + [app.config :as cf] [app.db :as db] [app.loggers.audit :as audit] [app.rpc.mutations.profile :as profile] @@ -35,11 +35,11 @@ :email email :fullname fullname :is-demo true - :deleted-at (dt/in-future cfg/deletion-delay) + :deleted-at (dt/in-future cf/deletion-delay) :password password :props {:onboarding-viewed true}}] - (when-not (cfg/get :allow-demo-users) + (when-not (contains? cf/flags :demo-users) (ex/raise :type :validation :code :demo-users-not-allowed :hint "Demo users are disabled by config.")) diff --git a/backend/src/app/rpc/mutations/profile.clj b/backend/src/app/rpc/mutations/profile.clj index 099092ba5..bbef6f54e 100644 --- a/backend/src/app/rpc/mutations/profile.clj +++ b/backend/src/app/rpc/mutations/profile.clj @@ -100,10 +100,9 @@ (sv/defmethod ::prepare-register-profile {:auth false} [{:keys [pool tokens] :as cfg} params] - (when-not (cf/get :registration-enabled) + (when-not (contains? cf/flags :registration) (ex/raise :type :restriction :code :registration-disabled)) - (when-let [domains (cf/get :registration-domain-whitelist)] (when-not (email-domain-in-whitelist? domains (:email params)) (ex/raise :type :validation @@ -458,7 +457,8 @@ params (assoc params :profile profile :email (str/lower email))] - (if (cf/get :smtp-enabled) + (if (or (cf/get :smtp-enabled) + (contains? cf/flags :smtp)) (request-email-change cfg params) (change-email-inmediatelly cfg params))))) diff --git a/backend/test/app/services_profile_test.clj b/backend/test/app/services_profile_test.clj index d5e6dac43..ab4d309c2 100644 --- a/backend/test/app/services_profile_test.clj +++ b/backend/test/app/services_profile_test.clj @@ -211,10 +211,7 @@ )) (t/deftest prepare-register-with-registration-disabled - (with-mocks [mock {:target 'app.config/get - :return (th/mock-config-get-with - {:registration-enabled false})}] - + (th/with-mocks {#'app.config/flags nil} (let [data {::th/type :prepare-register-profile :email "user@example.com" :password "foobar"}] diff --git a/backend/test/app/test_helpers.clj b/backend/test/app/test_helpers.clj index ba8999123..43334b883 100644 --- a/backend/test/app/test_helpers.clj +++ b/backend/test/app/test_helpers.clj @@ -7,6 +7,7 @@ (ns app.test-helpers (:require [app.common.data :as d] + [app.common.flags :as flags] [app.common.pages :as cp] [app.common.spec :as us] [app.common.uuid :as uuid] @@ -336,9 +337,15 @@ [data] (fn ([key] - (get data key (get @cf/config key))) + (get data key (get cf/config key))) ([key default] - (get data key (get @cf/config key default))))) + (get data key (get cf/config key default))))) + + +(defmacro with-mocks + [rebinds & body] + `(with-redefs-fn ~rebinds + (fn [] ~@body))) (defn reset-mock! [m] diff --git a/common/src/app/common/flags.cljc b/common/src/app/common/flags.cljc index a5a051f7c..c3f52bac2 100644 --- a/common/src/app/common/flags.cljc +++ b/common/src/app/common/flags.cljc @@ -9,24 +9,30 @@ (:require [cuerdas.core :as str])) +(def default + #{:backend-asserts + :registration + :demo-users}) + (defn parse - [default flags] - (loop [flags (seq flags) - result default] - (let [item (first flags)] - (if (nil? item) - result - (let [sname (name item)] - (cond - (str/starts-with? sname "enable-") - (recur (rest flags) - (conj result (keyword (subs sname 7)))) + ([flags] (parse flags #{})) + ([flags default] + (loop [flags (seq flags) + result default] + (let [item (first flags)] + (if (nil? item) + result + (let [sname (name item)] + (cond + (str/starts-with? sname "enable-") + (recur (rest flags) + (conj result (keyword (subs sname 7)))) - (str/starts-with? sname "disable-") - (recur (rest flags) - (disj result (keyword (subs sname 8)))) + (str/starts-with? sname "disable-") + (recur (rest flags) + (disj result (keyword (subs sname 8)))) - :else - (recur (rest flags) result))))))) + :else + (recur (rest flags) result)))))))) diff --git a/common/src/app/common/spec.cljc b/common/src/app/common/spec.cljc index ca369fb51..b4823ce9e 100644 --- a/common/src/app/common/spec.cljc +++ b/common/src/app/common/spec.cljc @@ -111,6 +111,16 @@ (s/def ::point gpt/point?) (s/def ::id ::uuid) +(s/def ::words + (s/conformer + (fn [s] + (cond + (set? s) s + (string? s) (into #{} (map keyword) (str/words s)) + :else ::s/invalid)) + (fn [s] + (str/join " " (map name s))))) + (defn bytes? "Test if a first parameter is a byte array or not." diff --git a/frontend/src/app/config.cljs b/frontend/src/app/config.cljs index 8e43d15aa..7e72d8d39 100644 --- a/frontend/src/app/config.cljs +++ b/frontend/src/app/config.cljs @@ -54,22 +54,11 @@ :browser :webworker)) -(def available-flags - #{:registration - :audit-log - :demo-users - :user-feedback - :demo-warning - :login-with-ldap}) - -(def default-flags - #{:registration :demo-users}) - (defn- parse-flags [global] (let [flags (obj/get global "penpotFlags" "") flags (into #{} (map keyword) (str/words flags))] - (flags/parse default-flags flags))) + (flags/parse flags flags/default))) (defn- parse-version [global]