Penpot/penpot
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-03-20 19:51:23 -05:00
Code Issues Activity

♻️ Unify flags parsing on backend.

Browse source
This commit is contained in:
Andrey Antukh 2021-09-16 14:40:50 +02:00 • committed by Andrés Moya
parent da1135c80f
commit 8a0bba3c7a
13 changed files with 140 additions and 147 deletions

2
backend/scripts/repl
View file

@ -1,6 +1,6 @@
#!/usr/bin/env bash
export PENPOT_ASSERTS_ENABLED=true
export PENPOT_FLAGS="enable-asserts $PENPOT_FLAGS"
export OPTIONS="-A:jmx-remote:dev -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -J-Dlog4j2.configurationFile=log4j2-devenv.xml -J-Djdk.attach.allowAttachSelf -J-XX:+UseZGC -J-XX:ConcGCThreads=1 -J-XX:-OmitStackTraceInFastThrow -J-Xms50m -J-Xmx512m";
# export OPTIONS="$OPTIONS -J-XX:+UnlockDiagnosticVMOptions";

75
backend/src/app/config.clj
View file

@ -10,6 +10,7 @@
(:require
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.flags :as flags]
[app.common.spec :as us]
[app.common.version :as v]
[app.util.time :as dt]
@ -50,8 +51,6 @@
:default-blob-version 3
:loggers-zmq-uri "tcp://localhost:45556"
:asserts-enabled false
:public-uri "http://localhost:3449"
:redis-uri "redis://redis/0"
@ -61,15 +60,11 @@
:assets-storage-backend :assets-fs
:storage-assets-fs-directory "assets"
:feedback-destination "info@example.com"
:feedback-enabled false
:assets-path "/internal/assets/"
:rlimits-password 10
:rlimits-image 2
:smtp-enabled false
:smtp-default-reply-to "Penpot <no-reply@example.com>"
:smtp-default-from "Penpot <no-reply@example.com>"
@ -79,10 +74,6 @@
:profile-bounce-max-age (dt/duration {:days 7})
:profile-bounce-threshold 10
:allow-demo-users true
:registration-enabled true
:telemetry-enabled false
:telemetry-uri "https://telemetry.penpot.app/"
:ldap-user-query "(|(uid=:username)(mail=:username))"
@ -95,24 +86,27 @@
:initial-project-skey "initial-project"
})
(s/def ::audit-enabled ::us/boolean)
(s/def ::audit-archive-enabled ::us/boolean)
(s/def ::audit-archive-uri ::us/string)
(s/def ::audit-archive-gc-enabled ::us/boolean)
(s/def ::audit-archive-gc-max-age ::dt/duration)
(s/def ::flags ::us/words)
;; DEPRECATED PROPERTIES: should be removed in 1.10
(s/def ::registration-enabled ::us/boolean)
(s/def ::smtp-enabled ::us/boolean)
(s/def ::telemetry-enabled ::us/boolean)
(s/def ::asserts-enabled ::us/boolean)
;; END DEPRECATED
(s/def ::audit-log-archive-uri ::us/string)
(s/def ::audit-log-gc-max-age ::dt/duration)
(s/def ::secret-key ::us/string)
(s/def ::allow-demo-users ::us/boolean)
(s/def ::asserts-enabled ::us/boolean)
(s/def ::assets-path ::us/string)
(s/def ::database-password (s/nilable ::us/string))
(s/def ::database-uri ::us/string)
(s/def ::database-username (s/nilable ::us/string))
(s/def ::default-blob-version ::us/integer)
(s/def ::error-report-webhook ::us/string)
(s/def ::feedback-destination ::us/string)
(s/def ::feedback-enabled ::us/boolean)
(s/def ::feedback-token ::us/string)
(s/def ::user-feedback-destination ::us/string)
(s/def ::github-client-id ::us/string)
(s/def ::github-client-secret ::us/string)
(s/def ::gitlab-base-uri ::us/string)
@ -158,12 +152,10 @@
(s/def ::public-uri ::us/string)
(s/def ::redis-uri ::us/string)
(s/def ::registration-domain-whitelist ::us/set-of-str)
(s/def ::registration-enabled ::us/boolean)
(s/def ::rlimits-image ::us/integer)
(s/def ::rlimits-password ::us/integer)
(s/def ::smtp-default-from ::us/string)
(s/def ::smtp-default-reply-to ::us/string)
(s/def ::smtp-enabled ::us/boolean)
(s/def ::smtp-host ::us/string)
(s/def ::smtp-password (s/nilable ::us/string))
(s/def ::smtp-port ::us/integer)
@ -180,28 +172,22 @@
(s/def ::storage-fdata-s3-bucket ::us/string)
(s/def ::storage-fdata-s3-region ::us/keyword)
(s/def ::storage-fdata-s3-prefix ::us/string)
(s/def ::telemetry-enabled ::us/boolean)
(s/def ::telemetry-uri ::us/string)
(s/def ::telemetry-with-taiga ::us/boolean)
(s/def ::tenant ::us/string)
(s/def ::config
(s/keys :opt-un [::secret-key
::flags
::allow-demo-users
::audit-enabled
::audit-archive-enabled
::audit-archive-uri
::audit-archive-gc-enabled
::audit-archive-gc-max-age
::asserts-enabled
::audit-log-archive-uri
::audit-log-gc-max-age
::database-password
::database-uri
::database-username
::default-blob-version
::error-report-webhook
::feedback-destination
::feedback-enabled
::feedback-token
::user-feedback-destination
::github-client-id
::github-client-secret
::gitlab-base-uri
@ -258,26 +244,26 @@
::smtp-ssl
::smtp-tls
::smtp-username
::srepl-host
::srepl-port
::assets-storage-backend
::storage-assets-fs-directory
::storage-assets-s3-bucket
::storage-assets-s3-region
::fdata-storage-backend
::storage-fdata-s3-bucket
::storage-fdata-s3-region
::storage-fdata-s3-prefix
::telemetry-enabled
::telemetry-uri
::telemetry-referer
::telemetry-with-taiga
::tenant]))
(defn- parse-flags
[{:keys [flags]}]
(flags/parse flags flags/default))
(defn read-env
[prefix]
(let [prefix (str prefix "-")
@ -304,11 +290,14 @@
(println ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;")))
(throw e))))
(def version (v/parse (or (some-> (io/resource "version.txt")
(slurp)
(str/trim))
"%version%")))
(def config (atom (read-config)))
(def version
(v/parse (or (some-> (io/resource "version.txt")
(slurp)
(str/trim))
"%version%")))
(def ^:dynamic config (read-config))
(def ^:dynamic flags (parse-flags config))
(def deletion-delay
(dt/duration {:days 7}))
@ -316,9 +305,9 @@
(defn get
"A configuration getter. Helps code be more testable."
([key]
(c/get @config key))
(c/get config key))
([key default]
(c/get @config key default)))
(c/get config key default)))
;; Set value for all new threads bindings.
(alter-var-root #'*assert* (constantly (get :asserts-enabled)))
(alter-var-root #'*assert* (constantly (contains? flags :backend-asserts)))

39
backend/src/app/emails.clj
View file

@ -8,7 +8,7 @@
"Main api for send emails."
(:require
[app.common.spec :as us]
[app.config :as cfg]
[app.config :as cf]
[app.db :as db]
[app.db.sql :as sql]
[app.util.emails :as emails]
@ -54,10 +54,10 @@
(defn allow-send-emails?
[conn profile]
(when-not (:is-muted profile false)
(let [complaint-threshold (cfg/get :profile-complaint-threshold)
complaint-max-age (cfg/get :profile-complaint-max-age)
bounce-threshold (cfg/get :profile-bounce-threshold)
bounce-max-age (cfg/get :profile-bounce-max-age)
(let [complaint-threshold (cf/get :profile-complaint-threshold)
complaint-max-age (cf/get :profile-complaint-max-age)
bounce-threshold (cf/get :profile-bounce-threshold)
bounce-max-age (cf/get :profile-bounce-max-age)
{:keys [complaints bounces] :as result}
(db/exec-one! conn [sql:profile-complaint-report
@ -140,19 +140,17 @@
(declare send-console!)
(s/def ::username ::cfg/smtp-username)
(s/def ::password ::cfg/smtp-password)
(s/def ::tls ::cfg/smtp-tls)
(s/def ::ssl ::cfg/smtp-ssl)
(s/def ::host ::cfg/smtp-host)
(s/def ::port ::cfg/smtp-port)
(s/def ::default-reply-to ::cfg/smtp-default-reply-to)
(s/def ::default-from ::cfg/smtp-default-from)
(s/def ::enabled ::cfg/smtp-enabled)
(s/def ::username ::cf/smtp-username)
(s/def ::password ::cf/smtp-password)
(s/def ::tls ::cf/smtp-tls)
(s/def ::ssl ::cf/smtp-ssl)
(s/def ::host ::cf/smtp-host)
(s/def ::port ::cf/smtp-port)
(s/def ::default-reply-to ::cf/smtp-default-reply-to)
(s/def ::default-from ::cf/smtp-default-from)
(defmethod ig/pre-init-spec ::sendmail-handler [_]
(s/keys :req-un [::enabled]
:opt-un [::username
(s/keys :opt-un [::username
::password
::tls
::ssl
@ -164,9 +162,12 @@
(defmethod ig/init-key ::sendmail-handler
[_ cfg]
(fn [{:keys [props] :as task}]
(if (:enabled cfg)
(emails/send! cfg props)
(send-console! cfg props))))
(let [enabled? (or (contains? cf/flags :smtp)
(cf/get :smtp-enabled)
(:enabled task))]
(if enabled?
(emails/send! cfg props)
(send-console! cfg props)))))
(defn- send-console!
[cfg email]

8
backend/src/app/http/feedback.clj
View file

@ -10,7 +10,7 @@
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.config :as cfg]
[app.config :as cf]
[app.db :as db]
[app.emails :as eml]
[app.rpc.queries.profile :as profile]
@ -24,8 +24,8 @@
(defmethod ig/init-key ::handler
[_ {:keys [pool] :as scfg}]
(let [ftoken (cfg/get :feedback-token ::no-token)
enabled (cfg/get :feedback-enabled)]
(let [ftoken (cf/get :feedback-token ::no-token)
enabled (contains? cf/flags :user-feedback)]
(fn [{:keys [profile-id] :as request}]
(let [token (get-in request [:headers "x-feedback-token"])
params (d/merge (:params request)
@ -58,7 +58,7 @@
(defn send-feedback
[pool profile params]
(let [params (us/conform ::feedback params)
destination (cfg/get :feedback-destination)]
destination (cf/get :feedback-destination)]
(eml/send! {::eml/conn pool
::eml/factory eml/feedback
:to destination

44
backend/src/app/loggers/audit.clj
View file

@ -88,9 +88,9 @@
(s/def ::events (s/every ::event))
(defmethod ig/init-key ::http-handler
[_ {:keys [executor enabled] :as cfg}]
(fn [{:keys [params _headers _cookies profile-id] :as request}]
(when enabled
[_ {:keys [executor] :as cfg}]
(fn [{:keys [params profile-id] :as request}]
(when (contains? cf/flags :audit-log)
(let [events (->> (:events params)
(remove #(not= profile-id (:profile-id %)))
(us/conform ::events))
@ -136,10 +136,9 @@
;; an external storage and data cleared.
(declare persist-events)
(s/def ::enabled ::us/boolean)
(defmethod ig/pre-init-spec ::collector [_]
(s/keys :req-un [::db/pool ::wrk/executor ::enabled]))
(s/keys :req-un [::db/pool ::wrk/executor]))
(def event-xform
(comp
@ -147,9 +146,9 @@
(map clean-props)))
(defmethod ig/init-key ::collector
[_ {:keys [enabled] :as cfg}]
(when enabled
(l/info :msg "intializing audit collector")
[_ cfg]
(when (contains? cf/flags :audit-log)
(l/info :msg "intializing audit log collector")
(let [input (a/chan 512 event-xform)
buffer (aa/batch input {:max-batch-size 100
:max-batch-age (* 10 1000) ; 10s
@ -202,15 +201,16 @@
(s/def ::tokens fn?)
(defmethod ig/pre-init-spec ::archive-task [_]
(s/keys :req-un [::db/pool ::tokens ::enabled]
(s/keys :req-un [::db/pool ::tokens]
:opt-un [::uri]))
(defmethod ig/init-key ::archive-task
[_ {:keys [uri enabled] :as cfg}]
[_ {:keys [uri] :as cfg}]
(fn [props]
;; NOTE: this let allows overwrite default configured values from
;; the repl, when manually invoking the task.
(let [enabled (or enabled (:enabled props false))
(let [enabled (or (contains? cf/flags :audit-log-archive)
(:enabled props false))
uri (or uri (:uri props))
cfg (assoc cfg :uri uri)]
(when (and enabled (not uri))
@ -298,18 +298,6 @@
;; GC Task
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(declare clean-archived)
(s/def ::max-age ::cf/audit-archive-gc-max-age)
(defmethod ig/pre-init-spec ::archive-gc-task [_]
(s/keys :req-un [::db/pool ::enabled ::max-age]))
(defmethod ig/init-key ::archive-gc-task
[_ cfg]
(fn [_]
(clean-archived cfg)))
(def sql:clean-archived
"delete from audit_log
where archived_at is not null
@ -322,3 +310,13 @@
result (:next.jdbc/update-count result)]
(l/debug :action "clean archived audit log" :removed result)
result))
(s/def ::max-age ::cf/audit-log-gc-max-age)
(defmethod ig/pre-init-spec ::gc-task [_]
(s/keys :req-un [::db/pool ::max-age]))
(defmethod ig/init-key ::gc-task
[_ cfg]
(fn [_]
(clean-archived cfg)))

30
backend/src/app/main.clj
View file

@ -210,15 +210,16 @@
{:cron #app/cron "0 0 * * * ?" ;; hourly
:task :file-offload})
(when (cf/get :audit-archive-enabled)
(when (contains? cf/flags :audit-log-archive)
{:cron #app/cron "0 0 * * * ?" ;; every 1h
:task :audit-archive})
:task :audit-log-archive})
(when (cf/get :audit-archive-gc-enabled)
(when (contains? cf/flags :audit-log-gc)
{:cron #app/cron "0 0 * * * ?" ;; every 1h
:task :audit-archive-gc})
:task :audit-log-gc})
(when (cf/get :telemetry-enabled)
(when (or (contains? cf/flags :telemetry)
(cf/get :telemetry-enabled))
{:cron #app/cron "0 0 */6 * * ?" ;; every 6h
:task :telemetry})]}
@ -238,15 +239,14 @@
:telemetry (ig/ref :app.tasks.telemetry/handler)
:session-gc (ig/ref :app.http.session/gc-task)
:file-offload (ig/ref :app.tasks.file-offload/handler)
:audit-archive (ig/ref :app.loggers.audit/archive-task)
:audit-archive-gc (ig/ref :app.loggers.audit/archive-gc-task)}}
:audit-log-archive (ig/ref :app.loggers.audit/archive-task)
:audit-log-gc (ig/ref :app.loggers.audit/gc-task)}}
:app.emails/sendmail-handler
{:host (cf/get :smtp-host)
:port (cf/get :smtp-port)
:ssl (cf/get :smtp-ssl)
:tls (cf/get :smtp-tls)
:enabled (cf/get :smtp-enabled)
:username (cf/get :smtp-username)
:password (cf/get :smtp-password)
:metrics (ig/ref :app.metrics/metrics)
@ -304,24 +304,20 @@
{:endpoint (cf/get :loggers-zmq-uri)}
:app.loggers.audit/http-handler
{:enabled (cf/get :audit-enabled false)
:pool (ig/ref :app.db/pool)
{:pool (ig/ref :app.db/pool)
:executor (ig/ref :app.worker/executor)}
:app.loggers.audit/collector
{:enabled (cf/get :audit-enabled false)
:pool (ig/ref :app.db/pool)
{:pool (ig/ref :app.db/pool)
:executor (ig/ref :app.worker/executor)}
:app.loggers.audit/archive-task
{:uri (cf/get :audit-archive-uri)
:enabled (cf/get :audit-archive-enabled false)
{:uri (cf/get :audit-log-archive-uri)
:tokens (ig/ref :app.tokens/tokens)
:pool (ig/ref :app.db/pool)}
:app.loggers.audit/archive-gc-task
{:enabled (cf/get :audit-archive-gc-enabled false)
:max-age (cf/get :audit-archive-gc-max-age cf/deletion-delay)
:app.loggers.audit/gc-task
{:max-age (cf/get :audit-log-gc-max-age cf/deletion-delay)
:pool (ig/ref :app.db/pool)}
:app.loggers.loki/reporter

6
backend/src/app/rpc/mutations/demo.clj
View file

@ -9,7 +9,7 @@
(:require
[app.common.exceptions :as ex]
[app.common.uuid :as uuid]
[app.config :as cfg]
[app.config :as cf]
[app.db :as db]
[app.loggers.audit :as audit]
[app.rpc.mutations.profile :as profile]
@ -35,11 +35,11 @@
:email email
:fullname fullname
:is-demo true
:deleted-at (dt/in-future cfg/deletion-delay)
:deleted-at (dt/in-future cf/deletion-delay)
:password password
:props {:onboarding-viewed true}}]
(when-not (cfg/get :allow-demo-users)
(when-not (contains? cf/flags :demo-users)
(ex/raise :type :validation
:code :demo-users-not-allowed
:hint "Demo users are disabled by config."))

6
backend/src/app/rpc/mutations/profile.clj
View file

@ -100,10 +100,9 @@
(sv/defmethod ::prepare-register-profile {:auth false}
[{:keys [pool tokens] :as cfg} params]
(when-not (cf/get :registration-enabled)
(when-not (contains? cf/flags :registration)
(ex/raise :type :restriction
:code :registration-disabled))
(when-let [domains (cf/get :registration-domain-whitelist)]
(when-not (email-domain-in-whitelist? domains (:email params))
(ex/raise :type :validation
@ -458,7 +457,8 @@
params (assoc params
:profile profile
:email (str/lower email))]
(if (cf/get :smtp-enabled)
(if (or (cf/get :smtp-enabled)
(contains? cf/flags :smtp))
(request-email-change cfg params)
(change-email-inmediatelly cfg params)))))

5
backend/test/app/services_profile_test.clj
View file

@ -211,10 +211,7 @@
))
(t/deftest prepare-register-with-registration-disabled
(with-mocks [mock {:target 'app.config/get
:return (th/mock-config-get-with
{:registration-enabled false})}]
(th/with-mocks {#'app.config/flags nil}
(let [data {::th/type :prepare-register-profile
:email "user@example.com"
:password "foobar"}]

11
backend/test/app/test_helpers.clj
View file

@ -7,6 +7,7 @@
(ns app.test-helpers
(:require
[app.common.data :as d]
[app.common.flags :as flags]
[app.common.pages :as cp]
[app.common.spec :as us]
[app.common.uuid :as uuid]
@ -336,9 +337,15 @@
[data]
(fn
([key]
(get data key (get @cf/config key)))
(get data key (get cf/config key)))
([key default]
(get data key (get @cf/config key default)))))
(get data key (get cf/config key default)))))
(defmacro with-mocks
[rebinds & body]
`(with-redefs-fn ~rebinds
(fn [] ~@body)))
(defn reset-mock!
[m]

38
common/src/app/common/flags.cljc
View file

@ -9,24 +9,30 @@
(:require
[cuerdas.core :as str]))
(def default
#{:backend-asserts
:registration
:demo-users})
(defn parse
[default flags]
(loop [flags (seq flags)
result default]
(let [item (first flags)]
(if (nil? item)
result
(let [sname (name item)]
(cond
(str/starts-with? sname "enable-")
(recur (rest flags)
(conj result (keyword (subs sname 7))))
([flags] (parse flags #{}))
([flags default]
(loop [flags (seq flags)
result default]
(let [item (first flags)]
(if (nil? item)
result
(let [sname (name item)]
(cond
(str/starts-with? sname "enable-")
(recur (rest flags)
(conj result (keyword (subs sname 7))))
(str/starts-with? sname "disable-")
(recur (rest flags)
(disj result (keyword (subs sname 8))))
(str/starts-with? sname "disable-")
(recur (rest flags)
(disj result (keyword (subs sname 8))))
:else
(recur (rest flags) result)))))))
:else
(recur (rest flags) result))))))))

10
common/src/app/common/spec.cljc
View file

@ -111,6 +111,16 @@
(s/def ::point gpt/point?)
(s/def ::id ::uuid)
(s/def ::words
(s/conformer
(fn [s]
(cond
(set? s) s
(string? s) (into #{} (map keyword) (str/words s))
:else ::s/invalid))
(fn [s]
(str/join " " (map name s)))))
(defn bytes?
"Test if a first parameter is a byte
array or not."

13
frontend/src/app/config.cljs
View file

@ -54,22 +54,11 @@
:browser
:webworker))
(def available-flags
#{:registration
:audit-log
:demo-users
:user-feedback
:demo-warning
:login-with-ldap})
(def default-flags
#{:registration :demo-users})
(defn- parse-flags
[global]
(let [flags (obj/get global "penpotFlags" "")
flags (into #{} (map keyword) (str/words flags))]
(flags/parse default-flags flags)))
(flags/parse flags flags/default)))
(defn- parse-version
[global]