From 5a641e4629e540a2d542dc1cab2c86d42e184561 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Sun, 22 Jan 2017 15:50:00 +0100 Subject: [PATCH] Refactor configuration loading. --- .gitignore | 2 +- backend/project.clj | 3 + backend/src/uxbox/config.clj | 66 +++++++++++++++------ backend/src/uxbox/db.clj | 13 +++- backend/src/uxbox/emails/core.clj | 11 ++-- backend/src/uxbox/frontend.clj | 15 +++-- backend/src/uxbox/main.clj | 62 ++++++------------- backend/src/uxbox/media.clj | 16 ++--- backend/src/uxbox/migrations.clj | 2 +- backend/src/uxbox/scheduled_jobs/emails.clj | 12 +++- backend/src/uxbox/services/auth.clj | 8 ++- backend/test/uxbox/tests/helpers.clj | 4 +- 12 files changed, 125 insertions(+), 89 deletions(-) diff --git a/.gitignore b/.gitignore index 9e18b1954..de5c9f411 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,7 @@ pom.xml .nrepl-port node_modules /backend/target/ -/backend/resources/public/media +/backend/resources/media /backend/dist/ /frontend/target/ /frontend/dist/ diff --git a/backend/project.clj b/backend/project.clj index 77c469150..8d09c807e 100644 --- a/backend/project.clj +++ b/backend/project.clj @@ -15,6 +15,8 @@ [funcool/suricatta "1.3.1"] [funcool/promesa "1.7.0"] [funcool/catacumba "2.0.0-SNAPSHOT"] + [funcool/cuerdas "2.0.2"] + [funcool/datoteka "1.0.0-SNAPSHOT"] [org.clojure/data.xml "0.1.0-beta2"] [org.jsoup/jsoup "1.10.2"] @@ -27,6 +29,7 @@ :exclusions [org.clojure/tools.reader]] [niwinz/migrante "0.1.0"] + [buddy/buddy-sign "1.3.0" :exclusions [org.clojure/tools.reader]] [buddy/buddy-hashers "1.1.0"] diff --git a/backend/src/uxbox/config.clj b/backend/src/uxbox/config.clj index 8b6ce12bc..bb2094990 100644 --- a/backend/src/uxbox/config.clj +++ b/backend/src/uxbox/config.clj @@ -2,37 +2,69 @@ ;; License, v. 2.0. If a copy of the MPL was not distributed with this ;; file, You can obtain one at http://mozilla.org/MPL/2.0/. ;; -;; Copyright (c) 2016 Andrey Antukh +;; Copyright (c) 2017 Andrey Antukh (ns uxbox.config "A configuration management." - (:require [mount.core :refer [defstate]] - [environ.core :refer (env)] - [buddy.core.hash :as hash] - [clojure.java.io :as io] + (:require [clojure.java.io :as io] + [clojure.tools.logging :as log] [clojure.edn :as edn] + [cuerdas.core :as str] + [buddy.core.hash :as hash] + [environ.core :refer [env]] + [mount.core :refer [defstate]] [uxbox.util.exceptions :as ex] [uxbox.util.data :refer [deep-merge]])) -;; --- Configuration Loading & Parsing +;; --- Configuration Reading & Loading -(def ^:dynamic *default-config-path* "config/default.edn") -(def ^:dynamic *local-config-path* "config/local.edn") +(defn lookup-env + [env key default] + (let [value (get env key ::empty)] + (if (= value ::empty) + default + (try + (read-string value) + (catch Exception e + (log/warn (str/istr "can't parse `~{key}` env value")) + default))))) + +;; --- Configuration Loading & Parsing (defn read-config [] - (let [builtin (io/resource *default-config-path*) - local (io/resource *local-config-path*) - external (io/file (:uxbox-config env))] - (deep-merge (edn/read-string (slurp builtin)) - (when local (edn/read-string (slurp local))) - (when (and external (.exists external)) - (edn/read-string (slurp external)))))) + {:http-server-port (lookup-env env :uxbox-http-server-port 6060) + :http-server-debug (lookup-env env :uxbox-http-server-debug true) + :database-username (lookup-env env :uxbox-database-username "") + :database-password (lookup-env env :uxbox-database-password "") + :database-name (lookup-env env :uxbox-database-name "uxbox") + :database-server (lookup-env env :uxbox-database-server "localhost") + :database-port (lookup-env env :uxbox-database-port 5432) + :media-directory (lookup-env env :uxbox-media-directory "media") + :media-uri (lookup-env env :uxbox-media-uri "http://localhost:6060/media/") + :assets-directory (lookup-env env :uxbox-assets-directory "static") + :assets-uri (lookup-env env :uxbox-assets-uri "http://localhost:6060/static/") + + :email-reply-to (lookup-env env :uxbox-email-reply-to "no-reply@uxbox.io") + :email-from (lookup-env env :uxbox-email-from "no-reply@uxbox.io") + + :smtp-host (lookup-env env :uxbox-smtp-host "localhost") + :smtp-port (lookup-env env :uxbox-smtp-port 25) + :smtp-user (lookup-env env :uxbox-smtp-user nil) + :smtp-password (lookup-env env :uxbox-smtp-password nil) + :smtp-tls (lookup-env env :uxbox-smtp-tls false) + :smtp-ssl (lookup-env env :uxbox-smtp-ssl false) + :smtp-enabled (lookup-env env :uxbox-smtp-enabled false) + + :secret (lookup-env env :uxbox-secret "5qjiAndGY3")}) (defn read-test-config [] - (binding [*local-config-path* "config/test.edn"] - (read-config))) + (assoc (read-config) + :database-name "test" + :media-directory "/tmp/uxbox/media" + :assets-directory "/tmp/uxbox/static" + :migrations-verbose false)) (defstate config :start (read-config)) diff --git a/backend/src/uxbox/db.clj b/backend/src/uxbox/db.clj index 9d60a3889..26c17281b 100644 --- a/backend/src/uxbox/db.clj +++ b/backend/src/uxbox/db.clj @@ -21,7 +21,7 @@ ;; --- State -(def ^:const +defaults+ +(def connection-defaults {:connection-timeout 30000 :idle-timeout 600000 :max-lifetime 1800000 @@ -34,9 +34,18 @@ :server-name "localhost" :port-number 5432}) +(defn get-db-config + [config] + (assoc connection-defaults + :username (:database-username config) + :password (:database-password config) + :database-name (:database-name config) + :server-name (:database-server config) + :port-number (:database-port config))) + (defn create-datasource [config] - (let [dbconf (merge +defaults+ config)] + (let [dbconf (get-db-config config)] (hikari/make-datasource dbconf))) (defstate datasource diff --git a/backend/src/uxbox/emails/core.clj b/backend/src/uxbox/emails/core.clj index 454bae340..09dc2ed7b 100644 --- a/backend/src/uxbox/emails/core.clj +++ b/backend/src/uxbox/emails/core.clj @@ -5,8 +5,8 @@ ;; Copyright (c) 2016 Andrey Antukh (ns uxbox.emails.core - (:require [hiccup.core :refer (html)] - [hiccup.page :refer (html4)] + (:require [hiccup.core :refer [html]] + [hiccup.page :refer [html4]] [suricatta.core :as sc] [uxbox.db :as db] [uxbox.config :as cfg] @@ -49,11 +49,10 @@ (defn render-email [email context] - (let [config (:email cfg/config) - from (or (:email/from context) - (:from config)) + (let [from (or (:email/from context) + (:email-from cfg/config)) reply-to (or (:email/reply-to context) - (:reply-to config) + (:email-reply-to cfg/config) from)] {:subject (render-subject email context) :body (render-body-alternatives email context) diff --git a/backend/src/uxbox/frontend.clj b/backend/src/uxbox/frontend.clj index 0111bbaf5..cb57e4f1a 100644 --- a/backend/src/uxbox/frontend.clj +++ b/backend/src/uxbox/frontend.clj @@ -2,7 +2,7 @@ ;; License, v. 2.0. If a copy of the MPL was not distributed with this ;; file, You can obtain one at http://mozilla.org/MPL/2.0/. ;; -;; Copyright (c) 2016 Andrey Antukh +;; Copyright (c) 2016-2017 Andrey Antukh (ns uxbox.frontend (:require [mount.core :refer [defstate]] @@ -23,6 +23,7 @@ [uxbox.frontend.kvstore :as kvstore] [uxbox.frontend.svgparse :as svgparse] [uxbox.frontend.debug-emails :as dbgemails] + [uxbox.services.auth :refer [auth-opts]] [uxbox.util.response :refer [rsp]] [uxbox.util.uuid :as uuid])) @@ -56,14 +57,16 @@ ([] (routes cfg/config)) ([config] (let [auth-opts {:secret cfg/secret - :options (:auth-options cfg/config)}] + :options auth-opts}] (ct/routes [[:any (cauth/auth (cauth/jwe-backend auth-opts))] [:any (cmisc/autoreloader)] [:get "api" #'welcome-api] - [:assets "media" {:dir "public/media"}] - [:assets "static" {:dir "public/static"}] + + ;; Serve assets on development server + [:assets "media" {:dir "media"}] + [:assets "static" {:dir "static"}] [:prefix "debug" [:any debug-only] @@ -148,7 +151,9 @@ (defn- start-server [config] - (let [config (:http config)] + (let [config {:port (:http-server-port config) + :debug (:http-server-debug config) + :max-body-size 52428800}] (ct/run-server (routes config) config))) (defstate server diff --git a/backend/src/uxbox/main.clj b/backend/src/uxbox/main.clj index a5ae1346c..1a637a13b 100644 --- a/backend/src/uxbox/main.clj +++ b/backend/src/uxbox/main.clj @@ -5,6 +5,7 @@ ;; Copyright (c) 2016 Andrey Antukh (ns uxbox.main + (:refer-clojure :exclude [test]) (:require [clojure.tools.namespace.repl :as repl] [clojure.walk :refer [macroexpand-all]] [clojure.pprint :refer [pprint]] @@ -27,6 +28,10 @@ [] (mount/start)) +(defn- stop + [] + (mount/stop)) + (defn- start-minimal [] (-> (mount/only #{#'uxbox.config/config @@ -34,56 +39,25 @@ #'uxbox.migrations/migrations}) (mount/start))) -(defn- stop - [] - (mount/stop)) - -(defn- refresh - [] - (stop) - (repl/refresh)) - -(defn- refresh-all - [] - (stop) - (repl/refresh-all)) - -(defn- go - "starts all states defined by defstate" - [] - (start) - :ready) - -(defn- reset - [] - (stop) - (repl/refresh :after 'uxbox.main/start)) - -(defn make-secret +(defn- make-secret [] (-> (nonce/random-bytes 64) (b64/encode true) (codecs/bytes->str))) -;; --- Entry point (only for uberjar) +(defn- test + ([] (test #"^uxbox.tests.*")) + ([o] + (repl/refresh) + (cond + (instance? java.util.regex.Pattern o) + (test/run-all-tests o) -(defn test-vars - [& vars] - (repl/refresh) - (test/test-vars - (map (fn [sym] - (require (symbol (namespace sym))) - (resolve sym)) - vars))) - -(defn test-ns - [ns] - (repl/refresh) - (test/test-ns ns)) - -(defn test-all - ([] (test/run-all-tests #"^uxbox.tests.*")) - ([re] (test/run-all-tests re))) + (symbol? o) + (if-let [sns (namespace o)] + (do (require (symbol sns)) + (test/test-vars [(resolve o)])) + (test/test-ns o))))) ;; --- Entry point (only for uberjar) diff --git a/backend/src/uxbox/media.clj b/backend/src/uxbox/media.clj index 37a1dbc59..b81c5a36c 100644 --- a/backend/src/uxbox/media.clj +++ b/backend/src/uxbox/media.clj @@ -2,7 +2,7 @@ ;; License, v. 2.0. If a copy of the MPL was not distributed with this ;; file, You can obtain one at http://mozilla.org/MPL/2.0/. ;; -;; Copyright (c) 2016 Andrey Antukh +;; Copyright (c) 2017 Andrey Antukh (ns uxbox.media "A media storage impl for uxbox." @@ -14,15 +14,17 @@ [storages.backend.misc :refer (hashed scoped)] [uxbox.config :refer (config)])) +;; FIXME: migrate from storages to datoteka + ;; --- State -(defstate static-storage - :start (let [{:keys [basedir baseuri]} (:static config)] - (localfs {:basedir basedir :baseuri baseuri}))) +(defstate assets-storage + :start (localfs {:basedir (:assets-directory config) + :baseuri (:assets-uri config)})) (defstate media-storage - :start (let [{:keys [basedir baseuri]} (:media config)] - (localfs {:basedir basedir :baseuri baseuri}))) + :start (localfs {:basedir (:media-directory config) + :baseuri (:media-uri config)})) (defstate images-storage :start (-> media-storage @@ -36,4 +38,4 @@ (defn resolve-asset [path] - (str (st/public-url static-storage path))) + (str (st/public-url assets-storage path))) diff --git a/backend/src/uxbox/migrations.clj b/backend/src/uxbox/migrations.clj index 4036f9317..93ed725e8 100644 --- a/backend/src/uxbox/migrations.clj +++ b/backend/src/uxbox/migrations.clj @@ -69,7 +69,7 @@ (defn- migrate [] - (let [options (:migrations cfg/config {})] + (let [options {:verbose (:migrations-verbose cfg/config true)}] (with-open [mctx (mg/context db/datasource options)] (mg/migrate mctx +migrations+) nil))) diff --git a/backend/src/uxbox/scheduled_jobs/emails.clj b/backend/src/uxbox/scheduled_jobs/emails.clj index abe4c7976..4ffa38eb8 100644 --- a/backend/src/uxbox/scheduled_jobs/emails.clj +++ b/backend/src/uxbox/scheduled_jobs/emails.clj @@ -65,9 +65,19 @@ (println "********** end email:" id "**********") {:error :SUCCESS}) +(defn- get-smtp-config + [config] + {:host (:smtp-host config) + :port (:smtp-port config) + :user (:smtp-user config) + :pass (:smtp-password config) + :ssl (:smtp-ssl config) + :tls (:smtp-tls config) + :noop (not (:smtp-enabled config))}) + (defn- send-email [{:keys [id data] :as entry}] - (let [config (:smtp cfg/config) + (let [config (get-smtp-config cfg/config) result (if (:noop config) (send-email-to-console entry) (postal/send-message config data))] diff --git a/backend/src/uxbox/services/auth.clj b/backend/src/uxbox/services/auth.clj index 00d3659af..69d1c60ea 100644 --- a/backend/src/uxbox/services/auth.clj +++ b/backend/src/uxbox/services/auth.clj @@ -17,6 +17,9 @@ [uxbox.services.users :as users] [uxbox.util.exceptions :as ex])) +(def auth-opts + {:alg :a256kw :enc :a128cbc-hs256}) + ;; --- Login (defn- check-user-password @@ -25,9 +28,8 @@ (defn generate-token [user] - (let [data {:id (:id user)} - opts (:auth-options cfg/config)] - (jwt/encrypt data cfg/secret opts))) + (let [data {:id (:id user) :scope :auth}] + (jwt/encrypt data cfg/secret auth-opts))) (s/def ::scope string?) (s/def ::login diff --git a/backend/test/uxbox/tests/helpers.clj b/backend/test/uxbox/tests/helpers.clj index bc248e7f9..57620376d 100644 --- a/backend/test/uxbox/tests/helpers.clj +++ b/backend/test/uxbox/tests/helpers.clj @@ -24,7 +24,7 @@ #'uxbox.config/secret #'uxbox.db/datasource #'uxbox.migrations/migrations - #'uxbox.media/static-storage + #'uxbox.media/assets-storage #'uxbox.media/media-storage #'uxbox.media/images-storage #'uxbox.media/thumbnails-storage}) @@ -53,7 +53,7 @@ (next) (finally (st/clear! uxbox.media/media-storage) - (st/clear! uxbox.media/static-storage)))))) + (st/clear! uxbox.media/assets-storage)))))) (defmacro await [expr]