mirror of
https://github.com/penpot/penpot.git
synced 2025-03-11 23:31:21 -05:00
🎉 Add proper lifecycle handling for http sessions.
This commit is contained in:
Andrey Antukh
Andrés Moya
parent
995017df5a
commit
fb51580740
5 changed files with 227 additions and 86 deletions
|
|
@ -5,7 +5,7 @@
|
|||
;; This Source Code Form is "Incompatible With Secondary Licenses", as
|
||||
;; defined by the Mozilla Public License, v. 2.0.
|
||||
;;
|
||||
;; Copyright (c) 2020 UXBOX Labs SL
|
||||
;; Copyright (c) 2020-2021 UXBOX Labs SL
|
||||
|
||||
(ns app.config
|
||||
"A configuration management."
|
||||
|
|
@ -80,92 +80,78 @@
|
|||
;; :initial-data-project-name "Penpot Oboarding"
|
||||
})
|
||||
|
||||
(s/def ::http-server-port ::us/integer)
|
||||
|
||||
(s/def ::host ::us/string)
|
||||
(s/def ::tenant ::us/string)
|
||||
|
||||
(s/def ::database-username (s/nilable ::us/string))
|
||||
(s/def ::allow-demo-users ::us/boolean)
|
||||
(s/def ::asserts-enabled ::us/boolean)
|
||||
(s/def ::assets-path ::us/string)
|
||||
(s/def ::database-password (s/nilable ::us/string))
|
||||
(s/def ::database-uri ::us/string)
|
||||
(s/def ::redis-uri ::us/string)
|
||||
|
||||
(s/def ::loggers-loki-uri ::us/string)
|
||||
(s/def ::loggers-zmq-uri ::us/string)
|
||||
|
||||
(s/def ::storage-backend ::us/keyword)
|
||||
(s/def ::storage-fs-directory ::us/string)
|
||||
(s/def ::assets-path ::us/string)
|
||||
(s/def ::storage-s3-region ::us/keyword)
|
||||
(s/def ::storage-s3-bucket ::us/string)
|
||||
|
||||
(s/def ::media-uri ::us/string)
|
||||
(s/def ::media-directory ::us/string)
|
||||
(s/def ::asserts-enabled ::us/boolean)
|
||||
|
||||
(s/def ::feedback-enabled ::us/boolean)
|
||||
(s/def ::feedback-destination ::us/string)
|
||||
|
||||
(s/def ::profile-complaint-max-age ::dt/duration)
|
||||
(s/def ::profile-complaint-threshold ::us/integer)
|
||||
(s/def ::profile-bounce-max-age ::dt/duration)
|
||||
(s/def ::profile-bounce-threshold ::us/integer)
|
||||
|
||||
(s/def ::database-username (s/nilable ::us/string))
|
||||
(s/def ::default-blob-version ::us/integer)
|
||||
(s/def ::error-report-webhook ::us/string)
|
||||
|
||||
(s/def ::smtp-enabled ::us/boolean)
|
||||
(s/def ::smtp-default-reply-to ::us/string)
|
||||
(s/def ::smtp-default-from ::us/string)
|
||||
(s/def ::smtp-host ::us/string)
|
||||
(s/def ::smtp-port ::us/integer)
|
||||
(s/def ::smtp-username (s/nilable ::us/string))
|
||||
(s/def ::smtp-password (s/nilable ::us/string))
|
||||
(s/def ::smtp-tls ::us/boolean)
|
||||
(s/def ::smtp-ssl ::us/boolean)
|
||||
(s/def ::allow-demo-users ::us/boolean)
|
||||
(s/def ::registration-enabled ::us/boolean)
|
||||
(s/def ::registration-domain-whitelist ::us/string)
|
||||
(s/def ::public-uri ::us/string)
|
||||
|
||||
(s/def ::srepl-host ::us/string)
|
||||
(s/def ::srepl-port ::us/integer)
|
||||
|
||||
(s/def ::rlimits-password ::us/integer)
|
||||
(s/def ::rlimits-image ::us/integer)
|
||||
|
||||
(s/def ::google-client-id ::us/string)
|
||||
(s/def ::google-client-secret ::us/string)
|
||||
|
||||
(s/def ::gitlab-client-id ::us/string)
|
||||
(s/def ::gitlab-client-secret ::us/string)
|
||||
(s/def ::gitlab-base-uri ::us/string)
|
||||
|
||||
(s/def ::feedback-destination ::us/string)
|
||||
(s/def ::feedback-enabled ::us/boolean)
|
||||
(s/def ::github-client-id ::us/string)
|
||||
(s/def ::github-client-secret ::us/string)
|
||||
|
||||
(s/def ::ldap-host ::us/string)
|
||||
(s/def ::ldap-port ::us/integer)
|
||||
(s/def ::ldap-bind-dn ::us/string)
|
||||
(s/def ::ldap-bind-password ::us/string)
|
||||
(s/def ::ldap-ssl ::us/boolean)
|
||||
(s/def ::ldap-starttls ::us/boolean)
|
||||
(s/def ::ldap-base-dn ::us/string)
|
||||
(s/def ::ldap-user-query ::us/string)
|
||||
(s/def ::ldap-attrs-username ::us/string)
|
||||
(s/def ::gitlab-base-uri ::us/string)
|
||||
(s/def ::gitlab-client-id ::us/string)
|
||||
(s/def ::gitlab-client-secret ::us/string)
|
||||
(s/def ::google-client-id ::us/string)
|
||||
(s/def ::google-client-secret ::us/string)
|
||||
(s/def ::host ::us/string)
|
||||
(s/def ::http-server-port ::us/integer)
|
||||
(s/def ::http-session-cookie-name ::us/string)
|
||||
(s/def ::http-session-idle-max-age ::dt/duration)
|
||||
(s/def ::http-session-updater-batch-max-age ::dt/duration)
|
||||
(s/def ::http-session-updater-batch-max-size ::us/integer)
|
||||
(s/def ::initial-data-file ::us/string)
|
||||
(s/def ::initial-data-project-name ::us/string)
|
||||
(s/def ::ldap-attrs-email ::us/string)
|
||||
(s/def ::ldap-attrs-fullname ::us/string)
|
||||
(s/def ::ldap-attrs-photo ::us/string)
|
||||
|
||||
(s/def ::ldap-attrs-username ::us/string)
|
||||
(s/def ::ldap-base-dn ::us/string)
|
||||
(s/def ::ldap-bind-dn ::us/string)
|
||||
(s/def ::ldap-bind-password ::us/string)
|
||||
(s/def ::ldap-host ::us/string)
|
||||
(s/def ::ldap-port ::us/integer)
|
||||
(s/def ::ldap-ssl ::us/boolean)
|
||||
(s/def ::ldap-starttls ::us/boolean)
|
||||
(s/def ::ldap-user-query ::us/string)
|
||||
(s/def ::loggers-loki-uri ::us/string)
|
||||
(s/def ::loggers-zmq-uri ::us/string)
|
||||
(s/def ::media-directory ::us/string)
|
||||
(s/def ::media-uri ::us/string)
|
||||
(s/def ::profile-bounce-max-age ::dt/duration)
|
||||
(s/def ::profile-bounce-threshold ::us/integer)
|
||||
(s/def ::profile-complaint-max-age ::dt/duration)
|
||||
(s/def ::profile-complaint-threshold ::us/integer)
|
||||
(s/def ::public-uri ::us/string)
|
||||
(s/def ::redis-uri ::us/string)
|
||||
(s/def ::registration-domain-whitelist ::us/string)
|
||||
(s/def ::registration-enabled ::us/boolean)
|
||||
(s/def ::rlimits-image ::us/integer)
|
||||
(s/def ::rlimits-password ::us/integer)
|
||||
(s/def ::smtp-default-from ::us/string)
|
||||
(s/def ::smtp-default-reply-to ::us/string)
|
||||
(s/def ::smtp-enabled ::us/boolean)
|
||||
(s/def ::smtp-host ::us/string)
|
||||
(s/def ::smtp-password (s/nilable ::us/string))
|
||||
(s/def ::smtp-port ::us/integer)
|
||||
(s/def ::smtp-ssl ::us/boolean)
|
||||
(s/def ::smtp-tls ::us/boolean)
|
||||
(s/def ::smtp-username (s/nilable ::us/string))
|
||||
(s/def ::srepl-host ::us/string)
|
||||
(s/def ::srepl-port ::us/integer)
|
||||
(s/def ::storage-backend ::us/keyword)
|
||||
(s/def ::storage-fs-directory ::us/string)
|
||||
(s/def ::storage-s3-bucket ::us/string)
|
||||
(s/def ::storage-s3-region ::us/keyword)
|
||||
(s/def ::telemetry-enabled ::us/boolean)
|
||||
(s/def ::telemetry-with-taiga ::us/boolean)
|
||||
(s/def ::telemetry-uri ::us/string)
|
||||
(s/def ::telemetry-server-enabled ::us/boolean)
|
||||
(s/def ::telemetry-server-port ::us/integer)
|
||||
|
||||
(s/def ::initial-data-file ::us/string)
|
||||
(s/def ::initial-data-project-name ::us/string)
|
||||
|
||||
(s/def ::default-blob-version ::us/integer)
|
||||
(s/def ::telemetry-uri ::us/string)
|
||||
(s/def ::telemetry-with-taiga ::us/boolean)
|
||||
(s/def ::tenant ::us/string)
|
||||
|
||||
(s/def ::config
|
||||
(s/keys :opt-un [::allow-demo-users
|
||||
|
|
@ -185,6 +171,9 @@
|
|||
::google-client-id
|
||||
::google-client-secret
|
||||
::http-server-port
|
||||
::http-session-updater-batch-max-age
|
||||
::http-session-updater-batch-max-size
|
||||
::http-session-idle-max-age
|
||||
::host
|
||||
::ldap-attrs-username
|
||||
::ldap-attrs-email
|
||||
|
|
|
|||
|
|
@ -9,11 +9,20 @@
|
|||
|
||||
(ns app.http.session
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.config :as cfg]
|
||||
[app.db :as db]
|
||||
[app.metrics :as mtx]
|
||||
[app.util.async :as aa]
|
||||
[app.util.log4j :refer [update-thread-context!]]
|
||||
[app.util.time :as dt]
|
||||
[app.worker :as wrk]
|
||||
[buddy.core.codecs :as bc]
|
||||
[buddy.core.nonce :as bn]
|
||||
[clojure.core.async :as a]
|
||||
[clojure.spec.alpha :as s]
|
||||
[clojure.tools.logging :as log]
|
||||
[integrant.core :as ig]))
|
||||
|
||||
;; --- IMPL
|
||||
|
|
@ -42,8 +51,7 @@
|
|||
(defn- retrieve
|
||||
[{:keys [conn] :as cfg} token]
|
||||
(when token
|
||||
(-> (db/exec-one! conn ["select profile_id from http_session where id = ?" token])
|
||||
(:profile-id))))
|
||||
(db/exec-one! conn ["select id, profile_id from http_session where id = ?" token])))
|
||||
|
||||
(defn- retrieve-from-request
|
||||
[{:keys [cookie-name] :as cfg} {:keys [cookies] :as request}]
|
||||
|
|
@ -57,24 +65,33 @@
|
|||
(defn- middleware
|
||||
[cfg handler]
|
||||
(fn [request]
|
||||
(if-let [profile-id (retrieve-from-request cfg request)]
|
||||
(do
|
||||
(if-let [{:keys [id profile-id] :as session} (retrieve-from-request cfg request)]
|
||||
(let [ech (::events-ch cfg)]
|
||||
(a/>!! ech id)
|
||||
(update-thread-context! {:profile-id profile-id})
|
||||
(handler (assoc request :profile-id profile-id)))
|
||||
(handler request))))
|
||||
|
||||
;; --- STATE INIT
|
||||
;; --- STATE INIT: SESSION
|
||||
|
||||
(s/def ::cookie-name ::cfg/http-session-cookie-name)
|
||||
|
||||
(defmethod ig/pre-init-spec ::session [_]
|
||||
(s/keys :req-un [::db/pool]))
|
||||
(s/keys :req-un [::db/pool]
|
||||
:opt-un [::cookie-name]))
|
||||
|
||||
(defmethod ig/prep-key ::session
|
||||
[_ cfg]
|
||||
(merge {:cookie-name "auth-token"} cfg))
|
||||
(merge {:cookie-name "auth-token"
|
||||
:buffer-size 64}
|
||||
(d/without-nils cfg)))
|
||||
|
||||
(defmethod ig/init-key ::session
|
||||
[_ {:keys [pool] :as cfg}]
|
||||
(let [cfg (assoc cfg :conn pool)]
|
||||
(let [events (a/chan (a/dropping-buffer (:buffer-size cfg)))
|
||||
cfg (assoc cfg
|
||||
:conn pool
|
||||
::events-ch events)]
|
||||
(-> cfg
|
||||
(assoc :middleware #(middleware cfg %))
|
||||
(assoc :create (fn [profile-id]
|
||||
|
|
@ -89,3 +106,113 @@
|
|||
:body ""
|
||||
:cookies (cookies cfg {:value "" :max-age -1})))))))
|
||||
|
||||
(defmethod ig/halt-key! ::session
|
||||
[_ data]
|
||||
(a/close! (::events-ch data)))
|
||||
|
||||
;; --- STATE INIT: SESSION UPDATER
|
||||
|
||||
(declare batch-events)
|
||||
(declare update-sessions)
|
||||
|
||||
(s/def ::session map?)
|
||||
(s/def ::max-batch-age ::cfg/http-session-updater-batch-max-age)
|
||||
(s/def ::max-batch-size ::cfg/http-session-updater-batch-max-size)
|
||||
|
||||
(defmethod ig/pre-init-spec ::updater [_]
|
||||
(s/keys :req-un [::db/pool ::wrk/executor ::mtx/metrics ::session]
|
||||
:opt-un [::max-batch-age
|
||||
::max-batch-size]))
|
||||
|
||||
(defmethod ig/prep-key ::updater
|
||||
[_ cfg]
|
||||
(merge {:max-batch-age (dt/duration {:minutes 5})
|
||||
:max-batch-size 200}
|
||||
(d/without-nils cfg)))
|
||||
|
||||
(defmethod ig/init-key ::updater
|
||||
[_ {:keys [session metrics] :as cfg}]
|
||||
(log/infof "initialize session updater (max-batch-age=%s, max-batch-size=%s)"
|
||||
(str (:max-batch-age cfg))
|
||||
(str (:max-batch-size cfg)))
|
||||
(let [input (batch-events cfg (::events-ch session))
|
||||
mcnt (mtx/create
|
||||
{:name "http_session_updater_count"
|
||||
:help "A counter of session update batch events."
|
||||
:registry (:registry metrics)
|
||||
:type :counter})]
|
||||
(a/go-loop []
|
||||
(when-let [[reason batch] (a/<! input)]
|
||||
(let [result (a/<! (update-sessions cfg batch))]
|
||||
(mcnt :inc)
|
||||
(if (ex/exception? result)
|
||||
(log/error result "updater: unexpected error on update sessions")
|
||||
(log/debugf "updater: updated %s sessions (reason: %s)." result (name reason)))
|
||||
(recur))))))
|
||||
|
||||
(defn- timeout-chan
|
||||
[cfg]
|
||||
(a/timeout (inst-ms (:max-batch-age cfg))))
|
||||
|
||||
(defn- batch-events
|
||||
[cfg in]
|
||||
(let [out (a/chan)]
|
||||
(a/go-loop [tch (timeout-chan cfg)
|
||||
buf #{}]
|
||||
(let [[val port] (a/alts! [tch in])]
|
||||
(cond
|
||||
(identical? port tch)
|
||||
(if (empty? buf)
|
||||
(recur (timeout-chan cfg) buf)
|
||||
(do
|
||||
(a/>! out [:timeout buf])
|
||||
(recur (timeout-chan cfg) #{})))
|
||||
|
||||
(nil? val)
|
||||
(a/close! out)
|
||||
|
||||
(identical? port in)
|
||||
(let [buf (conj buf val)]
|
||||
(if (>= (count buf) (:max-batch-size cfg))
|
||||
(do
|
||||
(a/>! out [:size buf])
|
||||
(recur (timeout-chan cfg) #{}))
|
||||
(recur tch buf))))))
|
||||
out))
|
||||
|
||||
(defn- update-sessions
|
||||
[{:keys [pool executor]} ids]
|
||||
(aa/with-thread executor
|
||||
(db/exec-one! pool ["update http_session set updated_at=now() where id = ANY(?)"
|
||||
(into-array String ids)])
|
||||
(count ids)))
|
||||
|
||||
;; --- STATE INIT: SESSION GC
|
||||
|
||||
(declare sql:delete-expired)
|
||||
|
||||
(s/def ::max-age ::dt/duration)
|
||||
|
||||
(defmethod ig/pre-init-spec ::gc-task [_]
|
||||
(s/keys :req-un [::db/pool]
|
||||
:opt-un [::max-age]))
|
||||
|
||||
(defmethod ig/prep-key ::gc-task
|
||||
[_ cfg]
|
||||
(merge {:max-age (dt/duration {:days 2})}
|
||||
(d/without-nils cfg)))
|
||||
|
||||
(defmethod ig/init-key ::gc-task
|
||||
[_ {:keys [pool max-age] :as cfg}]
|
||||
(fn [_]
|
||||
(db/with-atomic [conn pool]
|
||||
(let [interval (db/interval max-age)
|
||||
result (db/exec-one! conn [sql:delete-expired interval])
|
||||
result (:next.jdbc/update-count result)]
|
||||
(log/debugf "gc-task: removed %s rows from http-session table" result)
|
||||
result))))
|
||||
|
||||
(def ^:private
|
||||
sql:delete-expired
|
||||
"delete from http_session
|
||||
where updated_at < now() - ?::interval")
|
||||
|
|
|
|||
|
|
@ -69,7 +69,19 @@
|
|||
|
||||
:app.http.session/session
|
||||
{:pool (ig/ref :app.db/pool)
|
||||
:cookie-name "auth-token"}
|
||||
:cookie-name (:http-session-cookie-name config)}
|
||||
|
||||
:app.http.session/gc-task
|
||||
{:pool (ig/ref :app.db/pool)
|
||||
:max-age (:http-session-idle-max-age config)}
|
||||
|
||||
:app.http.session/updater
|
||||
{:pool (ig/ref :app.db/pool)
|
||||
:metrics (ig/ref :app.metrics/metrics)
|
||||
:executor (ig/ref :app.worker/executor)
|
||||
:session (ig/ref :app.http.session/session)
|
||||
:max-batch-age (:http-session-updater-batch-max-age config)
|
||||
:max-batch-size (:http-session-updater-batch-max-size config)}
|
||||
|
||||
:app.http.awsns/handler
|
||||
{:tokens (ig/ref :app.tokens/tokens)
|
||||
|
|
@ -197,6 +209,10 @@
|
|||
:cron #app/cron "0 0 2 */1 * ?" ;; daily (2 hour shift)
|
||||
:task :storage-touched-gc}
|
||||
|
||||
{:id "session-gc"
|
||||
:cron #app/cron "0 0 3 */1 * ?" ;; daily (3 hour shift)
|
||||
:task :session-gc}
|
||||
|
||||
{:id "storage-recheck"
|
||||
:cron #app/cron "0 0 */1 * * ?" ;; hourly
|
||||
:task :storage-recheck}
|
||||
|
|
@ -223,7 +239,8 @@
|
|||
:storage-touched-gc (ig/ref :app.storage/gc-touched-task)
|
||||
:storage-recheck (ig/ref :app.storage/recheck-task)
|
||||
:tasks-gc (ig/ref :app.tasks.tasks-gc/handler)
|
||||
:telemetry (ig/ref :app.tasks.telemetry/handler)}}
|
||||
:telemetry (ig/ref :app.tasks.telemetry/handler)
|
||||
:session-gc (ig/ref :app.http.session/gc-task)}}
|
||||
|
||||
:app.tasks.sendmail/handler
|
||||
{:host (:smtp-host config)
|
||||
|
|
|
|||
|
|
@ -158,6 +158,8 @@
|
|||
{:name "0048-mod-storage-tables"
|
||||
:fn (mg/resource "app/migrations/sql/0048-mod-storage-tables.sql")}
|
||||
|
||||
{:name "0049-mod-http-session-table"
|
||||
:fn (mg/resource "app/migrations/sql/0049-mod-http-session-table.sql")}
|
||||
])
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,6 @@
|
|||
ALTER TABLE http_session
|
||||
ADD COLUMN updated_at timestamptz NULL;
|
||||
|
||||
CREATE INDEX http_session__updated_at__idx
|
||||
ON http_session (updated_at)
|
||||
WHERE updated_at IS NOT NULL;
|
||||
Loading…
Add table
Reference in a new issue