Penpot/penpot
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-01-08 07:50:43 -05:00
Code Issues Activity

🎉 Add sentry integration (on backend).

Browse source
This commit is contained in:
Andrey Antukh 2021-09-15 14:10:43 +02:00 committed by Andrés Moya
parent 835b597af5
commit 26b28e2364
10 changed files with 399 additions and 136 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
backend/deps.edn
View file

@ -46,6 +46,8 @@
org.clojars.pntblnk/clj-ldap {:mvn/version "0.0.17"}
integrant/integrant {:mvn/version "0.8.0"}
io.sentry/sentry {:mvn/version "5.1.2"}
software.amazon.awssdk/s3 {:mvn/version "2.17.40"}}
:paths ["src" "resources"]

9
backend/src/app/config.clj
View file

@ -176,6 +176,11 @@
(s/def ::telemetry-with-taiga ::us/boolean)
(s/def ::tenant ::us/string)
(s/def ::sentry-trace-sample-rate ::us/number)
(s/def ::sentry-attach-stack-trace ::us/boolean)
(s/def ::sentry-debug ::us/boolean)
(s/def ::sentry-dsn ::us/string)
(s/def ::config
(s/keys :opt-un [::secret-key
::flags
@ -235,6 +240,10 @@
::registration-enabled
::rlimits-image
::rlimits-password
::sentry-dsn
::sentry-debug
::sentry-attach-stack-trace
::sentry-trace-sample-rate
::smtp-default-from
::smtp-default-reply-to
::smtp-enabled

60
backend/src/app/http/errors.clj
View file

@ -7,31 +7,49 @@
(ns app.http.errors
"A errors handling for the http server."
(:require
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.uuid :as uuid]
[app.util.logging :as l]
[cuerdas.core :as str]
[expound.alpha :as expound]))
[clojure.pprint]
[cuerdas.core :as str]))
(defn- explain-error
[error]
(with-out-str
(expound/printer (:data error))))
(defn- parse-client-ip
[{:keys [headers] :as request}]
(or (some-> (get headers "x-forwarded-for") (str/split ",") first)
(get headers "x-real-ip")
(get request :remote-addr)))
(defn- stringify-data
[data]
(binding [clojure.pprint/*print-right-margin* 200]
(let [result (with-out-str (clojure.pprint/pprint data))]
(str/prune result (* 1024 1024) "[...]"))))
(defn get-error-context
[request error]
(let [edata (ex-data error)]
(merge
{:id (uuid/next)
:path (:uri request)
:method (:request-method request)
:params (:params request)
:data edata}
(let [data (ex-data error)]
(d/without-nils
(merge
{:id (str (uuid/next))
:path (str (:uri request))
:method (name (:request-method request))
:hint (or (:hint data) (ex-message error))
:params (stringify-data (:params request))
:data (stringify-data (dissoc data :explain))
:ip-addr (parse-client-ip request)
:explain (str/prune (:explain data) (* 1024 1024) "[...]")}
(when-let [id (:profile-id request)]
{:profile-id id})
(let [headers (:headers request)]
{:user-agent (get headers "user-agent")
:frontend-version (get headers "x-frontend-version" "unknown")})
(when (and (map? edata) (:data edata))
{:explain (explain-error edata)}))))
(when (map? data)
{:error-type (:type data)
:error-code (:code data)})))))
(defmulti handle-exception
(fn [err & _rest]
@ -43,7 +61,6 @@
[err _]
{:status 401 :body (ex-data err)})
(defmethod handle-exception :restriction
[err _]
{:status 400 :body (ex-data err)})
@ -57,13 +74,10 @@
{:status 400
:headers {"content-type" "text/html"}
:body (str "<pre style='font-size:16px'>"
(explain-error edata)
(:explain edata)
"</pre>\n")}
{:status 400
:body (cond-> edata
(map? (:data edata))
(-> (assoc :explain (explain-error edata))
(dissoc :data)))})))
:body (dissoc edata :data)})))
(defmethod handle-exception :assertion
[error request]
@ -77,9 +91,7 @@
{:status 500
:body {:type :server-error
:code :assertion
:data (-> edata
(assoc :explain (explain-error edata))
(dissoc :data))}}))
:data (dissoc edata :data)}}))
(defmethod handle-exception :not-found
[err _]

125
backend/src/app/loggers/database.clj Normal file
View file

@ -0,0 +1,125 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) UXBOX Labs SL
(ns app.loggers.database
"A specific logger impl that persists errors on the database."
(:require
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.util.async :as aa]
[app.util.logging :as l]
[app.util.template :as tmpl]
[app.worker :as wrk]
[clojure.core.async :as a]
[clojure.java.io :as io]
[clojure.spec.alpha :as s]
[cuerdas.core :as str]
[integrant.core :as ig]))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Error Listener
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(declare handle-event)
(defonce enabled (atom true))
(defn- persist-on-database!
[{:keys [pool] :as cfg} {:keys [id] :as event}]
(db/with-atomic [conn pool]
(db/insert! conn :server-error-report
{:id id :content (db/tjson event)})))
(defn- parse-context
[event]
(reduce-kv
(fn [acc k v]
(cond
(= k :id) (assoc acc k (uuid/uuid v))
(= k :profile-id) (assoc acc k (uuid/uuid v))
(str/blank? v) acc
:else (assoc acc k v)))
{}
(:context event)))
(defn parse-event
[event]
(-> (parse-context event)
(merge (dissoc event :context))
(assoc :tenant (cf/get :tenant))
(assoc :host (cf/get :host))
(assoc :public-uri (cf/get :public-uri))
(assoc :version (:full cf/version))))
(defn handle-event
[{:keys [executor] :as cfg} event]
(aa/with-thread executor
(try
(let [event (parse-event event)]
(persist-on-database! cfg event))
(catch Exception e
(l/warn :hint "unexpected exception on database error logger"
:cause e)))))
(defmethod ig/pre-init-spec ::reporter [_]
(s/keys :req-un [::wrk/executor ::db/pool ::receiver]))
(defmethod ig/init-key ::reporter
[_ {:keys [receiver] :as cfg}]
(l/info :msg "initializing database error persistence")
(let [output (a/chan (a/sliding-buffer 128)
(filter #(= (:level %) "error")))]
(receiver :sub output)
(a/go-loop []
(let [msg (a/<! output)]
(if (nil? msg)
(l/info :msg "stoping error reporting loop")
(do
(a/<! (handle-event cfg msg))
(recur)))))
output))
(defmethod ig/halt-key! ::reporter
[_ output]
(a/close! output))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Http Handler
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defmethod ig/pre-init-spec ::handler [_]
(s/keys :req-un [::db/pool]))
(defmethod ig/init-key ::handler
[_ {:keys [pool] :as cfg}]
(letfn [(parse-id [request]
(let [id (get-in request [:path-params :id])
id (us/uuid-conformer id)]
(when (uuid? id)
id)))
(retrieve-report [id]
(ex/ignoring
(when-let [{:keys [content] :as row} (db/get-by-id pool :server-error-report id)]
(assoc row :content (db/decode-transit-pgobject content)))))
(render-template [{:keys [content] :as report}]
(some-> (io/resource "error-report.tmpl")
(tmpl/render content)))]
(fn [request]
(let [result (some-> (parse-id request)
(retrieve-report)
(render-template))]
(if result
{:status 200
:headers {"content-type" "text/html; charset=utf-8"}
:body result}
{:status 404
:body "not found"})))))

137
backend/src/app/loggers/mattermost.clj
View file

@ -7,32 +7,51 @@
(ns app.loggers.mattermost
"A mattermost integration for error reporting."
(:require
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.common.uuid :as uuid]
[app.config :as cfg]
[app.config :as cf]
[app.db :as db]
[app.loggers.database :as ldb]
[app.util.async :as aa]
[app.util.http :as http]
[app.util.json :as json]
[app.util.logging :as l]
[app.util.template :as tmpl]
[app.worker :as wrk]
[clojure.core.async :as a]
[clojure.java.io :as io]
[clojure.spec.alpha :as s]
[cuerdas.core :as str]
[integrant.core :as ig]))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Error Listener
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defonce enabled (atom true))
(declare handle-event)
(defn- send-mattermost-notification!
[cfg {:keys [host id public-uri] :as event}]
(try
(let [uri (:uri cfg)
text (str "Exception on (host: " host ", url: " public-uri "/dbg/error-by-id/" id ")\n"
(when-let [pid (:profile-id event)]
(str "- profile-id: #uuid-" pid "\n")))
rsp (http/send! {:uri uri
:method :post
:headers {"content-type" "application/json"}
:body (json/encode-str {:text text})})]
(when (not= (:status rsp) 200)
(l/error :hint "error on sending data to mattermost"
:response (pr-str rsp))))
(defonce enabled-mattermost (atom true))
(catch Exception e
(l/error :hint "unexpected exception on error reporter"
:cause e))))
(s/def ::uri ::us/string)
(defn handle-event
[{:keys [executor] :as cfg} event]
(aa/with-thread executor
(try
(let [event (ldb/parse-event event)]
(when @enabled
(send-mattermost-notification! cfg event)))
(catch Exception e
(l/warn :hint "unexpected exception on error reporter" :cause e)))))
(s/def ::uri ::cf/error-report-webhook)
(defmethod ig/pre-init-spec ::reporter [_]
(s/keys :req-un [::wrk/executor ::db/pool ::receiver]
@ -58,95 +77,3 @@
[_ output]
(when output
(a/close! output)))
(defn- send-mattermost-notification!
[cfg {:keys [host id] :as cdata}]
(try
(let [uri (:uri cfg)
text (str "Unhandled exception (host: " host ", url: " (cfg/get :public-uri) "/dbg/error-by-id/" id "\n"
"- profile-id: #" (:profile-id cdata) "\n")
rsp (http/send! {:uri uri
:method :post
:headers {"content-type" "application/json"}
:body (json/encode-str {:text text})})]
(when (not= (:status rsp) 200)
(l/error :hint "error on sending data to mattermost"
:response (pr-str rsp))))
(catch Exception e
(l/error :hint "unexpected exception on error reporter"
:cause e))))
(defn- persist-on-database!
[{:keys [pool] :as cfg} {:keys [id] :as cdata}]
(db/with-atomic [conn pool]
(db/insert! conn :server-error-report
{:id id :content (db/tjson cdata)})))
(defn- parse-context
[event]
(reduce-kv
(fn [acc k v]
(cond
(= k :id) (assoc acc k (uuid/uuid v))
(= k :profile-id) (assoc acc k (uuid/uuid v))
(str/blank? v) acc
:else (assoc acc k v)))
{:id (uuid/next)}
(:context event)))
(defn- parse-event
[event]
(-> (parse-context event)
(merge (dissoc event :context))
(assoc :tenant (cfg/get :tenant))
(assoc :host (cfg/get :host))
(assoc :public-uri (cfg/get :public-uri))
(assoc :version (:full cfg/version))))
(defn handle-event
[{:keys [executor] :as cfg} event]
(aa/with-thread executor
(try
(let [cdata (parse-event event)]
(when @enabled-mattermost
(send-mattermost-notification! cfg cdata))
(persist-on-database! cfg cdata))
(catch Exception e
(l/error :hint "unexpected exception on error reporter"
:cause e)))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Http Handler
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defmethod ig/pre-init-spec ::handler [_]
(s/keys :req-un [::db/pool]))
(defmethod ig/init-key ::handler
[_ {:keys [pool] :as cfg}]
(letfn [(parse-id [request]
(let [id (get-in request [:path-params :id])
id (us/uuid-conformer id)]
(when (uuid? id)
id)))
(retrieve-report [id]
(ex/ignoring
(when-let [{:keys [content] :as row} (db/get-by-id pool :server-error-report id)]
(assoc row :content (db/decode-transit-pgobject content)))))
(render-template [{:keys [content] :as report}]
(some-> (io/resource "error-report.tmpl")
(tmpl/render content)))]
(fn [request]
(let [result (some-> (parse-id request)
(retrieve-report)
(render-template))]
(if result
{:status 200
:headers {"content-type" "text/html; charset=utf-8"}
:body result}
{:status 404
:body "not found"})))))

172
backend/src/app/loggers/sentry.clj Normal file
View file

@ -0,0 +1,172 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) UXBOX Labs SL
(ns app.loggers.sentry
"A mattermost integration for error reporting."
(:require
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.util.async :as aa]
[app.util.logging :as l]
[app.worker :as wrk]
[clojure.core.async :as a]
[clojure.spec.alpha :as s]
[cuerdas.core :as str]
[integrant.core :as ig])
(:import
io.sentry.Scope
io.sentry.IHub
io.sentry.Hub
io.sentry.NoOpHub
io.sentry.protocol.User
io.sentry.SentryOptions
io.sentry.SentryLevel
io.sentry.ScopeCallback))
(defonce enabled (atom true))
(defn- parse-context
[event]
(reduce-kv
(fn [acc k v]
(cond
(= k :id) (assoc acc k (uuid/uuid v))
(= k :profile-id) (assoc acc k (uuid/uuid v))
(str/blank? v) acc
:else (assoc acc k v)))
{}
(:context event)))
(defn- parse-event
[event]
(assoc event :context (parse-context event)))
(defn- build-sentry-options
[cfg]
(let [version (:base cf/version)]
(doto (SentryOptions.)
(.setDebug (:debug cfg false))
(.setTracesSampleRate (:traces-sample-rate cfg 1.0))
(.setDsn (:dsn cfg))
(.setServerName (cf/get :host))
(.setEnvironment (cf/get :tenant))
(.setAttachServerName true)
(.setAttachStacktrace (:attach-stack-trace cfg false))
(.setRelease (str "backend@" (if (= version "0.0.0") "develop" version))))))
(defn handle-event
[^IHub shub event]
(letfn [(set-user! [^Scope scope {:keys [context] :as event}]
(let [user (User.)]
(.setIpAddress ^User user ^String (:ip-addr context))
(when-let [pid (:profile-id context)]
(.setId ^User user ^String (str pid)))
(.setUser scope ^User user)))
(set-level! [^Scope scope]
(.setLevel scope SentryLevel/ERROR))
(set-context! [^Scope scope {:keys [context] :as event}]
(let [uri (str (cf/get :public-uri) "/dbg/error-by-id/" (:id context))]
(.setContexts scope "detailed_error_uri" ^String uri))
(when-let [vers (:frontend-version event)]
(.setContexts scope "frontend_version" ^String vers))
(when-let [puri (:public-uri event)]
(.setContexts scope "public_uri" ^String (str puri)))
(when-let [uagent (:user-agent context)]
(.setContexts scope "user_agent" ^String uagent))
(when-let [tenant (:tenant event)]
(.setTag scope "tenant" ^String tenant))
(when-let [type (:error-type context)]
(.setTag scope "error_type" ^String (str type)))
(when-let [code (:error-code context)]
(.setTag scope "error_code" ^String (str code)))
)
(capture [^Scope scope {:keys [context error] :as event}]
(let [msg (str (:message error) "\n\n"
"======================================================\n"
"=================== Params ===========================\n"
"======================================================\n"
(:params context) "\n"
(when (:explain context)
(str "======================================================\n"
"=================== Explain ==========================\n"
"======================================================\n"
(:explain context) "\n"))
(when (:data context)
(str "======================================================\n"
"=================== Error Data =======================\n"
"======================================================\n"
(:data context) "\n"))
(str "======================================================\n"
"=================== Stack Trace ======================\n"
"======================================================\n"
(:trace error))
"\n")]
(set-user! scope event)
(set-level! scope)
(set-context! scope event)
(.captureMessage ^IHub shub msg)
))
]
;; (clojure.pprint/pprint event)
(when @enabled
(.withScope ^IHub shub (reify ScopeCallback
(run [_ scope]
(->> event
(parse-event)
(capture scope))))))
))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Error Listener
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(s/def ::receiver any?)
(s/def ::dsn ::cf/sentry-dsn)
(s/def ::trace-sample-rate ::cf/sentry-trace-sample-rate)
(s/def ::attach-stack-trace ::cf/sentry-attach-stack-trace)
(s/def ::debug ::cf/sentry-debug)
(defmethod ig/pre-init-spec ::reporter [_]
(s/keys :req-un [::wrk/executor ::db/pool ::receiver]
:opt-un [::dsn ::trace-sample-rate ::attach-stack-trace]))
(defmethod ig/init-key ::reporter
[_ {:keys [receiver dsn executor] :as cfg}]
(l/info :msg "initializing sentry reporter" :dsn dsn)
(let [opts (build-sentry-options cfg)
shub (if dsn
(Hub. ^SentryOptions opts)
(NoOpHub/getInstance))
output (a/chan (a/sliding-buffer 128)
(filter #(= (:level %) "error")))]
(receiver :sub output)
(a/go-loop []
(let [event (a/<! output)]
(if (nil? event)
(do
(l/info :msg "stoping error reporting loop")
(.close ^IHub shub))
(do
(a/<! (aa/with-thread executor (handle-event shub event)))
(recur)))))
output))
(defmethod ig/halt-key! ::reporter
[_ output]
(when output
(a/close! output)))

3
backend/src/app/loggers/zmq.clj
View file

@ -7,7 +7,6 @@
(ns app.loggers.zmq
"A generic ZMQ listener."
(:require
[app.common.data :as d]
[app.common.spec :as us]
[app.util.json :as json]
[app.util.logging :as l]
@ -74,7 +73,7 @@
(defn- prepare
[event]
(d/merge
(merge
{:logger (:loggerName event)
:level (str/lower (:level event))
:thread (:thread event)

18
backend/src/app/main.clj
View file

@ -109,7 +109,7 @@
:sns-webhook (ig/ref :app.http.awsns/handler)
:feedback (ig/ref :app.http.feedback/handler)
:audit-http-handler (ig/ref :app.loggers.audit/http-handler)
:error-report-handler (ig/ref :app.loggers.mattermost/handler)}
:error-report-handler (ig/ref :app.loggers.database/handler)}
:app.http.assets/handlers
{:metrics (ig/ref :app.metrics/metrics)
@ -331,9 +331,23 @@
:pool (ig/ref :app.db/pool)
:executor (ig/ref :app.worker/executor)}
:app.loggers.mattermost/handler
:app.loggers.database/reporter
{:receiver (ig/ref :app.loggers.zmq/receiver)
:pool (ig/ref :app.db/pool)
:executor (ig/ref :app.worker/executor)}
:app.loggers.database/handler
{:pool (ig/ref :app.db/pool)}
:app.loggers.sentry/reporter
{:dsn (cf/get :sentry-dsn)
:trace-sample-rate (cf/get :sentry-trace-sample-rate 1.0)
:attach-stack-trace (cf/get :sentry-attach-stack-trace false)
:debug (cf/get :sentry-debug false)
:receiver (ig/ref :app.loggers.zmq/receiver)
:pool (ig/ref :app.db/pool)
:executor (ig/ref :app.worker/executor)}
:app.storage/storage
{:pool (ig/ref :app.db/pool)
:executor (ig/ref :app.worker/executor)

5
common/src/app/common/exceptions.cljc
View file

@ -27,7 +27,10 @@
[& {:keys [message hint cause] :as params}]
(s/assert ::error-params params)
(let [message (or message hint "")
payload (dissoc params :cause)]
payload (-> params
(dissoc :cause)
(dissoc :message)
(assoc :hint message))]
(ex-info message payload cause)))
(defmacro raise

4
common/src/app/common/spec.cljc
View file

@ -206,7 +206,7 @@
:name (pr-str spec)
:line (:line &env)
:file (:file (:meta nsdata))})
message (str "Spec Assertion: '" (pr-str spec) "'")]
message (str "spec assert: '" (pr-str spec) "'")]
`(spec-assert* ~spec ~x ~message ~context))))
(defmacro verify
@ -218,7 +218,7 @@
:name (pr-str spec)
:line (:line &env)
:file (:file (:meta nsdata))})
message (str "Spec Assertion: '" (pr-str spec) "'")]
message (str "spec verify: '" (pr-str spec) "'")]
`(spec-assert* ~spec ~x ~message ~context)))
;; --- Public Api