Penpot/penpot
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-02-16 03:58:20 -05:00
Code Issues Activity

✨ Stream transit encoding to the response output-stream.

Browse source 
Instead of in-memmory encoding. This will prevent many OOM errors.
This commit is contained in:
Andrey Antukh 2022-01-17 23:39:26 +01:00 • committed by Alonso Torres
parent 9cf5258053
commit 7afb3e2c6d
2 changed files with 42 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
backend/src/app/http.clj
View file

@ -141,11 +141,11 @@
:get ws}] :get ws}]
["/api" {:middleware [[middleware/cors] ["/api" {:middleware [[middleware/cors]
[middleware/etag]
[middleware/params] [middleware/params]
[middleware/multipart-params] [middleware/multipart-params]
[middleware/keyword-params] [middleware/keyword-params]
[middleware/format-response-body] [middleware/format-response-body]
[middleware/etag]
[middleware/parse-request-body] [middleware/parse-request-body]
[middleware/errors errors/handle] [middleware/errors errors/handle]
[middleware/cookies]]} [middleware/cookies]]}

64
backend/src/app/http/middleware.clj
View file

@ -12,10 +12,12 @@
[app.util.json :as json] [app.util.json :as json]
[buddy.core.codecs :as bc] [buddy.core.codecs :as bc]
[buddy.core.hash :as bh] [buddy.core.hash :as bh]
[ring.core.protocols :as rp]
[ring.middleware.cookies :refer [wrap-cookies]] [ring.middleware.cookies :refer [wrap-cookies]]
[ring.middleware.keyword-params :refer [wrap-keyword-params]] [ring.middleware.keyword-params :refer [wrap-keyword-params]]
[ring.middleware.multipart-params :refer [wrap-multipart-params]] [ring.middleware.multipart-params :refer [wrap-multipart-params]]
[ring.middleware.params :refer [wrap-params]])) [ring.middleware.params :refer [wrap-params]]
[yetti.adapter :as yt]))
(defn wrap-server-timing (defn wrap-server-timing
[handler] [handler]
@ -65,11 +67,33 @@
{:name ::parse-request-body {:name ::parse-request-body
:compile (constantly wrap-parse-request-body)}) :compile (constantly wrap-parse-request-body)})
(defn buffered-output-stream
"Returns a buffered output stream that ignores flush calls. This is
needed because transit-java calls flush very aggresivelly on each
object write."
[^java.io.OutputStream os ^long chunk-size]
(proxy [java.io.BufferedOutputStream] [os (int chunk-size)]
;; Explicitly do not forward flush
(flush [])
(close []
(proxy-super flush)
(proxy-super close))))
(def ^:const buffer-size (:http/output-buffer-size yt/base-defaults))
(defn- transit-streamable-body
[data opts]
(reify rp/StreamableResponseBody
(write-body-to-stream [_ _ output-stream]
;; Use the same buffer as jetty output buffer size
(with-open [bos (buffered-output-stream output-stream buffer-size)]
(let [tw (t/writer bos opts)]
(t/write! tw data))))))
(defn- impl-format-response-body (defn- impl-format-response-body
[response request] [response {:keys [query-params] :as request}]
(let [body (:body response) (let [body (:body response)
params (:query-params request) opts {:type (if (contains? query-params "transit_verbose") :json-verbose :json)}]
opts {:type (if (contains? params "transit_verbose") :json-verbose :json)}]
(cond (cond
(:ws response) (:ws response)
@ -78,7 +102,7 @@
(coll? body) (coll? body)
(-> response (-> response
(update :headers assoc "content-type" "application/transit+json") (update :headers assoc "content-type" "application/transit+json")
(assoc :body (t/encode body opts))) (assoc :body (transit-streamable-body body opts)))
(nil? body) (nil? body)
(assoc response :status 204 :body "") (assoc response :status 204 :body "")
@ -131,24 +155,18 @@
(defn wrap-etag (defn wrap-etag
[handler] [handler]
(letfn [(generate-etag [{:keys [body] :as response}] (letfn [(encode [data]
(str "W/\"" (-> body bh/blake2b-128 bc/bytes->hex) "\"")) (when (string? data)
(get-match [{:keys [headers] :as request}] (str "W/\"" (-> data bh/blake2b-128 bc/bytes->hex) "\"")))]
(get headers "if-none-match"))] (fn [{method :request-method headers :headers :as request}]
(fn [request] (cond-> (handler request)
(let [response (handler request)] (= :get method)
(if (= :get (:request-method request)) (as-> $ (if-let [etag (-> $ :body meta :etag encode)]
(let [etag (generate-etag response) (cond-> (update $ :headers assoc "etag" etag)
match (get-match request) (= etag (get headers "if-none-match"))
response (update response :headers #(assoc % "ETag" etag))] (-> (assoc :body "")
(cond-> response (assoc :status 304)))
(and (string? match) $))))))
(= :get (:request-method request))
(= etag match))
(-> response
(assoc :body "")
(assoc :status 304))))
response)))))
(def etag (def etag
{:name ::etag {:name ::etag