From f43fc282d3ac1cc3960277c0d65fb37fbc189d46 Mon Sep 17 00:00:00 2001
From: Andrey Antukh <niwi@niwi.nz>
Date: Mon, 28 Oct 2024 13:41:58 +0100
Subject: [PATCH] :sparkles: Increase internal s3 http client limits

Tries to improve performance of accidental spikes/bursts of
requests to s3 service. This is not a final solution to all issues
caused by unexpected burst, is a simple improvement to the current
apprach.
---
 backend/src/app/storage/s3.clj | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/backend/src/app/storage/s3.clj b/backend/src/app/storage/s3.clj
index 9ddbb0ba5..2adde671f 100644
--- a/backend/src/app/storage/s3.clj
+++ b/backend/src/app/storage/s3.clj
@@ -62,6 +62,16 @@
   "A maximum number of retries on internal operations"
   3)
 
+(def ^:private max-concurrency
+  "Maximum concurrent request to S3 service"
+  128)
+
+(def ^:private max-pending-connection-acquires
+  20000)
+
+(def default-timeout
+  (dt/duration {:seconds 30}))
+
 (declare put-object)
 (declare get-object-bytes)
 (declare get-object-data)
@@ -177,9 +187,6 @@
 
 ;; --- HELPERS
 
-(def default-timeout
-  (dt/duration {:seconds 30}))
-
 (defn- lookup-region
   ^Region
   [region]
@@ -203,6 +210,8 @@
                      (.connectionTimeout default-timeout)
                      (.readTimeout default-timeout)
                      (.writeTimeout default-timeout)
+                     (.maxConcurrency (int max-concurrency))
+                     (.maxPendingConnectionAcquires (int max-pending-connection-acquires))
                      (.build))
 
         client   (let [builder (S3AsyncClient/builder)