Penpot/penpot
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-01-22 14:39:45 -05:00
Code Issues Activity

🐛 Fix s3 client issues with s3 compatible services

Browse source
This commit is contained in:
Andrey Antukh 2022-12-31 16:37:42 +01:00
parent 27451b9796
commit e372e8ba3e
2 changed files with 49 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
backend/deps.edn
View file

@ -55,7 +55,8 @@
;; Pretty Print specs
pretty-spec/pretty-spec {:mvn/version "0.1.4"}
software.amazon.awssdk/s3 {:mvn/version "2.19.5"}}
software.amazon.awssdk/s3 {:mvn/version "2.19.8"}
}
:paths ["src" "resources" "target/classes"]
:aliases

72
backend/src/app/storage/s3.clj
View file

@ -24,7 +24,9 @@
(:import
java.io.FilterInputStream
java.io.InputStream
java.net.URI
java.nio.ByteBuffer
java.nio.file.Path
java.time.Duration
java.util.Collection
java.util.Optional
@ -40,6 +42,7 @@
software.amazon.awssdk.http.nio.netty.SdkEventLoopGroup
software.amazon.awssdk.regions.Region
software.amazon.awssdk.services.s3.S3AsyncClient
software.amazon.awssdk.services.s3.S3Configuration
software.amazon.awssdk.services.s3.model.Delete
software.amazon.awssdk.services.s3.model.DeleteObjectRequest
software.amazon.awssdk.services.s3.model.DeleteObjectsRequest
@ -151,46 +154,51 @@
(defn build-s3-client
[{:keys [region endpoint executor]}]
(let [hclient (.. (NettyNioAsyncHttpClient/builder)
(eventLoopGroupBuilder (.. (SdkEventLoopGroup/builder)
(numberOfThreads (int default-eventloop-threads))))
(connectionAcquisitionTimeout default-timeout)
(connectionTimeout default-timeout)
(readTimeout default-timeout)
(writeTimeout default-timeout)
(build))
client (.. (S3AsyncClient/builder)
(asyncConfiguration (.. (ClientAsyncConfiguration/builder)
(advancedOption SdkAdvancedAsyncClientOption/FUTURE_COMPLETION_EXECUTOR
executor)
(build)))
(httpClient hclient)
(region (lookup-region region)))]
(let [aconfig (-> (ClientAsyncConfiguration/builder)
(.advancedOption SdkAdvancedAsyncClientOption/FUTURE_COMPLETION_EXECUTOR executor)
(.build))
(when-let [uri (some-> endpoint (java.net.URI.))]
(.endpointOverride client uri))
sconfig (-> (S3Configuration/builder)
(cond-> (some? endpoint) (.pathStyleAccessEnabled true))
(.build))
hclient (-> (NettyNioAsyncHttpClient/builder)
(.eventLoopGroupBuilder (-> (SdkEventLoopGroup/builder)
(.numberOfThreads (int default-eventloop-threads))))
(.connectionAcquisitionTimeout default-timeout)
(.connectionTimeout default-timeout)
(.readTimeout default-timeout)
(.writeTimeout default-timeout)
(.build))
client (-> (S3AsyncClient/builder)
(.serviceConfiguration ^S3Configuration sconfig)
(.asyncConfiguration ^ClientAsyncConfiguration aconfig)
(.httpClient ^NettyNioAsyncHttpClient hclient)
(.region (lookup-region region))
(cond-> (some? endpoint) (.endpointOverride (URI. endpoint)))
(.build))]
(let [client (.build client)]
(reify
clojure.lang.IDeref
(deref [_] client)
java.lang.AutoCloseable
(close [_]
(.close hclient)
(.close client))))))
(.close ^NettyNioAsyncHttpClient hclient)
(.close ^S3AsyncClient client)))))
(defn build-s3-presigner
[{:keys [region endpoint]}]
(if (string? endpoint)
(let [uri (java.net.URI. endpoint)]
(.. (S3Presigner/builder)
(endpointOverride uri)
(region (lookup-region region))
(build)))
(.. (S3Presigner/builder)
(region (lookup-region region))
(build))))
(let [config (-> (S3Configuration/builder)
(cond-> (some? endpoint) (.pathStyleAccessEnabled true))
(.build))]
(-> (S3Presigner/builder)
(cond-> (some? endpoint) (.endpointOverride (URI. endpoint)))
(.region (lookup-region region))
(.serviceConfiguration ^S3Configuration config)
(.build))))
(defn- make-request-body
[content]
@ -198,7 +206,7 @@
buff-size (* 1024 64)
sem (Semaphore. 0)
writer-fn (fn [s]
writer-fn (fn [^Subscriber s]
(try
(loop []
(.acquire sem 1)
@ -261,7 +269,7 @@
;; not, read the contento into memory using bytearrays.
(if (> size (* 1024 1024 2))
(p/let [path (tmp/tempfile :prefix "penpot.storage.s3.")
rxf (AsyncResponseTransformer/toFile path)
rxf (AsyncResponseTransformer/toFile ^Path path)
_ (.getObject ^S3AsyncClient client
^GetObjectRequest gor
^AsyncResponseTransformer rxf)]
@ -283,7 +291,7 @@
(key (str prefix (impl/id->path id)))
(build))
rxf (AsyncResponseTransformer/toBytes)
obj (.getObjectAsBytes ^S3AsyncClient client
obj (.getObject ^S3AsyncClient client
^GetObjectRequest gor
^AsyncResponseTransformer rxf)]
(.asByteArray ^ResponseBytes obj)))