3pp-mirror/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-05-25 16:06:11 +02:00
Code Issues Projects Releases Packages Wiki Activity

✨ Make the s3 client io-threads configurable and adaptable

Browse source
This commit is contained in:
Andrey Antukh 2023-11-15 13:26:52 +01:00
parent ed1c7dcc12
commit 4c190e385e
3 changed files with 12 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

2
backend/src/app/config.clj
View file

@ -203,6 +203,7 @@
(s/def ::storage-assets-s3-bucket ::us/string) (s/def ::storage-assets-s3-bucket ::us/string)
(s/def ::storage-assets-s3-region ::us/keyword) (s/def ::storage-assets-s3-region ::us/keyword)
(s/def ::storage-assets-s3-endpoint ::us/string) (s/def ::storage-assets-s3-endpoint ::us/string)
(s/def ::storage-assets-s3-io-threads ::us/integer)
(s/def ::telemetry-uri ::us/string) (s/def ::telemetry-uri ::us/string)
(s/def ::telemetry-with-taiga ::us/boolean) (s/def ::telemetry-with-taiga ::us/boolean)
(s/def ::tenant ::us/string) (s/def ::tenant ::us/string)
@ -320,6 +321,7 @@
::storage-assets-s3-bucket ::storage-assets-s3-bucket
::storage-assets-s3-region ::storage-assets-s3-region
::storage-assets-s3-endpoint ::storage-assets-s3-endpoint
::storage-assets-s3-io-threads
::telemetry-enabled ::telemetry-enabled
::telemetry-uri ::telemetry-uri
::telemetry-referer ::telemetry-referer

8
backend/src/app/main.clj
View file

@ -450,10 +450,10 @@
:assets-fs (ig/ref [::assets :app.storage.fs/backend])}} :assets-fs (ig/ref [::assets :app.storage.fs/backend])}}
[::assets :app.storage.s3/backend] [::assets :app.storage.s3/backend]
{::sto.s3/region (cf/get :storage-assets-s3-region) {::sto.s3/region (cf/get :storage-assets-s3-region)
::sto.s3/endpoint (cf/get :storage-assets-s3-endpoint) ::sto.s3/endpoint (cf/get :storage-assets-s3-endpoint)
::sto.s3/bucket (cf/get :storage-assets-s3-bucket) ::sto.s3/bucket (cf/get :storage-assets-s3-bucket)
::wrk/executor (ig/ref ::wrk/executor)} ::sto.s3/io-threads (cf/get :storage-assets-s3-io-threads)}
[::assets :app.storage.fs/backend] [::assets :app.storage.fs/backend]
{::sto.fs/directory (cf/get :storage-assets-fs-directory)} {::sto.fs/directory (cf/get :storage-assets-fs-directory)}

15
backend/src/app/storage/s3.clj
View file

@ -17,7 +17,6 @@
[app.storage.impl :as impl] [app.storage.impl :as impl]
[app.storage.tmp :as tmp] [app.storage.tmp :as tmp]
[app.util.time :as dt] [app.util.time :as dt]
[app.worker :as wrk]
[clojure.java.io :as io] [clojure.java.io :as io]
[clojure.spec.alpha :as s] [clojure.spec.alpha :as s]
[datoteka.fs :as fs] [datoteka.fs :as fs]
@ -40,7 +39,6 @@
software.amazon.awssdk.core.async.AsyncRequestBody software.amazon.awssdk.core.async.AsyncRequestBody
software.amazon.awssdk.core.async.AsyncResponseTransformer software.amazon.awssdk.core.async.AsyncResponseTransformer
software.amazon.awssdk.core.client.config.ClientAsyncConfiguration software.amazon.awssdk.core.client.config.ClientAsyncConfiguration
software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption
software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient
software.amazon.awssdk.http.nio.netty.SdkEventLoopGroup software.amazon.awssdk.http.nio.netty.SdkEventLoopGroup
software.amazon.awssdk.regions.Region software.amazon.awssdk.regions.Region
@ -77,9 +75,10 @@
(s/def ::bucket ::us/string) (s/def ::bucket ::us/string)
(s/def ::prefix ::us/string) (s/def ::prefix ::us/string)
(s/def ::endpoint ::us/string) (s/def ::endpoint ::us/string)
(s/def ::io-threads ::us/integer)
(defmethod ig/pre-init-spec ::backend [_] (defmethod ig/pre-init-spec ::backend [_]
(s/keys :opt [::region ::bucket ::prefix ::endpoint ::wrk/executor])) (s/keys :opt [::region ::bucket ::prefix ::endpoint ::io-threads]))
(defmethod ig/prep-key ::backend (defmethod ig/prep-key ::backend
[_ {:keys [::prefix ::region] :as cfg}] [_ {:keys [::prefix ::region] :as cfg}]
@ -114,8 +113,7 @@
::client ::client
::presigner] ::presigner]
:opt [::prefix :opt [::prefix
::sto/id ::sto/id]))
::wrk/executor]))
;; --- API IMPL ;; --- API IMPL
@ -161,7 +159,6 @@
;; --- HELPERS ;; --- HELPERS
(def default-eventloop-threads 4)
(def default-timeout (def default-timeout
(dt/duration {:seconds 30})) (dt/duration {:seconds 30}))
@ -171,18 +168,18 @@
(Region/of (name region))) (Region/of (name region)))
(defn- build-s3-client (defn- build-s3-client
[{:keys [::region ::endpoint ::wrk/executor]}] [{:keys [::region ::endpoint ::io-threads]}]
(let [aconfig (-> (ClientAsyncConfiguration/builder) (let [aconfig (-> (ClientAsyncConfiguration/builder)
(.advancedOption SdkAdvancedAsyncClientOption/FUTURE_COMPLETION_EXECUTOR executor)
(.build)) (.build))
sconfig (-> (S3Configuration/builder) sconfig (-> (S3Configuration/builder)
(cond-> (some? endpoint) (.pathStyleAccessEnabled true)) (cond-> (some? endpoint) (.pathStyleAccessEnabled true))
(.build)) (.build))
thr-num (or io-threads (min 16 (px/get-available-processors)))
hclient (-> (NettyNioAsyncHttpClient/builder) hclient (-> (NettyNioAsyncHttpClient/builder)
(.eventLoopGroupBuilder (-> (SdkEventLoopGroup/builder) (.eventLoopGroupBuilder (-> (SdkEventLoopGroup/builder)
(.numberOfThreads (int default-eventloop-threads)))) (.numberOfThreads (int thr-num))))
(.connectionAcquisitionTimeout default-timeout) (.connectionAcquisitionTimeout default-timeout)
(.connectionTimeout default-timeout) (.connectionTimeout default-timeout)
(.readTimeout default-timeout) (.readTimeout default-timeout)