mirror of
https://github.com/penpot/penpot.git
synced 2025-07-15 19:15:15 +02:00
♻️ Refactor backend to be more async friendly
This commit is contained in:
Andrey Antukh
Alonso Torres
parent
087d896569
commit
9e4a50fb15
49 changed files with 1503 additions and 1378 deletions
|
|
@ -10,7 +10,8 @@
|
|||
[app.db :as db]
|
||||
[app.storage.impl :as impl]
|
||||
[clojure.spec.alpha :as s]
|
||||
[integrant.core :as ig])
|
||||
[integrant.core :as ig]
|
||||
[promesa.exec :as px])
|
||||
(:import
|
||||
java.io.ByteArrayInputStream))
|
||||
|
||||
|
|
@ -30,26 +31,23 @@
|
|||
;; --- API IMPL
|
||||
|
||||
(defmethod impl/put-object :db
|
||||
[{:keys [conn] :as storage} {:keys [id] :as object} content]
|
||||
(let [data (impl/slurp-bytes content)]
|
||||
(db/insert! conn :storage-data {:id id :data data})
|
||||
object))
|
||||
|
||||
(defmethod impl/copy-object :db
|
||||
[{:keys [conn] :as storage} src-object dst-object]
|
||||
(db/exec-one! conn ["insert into storage_data (id, data) select ? as id, data from storage_data where id=?"
|
||||
(:id dst-object)
|
||||
(:id src-object)]))
|
||||
[{:keys [conn executor] :as storage} {:keys [id] :as object} content]
|
||||
(px/with-dispatch executor
|
||||
(let [data (impl/slurp-bytes content)]
|
||||
(db/insert! conn :storage-data {:id id :data data})
|
||||
object)))
|
||||
|
||||
(defmethod impl/get-object-data :db
|
||||
[{:keys [conn] :as backend} {:keys [id] :as object}]
|
||||
(let [result (db/exec-one! conn ["select data from storage_data where id=?" id])]
|
||||
(ByteArrayInputStream. (:data result))))
|
||||
[{:keys [conn executor] :as backend} {:keys [id] :as object}]
|
||||
(px/with-dispatch executor
|
||||
(let [result (db/exec-one! conn ["select data from storage_data where id=?" id])]
|
||||
(ByteArrayInputStream. (:data result)))))
|
||||
|
||||
(defmethod impl/get-object-bytes :db
|
||||
[{:keys [conn] :as backend} {:keys [id] :as object}]
|
||||
(let [result (db/exec-one! conn ["select data from storage_data where id=?" id])]
|
||||
(:data result)))
|
||||
[{:keys [conn executor] :as backend} {:keys [id] :as object}]
|
||||
(px/with-dispatch executor
|
||||
(let [result (db/exec-one! conn ["select data from storage_data where id=?" id])]
|
||||
(:data result))))
|
||||
|
||||
(defmethod impl/get-object-url :db
|
||||
[_ _]
|
||||
|
|
|
|||
|
|
@ -14,7 +14,8 @@
|
|||
[clojure.spec.alpha :as s]
|
||||
[cuerdas.core :as str]
|
||||
[datoteka.core :as fs]
|
||||
[integrant.core :as ig])
|
||||
[integrant.core :as ig]
|
||||
[promesa.exec :as px])
|
||||
(:import
|
||||
java.io.InputStream
|
||||
java.io.OutputStream
|
||||
|
|
@ -47,62 +48,57 @@
|
|||
;; --- API IMPL
|
||||
|
||||
(defmethod impl/put-object :fs
|
||||
[backend {:keys [id] :as object} content]
|
||||
(let [base (fs/path (:directory backend))
|
||||
path (fs/path (impl/id->path id))
|
||||
full (fs/normalize (fs/join base path))]
|
||||
(when-not (fs/exists? (fs/parent full))
|
||||
(fs/create-dir (fs/parent full)))
|
||||
(with-open [^InputStream src (io/input-stream content)
|
||||
^OutputStream dst (io/output-stream full)]
|
||||
(io/copy src dst))))
|
||||
|
||||
(defmethod impl/copy-object :fs
|
||||
[backend src-object dst-object]
|
||||
(let [base (fs/path (:directory backend))
|
||||
path (fs/path (impl/id->path (:id dst-object)))
|
||||
full (fs/normalize (fs/join base path))]
|
||||
(when-not (fs/exists? (fs/parent full))
|
||||
(fs/create-dir (fs/parent full)))
|
||||
(with-open [^InputStream src (impl/get-object-data backend src-object)
|
||||
^OutputStream dst (io/output-stream full)]
|
||||
(io/copy src dst))))
|
||||
[{:keys [executor] :as backend} {:keys [id] :as object} content]
|
||||
(px/with-dispatch executor
|
||||
(let [base (fs/path (:directory backend))
|
||||
path (fs/path (impl/id->path id))
|
||||
full (fs/normalize (fs/join base path))]
|
||||
(when-not (fs/exists? (fs/parent full))
|
||||
(fs/create-dir (fs/parent full)))
|
||||
(with-open [^InputStream src (io/input-stream content)
|
||||
^OutputStream dst (io/output-stream full)]
|
||||
(io/copy src dst)))))
|
||||
|
||||
(defmethod impl/get-object-data :fs
|
||||
[backend {:keys [id] :as object}]
|
||||
(let [^Path base (fs/path (:directory backend))
|
||||
^Path path (fs/path (impl/id->path id))
|
||||
^Path full (fs/normalize (fs/join base path))]
|
||||
(when-not (fs/exists? full)
|
||||
(ex/raise :type :internal
|
||||
:code :filesystem-object-does-not-exists
|
||||
:path (str full)))
|
||||
(io/input-stream full)))
|
||||
[{:keys [executor] :as backend} {:keys [id] :as object}]
|
||||
(px/with-dispatch executor
|
||||
(let [^Path base (fs/path (:directory backend))
|
||||
^Path path (fs/path (impl/id->path id))
|
||||
^Path full (fs/normalize (fs/join base path))]
|
||||
(when-not (fs/exists? full)
|
||||
(ex/raise :type :internal
|
||||
:code :filesystem-object-does-not-exists
|
||||
:path (str full)))
|
||||
(io/input-stream full))))
|
||||
|
||||
(defmethod impl/get-object-bytes :fs
|
||||
[backend object]
|
||||
(fs/slurp-bytes (impl/get-object-data backend object)))
|
||||
[{:keys [executor] :as backend} object]
|
||||
(px/with-dispatch executor
|
||||
(fs/slurp-bytes (impl/get-object-data backend object))))
|
||||
|
||||
(defmethod impl/get-object-url :fs
|
||||
[{:keys [uri] :as backend} {:keys [id] :as object} _]
|
||||
(update uri :path
|
||||
(fn [existing]
|
||||
(if (str/ends-with? existing "/")
|
||||
(str existing (impl/id->path id))
|
||||
(str existing "/" (impl/id->path id))))))
|
||||
[{:keys [uri executor] :as backend} {:keys [id] :as object} _]
|
||||
(px/with-dispatch executor
|
||||
(update uri :path
|
||||
(fn [existing]
|
||||
(if (str/ends-with? existing "/")
|
||||
(str existing (impl/id->path id))
|
||||
(str existing "/" (impl/id->path id)))))))
|
||||
|
||||
(defmethod impl/del-object :fs
|
||||
[backend {:keys [id] :as object}]
|
||||
(let [base (fs/path (:directory backend))
|
||||
path (fs/path (impl/id->path id))
|
||||
path (fs/join base path)]
|
||||
(Files/deleteIfExists ^Path path)))
|
||||
[{:keys [executor] :as backend} {:keys [id] :as object}]
|
||||
(px/with-dispatch executor
|
||||
(let [base (fs/path (:directory backend))
|
||||
path (fs/path (impl/id->path id))
|
||||
path (fs/join base path)]
|
||||
(Files/deleteIfExists ^Path path))))
|
||||
|
||||
(defmethod impl/del-objects-in-bulk :fs
|
||||
[backend ids]
|
||||
(let [base (fs/path (:directory backend))]
|
||||
(doseq [id ids]
|
||||
(let [path (fs/path (impl/id->path id))
|
||||
path (fs/join base path)]
|
||||
(Files/deleteIfExists ^Path path)))))
|
||||
[{:keys [executor] :as backend} ids]
|
||||
(px/with-dispatch executor
|
||||
(let [base (fs/path (:directory backend))]
|
||||
(doseq [id ids]
|
||||
(let [path (fs/path (impl/id->path id))
|
||||
path (fs/join base path)]
|
||||
(Files/deleteIfExists ^Path path))))))
|
||||
|
||||
|
|
|
|||
|
|
@ -7,17 +7,20 @@
|
|||
(ns app.storage.impl
|
||||
"Storage backends abstraction layer."
|
||||
(:require
|
||||
[app.common.data.macros :as dm]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.uuid :as uuid]
|
||||
[buddy.core.codecs :as bc]
|
||||
[clojure.java.io :as io]
|
||||
[cuerdas.core :as str])
|
||||
[buddy.core.hash :as bh]
|
||||
[clojure.java.io :as io])
|
||||
(:import
|
||||
java.nio.ByteBuffer
|
||||
java.util.UUID
|
||||
java.io.ByteArrayInputStream
|
||||
java.io.InputStream
|
||||
java.nio.file.Files))
|
||||
java.nio.file.Files
|
||||
org.apache.commons.io.input.BoundedInputStream
|
||||
))
|
||||
|
||||
;; --- API Definition
|
||||
|
||||
|
|
@ -29,14 +32,6 @@
|
|||
:code :invalid-storage-backend
|
||||
:context cfg))
|
||||
|
||||
(defmulti copy-object (fn [cfg _ _] (:type cfg)))
|
||||
|
||||
(defmethod copy-object :default
|
||||
[cfg _ _]
|
||||
(ex/raise :type :internal
|
||||
:code :invalid-storage-backend
|
||||
:context cfg))
|
||||
|
||||
(defmulti get-object-data (fn [cfg _] (:type cfg)))
|
||||
|
||||
(defmethod get-object-data :default
|
||||
|
|
@ -106,63 +101,26 @@
|
|||
:code :invalid-id-type
|
||||
:hint "id should be string or uuid")))
|
||||
|
||||
(defprotocol IContentObject
|
||||
(size [_] "get object size"))
|
||||
|
||||
(defprotocol IContentObject)
|
||||
(defprotocol IContentHash
|
||||
(get-hash [_] "get precalculated hash"))
|
||||
|
||||
(defn- path->content
|
||||
[path]
|
||||
(let [size (Files/size path)]
|
||||
(reify
|
||||
IContentObject
|
||||
io/IOFactory
|
||||
(make-reader [_ opts]
|
||||
(io/make-reader path opts))
|
||||
(make-writer [_ _]
|
||||
(throw (UnsupportedOperationException. "not implemented")))
|
||||
(make-input-stream [_ opts]
|
||||
(io/make-input-stream path opts))
|
||||
(make-output-stream [_ _]
|
||||
(throw (UnsupportedOperationException. "not implemented")))
|
||||
clojure.lang.Counted
|
||||
(count [_] size)
|
||||
|
||||
java.lang.AutoCloseable
|
||||
(close [_]))))
|
||||
|
||||
(defn string->content
|
||||
[^String v]
|
||||
(let [data (.getBytes v "UTF-8")
|
||||
bais (ByteArrayInputStream. ^bytes data)]
|
||||
(reify
|
||||
IContentObject
|
||||
io/IOFactory
|
||||
(make-reader [_ opts]
|
||||
(io/make-reader bais opts))
|
||||
(make-writer [_ _]
|
||||
(throw (UnsupportedOperationException. "not implemented")))
|
||||
(make-input-stream [_ opts]
|
||||
(io/make-input-stream bais opts))
|
||||
(make-output-stream [_ _]
|
||||
(throw (UnsupportedOperationException. "not implemented")))
|
||||
|
||||
clojure.lang.Counted
|
||||
(count [_]
|
||||
(alength data))
|
||||
|
||||
java.lang.AutoCloseable
|
||||
(close [_]))))
|
||||
|
||||
(defn- input-stream->content
|
||||
[^InputStream is size]
|
||||
(defn- make-content
|
||||
[^InputStream is ^long size]
|
||||
(reify
|
||||
IContentObject
|
||||
(size [_] size)
|
||||
|
||||
io/IOFactory
|
||||
(make-reader [_ opts]
|
||||
(io/make-reader is opts))
|
||||
(make-reader [this opts]
|
||||
(io/make-reader this opts))
|
||||
(make-writer [_ _]
|
||||
(throw (UnsupportedOperationException. "not implemented")))
|
||||
(make-input-stream [_ opts]
|
||||
(io/make-input-stream is opts))
|
||||
(make-input-stream [_ _]
|
||||
(doto (BoundedInputStream. is size)
|
||||
(.setPropagateClose false)))
|
||||
(make-output-stream [_ _]
|
||||
(throw (UnsupportedOperationException. "not implemented")))
|
||||
|
||||
|
|
@ -178,26 +136,63 @@
|
|||
([data size]
|
||||
(cond
|
||||
(instance? java.nio.file.Path data)
|
||||
(path->content data)
|
||||
(make-content (io/input-stream data)
|
||||
(Files/size data))
|
||||
|
||||
(instance? java.io.File data)
|
||||
(path->content (.toPath ^java.io.File data))
|
||||
(content (.toPath ^java.io.File data) nil)
|
||||
|
||||
(instance? String data)
|
||||
(string->content data)
|
||||
(let [data (.getBytes data "UTF-8")
|
||||
bais (ByteArrayInputStream. ^bytes data)]
|
||||
(make-content bais (alength data)))
|
||||
|
||||
(bytes? data)
|
||||
(input-stream->content (ByteArrayInputStream. ^bytes data) (alength ^bytes data))
|
||||
(let [size (alength ^bytes data)
|
||||
bais (ByteArrayInputStream. ^bytes data)]
|
||||
(make-content bais size))
|
||||
|
||||
(instance? InputStream data)
|
||||
(do
|
||||
(when-not size
|
||||
(throw (UnsupportedOperationException. "size should be provided on InputStream")))
|
||||
(input-stream->content data size))
|
||||
(make-content data size))
|
||||
|
||||
:else
|
||||
(throw (UnsupportedOperationException. "type not supported")))))
|
||||
|
||||
(defn wrap-with-hash
|
||||
[content ^String hash]
|
||||
(when-not (satisfies? IContentObject content)
|
||||
(throw (UnsupportedOperationException. "`content` should be an instance of IContentObject")))
|
||||
|
||||
(when-not (satisfies? io/IOFactory content)
|
||||
(throw (UnsupportedOperationException. "`content` should be an instance of IOFactory")))
|
||||
|
||||
(reify
|
||||
IContentObject
|
||||
(size [_] (size content))
|
||||
|
||||
IContentHash
|
||||
(get-hash [_] hash)
|
||||
|
||||
io/IOFactory
|
||||
(make-reader [_ opts]
|
||||
(io/make-reader content opts))
|
||||
(make-writer [_ opts]
|
||||
(io/make-writer content opts))
|
||||
(make-input-stream [_ opts]
|
||||
(io/make-input-stream content opts))
|
||||
(make-output-stream [_ opts]
|
||||
(io/make-output-stream content opts))
|
||||
|
||||
clojure.lang.Counted
|
||||
(count [_] (count content))
|
||||
|
||||
java.lang.AutoCloseable
|
||||
(close [_]
|
||||
(.close ^java.lang.AutoCloseable content))))
|
||||
|
||||
(defn content?
|
||||
[v]
|
||||
(satisfies? IContentObject v))
|
||||
|
|
@ -209,15 +204,29 @@
|
|||
(io/copy input output)
|
||||
(.toByteArray output)))
|
||||
|
||||
(defn resolve-backend
|
||||
[{:keys [conn pool] :as storage} backend-id]
|
||||
(when backend-id
|
||||
(let [backend (get-in storage [:backends backend-id])]
|
||||
(when-not backend
|
||||
(ex/raise :type :internal
|
||||
:code :backend-not-configured
|
||||
:hint (str/fmt "backend '%s' not configured" backend-id)))
|
||||
(assoc backend
|
||||
:conn (or conn pool)
|
||||
:id backend-id))))
|
||||
(defn calculate-hash
|
||||
[path-or-stream]
|
||||
(let [result (cond
|
||||
(instance? InputStream path-or-stream)
|
||||
(let [result (-> (bh/blake2b-256 path-or-stream)
|
||||
(bc/bytes->hex))]
|
||||
(.reset path-or-stream)
|
||||
result)
|
||||
|
||||
:else
|
||||
(with-open [is (io/input-stream path-or-stream)]
|
||||
(-> (bh/blake2b-256 is)
|
||||
(bc/bytes->hex))))]
|
||||
(str "blake2b:" result)))
|
||||
|
||||
(defn resolve-backend
|
||||
[{:keys [conn pool executor] :as storage} backend-id]
|
||||
(let [backend (get-in storage [:backends backend-id])]
|
||||
(when-not backend
|
||||
(ex/raise :type :internal
|
||||
:code :backend-not-configured
|
||||
:hint (dm/fmt "backend '%' not configured" backend-id)))
|
||||
(assoc backend
|
||||
:executor executor
|
||||
:conn (or conn pool)
|
||||
:id backend-id)))
|
||||
|
|
|
|||
|
|
@ -13,36 +13,42 @@
|
|||
[app.common.uri :as u]
|
||||
[app.storage.impl :as impl]
|
||||
[app.util.time :as dt]
|
||||
[app.worker :as wrk]
|
||||
[clojure.java.io :as io]
|
||||
[clojure.spec.alpha :as s]
|
||||
[integrant.core :as ig])
|
||||
[integrant.core :as ig]
|
||||
[promesa.core :as p]
|
||||
[promesa.exec :as px])
|
||||
(:import
|
||||
java.time.Duration
|
||||
java.io.InputStream
|
||||
java.nio.ByteBuffer
|
||||
java.time.Duration
|
||||
java.util.Collection
|
||||
software.amazon.awssdk.core.sync.RequestBody
|
||||
java.util.Optional
|
||||
java.util.concurrent.Semaphore
|
||||
org.reactivestreams.Subscriber
|
||||
org.reactivestreams.Subscription
|
||||
software.amazon.awssdk.core.ResponseBytes
|
||||
;; software.amazon.awssdk.core.ResponseInputStream
|
||||
software.amazon.awssdk.core.async.AsyncRequestBody
|
||||
software.amazon.awssdk.core.client.config.ClientAsyncConfiguration
|
||||
software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption
|
||||
software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient
|
||||
software.amazon.awssdk.http.nio.netty.SdkEventLoopGroup
|
||||
software.amazon.awssdk.regions.Region
|
||||
software.amazon.awssdk.services.s3.S3Client
|
||||
software.amazon.awssdk.services.s3.S3AsyncClient
|
||||
software.amazon.awssdk.services.s3.model.Delete
|
||||
software.amazon.awssdk.services.s3.model.CopyObjectRequest
|
||||
software.amazon.awssdk.services.s3.model.DeleteObjectRequest
|
||||
software.amazon.awssdk.services.s3.model.DeleteObjectsRequest
|
||||
software.amazon.awssdk.services.s3.model.DeleteObjectsResponse
|
||||
software.amazon.awssdk.services.s3.model.DeleteObjectRequest
|
||||
software.amazon.awssdk.services.s3.model.GetObjectRequest
|
||||
software.amazon.awssdk.services.s3.model.ObjectIdentifier
|
||||
software.amazon.awssdk.services.s3.model.PutObjectRequest
|
||||
software.amazon.awssdk.services.s3.model.S3Error
|
||||
;; software.amazon.awssdk.services.s3.model.GetObjectResponse
|
||||
software.amazon.awssdk.services.s3.presigner.S3Presigner
|
||||
software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest
|
||||
software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest
|
||||
|
||||
))
|
||||
software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest))
|
||||
|
||||
(declare put-object)
|
||||
(declare copy-object)
|
||||
(declare get-object-bytes)
|
||||
(declare get-object-data)
|
||||
(declare get-object-url)
|
||||
|
|
@ -59,7 +65,7 @@
|
|||
(s/def ::endpoint ::us/string)
|
||||
|
||||
(defmethod ig/pre-init-spec ::backend [_]
|
||||
(s/keys :opt-un [::region ::bucket ::prefix ::endpoint]))
|
||||
(s/keys :opt-un [::region ::bucket ::prefix ::endpoint ::wrk/executor]))
|
||||
|
||||
(defmethod ig/prep-key ::backend
|
||||
[_ {:keys [prefix] :as cfg}]
|
||||
|
|
@ -75,12 +81,18 @@
|
|||
(let [client (build-s3-client cfg)
|
||||
presigner (build-s3-presigner cfg)]
|
||||
(assoc cfg
|
||||
:client client
|
||||
:client @client
|
||||
:presigner presigner
|
||||
:type :s3))))
|
||||
:type :s3
|
||||
::close-fn #(.close ^java.lang.AutoCloseable client)))))
|
||||
|
||||
(defmethod ig/halt-key! ::backend
|
||||
[_ {:keys [::close-fn]}]
|
||||
(when (fn? close-fn)
|
||||
(px/run! close-fn)))
|
||||
|
||||
(s/def ::type ::us/keyword)
|
||||
(s/def ::client #(instance? S3Client %))
|
||||
(s/def ::client #(instance? S3AsyncClient %))
|
||||
(s/def ::presigner #(instance? S3Presigner %))
|
||||
(s/def ::backend
|
||||
(s/keys :req-un [::region ::bucket ::client ::type ::presigner]
|
||||
|
|
@ -92,10 +104,6 @@
|
|||
[backend object content]
|
||||
(put-object backend object content))
|
||||
|
||||
(defmethod impl/copy-object :s3
|
||||
[backend src-object dst-object]
|
||||
(copy-object backend src-object dst-object))
|
||||
|
||||
(defmethod impl/get-object-data :s3
|
||||
[backend object]
|
||||
(get-object-data backend object))
|
||||
|
|
@ -118,21 +126,44 @@
|
|||
|
||||
;; --- HELPERS
|
||||
|
||||
(def default-eventloop-threads 4)
|
||||
(def default-timeout
|
||||
(dt/duration {:seconds 30}))
|
||||
|
||||
(defn- ^Region lookup-region
|
||||
[region]
|
||||
(Region/of (name region)))
|
||||
|
||||
(defn build-s3-client
|
||||
[{:keys [region endpoint]}]
|
||||
(if (string? endpoint)
|
||||
(let [uri (java.net.URI. endpoint)]
|
||||
(.. (S3Client/builder)
|
||||
(endpointOverride uri)
|
||||
(region (lookup-region region))
|
||||
(build)))
|
||||
(.. (S3Client/builder)
|
||||
(region (lookup-region region))
|
||||
(build))))
|
||||
[{:keys [region endpoint executor]}]
|
||||
(let [hclient (.. (NettyNioAsyncHttpClient/builder)
|
||||
(eventLoopGroupBuilder (.. (SdkEventLoopGroup/builder)
|
||||
(numberOfThreads (int default-eventloop-threads))))
|
||||
(connectionAcquisitionTimeout default-timeout)
|
||||
(connectionTimeout default-timeout)
|
||||
(readTimeout default-timeout)
|
||||
(writeTimeout default-timeout)
|
||||
(build))
|
||||
client (.. (S3AsyncClient/builder)
|
||||
(asyncConfiguration (.. (ClientAsyncConfiguration/builder)
|
||||
(advancedOption SdkAdvancedAsyncClientOption/FUTURE_COMPLETION_EXECUTOR
|
||||
executor)
|
||||
(build)))
|
||||
(httpClient hclient)
|
||||
(region (lookup-region region)))]
|
||||
|
||||
(when-let [uri (some-> endpoint (java.net.URI.))]
|
||||
(.endpointOverride client uri))
|
||||
|
||||
(let [client (.build client)]
|
||||
(reify
|
||||
clojure.lang.IDeref
|
||||
(deref [_] client)
|
||||
|
||||
java.lang.AutoCloseable
|
||||
(close [_]
|
||||
(.close hclient)
|
||||
(.close client))))))
|
||||
|
||||
(defn build-s3-presigner
|
||||
[{:keys [region endpoint]}]
|
||||
|
|
@ -146,58 +177,83 @@
|
|||
(region (lookup-region region))
|
||||
(build))))
|
||||
|
||||
(defn- make-request-body
|
||||
[content]
|
||||
(let [is (io/input-stream content)
|
||||
buff-size (* 1024 64)
|
||||
sem (Semaphore. 0)
|
||||
|
||||
writer-fn (fn [s]
|
||||
(try
|
||||
(loop []
|
||||
(.acquire sem 1)
|
||||
(let [buffer (byte-array buff-size)
|
||||
readed (.read is buffer)]
|
||||
(when (pos? readed)
|
||||
(.onNext ^Subscriber s (ByteBuffer/wrap buffer 0 readed))
|
||||
(when (= readed buff-size)
|
||||
(recur)))))
|
||||
(.onComplete s)
|
||||
(catch Throwable cause
|
||||
(.onError s cause))
|
||||
(finally
|
||||
(.close ^InputStream is))))]
|
||||
|
||||
(reify
|
||||
AsyncRequestBody
|
||||
(contentLength [_]
|
||||
(Optional/of (long (count content))))
|
||||
|
||||
(^void subscribe [_ ^Subscriber s]
|
||||
(let [thread (Thread. #(writer-fn s))]
|
||||
(.setDaemon thread true)
|
||||
(.setName thread "penpot/storage:s3")
|
||||
(.start thread)
|
||||
|
||||
(.onSubscribe s (reify Subscription
|
||||
(cancel [_]
|
||||
(.interrupt thread)
|
||||
(.release sem 1))
|
||||
|
||||
(request [_ n]
|
||||
(.release sem (int n))))))))))
|
||||
|
||||
|
||||
(defn put-object
|
||||
[{:keys [client bucket prefix]} {:keys [id] :as object} content]
|
||||
(let [path (str prefix (impl/id->path id))
|
||||
mdata (meta object)
|
||||
mtype (:content-type mdata "application/octet-stream")
|
||||
request (.. (PutObjectRequest/builder)
|
||||
(bucket bucket)
|
||||
(contentType mtype)
|
||||
(key path)
|
||||
(build))]
|
||||
(p/let [path (str prefix (impl/id->path id))
|
||||
mdata (meta object)
|
||||
mtype (:content-type mdata "application/octet-stream")
|
||||
request (.. (PutObjectRequest/builder)
|
||||
(bucket bucket)
|
||||
(contentType mtype)
|
||||
(key path)
|
||||
(build))]
|
||||
|
||||
(with-open [^InputStream is (io/input-stream content)]
|
||||
(let [content (RequestBody/fromInputStream is (count content))]
|
||||
(.putObject ^S3Client client
|
||||
^PutObjectRequest request
|
||||
^RequestBody content)))))
|
||||
|
||||
(defn copy-object
|
||||
[{:keys [client bucket prefix]} src-object dst-object]
|
||||
(let [source-path (str prefix (impl/id->path (:id src-object)))
|
||||
source-mdata (meta src-object)
|
||||
source-mtype (:content-type source-mdata "application/octet-stream")
|
||||
dest-path (str prefix (impl/id->path (:id dst-object)))
|
||||
|
||||
request (.. (CopyObjectRequest/builder)
|
||||
(copySource (u/query-encode (str bucket "/" source-path)))
|
||||
(destinationBucket bucket)
|
||||
(destinationKey dest-path)
|
||||
(contentType source-mtype)
|
||||
(build))]
|
||||
|
||||
(.copyObject ^S3Client client ^CopyObjectRequest request)))
|
||||
(let [content (make-request-body content)]
|
||||
(.putObject ^S3AsyncClient client
|
||||
^PutObjectRequest request
|
||||
^AsyncRequestBody content))))
|
||||
|
||||
(defn get-object-data
|
||||
[{:keys [client bucket prefix]} {:keys [id]}]
|
||||
(let [gor (.. (GetObjectRequest/builder)
|
||||
(bucket bucket)
|
||||
(key (str prefix (impl/id->path id)))
|
||||
(build))
|
||||
obj (.getObject ^S3Client client ^GetObjectRequest gor)
|
||||
;; rsp (.response ^ResponseInputStream obj)
|
||||
;; len (.contentLength ^GetObjectResponse rsp)
|
||||
]
|
||||
(p/let [gor (.. (GetObjectRequest/builder)
|
||||
(bucket bucket)
|
||||
(key (str prefix (impl/id->path id)))
|
||||
(build))
|
||||
obj (.getObject ^S3AsyncClient client ^GetObjectRequest gor)
|
||||
;; rsp (.response ^ResponseInputStream obj)
|
||||
;; len (.contentLength ^GetObjectResponse rsp)
|
||||
]
|
||||
(io/input-stream obj)))
|
||||
|
||||
(defn get-object-bytes
|
||||
[{:keys [client bucket prefix]} {:keys [id]}]
|
||||
(let [gor (.. (GetObjectRequest/builder)
|
||||
(bucket bucket)
|
||||
(key (str prefix (impl/id->path id)))
|
||||
(build))
|
||||
obj (.getObjectAsBytes ^S3Client client ^GetObjectRequest gor)]
|
||||
(p/let [gor (.. (GetObjectRequest/builder)
|
||||
(bucket bucket)
|
||||
(key (str prefix (impl/id->path id)))
|
||||
(build))
|
||||
obj (.getObjectAsBytes ^S3AsyncClient client ^GetObjectRequest gor)]
|
||||
(.asByteArray ^ResponseBytes obj)))
|
||||
|
||||
(def default-max-age
|
||||
|
|
@ -206,42 +262,43 @@
|
|||
(defn get-object-url
|
||||
[{:keys [presigner bucket prefix]} {:keys [id]} {:keys [max-age] :or {max-age default-max-age}}]
|
||||
(us/assert dt/duration? max-age)
|
||||
(let [gor (.. (GetObjectRequest/builder)
|
||||
(bucket bucket)
|
||||
(key (str prefix (impl/id->path id)))
|
||||
(build))
|
||||
gopr (.. (GetObjectPresignRequest/builder)
|
||||
(signatureDuration ^Duration max-age)
|
||||
(getObjectRequest ^GetObjectRequest gor)
|
||||
(build))
|
||||
pgor (.presignGetObject ^S3Presigner presigner ^GetObjectPresignRequest gopr)]
|
||||
(u/uri (str (.url ^PresignedGetObjectRequest pgor)))))
|
||||
(p/do
|
||||
(let [gor (.. (GetObjectRequest/builder)
|
||||
(bucket bucket)
|
||||
(key (str prefix (impl/id->path id)))
|
||||
(build))
|
||||
gopr (.. (GetObjectPresignRequest/builder)
|
||||
(signatureDuration ^Duration max-age)
|
||||
(getObjectRequest ^GetObjectRequest gor)
|
||||
(build))
|
||||
pgor (.presignGetObject ^S3Presigner presigner ^GetObjectPresignRequest gopr)]
|
||||
(u/uri (str (.url ^PresignedGetObjectRequest pgor))))))
|
||||
|
||||
(defn del-object
|
||||
[{:keys [bucket client prefix]} {:keys [id] :as obj}]
|
||||
(let [dor (.. (DeleteObjectRequest/builder)
|
||||
(bucket bucket)
|
||||
(key (str prefix (impl/id->path id)))
|
||||
(build))]
|
||||
(.deleteObject ^S3Client client
|
||||
(p/let [dor (.. (DeleteObjectRequest/builder)
|
||||
(bucket bucket)
|
||||
(key (str prefix (impl/id->path id)))
|
||||
(build))]
|
||||
(.deleteObject ^S3AsyncClient client
|
||||
^DeleteObjectRequest dor)))
|
||||
|
||||
(defn del-object-in-bulk
|
||||
[{:keys [bucket client prefix]} ids]
|
||||
(let [oids (map (fn [id]
|
||||
(.. (ObjectIdentifier/builder)
|
||||
(key (str prefix (impl/id->path id)))
|
||||
(build)))
|
||||
ids)
|
||||
delc (.. (Delete/builder)
|
||||
(objects ^Collection oids)
|
||||
(build))
|
||||
dor (.. (DeleteObjectsRequest/builder)
|
||||
(bucket bucket)
|
||||
(delete ^Delete delc)
|
||||
(build))
|
||||
dres (.deleteObjects ^S3Client client
|
||||
^DeleteObjectsRequest dor)]
|
||||
(p/let [oids (map (fn [id]
|
||||
(.. (ObjectIdentifier/builder)
|
||||
(key (str prefix (impl/id->path id)))
|
||||
(build)))
|
||||
ids)
|
||||
delc (.. (Delete/builder)
|
||||
(objects ^Collection oids)
|
||||
(build))
|
||||
dor (.. (DeleteObjectsRequest/builder)
|
||||
(bucket bucket)
|
||||
(delete ^Delete delc)
|
||||
(build))
|
||||
dres (.deleteObjects ^S3AsyncClient client
|
||||
^DeleteObjectsRequest dor)]
|
||||
(when (.hasErrors ^DeleteObjectsResponse dres)
|
||||
(let [errors (seq (.errors ^DeleteObjectsResponse dres))]
|
||||
(ex/raise :type :internal
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue