🐛 Add retry mechanism for internal tmp file handling on s3 backend

This commit is contained in:
Andrey Antukh 2024-10-25 14:22:01 +02:00
parent e7d7291947
commit 15c42fba5e

View file

@ -59,6 +59,10 @@
software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest
software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest)) software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest))
(def ^:private max-retries
"A maximum number of retries on internal operations"
3)
(declare put-object) (declare put-object)
(declare get-object-bytes) (declare get-object-bytes)
(declare get-object-data) (declare get-object-data)
@ -128,7 +132,10 @@
[backend object] [backend object]
(us/assert! ::backend backend) (us/assert! ::backend backend)
(let [result (p/await (get-object-data backend object))] (loop [result (get-object-data backend object)
retryn 0]
(let [result (p/await result)]
(if (ex/exception? result) (if (ex/exception? result)
(cond (cond
(ex/instance? NoSuchKeyException result) (ex/instance? NoSuchKeyException result)
@ -136,10 +143,16 @@
:code :object-not-found :code :object-not-found
:hint "s3 object not found" :hint "s3 object not found"
:cause result) :cause result)
(and (ex/instance? java.nio.file.FileAlreadyExistsException result)
(< retryn max-retries))
(recur (get-object-data backend object)
(inc retryn))
:else :else
(throw result)) (throw result))
result))) result))))
(defmethod impl/get-object-bytes :s3 (defmethod impl/get-object-bytes :s3
[backend object] [backend object]