🎉 Add file offloading to external storage mechanism.

backend/src/app/storage/s3.clj

@@ -5,7 +5,7 @@
 ;; Copyright (c) UXBOX Labs SL
 
 (ns app.storage.s3
-  "Storage backends abstraction layer."
+  "S3 Storage backend implementation."
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
@@ -18,8 +18,11 @@
    [integrant.core :as ig])
   (:import
    java.time.Duration
+   java.io.InputStream
    java.util.Collection
    software.amazon.awssdk.core.sync.RequestBody
+   software.amazon.awssdk.core.ResponseBytes
+   ;; software.amazon.awssdk.core.ResponseInputStream
    software.amazon.awssdk.regions.Region
    software.amazon.awssdk.services.s3.S3Client
    software.amazon.awssdk.services.s3.model.Delete
@@ -29,13 +32,17 @@
    software.amazon.awssdk.services.s3.model.GetObjectRequest
    software.amazon.awssdk.services.s3.model.ObjectIdentifier
    software.amazon.awssdk.services.s3.model.PutObjectRequest
+   ;; software.amazon.awssdk.services.s3.model.GetObjectResponse
    software.amazon.awssdk.services.s3.presigner.S3Presigner
    software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest
-   software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest))
+   software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest
+
+   ))
 
 (declare put-object)
 (declare copy-object)
-(declare get-object)
+(declare get-object-bytes)
+(declare get-object-data)
 (declare get-object-url)
 (declare del-object-in-bulk)
 (declare build-s3-client)
@@ -87,7 +94,11 @@
 
 (defmethod impl/get-object-data :s3
   [backend object]
-  (get-object backend object))
+  (get-object-data backend object))
+
+(defmethod impl/get-object-bytes :s3
+  [backend object]
+  (get-object-bytes backend object))
 
 (defmethod impl/get-object-url :s3
   [backend object options]
@@ -104,19 +115,19 @@
   (case region
     :eu-central-1 Region/EU_CENTRAL_1))
 
-(defn- build-s3-client
+(defn build-s3-client
   [{:keys [region]}]
   (.. (S3Client/builder)
       (region (lookup-region region))
       (build)))
 
-(defn- build-s3-presigner
+(defn build-s3-presigner
   [{:keys [region]}]
   (.. (S3Presigner/builder)
       (region (lookup-region region))
       (build)))
 
-(defn- put-object
+(defn put-object
   [{:keys [client bucket prefix]} {:keys [id] :as object} content]
   (let [path    (str prefix (impl/id->path id))
         mdata   (meta object)
@@ -125,14 +136,15 @@
                     (bucket bucket)
                     (contentType mtype)
                     (key path)
-                    (build))
-        content (RequestBody/fromInputStream (io/input-stream content)
-                                             (count content))]
-    (.putObject ^S3Client client
-                ^PutObjectRequest request
-                ^RequestBody content)))
+                    (build))]
 
-(defn- copy-object
+    (with-open [^InputStream is (io/input-stream content)]
+      (let [content (RequestBody/fromInputStream is (count content))]
+        (.putObject ^S3Client client
+                    ^PutObjectRequest request
+                    ^RequestBody content)))))
+
+(defn copy-object
   [{:keys [client bucket prefix]} src-object dst-object]
   (let [source-path  (str prefix (impl/id->path (:id src-object)))
         source-mdata (meta src-object)
@@ -146,22 +158,33 @@
                          (contentType source-mtype)
                          (build))]
 
-    (.copyObject ^S3Client client
-                 ^CopyObjectRequest request)))
+    (.copyObject ^S3Client client ^CopyObjectRequest request)))
 
-(defn- get-object
+(defn get-object-data
   [{:keys [client bucket prefix]} {:keys [id]}]
   (let [gor (.. (GetObjectRequest/builder)
                 (bucket bucket)
                 (key (str prefix (impl/id->path id)))
                 (build))
-        obj (.getObject ^S3Client client ^GetObjectRequest gor)]
+        obj (.getObject ^S3Client client ^GetObjectRequest gor)
+        ;; rsp (.response ^ResponseInputStream obj)
+        ;; len (.contentLength ^GetObjectResponse rsp)
+        ]
     (io/input-stream obj)))
 
+(defn get-object-bytes
+  [{:keys [client bucket prefix]} {:keys [id]}]
+  (let [gor (.. (GetObjectRequest/builder)
+                (bucket bucket)
+                (key (str prefix (impl/id->path id)))
+                (build))
+        obj (.getObjectAsBytes ^S3Client client ^GetObjectRequest gor)]
+    (.asByteArray ^ResponseBytes obj)))
+
 (def default-max-age
   (dt/duration {:minutes 10}))
 
-(defn- get-object-url
+(defn get-object-url
   [{:keys [presigner bucket prefix]} {:keys [id]} {:keys [max-age] :or {max-age default-max-age}}]
   (us/assert dt/duration? max-age)
   (let [gor  (.. (GetObjectRequest/builder)
@@ -175,7 +198,7 @@
         pgor (.presignGetObject ^S3Presigner presigner ^GetObjectPresignRequest gopr)]
     (u/uri (str (.url ^PresignedGetObjectRequest pgor)))))
 
-(defn- del-object-in-bulk
+(defn del-object-in-bulk
   [{:keys [bucket client prefix]} ids]
   (let [oids (map (fn [id]
                     (.. (ObjectIdentifier/builder)