♻️ Integrate new storage subsystem.

backend/src/app/storage/db.clj

@@ -46,7 +46,7 @@
     (db/insert! conn :storage-data {:id id :data data})
     object))
 
-(defmethod impl/get-object :db
+(defmethod impl/get-object-data :db
   [{:keys [conn] :as backend} {:keys [id] :as object}]
   (let [result (db/exec-one! conn ["select data from storage_data where id=?" id])]
     (ByteArrayInputStream. (:data result))))

backend/src/app/storage/fs.clj

@@ -9,6 +9,7 @@
 
 (ns app.storage.fs
   (:require
+   [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.spec :as us]
    [app.db :as db]
@@ -16,6 +17,7 @@
    [clojure.java.io :as io]
    [clojure.spec.alpha :as s]
    [datoteka.core :as fs]
+   [cuerdas.core :as str]
    [lambdaisland.uri :as u]
    [integrant.core :as ig])
   (:import
@@ -48,20 +50,20 @@
 
 (defmethod impl/put-object :fs
   [backend {:keys [id] :as object} content]
-  (let [^Path base (fs/path (:directory backend))
-        ^Path path (fs/path (impl/id->path id))
-        ^Path full (.resolve base path)]
-    (when-not (fs/exists? (.getParent full))
-      (fs/create-dir (.getParent full)))
+  (let [base (fs/path (:directory backend))
+        path (fs/path (impl/id->path id))
+        full (fs/normalize (fs/join base path))]
+    (when-not (fs/exists? (fs/parent full))
+      (fs/create-dir (fs/parent full)))
     (with-open [^InputStream src  (io/input-stream content)
                 ^OutputStream dst (io/output-stream full)]
       (io/copy src dst))))
 
-(defmethod impl/get-object :fs
+(defmethod impl/get-object-data :fs
   [backend {:keys [id] :as object}]
   (let [^Path base (fs/path (:directory backend))
         ^Path path (fs/path (impl/id->path id))
-        ^Path full (.resolve base path)]
+        ^Path full (fs/normalize (fs/join base path))]
     (when-not (fs/exists? full)
       (ex/raise :type :internal
                 :code :filesystem-object-does-not-exists
@@ -73,12 +75,14 @@
   (let [uri (u/uri (:uri backend))]
     (update uri :path
             (fn [existing]
-              (str existing (impl/id->path id))))))
+              (if (str/ends-with? existing "/")
+                (str existing (impl/id->path id))
+                (str existing "/" (impl/id->path id)))))))
 
 (defmethod impl/del-objects-in-bulk :fs
   [backend ids]
   (let [base (fs/path (:directory backend))]
     (doseq [id ids]
       (let [path (fs/path (impl/id->path id))
-            path (.resolve ^Path base ^Path path)]
+            path (fs/join base path)]
         (Files/deleteIfExists ^Path path)))))

backend/src/app/storage/impl.clj

@@ -33,9 +33,9 @@
             :code :invalid-storage-backend
             :context cfg))
 
-(defmulti get-object (fn [cfg _] (:type cfg)))
+(defmulti get-object-data (fn [cfg _] (:type cfg)))
 
-(defmethod get-object :default
+(defmethod get-object-data :default
   [cfg _]
   (ex/raise :type :internal
             :code :invalid-storage-backend
@@ -90,7 +90,7 @@
 
 (defprotocol IContentObject)
 
-(defn- path->content-object
+(defn- path->content
   [path]
   (let [size (Files/size path)]
     (reify
@@ -107,7 +107,7 @@
       clojure.lang.Counted
       (count [_] size))))
 
-(defn string->content-object
+(defn string->content
   [^String v]
   (let [data (.getBytes v "UTF-8")
         bais (ByteArrayInputStream. ^bytes data)]
@@ -127,7 +127,7 @@
       (count [_]
         (alength data)))))
 
-(defn- input-stream->content-object
+(defn- input-stream->content
   [^InputStream is size]
   (reify
     IContentObject
@@ -144,35 +144,35 @@
       clojure.lang.Counted
       (count [_] size)))
 
-(defn content-object
-  ([data] (content-object data nil))
+(defn content
+  ([data] (content data nil))
   ([data size]
    (cond
      (instance? java.nio.file.Path data)
-     (path->content-object data)
+     (path->content data)
 
      (instance? java.io.File data)
-     (path->content-object (.toPath ^java.io.File data))
+     (path->content (.toPath ^java.io.File data))
 
      (instance? String data)
-     (string->content-object data)
+     (string->content data)
 
      (instance? InputStream data)
      (do
        (when-not size
          (throw (UnsupportedOperationException. "size should be provided on InputStream")))
-       (input-stream->content-object data size))
+       (input-stream->content data size))
 
      :else
      (throw (UnsupportedOperationException. "type not supported")))))
 
-(defn content-object?
+(defn content?
   [v]
   (satisfies? IContentObject v))
 
 (defn slurp-bytes
   [content]
-  (us/assert content-object? content)
+  (us/assert content? content)
   (with-open [input  (io/input-stream content)
               output (java.io.ByteArrayOutputStream. (count content))]
     (io/copy input output)

backend/src/app/storage/s3.clj

@@ -10,6 +10,7 @@
 (ns app.storage.s3
   "Storage backends abstraction layer."
   (:require
+   [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.spec :as us]
    [app.db :as db]
@@ -23,20 +24,19 @@
    java.io.InputStream
    java.io.OutputStream
    java.nio.file.Path
+   software.amazon.awssdk.core.sync.RequestBody
    software.amazon.awssdk.regions.Region
    software.amazon.awssdk.services.s3.S3Client
    software.amazon.awssdk.services.s3.S3ClientBuilder
-   software.amazon.awssdk.core.sync.RequestBody
-   software.amazon.awssdk.services.s3.model.PutObjectRequest
-   software.amazon.awssdk.services.s3.model.GetObjectRequest
-   software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest
-   software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest
-   software.amazon.awssdk.services.s3.presigner.S3Presigner
-   software.amazon.awssdk.services.s3.model.DeleteObjectsRequest
    software.amazon.awssdk.services.s3.model.Delete
+   software.amazon.awssdk.services.s3.model.DeleteObjectsRequest
+   software.amazon.awssdk.services.s3.model.DeleteObjectsResponse
+   software.amazon.awssdk.services.s3.model.GetObjectRequest
    software.amazon.awssdk.services.s3.model.ObjectIdentifier
-   software.amazon.awssdk.services.s3.model.DeleteObjectsResponse))
-
+   software.amazon.awssdk.services.s3.model.PutObjectRequest
+   software.amazon.awssdk.services.s3.presigner.S3Presigner
+   software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest
+   software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest))
 
 (declare put-object)
 (declare get-object)
@@ -49,9 +49,14 @@
 
 (s/def ::region #{:eu-central-1})
 (s/def ::bucket ::us/string)
+(s/def ::prefix ::us/string)
 
 (defmethod ig/pre-init-spec ::backend [_]
-  (s/keys :opt-un [::region ::bucket]))
+  (s/keys :opt-un [::region ::bucket ::prefix]))
+
+(defmethod ig/prep-key ::backend
+  [_ cfg]
+  (merge {:prefix ""} (d/without-nils cfg)))
 
 (defmethod ig/init-key ::backend
   [_ cfg]
@@ -70,7 +75,7 @@
 (s/def ::client #(instance? S3Client %))
 (s/def ::presigner #(instance? S3Presigner %))
 (s/def ::backend
-  (s/keys :req-un [::region ::bucket ::client ::type ::presigner]))
+  (s/keys :req-un [::region ::bucket ::client ::type ::presigner ::prefix]))
 
 ;; --- API IMPL
 
@@ -78,7 +83,7 @@
   [backend object content]
   (put-object backend object content))
 
-(defmethod impl/get-object :s3
+(defmethod impl/get-object-data :s3
   [backend object]
   (get-object backend object))
 
@@ -110,8 +115,8 @@
       (build)))
 
 (defn- put-object
-  [{:keys [client bucket]} {:keys [id] :as object} content]
-  (let [path    (impl/id->path id)
+  [{:keys [client bucket prefix]} {:keys [id] :as object} content]
+  (let [path    (str prefix "-" (impl/id->path id))
         mdata   (meta object)
         mtype   (:content-type mdata "application/octet-stream")
         request (.. (PutObjectRequest/builder)
@@ -126,10 +131,10 @@
                 ^RequestBody content)))
 
 (defn- get-object
-  [{:keys [client bucket]} {:keys [id]}]
+  [{:keys [client bucket prefix]} {:keys [id]}]
   (let [gor (.. (GetObjectRequest/builder)
                 (bucket bucket)
-                (key (impl/id->path id))
+                (key (str prefix "-" (impl/id->path id)))
                 (build))
         obj (.getObject ^S3Client client gor)]
     (io/input-stream obj)))
@@ -138,11 +143,11 @@
   (dt/duration {:minutes 10}))
 
 (defn- get-object-url
-  [{:keys [presigner bucket]} {:keys [id]} {:keys [max-age] :or {max-age default-max-age}}]
+  [{:keys [presigner bucket prefix]} {:keys [id]} {:keys [max-age] :or {max-age default-max-age}}]
   (us/assert dt/duration? max-age)
   (let [gor  (.. (GetObjectRequest/builder)
                  (bucket bucket)
-                 (key (impl/id->path id))
+                 (key (str prefix "-" (impl/id->path id)))
                  (build))
         gopr (.. (GetObjectPresignRequest/builder)
                  (signatureDuration max-age)
@@ -152,10 +157,10 @@
     (u/uri (str (.url ^PresignedGetObjectRequest pgor)))))
 
 (defn- del-object-in-bulk
-  [{:keys [bucket client]} ids]
+  [{:keys [bucket client prefix]} ids]
   (let [oids (map (fn [id]
                     (.. (ObjectIdentifier/builder)
-                        (key (impl/id->path id))
+                        (key (str prefix "-" (impl/id->path id)))
                         (build)))
                   ids)
         delc (.. (Delete/builder)