♻️ Refactor storage internal concurrency model

backend/test/backend_tests/helpers.clj

@@ -8,6 +8,7 @@
   (:require
    [app.auth]
    [app.common.data :as d]
+   [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.flags :as flags]
    [app.common.pages :as cp]
@@ -208,7 +209,7 @@
                         :password "123123"
                         :is-demo false}
                        params)]
-     (with-open [conn (db/open pool)]
+     (dm/with-open [conn (db/open pool)]
        (->> params
             (cmd.auth/create-profile! conn)
             (cmd.auth/create-profile-rels! conn))))))
@@ -218,7 +219,7 @@
   ([pool i {:keys [profile-id team-id] :as params}]
    (us/assert uuid? profile-id)
    (us/assert uuid? team-id)
-   (with-open [conn (db/open pool)]
+   (dm/with-open [conn (db/open pool)]
      (->> (merge {:id (mk-uuid "project" i)
                   :name (str "project" i)}
                  params)
@@ -230,7 +231,7 @@
   ([pool i {:keys [profile-id project-id] :as params}]
    (us/assert uuid? profile-id)
    (us/assert uuid? project-id)
-   (with-open [conn (db/open pool)]
+   (dm/with-open [conn (db/open pool)]
      (files.create/create-file conn
                                (merge {:id (mk-uuid "file" i)
                                        :name (str "file" i)
@@ -246,7 +247,7 @@
   ([i params] (create-team* *pool* i params))
   ([pool i {:keys [profile-id] :as params}]
    (us/assert uuid? profile-id)
-   (with-open [conn (db/open pool)]
+   (dm/with-open [conn (db/open pool)]
      (let [id   (mk-uuid "team" i)]
        (teams/create-team conn {:id id
                                 :profile-id profile-id
@@ -257,7 +258,7 @@
   ([pool {:keys [name width height mtype file-id is-local media-id]
           :or {name "sample" width 100 height 100 mtype "image/svg+xml" is-local true}}]
 
-   (with-open [conn (db/open pool)]
+   (dm/with-open [conn (db/open pool)]
      (db/insert! conn :file-media-object
                  {:id (uuid/next)
                   :file-id file-id
@@ -271,12 +272,12 @@
 (defn link-file-to-library*
   ([params] (link-file-to-library* *pool* params))
   ([pool {:keys [file-id library-id] :as params}]
-   (with-open [conn (db/open pool)]
+   (dm/with-open [conn (db/open pool)]
      (#'files/link-file-to-library conn {:file-id file-id :library-id library-id}))))
 
 (defn create-complaint-for
   [pool {:keys [id created-at type]}]
-  (with-open [conn (db/open pool)]
+  (dm/with-open [conn (db/open pool)]
     (db/insert! conn :profile-complaint-report
                 {:profile-id id
                  :created-at (or created-at (dt/now))
@@ -285,7 +286,7 @@
 
 (defn create-global-complaint-for
   [pool {:keys [email type created-at]}]
-  (with-open [conn (db/open pool)]
+  (dm/with-open [conn (db/open pool)]
     (db/insert! conn :global-complaint-report
                 {:email email
                  :type (name type)
@@ -295,7 +296,7 @@
 (defn create-team-role*
   ([params] (create-team-role* *pool* params))
   ([pool {:keys [team-id profile-id role] :or {role :owner}}]
-   (with-open [conn (db/open pool)]
+   (dm/with-open [conn (db/open pool)]
      (#'teams/create-team-role conn {:team-id team-id
                                      :profile-id profile-id
                                      :role role}))))
@@ -303,7 +304,7 @@
 (defn create-project-role*
   ([params] (create-project-role* *pool* params))
   ([pool {:keys [project-id profile-id role] :or {role :owner}}]
-   (with-open [conn (db/open pool)]
+   (dm/with-open [conn (db/open pool)]
      (#'teams/create-project-role conn {:project-id project-id
                                            :profile-id profile-id
                                            :role role}))))
@@ -311,7 +312,7 @@
 (defn create-file-role*
   ([params] (create-file-role* *pool* params))
   ([pool {:keys [file-id profile-id role] :or {role :owner}}]
-   (with-open [conn (db/open pool)]
+   (dm/with-open [conn (db/open pool)]
      (files.create/create-file-role! conn {:file-id file-id
                                            :profile-id profile-id
                                            :role role}))))
@@ -320,7 +321,7 @@
   ([params] (update-file* *pool* params))
   ([pool {:keys [file-id changes session-id profile-id revn]
           :or {session-id (uuid/next) revn 0}}]
-   (with-open [conn (db/open pool)]
+   (dm/with-open [conn (db/open pool)]
      (let [features #{"components/v2"}
            cfg      (-> (select-keys *system* [::mbus/msgbus ::mtx/metrics])
                         (assoc ::db/conn conn))]

backend/test/backend_tests/rpc_file_test.clj

@@ -215,10 +215,10 @@
         (t/is (= 1 (count rows))))
 
       ;; The underlying storage objects are still available.
-      (t/is (some? @(sto/get-object storage (:media-id fmo2))))
-      (t/is (some? @(sto/get-object storage (:thumbnail-id fmo2))))
-      (t/is (some? @(sto/get-object storage (:media-id fmo1))))
-      (t/is (some? @(sto/get-object storage (:thumbnail-id fmo1))))
+      (t/is (some? (sto/get-object storage (:media-id fmo2))))
+      (t/is (some? (sto/get-object storage (:thumbnail-id fmo2))))
+      (t/is (some? (sto/get-object storage (:media-id fmo1))))
+      (t/is (some? (sto/get-object storage (:thumbnail-id fmo1))))
 
       ;; proceed to remove usage of the file
       (update-file {:file-id (:id file)
@@ -246,10 +246,10 @@
       ;; Finally, check that some of the objects that are marked as
       ;; deleted we are unable to retrieve them using standard storage
       ;; public api.
-      (t/is (nil? @(sto/get-object storage (:media-id fmo2))))
-      (t/is (nil? @(sto/get-object storage (:thumbnail-id fmo2))))
-      (t/is (nil? @(sto/get-object storage (:media-id fmo1))))
-      (t/is (nil? @(sto/get-object storage (:thumbnail-id fmo1))))
+      (t/is (nil? (sto/get-object storage (:media-id fmo2))))
+      (t/is (nil? (sto/get-object storage (:thumbnail-id fmo2))))
+      (t/is (nil? (sto/get-object storage (:media-id fmo1))))
+      (t/is (nil? (sto/get-object storage (:thumbnail-id fmo1))))
       )))
 
 (t/deftest permissions-checks-creating-file

backend/test/backend_tests/rpc_management_test.clj

@@ -26,9 +26,9 @@
   (let [storage (-> (:app.storage/storage th/*system*)
                     (configure-storage-backend))
 
-        sobject @(sto/put-object! storage {::sto/content (sto/content "content")
-                                           :content-type "text/plain"
-                                           :other "data"})
+        sobject (sto/put-object! storage {::sto/content (sto/content "content")
+                                          :content-type "text/plain"
+                                          :other "data"})
         profile (th/create-profile* 1 {:is-active true})
         project (th/create-project* 1 {:team-id (:default-team-id profile)
                                        :profile-id (:id profile)})
@@ -98,9 +98,9 @@
 (t/deftest duplicate-file-with-deleted-relations
   (let [storage (-> (:app.storage/storage th/*system*)
                     (configure-storage-backend))
-        sobject @(sto/put-object! storage {::sto/content (sto/content "content")
-                                           :content-type "text/plain"
-                                           :other "data"})
+        sobject (sto/put-object! storage {::sto/content (sto/content "content")
+                                          :content-type "text/plain"
+                                          :other "data"})
         profile (th/create-profile* 1 {:is-active true})
 
         project (th/create-project* 1 {:team-id (:default-team-id profile)
@@ -120,7 +120,7 @@
                                                :media-id (:id sobject)})]
 
     (th/mark-file-deleted* {:id (:id file2)})
-    @(sto/del-object! storage sobject)
+    (sto/del-object! storage sobject)
 
     (let [data {::th/type :duplicate-file
                 ::rpc/profile-id (:id profile)
@@ -157,9 +157,9 @@
   (let [storage (-> (:app.storage/storage th/*system*)
                     (configure-storage-backend))
 
-        sobject @(sto/put-object! storage {::sto/content (sto/content "content")
-                                           :content-type "text/plain"
-                                           :other "data"})
+        sobject (sto/put-object! storage {::sto/content (sto/content "content")
+                                          :content-type "text/plain"
+                                          :other "data"})
 
         profile (th/create-profile* 1 {:is-active true})
         project (th/create-project* 1 {:team-id (:default-team-id profile)
@@ -230,9 +230,9 @@
 (t/deftest duplicate-project-with-deleted-files
   (let [storage (-> (:app.storage/storage th/*system*)
                     (configure-storage-backend))
-        sobject @(sto/put-object! storage {::sto/content (sto/content "content")
-                                           :content-type "text/plain"
-                                           :other "data"})
+        sobject (sto/put-object! storage {::sto/content (sto/content "content")
+                                          :content-type "text/plain"
+                                          :other "data"})
         profile (th/create-profile* 1 {:is-active true})
         project (th/create-project* 1 {:team-id (:default-team-id profile)
                                        :profile-id (:id profile)})

backend/test/backend_tests/rpc_media_test.clj

@@ -42,8 +42,8 @@
       (t/is (uuid? media-id))
       (t/is (uuid? thumbnail-id))
       (let [storage (:app.storage/storage th/*system*)
-            mobj1   @(sto/get-object storage media-id)
-            mobj2   @(sto/get-object storage thumbnail-id)]
+            mobj1   (sto/get-object storage media-id)
+            mobj2   (sto/get-object storage thumbnail-id)]
         (t/is (sto/object? mobj1))
         (t/is (sto/object? mobj2))
         (t/is (= 122785 (:size mobj1)))
@@ -83,8 +83,8 @@
       (t/is (uuid? media-id))
       (t/is (uuid? thumbnail-id))
       (let [storage (:app.storage/storage th/*system*)
-            mobj1   @(sto/get-object storage media-id)
-            mobj2   @(sto/get-object storage thumbnail-id)]
+            mobj1   (sto/get-object storage media-id)
+            mobj2   (sto/get-object storage thumbnail-id)]
         (t/is (sto/object? mobj1))
         (t/is (sto/object? mobj2))
         (t/is (= 312043 (:size mobj1)))
@@ -162,8 +162,8 @@
       (t/is (uuid? media-id))
       (t/is (uuid? thumbnail-id))
       (let [storage (:app.storage/storage th/*system*)
-            mobj1   @(sto/get-object storage media-id)
-            mobj2   @(sto/get-object storage thumbnail-id)]
+            mobj1   (sto/get-object storage media-id)
+            mobj2   (sto/get-object storage thumbnail-id)]
         (t/is (sto/object? mobj1))
         (t/is (sto/object? mobj2))
         (t/is (= 122785 (:size mobj1)))
@@ -203,8 +203,8 @@
       (t/is (uuid? media-id))
       (t/is (uuid? thumbnail-id))
       (let [storage (:app.storage/storage th/*system*)
-            mobj1   @(sto/get-object storage media-id)
-            mobj2   @(sto/get-object storage thumbnail-id)]
+            mobj1   (sto/get-object storage media-id)
+            mobj2   (sto/get-object storage thumbnail-id)]
         (t/is (sto/object? mobj1))
         (t/is (sto/object? mobj2))
         (t/is (= 312043 (:size mobj1)))

backend/test/backend_tests/storage_test.clj

@@ -37,61 +37,61 @@
   (let [storage (-> (:app.storage/storage th/*system*)
                     (configure-storage-backend))
         content (sto/content "content")
-        object  @(sto/put-object! storage {::sto/content content
-                                           :content-type "text/plain"
-                                           :other "data"})]
+        object  (sto/put-object! storage {::sto/content content
+                                          :content-type "text/plain"
+                                          :other "data"})]
 
     (t/is (sto/object? object))
-    (t/is (fs/path? @(sto/get-object-path storage object)))
+    (t/is (fs/path? (sto/get-object-path storage object)))
 
     (t/is (nil? (:expired-at object)))
     (t/is (= :assets-fs (:backend object)))
     (t/is (= "data" (:other (meta object))))
     (t/is (= "text/plain" (:content-type (meta object))))
-    (t/is (= "content" (slurp @(sto/get-object-data storage object))))
-    (t/is (= "content" (slurp @(sto/get-object-path storage object))))
+    (t/is (= "content" (slurp (sto/get-object-data storage object))))
+    (t/is (= "content" (slurp (sto/get-object-path storage object))))
     ))
 
 (t/deftest put-and-retrieve-expired-object
   (let [storage (-> (:app.storage/storage th/*system*)
                     (configure-storage-backend))
         content (sto/content "content")
-        object  @(sto/put-object! storage {::sto/content content
-                                           ::sto/expired-at (dt/in-future {:seconds 1})
-                                           :content-type "text/plain"
-                                           })]
+        object  (sto/put-object! storage {::sto/content content
+                                          ::sto/expired-at (dt/in-future {:seconds 1})
+                                          :content-type "text/plain"
+                                          })]
 
     (t/is (sto/object? object))
     (t/is (dt/instant? (:expired-at object)))
     (t/is (dt/is-after? (:expired-at object) (dt/now)))
-    (t/is (= object @(sto/get-object storage (:id object))))
+    (t/is (= object (sto/get-object storage (:id object))))
 
     (th/sleep 1000)
-    (t/is (nil? @(sto/get-object storage (:id object))))
-    (t/is (nil? @(sto/get-object-data storage object)))
-    (t/is (nil? @(sto/get-object-url storage object)))
-    (t/is (nil? @(sto/get-object-path storage object)))
+    (t/is (nil? (sto/get-object storage (:id object))))
+    (t/is (nil? (sto/get-object-data storage object)))
+    (t/is (nil? (sto/get-object-url storage object)))
+    (t/is (nil? (sto/get-object-path storage object)))
     ))
 
 (t/deftest put-and-delete-object
   (let [storage (-> (:app.storage/storage th/*system*)
                     (configure-storage-backend))
         content (sto/content "content")
-        object  @(sto/put-object! storage {::sto/content content
-                                           :content-type "text/plain"
-                                           :expired-at (dt/in-future {:seconds 1})})]
+        object  (sto/put-object! storage {::sto/content content
+                                          :content-type "text/plain"
+                                          :expired-at (dt/in-future {:seconds 1})})]
     (t/is (sto/object? object))
-    (t/is (true? @(sto/del-object! storage object)))
+    (t/is (true? (sto/del-object! storage object)))
 
     ;; retrieving the same object should be not nil because the
     ;; deletion is not immediate
-    (t/is (some? @(sto/get-object-data storage object)))
-    (t/is (some? @(sto/get-object-url storage object)))
-    (t/is (some? @(sto/get-object-path storage object)))
+    (t/is (some? (sto/get-object-data storage object)))
+    (t/is (some? (sto/get-object-url storage object)))
+    (t/is (some? (sto/get-object-path storage object)))
 
     ;; But you can't retrieve the object again because in database is
     ;; marked as deleted/expired.
-    (t/is (nil? @(sto/get-object storage (:id object))))
+    (t/is (nil? (sto/get-object storage (:id object))))
     ))
 
 (t/deftest test-deleted-gc-task
@@ -99,14 +99,14 @@
                     (configure-storage-backend))
         content1 (sto/content "content1")
         content2 (sto/content "content2")
-        object1  @(sto/put-object! storage {::sto/content content1
-                                            ::sto/expired-at (dt/now)
-                                            :content-type "text/plain"
-                                            })
-        object2  @(sto/put-object! storage {::sto/content content2
-                                            ::sto/expired-at (dt/in-past {:hours 2})
-                                            :content-type "text/plain"
-                                            })]
+        object1  (sto/put-object! storage {::sto/content content1
+                                           ::sto/expired-at (dt/now)
+                                           :content-type "text/plain"
+                                           })
+        object2  (sto/put-object! storage {::sto/content content2
+                                           ::sto/expired-at (dt/in-past {:hours 2})
+                                           :content-type "text/plain"
+                                           })]
 
     (th/sleep 200)