✨ Improve handling of temporal files.

Store temporal files outside of main fs backend.

backend/src/app/main.clj

@@ -287,24 +287,25 @@
     :app.storage/storage
     {:pool     (ig/ref :app.db/pool)
      :executor (ig/ref :app.worker/executor)
-     :backends {:s3  (ig/ref :app.storage.s3/backend)
+     :backend  (:storage-backend cfg/config :fs)
-                :db  (ig/ref :app.storage.db/backend)
+     :backends {:s3  (ig/ref [::main :app.storage.s3/backend])
-                :fs  (ig/ref :app.storage.fs/backend)}}
+                :db  (ig/ref [::main :app.storage.db/backend])
+                :fs  (ig/ref [::main :app.storage.fs/backend])
+                :tmp (ig/ref [::tmp  :app.storage.fs/backend])}}
 
-    :app.storage.s3/backend
+    [::main :app.storage.s3/backend]
     {:region (:storage-s3-region cfg/config)
      :bucket (:storage-s3-bucket cfg/config)}
 
-    :app.storage.fs/backend
+    [::main :app.storage.fs/backend]
-    {:directory (:storage-fs-directory cfg/config)
+    {:directory (:storage-fs-directory cfg/config)}
-     :uri       (:storage-fs-uri cfg/config)}
 
-    :app.storage.db/backend
+    [::tmp :app.storage.fs/backend]
+    {:directory "/tmp/penpot"}
+
+    [::main :app.storage.db/backend]
     {:pool (ig/ref :app.db/pool)}}
 
-   (let [backend (:storage-backend cfg/config :fs)]
-     {:app.storage/storage {:backend backend}})
-
    (when (:telemetry-server-enabled cfg/config)
      {:app.telemetry/handler
       {:pool     (ig/ref :app.db/pool)

backend/src/app/rpc/mutations/media.clj

@@ -19,7 +19,7 @@
    [app.storage :as sto]
    [app.util.http :as http]
    [app.util.services :as sv]
-   [app.util.storage :as ust]
+   [app.util.time :as dt]
    [clojure.java.io :as io]
    [clojure.spec.alpha :as s]
    [datoteka.core :as fs]))
@@ -73,12 +73,15 @@
         data   (:body result)
         mtype  (get (:headers result) "content-type")
         format (cm/mtype->format mtype)]
-    (if (nil? format)
+    (when (nil? format)
       (ex/raise :type :validation
                 :code :media-type-not-allowed
-                :hint "Seems like the url points to an invalid media object.")
+                :hint "Seems like the url points to an invalid media object."))
-      (sto/put-tmp-object storage {:content (sto/content data)
+    (-> (assoc storage :backend :tmp)
-                                   :content-type mtype}))))
+        (sto/put-object {:content (sto/content data)
+                         :content-type mtype
+                         :expired-at (dt/in-future {:minutes 30})}))))
+
 
 (defn create-file-media-object
   [{:keys [conn storage svgc] :as cfg} {:keys [id file-id is-local name content] :as params}]
@@ -132,8 +135,7 @@
       (let [mobj    (download-media cfg url)
             content {:filename "tempfile"
                      :size (:size mobj)
-                     :tempfile (-> (sto/get-object-url storage mobj)
+                     :tempfile (sto/get-object-path storage mobj)
-                                   (sto/file-url->path))
                      :content-type (:content-type (meta mobj))}
             params' (merge params {:content content
                                    :name (or name (:filename content))})]

backend/src/app/storage.clj

@@ -87,6 +87,8 @@
 (defn- create-database-object
   [{:keys [conn backend]} {:keys [content] :as object}]
   (if (instance? StorageObject object)
+    ;; If we in this condition branch, this means we come from the
+    ;; clone-object, so we just need to clone it with a new backend.
     (let [id     (uuid/random)
           mdata  (meta object)
           result (insert-object conn
@@ -151,6 +153,14 @@
 
 (declare resolve-backend)
 
+(defn object->relative-path
+  [{:keys [id] :as obj}]
+  (impl/id->path id))
+
+(defn file-url->path
+  [url]
+  (fs/path (java.net.URI. (str url))))
+
 (defn content
   ([data] (impl/content data nil))
   ([data size] (impl/content data size)))
@@ -162,6 +172,7 @@
       (retrieve-database-object id)))
 
 (defn put-object
+  "Creates a new object with the provided content."
   [{:keys [pool conn backend executor] :as storage} {:keys [content] :as params}]
   (us/assert ::storage storage)
   (us/assert impl/content? content)
@@ -180,6 +191,8 @@
     object))
 
 (defn clone-object
+  "Creates a clone of the provided object using backend basded efficient
+  method. Always clones objects to the configured default."
   [{:keys [pool conn executor] :as storage} object]
   (us/assert ::storage storage)
   (let [storage (assoc storage :conn (or conn pool))
@@ -217,13 +230,17 @@
        (resolve-backend (:backend object))
        (impl/get-object-url object options))))
 
-(defn object->relative-path
+(defn get-object-path
-  [{:keys [id] :as obj}]
+  "Get the Path to the object. Only works with `:fs` type of
-  (impl/id->path id))
+  storages."
-
+  [{:keys [backend conn path] :as storage} object]
-(defn file-url->path
+  (let [backend (resolve-backend storage (:backend object))]
-  [url]
+    (when (not= :fs (:type backend))
-  (fs/path (java.net.URI. (str url))))
+      (ex/raise :type :internal
+                :code :operation-not-allowed
+                :hint "get-object-path only works with fs type backends"))
+    (-> (impl/get-object-url backend object nil)
+        (file-url->path))))
 
 (defn del-object
   [{:keys [conn pool] :as storage} id-or-obj]
@@ -231,16 +248,6 @@
   (-> (assoc storage :conn (or conn pool))
       (delete-database-object (if (uuid? id-or-obj) id-or-obj (:id id-or-obj)))))
 
-(defn put-tmp-object
-  "A special function for create an object explicitly setting the TMP backend
-  and marking the object as deleted."
-  [storage params]
-  (let [storage (assoc storage :backend :fs)
-        params  (assoc params
-                       :expired-at (dt/in-future {:minutes 30})
-                       :temporal true)]
-    (put-object storage params)))
-
 ;; --- impl
 
 (defn resolve-backend