Merge pull request #4145 from penpot/niwinz-staging-tmp

✨ Minor improvements on TMP storage API

backend/dev/user.clj

@@ -44,7 +44,7 @@
    [clojure.walk :refer [macroexpand-all]]
    [criterium.core  :as crit]
    [cuerdas.core :as str]
-   [datoteka.core]
+   [datoteka.fs :as fs]
    [integrant.core :as ig]
    [malli.core :as m]
    [malli.dev.pretty :as mdp]

backend/src/app/features/components_v2.clj

@@ -1524,21 +1524,9 @@
   (cfv/validate-file! file libs)
   (cfv/validate-file-schema! file))
 
-(defn- process-file
+(defn- persist-file!
-  [{:keys [::db/conn] :as system} {:keys [id] :as file} & {:keys [validate?]}]
+  [{:keys [::db/conn] :as system} {:keys [id] :as file}]
-  (let [libs  (->> (files/get-file-libraries conn id)
+  (let [file (if (contains? (:features file) "fdata/objects-map")
-                   (into [file] (comp (map :id)
-                                      (map (partial get-file system))))
-                   (d/index-by :id))
-
-        file  (-> file
-                  (update :data migrate-fdata libs)
-                  (update :features conj "components/v2"))
-
-        _     (when validate?
-                (validate-file! file libs))
-
-        file (if (contains? (:features file) "fdata/objects-map")
                (fdata/enable-objects-map file)
                file)
 
@@ -1547,15 +1535,32 @@
                  (let [file (fdata/enable-pointer-map file)]
                    (fdata/persist-pointers! system id)
                    file))
-               file)]
+               file)
+
+        ;; Ensure all files has :data with id
+        file (update file :data assoc :id id)]
 
     (db/update! conn :file
                 {:data (blob/encode (:data file))
                  :features (db/create-array conn "text" (:features file))
                  :revn (:revn file)}
-                {:id (:id file)})
+                {:id (:id file)})))
 
-    (dissoc file :data)))
+(defn- process-file!
+  [{:keys [::db/conn] :as system} {:keys [id] :as file} & {:keys [validate?]}]
+  (let [libs  (->> (files/get-file-libraries conn id)
+                   (into [file] (comp (map :id)
+                                      (map (partial get-file system))))
+                   (d/index-by :id))
+
+        file  (-> file
+                  (update :data migrate-fdata libs)
+                  (update :features conj "components/v2"))]
+
+    (when validate?
+      (validate-file! file libs))
+
+    file))
 
 (def ^:private sql:get-and-lock-team-files
   "SELECT f.id
@@ -1605,13 +1610,15 @@
                         (when (string? label)
                           (fsnap/take-file-snapshot! system {:file-id file-id
                                                              :label (str "migration/" label)}))
-                        (let [file (get-file system file-id)]
+                        (let [file (get-file system file-id)
+                              file (process-file! system file :validate? validate?)]
+
                           (events/tap :progress
                                       {:op :migrate-file
                                        :name (:name file)
                                        :id (:id file)})
 
-                          (process-file system file :validate? validate?)))))
+                          (persist-file! system file)))))
 
         (catch Throwable cause
           (vreset! err true)

backend/src/app/storage/tmp.clj

@@ -19,6 +19,8 @@
    [promesa.exec :as px]
    [promesa.exec.csp :as sp]))
 
+(def default-tmp-dir "/tmp/penpot")
+
 (declare ^:private remove-temp-file)
 (declare ^:private io-loop)
 
@@ -33,6 +35,7 @@
 
 (defmethod ig/init-key ::cleaner
   [_ cfg]
+  (fs/create-dir default-tmp-dir)
   (px/fn->thread (partial io-loop cfg)
                  {:name "penpot/storage/tmp-cleaner" :virtual true}))
 
@@ -70,18 +73,14 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (defn tempfile
-  "Returns a tmpfile candidate (without creating it)"
   [& {:keys [suffix prefix min-age]
       :or {prefix "penpot."
            suffix ".tmp"}}]
-  (let [path (fs/tempfile :suffix suffix :prefix prefix)]
+  (let [path (fs/create-tempfile
-    (sp/offer! queue [path (some-> min-age dt/duration)])
+              :perms "rw-r--r--"
-    path))
+              :dir default-tmp-dir
-
+              :suffix suffix
-(defn create-tempfile
+              :prefix prefix)]
-  [& {:keys [suffix prefix min-age]
+    (fs/delete-on-exit! path)
-      :or {prefix "penpot."
-           suffix ".tmp"}}]
-  (let [path (fs/create-tempfile :suffix suffix :prefix prefix)]
     (sp/offer! queue [path (some-> min-age dt/duration)])
     path))

backend/test/backend_tests/helpers.clj

@@ -38,7 +38,7 @@
    [clojure.spec.alpha :as s]
    [clojure.test :as t]
    [cuerdas.core :as str]
-   [datoteka.core :as fs]
+   [datoteka.fs :as fs]
    [environ.core :refer [env]]
    [expound.alpha :as expound]
    [integrant.core :as ig]
@@ -127,6 +127,8 @@
                 app.auth/verify-password (fn [a b] {:valid (= a b)})
                 app.common.features/get-enabled-features (fn [& _] app.common.features/supported-features)]
 
+    (fs/create-dir "/tmp/penpot")
+
     (let [templates [{:id "test"
                       :name "test"
                       :file-uri "test"
@@ -191,6 +193,7 @@
   (let [path (fs/path "/tmp/penpot")]
     (when (fs/exists? path)
       (fs/delete (fs/path "/tmp/penpot")))
+    (fs/create-dir "/tmp/penpot")
     (next)))
 
 (defn serial
@@ -496,7 +499,7 @@
 (defn tempfile
   [source]
   (let [rsc (io/resource source)
-        tmp (fs/create-tempfile)]
+        tmp (fs/create-tempfile :dir "/tmp/penpot" :prefix "test-")]
     (io/copy (io/file rsc)
              (io/file tmp))
     tmp))

backend/test/backend_tests/rpc_comment_test.clj

@@ -17,7 +17,7 @@
    [app.util.time :as dt]
    [backend-tests.helpers :as th]
    [clojure.test :as t]
-   [datoteka.core :as fs]
+   [datoteka.fs :as fs]
    [mockery.core :refer [with-mocks]]))
 
 (t/use-fixtures :once th/state-init)

backend/test/backend_tests/rpc_cond_middleware_test.clj

@@ -15,7 +15,7 @@
    [backend-tests.helpers :as th]
    [backend-tests.storage-test :refer [configure-storage-backend]]
    [clojure.test :as t]
-   [datoteka.core :as fs]))
+   [datoteka.fs :as fs]))
 
 (t/use-fixtures :once th/state-init)
 (t/use-fixtures :each th/database-reset)

backend/test/backend_tests/rpc_file_thumbnails_test.clj

@@ -21,7 +21,7 @@
    [clojure.java.io :as io]
    [clojure.test :as t]
    [cuerdas.core :as str]
-   [datoteka.core :as fs]
+   [datoteka.fs :as fs]
    [mockery.core :refer [with-mocks]]))
 
 (t/use-fixtures :once th/state-init)

backend/test/backend_tests/rpc_management_test.clj

@@ -16,7 +16,7 @@
    [backend-tests.storage-test :refer [configure-storage-backend]]
    [buddy.core.bytes :as b]
    [clojure.test :as t]
-   [datoteka.core :as fs]))
+   [datoteka.fs :as fs]))
 
 (t/use-fixtures :once th/state-init)
 (t/use-fixtures :each th/database-reset)

backend/test/backend_tests/rpc_media_test.clj

@@ -12,7 +12,7 @@
    [app.storage :as sto]
    [backend-tests.helpers :as th]
    [clojure.test :as t]
-   [datoteka.core :as fs]))
+   [datoteka.fs :as fs]))
 
 (t/use-fixtures :once th/state-init)
 (t/use-fixtures :each th/database-reset)

backend/test/backend_tests/rpc_profile_test.clj

@@ -18,7 +18,7 @@
    [clojure.java.io :as io]
    [clojure.test :as t]
    [cuerdas.core :as str]
-   [datoteka.core :as fs]
+   [datoteka.fs :as fs]
    [mockery.core :refer [with-mocks]]))
 
 ;; TODO: profile deletion with teams

backend/test/backend_tests/rpc_quotes_test.clj

@@ -14,7 +14,7 @@
    [app.rpc.quotes :as-alias quotes]
    [backend-tests.helpers :as th]
    [clojure.test :as t]
-   [datoteka.core :as fs]
+   [datoteka.fs :as fs]
    [mockery.core :refer [with-mocks]]))
 
 (t/use-fixtures :once th/state-init)

backend/test/backend_tests/rpc_team_test.clj

@@ -16,7 +16,7 @@
    [app.util.time :as dt]
    [backend-tests.helpers :as th]
    [clojure.test :as t]
-   [datoteka.core :as fs]
+   [datoteka.fs :as fs]
    [mockery.core :refer [with-mocks]]))
 
 (t/use-fixtures :once th/state-init)

backend/test/backend_tests/rpc_viewer_test.clj

@@ -11,7 +11,7 @@
    [app.rpc :as-alias rpc]
    [backend-tests.helpers :as th]
    [clojure.test :as t]
-   [datoteka.core :as fs]))
+   [datoteka.fs :as fs]))
 
 (t/use-fixtures :once th/state-init)
 (t/use-fixtures :each th/database-reset)

backend/test/backend_tests/storage_test.clj

@@ -15,7 +15,7 @@
    [backend-tests.helpers :as th]
    [clojure.test :as t]
    [cuerdas.core :as str]
-   [datoteka.core :as fs]
+   [datoteka.fs :as fs]
    [datoteka.io :as io]
    [mockery.core :refer [with-mocks]]))
 

common/deps.edn

@@ -32,11 +32,14 @@
 
   funcool/tubax {:mvn/version "2021.05.20-0"}
   funcool/cuerdas {:mvn/version "2023.11.09-407"}
-  funcool/promesa {:git/sha "0c5ed6ad033515a2df4b55addea044f60e9653d0"
+  funcool/promesa
+  {:git/sha "0c5ed6ad033515a2df4b55addea044f60e9653d0"
    :git/url "https://github.com/funcool/promesa"}
 
-  funcool/datoteka {:mvn/version "3.0.66"
+  funcool/datoteka
-                    :exclusions [funcool/promesa]}
+  {:git/sha "5ac3781"
+   :git/tag "3.0.0"
+   :git/url "https://github.com/funcool/datoteka"}
 
   lambdaisland/uri {:mvn/version "1.16.134"
                     :exclusions [org.clojure/data.json]}