Merge pull request #2058 from penpot/niwinz-exporter-tmp-files

✨ Put all temporal files under the same directory

exporter/src/app/browser.cljs

@@ -9,10 +9,8 @@
    ["generic-pool" :as gp]
    ["generic-pool/lib/errors" :as gpe]
    ["playwright" :as pw]
-   [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
-   [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.util.object :as obj]
    [promesa.core :as p]))

exporter/src/app/config.cljs

@@ -7,13 +7,10 @@
 (ns app.config
   (:refer-clojure :exclude [get])
   (:require
-   ["fs" :as fs]
    ["process" :as process]
-   [app.common.exceptions :as ex]
    [app.common.data :as d]
    [app.common.spec :as us]
    [app.common.version :as v]
-   [app.common.uri :as u]
    [cljs.core :as c]
    [cljs.pprint]
    [cljs.spec.alpha :as s]

exporter/src/app/core.cljs

@@ -8,17 +8,17 @@
   (:require
    ["process" :as proc]
    [app.browser :as bwr]
-   [app.redis :as redis]
    [app.common.logging :as l]
    [app.config]
    [app.http :as http]
+   [app.redis :as redis]
    [promesa.core :as p]))
 
 (enable-console-print!)
 (l/initialize!)
 
 (defn start
-  [& args]
+  [& _]
   (l/info :msg "initializing")
   (p/do!
    (bwr/init)

exporter/src/app/handlers.cljs

@@ -10,22 +10,18 @@
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
    [app.common.spec :as us]
-   [app.common.uri :as u]
-   [app.config :as cf]
    [app.handlers.export-frames :as export-frames]
    [app.handlers.export-shapes :as export-shapes]
    [app.handlers.resources :as resources]
    [app.util.transit :as t]
    [clojure.spec.alpha :as s]
-   [cuerdas.core :as str]
+   [cuerdas.core :as str]))
-   [promesa.core :as p]
-   [reitit.core :as r]))
 
 (l/set-level! :debug)
 
 (defn on-error
   [error exchange]
-  (let [{:keys [type message code] :as data} (ex-data error)]
+  (let [{:keys [type code] :as data} (ex-data error)]
     (cond
       (or (= :validation type)
           (= :assertion type))

exporter/src/app/handlers/export_frames.cljs

@@ -6,13 +6,10 @@
 
 (ns app.handlers.export-frames
   (:require
-   ["path" :as path]
    [app.common.logging :as l]
-   [app.common.exceptions :as exc]
    [app.common.spec :as us]
-   [app.common.pprint :as pp]
-   [app.handlers.resources :as rsc]
    [app.handlers.export-shapes :refer [prepare-exports]]
+   [app.handlers.resources :as rsc]
    [app.redis :as redis]
    [app.renderer :as rd]
    [app.util.shell :as sh]
@@ -41,7 +38,7 @@
           :opt-un [::name]))
 
 (defn handler
-  [{:keys [:request/auth-token] :as exchange} {:keys [exports profile-id] :as params}]
+  [{:keys [:request/auth-token] :as exchange} {:keys [exports] :as params}]
   ;; NOTE: we need to have the `:type` prop because the exports
   ;; datastructure preparation uses it for creating the groups.
   (let [exports  (-> (map #(assoc % :type :pdf :scale 1 :suffix "") exports)
@@ -111,7 +108,8 @@
 
     (-> (p/loop [exports (seq exports)]
           (when-let [export (first exports)]
-            (p/let [proc (rd/render export on-object)]
+            (p/do
+              (rd/render export on-object)
               (p/recur (rest exports)))))
 
         (p/then (fn [_] (deref result)))
@@ -122,14 +120,14 @@
                   (-> (sh/stat (:path resource))
                       (p/then #(merge resource %)))))
         (p/catch on-error)
-        (p/finally (fn [result cause]
+        (p/finally (fn [_ cause]
                      (when-not cause
                        (on-complete)))))))
 
 (defn- join-pdf
   [file-id paths]
-  (p/let [tmpdir (sh/mktmpdir! "join-pdf")
+  (p/let [prefix (str/concat "penpot.tmp.pdfunite." file-id ".")
-          path   (path/join tmpdir (str/concat file-id ".pdf"))]
+          path   (sh/tempfile :prefix prefix :suffix ".pdf")]
     (sh/run-cmd! (str "pdfunite " (str/join " " paths) " " path))
     path))
 
@@ -137,5 +135,4 @@
   [{:keys [path] :as resource} output-path]
   (p/do
     (sh/move! output-path path)
-    (sh/rmdir! (path/dirname output-path))
     resource))

exporter/src/app/handlers/export_shapes.cljs

@@ -6,9 +6,7 @@
 
 (ns app.handlers.export-shapes
   (:require
-   ["path" :as path]
    [app.common.data :as d]
-   [app.common.exceptions :as exc]
    [app.common.logging :as l]
    [app.common.spec :as us]
    [app.handlers.resources :as rsc]
@@ -102,8 +100,6 @@
         total       (count exports)
         topic       (str profile-id)
 
-        to-delete   (atom #{})
-
         on-progress (fn [{:keys [done]}]
                       (when-not wait
                         (let [data {:type :export-update
@@ -137,16 +133,15 @@
                                     :on-progress on-progress)
 
         append      (fn [{:keys [filename path] :as object}]
-                      (swap! to-delete conj path)
                       (rsc/add-to-zip! zip path filename))
 
         proc        (-> (p/do
                           (p/loop [exports (seq exports)]
                             (when-let [export (first exports)]
-                              (p/let [proc (rd/render export append)]
+                              (p/do
+                                (rd/render export append)
                                 (p/recur (rest exports)))))
                           (.finalize zip))
-                        (p/then (fn [_] (p/run! #(sh/rmdir! (path/dirname %)) @to-delete)))
                         (p/then (constantly resource))
                         (p/catch on-error))
         ]

exporter/src/app/handlers/resources.cljs

@@ -9,20 +9,18 @@
   (:require
    ["archiver" :as arc]
    ["fs" :as fs]
-   ["os" :as os]
    ["path" :as path]
-   [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.uuid :as uuid]
-   [app.util.shell :as sh]
    [app.util.mime :as mime]
+   [app.util.shell :as sh]
    [cljs.core :as c]
    [cuerdas.core :as str]
    [promesa.core :as p]))
 
 (defn- get-path
   [type id]
-  (path/join (os/tmpdir) (str/concat  "exporter-resource." (c/name type) "." id)))
+  (path/join sh/tmpdir (str/concat  "penpot.resource." (c/name type) "." id)))
 
 (defn create
   "Generates ephimeral resource object."
@@ -49,7 +47,7 @@
                "content-length" (:size stat)}}))
 
 (defn handler
-  [{:keys [:request/params response] :as exchange}]
+  [{:keys [:request/params] :as exchange}]
   (when-not (contains? params :id)
     (ex/raise :type :validation
               :code :missing-id))

exporter/src/app/http.cljs

@@ -12,7 +12,6 @@
    ["raw-body" :as raw-body]
    ["stream" :as stream]
    [app.common.logging :as l]
-   [app.common.spec :as us]
    [app.common.transit :as t]
    [app.config :as cf]
    [app.handlers :as handlers]

exporter/src/app/redis.cljs

@@ -28,7 +28,7 @@
     (.on client "reconnect"
          (fn [ms] (l/warn :hint "reconnecting to redis" :ms ms)))
     (.on client "end"
-         (fn [ms] (l/warn :hint "client ended, no more connections will be attempted")))
+         (fn [] (l/warn :hint "client ended, no more connections will be attempted")))
     client))
 
 (defn init

exporter/src/app/renderer.cljs

@@ -31,7 +31,7 @@
 (s/def ::render-params
   (s/keys :req-un [::file-id ::page-id ::scale ::token ::type ::objects]))
 
-(defn- render
+(defn render
   [{:keys [type] :as params} on-object]
   (us/verify ::render-params params)
   (us/verify fn? on-object)

exporter/src/app/renderer/bitmap.cljs

@@ -7,19 +7,12 @@
 (ns app.renderer.bitmap
   "A bitmap renderer."
   (:require
-   ["path" :as path]
    [app.browser :as bw]
-   [app.common.data :as d]
-   [app.common.data.macros :as dm]
-   [app.common.exceptions :as ex]
    [app.common.logging :as l]
-   [app.common.pages :as cp]
-   [app.common.spec :as us]
    [app.common.uri :as u]
    [app.config :as cf]
    [app.util.mime :as mime]
    [app.util.shell :as sh]
-   [cljs.spec.alpha :as s]
    [cuerdas.core :as str]
    [promesa.core :as p]))
 
@@ -36,8 +29,7 @@
                  :userAgent bw/default-user-agent})
 
           (render-object [page {:keys [id] :as object}]
-            (p/let [tmpdir (sh/mktmpdir! "bitmap-render")
+            (p/let [path (sh/tempfile :prefix "penpot.tmp.render.bitmap." :suffix (mime/get-extension type))
-                    path   (path/join tmpdir (str/concat id (mime/get-extension type)))
                     node (bw/select page (str/concat "#screenshot-" id))]
               (bw/wait-for node)
               (case type

exporter/src/app/renderer/pdf.cljs

@@ -7,18 +7,13 @@
 (ns app.renderer.pdf
   "A pdf renderer."
   (:require
-   ["path" :as path]
    [app.browser :as bw]
    [app.common.data.macros :as dm]
-   [app.common.exceptions :as ex :include-macros true]
    [app.common.logging :as l]
-   [app.common.spec :as us]
    [app.common.uri :as u]
    [app.config :as cf]
    [app.util.mime :as mime]
    [app.util.shell :as sh]
-   [cuerdas.core :as str]
-   [cljs.spec.alpha :as s]
    [promesa.core :as p]))
 
 (defn render
@@ -44,8 +39,7 @@
 
           (render-object [page base-uri {:keys [id] :as object}]
             (p/let [uri  (prepare-uri base-uri id)
-                    tmp  (sh/mktmpdir! "pdf-render")
+                    path (sh/tempfile :prefix "penpot.tmp.render.pdf." :suffix (mime/get-extension type))]
-                    path (path/join tmp (str/concat id (mime/get-extension type)))]
               (l/info :uri uri)
               (bw/nav! page uri)
               (p/let [dom (bw/select page (dm/str "#screenshot-" id))]
@@ -58,8 +52,7 @@
           (render [base-uri page]
             (p/loop [objects (seq objects)]
               (when-let [object (first objects)]
-                (p/let [uri  (prepare-uri base-uri (:id object))
+                (p/let [path (render-object page base-uri object)]
-                        path (render-object page base-uri object)]
                   (on-object (assoc object :path path))
                   (p/recur (rest objects))))))]
 

exporter/src/app/renderer/svg.cljs

@@ -6,20 +6,14 @@
 
 (ns app.renderer.svg
   (:require
-   ["path" :as path]
    ["xml-js" :as xml]
    [app.browser :as bw]
    [app.common.data :as d]
-   [app.common.data.macros :as dm]
-   [app.common.exceptions :as ex :include-macros true]
    [app.common.logging :as l]
-   [app.common.pages :as cp]
-   [app.common.spec :as us]
    [app.common.uri :as u]
    [app.config :as cf]
    [app.util.mime :as mime]
    [app.util.shell :as sh]
-   [cljs.spec.alpha :as s]
    [clojure.walk :as walk]
    [cuerdas.core :as str]
    [promesa.core :as p]))
@@ -116,24 +110,20 @@
 (defn render
   [{:keys [page-id file-id objects token scale type]} on-object]
   (letfn [(convert-to-ppm [pngpath]
-            (l/trace :fn :convert-to-ppm)
+            (let [ppmpath (str/concat pngpath "origin.ppm")]
-            (let [basepath (path/dirname pngpath)
+              (l/trace :fn :convert-to-ppm :path ppmpath)
-                  ppmpath  (path/join basepath "origin.ppm")]
               (-> (sh/run-cmd! (str "convert " pngpath " " ppmpath))
                   (p/then (constantly ppmpath)))))
 
           (trace-color-mask [pbmpath]
             (l/trace :fn :trace-color-mask :pbmpath pbmpath)
-            (let [basepath (path/dirname pbmpath)
+            (let [svgpath (str/concat pbmpath ".svg")]
-                  basename (path/basename pbmpath ".pbm")
-                  svgpath  (path/join basepath (str basename ".svg"))]
               (-> (sh/run-cmd! (str "potrace --flat -b svg " pbmpath " -o " svgpath))
                   (p/then (constantly svgpath)))))
 
           (generate-color-layer [ppmpath color]
             (l/trace :fn :generate-color-layer :ppmpath ppmpath :color color)
-            (let [basepath (path/dirname ppmpath)
+            (let [pbmpath (str/concat ppmpath ".mask-" (subs color 1) ".pbm")]
-                  pbmpath  (path/join basepath (str "mask-" (subs color 1) ".pbm"))]
               (-> (sh/run-cmd! (str/format "ppmcolormask \"%s\" %s" color ppmpath))
                   (p/then (fn [stdout]
                             (-> (sh/write-file! pbmpath stdout)
@@ -188,7 +178,7 @@
 
           (get-gradients [id mapping]
             (->> mapping
-                 (filter (fn [[color data]]
+                 (filter (fn [[_color data]]
                            (= (get data "type") "gradient")))
                  (mapv (partial data->gradient-def id))))
 
@@ -231,7 +221,7 @@
 
 
                       elements (cond->> elements
-                                 (not (empty? gradient-defs))
+                                 (seq gradient-defs)
                                  (into [{"type" "element" "name" "defs" "attributes" {}
                                          "elements" gradient-defs}]))]
 
@@ -247,15 +237,14 @@
 
           (trace-node [{:keys [data] :as node}]
             (l/trace :fn :trace-node)
-            (p/let [tdpath  (sh/mktmpdir! "svgexport")
+            (p/let [pngpath (sh/tempfile :prefix "penpot.tmp.render.svg.parse."
-                    pngpath (path/join tdpath "origin.png")
+                                         :suffix ".origin.png")
                     _       (sh/write-file! pngpath data)
                     ppmpath (convert-to-ppm pngpath)
                     svgdata (convert-to-svg ppmpath node)]
               (-> node
                   (dissoc :data)
-                  (assoc :tempdir tdpath
+                  (assoc :svgdata svgdata))))
-                         :svgdata svgdata))))
 
           (extract-element-attrs [^js element]
             (let [^js attrs   (.. element -attributes)
@@ -289,17 +278,11 @@
                     shot (bw/screenshot text-node {:omit-background? true :type "png"})]
               [shot node]))
 
-          (clean-temp-data [{:keys [tempdir] :as node}]
-            (p/do!
-             (sh/rmdir! tempdir)
-             (dissoc node :tempdir)))
-
           (extract-txt-node [page item]
             (-> (p/resolved item)
                 (p/then (partial resolve-text-node page))
                 (p/then extract-single-node)
-                (p/then trace-node)
+                (p/then trace-node)))
-                (p/then clean-temp-data)))
 
           (extract-txt-nodes [page {:keys [id] :as objects}]
             (l/trace :fn :process-text-nodes)
@@ -323,8 +306,7 @@
                  :userAgent bw/default-user-agent})
 
           (render-object [page {:keys [id] :as object}]
-            (p/let [tmpdir (sh/mktmpdir! "svg-render")
+            (p/let [path (sh/tempfile :prefix "penpot.tmp.render.svg." :suffix (mime/get-extension type))
-                    path   (path/join tmpdir (str/concat id (mime/get-extension type)))
                     node (bw/select page (str/concat "#screenshot-" id))]
               (bw/wait-for node)
               (p/let [xmldata (extract-svg page object)

exporter/src/app/util/mime.cljs

@@ -8,7 +8,6 @@
   "Mimetype and file extension helpers."
   (:refer-clojure :exclude [get])
   (:require
-   [app.common.data :as d]
    [cljs.core :as c]))
 
 (defn get-extension
@@ -20,7 +19,7 @@
     :pdf  ".pdf"
     :zip  ".zip"))
 
-(defn- get
+(defn get
   [type]
   (case type
     :zip  "application/zip"

exporter/src/app/util/shell.cljs

@@ -11,14 +11,49 @@
    ["fs" :as fs]
    ["os" :as os]
    ["path" :as path]
+   [app.common.exceptions :as ex]
    [app.common.logging :as l]
+   [app.common.uuid :as uuid]
+   [cuerdas.core :as str]
    [promesa.core :as p]))
 
 (l/set-level! :trace)
 
-(defn mktmpdir!
+(def tempfile-minage (* 1000 60 60 1)) ;; 1h
-  [prefix]
+
-  (.mkdtemp fs/promises (path/join (os/tmpdir) prefix)))
+(def tmpdir
+  (let [path (path/join (os/tmpdir) "penpot")]
+    (when-not (fs/existsSync path)
+      (fs/mkdirSync path #js {:recursive true}))
+    path))
+
+
+(defn- schedule-deletion!
+  [path]
+  (letfn [(remote-tempfile []
+            (when (fs/existsSync path)
+              (l/trace :hint "permanently remove tempfile" :path path)
+              (fs/rmSync path #js {:recursive true})))]
+    (l/trace :hint "schedule tempfile deletion"
+             :path path
+             :scheduled-at (.. (js/Date. (+ (js/Date.now) tempfile-minage)) toString))
+    (js/setTimeout remote-tempfile tempfile-minage)))
+
+(defn tempfile
+  [& {:keys [prefix suffix]
+      :or {prefix "penpot."
+           suffix ".tmp"}}]
+  (loop [i 0]
+    (if (< i 1000)
+      (let [path (path/join tmpdir (str/concat prefix (uuid/next) "-" i  suffix))]
+        (if (fs/existsSync path)
+          (recur (inc i))
+          (do
+            (schedule-deletion! path)
+            path)))
+      (ex/raise :type :internal
+                :code :unable-to-locate-temporal-file
+                :hint "unable to find a tempfile candidate"))))
 
 (defn move!
   [origin-path dest-path]
@@ -50,7 +85,7 @@
    (fn [resolve reject]
      (l/trace :fn :run-cmd :cmd cmd)
      (proc/exec cmd #js {:encoding "buffer"}
-                (fn [error stdout stderr]
+                (fn [error stdout _stderr]
                   ;; (l/trace :fn :run-cmd :stdout stdout)
                   (if error
                     (reject error)

frontend/src/app/main/data/websocket.cljs

@@ -55,6 +55,7 @@
                        (rx/filter (ptk/type? ::initialize) stream))]
 
           (->> (rx/merge
+                (rx/of #(assoc % :ws-conn ws))
                 (->> (ws/get-rcv-stream ws)
                      (rx/filter ws/message-event?)
                      (rx/map :payload)
@@ -69,6 +70,10 @@
 (defn finalize
   []
   (ptk/reify ::finalize
+    ptk/UpdateEvent
+    (update [_ state]
+      (dissoc state :ws-conn))
+
     ptk/EffectEvent
     (effect [_ _ _]
       (l/trace :hint "event:finalize" :fn "effect")