♻️ Refactor binfile implementation

backend/resources/log4j2-devenv.xml

@@ -25,7 +25,7 @@
     <Logger name="org.postgresql" level="error" />
 
     <Logger name="app.rpc.commands.binfile" level="debug" />
-    <Logger name="app.storage.tmp" level="trace" />
+    <Logger name="app.storage.tmp" level="debug" />
     <Logger name="app.worker" level="info" />
     <Logger name="app.msgbus" level="info" />
     <Logger name="app.http.websocket" level="info" />

backend/src/app/rpc/commands/binfile.clj

@@ -42,7 +42,7 @@
 (set! *warn-on-reflection* true)
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; VARS & DEFAULTS
+;; DEFAULTS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Threshold in MiB when we pass from using
@@ -50,22 +50,6 @@
 (def temp-file-threshold
   (* 1024 1024 2))
 
-;; Represents the current processing file-id on
-;; export process.
-(def ^:dynamic *file-id*)
-
-;; Stores all media file object references of
-;; processed files on import process.
-(def ^:dynamic *media*)
-
-;; Stores the objects index on reamping subprocess
-;; part of the import process.
-(def ^:dynamic *index*)
-
-;; Has the current connection used on the import
-;; process.
-(def ^:dynamic *conn*)
-
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; LOW LEVEL STREAM IO API
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -211,33 +195,33 @@
     (read-obj! input)))
 
 (defn write-header!
-  [^DataOutputStream output & {:keys [version metadata]}]
+  [^OutputStream output version]
   (l/trace :fn "write-header!"
            :version version
-           :metadata metadata
            :position @*position*
            ::l/async false)
-
+  (let [vers   (-> version name (subs 1) parse-long)
+        output (bs/data-output-stream output)]
     (doto output
       (write-byte! (get-mark :header))
       (write-long! penpot-magic-number)
-    (write-long! version)
+      (write-long! vers))))
-    (write-obj! metadata)))
 
 (defn read-header!
-  [^DataInputStream input]
+  [^InputStream input]
   (l/trace :fn "read-header!" :position @*position* ::l/async false)
-  (let [mark  (read-byte! input)
+  (let [input (bs/data-input-stream input)
+        mark  (read-byte! input)
         mnum  (read-long! input)
         vers  (read-long! input)]
 
     (when (or (not= mark (get-mark :header))
               (not= mnum penpot-magic-number))
       (ex/raise :type :validation
-                :code :invalid-penpot-file))
+                :code :invalid-penpot-file
+                :hint "invalid penpot file"))
 
-    (-> (read-obj! input)
+    (keyword (str "v" vers))))
-        (assoc ::version vers))))
 
 (defn copy-stream!
   [^OutputStream output ^InputStream input ^long size]
@@ -349,8 +333,84 @@
   (with-open [^AutoCloseable conn (db/open pool)]
     (db/exec! conn [sql:file-library-rels (db/create-array conn "uuid" ids)])))
 
+
+(defn- create-or-update-file
+  [conn params]
+  (let [sql (str "INSERT INTO file (id, project_id, name, revn, is_shared, data, created_at, modified_at) "
+                 "VALUES (?, ?, ?, ?, ?, ?, ?, ?) "
+                 "ON CONFLICT (id) DO UPDATE SET data=?")]
+    (db/exec-one! conn [sql
+                        (:id params)
+                        (:project-id params)
+                        (:name params)
+                        (:revn params)
+                        (:is-shared params)
+                        (:data params)
+                        (:created-at params)
+                        (:modified-at params)
+                        (:data params)])))
+
+;; --- GENERAL PURPOSE DYNAMIC VARS
+
+(def ^:dynamic *state*)
+(def ^:dynamic *options*)
+
 ;; --- EXPORT WRITTER
 
+(defn- embed-file-assets
+  [data conn file-id]
+  (letfn [(walk-map-form [form state]
+            (cond
+              (uuid? (:fill-color-ref-file form))
+              (do
+                (vswap! state conj [(:fill-color-ref-file form) :colors (:fill-color-ref-id form)])
+                (assoc form :fill-color-ref-file file-id))
+
+              (uuid? (:stroke-color-ref-file form))
+              (do
+                (vswap! state conj [(:stroke-color-ref-file form) :colors (:stroke-color-ref-id form)])
+                (assoc form :stroke-color-ref-file file-id))
+
+              (uuid? (:typography-ref-file form))
+              (do
+                (vswap! state conj [(:typography-ref-file form) :typographies (:typography-ref-id form)])
+                (assoc form :typography-ref-file file-id))
+
+              (uuid? (:component-file form))
+              (do
+                (vswap! state conj [(:component-file form) :components (:component-id form)])
+                (assoc form :component-file file-id))
+
+              :else
+              form))
+
+          (process-group-of-assets [data [lib-id items]]
+            ;; NOTE: there are a posibility that shape refers to a not
+            ;; existing file because the file was removed. In this
+            ;; case we just ignore the asset.
+            (if-let [lib (retrieve-file conn lib-id)]
+              (reduce (partial process-asset lib) data items)
+              data))
+
+          (process-asset [lib data [bucket asset-id]]
+            (let [asset (get-in lib [:data bucket asset-id])
+                  ;; Add a special case for colors that need to have
+                  ;; correctly set the :file-id prop (pending of the
+                  ;; refactor that will remove it).
+                  asset (cond-> asset
+                          (= bucket :colors) (assoc :file-id file-id))]
+              (update data bucket assoc asset-id asset)))]
+
+    (let [assets (volatile! [])]
+      (walk/postwalk #(cond-> % (map? %) (walk-map-form assets)) data)
+      (->> (deref assets)
+           (filter #(as-> (first %) $ (and (uuid? $) (not= $ file-id))))
+           (d/group-by first rest)
+           (reduce (partial process-group-of-assets) data)))))
+
+(defmulti write-export ::version)
+(defmulti write-section ::section)
+
 (s/def ::output bs/output-stream?)
 (s/def ::file-ids (s/every ::us/uuid :kind vector? :min-count 1))
 (s/def ::include-libraries? (s/nilable ::us/boolean))
@@ -370,68 +430,84 @@
   dependencies).
 
   `::embed-assets?`: instead of including the libraryes, embedd in the
-  same file library all assets used from external libraries.
+  same file library all assets used from external libraries."
-  "
+  [{:keys [::include-libraries? ::embed-assets?] :as options}]
-
-  [{:keys [pool storage ::output ::file-ids ::include-libraries? ::embed-assets?] :as options}]
-
   (us/assert! ::write-export-options options)
-
   (us/verify!
    :expr (not (and include-libraries? embed-assets?))
    :hint "the `include-libraries?` and `embed-assets?` are mutally excluding options")
+  (write-export options))
 
-  (letfn [(write-header [output files]
+(defmethod write-export :default
-            (let [sections [:v1/files :v1/rels :v1/sobjects]
+  [{:keys [::output] :as options}]
-                  mdata    {:penpot-version (:full cf/version)
+  (write-header! output :v1)
-                            :sections sections
+  (with-open [output (bs/zstd-output-stream output :level 12)]
-                            :files files}]
+    (with-open [output (bs/data-output-stream output)]
-              (write-header! output :version 1 :metadata mdata)))
+      (binding [*state* (volatile! {})]
+        (run! (fn [section]
+                (l/debug :hint "write section" :section section ::l/async false)
+                (write-label! output section)
+                (let [options (-> options
+                                  (assoc ::output output)
+                                  (assoc ::section section))]
+                  (binding [*options* options]
+                    (write-section options))))
 
-          (write-files [output files sids]
+              [:v1/metadata :v1/files :v1/rels :v1/sobjects])))))
-            (l/debug :hint "write section" :section :v1/files :total (count files) ::l/async false)
+
-            (write-label! output :v1/files)
+(defmethod write-section :v1/metadata
-            (doseq [file-id files]
+  [{:keys [pool ::output ::file-ids ::include-libraries?]}]
+  (let [libs  (when include-libraries?
+                (retrieve-libraries pool file-ids))
+        files (into file-ids libs)]
+    (write-obj! output {:version cf/version :files files})
+    (vswap! *state* assoc :files files)))
+
+(defmethod write-section :v1/files
+  [{:keys [pool ::output ::embed-assets?]}]
+
+  ;; Initialize SIDS with empty vector
+  (vswap! *state* assoc :sids [])
+
+  (doseq [file-id (-> *state* deref :files)]
     (let [file  (cond-> (retrieve-file pool file-id)
-                            embed-assets? (update :data embed-file-assets file-id))
+                  embed-assets?
+                  (update :data embed-file-assets pool file-id))
+
           media (retrieve-file-media pool file)]
 
-                ;; Collect all storage ids for later write them all under
+      (l/debug :hint "write penpot file"
-                ;; specific storage objects section.
-                (vswap! sids into (sequence storage-object-id-xf media))
-
-                (l/trace :hint "write penpot file"
                :id file-id
                :media (count media)
                ::l/async false)
 
       (doto output
         (write-obj! file)
-                  (write-obj! media)))))
+        (write-obj! media))
 
-          (write-rels [output files]
+      (vswap! *state* update :sids into storage-object-id-xf media))))
-            (let [rels (when include-libraries? (retrieve-library-relations pool files))]
-              (l/debug :hint "write section" :section :v1/rels :total (count rels) ::l/async false)
-              (doto output
-                (write-label! :v1/rels)
-                (write-obj! rels))))
 
-          (write-sobjects [output sids]
+(defmethod write-section :v1/rels
-            (l/debug :hint "write section"
+  [{:keys [pool  ::output ::include-libraries?]}]
-                     :section :v1/sobjects
+  (let [rels  (when include-libraries?
+                (retrieve-library-relations pool (-> *state* deref :files)))]
+    (l/debug :hint "found rels" :total (count rels) ::l/async false)
+    (write-obj! output rels)))
+
+(defmethod write-section :v1/sobjects
+  [{:keys [storage ::output]}]
+  (let [sids    (-> *state* deref :sids)
+        storage (media/configure-assets-storage storage)]
+    (l/debug :hint "found sobjects"
              :items (count sids)
              ::l/async false)
 
     ;; Write all collected storage objects
-            (doto output
+    (write-obj! output sids)
-              (write-label! :v1/sobjects)
-              (write-obj! sids))
 
-            (let [storage (media/configure-assets-storage storage)]
     (doseq [id sids]
       (let [{:keys [size] :as obj} @(sto/get-object storage id)]
-                  (l/trace :hint "write sobject" :id id ::l/async false)
+        (l/debug :hint "write sobject" :id id ::l/async false)
-
         (doto output
           (write-uuid! id)
           (write-obj! (meta obj)))
@@ -443,74 +519,15 @@
                         :code :mismatch-readed-size
                         :hint (str/ffmt "found unexpected object size; size=% written=%" size written)))))))))
 
-          (embed-file-assets [data file-id]
+;; --- EXPORT READER
-            (binding [*file-id* file-id]
-              (let [assets (volatile! [])]
-                (walk/postwalk #(cond-> % (map? %) (walk-map-form assets)) data)
-                (->> (deref assets)
-                     (filter #(as-> (first %) $ (and (uuid? $) (not= $ file-id))))
-                     (d/group-by first rest)
-                     (reduce process-group-of-assets data)))))
 
-          (walk-map-form [form state]
+(declare lookup-index)
-            (cond
+(declare update-index)
-              (uuid? (:fill-color-ref-file form))
+(declare relink-media)
-              (do
+(declare relink-shapes)
-                (vswap! state conj [(:fill-color-ref-file form) :colors (:fill-color-ref-id form)])
-                (assoc form :fill-color-ref-file *file-id*))
 
-              (uuid? (:stroke-color-ref-file form))
+(defmulti read-import ::version)
-              (do
+(defmulti read-section ::section)
-                (vswap! state conj [(:stroke-color-ref-file form) :colors (:stroke-color-ref-id form)])
-                (assoc form :stroke-color-ref-file *file-id*))
-
-              (uuid? (:typography-ref-file form))
-              (do
-                (vswap! state conj [(:typography-ref-file form) :typographies (:typography-ref-id form)])
-                (assoc form :typography-ref-file *file-id*))
-
-              (uuid? (:component-file form))
-              (do
-                (vswap! state conj [(:component-file form) :components (:component-id form)])
-                (assoc form :component-file *file-id*))
-
-              :else
-              form))
-
-          (process-group-of-assets [data [lib-id items]]
-            ;; NOTE: there are a posibility that shape refers to a not
-            ;; existing file because the file was removed. In this
-            ;; case we just ignore the asset.
-            (if-let [lib (retrieve-file pool lib-id)]
-              (reduce #(process-asset %1 lib %2) data items)
-              data))
-
-          (process-asset [data lib [bucket asset-id]]
-            (let [asset (get-in lib [:data bucket asset-id])
-                  ;; Add a special case for colors that need to have
-                  ;; correctly set the :file-id prop (pending of the
-                  ;; refactor that will remove it).
-                  asset (cond-> asset
-                          (= bucket :colors) (assoc :file-id *file-id*))]
-              (update data bucket assoc asset-id asset)))]
-
-    (with-open [output (bs/zstd-output-stream output :level 12)]
-      (with-open [output (bs/data-output-stream output)]
-        (let [libs  (when include-libraries? (retrieve-libraries pool file-ids))
-              files (into file-ids libs)
-              sids  (volatile! #{})]
-
-          ;; Write header with metadata
-          (l/debug :hint "exportation summary"
-                   :files (count files)
-                   :embed-assets? embed-assets?
-                   :include-libs? include-libraries?
-                   ::l/async false)
-
-          (write-header output files)
-          (write-files output files sids)
-          (write-rels output files)
-          (write-sobjects output (vec @sids)))))))
 
 (s/def ::project-id ::us/uuid)
 (s/def ::input bs/input-stream?)
@@ -538,31 +555,178 @@
   happen with broken files; defaults to: `false`.
   "
 
-  [{:keys [pool storage ::project-id ::timestamp ::input ::overwrite? ::migrate? ::ignore-index-errors?]
+  [{:keys [::input ::timestamp] :or {timestamp (dt/now)} :as options}]
-    :or {overwrite? false migrate? false timestamp (dt/now)}
-    :as options}]
-
   (us/verify! ::read-import-options options)
+  (let [version (read-header! input)]
+    (read-import (assoc options ::version version ::timestamp timestamp))))
 
-  (letfn [(lookup-index [id]
+(defmethod read-import :v1
-            (let [val (get @*index* id)]
+  [{:keys [pool ::input] :as options}]
+  (with-open [input (bs/zstd-input-stream input)]
+    (with-open [input (bs/data-input-stream input)]
+      (db/with-atomic [conn pool]
+        (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED;"])
+        (binding [*state* (volatile! {:media [] :index {}})]
+          (run! (fn [section]
+                  (l/debug :hint "reading section" :section section ::l/async false)
+                  (assert-read-label! input section)
+                  (let [options (-> options
+                                    (assoc ::section section)
+                                    (assoc ::input input)
+                                    (assoc :conn conn))]
+                    (binding [*options* options]
+                      (read-section options))))
+                [:v1/metadata :v1/files :v1/rels :v1/sobjects])
+
+          ;; Knowing that the ids of the created files are in
+          ;; index, just lookup them and return it as a set
+          (let [files (-> *state* deref :files)]
+            (into #{} (keep #(get-in @*state* [:index %])) files)))))))
+
+(defmethod read-section :v1/metadata
+  [{:keys [::input]}]
+  (let [{:keys [version files]} (read-obj! input)]
+    (l/debug :hint "metadata readed" :version (:full version) :files files ::l/async false)
+    (vswap! *state* update :index update-index files)
+    (vswap! *state* assoc :version version :files files)))
+
+(defmethod read-section :v1/files
+  [{:keys [conn ::input ::migrate? ::project-id ::timestamp ::overwrite?]}]
+  (doseq [expected-file-id (-> *state* deref :files)]
+    (let [file    (read-obj! input)
+          media'  (read-obj! input)
+          file-id (:id file)]
+
+      (when (not= file-id expected-file-id)
+        (ex/raise :type :validation
+                  :code :inconsistent-penpot-file
+                  :hint "the penpot file seems corrupt, found unexpected uuid (file-id)"))
+
+      ;; Update index using with media
+      (l/debug :hint "update index with media" ::l/async false)
+      (vswap! *state* update :index update-index (map :id media'))
+
+      ;; Store file media for later insertion
+      (l/debug :hint "update media references" ::l/async false)
+      (vswap! *state* update :media into (map #(update % :id lookup-index)) media')
+
+      (l/debug :hint "procesing file" :file-id file-id ::l/async false)
+
+      (let [file-id' (lookup-index file-id)
+            data     (-> (:data file)
+                         (assoc :id file-id')
+                         (cond-> migrate? (pmg/migrate-data))
+                         (update :pages-index relink-shapes)
+                         (update :components relink-shapes)
+                         (update :media relink-media))
+
+            params  {:id file-id'
+                     :project-id project-id
+                     :name (str "Imported: " (:name file))
+                     :revn (:revn file)
+                     :is-shared (:is-shared file)
+                     :data (blob/encode data)
+                     :created-at timestamp
+                     :modified-at timestamp}]
+
+        (l/debug :hint "create file" :id file-id' ::l/async false)
+
+        (if overwrite?
+          (create-or-update-file conn params)
+          (db/insert! conn :file params))
+
+        (when overwrite?
+          (db/delete! conn :file-thumbnail {:file-id file-id'}))))))
+
+(defmethod read-section :v1/rels
+  [{:keys [conn ::input ::timestamp]}]
+  (let [rels (read-obj! input)]
+    ;; Insert all file relations
+    (doseq [rel rels]
+      (let [rel (-> rel
+                    (assoc :synced-at timestamp)
+                    (update :file-id lookup-index)
+                    (update :library-file-id lookup-index))]
+        (l/debug :hint "create file library link"
+                 :file-id (:file-id rel)
+                 :lib-id (:library-file-id rel)
+                 ::l/async false)
+        (db/insert! conn :file-library-rel rel)))))
+
+(defmethod read-section :v1/sobjects
+  [{:keys [storage conn ::input ::overwrite?]}]
+  (let [storage (media/configure-assets-storage storage)
+        ids     (read-obj! input)]
+
+    (doseq [expected-storage-id ids]
+      (let [id    (read-uuid! input)
+            mdata (read-obj! input)]
+
+        (when (not= id expected-storage-id)
+          (ex/raise :type :validation
+                    :code :inconsistent-penpot-file
+                    :hint "the penpot file seems corrupt, found unexpected uuid (storage-object-id)"))
+
+        (l/debug :hint "readed storage object" :id id ::l/async false)
+
+        (let [[size resource] (read-stream! input)
+              hash            (sto/calculate-hash resource)
+              content         (-> (sto/content resource size)
+                                  (sto/wrap-with-hash hash))
+              params          (-> mdata
+                                  (assoc ::sto/deduplicate? true)
+                                  (assoc ::sto/content content)
+                                  (assoc ::sto/touched-at (dt/now))
+                                  (assoc :bucket "file-media-object"))
+
+              sobject         @(sto/put-object! storage params)]
+
+          (l/debug :hint "persisted storage object" :id id :new-id (:id sobject) ::l/async false)
+          (vswap! *state* update :index assoc id (:id sobject)))))
+
+    (doseq [item (:media @*state*)]
+      (l/debug :hint "inserting file media object"
+               :id (:id item)
+               :file-id (:file-id item)
+               ::l/async false)
+
+      (let [file-id (lookup-index (:file-id item))]
+        (if (= file-id (:file-id item))
+          (l/warn :hint "ignoring file media object" :file-id (:file-id item) ::l/async false)
+          (db/insert! conn :file-media-object
+                      (-> item
+                          (assoc :file-id file-id)
+                          (d/update-when :media-id lookup-index)
+                          (d/update-when :thumbnail-id lookup-index))
+                      {:on-conflict-do-nothing overwrite?}))))))
+
+(defn- lookup-index
+  [id]
+  (let [val (get-in @*state* [:index id])]
     (l/trace :fn "lookup-index" :id id :val val ::l/async false)
-              (when (and (not ignore-index-errors?) (not val))
+    (when (and (not (::ignore-index-errors? *options*)) (not val))
       (ex/raise :type :validation
                 :code :incomplete-index
                 :hint "looks like index has missing data"))
     (or val id)))
-          (update-index [index coll]
+
+(defn- update-index
+  [index coll]
   (loop [items (seq coll)
          index index]
     (if-let [id (first items)]
-                (let [new-id (if overwrite? id (uuid/next))]
+      (let [new-id (if (::overwrite? *options*) id (uuid/next))]
         (l/trace :fn "update-index" :id id :new-id new-id ::l/async false)
         (recur (rest items)
                (assoc index id new-id)))
       index)))
 
-          (process-map-form [form]
+(defn- relink-shapes
+  "A function responsible to analyze all file data and
+  replace the old :component-file reference with the new
+  ones, using the provided file-index."
+  [data]
+  (letfn [(process-map-form [form]
             (cond-> form
               ;; Relink Image Shapes
               (and (map? (:metadata form))
@@ -584,25 +748,22 @@
               ;; This covers the shadows and grids (they have directly
               ;; the :file-id prop)
               (uuid? (:file-id form))
-              (update :file-id lookup-index)))
+              (update :file-id lookup-index)))]
 
-          ;; a function responsible to analyze all file data and
-          ;; replace the old :component-file reference with the new
-          ;; ones, using the provided file-index
-          (relink-shapes [data]
     (walk/postwalk (fn [form]
                      (if (map? form)
                        (try
                          (process-map-form form)
                          (catch Throwable cause
-                                   (l/trace :hint "failed form" :form (pr-str form) ::l/async false)
+                           (l/warn :hint "failed form" :form (pr-str form) ::l/async false)
                            (throw cause)))
                        form))
-                           data))
+                   data)))
 
-          ;; A function responsible of process the :media attr of file
+(defn- relink-media
-          ;; data and remap the old ids with the new ones.
+  "A function responsible of process the :media attr of file data and
-          (relink-media [media]
+  remap the old ids with the new ones."
+  [media]
   (reduce-kv (fn [res k v]
                (let [id (lookup-index k)]
                  (if (uuid? id)
@@ -613,164 +774,9 @@
              media
              media))
 
-          (create-or-update-file [params]
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-            (let [sql (str "INSERT INTO file (id, project_id, name, revn, is_shared, data, created_at, modified_at) "
+;; HIGH LEVEL API
-                           "VALUES (?, ?, ?, ?, ?, ?, ?, ?) "
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-                           "ON CONFLICT (id) DO UPDATE SET data=?")]
-              (db/exec-one! *conn* [sql
-                                    (:id params)
-                                    (:project-id params)
-                                    (:name params)
-                                    (:revn params)
-                                    (:is-shared params)
-                                    (:data params)
-                                    (:created-at params)
-                                    (:modified-at params)
-                                    (:data params)])))
-
-          (read-files-section! [input expected-files]
-            (l/debug :hint "reading section" :section :v1/files ::l/async false)
-            (assert-read-label! input :v1/files)
-
-            ;; Process/Read all file
-            (doseq [expected-file-id expected-files]
-              (let [file    (read-obj! input)
-                    media'  (read-obj! input)
-                    file-id (:id file)]
-
-                (when (not= file-id expected-file-id)
-                  (ex/raise :type :validation
-                            :code :inconsistent-penpot-file
-                            :hint "the penpot file seems corrupt, found unexpected uuid (file-id)"))
-
-
-                ;; Update index using with media
-                (l/trace :hint "update index with media" ::l/async false)
-                (vswap! *index* update-index (map :id media'))
-
-                ;; Store file media for later insertion
-                (l/trace :hint "update media references" ::l/async false)
-                (vswap! *media* into (map #(update % :id lookup-index)) media')
-
-                (l/trace :hint "procesing file" :file-id file-id ::l/async false)
-
-                (let [file-id' (lookup-index file-id)
-                      data     (-> (:data file)
-                                   (assoc :id file-id')
-                                   (cond-> migrate? (pmg/migrate-data))
-                                   (update :pages-index relink-shapes)
-                                   (update :components relink-shapes)
-                                   (update :media relink-media))
-
-                      params  {:id file-id'
-                               :project-id project-id
-                               :name (str "Imported: " (:name file))
-                               :revn (:revn file)
-                               :is-shared (:is-shared file)
-                               :data (blob/encode data)
-                               :created-at timestamp
-                               :modified-at timestamp}]
-
-                  (l/trace :hint "create file" :id file-id' ::l/async false)
-
-                  (if overwrite?
-                    (create-or-update-file params)
-                    (db/insert! *conn* :file params))
-
-                  (when overwrite?
-                    (db/delete! *conn* :file-thumbnail {:file-id file-id'}))))))
-
-          (read-rels-section! [input]
-            (l/debug :hint "reading section" :section :v1/rels ::l/async false)
-            (assert-read-label! input :v1/rels)
-
-            (let [rels (read-obj! input)]
-              ;; Insert all file relations
-              (doseq [rel rels]
-                (let [rel (-> rel
-                              (assoc :synced-at timestamp)
-                              (update :file-id lookup-index)
-                              (update :library-file-id lookup-index))]
-                  (l/trace :hint "create file library link"
-                           :file-id (:file-id rel)
-                           :lib-id (:library-file-id rel)
-                           ::l/async false)
-                  (db/insert! *conn* :file-library-rel rel)))))
-
-          (read-sobjects-section! [input]
-            (l/debug :hint "reading section" :section :v1/sobjects ::l/async false)
-            (assert-read-label! input :v1/sobjects)
-
-            (let [storage (media/configure-assets-storage storage)
-                  ids     (read-obj! input)]
-
-              (doseq [expected-storage-id ids]
-                (let [id    (read-uuid! input)
-                      mdata (read-obj! input)]
-
-                  (when (not= id expected-storage-id)
-                    (ex/raise :type :validation
-                              :code :inconsistent-penpot-file
-                              :hint "the penpot file seems corrupt, found unexpected uuid (storage-object-id)"))
-
-                  (l/trace :hint "readed storage object" :id id ::l/async false)
-
-                  (let [[size resource] (read-stream! input)
-                        hash            (sto/calculate-hash resource)
-                        content         (-> (sto/content resource size)
-                                            (sto/wrap-with-hash hash))
-                        params          (-> mdata
-                                            (assoc ::sto/deduplicate? true)
-                                            (assoc ::sto/content content)
-                                            (assoc ::sto/touched-at (dt/now)))
-                        sobject         @(sto/put-object! storage params)]
-                    (l/trace :hint "persisted storage object" :id id :new-id (:id sobject) ::l/async false)
-                    (vswap! *index* assoc id (:id sobject)))))))
-
-          (persist-file-media-objects! []
-            (l/debug :hint "processing file media objects" :section :v1/sobjects ::l/async false)
-
-            ;; Step 2: insert all file-media-object rows with correct
-            ;; storage-id reference.
-            (doseq [item @*media*]
-              (l/trace :hint "inserting file media object"
-                       :id (:id item)
-                       :file-id (:file-id item)
-                       ::l/async false)
-
-              (let [file-id (lookup-index (:file-id item))]
-                (if (= file-id (:file-id item))
-                  (l/warn :hint "ignoring file media object" :file-id (:file-id item) ::l/async false)
-                  (db/insert! *conn* :file-media-object
-                              (-> item
-                                  (assoc :file-id file-id)
-                                  (d/update-when :media-id lookup-index)
-                                  (d/update-when :thumbnail-id lookup-index))
-                              {:on-conflict-do-nothing overwrite?})))))]
-
-    (with-open [input (bs/zstd-input-stream input)]
-      (with-open [input (bs/data-input-stream input)]
-        (db/with-atomic [conn pool]
-          (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED;"])
-
-          ;; Verify that we received a proper .penpot file
-          (let [{:keys [sections files]} (read-header! input)]
-            (l/debug :hint "import verified" :files files :overwrite? overwrite?)
-            (binding [*index* (volatile! (update-index {} files))
-                      *media* (volatile! [])
-                      *conn*  conn]
-
-              (doseq [section sections]
-                (case section
-                  :v1/rels     (read-rels-section! input)
-                  :v1/files    (read-files-section! input files)
-                  :v1/sobjects (do
-                                 (read-sobjects-section! input)
-                                 (persist-file-media-objects!))))
-
-              ;; Knowing that the ids of the created files are in
-              ;; index, just lookup them and return it as a set
-              (into #{} (keep #(get @*index* %)) files))))))))
 
 (defn export!
   [cfg]

frontend/resources/styles/main/partials/dashboard-grid.scss

@@ -203,7 +203,7 @@
         right: 5px;
         justify-content: center;
         position: absolute;
-        width: 15px;
+        width: 30px;
         height: 30px;
 
         span {
@@ -218,9 +218,11 @@
           &.menu {
             margin-right: 0;
             display: flex;
-            justify-content: flex-end;
+            justify-content: center;
             align-items: flex-end;
             flex-direction: column;
+            width: 100%;
+            height: 100%;
 
             > svg {
               fill: $color-gray-60;

frontend/src/app/main/ui/dashboard/grid.cljs

@@ -142,10 +142,10 @@
          (fn [event]
            (dom/prevent-default event)
            (when-not selected?
-             (let [shift? (kbd/shift? event)]
+             (when-not (kbd/shift? event)
-               (when-not shift?
                (st/emit! (dd/clear-selected-files)))
-               (st/emit! (dd/toggle-file-select file))))
+             (st/emit! (dd/toggle-file-select file)))
+
            (let [position (dom/get-client-position event)]
              (swap! local assoc
                     :menu-open true