⚡ Improve performance on import .penpot files.

CHANGES.md

@@ -22,6 +22,7 @@
 - Add actions to go to main component context menu option [Taiga #2053](https://tree.taiga.io/project/penpot/us/2053).
 - Add contrast between component select color and shape select color [Taiga #2121](https://tree.taiga.io/project/penpot/issue/2121).
 - Add animations in interactions [Taiga #2244](https://tree.taiga.io/project/penpot/us/2244).
+- Add performance improvements on .penpot file import process [Taiga 2497](https://tree.taiga.io/project/penpot/us/2497).
 
 ### :bug: Bugs fixed
 

backend/src/app/rpc/mutations/files.clj

@@ -414,7 +414,9 @@
   [conn project-id]
   (:team-id (db/get-by-id conn :project project-id {:columns [:team-id]})))
 
-;; TEMPORARY FILE CREATION
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; TEMPORARY FILES (behaves differently)
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (s/def ::create-temp-file ::create-file)
 
@@ -424,6 +426,23 @@
     (proj/check-edition-permissions! conn profile-id project-id)
     (create-file conn (assoc params :deleted-at (dt/in-future {:days 1})))))
 
+(s/def ::update-temp-file
+  (s/keys :req-un [::changes ::revn ::session-id ::id]))
+
+(sv/defmethod ::update-temp-file
+  [{:keys [pool] :as cfg} {:keys [profile-id session-id id revn changes] :as params}]
+  (db/with-atomic [conn pool]
+    (db/insert! conn :file-change
+                {:id (uuid/next)
+                 :session-id session-id
+                 :profile-id profile-id
+                 :created-at (dt/now)
+                 :file-id id
+                 :revn revn
+                 :data nil
+                 :changes (blob/encode changes)})
+    nil))
+
 (s/def ::persist-temp-file
   (s/keys :req-un [::id ::profile-id]))
 
@@ -431,6 +450,25 @@
   [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}]
   (db/with-atomic [conn pool]
     (files/check-edition-permissions! conn profile-id id)
+    (let [file (db/get-by-id conn :file id)
+          revs (db/query conn :file-change
+                         {:file-id id}
+                         {:order-by [[:revn :asc]]})
+          revn (count revs)]
+
+      (when (nil? (:deleted-at file))
+        (ex/raise :type :validation
+                  :code :cant-persist-already-persisted-file))
+
+      (loop [revs (seq revs)
+             data (blob/decode (:data file))]
+        (if-let [rev (first revs)]
+          (recur (rest revs)
+                 (->> rev :changes blob/decode (cp/process-changes data)))
           (db/update! conn :file
-                {:deleted-at nil}
+                      {:deleted-at nil
+                       :revn revn
+                       :data (blob/encode data)}
                       {:id id})))
+
+      nil)))

frontend/src/app/worker/import.cljs

@@ -141,38 +141,30 @@
          (rx/map #(hash-map :file-id file-id :library-id %))
          (rx/flat-map (partial rp/mutation :link-file-to-library)))))
 
-(defn persist-file [file]
-  (rp/mutation :persist-temp-file {:id (:id file)}))
-
 (defn send-changes
   "Creates batches of changes to be sent to the backend"
   [context file]
-  (let [revn (atom (:revn file))
+  (let [file-id    (:id file)
-        file-id (:id file)
         session-id (uuid/next)
-        changes-batches
+        batches    (->> (fb/generate-changes file)
-        (->> (fb/generate-changes file)
                         (partition change-batch-size change-batch-size nil)
                         (mapv vec))
 
-        current (atom 0)
+        processed  (atom 0)
-        total (count changes-batches)]
+        total      (count batches)]
 
     (rx/concat
-     (->> (rx/from changes-batches)
+     (->> (rx/from (d/enumerate batches))
-          (rx/mapcat
+          (rx/merge-map
-           (fn [change-batch]
+           (fn [[i change-batch]]
-             (->> (rp/mutation :update-file
+             (->> (rp/mutation :update-temp-file
                                {:id file-id
                                 :session-id session-id
-                                :revn @revn
+                                :revn i
                                 :changes change-batch})
-                  (rx/tap #(do (swap! current inc)
+                  (rx/tap #(do (swap! processed inc)
-                               (progress! context
+                               (progress! context :upload-data @processed total))))))
-                                          :upload-data @current total))))))
-
           (rx/map first)
-          (rx/tap #(reset! revn (:revn %)))
           (rx/ignore))
 
      (->> (rp/mutation :persist-temp-file {:id file-id})
@@ -340,7 +332,7 @@
         file      (-> file (fb/add-page page-data))]
     (->> (rx/from nodes)
          (rx/filter cip/shape?)
-         (rx/mapcat (partial resolve-media context file-id))
+         (rx/merge-map (partial resolve-media context file-id))
          (rx/reduce (partial process-import-node context) file)
          (rx/map (comp fb/close-page setup-interactions)))))
 
@@ -362,7 +354,7 @@
          (rx/filter cip/shape?)
          (rx/skip 1)
          (rx/skip-last 1)
-         (rx/mapcat (partial resolve-media context file-id))
+         (rx/merge-map (partial resolve-media context file-id))
          (rx/reduce (partial process-import-node context) file)
          (rx/map fb/finish-component))))
 
@@ -382,6 +374,7 @@
           (fn [[page-id page-name]]
             (->> (get-file context :page page-id)
                  (rx/map (fn [page-data] [page-id page-name page-data])))))
+
          (rx/concat-reduce (partial import-page context) file))))
 
 (defn process-library-colors
@@ -420,7 +413,7 @@
     (let [resolve (:resolve context)]
       (->> (get-file context :media-list)
            (rx/flat-map (comp d/kebab-keys cip/string->uuid))
-           (rx/mapcat
+           (rx/merge-map
             (fn [[id media]]
               (let [media (assoc media :id (resolve id))]
                 (->> (get-file context :media id media)
@@ -432,12 +425,11 @@
                                   :content content
                                   :is-local false})))
                      (rx/tap #(progress! context :upload-media (:name %)))
-                     (rx/flat-map #(rp/mutation! :upload-file-media-object %))
+                     (rx/merge-map #(rp/mutation! :upload-file-media-object %))
                      (rx/map (constantly media))
                      (rx/catch #(do (.error js/console (str "Error uploading media: " (:name media)) )
                                     (rx/empty)))))))
            (rx/reduce fb/add-library-media file)))
-
     (rx/of file)))
 
 (defn process-library-components
@@ -511,7 +503,7 @@
           (fn [[file data]]
             (->> (uz/load-from-url (:uri data))
                  (rx/map #(-> context (assoc :zip %) (merge data)))
-                 (rx/flat-map
+                 (rx/merge-map
                   (fn [context]
                     ;; process file retrieves a stream that will emit progress notifications
                     ;; and other that will emit the files once imported