⚡ Improve migration script performance and api usability

backend/dev/user.clj

@@ -8,6 +8,7 @@
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.files.helpers :as cfh]
    [app.common.fressian :as fres]
    [app.common.geom.matrix :as gmt]
    [app.common.logging :as l]
@@ -136,3 +137,12 @@
     (add-tap #(locking debug-tap
                 (prn "tap debug:" %)))
     1))
+
+
+(defn calculate-frames
+  [{:keys [data]}]
+  (->> (vals (:pages-index data))
+       (mapcat (comp vals :objects))
+       (filter cfh/is-direct-child-of-root?)
+       (filter cfh/frame-shape?)
+       (count)))

backend/src/app/config.clj

@@ -207,6 +207,7 @@
 (s/def ::telemetry-uri ::us/string)
 (s/def ::telemetry-with-taiga ::us/boolean)
 (s/def ::tenant ::us/string)
+(s/def ::svgo-max-procs ::us/integer)
 
 (s/def ::config
   (s/keys :opt-un [::secret-key
@@ -326,7 +327,9 @@
                    ::telemetry-uri
                    ::telemetry-referer
                    ::telemetry-with-taiga
-                   ::tenant]))
+                   ::tenant
+
+                   ::svgo-max-procs]))
 
 (def default-flags
   [:enable-backend-api-doc

backend/src/app/features/components_v2.clj

@@ -39,27 +39,54 @@
    [app.rpc.commands.media :as cmd.media]
    [app.storage :as sto]
    [app.storage.tmp :as tmp]
+   [app.svgo :as svgo]
    [app.util.blob :as blob]
    [app.util.pointer-map :as pmap]
    [app.util.time :as dt]
    [buddy.core.codecs :as bc]
    [cuerdas.core :as str]
    [datoteka.io :as io]
-   [promesa.exec :as px]
+   [promesa.core :as p]))
-   [promesa.exec.semaphore :as ps]
-   [promesa.util :as pu]))
 
-(def ^:dynamic *system* nil)
+(def ^:dynamic *stats*
-(def ^:dynamic *stats* nil)
+  "A dynamic var for setting up state for collect stats globally."
-(def ^:dynamic *file-stats* nil)
+  nil)
-(def ^:dynamic *team-stats* nil)
+
-(def ^:dynamic *semaphore* nil)
+(def ^:dynamic *skip-on-error*
-(def ^:dynamic *skip-on-error* true)
+  "A dynamic var for setting up the default error behavior."
+  true)
+
+(def ^:dynamic ^:private *system*
+  "An internal var for making the current `system` available to all
+  internal functions without the need to explicitly pass it top down."
+  nil)
+
+(def ^:dynamic ^:private *max-procs*
+  "A dynamic variable that can optionally indicates the maxumum number
+  of concurrent graphics migration processes."
+  nil)
+
+(def ^:dynamic ^:private *file-stats*
+  "An internal dynamic var for collect stats by file."
+  nil)
+
+(def ^:dynamic ^:private *team-stats*
+  "An internal dynamic var for collect stats by team."
+  nil)
 
 (def grid-gap 50)
 (def frame-gap 200)
 (def max-group-size 50)
 
+(defn decode-row
+  [{:keys [features data] :as row}]
+  (cond-> row
+    (some? features)
+    (assoc :features (db/decode-pgarray features #{}))
+
+    (some? data)
+    (assoc :data (blob/decode data))))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; FILE PREPARATION BEFORE MIGRATION
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -220,12 +247,10 @@
         (fn [file-data]
           ;; Transform component and copy heads to frames, and set the
           ;; frame-id of its childrens
-          (letfn [(fix-container
+          (letfn [(fix-container [container]
-                    [container]
                     (update container :objects update-vals fix-shape))
 
-                  (fix-shape
+                  (fix-shape [shape]
-                    [shape]
                     (if (or (nil? (:parent-id shape)) (ctk/instance-head? shape))
                       (assoc shape
                              :type :frame                  ; Old groups must be converted
@@ -310,10 +335,10 @@
 
 (defn- get-asset-groups
   [assets generic-name]
-  (let [; Group by first element of the path.
+  (let [;; Group by first element of the path.
         groups (d/group-by #(first (cfh/split-path (:path %))) assets)
 
-        ; Split large groups in chunks of max-group-size elements
+        ;; Split large groups in chunks of max-group-size elements
         groups (loop [groups (seq groups)
                       result {}]
                  (if (empty? groups)
@@ -334,15 +359,14 @@
                                         result
                                         splits)))))))
 
-        ; Sort assets in each group by path
+        ;; Sort assets in each group by path
         groups (update-vals groups (fn [assets]
                                      (sort-by (fn [{:keys [path name]}]
                                                 (str/lower (cfh/merge-path-item path name)))
-                                              assets)))
+                                              assets)))]
 
-        ; Sort groups by name
+    ;; Sort groups by name
-        groups (into (sorted-map) groups)]
+    (into (sorted-map) groups)))
-    groups))
 
 (defn- create-frame
   [name position width height]
@@ -613,10 +637,7 @@
 (defn- create-shapes-for-svg
   [{:keys [id] :as mobj} file-id objects frame-id position]
   (let [svg-text (get-svg-content id)
-
+        svg-text (svgo/optimize *system* svg-text)
-        optimizer (::csvg/optimizer *system*)
-        svg-text  (csvg/optimize optimizer svg-text)
-
         svg-data (-> (csvg/parse svg-text)
                      (assoc :name (:name mobj))
                      (collect-and-persist-images file-id))]
@@ -678,9 +699,7 @@
 
 (defn- create-media-grid
   [fdata page-id frame-id grid media-group]
-  (let [factory  (px/thread-factory :virtual true)
+  (let [process  (fn [mobj position]
-        executor (px/fixed-executor :parallelism 10 :factory factory)
-        process  (fn [mobj position]
                    (let [position (gpt/add position (gpt/point grid-gap grid-gap))
                          tp1      (dt/tpoint)]
                      (try
@@ -690,7 +709,6 @@
                                 :file-id (str (:id fdata))
                                 :id (str (:id mobj))
                                 :cause cause)
-
                          (if-not *skip-on-error*
                            (throw cause)
                            nil))
@@ -699,21 +717,24 @@
                                 :file-id (str (:id fdata))
                                 :media-id (str (:id mobj))
                                 :elapsed (dt/format-duration (tp1)))))))]
-    (try
+
     (->> (d/zip media-group grid)
+         (partition-all (or *max-procs* 1))
+         (mapcat (fn [partition]
+                   (->> partition
                         (map (fn [[mobj position]]
                                (sse/tap {:type :migration-progress
                                          :section :graphics
                                          :name (:name mobj)})
-                  (px/submit! executor (partial process mobj position))))
+                               (p/vthread (process mobj position))))
-           (reduce (fn [fdata promise]
+                        (doall)
-                     (if-let [changes (deref promise)]
+                        (map deref)
+                        (doall))))
+         (filter some?)
+         (reduce (fn [fdata changes]
                    (-> (assoc-in fdata [:options :components-v2] true)
-                           (cp/process-changes changes false))
+                       (cp/process-changes changes false)))
-                       fdata))
+                 fdata))))
-                   fdata))
-      (finally
-        (pu/close! executor)))))
 
 (defn- migrate-graphics
   [fdata]
@@ -759,6 +780,11 @@
                    (create-media-grid fdata page-id (:id frame) grid assets)
                    (gpt/add position (gpt/point 0 (+ height (* 2 grid-gap) frame-gap))))))))))
 
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; PRIVATE HELPERS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
 (defn- migrate-fdata
   [fdata libs]
   (let [migrated? (dm/get-in fdata [:options :components-v2])]
@@ -771,11 +797,22 @@
 (defn- get-file
   [system id]
   (binding [pmap/*load-fn* (partial fdata/load-pointer system id)]
-    (-> (files/get-file system id :migrate? false)
+    (-> (db/get system :file {:id id}
+                {::db/remove-deleted false
+                 ::db/check-deleted false})
+        (decode-row)
         (update :data assoc :id id)
         (update :data fdata/process-pointers deref)
         (fmg/migrate-file))))
 
+
+(defn- get-team
+  [system team-id]
+  (-> (db/get system :team {:id team-id}
+              {::db/remove-deleted false
+               ::db/check-deleted false})
+      (decode-row)))
+
 (defn- validate-file!
   [file libs throw-on-validate?]
   (try
@@ -791,7 +828,8 @@
   (let [file  (get-file system id)
 
         libs  (->> (files/get-file-libraries conn id)
-                   (into [file] (comp (map :id) (map (partial get-file system))))
+                   (into [file] (comp (map :id)
+                                      (map (partial get-file system))))
                    (d/index-by :id))
 
         file  (-> file
@@ -820,13 +858,35 @@
 
     (dissoc file :data)))
 
+
+(def ^:private sql:get-and-lock-team-files
+  "SELECT f.id
+     FROM file AS f
+     JOIN project AS p ON (p.id = f.project_id)
+    WHERE p.team_id = ?
+      FOR UPDATE")
+
+(defn- get-and-lock-files
+  [conn team-id]
+  (->> (db/cursor conn [sql:get-and-lock-team-files team-id])
+       (map :id)))
+
+(defn- update-team-features!
+  [conn team-id features]
+  (let [features (db/create-array conn "text" features)]
+    (db/update! conn :team
+                {:features features}
+                {:id team-id})))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; PUBLIC API
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
 (defn migrate-file!
-  [system file-id & {:keys [validate? throw-on-validate?]}]
+  [system file-id & {:keys [validate? throw-on-validate? max-procs]}]
-  (let [tpoint  (dt/tpoint)
+  (let [tpoint  (dt/tpoint)]
-        file-id (if (string? file-id)
+    (binding [*file-stats* (atom {})
-                  (parse-uuid file-id)
+              *max-procs* max-procs]
-                  file-id)]
-    (binding [*file-stats* (atom {})]
       (try
         (l/dbg :hint "migrate:file:start" :file-id (str file-id))
 
@@ -838,7 +898,6 @@
                           (process-file system file-id
                                         :validate? validate?
                                         :throw-on-validate? throw-on-validate?)))))
-
         (finally
           (let [elapsed    (tpoint)
                 components (get @*file-stats* :processed/components 0)
@@ -854,75 +913,51 @@
             (some-> *team-stats* (swap! update :processed/files (fnil inc 0)))))))))
 
 (defn migrate-team!
-  [system team-id & {:keys [validate? throw-on-validate?]}]
+  [system team-id & {:keys [validate? throw-on-validate? max-procs]}]
+
+  (l/dbg :hint "migrate:team:start"
+         :team-id (dm/str team-id))
+
   (let [tpoint (dt/tpoint)
-        team-id (if (string? team-id)
-                  (parse-uuid team-id)
-                  team-id)]
-    (l/dbg :hint "migrate:team:start" :team-id (dm/str team-id))
-    (binding [*team-stats* (atom {})]
-      (try
-        ;; We execute this out of transaction because we want this
-        ;; change to be visible to all other sessions before starting
-        ;; the migration
-        (let [sql (str "UPDATE team SET features = "
-                       "    array_append(features, 'ephimeral/v2-migration') "
-                       " WHERE id = ?")]
-          (db/exec-one! system [sql team-id]))
 
-        (db/tx-run! system
+        migrate-file
-                    (fn [{:keys [::db/conn] :as system}]
+        (fn [system file-id]
-                      ;; Lock the team
-                      (db/exec-one! conn ["SET idle_in_transaction_session_timeout = 0"])
-
-                      (let [{:keys [features] :as team} (-> (db/get conn :team {:id team-id})
-                                                            (update :features db/decode-pgarray #{}))]
-
-                        (if (contains? features "components/v2")
-                          (l/dbg :hint "team already migrated")
-                          (let [sql  (str/concat
-                                      "SELECT f.id FROM file AS f "
-                                      "  JOIN project AS p ON (p.id = f.project_id) "
-                                      "WHERE p.team_id = ? AND f.deleted_at IS NULL AND p.deleted_at IS NULL "
-                                      "FOR UPDATE")]
-
-                            (doseq [file-id (->> (db/exec! conn [sql team-id])
-                                                 (map :id))]
           (migrate-file! system file-id
+                         :max-procs max-procs
                          :validate? validate?
                          :throw-on-validate? throw-on-validate?))
-
+        migrate-team
+        (fn [{:keys [::db/conn] :as system} {:keys [id features] :as team}]
           (let [features (-> features
                              (disj "ephimeral/v2-migration")
                              (conj "components/v2")
                              (conj "layout/grid")
                              (conj "styles/v2"))]
-                              (db/update! conn :team
-                                          {:features (db/create-array conn "text" features)}
-                                          {:id team-id})
 
-                              nil))))))
+            (run! (partial migrate-file system)
+                  (get-and-lock-files conn id))
+
+            (update-team-features! conn id features)))]
+
+    (binding [*team-stats* (atom {})]
+      (try
+        (db/tx-run! system (fn [system]
+                             (db/exec-one! system ["SET idle_in_transaction_session_timeout = 0"])
+                             (let [team (get-team system team-id)]
+                               (if (contains? (:features team) "components/v2")
+                                 (l/inf :hint "team already migrated")
+                                 (migrate-team system team)))))
         (finally
-          (some-> *semaphore* ps/release!)
+          (let [elapsed    (tpoint)
-          (let [elapsed (tpoint)]
+                components (get @*team-stats* :processed/components 0)
-            (some-> *stats* (swap! update :processed/teams (fnil inc 0)))
-
-            ;; We execute this out of transaction because we want this
-            ;; change to be visible to all other sessions before starting
-            ;; the migration
-            (let [sql (str "UPDATE team SET features = "
-                           "    array_remove(features, 'ephimeral/v2-migration') "
-                           " WHERE id = ?")]
-              (db/exec-one! system [sql team-id]))
-
-            (let [components (get @*team-stats* :processed/components 0)
                 graphics   (get @*team-stats* :processed/graphics 0)
                 files      (get @*team-stats* :processed/files 0)]
+
+            (some-> *stats* (swap! update :processed/teams (fnil inc 0)))
+
             (l/dbg :hint "migrate:team:end"
                    :team-id (dm/str team-id)
                    :files files
                    :components components
                    :graphics graphics
-                     :elapsed (dt/format-duration elapsed)))))))))
+                   :elapsed (dt/format-duration elapsed))))))))
-
-

backend/src/app/main.clj

@@ -10,7 +10,6 @@
    [app.auth.oidc :as-alias oidc]
    [app.auth.oidc.providers :as-alias oidc.providers]
    [app.common.logging :as l]
-   [app.common.svg :as csvg]
    [app.config :as cf]
    [app.db :as-alias db]
    [app.email :as-alias email]
@@ -37,6 +36,7 @@
    [app.storage.gc-deleted :as-alias sto.gc-deleted]
    [app.storage.gc-touched :as-alias sto.gc-touched]
    [app.storage.s3 :as-alias sto.s3]
+   [app.svgo :as-alias svgo]
    [app.util.time :as dt]
    [app.worker :as-alias wrk]
    [cider.nrepl :refer [cider-nrepl-handler]]
@@ -316,7 +316,7 @@
     ::mtx/metrics        (ig/ref ::mtx/metrics)
     ::mbus/msgbus        (ig/ref ::mbus/msgbus)
     ::rds/redis          (ig/ref ::rds/redis)
-    ::csvg/optimizer     (ig/ref ::csvg/optimizer)
+    ::svgo/optimizer     (ig/ref ::svgo/optimizer)
 
     ::rpc/climit         (ig/ref ::rpc/climit)
     ::rpc/rlimit         (ig/ref ::rpc/rlimit)
@@ -409,8 +409,9 @@
     ;; module requires the migrations to run before initialize.
     ::migrations (ig/ref :app.migrations/migrations)}
 
-   ::csvg/optimizer
+   ::svgo/optimizer
-   {}
+   {::wrk/executor   (ig/ref ::wrk/executor)
+    ::svgo/max-procs (cf/get :svgo-max-procs)}
 
    ::audit.tasks/archive
    {::props              (ig/ref ::setup/props)

backend/src/app/rpc/commands/binfile.clj

@@ -664,6 +664,7 @@
           (case feature
             "components/v2"
             (feat.compv2/migrate-file! options file-id
+                                       :max-procs 2
                                        :validate? validate?
                                        :throw-on-validate? true)
 

backend/src/app/srepl/components_v2.clj

@@ -6,8 +6,6 @@
 
 (ns app.srepl.components-v2
   (:require
-   [app.common.data :as d]
-   [app.common.data.macros :as dm]
    [app.common.logging :as l]
    [app.common.pprint :as pp]
    [app.db :as db]
@@ -19,6 +17,13 @@
    [promesa.exec.semaphore :as ps]
    [promesa.util :as pu]))
 
+(def ^:dynamic *scope* nil)
+(def ^:dynamic *semaphore* nil)
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; PRIVATE HELPERS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
 (defn- print-stats!
   [stats]
   (->> stats
@@ -87,204 +92,220 @@
         res (db/exec-one! pool [sql])]
     (:count res)))
 
-(defn migrate-file!
-  [system file-id & {:keys [rollback?] :or {rollback? true}}]
 
-  (l/dbg :hint "migrate:start")
+(defn- mark-team-migration!
-  (let [tpoint (dt/tpoint)]
+  [{:keys [::db/pool]} team-id]
-    (try
+  ;; We execute this out of transaction because we want this
+  ;; change to be visible to all other sessions before starting
+  ;; the migration
+  (let [sql (str "UPDATE team SET features = "
+                 "    array_append(features, 'ephimeral/v2-migration') "
+                 " WHERE id = ?")]
+    (db/exec-one! pool [sql team-id])))
+
+(defn- unmark-team-migration!
+  [{:keys [::db/pool]} team-id]
+  ;; We execute this out of transaction because we want this
+  ;; change to be visible to all other sessions before starting
+  ;; the migration
+  (let [sql (str "UPDATE team SET features = "
+                 "    array_remove(features, 'ephimeral/v2-migration') "
+                 " WHERE id = ?")]
+    (db/exec-one! pool [sql team-id])))
+
+(def ^:private sql:get-teams
+  "SELECT id, features
+     FROM team
+    WHERE deleted_at IS NULL
+    ORDER BY created_at ASC")
+
+(defn- get-teams
+  [conn]
+  (->> (db/cursor conn sql:get-teams)
+       (map feat/decode-row)))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; PUBLIC API
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn migrate-file!
+  [system file-id & {:keys [rollback? max-procs]
+                     :or {rollback? true}}]
+
+  (l/dbg :hint "migrate:start" :rollback rollback?)
+  (let [tpoint (dt/tpoint)
+        file-id (if (string? file-id)
+                  (parse-uuid file-id)
+                  file-id)]
     (binding [feat/*stats* (atom {})]
+      (try
         (-> (assoc system ::db/rollback rollback?)
-            (feat/migrate-file! file-id))
+            (feat/migrate-file! file-id :max-procs max-procs))
 
         (-> (deref feat/*stats*)
-            (assoc :elapsed (dt/format-duration (tpoint)))))
+            (assoc :elapsed (dt/format-duration (tpoint))))
 
         (catch Throwable cause
           (l/wrn :hint "migrate:error" :cause cause))
 
         (finally
           (let [elapsed (dt/format-duration (tpoint))]
-          (l/dbg :hint "migrate:end" :elapsed elapsed))))))
+            (l/dbg :hint "migrate:end" :rollback rollback? :elapsed elapsed)))))))
-
-(defn migrate-files!
-  [{:keys [::db/pool] :as system}
-   & {:keys [chunk-size max-jobs max-items start-at preset rollback? skip-on-error validate?]
-      :or {chunk-size 10
-           skip-on-error true
-           max-jobs 10
-           max-items Long/MAX_VALUE
-           preset :shutdown-on-failure
-           rollback? true
-           validate? false}}]
-  (letfn [(get-chunk [cursor]
-            (let [sql  (str/concat
-                        "SELECT id, created_at FROM file "
-                        " WHERE created_at < ? AND deleted_at IS NULL "
-                        " ORDER BY created_at desc LIMIT ?")
-                  rows (db/exec! pool [sql cursor chunk-size])]
-              [(some->> rows peek :created-at) (seq rows)]))
-
-          (get-candidates []
-            (->> (d/iteration get-chunk
-                              :vf second
-                              :kf first
-                              :initk (or start-at (dt/now)))
-                 (take max-items)
-                 (map :id)))]
-
-    (l/dbg :hint "migrate:start")
-    (let [fsem   (ps/create :permits max-jobs)
-          total  (get-total-files pool)
-          stats  (atom {:files/total total})
-          tpoint (dt/tpoint)]
-
-      (add-watch stats :progress-report (report-progress-files tpoint))
-
-      (binding [feat/*stats* stats
-                feat/*semaphore* fsem
-                feat/*skip-on-error* skip-on-error]
-        (try
-          (pu/with-open [scope (px/structured-task-scope :preset preset :factory :virtual)]
-
-            (run! (fn [file-id]
-                    (ps/acquire! feat/*semaphore*)
-                    (px/submit! scope (fn []
-                                        (-> (assoc system ::db/rollback rollback?)
-                                            (feat/migrate-file! file-id
-                                                                :validate? validate?
-                                                                :throw-on-validate? (not skip-on-error))))))
-                  (get-candidates))
-
-            (p/await! scope))
-
-          (-> (deref feat/*stats*)
-              (assoc :elapsed (dt/format-duration (tpoint))))
-
-          (catch Throwable cause
-            (l/dbg :hint "migrate:error" :cause cause))
-
-          (finally
-            (let [elapsed (dt/format-duration (tpoint))]
-              (l/dbg :hint "migrate:end" :elapsed elapsed))))))))
 
 (defn migrate-team!
-  [{:keys [::db/pool] :as system} team-id
+  [{:keys [::db/pool] :as system} team-id & {:keys [rollback? skip-on-error validate? max-procs]
-   & {:keys [rollback? skip-on-error validate?]
+                                             :or {rollback? true
-      :or {rollback? true skip-on-error true validate? false}}]
+                                                  skip-on-error true
-  (l/dbg :hint "migrate:start")
+                                                  validate? false
+                                                  max-procs 1 }
+                                             :as opts}]
 
-  (let [total  (get-total-files pool :team-id team-id)
+  (l/dbg :hint "migrate:start" :rollback rollback?)
+
+  (let [team-id   (if (string? team-id)
+                    (parse-uuid team-id)
+                    team-id)
+        total     (get-total-files pool :team-id team-id)
         stats     (atom {:total/files total})
         tpoint    (dt/tpoint)]
 
     (add-watch stats :progress-report (report-progress-files tpoint))
 
-    (try
     (binding [feat/*stats* stats
               feat/*skip-on-error* skip-on-error]
+
+      (try
+        (mark-team-migration! system team-id)
+
         (-> (assoc system ::db/rollback rollback?)
             (feat/migrate-team! team-id
+                                :max-procs max-procs
                                 :validate? validate?
                                 :throw-on-validate? (not skip-on-error)))
 
         (print-stats!
          (-> (deref feat/*stats*)
              (dissoc :total/files)
-             (assoc :elapsed (dt/format-duration (tpoint))))))
+             (assoc :elapsed (dt/format-duration (tpoint)))))
 
         (catch Throwable cause
           (l/dbg :hint "migrate:error" :cause cause))
 
         (finally
-        (let [elapsed (dt/format-duration (tpoint))]
+          (unmark-team-migration! system team-id)
-          (l/dbg :hint "migrate:end" :elapsed elapsed))))))
 
-(defn default-on-end
+          (let [elapsed (dt/format-duration (tpoint))]
-  [stats]
+            (l/dbg :hint "migrate:end" :rollback rollback? :elapsed elapsed)))))))
-  (print-stats!
-   (-> stats
-       (update :elapsed/total dt/format-duration)
-       (dissoc :total/teams))))
 
 (defn migrate-teams!
-  [{:keys [::db/pool] :as system}
+  "A REPL helper for migrate all teams.
-   & {:keys [chunk-size max-jobs max-items start-at
+
-             rollback? validate? preset skip-on-error
+  This function starts multiple concurrent team migration processes
-             max-time on-start on-progress on-error on-end]
+  until thw maximum number of jobs is reached which by default has the
-      :or {chunk-size 10000
+  value of `1`. This is controled with the `:max-jobs` option.
-           validate? false
+
+  Each tram migration process also can start multiple procs for
+  graphics migration, the total of that procs is controled with the
+  `:max-procs` option.
+
+  Internally, the graphics migration process uses SVGO module which by
+  default has a limited number of maximum concurent
+  operations (globally), ensure setting up correct number with
+  PENPOT_SVGO_MAX_PROCS environment variable."
+
+  [{:keys [::db/pool] :as system} & {:keys [max-jobs max-procs max-items
+                                            rollback? validate? preset
+                                            skip-on-error max-time
+                                            on-start on-progress on-error on-end]
+                                     :or {validate? false
                                           rollback? true
                                           skip-on-error true
-           on-end default-on-end
                                           preset :shutdown-on-failure
-           max-jobs Integer/MAX_VALUE
+                                          max-jobs 1
-           max-items Long/MAX_VALUE}}]
+                                          max-procs 10
+                                          max-items Long/MAX_VALUE}
+                                     :as opts}]
 
-  (letfn [(get-chunk [cursor]
+  (let [total  (get-total-teams pool)
-            (let [sql  (str/concat
+        stats  (atom {:total/teams (min total max-items)})
-                        "SELECT id, created_at, features FROM team "
-                        " WHERE created_at < ? AND deleted_at IS NULL "
-                        " ORDER BY created_at desc LIMIT ?")
-                  rows (db/exec! pool [sql cursor chunk-size])]
-              [(some->> rows peek :created-at) (seq rows)]))
 
-          (get-candidates []
+        tpoint (dt/tpoint)
-            (->> (d/iteration get-chunk
+        mtime  (some-> max-time dt/duration)
-                              :vf second
-                              :kf first
-                              :initk (or start-at (dt/now)))
-                 (map #(update % :features db/decode-pgarray #{}))
-                 (remove #(contains? (:features %) "ephimeral/v2-migration"))
-                 (take max-items)
-                 (map :id)))
 
-          (migrate-team [team-id]
+        scope  (px/structured-task-scope :preset preset :factory :virtual)
+        sjobs  (ps/create :permits max-jobs)
+
+        migrate-team
+        (fn [{:keys [id features] :as team}]
+          (ps/acquire! sjobs)
+          (let [ts (tpoint)]
+            (cond
+              (and mtime (neg? (compare mtime ts)))
+              (do
+                (l/inf :hint "max time constraint reached"
+                       :team-id (str id)
+                       :elapsed (dt/format-duration ts))
+                (ps/release! sjobs)
+                (reduced nil))
+
+              (or (contains? features "ephimeral/v2-migration")
+                  (contains? features "components/v2"))
+              (do
+                (l/dbg :hint "skip team" :team-id (str id))
+                (ps/release! sjobs))
+
+              :else
+              (px/submit! scope (fn []
                                     (try
+                                      (mark-team-migration! system id)
                                       (-> (assoc system ::db/rollback rollback?)
-                  (feat/migrate-team! team-id
+                                          (feat/migrate-team! id
+                                                              :max-procs max-procs
                                                               :validate? validate?
                                                               :throw-on-validate? (not skip-on-error)))
                                       (catch Throwable cause
-                (l/err :hint "unexpected error on processing team" :team-id (dm/str team-id) :cause cause))))
+                                        (l/err :hint "unexpected error on processing team"
+                                               :team-id (str id)
+                                               :cause cause))
+                                      (finally
+                                        (ps/release! sjobs)
+                                        (unmark-team-migration! system id))))))))]
 
-          (process-team [scope tpoint mtime team-id]
+    (l/dbg :hint "migrate:start"
-            (ps/acquire! feat/*semaphore*)
+           :rollback rollback?
-            (let [ts (tpoint)]
+           :total total
-              (if (and mtime (neg? (compare mtime ts)))
+           :max-jobs max-jobs
-                (l/inf :hint "max time constraint reached" :elapsed (dt/format-duration ts))
+           :max-procs max-procs
-                (px/submit! scope (partial migrate-team team-id)))))]
+           :max-items max-items)
-
-    (l/dbg :hint "migrate:start")
-
-    (let [sem    (ps/create :permits max-jobs)
-          total  (get-total-teams pool)
-          stats  (atom {:total/teams (min total max-items)})
-          tpoint (dt/tpoint)
-          mtime  (some-> max-time dt/duration)]
-
-      (when (fn? on-start)
-        (on-start {:total total :rollback rollback?}))
 
     (add-watch stats :progress-report (report-progress-teams tpoint on-progress))
 
     (binding [feat/*stats* stats
-                feat/*semaphore* sem
               feat/*skip-on-error* skip-on-error]
       (try
-          (pu/with-open [scope (px/structured-task-scope :preset preset
+        (when (fn? on-start)
-                                                         :factory :virtual)]
+          (on-start {:total total :rollback rollback?}))
-            (loop [candidates (get-candidates)]
-              (when-let [team-id (first candidates)]
-                (when (process-team scope tpoint mtime team-id)
-                  (recur (rest candidates)))))
 
-            (p/await! scope))
+        (db/tx-run! system
+                    (fn [{:keys [::db/conn]}]
+                      (run! (partial migrate-team)
+                            (->> (get-teams conn)
+                                 (take max-items)))))
+        (try
+          (p/await! scope)
+          (finally
+            (pu/close! scope)))
 
-          (when (fn? on-end)
+
+        (if (fn? on-end)
           (-> (deref stats)
               (assoc :elapsed/total (tpoint))
-                (on-end)))
+              (on-end))
+          (-> (deref stats)
+              (assoc :elapsed/total (tpoint))
+              (update :elapsed/total dt/format-duration)
+              (dissoc :total/teams)
+              (print-stats!)))
 
         (catch Throwable cause
           (l/dbg :hint "migrate:error" :cause cause)
@@ -293,4 +314,6 @@
 
         (finally
           (let [elapsed (dt/format-duration (tpoint))]
-              (l/dbg :hint "migrate:end" :elapsed elapsed))))))))
+            (l/dbg :hint "migrate:end"
+                   :rollback rollback?
+                   :elapsed elapsed)))))))

backend/src/app/svgo.clj

@@ -0,0 +1,65 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.svgo
+  "A SVG Optimizer service"
+  (:require
+   [app.common.data :as d]
+   [app.common.data.macros :as dm]
+   [app.common.jsrt :as jsrt]
+   [app.common.logging :as l]
+   [app.common.spec :as us]
+   [app.worker :as-alias wrk]
+   [clojure.spec.alpha :as s]
+   [integrant.core :as ig]
+   [promesa.exec :as px]
+   [promesa.exec.bulkhead :as bh]
+   [promesa.exec.semaphore :as ps]
+   [promesa.util :as pu]))
+
+(def ^:dynamic *semaphore*
+  "A dynamic variable that can optionally contain a traffic light to
+  appropriately delimit the use of resources, managed externally."
+  nil)
+
+(defn optimize
+  [system data]
+  (dm/assert! "expect data to be a string" (string? data))
+
+  (letfn [(optimize-fn [pool]
+            (jsrt/run! pool
+                       (fn [context]
+                         (jsrt/set! context "svgData" data)
+                         (jsrt/eval! context "penpotSvgo.optimize(svgData, {plugins: ['safeAndFastPreset']})"))))]
+    (try
+      (some-> *semaphore* ps/acquire!)
+      (let [{:keys [::jsrt/pool ::wrk/executor]} (::optimizer system)]
+        (dm/assert! "expect optimizer instance" (jsrt/pool? pool))
+        (px/invoke! executor (partial optimize-fn pool)))
+      (finally
+        (some-> *semaphore* ps/release!)))))
+
+(s/def ::max-procs (s/nilable ::us/integer))
+
+(defmethod ig/pre-init-spec ::optimizer [_]
+  (s/keys :req [::wrk/executor ::max-procs]))
+
+(defmethod ig/prep-key ::optimizer
+  [_ cfg]
+  (merge {::max-procs 20} (d/without-nils cfg)))
+
+(defmethod ig/init-key ::optimizer
+  [_ {:keys [::wrk/executor ::max-procs]}]
+  (l/inf :hint "initializing svg optimizer pool" :max-procs max-procs)
+  (let [init     (jsrt/resource->source "app/common/svg/optimizer.js")
+        executor (bh/create :type :executor :executor executor :permits max-procs)]
+    {::jsrt/pool (jsrt/pool :init init)
+     ::wrk/executor executor}))
+
+(defmethod ig/halt-key! ::optimizer
+  [_ {:keys [::jsrt/pool]}]
+  (l/info :hint "stopping svg optimizer pool")
+  (pu/close! pool))

backend/src/app/worker.clj

@@ -43,7 +43,7 @@
 (defmethod ig/init-key ::executor
   [_ _]
   (let [factory  (px/thread-factory :prefix "penpot/default/")
-        executor (px/cached-executor :factory factory :keepalive 30000)]
+        executor (px/cached-executor :factory factory :keepalive 60000)]
     (l/inf :hint "starting executor")
     (reify
       java.lang.AutoCloseable

common/src/app/common/svg.cljc

@@ -9,9 +9,6 @@
    #?(:cljs ["./svg/optimizer.js" :as svgo])
    #?(:clj  [clojure.xml :as xml]
       :cljs [tubax.core :as tubax])
-   #?(:clj [integrant.core :as ig])
-   #?(:clj [app.common.jsrt :as jsrt])
-   #?(:clj [app.common.logging :as l])
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.geom.matrix :as gmt]
@@ -1053,21 +1050,6 @@
                          :height (d/parse-integer (:height attrs) 0)})))]
     (reduce-nodes redfn [] svg-data )))
 
-#?(:cljs
-   (defn optimize
-     ([input] (optimize input nil))
-     ([input options]
-      (svgo/optimize input (clj->js options))))
-   :clj
-   (defn optimize
-     [pool data]
-     (dm/assert! "expected a valid pool" (jsrt/pool? pool))
-     (dm/assert! "expect data to be a string" (string? data))
-     (jsrt/run! pool
-                (fn [context]
-                  (jsrt/set! context "svgData" data)
-                  (jsrt/eval! context "penpotSvgo.optimize(svgData, {})")))))
-
 #?(:clj
    (defn- secure-parser-factory
      [^InputStream input ^XMLHandler handler]
@@ -1091,15 +1073,9 @@
              (dm/with-open [istream (IOUtils/toInputStream text "UTF-8")]
                (xml/parse istream secure-parser-factory)))))
 
-#?(:clj
+;; FIXME pass correct plugin set
-   (defmethod ig/init-key ::optimizer
+#?(:cljs
-     [_ _]
+   (defn optimize
-     (l/info :hint "initializing svg optimizer pool")
+     ([input] (optimize input nil))
-     (let [init (jsrt/resource->source "app/common/svg/optimizer.js")]
+     ([input options]
-       (jsrt/pool :init init))))
+      (svgo/optimize input (clj->js options)))))
-
-#?(:clj
-   (defmethod ig/halt-key! ::optimizer
-     [_ pool]
-     (l/info :hint "stopping svg optimizer pool")
-     (.close ^java.lang.AutoCloseable pool)))

common/src/app/common/svg/optimizer.js

@@ -29431,12 +29431,13 @@ const optimize = (input, config) => {
 
 exports.optimize = optimize;
 
-},{"./svgo/parser.js":322,"./svgo/plugins.js":324,"./svgo/stringifier.js":381,"./svgo/tools.js":383,"lodash/isPlainObject":308}],320:[function(require,module,exports){
+},{"./svgo/parser.js":322,"./svgo/plugins.js":324,"./svgo/stringifier.js":382,"./svgo/tools.js":384,"lodash/isPlainObject":308}],320:[function(require,module,exports){
 'use strict';
 
 exports.builtin = [
   require('./plugins/default.js'),
   require('./plugins/safe.js'),
+  require('./plugins/safeAndFast.js'),
   require('./plugins/addAttributesToSVGElement.js'),
   require('./plugins/addClassesToSVGElement.js'),
   require('./plugins/cleanupAttrs.js'),
@@ -29489,7 +29490,7 @@ exports.builtin = [
   require('./plugins/sortDefsChildren.js'),
 ];
 
-},{"./plugins/addAttributesToSVGElement.js":328,"./plugins/addClassesToSVGElement.js":329,"./plugins/cleanupAttrs.js":331,"./plugins/cleanupEnableBackground.js":332,"./plugins/cleanupIds.js":333,"./plugins/cleanupListOfValues.js":334,"./plugins/cleanupNumericValues.js":335,"./plugins/collapseGroups.js":336,"./plugins/convertColors.js":337,"./plugins/convertEllipseToCircle.js":338,"./plugins/convertPathData.js":339,"./plugins/convertShapeToPath.js":340,"./plugins/convertStyleToAttrs.js":341,"./plugins/convertTransform.js":342,"./plugins/default.js":343,"./plugins/inlineStyles.js":344,"./plugins/mergePaths.js":345,"./plugins/mergeStyles.js":346,"./plugins/minifyStyles.js":347,"./plugins/moveElemsAttrsToGroup.js":348,"./plugins/moveGroupAttrsToElems.js":349,"./plugins/prefixIds.js":350,"./plugins/removeAttributesBySelector.js":351,"./plugins/removeAttrs.js":352,"./plugins/removeComments.js":353,"./plugins/removeDesc.js":354,"./plugins/removeDimensions.js":355,"./plugins/removeDoctype.js":356,"./plugins/removeEditorsNSData.js":357,"./plugins/removeElementsByAttr.js":358,"./plugins/removeEmptyAttrs.js":359,"./plugins/removeEmptyContainers.js":360,"./plugins/removeEmptyText.js":361,"./plugins/removeHiddenElems.js":362,"./plugins/removeMetadata.js":363,"./plugins/removeNonInheritableGroupAttrs.js":364,"./plugins/removeOffCanvasPaths.js":365,"./plugins/removeRasterImages.js":366,"./plugins/removeScriptElement.js":367,"./plugins/removeStyleElement.js":368,"./plugins/removeTitle.js":369,"./plugins/removeUnknownsAndDefaults.js":370,"./plugins/removeUnusedNS.js":371,"./plugins/removeUselessDefs.js":372,"./plugins/removeUselessStrokeAndFill.js":373,"./plugins/removeViewBox.js":374,"./plugins/removeXMLNS.js":375,"./plugins/removeXMLProcInst.js":376,"./plugins/reusePaths.js":377,"./plugins/safe.js":378,"./plugins/sortAttrs.js":379,"./plugins/sortDefsChildren.js":380}],321:[function(require,module,exports){
+},{"./plugins/addAttributesToSVGElement.js":328,"./plugins/addClassesToSVGElement.js":329,"./plugins/cleanupAttrs.js":331,"./plugins/cleanupEnableBackground.js":332,"./plugins/cleanupIds.js":333,"./plugins/cleanupListOfValues.js":334,"./plugins/cleanupNumericValues.js":335,"./plugins/collapseGroups.js":336,"./plugins/convertColors.js":337,"./plugins/convertEllipseToCircle.js":338,"./plugins/convertPathData.js":339,"./plugins/convertShapeToPath.js":340,"./plugins/convertStyleToAttrs.js":341,"./plugins/convertTransform.js":342,"./plugins/default.js":343,"./plugins/inlineStyles.js":344,"./plugins/mergePaths.js":345,"./plugins/mergeStyles.js":346,"./plugins/minifyStyles.js":347,"./plugins/moveElemsAttrsToGroup.js":348,"./plugins/moveGroupAttrsToElems.js":349,"./plugins/prefixIds.js":350,"./plugins/removeAttributesBySelector.js":351,"./plugins/removeAttrs.js":352,"./plugins/removeComments.js":353,"./plugins/removeDesc.js":354,"./plugins/removeDimensions.js":355,"./plugins/removeDoctype.js":356,"./plugins/removeEditorsNSData.js":357,"./plugins/removeElementsByAttr.js":358,"./plugins/removeEmptyAttrs.js":359,"./plugins/removeEmptyContainers.js":360,"./plugins/removeEmptyText.js":361,"./plugins/removeHiddenElems.js":362,"./plugins/removeMetadata.js":363,"./plugins/removeNonInheritableGroupAttrs.js":364,"./plugins/removeOffCanvasPaths.js":365,"./plugins/removeRasterImages.js":366,"./plugins/removeScriptElement.js":367,"./plugins/removeStyleElement.js":368,"./plugins/removeTitle.js":369,"./plugins/removeUnknownsAndDefaults.js":370,"./plugins/removeUnusedNS.js":371,"./plugins/removeUselessDefs.js":372,"./plugins/removeUselessStrokeAndFill.js":373,"./plugins/removeViewBox.js":374,"./plugins/removeXMLNS.js":375,"./plugins/removeXMLProcInst.js":376,"./plugins/reusePaths.js":377,"./plugins/safe.js":378,"./plugins/safeAndFast.js":379,"./plugins/sortAttrs.js":380,"./plugins/sortDefsChildren.js":381}],321:[function(require,module,exports){
 'use strict';
 
 const isTag = (node) => {
@@ -30102,7 +30103,7 @@ const stringifyPathData = ({ pathData, precision, disableSpaceAfterFlags }) => {
 };
 exports.stringifyPathData = stringifyPathData;
 
-},{"./tools.js":383}],324:[function(require,module,exports){
+},{"./tools.js":384}],324:[function(require,module,exports){
 'use strict';
 
 const { builtin } = require('./builtin.js');
@@ -33826,7 +33827,7 @@ const applyMatrixToPathData = (pathData, matrix) => {
   }
 };
 
-},{"../style.js":382,"../tools.js":383,"./_collections.js":325,"./_path.js":326,"./_transforms.js":327}],331:[function(require,module,exports){
+},{"../style.js":383,"../tools.js":384,"./_collections.js":325,"./_path.js":326,"./_transforms.js":327}],331:[function(require,module,exports){
 'use strict';
 
 exports.name = 'cleanupAttrs';
@@ -33948,7 +33949,7 @@ exports.fn = (root) => {
   };
 };
 
-},{"../xast.js":384}],333:[function(require,module,exports){
+},{"../xast.js":385}],333:[function(require,module,exports){
 'use strict';
 
 const { visitSkip } = require('../xast.js');
@@ -34207,7 +34208,7 @@ exports.fn = (_root, params) => {
   };
 };
 
-},{"../xast.js":384,"./_collections.js":325}],334:[function(require,module,exports){
+},{"../xast.js":385,"./_collections.js":325}],334:[function(require,module,exports){
 'use strict';
 
 const { removeLeadingZero } = require('../tools.js');
@@ -34345,7 +34346,7 @@ exports.fn = (_root, params) => {
   };
 };
 
-},{"../tools.js":383}],335:[function(require,module,exports){
+},{"../tools.js":384}],335:[function(require,module,exports){
 'use strict';
 
 const { removeLeadingZero } = require('../tools.js');
@@ -34451,7 +34452,7 @@ exports.fn = (_root, params) => {
   };
 };
 
-},{"../tools.js":383}],336:[function(require,module,exports){
+},{"../tools.js":384}],336:[function(require,module,exports){
 'use strict';
 
 const { inheritableAttrs, elemsGroups } = require('./_collections.js');
@@ -35838,7 +35839,7 @@ function data2Path(params, pathData) {
   }, '');
 }
 
-},{"../style.js":382,"../tools.js":383,"../xast.js":384,"./_collections.js":325,"./_path.js":326,"./applyTransforms.js":330}],340:[function(require,module,exports){
+},{"../style.js":383,"../tools.js":384,"../xast.js":385,"./_collections.js":325,"./_path.js":326,"./applyTransforms.js":330}],340:[function(require,module,exports){
 'use strict';
 
 const { stringifyPathData } = require('../path.js');
@@ -36004,7 +36005,7 @@ exports.fn = (root, params) => {
   };
 };
 
-},{"../path.js":323,"../xast.js":384}],341:[function(require,module,exports){
+},{"../path.js":323,"../xast.js":385}],341:[function(require,module,exports){
 'use strict';
 
 const { attrsGroups } = require('./_collections');
@@ -36506,7 +36507,7 @@ const smartRound = (precision, data) => {
   return data;
 };
 
-},{"../tools.js":383,"./_transforms.js":327}],343:[function(require,module,exports){
+},{"../tools.js":384,"./_transforms.js":327}],343:[function(require,module,exports){
 'use strict';
 
 const { createPreset } = require('../tools.js');
@@ -36590,7 +36591,7 @@ const presetDefault = createPreset({
 
 module.exports = presetDefault;
 
-},{"../tools.js":383,"./cleanupAttrs.js":331,"./cleanupEnableBackground.js":332,"./cleanupIds.js":333,"./cleanupNumericValues.js":335,"./collapseGroups.js":336,"./convertColors.js":337,"./convertEllipseToCircle.js":338,"./convertPathData.js":339,"./convertShapeToPath.js":340,"./convertTransform.js":342,"./inlineStyles.js":344,"./mergePaths.js":345,"./mergeStyles.js":346,"./minifyStyles.js":347,"./moveElemsAttrsToGroup.js":348,"./moveGroupAttrsToElems.js":349,"./removeComments.js":353,"./removeDesc.js":354,"./removeDoctype.js":356,"./removeEditorsNSData.js":357,"./removeEmptyAttrs.js":359,"./removeEmptyContainers.js":360,"./removeEmptyText.js":361,"./removeHiddenElems.js":362,"./removeMetadata.js":363,"./removeNonInheritableGroupAttrs.js":364,"./removeTitle.js":369,"./removeUnknownsAndDefaults.js":370,"./removeUnusedNS.js":371,"./removeUselessDefs.js":372,"./removeUselessStrokeAndFill.js":373,"./removeViewBox.js":374,"./removeXMLProcInst.js":376,"./sortAttrs.js":379,"./sortDefsChildren.js":380}],344:[function(require,module,exports){
+},{"../tools.js":384,"./cleanupAttrs.js":331,"./cleanupEnableBackground.js":332,"./cleanupIds.js":333,"./cleanupNumericValues.js":335,"./collapseGroups.js":336,"./convertColors.js":337,"./convertEllipseToCircle.js":338,"./convertPathData.js":339,"./convertShapeToPath.js":340,"./convertTransform.js":342,"./inlineStyles.js":344,"./mergePaths.js":345,"./mergeStyles.js":346,"./minifyStyles.js":347,"./moveElemsAttrsToGroup.js":348,"./moveGroupAttrsToElems.js":349,"./removeComments.js":353,"./removeDesc.js":354,"./removeDoctype.js":356,"./removeEditorsNSData.js":357,"./removeEmptyAttrs.js":359,"./removeEmptyContainers.js":360,"./removeEmptyText.js":361,"./removeHiddenElems.js":362,"./removeMetadata.js":363,"./removeNonInheritableGroupAttrs.js":364,"./removeTitle.js":369,"./removeUnknownsAndDefaults.js":370,"./removeUnusedNS.js":371,"./removeUselessDefs.js":372,"./removeUselessStrokeAndFill.js":373,"./removeViewBox.js":374,"./removeXMLProcInst.js":376,"./sortAttrs.js":380,"./sortDefsChildren.js":381}],344:[function(require,module,exports){
 'use strict';
 
 const csstree = require('css-tree');
@@ -36937,7 +36938,7 @@ exports.fn = (root, params) => {
   };
 };
 
-},{"../xast.js":384,"css-tree":25,"csso":138}],345:[function(require,module,exports){
+},{"../xast.js":385,"css-tree":25,"csso":138}],345:[function(require,module,exports){
 'use strict';
 
 const { detachNodeFromParent } = require('../xast.js');
@@ -37035,7 +37036,7 @@ exports.fn = (root, params) => {
   };
 };
 
-},{"../style.js":382,"../xast.js":384,"./_path.js":326}],346:[function(require,module,exports){
+},{"../style.js":383,"../xast.js":385,"./_path.js":326}],346:[function(require,module,exports){
 'use strict';
 
 const { visitSkip, detachNodeFromParent } = require('../xast.js');
@@ -37119,7 +37120,7 @@ exports.fn = () => {
   };
 };
 
-},{"../xast.js":384}],347:[function(require,module,exports){
+},{"../xast.js":385}],347:[function(require,module,exports){
 'use strict';
 
 const csso = require('csso');
@@ -37364,7 +37365,7 @@ exports.fn = (root) => {
   };
 };
 
-},{"../xast.js":384,"./_collections.js":325}],349:[function(require,module,exports){
+},{"../xast.js":385,"./_collections.js":325}],349:[function(require,module,exports){
 'use strict';
 
 const { pathElems, referencesProps } = require('./_collections.js');
@@ -37742,7 +37743,7 @@ exports.fn = (root, params) => {
   return {};
 };
 
-},{"../xast.js":384}],352:[function(require,module,exports){
+},{"../xast.js":385}],352:[function(require,module,exports){
 'use strict';
 
 exports.name = 'removeAttrs';
@@ -37924,7 +37925,7 @@ exports.fn = () => {
   };
 };
 
-},{"../xast.js":384}],354:[function(require,module,exports){
+},{"../xast.js":385}],354:[function(require,module,exports){
 'use strict';
 
 const { detachNodeFromParent } = require('../xast.js');
@@ -37963,7 +37964,7 @@ exports.fn = (root, params) => {
   };
 };
 
-},{"../xast.js":384}],355:[function(require,module,exports){
+},{"../xast.js":385}],355:[function(require,module,exports){
 'use strict';
 
 exports.name = 'removeDimensions';
@@ -38046,7 +38047,7 @@ exports.fn = () => {
   };
 };
 
-},{"../xast.js":384}],357:[function(require,module,exports){
+},{"../xast.js":385}],357:[function(require,module,exports){
 'use strict';
 
 const { detachNodeFromParent } = require('../xast.js');
@@ -38110,7 +38111,7 @@ exports.fn = (_root, params) => {
   };
 };
 
-},{"../xast.js":384,"./_collections.js":325}],358:[function(require,module,exports){
+},{"../xast.js":385,"./_collections.js":325}],358:[function(require,module,exports){
 'use strict';
 
 const { detachNodeFromParent } = require('../xast.js');
@@ -38183,7 +38184,7 @@ exports.fn = (root, params) => {
   };
 };
 
-},{"../xast.js":384}],359:[function(require,module,exports){
+},{"../xast.js":385}],359:[function(require,module,exports){
 'use strict';
 
 const { attrsGroups } = require('./_collections.js');
@@ -38270,7 +38271,7 @@ exports.fn = () => {
   };
 };
 
-},{"../xast.js":384,"./_collections.js":325}],361:[function(require,module,exports){
+},{"../xast.js":385,"./_collections.js":325}],361:[function(require,module,exports){
 'use strict';
 
 const { detachNodeFromParent } = require('../xast.js');
@@ -38321,7 +38322,7 @@ exports.fn = (root, params) => {
   };
 };
 
-},{"../xast.js":384}],362:[function(require,module,exports){
+},{"../xast.js":385}],362:[function(require,module,exports){
 'use strict';
 
 const {
@@ -38631,7 +38632,7 @@ exports.fn = (root, params) => {
   };
 };
 
-},{"../path.js":323,"../style.js":382,"../xast.js":384}],363:[function(require,module,exports){
+},{"../path.js":323,"../style.js":383,"../xast.js":385}],363:[function(require,module,exports){
 'use strict';
 
 const { detachNodeFromParent } = require('../xast.js');
@@ -38658,7 +38659,7 @@ exports.fn = () => {
   };
 };
 
-},{"../xast.js":384}],364:[function(require,module,exports){
+},{"../xast.js":385}],364:[function(require,module,exports){
 'use strict';
 
 const {
@@ -38815,7 +38816,7 @@ exports.fn = () => {
   };
 };
 
-},{"../path.js":323,"../xast.js":384,"./_path.js":326}],366:[function(require,module,exports){
+},{"../path.js":323,"../xast.js":385,"./_path.js":326}],366:[function(require,module,exports){
 'use strict';
 
 const { detachNodeFromParent } = require('../xast.js');
@@ -38846,7 +38847,7 @@ exports.fn = () => {
   };
 };
 
-},{"../xast.js":384}],367:[function(require,module,exports){
+},{"../xast.js":385}],367:[function(require,module,exports){
 'use strict';
 
 const { detachNodeFromParent } = require('../xast.js');
@@ -38873,7 +38874,7 @@ exports.fn = () => {
   };
 };
 
-},{"../xast.js":384}],368:[function(require,module,exports){
+},{"../xast.js":385}],368:[function(require,module,exports){
 'use strict';
 
 const { detachNodeFromParent } = require('../xast.js');
@@ -38900,7 +38901,7 @@ exports.fn = () => {
   };
 };
 
-},{"../xast.js":384}],369:[function(require,module,exports){
+},{"../xast.js":385}],369:[function(require,module,exports){
 'use strict';
 
 const { detachNodeFromParent } = require('../xast.js');
@@ -38927,7 +38928,7 @@ exports.fn = () => {
   };
 };
 
-},{"../xast.js":384}],370:[function(require,module,exports){
+},{"../xast.js":385}],370:[function(require,module,exports){
 'use strict';
 
 const { visitSkip, detachNodeFromParent } = require('../xast.js');
@@ -39135,7 +39136,7 @@ exports.fn = (root, params) => {
   };
 };
 
-},{"../style.js":382,"../xast.js":384,"./_collections":325}],371:[function(require,module,exports){
+},{"../style.js":383,"../xast.js":385,"./_collections":325}],371:[function(require,module,exports){
 'use strict';
 
 exports.name = 'removeUnusedNS';
@@ -39252,7 +39253,7 @@ const collectUsefulNodes = (node, usefulNodes) => {
   }
 };
 
-},{"../xast.js":384,"./_collections.js":325}],373:[function(require,module,exports){
+},{"../xast.js":385,"./_collections.js":325}],373:[function(require,module,exports){
 'use strict';
 
 const { visit, visitSkip, detachNodeFromParent } = require('../xast.js');
@@ -39390,7 +39391,7 @@ exports.fn = (root, params) => {
   };
 };
 
-},{"../style.js":382,"../xast.js":384,"./_collections.js":325}],374:[function(require,module,exports){
+},{"../style.js":383,"../xast.js":385,"./_collections.js":325}],374:[function(require,module,exports){
 'use strict';
 
 exports.name = 'removeViewBox';
@@ -39497,7 +39498,7 @@ exports.fn = () => {
   };
 };
 
-},{"../xast.js":384}],377:[function(require,module,exports){
+},{"../xast.js":385}],377:[function(require,module,exports){
 'use strict';
 
 exports.name = 'reusePaths';
@@ -39678,7 +39679,72 @@ const presetSafe = createPreset({
 
 module.exports = presetSafe;
 
-},{"../tools.js":383,"./cleanupAttrs.js":331,"./cleanupEnableBackground.js":332,"./cleanupIds.js":333,"./cleanupNumericValues.js":335,"./collapseGroups.js":336,"./convertColors.js":337,"./convertEllipseToCircle.js":338,"./convertPathData.js":339,"./convertTransform.js":342,"./inlineStyles.js":344,"./mergePaths.js":345,"./mergeStyles.js":346,"./minifyStyles.js":347,"./removeComments.js":353,"./removeDesc.js":354,"./removeDoctype.js":356,"./removeEditorsNSData.js":357,"./removeEmptyAttrs.js":359,"./removeEmptyContainers.js":360,"./removeEmptyText.js":361,"./removeHiddenElems.js":362,"./removeMetadata.js":363,"./removeNonInheritableGroupAttrs.js":364,"./removeTitle.js":369,"./removeUnknownsAndDefaults.js":370,"./removeUnusedNS.js":371,"./removeUselessDefs.js":372,"./removeUselessStrokeAndFill.js":373,"./removeViewBox.js":374,"./removeXMLProcInst.js":376,"./sortDefsChildren.js":380}],379:[function(require,module,exports){
+},{"../tools.js":384,"./cleanupAttrs.js":331,"./cleanupEnableBackground.js":332,"./cleanupIds.js":333,"./cleanupNumericValues.js":335,"./collapseGroups.js":336,"./convertColors.js":337,"./convertEllipseToCircle.js":338,"./convertPathData.js":339,"./convertTransform.js":342,"./inlineStyles.js":344,"./mergePaths.js":345,"./mergeStyles.js":346,"./minifyStyles.js":347,"./removeComments.js":353,"./removeDesc.js":354,"./removeDoctype.js":356,"./removeEditorsNSData.js":357,"./removeEmptyAttrs.js":359,"./removeEmptyContainers.js":360,"./removeEmptyText.js":361,"./removeHiddenElems.js":362,"./removeMetadata.js":363,"./removeNonInheritableGroupAttrs.js":364,"./removeTitle.js":369,"./removeUnknownsAndDefaults.js":370,"./removeUnusedNS.js":371,"./removeUselessDefs.js":372,"./removeUselessStrokeAndFill.js":373,"./removeViewBox.js":374,"./removeXMLProcInst.js":376,"./sortDefsChildren.js":381}],379:[function(require,module,exports){
+'use strict';
+
+const { createPreset } = require('../tools.js');
+
+const removeDoctype = require('./removeDoctype.js');
+const removeXMLProcInst = require('./removeXMLProcInst.js');
+const removeComments = require('./removeComments.js');
+const removeMetadata = require('./removeMetadata.js');
+const removeEditorsNSData = require('./removeEditorsNSData.js');
+const cleanupAttrs = require('./cleanupAttrs.js');
+const mergeStyles = require('./mergeStyles.js');
+const minifyStyles = require('./minifyStyles.js');
+const cleanupIds = require('./cleanupIds.js');
+const removeUselessDefs = require('./removeUselessDefs.js');
+const cleanupNumericValues = require('./cleanupNumericValues.js');
+const convertColors = require('./convertColors.js');
+const removeUnknownsAndDefaults = require('./removeUnknownsAndDefaults.js');
+const removeNonInheritableGroupAttrs = require('./removeNonInheritableGroupAttrs.js');
+const removeUselessStrokeAndFill = require('./removeUselessStrokeAndFill.js');
+const removeViewBox = require('./removeViewBox.js');
+const cleanupEnableBackground = require('./cleanupEnableBackground.js');
+const removeHiddenElems = require('./removeHiddenElems.js');
+const removeEmptyText = require('./removeEmptyText.js');
+const collapseGroups = require('./collapseGroups.js');
+const removeEmptyAttrs = require('./removeEmptyAttrs.js');
+const removeEmptyContainers = require('./removeEmptyContainers.js');
+const mergePaths = require('./mergePaths.js');
+const removeUnusedNS = require('./removeUnusedNS.js');
+const sortDefsChildren = require('./sortDefsChildren.js');
+const removeTitle = require('./removeTitle.js');
+const removeDesc = require('./removeDesc.js');
+
+const presetSafe = createPreset({
+  name: 'safeAndFastPreset',
+  plugins: [
+    removeDoctype,
+    removeXMLProcInst,
+    removeComments,
+    removeMetadata,
+    removeEditorsNSData,
+    cleanupAttrs,
+    mergeStyles,
+    cleanupIds,
+    removeUselessDefs,
+    cleanupNumericValues,
+    convertColors,
+    removeUnknownsAndDefaults,
+    removeNonInheritableGroupAttrs,
+    removeUselessStrokeAndFill,
+    removeViewBox,
+    cleanupEnableBackground,
+    removeHiddenElems,
+    removeEmptyText,
+    collapseGroups,
+    removeEmptyAttrs,
+    removeEmptyContainers,
+    removeUnusedNS,
+    removeTitle,
+    removeDesc,
+  ],
+});
+
+module.exports = presetSafe;
+
+},{"../tools.js":384,"./cleanupAttrs.js":331,"./cleanupEnableBackground.js":332,"./cleanupIds.js":333,"./cleanupNumericValues.js":335,"./collapseGroups.js":336,"./convertColors.js":337,"./mergePaths.js":345,"./mergeStyles.js":346,"./minifyStyles.js":347,"./removeComments.js":353,"./removeDesc.js":354,"./removeDoctype.js":356,"./removeEditorsNSData.js":357,"./removeEmptyAttrs.js":359,"./removeEmptyContainers.js":360,"./removeEmptyText.js":361,"./removeHiddenElems.js":362,"./removeMetadata.js":363,"./removeNonInheritableGroupAttrs.js":364,"./removeTitle.js":369,"./removeUnknownsAndDefaults.js":370,"./removeUnusedNS.js":371,"./removeUselessDefs.js":372,"./removeUselessStrokeAndFill.js":373,"./removeViewBox.js":374,"./removeXMLProcInst.js":376,"./sortDefsChildren.js":381}],380:[function(require,module,exports){
 'use strict';
 
 exports.name = 'sortAttrs';
@@ -39778,7 +39844,7 @@ exports.fn = (_root, params) => {
   };
 };
 
-},{}],380:[function(require,module,exports){
+},{}],381:[function(require,module,exports){
 'use strict';
 
 exports.name = 'sortDefsChildren';
@@ -39836,7 +39902,7 @@ exports.fn = () => {
   };
 };
 
-},{}],381:[function(require,module,exports){
+},{}],382:[function(require,module,exports){
 'use strict';
 
 const { textElems } = require('./plugins/_collections.js');
@@ -40070,7 +40136,7 @@ const stringifyText = (node, config, state) => {
   );
 };
 
-},{"./plugins/_collections.js":325}],382:[function(require,module,exports){
+},{"./plugins/_collections.js":325}],383:[function(require,module,exports){
 'use strict';
 
 const csstree = require('css-tree');
@@ -40300,7 +40366,7 @@ const computeStyle = (stylesheet, node) => {
 };
 exports.computeStyle = computeStyle;
 
-},{"./plugins/_collections.js":325,"./xast.js":384,"css-tree":25,"csso":138}],383:[function(require,module,exports){
+},{"./plugins/_collections.js":325,"./xast.js":385,"css-tree":25,"csso":138}],384:[function(require,module,exports){
 (function (Buffer){(function (){
 'use strict';
 
@@ -40455,7 +40521,7 @@ exports.invokePlugins = invokePlugins;
 exports.createPreset = createPreset;
 
 }).call(this)}).call(this,require("buffer").Buffer)
-},{"./xast.js":384,"buffer":4}],384:[function(require,module,exports){
+},{"./xast.js":385,"buffer":4}],385:[function(require,module,exports){
 'use strict';
 
 const { selectAll, selectOne, is } = require('css-select');