3pp-mirror/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-06-13 12:21:38 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #3959 from penpot/niwinz-staging-storage-improvements-2

Browse source 
✨ Improvements to components migration
This commit is contained in:
Alejandro 2024-01-04 15:55:31 +01:00 • committed by GitHub
commit 65b3c62a87
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
40 changed files with 904 additions and 678 deletions
Download patch file Download diff file Expand all files Collapse all files

2
backend/deps.edn
View file

@ -26,7 +26,7 @@
:git/url "https://github.com/funcool/yetti.git" :git/url "https://github.com/funcool/yetti.git"
:exclusions [org.slf4j/slf4j-api]} :exclusions [org.slf4j/slf4j-api]}
com.github.seancorfield/next.jdbc {:mvn/version "1.3.894"} com.github.seancorfield/next.jdbc {:mvn/version "1.3.909"}
metosin/reitit-core {:mvn/version "0.6.0"} metosin/reitit-core {:mvn/version "0.6.0"}
nrepl/nrepl {:mvn/version "1.1.0"} nrepl/nrepl {:mvn/version "1.1.0"}
cider/cider-nrepl {:mvn/version "0.43.1"} cider/cider-nrepl {:mvn/version "0.43.1"}

10
backend/dev/user.clj
View file

@ -8,6 +8,7 @@
(:require (:require
[app.common.data :as d] [app.common.data :as d]
[app.common.exceptions :as ex] [app.common.exceptions :as ex]
[app.common.files.helpers :as cfh]
[app.common.fressian :as fres] [app.common.fressian :as fres]
[app.common.geom.matrix :as gmt] [app.common.geom.matrix :as gmt]
[app.common.logging :as l] [app.common.logging :as l]
@ -136,3 +137,12 @@
(add-tap #(locking debug-tap (add-tap #(locking debug-tap
(prn "tap debug:" %))) (prn "tap debug:" %)))
1)) 1))
(defn calculate-frames
[{:keys [data]}]
(->> (vals (:pages-index data))
(mapcat (comp vals :objects))
(filter cfh/is-direct-child-of-root?)
(filter cfh/frame-shape?)
(count)))

5
backend/src/app/config.clj
View file

@ -207,6 +207,7 @@
(s/def ::telemetry-uri ::us/string) (s/def ::telemetry-uri ::us/string)
(s/def ::telemetry-with-taiga ::us/boolean) (s/def ::telemetry-with-taiga ::us/boolean)
(s/def ::tenant ::us/string) (s/def ::tenant ::us/string)
(s/def ::svgo-max-procs ::us/integer)
(s/def ::config (s/def ::config
(s/keys :opt-un [::secret-key (s/keys :opt-un [::secret-key
@ -326,7 +327,9 @@
::telemetry-uri ::telemetry-uri
::telemetry-referer ::telemetry-referer
::telemetry-with-taiga ::telemetry-with-taiga
::tenant])) ::tenant
::svgo-max-procs]))
(def default-flags (def default-flags
[:enable-backend-api-doc [:enable-backend-api-doc

144
backend/src/app/db.clj
View file

@ -19,6 +19,7 @@
[app.util.json :as json] [app.util.json :as json]
[app.util.time :as dt] [app.util.time :as dt]
[clojure.java.io :as io] [clojure.java.io :as io]
[clojure.set :as set]
[clojure.spec.alpha :as s] [clojure.spec.alpha :as s]
[integrant.core :as ig] [integrant.core :as ig]
[next.jdbc :as jdbc] [next.jdbc :as jdbc]
@ -239,6 +240,10 @@
(ex/raise :type :internal (ex/raise :type :internal
:code :unable-resolve-pool)))) :code :unable-resolve-pool))))
(defn get-update-count
[result]
(:next.jdbc/update-count result))
(defn get-connection (defn get-connection
[cfg-or-conn] [cfg-or-conn]
(if (connection? cfg-or-conn) (if (connection? cfg-or-conn)
@ -265,48 +270,120 @@
:code :unable-resolve-connectable :code :unable-resolve-connectable
:hint "expected conn, pool or system"))) :hint "expected conn, pool or system")))
(def ^:private params-mapping
{::return-keys? :return-keys
::return-keys :return-keys})
(defn rename-opts
[opts]
(set/rename-keys opts params-mapping))
(def ^:private default-insert-opts
{:builder-fn sql/as-kebab-maps
:return-keys true})
(def ^:private default-opts (def ^:private default-opts
{:builder-fn sql/as-kebab-maps}) {:builder-fn sql/as-kebab-maps})
(defn exec! (defn exec!
([ds sv] ([ds sv] (exec! ds sv nil))
(-> (get-connectable ds)
(jdbc/execute! sv default-opts)))
([ds sv opts] ([ds sv opts]
(-> (get-connectable ds) (let [conn (get-connectable ds)
(jdbc/execute! sv (into default-opts (sql/adapt-opts opts)))))) opts (if (empty? opts)
default-opts
(into default-opts (rename-opts opts)))]
(jdbc/execute! conn sv opts))))
(defn exec-one! (defn exec-one!
([ds sv] ([ds sv] (exec-one! ds sv nil))
(-> (get-connectable ds)
(jdbc/execute-one! sv default-opts)))
([ds sv opts] ([ds sv opts]
(-> (get-connectable ds) (let [conn (get-connectable ds)
(jdbc/execute-one! sv (into default-opts (sql/adapt-opts opts)))))) opts (if (empty? opts)
default-opts
(into default-opts (rename-opts opts)))]
(jdbc/execute-one! conn sv opts))))
(defn insert! (defn insert!
[ds table params & {:as opts :keys [::return-keys?] :or {return-keys? true}}] "A helper that builds an insert sql statement and executes it. By
(-> (get-connectable ds) default returns the inserted row with all the field; you can delimit
(exec-one! (sql/insert table params opts) the returned columns with the `::columns` option."
(assoc opts ::return-keys? return-keys?)))) [ds table params & {:as opts}]
(let [conn (get-connectable ds)
sql (sql/insert table params opts)
opts (if (empty? opts)
default-insert-opts
(into default-insert-opts (rename-opts opts)))]
(jdbc/execute-one! conn sql opts)))
(defn insert-multi! (defn insert-many!
[ds table cols rows & {:as opts :keys [::return-keys?] :or {return-keys? true}}] "An optimized version of `insert!` that perform insertion of multiple
(-> (get-connectable ds) values at once.
(exec! (sql/insert-multi table cols rows opts)
(assoc opts ::return-keys? return-keys?)))) This expands to a single SQL statement with placeholders for every
value being inserted. For large data sets, this may exceed the limit
of sql string size and/or number of parameters."
[ds table cols rows & {:as opts}]
(let [conn (get-connectable ds)
sql (sql/insert-many table cols rows opts)
opts (if (empty? opts)
default-insert-opts
(into default-insert-opts (rename-opts opts)))
opts (update opts :return-keys boolean)]
(jdbc/execute! conn sql opts)))
(defn update! (defn update!
[ds table params where & {:as opts :keys [::return-keys?] :or {return-keys? true}}] "A helper that build an UPDATE SQL statement and executes it.
(-> (get-connectable ds)
(exec-one! (sql/update table params where opts) Given a connectable object, a table name, a hash map of columns and
(assoc opts ::return-keys? return-keys?)))) values to set, and either a hash map of columns and values to search
on or a vector of a SQL where clause and parameters, perform an
update on the table.
By default returns an object with the number of affected rows; a
complete row can be returned if you pass `::return-keys` with `true`
or with a vector of columns.
Also it can be combined with the `::many` option if you perform an
update to multiple rows and you want all the affected rows to be
returned."
[ds table params where & {:as opts}]
(let [conn (get-connectable ds)
sql (sql/update table params where opts)
opts (if (empty? opts)
default-opts
(into default-opts (rename-opts opts)))
opts (update opts :return-keys boolean)]
(if (::many opts)
(jdbc/execute! conn sql opts)
(jdbc/execute-one! conn sql opts))))
(defn delete! (defn delete!
[ds table params & {:as opts :keys [::return-keys?] :or {return-keys? true}}] "A helper that builds an DELETE SQL statement and executes it.
(-> (get-connectable ds)
(exec-one! (sql/delete table params opts) Given a connectable object, a table name, and either a hash map of columns
(assoc opts ::return-keys? return-keys?)))) and values to search on or a vector of a SQL where clause and parameters,
perform a delete on the table.
By default returns an object with the number of affected rows; a
complete row can be returned if you pass `::return-keys` with `true`
or with a vector of columns.
Also it can be combined with the `::many` option if you perform an
update to multiple rows and you want all the affected rows to be
returned."
[ds table params & {:as opts}]
(let [conn (get-connectable ds)
sql (sql/delete table params opts)
opts (if (empty? opts)
default-opts
(into default-opts (rename-opts opts)))]
(if (::many opts)
(jdbc/execute! conn sql opts)
(jdbc/execute-one! conn sql opts))))
(defn query
[ds table params & {:as opts}]
(exec! ds (sql/select table params opts) opts))
(defn is-row-deleted? (defn is-row-deleted?
[{:keys [deleted-at]}] [{:keys [deleted-at]}]
@ -320,7 +397,7 @@
[ds table params & {:as opts}] [ds table params & {:as opts}]
(let [rows (exec! ds (sql/select table params opts)) (let [rows (exec! ds (sql/select table params opts))
rows (cond->> rows rows (cond->> rows
(::remove-deleted? opts true) (::remove-deleted opts true)
(remove is-row-deleted?))] (remove is-row-deleted?))]
(first rows))) (first rows)))
@ -329,7 +406,7 @@
filters. Raises :not-found exception if no object is found." filters. Raises :not-found exception if no object is found."
[ds table params & {:as opts}] [ds table params & {:as opts}]
(let [row (get* ds table params opts)] (let [row (get* ds table params opts)]
(when (and (not row) (::check-deleted? opts true)) (when (and (not row) (::check-deleted opts true))
(ex/raise :type :not-found (ex/raise :type :not-found
:code :object-not-found :code :object-not-found
:table table :table table
@ -364,10 +441,6 @@
[ds table id & {:as opts}] [ds table id & {:as opts}]
(get ds table {:id id} opts)) (get ds table {:id id} opts))
(defn query
[ds table params & {:as opts}]
(exec! ds (sql/select table params opts)))
(defn pgobject? (defn pgobject?
([v] ([v]
(instance? PGobject v)) (instance? PGobject v))
@ -567,11 +640,6 @@
(.setType "jsonb") (.setType "jsonb")
(.setValue (json/encode-str data))))) (.setValue (json/encode-str data)))))
(defn get-update-count
[result]
(:next.jdbc/update-count result))
;; --- Locks ;; --- Locks
(def ^:private siphash-state (def ^:private siphash-state

33
backend/src/app/db/sql.clj
View file

@ -8,7 +8,6 @@
(:refer-clojure :exclude [update]) (:refer-clojure :exclude [update])
(:require (:require
[app.db :as-alias db] [app.db :as-alias db]
[clojure.set :as set]
[clojure.string :as str] [clojure.string :as str]
[next.jdbc.optional :as jdbc-opt] [next.jdbc.optional :as jdbc-opt]
[next.jdbc.sql.builder :as sql])) [next.jdbc.sql.builder :as sql]))
@ -20,14 +19,6 @@
{:table-fn snake-case {:table-fn snake-case
:column-fn snake-case}) :column-fn snake-case})
(def params-mapping
{::db/return-keys? :return-keys
::db/columns :columns})
(defn adapt-opts
[opts]
(set/rename-keys opts params-mapping))
(defn as-kebab-maps (defn as-kebab-maps
[rs opts] [rs opts]
(jdbc-opt/as-unqualified-modified-maps rs (assoc opts :label-fn kebab-case))) (jdbc-opt/as-unqualified-modified-maps rs (assoc opts :label-fn kebab-case)))
@ -42,7 +33,7 @@
(assoc :suffix "ON CONFLICT DO NOTHING"))] (assoc :suffix "ON CONFLICT DO NOTHING"))]
(sql/for-insert table key-map opts)))) (sql/for-insert table key-map opts))))
(defn insert-multi (defn insert-many
[table cols rows opts] [table cols rows opts]
(let [opts (merge default-opts opts)] (let [opts (merge default-opts opts)]
(sql/for-insert-multi table cols rows opts))) (sql/for-insert-multi table cols rows opts)))
@ -53,11 +44,9 @@
([table where-params opts] ([table where-params opts]
(let [opts (merge default-opts opts) (let [opts (merge default-opts opts)
opts (cond-> opts opts (cond-> opts
(::db/columns opts) (assoc :columns (::db/columns opts)) (::columns opts) (assoc :columns (::columns opts))
(::db/for-update? opts) (assoc :suffix "FOR UPDATE") (::for-update opts) (assoc :suffix "FOR UPDATE")
(::db/for-share? opts) (assoc :suffix "FOR KEY SHARE") (::for-share opts) (assoc :suffix "FOR KEY SHARE"))]
(:for-update opts) (assoc :suffix "FOR UPDATE")
(:for-key-share opts) (assoc :suffix "FOR KEY SHARE"))]
(sql/for-query table where-params opts)))) (sql/for-query table where-params opts))))
(defn update (defn update
@ -65,11 +54,9 @@
(update table key-map where-params nil)) (update table key-map where-params nil))
([table key-map where-params opts] ([table key-map where-params opts]
(let [opts (into default-opts opts) (let [opts (into default-opts opts)
opts (if-let [columns (::db/columns opts)] keys (::db/return-keys opts)
(let [columns (if (seq columns) opts (if (vector? keys)
(sql/as-cols columns opts) (assoc opts :suffix (str "RETURNING " (sql/as-cols keys opts)))
"*")]
(assoc opts :suffix (str "RETURNING " columns)))
opts)] opts)]
(sql/for-update table key-map where-params opts)))) (sql/for-update table key-map where-params opts))))
@ -77,5 +64,9 @@
([table where-params] ([table where-params]
(delete table where-params nil)) (delete table where-params nil))
([table where-params opts] ([table where-params opts]
(let [opts (merge default-opts opts)] (let [opts (merge default-opts opts)
keys (::db/return-keys opts)
opts (if (vector? keys)
(assoc opts :suffix (str "RETURNING " (sql/as-cols keys opts)))
opts)]
(sql/for-delete table where-params opts)))) (sql/for-delete table where-params opts))))

292
backend/src/app/features/components_v2.clj
View file

@ -39,27 +39,54 @@
[app.rpc.commands.media :as cmd.media] [app.rpc.commands.media :as cmd.media]
[app.storage :as sto] [app.storage :as sto]
[app.storage.tmp :as tmp] [app.storage.tmp :as tmp]
[app.svgo :as svgo]
[app.util.blob :as blob] [app.util.blob :as blob]
[app.util.pointer-map :as pmap] [app.util.pointer-map :as pmap]
[app.util.time :as dt] [app.util.time :as dt]
[buddy.core.codecs :as bc] [buddy.core.codecs :as bc]
[cuerdas.core :as str] [cuerdas.core :as str]
[datoteka.io :as io] [datoteka.io :as io]
[promesa.exec :as px] [promesa.core :as p]))
[promesa.exec.semaphore :as ps]
[promesa.util :as pu]))
(def ^:dynamic *system* nil) (def ^:dynamic *stats*
(def ^:dynamic *stats* nil) "A dynamic var for setting up state for collect stats globally."
(def ^:dynamic *file-stats* nil) nil)
(def ^:dynamic *team-stats* nil)
(def ^:dynamic *semaphore* nil) (def ^:dynamic *skip-on-error*
(def ^:dynamic *skip-on-error* true) "A dynamic var for setting up the default error behavior."
true)
(def ^:dynamic ^:private *system*
"An internal var for making the current `system` available to all
internal functions without the need to explicitly pass it top down."
nil)
(def ^:dynamic ^:private *max-procs*
"A dynamic variable that can optionally indicates the maxumum number
of concurrent graphics migration processes."
nil)
(def ^:dynamic ^:private *file-stats*
"An internal dynamic var for collect stats by file."
nil)
(def ^:dynamic ^:private *team-stats*
"An internal dynamic var for collect stats by team."
nil)
(def grid-gap 50) (def grid-gap 50)
(def frame-gap 200) (def frame-gap 200)
(def max-group-size 50) (def max-group-size 50)
(defn decode-row
[{:keys [features data] :as row}]
(cond-> row
(some? features)
(assoc :features (db/decode-pgarray features #{}))
(some? data)
(assoc :data (blob/decode data))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; FILE PREPARATION BEFORE MIGRATION ;; FILE PREPARATION BEFORE MIGRATION
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@ -220,19 +247,17 @@
(fn [file-data] (fn [file-data]
;; Transform component and copy heads to frames, and set the ;; Transform component and copy heads to frames, and set the
;; frame-id of its childrens ;; frame-id of its childrens
(letfn [(fix-container (letfn [(fix-container [container]
[container]
(update container :objects update-vals fix-shape)) (update container :objects update-vals fix-shape))
(fix-shape (fix-shape [shape]
[shape]
(if (or (nil? (:parent-id shape)) (ctk/instance-head? shape)) (if (or (nil? (:parent-id shape)) (ctk/instance-head? shape))
(assoc shape (assoc shape
:type :frame ; Old groups must be converted :type :frame ; Old groups must be converted
:fills (or (:fills shape) []) ; to frames and conform to spec :fills (or (:fills shape) []) ; to frames and conform to spec
:hide-in-viewer (or (:hide-in-viewer shape) true) :hide-in-viewer (or (:hide-in-viewer shape) true)
:rx (or (:rx shape) 0) :rx (or (:rx shape) 0)
:ry (or (:ry shape) 0)) :ry (or (:ry shape) 0))
shape))] shape))]
(-> file-data (-> file-data
(update :pages-index update-vals fix-container) (update :pages-index update-vals fix-container)
@ -310,10 +335,10 @@
(defn- get-asset-groups (defn- get-asset-groups
[assets generic-name] [assets generic-name]
(let [; Group by first element of the path. (let [;; Group by first element of the path.
groups (d/group-by #(first (cfh/split-path (:path %))) assets) groups (d/group-by #(first (cfh/split-path (:path %))) assets)
; Split large groups in chunks of max-group-size elements ;; Split large groups in chunks of max-group-size elements
groups (loop [groups (seq groups) groups (loop [groups (seq groups)
result {}] result {}]
(if (empty? groups) (if (empty? groups)
@ -334,15 +359,14 @@
result result
splits))))))) splits)))))))
; Sort assets in each group by path ;; Sort assets in each group by path
groups (update-vals groups (fn [assets] groups (update-vals groups (fn [assets]
(sort-by (fn [{:keys [path name]}] (sort-by (fn [{:keys [path name]}]
(str/lower (cfh/merge-path-item path name))) (str/lower (cfh/merge-path-item path name)))
assets))) assets)))]
; Sort groups by name ;; Sort groups by name
groups (into (sorted-map) groups)] (into (sorted-map) groups)))
groups))
(defn- create-frame (defn- create-frame
[name position width height] [name position width height]
@ -612,14 +636,11 @@
(defn- create-shapes-for-svg (defn- create-shapes-for-svg
[{:keys [id] :as mobj} file-id objects frame-id position] [{:keys [id] :as mobj} file-id objects frame-id position]
(let [svg-text (get-svg-content id) (let [svg-text (get-svg-content id)
svg-text (svgo/optimize *system* svg-text)
optimizer (::csvg/optimizer *system*) svg-data (-> (csvg/parse svg-text)
svg-text (csvg/optimize optimizer svg-text) (assoc :name (:name mobj))
(collect-and-persist-images file-id))]
svg-data (-> (csvg/parse svg-text)
(assoc :name (:name mobj))
(collect-and-persist-images file-id))]
(sbuilder/create-svg-shapes svg-data position objects frame-id frame-id #{} false))) (sbuilder/create-svg-shapes svg-data position objects frame-id frame-id #{} false)))
@ -678,9 +699,7 @@
(defn- create-media-grid (defn- create-media-grid
[fdata page-id frame-id grid media-group] [fdata page-id frame-id grid media-group]
(let [factory (px/thread-factory :virtual true) (let [process (fn [mobj position]
executor (px/fixed-executor :parallelism 10 :factory factory)
process (fn [mobj position]
(let [position (gpt/add position (gpt/point grid-gap grid-gap)) (let [position (gpt/add position (gpt/point grid-gap grid-gap))
tp1 (dt/tpoint)] tp1 (dt/tpoint)]
(try (try
@ -690,7 +709,6 @@
:file-id (str (:id fdata)) :file-id (str (:id fdata))
:id (str (:id mobj)) :id (str (:id mobj))
:cause cause) :cause cause)
(if-not *skip-on-error* (if-not *skip-on-error*
(throw cause) (throw cause)
nil)) nil))
@ -699,21 +717,24 @@
:file-id (str (:id fdata)) :file-id (str (:id fdata))
:media-id (str (:id mobj)) :media-id (str (:id mobj))
:elapsed (dt/format-duration (tp1)))))))] :elapsed (dt/format-duration (tp1)))))))]
(try
(->> (d/zip media-group grid) (->> (d/zip media-group grid)
(map (fn [[mobj position]] (partition-all (or *max-procs* 1))
(sse/tap {:type :migration-progress (mapcat (fn [partition]
:section :graphics (->> partition
:name (:name mobj)}) (map (fn [[mobj position]]
(px/submit! executor (partial process mobj position)))) (sse/tap {:type :migration-progress
(reduce (fn [fdata promise] :section :graphics
(if-let [changes (deref promise)] :name (:name mobj)})
(-> (assoc-in fdata [:options :components-v2] true) (p/vthread (process mobj position))))
(cp/process-changes changes false)) (doall)
fdata)) (map deref)
fdata)) (doall))))
(finally (filter some?)
(pu/close! executor))))) (reduce (fn [fdata changes]
(-> (assoc-in fdata [:options :components-v2] true)
(cp/process-changes changes false)))
fdata))))
(defn- migrate-graphics (defn- migrate-graphics
[fdata] [fdata]
@ -759,6 +780,11 @@
(create-media-grid fdata page-id (:id frame) grid assets) (create-media-grid fdata page-id (:id frame) grid assets)
(gpt/add position (gpt/point 0 (+ height (* 2 grid-gap) frame-gap)))))))))) (gpt/add position (gpt/point 0 (+ height (* 2 grid-gap) frame-gap))))))))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; PRIVATE HELPERS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn- migrate-fdata (defn- migrate-fdata
[fdata libs] [fdata libs]
(let [migrated? (dm/get-in fdata [:options :components-v2])] (let [migrated? (dm/get-in fdata [:options :components-v2])]
@ -771,11 +797,22 @@
(defn- get-file (defn- get-file
[system id] [system id]
(binding [pmap/*load-fn* (partial fdata/load-pointer system id)] (binding [pmap/*load-fn* (partial fdata/load-pointer system id)]
(-> (files/get-file system id :migrate? false) (-> (db/get system :file {:id id}
{::db/remove-deleted false
::db/check-deleted false})
(decode-row)
(update :data assoc :id id) (update :data assoc :id id)
(update :data fdata/process-pointers deref) (update :data fdata/process-pointers deref)
(fmg/migrate-file)))) (fmg/migrate-file))))
(defn- get-team
[system team-id]
(-> (db/get system :team {:id team-id}
{::db/remove-deleted false
::db/check-deleted false})
(decode-row)))
(defn- validate-file! (defn- validate-file!
[file libs throw-on-validate?] [file libs throw-on-validate?]
(try (try
@ -791,7 +828,8 @@
(let [file (get-file system id) (let [file (get-file system id)
libs (->> (files/get-file-libraries conn id) libs (->> (files/get-file-libraries conn id)
(into [file] (comp (map :id) (map (partial get-file system)))) (into [file] (comp (map :id)
(map (partial get-file system))))
(d/index-by :id)) (d/index-by :id))
file (-> file file (-> file
@ -816,18 +854,39 @@
{:data (blob/encode (:data file)) {:data (blob/encode (:data file))
:features (db/create-array conn "text" (:features file)) :features (db/create-array conn "text" (:features file))
:revn (:revn file)} :revn (:revn file)}
{:id (:id file)} {:id (:id file)})
{::db/return-keys? false})
(dissoc file :data))) (dissoc file :data)))
(def ^:private sql:get-and-lock-team-files
"SELECT f.id
FROM file AS f
JOIN project AS p ON (p.id = f.project_id)
WHERE p.team_id = ?
FOR UPDATE")
(defn- get-and-lock-files
[conn team-id]
(->> (db/cursor conn [sql:get-and-lock-team-files team-id])
(map :id)))
(defn- update-team-features!
[conn team-id features]
(let [features (db/create-array conn "text" features)]
(db/update! conn :team
{:features features}
{:id team-id})))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; PUBLIC API
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn migrate-file! (defn migrate-file!
[system file-id & {:keys [validate? throw-on-validate?]}] [system file-id & {:keys [validate? throw-on-validate? max-procs]}]
(let [tpoint (dt/tpoint) (let [tpoint (dt/tpoint)]
file-id (if (string? file-id) (binding [*file-stats* (atom {})
(parse-uuid file-id) *max-procs* max-procs]
file-id)]
(binding [*file-stats* (atom {})]
(try (try
(l/dbg :hint "migrate:file:start" :file-id (str file-id)) (l/dbg :hint "migrate:file:start" :file-id (str file-id))
@ -839,7 +898,6 @@
(process-file system file-id (process-file system file-id
:validate? validate? :validate? validate?
:throw-on-validate? throw-on-validate?))))) :throw-on-validate? throw-on-validate?)))))
(finally (finally
(let [elapsed (tpoint) (let [elapsed (tpoint)
components (get @*file-stats* :processed/components 0) components (get @*file-stats* :processed/components 0)
@ -855,73 +913,51 @@
(some-> *team-stats* (swap! update :processed/files (fnil inc 0))))))))) (some-> *team-stats* (swap! update :processed/files (fnil inc 0)))))))))
(defn migrate-team! (defn migrate-team!
[system team-id & {:keys [validate? throw-on-validate?]}] [system team-id & {:keys [validate? throw-on-validate? max-procs]}]
(let [tpoint (dt/tpoint)
team-id (if (string? team-id) (l/dbg :hint "migrate:team:start"
(parse-uuid team-id) :team-id (dm/str team-id))
team-id)]
(l/dbg :hint "migrate:team:start" :team-id (dm/str team-id)) (let [tpoint (dt/tpoint)
migrate-file
(fn [system file-id]
(migrate-file! system file-id
:max-procs max-procs
:validate? validate?
:throw-on-validate? throw-on-validate?))
migrate-team
(fn [{:keys [::db/conn] :as system} {:keys [id features] :as team}]
(let [features (-> features
(disj "ephimeral/v2-migration")
(conj "components/v2")
(conj "layout/grid")
(conj "styles/v2"))]
(run! (partial migrate-file system)
(get-and-lock-files conn id))
(update-team-features! conn id features)))]
(binding [*team-stats* (atom {})] (binding [*team-stats* (atom {})]
(try (try
;; We execute this out of transaction because we want this (db/tx-run! system (fn [system]
;; change to be visible to all other sessions before starting (db/exec-one! system ["SET idle_in_transaction_session_timeout = 0"])
;; the migration (let [team (get-team system team-id)]
(let [sql (str "UPDATE team SET features = " (if (contains? (:features team) "components/v2")
" array_append(features, 'ephimeral/v2-migration') " (l/inf :hint "team already migrated")
" WHERE id = ?")] (migrate-team system team)))))
(db/exec-one! system [sql team-id]))
(db/tx-run! system
(fn [{:keys [::db/conn] :as system}]
;; Lock the team
(db/exec-one! conn ["SET idle_in_transaction_session_timeout = 0"])
(let [{:keys [features] :as team} (-> (db/get conn :team {:id team-id})
(update :features db/decode-pgarray #{}))]
(if (contains? features "components/v2")
(l/dbg :hint "team already migrated")
(let [sql (str/concat
"SELECT f.id FROM file AS f "
" JOIN project AS p ON (p.id = f.project_id) "
"WHERE p.team_id = ? AND f.deleted_at IS NULL AND p.deleted_at IS NULL "
"FOR UPDATE")]
(doseq [file-id (->> (db/exec! conn [sql team-id])
(map :id))]
(migrate-file! system file-id
:validate? validate?
:throw-on-validate? throw-on-validate?))
(let [features (-> features
(disj "ephimeral/v2-migration")
(conj "components/v2")
(conj "layout/grid")
(conj "styles/v2"))]
(db/update! conn :team
{:features (db/create-array conn "text" features)}
{:id team-id})))))))
(finally (finally
(some-> *semaphore* ps/release!) (let [elapsed (tpoint)
(let [elapsed (tpoint)] components (get @*team-stats* :processed/components 0)
graphics (get @*team-stats* :processed/graphics 0)
files (get @*team-stats* :processed/files 0)]
(some-> *stats* (swap! update :processed/teams (fnil inc 0))) (some-> *stats* (swap! update :processed/teams (fnil inc 0)))
;; We execute this out of transaction because we want this (l/dbg :hint "migrate:team:end"
;; change to be visible to all other sessions before starting :team-id (dm/str team-id)
;; the migration :files files
(let [sql (str "UPDATE team SET features = " :components components
" array_remove(features, 'ephimeral/v2-migration') " :graphics graphics
" WHERE id = ?")] :elapsed (dt/format-duration elapsed))))))))
(db/exec-one! system [sql team-id]))
(let [components (get @*team-stats* :processed/components 0)
graphics (get @*team-stats* :processed/graphics 0)
files (get @*team-stats* :processed/files 0)]
(l/dbg :hint "migrate:team:end"
:team-id (dm/str team-id)
:files files
:components components
:graphics graphics
:elapsed (dt/format-duration elapsed)))))))))

5
backend/src/app/features/fdata.clj
View file

@ -11,6 +11,7 @@
[app.common.exceptions :as ex] [app.common.exceptions :as ex]
[app.common.logging :as l] [app.common.logging :as l]
[app.db :as db] [app.db :as db]
[app.db.sql :as-alias sql]
[app.util.blob :as blob] [app.util.blob :as blob]
[app.util.objects-map :as omap] [app.util.objects-map :as omap]
[app.util.pointer-map :as pmap])) [app.util.pointer-map :as pmap]))
@ -38,8 +39,8 @@
[system file-id id] [system file-id id]
(let [{:keys [content]} (db/get system :file-data-fragment (let [{:keys [content]} (db/get system :file-data-fragment
{:id id :file-id file-id} {:id id :file-id file-id}
{::db/columns [:content] {::sql/columns [:content]
::db/check-deleted? false})] ::db/check-deleted false})]
(when-not content (when-not content
(ex/raise :type :internal (ex/raise :type :internal
:code :fragment-not-found :code :fragment-not-found

6
backend/src/app/loggers/webhooks.clj
View file

@ -111,9 +111,11 @@
" where id=?") " where id=?")
err err
(:id whook)] (:id whook)]
res (db/exec-one! pool sql {::db/return-keys? true})] res (db/exec-one! pool sql {::db/return-keys true})]
(when (>= (:error-count res) max-errors) (when (>= (:error-count res) max-errors)
(db/update! pool :webhook {:is-active false} {:id (:id whook)}))) (db/update! pool :webhook
{:is-active false}
{:id (:id whook)})))
(db/update! pool :webhook (db/update! pool :webhook
{:updated-at (dt/now) {:updated-at (dt/now)

9
backend/src/app/main.clj
View file

@ -10,7 +10,6 @@
[app.auth.oidc :as-alias oidc] [app.auth.oidc :as-alias oidc]
[app.auth.oidc.providers :as-alias oidc.providers] [app.auth.oidc.providers :as-alias oidc.providers]
[app.common.logging :as l] [app.common.logging :as l]
[app.common.svg :as csvg]
[app.config :as cf] [app.config :as cf]
[app.db :as-alias db] [app.db :as-alias db]
[app.email :as-alias email] [app.email :as-alias email]
@ -37,6 +36,7 @@
[app.storage.gc-deleted :as-alias sto.gc-deleted] [app.storage.gc-deleted :as-alias sto.gc-deleted]
[app.storage.gc-touched :as-alias sto.gc-touched] [app.storage.gc-touched :as-alias sto.gc-touched]
[app.storage.s3 :as-alias sto.s3] [app.storage.s3 :as-alias sto.s3]
[app.svgo :as-alias svgo]
[app.util.time :as dt] [app.util.time :as dt]
[app.worker :as-alias wrk] [app.worker :as-alias wrk]
[cider.nrepl :refer [cider-nrepl-handler]] [cider.nrepl :refer [cider-nrepl-handler]]
@ -316,7 +316,7 @@
::mtx/metrics (ig/ref ::mtx/metrics) ::mtx/metrics (ig/ref ::mtx/metrics)
::mbus/msgbus (ig/ref ::mbus/msgbus) ::mbus/msgbus (ig/ref ::mbus/msgbus)
::rds/redis (ig/ref ::rds/redis) ::rds/redis (ig/ref ::rds/redis)
::csvg/optimizer (ig/ref ::csvg/optimizer) ::svgo/optimizer (ig/ref ::svgo/optimizer)
::rpc/climit (ig/ref ::rpc/climit) ::rpc/climit (ig/ref ::rpc/climit)
::rpc/rlimit (ig/ref ::rpc/rlimit) ::rpc/rlimit (ig/ref ::rpc/rlimit)
@ -409,8 +409,9 @@
;; module requires the migrations to run before initialize. ;; module requires the migrations to run before initialize.
::migrations (ig/ref :app.migrations/migrations)} ::migrations (ig/ref :app.migrations/migrations)}
::csvg/optimizer ::svgo/optimizer
{} {::wrk/executor (ig/ref ::wrk/executor)
::svgo/max-procs (cf/get :svgo-max-procs)}
::audit.tasks/archive ::audit.tasks/archive
{::props (ig/ref ::setup/props) {::props (ig/ref ::setup/props)

2
backend/src/app/rpc/commands/audit.clj
View file

@ -48,7 +48,7 @@
(map event->row)) (map event->row))
events (sequence xform events)] events (sequence xform events)]
(when (seq events) (when (seq events)
(db/insert-multi! pool :audit-log event-columns events)))) (db/insert-many! pool :audit-log event-columns events))))
(def schema:event (def schema:event
[:map {:title "Event"} [:map {:title "Event"}

6
backend/src/app/rpc/commands/auth.clj
View file

@ -133,7 +133,8 @@
(update-password [conn profile-id] (update-password [conn profile-id]
(let [pwd (profile/derive-password cfg password)] (let [pwd (profile/derive-password cfg password)]
(db/update! conn :profile {:password pwd} {:id profile-id})))] (db/update! conn :profile {:password pwd} {:id profile-id})
nil))]
(db/with-atomic [conn pool] (db/with-atomic [conn pool]
(->> (validate-token token) (->> (validate-token token)
@ -303,7 +304,8 @@
(-> (db/update! conn :profile (-> (db/update! conn :profile
{:default-team-id (:id team) {:default-team-id (:id team)
:default-project-id (:default-project-id team)} :default-project-id (:default-project-id team)}
{:id id}) {:id id}
{::db/return-keys true})
(profile/decode-row)))) (profile/decode-row))))

3
backend/src/app/rpc/commands/binfile.clj
View file

@ -317,7 +317,7 @@
[cfg file-id] [cfg file-id]
(db/run! cfg (fn [{:keys [::db/conn] :as cfg}] (db/run! cfg (fn [{:keys [::db/conn] :as cfg}]
(binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)] (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
(some-> (db/get* conn :file {:id file-id} {::db/remove-deleted? false}) (some-> (db/get* conn :file {:id file-id} {::db/remove-deleted false})
(files/decode-row) (files/decode-row)
(update :data feat.fdata/process-pointers deref)))))) (update :data feat.fdata/process-pointers deref))))))
@ -664,6 +664,7 @@
(case feature (case feature
"components/v2" "components/v2"
(feat.compv2/migrate-file! options file-id (feat.compv2/migrate-file! options file-id
:max-procs 2
:validate? validate? :validate? validate?
:throw-on-validate? true) :throw-on-validate? true)

27
backend/src/app/rpc/commands/comments.clj
View file

@ -12,6 +12,7 @@
[app.common.spec :as us] [app.common.spec :as us]
[app.common.uuid :as uuid] [app.common.uuid :as uuid]
[app.db :as db] [app.db :as db]
[app.db.sql :as sql]
[app.features.fdata :as feat.fdata] [app.features.fdata :as feat.fdata]
[app.loggers.audit :as-alias audit] [app.loggers.audit :as-alias audit]
[app.loggers.webhooks :as-alias webhooks] [app.loggers.webhooks :as-alias webhooks]
@ -62,8 +63,8 @@
(decode-row))) (decode-row)))
(defn- get-comment (defn- get-comment
[conn comment-id & {:keys [for-update?]}] [conn comment-id & {:as opts}]
(db/get-by-id conn :comment comment-id {:for-update for-update?})) (db/get-by-id conn :comment comment-id opts))
(defn- get-next-seqn (defn- get-next-seqn
[conn file-id] [conn file-id]
@ -375,7 +376,7 @@
{::doc/added "1.15"} {::doc/added "1.15"}
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id share-id] :as params}] [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id share-id] :as params}]
(db/with-atomic [conn pool] (db/with-atomic [conn pool]
(let [{:keys [file-id] :as thread} (get-comment-thread conn id ::db/for-update? true)] (let [{:keys [file-id] :as thread} (get-comment-thread conn id ::sql/for-update true)]
(files/check-comment-permissions! conn profile-id file-id share-id) (files/check-comment-permissions! conn profile-id file-id share-id)
(upsert-comment-thread-status! conn profile-id id)))) (upsert-comment-thread-status! conn profile-id id))))
@ -392,7 +393,7 @@
{::doc/added "1.15"} {::doc/added "1.15"}
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id is-resolved share-id] :as params}] [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id is-resolved share-id] :as params}]
(db/with-atomic [conn pool] (db/with-atomic [conn pool]
(let [{:keys [file-id] :as thread} (get-comment-thread conn id ::db/for-update? true)] (let [{:keys [file-id] :as thread} (get-comment-thread conn id ::sql/for-update true)]
(files/check-comment-permissions! conn profile-id file-id share-id) (files/check-comment-permissions! conn profile-id file-id share-id)
(db/update! conn :comment-thread (db/update! conn :comment-thread
{:is-resolved is-resolved} {:is-resolved is-resolved}
@ -415,7 +416,7 @@
[cfg {:keys [::rpc/profile-id ::rpc/request-at thread-id share-id content]}] [cfg {:keys [::rpc/profile-id ::rpc/request-at thread-id share-id content]}]
(db/tx-run! cfg (db/tx-run! cfg
(fn [{:keys [::db/conn] :as cfg}] (fn [{:keys [::db/conn] :as cfg}]
(let [{:keys [file-id page-id] :as thread} (get-comment-thread conn thread-id ::db/for-update? true) (let [{:keys [file-id page-id] :as thread} (get-comment-thread conn thread-id ::sql/for-update true)
{:keys [team-id project-id page-name] :as file} (get-file cfg file-id page-id)] {:keys [team-id project-id page-name] :as file} (get-file cfg file-id page-id)]
(files/check-comment-permissions! conn profile-id (:id file) share-id) (files/check-comment-permissions! conn profile-id (:id file) share-id)
@ -471,8 +472,8 @@
(db/tx-run! cfg (db/tx-run! cfg
(fn [{:keys [::db/conn] :as cfg}] (fn [{:keys [::db/conn] :as cfg}]
(let [{:keys [thread-id owner-id] :as comment} (get-comment conn id ::db/for-update? true) (let [{:keys [thread-id owner-id] :as comment} (get-comment conn id ::sql/for-update true)
{:keys [file-id page-id] :as thread} (get-comment-thread conn thread-id ::db/for-update? true)] {:keys [file-id page-id] :as thread} (get-comment-thread conn thread-id ::sql/for-update true)]
(files/check-comment-permissions! conn profile-id file-id share-id) (files/check-comment-permissions! conn profile-id file-id share-id)
@ -504,7 +505,7 @@
{::doc/added "1.15"} {::doc/added "1.15"}
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id share-id]}] [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id share-id]}]
(db/with-atomic [conn pool] (db/with-atomic [conn pool]
(let [{:keys [owner-id file-id] :as thread} (get-comment-thread conn id ::db/for-update? true)] (let [{:keys [owner-id file-id] :as thread} (get-comment-thread conn id ::sql/for-update true)]
(files/check-comment-permissions! conn profile-id file-id share-id) (files/check-comment-permissions! conn profile-id file-id share-id)
(when-not (= owner-id profile-id) (when-not (= owner-id profile-id)
(ex/raise :type :validation (ex/raise :type :validation
@ -524,14 +525,14 @@
{::doc/added "1.15"} {::doc/added "1.15"}
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id share-id] :as params}] [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id share-id] :as params}]
(db/with-atomic [conn pool] (db/with-atomic [conn pool]
(let [{:keys [owner-id thread-id] :as comment} (get-comment conn id ::db/for-update? true) (let [{:keys [owner-id thread-id] :as comment} (get-comment conn id ::sql/for-update true)
{:keys [file-id] :as thread} (get-comment-thread conn thread-id)] {:keys [file-id] :as thread} (get-comment-thread conn thread-id)]
(files/check-comment-permissions! conn profile-id file-id share-id) (files/check-comment-permissions! conn profile-id file-id share-id)
(when-not (= owner-id profile-id) (when-not (= owner-id profile-id)
(ex/raise :type :validation (ex/raise :type :validation
:code :not-allowed)) :code :not-allowed))
(db/delete! conn :comment {:id id})))) (db/delete! conn :comment {:id id})
nil)))
;; --- COMMAND: Update comment thread position ;; --- COMMAND: Update comment thread position
@ -544,7 +545,7 @@
{::doc/added "1.15"} {::doc/added "1.15"}
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id ::rpc/request-at id position frame-id share-id]}] [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id ::rpc/request-at id position frame-id share-id]}]
(db/with-atomic [conn pool] (db/with-atomic [conn pool]
(let [{:keys [file-id] :as thread} (get-comment-thread conn id ::db/for-update? true)] (let [{:keys [file-id] :as thread} (get-comment-thread conn id ::sql/for-update true)]
(files/check-comment-permissions! conn profile-id file-id share-id) (files/check-comment-permissions! conn profile-id file-id share-id)
(db/update! conn :comment-thread (db/update! conn :comment-thread
{:modified-at request-at {:modified-at request-at
@ -564,7 +565,7 @@
{::doc/added "1.15"} {::doc/added "1.15"}
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id ::rpc/request-at id frame-id share-id]}] [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id ::rpc/request-at id frame-id share-id]}]
(db/with-atomic [conn pool] (db/with-atomic [conn pool]
(let [{:keys [file-id] :as thread} (get-comment-thread conn id ::db/for-update? true)] (let [{:keys [file-id] :as thread} (get-comment-thread conn id ::sql/for-update true)]
(files/check-comment-permissions! conn profile-id file-id share-id) (files/check-comment-permissions! conn profile-id file-id share-id)
(db/update! conn :comment-thread (db/update! conn :comment-thread
{:modified-at request-at {:modified-at request-at

29
backend/src/app/rpc/commands/files.clj
View file

@ -20,6 +20,7 @@
[app.common.types.file :as ctf] [app.common.types.file :as ctf]
[app.config :as cf] [app.config :as cf]
[app.db :as db] [app.db :as db]
[app.db.sql :as-alias sql]
[app.features.fdata :as feat.fdata] [app.features.fdata :as feat.fdata]
[app.loggers.audit :as-alias audit] [app.loggers.audit :as-alias audit]
[app.loggers.webhooks :as-alias webhooks] [app.loggers.webhooks :as-alias webhooks]
@ -238,8 +239,7 @@
(db/update! conn :file (db/update! conn :file
{:data (blob/encode (:data file)) {:data (blob/encode (:data file))
:features (db/create-array conn "text" (:features file))} :features (db/create-array conn "text" (:features file))}
{:id id} {:id id})
{::db/return-keys? false})
(when (contains? (:features file) "fdata/pointer-map") (when (contains? (:features file) "fdata/pointer-map")
(feat.fdata/persist-pointers! cfg id))) (feat.fdata/persist-pointers! cfg id)))
@ -262,9 +262,9 @@
(when (some? project-id) (when (some? project-id)
{:project-id project-id})) {:project-id project-id}))
file (-> (db/get conn :file params file (-> (db/get conn :file params
{::db/check-deleted? (not include-deleted?) {::db/check-deleted (not include-deleted?)
::db/remove-deleted? (not include-deleted?) ::db/remove-deleted (not include-deleted?)
::db/for-update? lock-for-update?}) ::sql/for-update lock-for-update?})
(decode-row))] (decode-row))]
(if migrate? (if migrate?
(migrate-file cfg file) (migrate-file cfg file)
@ -733,7 +733,8 @@
(db/update! conn :file (db/update! conn :file
{:name name {:name name
:modified-at (dt/now)} :modified-at (dt/now)}
{:id id})) {:id id}
{::db/return-keys true}))
(sv/defmethod ::rename-file (sv/defmethod ::rename-file
{::doc/added "1.17" {::doc/added "1.17"
@ -860,9 +861,7 @@
(let [file (assoc file :is-shared true)] (let [file (assoc file :is-shared true)]
(db/update! conn :file (db/update! conn :file
{:is-shared true} {:is-shared true}
{:id id} {:id id})
::db/return-keys? false)
file) file)
:else :else
@ -899,7 +898,7 @@
(db/update! conn :file (db/update! conn :file
{:deleted-at (dt/now)} {:deleted-at (dt/now)}
{:id file-id} {:id file-id}
{::db/columns [:id :name :is-shared :project-id :created-at :modified-at]})) {::db/return-keys [:id :name :is-shared :project-id :created-at :modified-at]}))
(def ^:private (def ^:private
schema:delete-file schema:delete-file
@ -998,8 +997,8 @@
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}] [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
(db/with-atomic [conn pool] (db/with-atomic [conn pool]
(check-edition-permissions! conn profile-id file-id) (check-edition-permissions! conn profile-id file-id)
(unlink-file-from-library conn params))) (unlink-file-from-library conn params)
nil))
;; --- MUTATION COMMAND: update-sync ;; --- MUTATION COMMAND: update-sync
@ -1008,7 +1007,8 @@
(db/update! conn :file-library-rel (db/update! conn :file-library-rel
{:synced-at (dt/now)} {:synced-at (dt/now)}
{:file-id file-id {:file-id file-id
:library-file-id library-id})) :library-file-id library-id}
{::db/return-keys true}))
(def ^:private schema:update-file-library-sync-status (def ^:private schema:update-file-library-sync-status
[:map {:title "update-file-library-sync-status"} [:map {:title "update-file-library-sync-status"}
@ -1031,7 +1031,8 @@
[conn {:keys [file-id date] :as params}] [conn {:keys [file-id date] :as params}]
(db/update! conn :file (db/update! conn :file
{:ignore-sync-until date} {:ignore-sync-until date}
{:id file-id})) {:id file-id}
{::db/return-keys true}))
(s/def ::ignore-file-library-sync-status (s/def ::ignore-file-library-sync-status
(s/keys :req [::rpc/profile-id] (s/keys :req [::rpc/profile-id]

14
backend/src/app/rpc/commands/files_thumbnails.clj
View file

@ -17,6 +17,7 @@
[app.common.types.shape-tree :as ctt] [app.common.types.shape-tree :as ctt]
[app.config :as cf] [app.config :as cf]
[app.db :as db] [app.db :as db]
[app.db.sql :as-alias sql]
[app.features.fdata :as feat.fdata] [app.features.fdata :as feat.fdata]
[app.loggers.audit :as-alias audit] [app.loggers.audit :as-alias audit]
[app.loggers.webhooks :as-alias webhooks] [app.loggers.webhooks :as-alias webhooks]
@ -236,8 +237,8 @@
{:file-id file-id {:file-id file-id
:object-id object-id :object-id object-id
:tag tag} :tag tag}
{::db/remove-deleted? false {::db/remove-deleted false
::db/for-update? true}) ::sql/for-update true})
path (:path media) path (:path media)
mtype (:mtype media) mtype (:mtype media)
@ -312,14 +313,13 @@
(when-let [{:keys [media-id tag]} (db/get* conn :file-tagged-object-thumbnail (when-let [{:keys [media-id tag]} (db/get* conn :file-tagged-object-thumbnail
{:file-id file-id {:file-id file-id
:object-id object-id} :object-id object-id}
{::db/for-update? true})] {::sql/for-update true})]
(sto/touch-object! storage media-id) (sto/touch-object! storage media-id)
(db/update! conn :file-tagged-object-thumbnail (db/update! conn :file-tagged-object-thumbnail
{:deleted-at (dt/now)} {:deleted-at (dt/now)}
{:file-id file-id {:file-id file-id
:object-id object-id :object-id object-id
:tag tag} :tag tag})))
{::db/return-keys? false})))
(s/def ::delete-file-object-thumbnail (s/def ::delete-file-object-thumbnail
(s/keys :req [::rpc/profile-id] (s/keys :req [::rpc/profile-id]
@ -365,8 +365,8 @@
thumb (db/get* conn :file-thumbnail thumb (db/get* conn :file-thumbnail
{:file-id file-id {:file-id file-id
:revn revn} :revn revn}
{::db/remove-deleted? false {::db/remove-deleted false
::db/for-update? true})] ::sql/for-update true})]
(if (some? thumb) (if (some? thumb)
(do (do

3
backend/src/app/rpc/commands/files_update.clj
View file

@ -250,7 +250,8 @@
:features (db/create-array conn "text" (:features file)) :features (db/create-array conn "text" (:features file))
:data (when (take-snapshot? file) :data (when (take-snapshot? file)
(:data file)) (:data file))
:changes (blob/encode changes)}) :changes (blob/encode changes)}
{::db/return-keys false})
(db/update! conn :file (db/update! conn :file
{:revn (:revn file) {:revn (:revn file)

15
backend/src/app/rpc/commands/fonts.clj
View file

@ -11,6 +11,7 @@
[app.common.schema :as sm] [app.common.schema :as sm]
[app.common.uuid :as uuid] [app.common.uuid :as uuid]
[app.db :as db] [app.db :as db]
[app.db.sql :as-alias sql]
[app.loggers.audit :as-alias audit] [app.loggers.audit :as-alias audit]
[app.loggers.webhooks :as-alias webhooks] [app.loggers.webhooks :as-alias webhooks]
[app.media :as media] [app.media :as media]
@ -179,8 +180,7 @@
(db/update! conn :team-font-variant (db/update! conn :team-font-variant
{:font-family name} {:font-family name}
{:font-id id {:font-id id
:team-id team-id} :team-id team-id})
{::db/return-keys? false})
(rph/with-meta (rph/wrap nil) (rph/with-meta (rph/wrap nil)
{::audit/replace-props {:id id {::audit/replace-props {:id id
@ -201,7 +201,6 @@
::webhooks/event? true ::webhooks/event? true
::sm/params schema:delete-font} ::sm/params schema:delete-font}
[cfg {:keys [::rpc/profile-id id team-id]}] [cfg {:keys [::rpc/profile-id id team-id]}]
(db/tx-run! cfg (db/tx-run! cfg
(fn [{:keys [::db/conn ::sto/storage] :as cfg}] (fn [{:keys [::db/conn ::sto/storage] :as cfg}]
(teams/check-edition-permissions! conn profile-id team-id) (teams/check-edition-permissions! conn profile-id team-id)
@ -209,7 +208,7 @@
{:team-id team-id {:team-id team-id
:font-id id :font-id id
:deleted-at nil} :deleted-at nil}
{::db/for-update? true}) {::sql/for-update true})
storage (media/configure-assets-storage storage conn) storage (media/configure-assets-storage storage conn)
tnow (dt/now)] tnow (dt/now)]
@ -220,8 +219,7 @@
(doseq [font fonts] (doseq [font fonts]
(db/update! conn :team-font-variant (db/update! conn :team-font-variant
{:deleted-at tnow} {:deleted-at tnow}
{:id (:id font)} {:id (:id font)})
{::db/return-keys? false})
(some->> (:woff1-file-id font) (sto/touch-object! storage)) (some->> (:woff1-file-id font) (sto/touch-object! storage))
(some->> (:woff2-file-id font) (sto/touch-object! storage)) (some->> (:woff2-file-id font) (sto/touch-object! storage))
(some->> (:ttf-file-id font) (sto/touch-object! storage)) (some->> (:ttf-file-id font) (sto/touch-object! storage))
@ -250,13 +248,12 @@
(teams/check-edition-permissions! conn profile-id team-id) (teams/check-edition-permissions! conn profile-id team-id)
(let [variant (db/get conn :team-font-variant (let [variant (db/get conn :team-font-variant
{:id id :team-id team-id} {:id id :team-id team-id}
{::db/for-update? true}) {::sql/for-update true})
storage (media/configure-assets-storage storage conn)] storage (media/configure-assets-storage storage conn)]
(db/update! conn :team-font-variant (db/update! conn :team-font-variant
{:deleted-at (dt/now)} {:deleted-at (dt/now)}
{:id (:id variant)} {:id (:id variant)})
{::db/return-keys? false})
(some->> (:woff1-file-id variant) (sto/touch-object! storage)) (some->> (:woff1-file-id variant) (sto/touch-object! storage))
(some->> (:woff2-file-id variant) (sto/touch-object! storage)) (some->> (:woff2-file-id variant) (sto/touch-object! storage))

6
backend/src/app/rpc/commands/management.clj
View file

@ -215,7 +215,7 @@
(-> file (-> file
(update :features #(db/create-array conn "text" %)) (update :features #(db/create-array conn "text" %))
(update :data blob/encode)) (update :data blob/encode))
{::db/return-keys? false}) {::db/return-keys false})
;; The file profile creation is optional, so when no profile is ;; The file profile creation is optional, so when no profile is
;; present (when this function is called from profile less ;; present (when this function is called from profile less
@ -231,10 +231,10 @@
{::db/return-keys? false})) {::db/return-keys? false}))
(doseq [params flibs] (doseq [params flibs]
(db/insert! conn :file-library-rel params ::db/return-keys? false)) (db/insert! conn :file-library-rel params ::db/return-keys false))
(doseq [params fmeds] (doseq [params fmeds]
(db/insert! conn :file-media-object params ::db/return-keys? false)) (db/insert! conn :file-media-object params ::db/return-keys false))
file)) file))

3
backend/src/app/rpc/commands/media.clj
View file

@ -157,8 +157,7 @@
(db/update! conn :file (db/update! conn :file
{:modified-at (dt/now) {:modified-at (dt/now)
:has-media-trimmed false} :has-media-trimmed false}
{:id file-id} {:id file-id})
{::db/return-keys? false})
(db/exec-one! conn [sql:create-file-media-object (db/exec-one! conn [sql:create-file-media-object
(or id (uuid/next)) (or id (uuid/next))

12
backend/src/app/rpc/commands/profile.clj
View file

@ -13,6 +13,7 @@
[app.common.uuid :as uuid] [app.common.uuid :as uuid]
[app.config :as cf] [app.config :as cf]
[app.db :as db] [app.db :as db]
[app.db.sql :as-alias sql]
[app.email :as eml] [app.email :as eml]
[app.http.session :as session] [app.http.session :as session]
[app.loggers.audit :as audit] [app.loggers.audit :as audit]
@ -99,7 +100,7 @@
;; NOTE: we need to retrieve the profile independently if we use ;; NOTE: we need to retrieve the profile independently if we use
;; it or not for explicit locking and avoid concurrent updates of ;; it or not for explicit locking and avoid concurrent updates of
;; the same row/object. ;; the same row/object.
(let [profile (-> (db/get-by-id conn :profile profile-id ::db/for-update? true) (let [profile (-> (db/get-by-id conn :profile profile-id ::sql/for-update true)
(decode-row)) (decode-row))
;; Update the profile map with direct params ;; Update the profile map with direct params
@ -164,7 +165,7 @@
(defn- validate-password! (defn- validate-password!
[{:keys [::db/conn] :as cfg} {:keys [profile-id old-password] :as params}] [{:keys [::db/conn] :as cfg} {:keys [profile-id old-password] :as params}]
(let [profile (db/get-by-id conn :profile profile-id ::db/for-update? true)] (let [profile (db/get-by-id conn :profile profile-id ::sql/for-update true)]
(when (and (not= (:password profile) "!") (when (and (not= (:password profile) "!")
(not (:valid (verify-password cfg old-password (:password profile))))) (not (:valid (verify-password cfg old-password (:password profile)))))
(ex/raise :type :validation (ex/raise :type :validation
@ -176,7 +177,8 @@
(when-not (db/read-only? conn) (when-not (db/read-only? conn)
(db/update! conn :profile (db/update! conn :profile
{:password (auth/derive-password password)} {:password (auth/derive-password password)}
{:id id}))) {:id id})
nil))
;; --- MUTATION: Update Photo ;; --- MUTATION: Update Photo
@ -202,7 +204,7 @@
(defn update-profile-photo (defn update-profile-photo
[{:keys [::db/pool ::sto/storage] :as cfg} {:keys [profile-id file] :as params}] [{:keys [::db/pool ::sto/storage] :as cfg} {:keys [profile-id file] :as params}]
(let [photo (upload-photo cfg params) (let [photo (upload-photo cfg params)
profile (db/get-by-id pool :profile profile-id ::db/for-update? true)] profile (db/get-by-id pool :profile profile-id ::sql/for-update true)]
;; Schedule deletion of old photo ;; Schedule deletion of old photo
(when-let [id (:photo-id profile)] (when-let [id (:photo-id profile)]
@ -329,7 +331,7 @@
::sm/params schema:update-profile-props} ::sm/params schema:update-profile-props}
[{:keys [::db/pool]} {:keys [::rpc/profile-id props]}] [{:keys [::db/pool]} {:keys [::rpc/profile-id props]}]
(db/with-atomic [conn pool] (db/with-atomic [conn pool]
(let [profile (get-profile conn profile-id ::db/for-update? true) (let [profile (get-profile conn profile-id ::sql/for-update true)
props (reduce-kv (fn [props k v] props (reduce-kv (fn [props k v]
;; We don't accept namespaced keys ;; We don't accept namespaced keys
(if (simple-ident? k) (if (simple-ident? k)

6
backend/src/app/rpc/commands/projects.clj
View file

@ -9,6 +9,7 @@
[app.common.data.macros :as dm] [app.common.data.macros :as dm]
[app.common.spec :as us] [app.common.spec :as us]
[app.db :as db] [app.db :as db]
[app.db.sql :as-alias sql]
[app.loggers.audit :as-alias audit] [app.loggers.audit :as-alias audit]
[app.loggers.webhooks :as webhooks] [app.loggers.webhooks :as webhooks]
[app.rpc :as-alias rpc] [app.rpc :as-alias rpc]
@ -233,7 +234,7 @@
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id name] :as params}] [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id name] :as params}]
(db/with-atomic [conn pool] (db/with-atomic [conn pool]
(check-edition-permissions! conn profile-id id) (check-edition-permissions! conn profile-id id)
(let [project (db/get-by-id conn :project id ::db/for-update? true)] (let [project (db/get-by-id conn :project id ::sql/for-update true)]
(db/update! conn :project (db/update! conn :project
{:name name} {:name name}
{:id id}) {:id id})
@ -259,7 +260,8 @@
(check-edition-permissions! conn profile-id id) (check-edition-permissions! conn profile-id id)
(let [project (db/update! conn :project (let [project (db/update! conn :project
{:deleted-at (dt/now)} {:deleted-at (dt/now)}
{:id id :is-default false})] {:id id :is-default false}
{::db/return-keys true})]
(rph/with-meta (rph/wrap) (rph/with-meta (rph/wrap)
{::audit/props {:team-id (:team-id project) {::audit/props {:team-id (:team-id project)
:name (:name project) :name (:name project)

3
backend/src/app/rpc/commands/teams.clj
View file

@ -963,5 +963,6 @@
(let [invitation (db/delete! conn :team-invitation (let [invitation (db/delete! conn :team-invitation
{:team-id team-id {:team-id team-id
:email-to (str/lower email)})] :email-to (str/lower email)}
{::db/return-keys true})]
(rph/wrap nil {::audit/props {:invitation-id (:id invitation)}}))))) (rph/wrap nil {::audit/props {:invitation-id (:id invitation)}})))))

3
backend/src/app/rpc/commands/webhooks.clj
View file

@ -95,7 +95,8 @@
:mtype mtype :mtype mtype
:error-code nil :error-code nil
:error-count 0} :error-count 0}
{:id id}) {:id id}
{::db/return-keys true})
(decode-row))) (decode-row)))
(sv/defmethod ::create-webhook (sv/defmethod ::create-webhook

13
backend/src/app/srepl/cli.clj
View file

@ -65,9 +65,8 @@
(let [res (db/update! conn :profile (let [res (db/update! conn :profile
params params
{:email email {:email email
:deleted-at nil} :deleted-at nil})]
{::db/return-keys? false})] (pos? (db/get-update-count res))))))))
(pos? (:next.jdbc/update-count res))))))))
(defmethod exec-command :delete-profile (defmethod exec-command :delete-profile
[{:keys [email soft]}] [{:keys [email soft]}]
@ -82,12 +81,10 @@
(let [res (if soft (let [res (if soft
(db/update! conn :profile (db/update! conn :profile
{:deleted-at (dt/now)} {:deleted-at (dt/now)}
{:email email :deleted-at nil} {:email email :deleted-at nil})
{::db/return-keys? false})
(db/delete! conn :profile (db/delete! conn :profile
{:email email} {:email email}))]
{::db/return-keys? false}))] (pos? (db/get-update-count res))))))
(pos? (:next.jdbc/update-count res))))))
(defmethod exec-command :search-profile (defmethod exec-command :search-profile
[{:keys [email]}] [{:keys [email]}]

369
backend/src/app/srepl/components_v2.clj
View file

@ -6,8 +6,6 @@
(ns app.srepl.components-v2 (ns app.srepl.components-v2
(:require (:require
[app.common.data :as d]
[app.common.data.macros :as dm]
[app.common.logging :as l] [app.common.logging :as l]
[app.common.pprint :as pp] [app.common.pprint :as pp]
[app.db :as db] [app.db :as db]
@ -19,6 +17,13 @@
[promesa.exec.semaphore :as ps] [promesa.exec.semaphore :as ps]
[promesa.util :as pu])) [promesa.util :as pu]))
(def ^:dynamic *scope* nil)
(def ^:dynamic *semaphore* nil)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; PRIVATE HELPERS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn- print-stats! (defn- print-stats!
[stats] [stats]
(->> stats (->> stats
@ -87,210 +92,228 @@
res (db/exec-one! pool [sql])] res (db/exec-one! pool [sql])]
(:count res))) (:count res)))
(defn migrate-file!
[system file-id & {:keys [rollback?] :or {rollback? true}}]
(l/dbg :hint "migrate:start") (defn- mark-team-migration!
(let [tpoint (dt/tpoint)] [{:keys [::db/pool]} team-id]
(try ;; We execute this out of transaction because we want this
(binding [feat/*stats* (atom {})] ;; change to be visible to all other sessions before starting
;; the migration
(let [sql (str "UPDATE team SET features = "
" array_append(features, 'ephimeral/v2-migration') "
" WHERE id = ?")]
(db/exec-one! pool [sql team-id])))
(defn- unmark-team-migration!
[{:keys [::db/pool]} team-id]
;; We execute this out of transaction because we want this
;; change to be visible to all other sessions before starting
;; the migration
(let [sql (str "UPDATE team SET features = "
" array_remove(features, 'ephimeral/v2-migration') "
" WHERE id = ?")]
(db/exec-one! pool [sql team-id])))
(def ^:private sql:get-teams
"SELECT id, features
FROM team
WHERE deleted_at IS NULL
ORDER BY created_at ASC")
(defn- get-teams
[conn]
(->> (db/cursor conn sql:get-teams)
(map feat/decode-row)))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; PUBLIC API
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn migrate-file!
[system file-id & {:keys [rollback? max-procs]
:or {rollback? true}}]
(l/dbg :hint "migrate:start" :rollback rollback?)
(let [tpoint (dt/tpoint)
file-id (if (string? file-id)
(parse-uuid file-id)
file-id)]
(binding [feat/*stats* (atom {})]
(try
(-> (assoc system ::db/rollback rollback?) (-> (assoc system ::db/rollback rollback?)
(feat/migrate-file! file-id)) (feat/migrate-file! file-id :max-procs max-procs))
(-> (deref feat/*stats*) (-> (deref feat/*stats*)
(assoc :elapsed (dt/format-duration (tpoint))))) (assoc :elapsed (dt/format-duration (tpoint))))
(catch Throwable cause (catch Throwable cause
(l/wrn :hint "migrate:error" :cause cause)) (l/wrn :hint "migrate:error" :cause cause))
(finally (finally
(let [elapsed (dt/format-duration (tpoint))] (let [elapsed (dt/format-duration (tpoint))]
(l/dbg :hint "migrate:end" :elapsed elapsed)))))) (l/dbg :hint "migrate:end" :rollback rollback? :elapsed elapsed)))))))
(defn migrate-files!
[{:keys [::db/pool] :as system}
& {:keys [chunk-size max-jobs max-items start-at preset rollback? skip-on-error validate?]
:or {chunk-size 10
skip-on-error true
max-jobs 10
max-items Long/MAX_VALUE
preset :shutdown-on-failure
rollback? true
validate? false}}]
(letfn [(get-chunk [cursor]
(let [sql (str/concat
"SELECT id, created_at FROM file "
" WHERE created_at < ? AND deleted_at IS NULL "
" ORDER BY created_at desc LIMIT ?")
rows (db/exec! pool [sql cursor chunk-size])]
[(some->> rows peek :created-at) (seq rows)]))
(get-candidates []
(->> (d/iteration get-chunk
:vf second
:kf first
:initk (or start-at (dt/now)))
(take max-items)
(map :id)))]
(l/dbg :hint "migrate:start")
(let [fsem (ps/create :permits max-jobs)
total (get-total-files pool)
stats (atom {:files/total total})
tpoint (dt/tpoint)]
(add-watch stats :progress-report (report-progress-files tpoint))
(binding [feat/*stats* stats
feat/*semaphore* fsem
feat/*skip-on-error* skip-on-error]
(try
(pu/with-open [scope (px/structured-task-scope :preset preset :factory :virtual)]
(run! (fn [file-id]
(ps/acquire! feat/*semaphore*)
(px/submit! scope (fn []
(-> (assoc system ::db/rollback rollback?)
(feat/migrate-file! file-id
:validate? validate?
:throw-on-validate? (not skip-on-error))))))
(get-candidates))
(p/await! scope))
(-> (deref feat/*stats*)
(assoc :elapsed (dt/format-duration (tpoint))))
(catch Throwable cause
(l/dbg :hint "migrate:error" :cause cause))
(finally
(let [elapsed (dt/format-duration (tpoint))]
(l/dbg :hint "migrate:end" :elapsed elapsed))))))))
(defn migrate-team! (defn migrate-team!
[{:keys [::db/pool] :as system} team-id [{:keys [::db/pool] :as system} team-id & {:keys [rollback? skip-on-error validate? max-procs]
& {:keys [rollback? skip-on-error validate?] :or {rollback? true
:or {rollback? true skip-on-error true validate? false}}] skip-on-error true
(l/dbg :hint "migrate:start") validate? false
max-procs 1 }
:as opts}]
(let [total (get-total-files pool :team-id team-id) (l/dbg :hint "migrate:start" :rollback rollback?)
stats (atom {:total/files total})
tpoint (dt/tpoint)] (let [team-id (if (string? team-id)
(parse-uuid team-id)
team-id)
total (get-total-files pool :team-id team-id)
stats (atom {:total/files total})
tpoint (dt/tpoint)]
(add-watch stats :progress-report (report-progress-files tpoint)) (add-watch stats :progress-report (report-progress-files tpoint))
(try (binding [feat/*stats* stats
(binding [feat/*stats* stats feat/*skip-on-error* skip-on-error]
feat/*skip-on-error* skip-on-error]
(try
(mark-team-migration! system team-id)
(-> (assoc system ::db/rollback rollback?) (-> (assoc system ::db/rollback rollback?)
(feat/migrate-team! team-id (feat/migrate-team! team-id
:max-procs max-procs
:validate? validate? :validate? validate?
:throw-on-validate? (not skip-on-error))) :throw-on-validate? (not skip-on-error)))
(print-stats! (print-stats!
(-> (deref feat/*stats*) (-> (deref feat/*stats*)
(dissoc :total/files) (dissoc :total/files)
(assoc :elapsed (dt/format-duration (tpoint)))))) (assoc :elapsed (dt/format-duration (tpoint)))))
(catch Throwable cause (catch Throwable cause
(l/dbg :hint "migrate:error" :cause cause)) (l/dbg :hint "migrate:error" :cause cause))
(finally (finally
(let [elapsed (dt/format-duration (tpoint))] (unmark-team-migration! system team-id)
(l/dbg :hint "migrate:end" :elapsed elapsed))))))
(defn default-on-end (let [elapsed (dt/format-duration (tpoint))]
[stats] (l/dbg :hint "migrate:end" :rollback rollback? :elapsed elapsed)))))))
(print-stats!
(-> stats
(update :elapsed/total dt/format-duration)
(dissoc :total/teams))))
(defn migrate-teams! (defn migrate-teams!
[{:keys [::db/pool] :as system} "A REPL helper for migrate all teams.
& {:keys [chunk-size max-jobs max-items start-at
rollback? validate? preset skip-on-error
max-time on-start on-progress on-error on-end]
:or {chunk-size 10000
validate? false
rollback? true
skip-on-error true
on-end default-on-end
preset :shutdown-on-failure
max-jobs Integer/MAX_VALUE
max-items Long/MAX_VALUE}}]
(letfn [(get-chunk [cursor] This function starts multiple concurrent team migration processes
(let [sql (str/concat until thw maximum number of jobs is reached which by default has the
"SELECT id, created_at, features FROM team " value of `1`. This is controled with the `:max-jobs` option.
" WHERE created_at < ? AND deleted_at IS NULL "
" ORDER BY created_at desc LIMIT ?")
rows (db/exec! pool [sql cursor chunk-size])]
[(some->> rows peek :created-at) (seq rows)]))
(get-candidates [] Each tram migration process also can start multiple procs for
(->> (d/iteration get-chunk graphics migration, the total of that procs is controled with the
:vf second `:max-procs` option.
:kf first
:initk (or start-at (dt/now)))
(map #(update % :features db/decode-pgarray #{}))
(remove #(contains? (:features %) "ephimeral/v2-migration"))
(take max-items)
(map :id)))
(migrate-team [team-id] Internally, the graphics migration process uses SVGO module which by
(try default has a limited number of maximum concurent
(-> (assoc system ::db/rollback rollback?) operations (globally), ensure setting up correct number with
(feat/migrate-team! team-id PENPOT_SVGO_MAX_PROCS environment variable."
:validate? validate?
:throw-on-validate? (not skip-on-error)))
(catch Throwable cause
(l/err :hint "unexpected error on processing team" :team-id (dm/str team-id) :cause cause))))
(process-team [scope tpoint mtime team-id] [{:keys [::db/pool] :as system} & {:keys [max-jobs max-procs max-items
(ps/acquire! feat/*semaphore*) rollback? validate? preset
(let [ts (tpoint)] skip-on-error max-time
(if (and mtime (neg? (compare mtime ts))) on-start on-progress on-error on-end]
(l/inf :hint "max time constraint reached" :elapsed (dt/format-duration ts)) :or {validate? false
(px/submit! scope (partial migrate-team team-id)))))] rollback? true
skip-on-error true
preset :shutdown-on-failure
max-jobs 1
max-procs 10
max-items Long/MAX_VALUE}
:as opts}]
(l/dbg :hint "migrate:start") (let [total (get-total-teams pool)
stats (atom {:total/teams (min total max-items)})
(let [sem (ps/create :permits max-jobs) tpoint (dt/tpoint)
total (get-total-teams pool) mtime (some-> max-time dt/duration)
stats (atom {:total/teams (min total max-items)})
tpoint (dt/tpoint)
mtime (some-> max-time dt/duration)]
(when (fn? on-start) scope (px/structured-task-scope :preset preset :factory :virtual)
(on-start {:total total :rollback rollback?})) sjobs (ps/create :permits max-jobs)
(add-watch stats :progress-report (report-progress-teams tpoint on-progress)) migrate-team
(fn [{:keys [id features] :as team}]
(ps/acquire! sjobs)
(let [ts (tpoint)]
(cond
(and mtime (neg? (compare mtime ts)))
(do
(l/inf :hint "max time constraint reached"
:team-id (str id)
:elapsed (dt/format-duration ts))
(ps/release! sjobs)
(reduced nil))
(binding [feat/*stats* stats (or (contains? features "ephimeral/v2-migration")
feat/*semaphore* sem (contains? features "components/v2"))
feat/*skip-on-error* skip-on-error] (do
(l/dbg :hint "skip team" :team-id (str id))
(ps/release! sjobs))
:else
(px/submit! scope (fn []
(try
(mark-team-migration! system id)
(-> (assoc system ::db/rollback rollback?)
(feat/migrate-team! id
:max-procs max-procs
:validate? validate?
:throw-on-validate? (not skip-on-error)))
(catch Throwable cause
(l/err :hint "unexpected error on processing team"
:team-id (str id)
:cause cause))
(finally
(ps/release! sjobs)
(unmark-team-migration! system id))))))))]
(l/dbg :hint "migrate:start"
:rollback rollback?
:total total
:max-jobs max-jobs
:max-procs max-procs
:max-items max-items)
(add-watch stats :progress-report (report-progress-teams tpoint on-progress))
(binding [feat/*stats* stats
feat/*skip-on-error* skip-on-error]
(try
(when (fn? on-start)
(on-start {:total total :rollback rollback?}))
(db/tx-run! system
(fn [{:keys [::db/conn]}]
(run! (partial migrate-team)
(->> (get-teams conn)
(take max-items)))))
(try (try
(pu/with-open [scope (px/structured-task-scope :preset preset (p/await! scope)
:factory :virtual)]
(loop [candidates (get-candidates)]
(when-let [team-id (first candidates)]
(when (process-team scope tpoint mtime team-id)
(recur (rest candidates)))))
(p/await! scope))
(when (fn? on-end)
(-> (deref stats)
(assoc :elapsed/total (tpoint))
(on-end)))
(catch Throwable cause
(l/dbg :hint "migrate:error" :cause cause)
(when (fn? on-error)
(on-error cause)))
(finally (finally
(let [elapsed (dt/format-duration (tpoint))] (pu/close! scope)))
(l/dbg :hint "migrate:end" :elapsed elapsed))))))))
(if (fn? on-end)
(-> (deref stats)
(assoc :elapsed/total (tpoint))
(on-end))
(-> (deref stats)
(assoc :elapsed/total (tpoint))
(update :elapsed/total dt/format-duration)
(dissoc :total/teams)
(print-stats!)))
(catch Throwable cause
(l/dbg :hint "migrate:error" :cause cause)
(when (fn? on-error)
(on-error cause)))
(finally
(let [elapsed (dt/format-duration (tpoint))]
(l/dbg :hint "migrate:end"
:rollback rollback?
:elapsed elapsed)))))))

6
backend/src/app/storage.clj
View file

@ -170,8 +170,7 @@
(let [id (if (impl/object? object-or-id) (:id object-or-id) object-or-id) (let [id (if (impl/object? object-or-id) (:id object-or-id) object-or-id)
rs (db/update! pool-or-conn :storage-object rs (db/update! pool-or-conn :storage-object
{:touched-at (dt/now)} {:touched-at (dt/now)}
{:id id} {:id id})]
{::db/return-keys? false})]
(pos? (db/get-update-count rs)))) (pos? (db/get-update-count rs))))
(defn get-object-data (defn get-object-data
@ -220,8 +219,7 @@
(let [id (if (impl/object? object-or-id) (:id object-or-id) object-or-id) (let [id (if (impl/object? object-or-id) (:id object-or-id) object-or-id)
res (db/update! pool-or-conn :storage-object res (db/update! pool-or-conn :storage-object
{:deleted-at (dt/now)} {:deleted-at (dt/now)}
{:id id} {:id id})]
{::db/return-keys? false})]
(pos? (db/get-update-count res)))) (pos? (db/get-update-count res))))
(dm/export impl/resolve-backend) (dm/export impl/resolve-backend)

5
backend/src/app/storage/gc_deleted.clj
View file

@ -47,8 +47,7 @@
[conn ids] [conn ids]
(let [ids (db/create-array conn "uuid" ids)] (let [ids (db/create-array conn "uuid" ids)]
(-> (db/exec-one! conn [sql:delete-sobjects ids]) (-> (db/exec-one! conn [sql:delete-sobjects ids])
(db/get-update-count)))) (db/get-update-count))))
(defn- delete-in-bulk! (defn- delete-in-bulk!
[cfg backend-id ids] [cfg backend-id ids]
@ -60,7 +59,6 @@
(fn [{:keys [::db/conn ::sto/storage]}] (fn [{:keys [::db/conn ::sto/storage]}]
(when-let [ids (lock-ids conn ids)] (when-let [ids (lock-ids conn ids)]
(let [total (delete-sobjects! conn ids)] (let [total (delete-sobjects! conn ids)]
(-> (impl/resolve-backend storage backend-id) (-> (impl/resolve-backend storage backend-id)
(impl/del-objects-in-bulk ids)) (impl/del-objects-in-bulk ids))
@ -68,7 +66,6 @@
(l/dbg :hint "permanently delete storage object" (l/dbg :hint "permanently delete storage object"
:id (str id) :id (str id)
:backend (name backend-id))) :backend (name backend-id)))
total)))) total))))
(catch Throwable cause (catch Throwable cause
(l/err :hint "unexpected error on bulk deletion" (l/err :hint "unexpected error on bulk deletion"

49
backend/src/app/storage/s3.clj
View file

@ -39,6 +39,7 @@
software.amazon.awssdk.core.async.AsyncRequestBody software.amazon.awssdk.core.async.AsyncRequestBody
software.amazon.awssdk.core.async.AsyncResponseTransformer software.amazon.awssdk.core.async.AsyncResponseTransformer
software.amazon.awssdk.core.client.config.ClientAsyncConfiguration software.amazon.awssdk.core.client.config.ClientAsyncConfiguration
software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption
software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient
software.amazon.awssdk.http.nio.netty.SdkEventLoopGroup software.amazon.awssdk.http.nio.netty.SdkEventLoopGroup
software.amazon.awssdk.regions.Region software.amazon.awssdk.regions.Region
@ -169,32 +170,34 @@
(defn- build-s3-client (defn- build-s3-client
[{:keys [::region ::endpoint ::io-threads]}] [{:keys [::region ::endpoint ::io-threads]}]
(let [aconfig (-> (ClientAsyncConfiguration/builder) (let [executor (px/resolve-executor :virtual)
(.build)) aconfig (-> (ClientAsyncConfiguration/builder)
(.advancedOption SdkAdvancedAsyncClientOption/FUTURE_COMPLETION_EXECUTOR executor)
(.build))
sconfig (-> (S3Configuration/builder) sconfig (-> (S3Configuration/builder)
(cond-> (some? endpoint) (.pathStyleAccessEnabled true)) (cond-> (some? endpoint) (.pathStyleAccessEnabled true))
(.build)) (.build))
thr-num (or io-threads (min 16 (px/get-available-processors))) thr-num (or io-threads (min 16 (px/get-available-processors)))
hclient (-> (NettyNioAsyncHttpClient/builder) hclient (-> (NettyNioAsyncHttpClient/builder)
(.eventLoopGroupBuilder (-> (SdkEventLoopGroup/builder) (.eventLoopGroupBuilder (-> (SdkEventLoopGroup/builder)
(.numberOfThreads (int thr-num)))) (.numberOfThreads (int thr-num))))
(.connectionAcquisitionTimeout default-timeout) (.connectionAcquisitionTimeout default-timeout)
(.connectionTimeout default-timeout) (.connectionTimeout default-timeout)
(.readTimeout default-timeout) (.readTimeout default-timeout)
(.writeTimeout default-timeout) (.writeTimeout default-timeout)
(.build)) (.build))
client (let [builder (S3AsyncClient/builder) client (let [builder (S3AsyncClient/builder)
builder (.serviceConfiguration ^S3AsyncClientBuilder builder ^S3Configuration sconfig) builder (.serviceConfiguration ^S3AsyncClientBuilder builder ^S3Configuration sconfig)
builder (.asyncConfiguration ^S3AsyncClientBuilder builder ^ClientAsyncConfiguration aconfig) builder (.asyncConfiguration ^S3AsyncClientBuilder builder ^ClientAsyncConfiguration aconfig)
builder (.httpClient ^S3AsyncClientBuilder builder ^NettyNioAsyncHttpClient hclient) builder (.httpClient ^S3AsyncClientBuilder builder ^NettyNioAsyncHttpClient hclient)
builder (.region ^S3AsyncClientBuilder builder (lookup-region region)) builder (.region ^S3AsyncClientBuilder builder (lookup-region region))
builder (cond-> ^S3AsyncClientBuilder builder builder (cond-> ^S3AsyncClientBuilder builder
(some? endpoint) (some? endpoint)
(.endpointOverride (URI. endpoint)))] (.endpointOverride (URI. endpoint)))]
(.build ^S3AsyncClientBuilder builder))] (.build ^S3AsyncClientBuilder builder))]
(reify (reify
clojure.lang.IDeref clojure.lang.IDeref

65
backend/src/app/svgo.clj Normal file
View file

@ -0,0 +1,65 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC
(ns app.svgo
"A SVG Optimizer service"
(:require
[app.common.data :as d]
[app.common.data.macros :as dm]
[app.common.jsrt :as jsrt]
[app.common.logging :as l]
[app.common.spec :as us]
[app.worker :as-alias wrk]
[clojure.spec.alpha :as s]
[integrant.core :as ig]
[promesa.exec :as px]
[promesa.exec.bulkhead :as bh]
[promesa.exec.semaphore :as ps]
[promesa.util :as pu]))
(def ^:dynamic *semaphore*
"A dynamic variable that can optionally contain a traffic light to
appropriately delimit the use of resources, managed externally."
nil)
(defn optimize
[system data]
(dm/assert! "expect data to be a string" (string? data))
(letfn [(optimize-fn [pool]
(jsrt/run! pool
(fn [context]
(jsrt/set! context "svgData" data)
(jsrt/eval! context "penpotSvgo.optimize(svgData, {plugins: ['safeAndFastPreset']})"))))]
(try
(some-> *semaphore* ps/acquire!)
(let [{:keys [::jsrt/pool ::wrk/executor]} (::optimizer system)]
(dm/assert! "expect optimizer instance" (jsrt/pool? pool))
(px/invoke! executor (partial optimize-fn pool)))
(finally
(some-> *semaphore* ps/release!)))))
(s/def ::max-procs (s/nilable ::us/integer))
(defmethod ig/pre-init-spec ::optimizer [_]
(s/keys :req [::wrk/executor ::max-procs]))
(defmethod ig/prep-key ::optimizer
[_ cfg]
(merge {::max-procs 20} (d/without-nils cfg)))
(defmethod ig/init-key ::optimizer
[_ {:keys [::wrk/executor ::max-procs]}]
(l/inf :hint "initializing svg optimizer pool" :max-procs max-procs)
(let [init (jsrt/resource->source "app/common/svg/optimizer.js")
executor (bh/create :type :executor :executor executor :permits max-procs)]
{::jsrt/pool (jsrt/pool :init init)
::wrk/executor executor}))
(defmethod ig/halt-key! ::optimizer
[_ {:keys [::jsrt/pool]}]
(l/info :hint "stopping svg optimizer pool")
(pu/close! pool))

6
backend/src/app/tasks/file_gc.clj
View file

@ -249,8 +249,7 @@
(blob/encode))] (blob/encode))]
(db/update! conn :file (db/update! conn :file
{:data data} {:data data}
{:id file-id} {:id file-id}))
{::db/return-keys? false}))
(count unused)))) (count unused))))
@ -315,7 +314,6 @@
;; Mark file as trimmed ;; Mark file as trimmed
(db/update! conn :file (db/update! conn :file
{:has-media-trimmed true} {:has-media-trimmed true}
{:id id} {:id id})
{::db/return-keys? false})
(feat.fdata/persist-pointers! cfg id)))) (feat.fdata/persist-pointers! cfg id))))

40
backend/src/app/tasks/objects_gc.clj
View file

@ -89,9 +89,7 @@
;; CASCADE database triggers. This may leave orphan ;; CASCADE database triggers. This may leave orphan
;; teams, but there is a special task for deleting ;; teams, but there is a special task for deleting
;; orphaned teams. ;; orphaned teams.
(db/delete! conn :profile (db/delete! conn :profile {:id id})
{:id id}
{::db/return-keys? false})
(inc total)) (inc total))
0))) 0)))
@ -118,20 +116,16 @@
(some->> photo-id (sto/touch-object! storage)) (some->> photo-id (sto/touch-object! storage))
;; And finally, permanently delete the team. ;; And finally, permanently delete the team.
(db/delete! conn :team (db/delete! conn :team {:id id})
{:id id}
{::db/return-keys? false})
;; Mark for deletion in cascade ;; Mark for deletion in cascade
(db/update! conn :team-font-variant (db/update! conn :team-font-variant
{:deleted-at deleted-at} {:deleted-at deleted-at}
{:team-id id} {:team-id id})
{::db/return-keys? false})
(db/update! conn :project (db/update! conn :project
{:deleted-at deleted-at} {:deleted-at deleted-at}
{:team-id id} {:team-id id})
{::db/return-keys? false})
(inc total)) (inc total))
0))) 0)))
@ -162,9 +156,7 @@
(some->> (:ttf-file-id font) (sto/touch-object! storage)) (some->> (:ttf-file-id font) (sto/touch-object! storage))
;; And finally, permanently delete the team font variant ;; And finally, permanently delete the team font variant
(db/delete! conn :team-font-variant (db/delete! conn :team-font-variant {:id id})
{:id id}
{::db/return-keys? false})
(inc total)) (inc total))
0))) 0)))
@ -187,16 +179,14 @@
:id (str id) :id (str id)
:team-id (str team-id) :team-id (str team-id)
:deleted-at (dt/format-instant deleted-at)) :deleted-at (dt/format-instant deleted-at))
;; And finally, permanently delete the project. ;; And finally, permanently delete the project.
(db/delete! conn :project (db/delete! conn :project {:id id})
{:id id}
{::db/return-keys? false})
;; Mark files to be deleted ;; Mark files to be deleted
(db/update! conn :file (db/update! conn :file
{:deleted-at deleted-at} {:deleted-at deleted-at}
{:project-id id} {:project-id id})
{::db/return-keys? false})
(inc total)) (inc total))
0))) 0)))
@ -221,25 +211,21 @@
:deleted-at (dt/format-instant deleted-at)) :deleted-at (dt/format-instant deleted-at))
;; And finally, permanently delete the file. ;; And finally, permanently delete the file.
(db/delete! conn :file (db/delete! conn :file {:id id})
{:id id}
{::db/return-keys? false})
;; Mark file media objects to be deleted ;; Mark file media objects to be deleted
(db/update! conn :file-media-object (db/update! conn :file-media-object
{:deleted-at deleted-at} {:deleted-at deleted-at}
{:file-id id} {:file-id id})
{::db/return-keys? false})
;; Mark thumbnails to be deleted ;; Mark thumbnails to be deleted
(db/update! conn :file-thumbnail (db/update! conn :file-thumbnail
{:deleted-at deleted-at} {:deleted-at deleted-at}
{:file-id id} {:file-id id})
{::db/return-keys? false})
(db/update! conn :file-tagged-object-thumbnail (db/update! conn :file-tagged-object-thumbnail
{:deleted-at deleted-at} {:deleted-at deleted-at}
{:file-id id} {:file-id id})
{::db/return-keys? false})
(inc total)) (inc total))
0))) 0)))

3
backend/src/app/tasks/orphan_teams_gc.clj
View file

@ -54,7 +54,6 @@
(l/trc :hint "mark orphan team for deletion" :id (str team-id)) (l/trc :hint "mark orphan team for deletion" :id (str team-id))
(db/update! conn :team (db/update! conn :team
{:deleted-at (dt/now)} {:deleted-at (dt/now)}
{:id team-id} {:id team-id})
{::db/return-keys? false})
(inc total)) (inc total))
0))) 0)))

4
backend/src/app/worker.clj
View file

@ -42,8 +42,8 @@
(defmethod ig/init-key ::executor (defmethod ig/init-key ::executor
[_ _] [_ _]
(let [factory (px/thread-factory :prefix "penpot/default/") (let [factory (px/thread-factory :prefix "penpot/default/")
executor (px/cached-executor :factory factory :keepalive 30000)] executor (px/cached-executor :factory factory :keepalive 60000)]
(l/inf :hint "starting executor") (l/inf :hint "starting executor")
(reify (reify
java.lang.AutoCloseable java.lang.AutoCloseable

9
backend/test/backend_tests/rpc_file_thumbnails_test.clj
View file

@ -140,7 +140,7 @@
(t/is (= 0 (:freeze res)))) (t/is (= 0 (:freeze res))))
;; check that storage object is still exists but is marked as deleted ;; check that storage object is still exists but is marked as deleted
(let [row (th/db-get :storage-object {:id (:media-id row1)} {::db/remove-deleted? false})] (let [row (th/db-get :storage-object {:id (:media-id row1)} {::db/remove-deleted false})]
(t/is (some? (:deleted-at row)))) (t/is (some? (:deleted-at row))))
;; Run the storage gc deleted task, it should permanently delete ;; Run the storage gc deleted task, it should permanently delete
@ -152,7 +152,7 @@
(t/is (some? (sto/get-object storage (:media-id row2)))) (t/is (some? (sto/get-object storage (:media-id row2))))
;; check that storage object is still exists but is marked as deleted ;; check that storage object is still exists but is marked as deleted
(let [row (th/db-get :storage-object {:id (:media-id row1)} {::db/remove-deleted? false})] (let [row (th/db-get :storage-object {:id (:media-id row1)} {::db/remove-deleted false})]
(t/is (nil? row)))))) (t/is (nil? row))))))
(t/deftest create-file-thumbnail (t/deftest create-file-thumbnail
@ -240,7 +240,7 @@
(t/is (nil? (sto/get-object storage (:media-id row1)))) (t/is (nil? (sto/get-object storage (:media-id row1))))
(t/is (some? (sto/get-object storage (:media-id row2)))) (t/is (some? (sto/get-object storage (:media-id row2))))
(let [row (th/db-get :storage-object {:id (:media-id row1)} {::db/remove-deleted? false})] (let [row (th/db-get :storage-object {:id (:media-id row1)} {::db/remove-deleted false})]
(t/is (some? (:deleted-at row)))) (t/is (some? (:deleted-at row))))
;; Run the storage gc deleted task, it should permanently delete ;; Run the storage gc deleted task, it should permanently delete
@ -320,6 +320,3 @@
(let [result (:result out)] (let [result (:result out)]
(t/is (contains? result "test-key-2")))))) (t/is (contains? result "test-key-2"))))))

4
backend/test/backend_tests/rpc_profile_test.clj
View file

@ -149,7 +149,7 @@
(let [row (th/db-get :team (let [row (th/db-get :team
{:id (:default-team-id prof)} {:id (:default-team-id prof)}
{::db/remove-deleted? false})] {::db/remove-deleted false})]
(t/is (nil? (:deleted-at row)))) (t/is (nil? (:deleted-at row))))
(let [result (th/run-task! :orphan-teams-gc {:min-age 0})] (let [result (th/run-task! :orphan-teams-gc {:min-age 0})]
@ -157,7 +157,7 @@
(let [row (th/db-get :team (let [row (th/db-get :team
{:id (:default-team-id prof)} {:id (:default-team-id prof)}
{::db/remove-deleted? false})] {::db/remove-deleted false})]
(t/is (dt/instant? (:deleted-at row)))) (t/is (dt/instant? (:deleted-at row))))
;; query profile after delete ;; query profile after delete

36
common/src/app/common/svg.cljc
View file

@ -9,9 +9,6 @@
#?(:cljs ["./svg/optimizer.js" :as svgo]) #?(:cljs ["./svg/optimizer.js" :as svgo])
#?(:clj [clojure.xml :as xml] #?(:clj [clojure.xml :as xml]
:cljs [tubax.core :as tubax]) :cljs [tubax.core :as tubax])
#?(:clj [integrant.core :as ig])
#?(:clj [app.common.jsrt :as jsrt])
#?(:clj [app.common.logging :as l])
[app.common.data :as d] [app.common.data :as d]
[app.common.data.macros :as dm] [app.common.data.macros :as dm]
[app.common.geom.matrix :as gmt] [app.common.geom.matrix :as gmt]
@ -1053,21 +1050,6 @@
:height (d/parse-integer (:height attrs) 0)})))] :height (d/parse-integer (:height attrs) 0)})))]
(reduce-nodes redfn [] svg-data ))) (reduce-nodes redfn [] svg-data )))
#?(:cljs
(defn optimize
([input] (optimize input nil))
([input options]
(svgo/optimize input (clj->js options))))
:clj
(defn optimize
[pool data]
(dm/assert! "expected a valid pool" (jsrt/pool? pool))
(dm/assert! "expect data to be a string" (string? data))
(jsrt/run! pool
(fn [context]
(jsrt/set! context "svgData" data)
(jsrt/eval! context "penpotSvgo.optimize(svgData, {})")))))
#?(:clj #?(:clj
(defn- secure-parser-factory (defn- secure-parser-factory
[^InputStream input ^XMLHandler handler] [^InputStream input ^XMLHandler handler]
@ -1091,15 +1073,9 @@
(dm/with-open [istream (IOUtils/toInputStream text "UTF-8")] (dm/with-open [istream (IOUtils/toInputStream text "UTF-8")]
(xml/parse istream secure-parser-factory))))) (xml/parse istream secure-parser-factory)))))
#?(:clj ;; FIXME pass correct plugin set
(defmethod ig/init-key ::optimizer #?(:cljs
[_ _] (defn optimize
(l/info :hint "initializing svg optimizer pool") ([input] (optimize input nil))
(let [init (jsrt/resource->source "app/common/svg/optimizer.js")] ([input options]
(jsrt/pool :init init)))) (svgo/optimize input (clj->js options)))))
#?(:clj
(defmethod ig/halt-key! ::optimizer
[_ pool]
(l/info :hint "stopping svg optimizer pool")
(.close ^java.lang.AutoCloseable pool)))

148
common/src/app/common/svg/optimizer.js
View file

@ -29431,12 +29431,13 @@ const optimize = (input, config) => {
exports.optimize = optimize; exports.optimize = optimize;
},{"./svgo/parser.js":322,"./svgo/plugins.js":324,"./svgo/stringifier.js":381,"./svgo/tools.js":383,"lodash/isPlainObject":308}],320:[function(require,module,exports){ },{"./svgo/parser.js":322,"./svgo/plugins.js":324,"./svgo/stringifier.js":382,"./svgo/tools.js":384,"lodash/isPlainObject":308}],320:[function(require,module,exports){
'use strict'; 'use strict';
exports.builtin = [ exports.builtin = [
require('./plugins/default.js'), require('./plugins/default.js'),
require('./plugins/safe.js'), require('./plugins/safe.js'),
require('./plugins/safeAndFast.js'),
require('./plugins/addAttributesToSVGElement.js'), require('./plugins/addAttributesToSVGElement.js'),
require('./plugins/addClassesToSVGElement.js'), require('./plugins/addClassesToSVGElement.js'),
require('./plugins/cleanupAttrs.js'), require('./plugins/cleanupAttrs.js'),
@ -29489,7 +29490,7 @@ exports.builtin = [
require('./plugins/sortDefsChildren.js'), require('./plugins/sortDefsChildren.js'),
]; ];
},{"./plugins/addAttributesToSVGElement.js":328,"./plugins/addClassesToSVGElement.js":329,"./plugins/cleanupAttrs.js":331,"./plugins/cleanupEnableBackground.js":332,"./plugins/cleanupIds.js":333,"./plugins/cleanupListOfValues.js":334,"./plugins/cleanupNumericValues.js":335,"./plugins/collapseGroups.js":336,"./plugins/convertColors.js":337,"./plugins/convertEllipseToCircle.js":338,"./plugins/convertPathData.js":339,"./plugins/convertShapeToPath.js":340,"./plugins/convertStyleToAttrs.js":341,"./plugins/convertTransform.js":342,"./plugins/default.js":343,"./plugins/inlineStyles.js":344,"./plugins/mergePaths.js":345,"./plugins/mergeStyles.js":346,"./plugins/minifyStyles.js":347,"./plugins/moveElemsAttrsToGroup.js":348,"./plugins/moveGroupAttrsToElems.js":349,"./plugins/prefixIds.js":350,"./plugins/removeAttributesBySelector.js":351,"./plugins/removeAttrs.js":352,"./plugins/removeComments.js":353,"./plugins/removeDesc.js":354,"./plugins/removeDimensions.js":355,"./plugins/removeDoctype.js":356,"./plugins/removeEditorsNSData.js":357,"./plugins/removeElementsByAttr.js":358,"./plugins/removeEmptyAttrs.js":359,"./plugins/removeEmptyContainers.js":360,"./plugins/removeEmptyText.js":361,"./plugins/removeHiddenElems.js":362,"./plugins/removeMetadata.js":363,"./plugins/removeNonInheritableGroupAttrs.js":364,"./plugins/removeOffCanvasPaths.js":365,"./plugins/removeRasterImages.js":366,"./plugins/removeScriptElement.js":367,"./plugins/removeStyleElement.js":368,"./plugins/removeTitle.js":369,"./plugins/removeUnknownsAndDefaults.js":370,"./plugins/removeUnusedNS.js":371,"./plugins/removeUselessDefs.js":372,"./plugins/removeUselessStrokeAndFill.js":373,"./plugins/removeViewBox.js":374,"./plugins/removeXMLNS.js":375,"./plugins/removeXMLProcInst.js":376,"./plugins/reusePaths.js":377,"./plugins/safe.js":378,"./plugins/sortAttrs.js":379,"./plugins/sortDefsChildren.js":380}],321:[function(require,module,exports){ },{"./plugins/addAttributesToSVGElement.js":328,"./plugins/addClassesToSVGElement.js":329,"./plugins/cleanupAttrs.js":331,"./plugins/cleanupEnableBackground.js":332,"./plugins/cleanupIds.js":333,"./plugins/cleanupListOfValues.js":334,"./plugins/cleanupNumericValues.js":335,"./plugins/collapseGroups.js":336,"./plugins/convertColors.js":337,"./plugins/convertEllipseToCircle.js":338,"./plugins/convertPathData.js":339,"./plugins/convertShapeToPath.js":340,"./plugins/convertStyleToAttrs.js":341,"./plugins/convertTransform.js":342,"./plugins/default.js":343,"./plugins/inlineStyles.js":344,"./plugins/mergePaths.js":345,"./plugins/mergeStyles.js":346,"./plugins/minifyStyles.js":347,"./plugins/moveElemsAttrsToGroup.js":348,"./plugins/moveGroupAttrsToElems.js":349,"./plugins/prefixIds.js":350,"./plugins/removeAttributesBySelector.js":351,"./plugins/removeAttrs.js":352,"./plugins/removeComments.js":353,"./plugins/removeDesc.js":354,"./plugins/removeDimensions.js":355,"./plugins/removeDoctype.js":356,"./plugins/removeEditorsNSData.js":357,"./plugins/removeElementsByAttr.js":358,"./plugins/removeEmptyAttrs.js":359,"./plugins/removeEmptyContainers.js":360,"./plugins/removeEmptyText.js":361,"./plugins/removeHiddenElems.js":362,"./plugins/removeMetadata.js":363,"./plugins/removeNonInheritableGroupAttrs.js":364,"./plugins/removeOffCanvasPaths.js":365,"./plugins/removeRasterImages.js":366,"./plugins/removeScriptElement.js":367,"./plugins/removeStyleElement.js":368,"./plugins/removeTitle.js":369,"./plugins/removeUnknownsAndDefaults.js":370,"./plugins/removeUnusedNS.js":371,"./plugins/removeUselessDefs.js":372,"./plugins/removeUselessStrokeAndFill.js":373,"./plugins/removeViewBox.js":374,"./plugins/removeXMLNS.js":375,"./plugins/removeXMLProcInst.js":376,"./plugins/reusePaths.js":377,"./plugins/safe.js":378,"./plugins/safeAndFast.js":379,"./plugins/sortAttrs.js":380,"./plugins/sortDefsChildren.js":381}],321:[function(require,module,exports){
'use strict'; 'use strict';
const isTag = (node) => { const isTag = (node) => {
@ -30102,7 +30103,7 @@ const stringifyPathData = ({ pathData, precision, disableSpaceAfterFlags }) => {
}; };
exports.stringifyPathData = stringifyPathData; exports.stringifyPathData = stringifyPathData;
},{"./tools.js":383}],324:[function(require,module,exports){ },{"./tools.js":384}],324:[function(require,module,exports){
'use strict'; 'use strict';
const { builtin } = require('./builtin.js'); const { builtin } = require('./builtin.js');
@ -33826,7 +33827,7 @@ const applyMatrixToPathData = (pathData, matrix) => {
} }
}; };
},{"../style.js":382,"../tools.js":383,"./_collections.js":325,"./_path.js":326,"./_transforms.js":327}],331:[function(require,module,exports){ },{"../style.js":383,"../tools.js":384,"./_collections.js":325,"./_path.js":326,"./_transforms.js":327}],331:[function(require,module,exports){
'use strict'; 'use strict';
exports.name = 'cleanupAttrs'; exports.name = 'cleanupAttrs';
@ -33948,7 +33949,7 @@ exports.fn = (root) => {
}; };
}; };
},{"../xast.js":384}],333:[function(require,module,exports){ },{"../xast.js":385}],333:[function(require,module,exports){
'use strict'; 'use strict';
const { visitSkip } = require('../xast.js'); const { visitSkip } = require('../xast.js');
@ -34207,7 +34208,7 @@ exports.fn = (_root, params) => {
}; };
}; };
},{"../xast.js":384,"./_collections.js":325}],334:[function(require,module,exports){ },{"../xast.js":385,"./_collections.js":325}],334:[function(require,module,exports){
'use strict'; 'use strict';
const { removeLeadingZero } = require('../tools.js'); const { removeLeadingZero } = require('../tools.js');
@ -34345,7 +34346,7 @@ exports.fn = (_root, params) => {
}; };
}; };
},{"../tools.js":383}],335:[function(require,module,exports){ },{"../tools.js":384}],335:[function(require,module,exports){
'use strict'; 'use strict';
const { removeLeadingZero } = require('../tools.js'); const { removeLeadingZero } = require('../tools.js');
@ -34451,7 +34452,7 @@ exports.fn = (_root, params) => {
}; };
}; };
},{"../tools.js":383}],336:[function(require,module,exports){ },{"../tools.js":384}],336:[function(require,module,exports){
'use strict'; 'use strict';
const { inheritableAttrs, elemsGroups } = require('./_collections.js'); const { inheritableAttrs, elemsGroups } = require('./_collections.js');
@ -35838,7 +35839,7 @@ function data2Path(params, pathData) {
}, ''); }, '');
} }
},{"../style.js":382,"../tools.js":383,"../xast.js":384,"./_collections.js":325,"./_path.js":326,"./applyTransforms.js":330}],340:[function(require,module,exports){ },{"../style.js":383,"../tools.js":384,"../xast.js":385,"./_collections.js":325,"./_path.js":326,"./applyTransforms.js":330}],340:[function(require,module,exports){
'use strict'; 'use strict';
const { stringifyPathData } = require('../path.js'); const { stringifyPathData } = require('../path.js');
@ -36004,7 +36005,7 @@ exports.fn = (root, params) => {
}; };
}; };
},{"../path.js":323,"../xast.js":384}],341:[function(require,module,exports){ },{"../path.js":323,"../xast.js":385}],341:[function(require,module,exports){
'use strict'; 'use strict';
const { attrsGroups } = require('./_collections'); const { attrsGroups } = require('./_collections');
@ -36506,7 +36507,7 @@ const smartRound = (precision, data) => {
return data; return data;
}; };
},{"../tools.js":383,"./_transforms.js":327}],343:[function(require,module,exports){ },{"../tools.js":384,"./_transforms.js":327}],343:[function(require,module,exports){
'use strict'; 'use strict';
const { createPreset } = require('../tools.js'); const { createPreset } = require('../tools.js');
@ -36590,7 +36591,7 @@ const presetDefault = createPreset({
module.exports = presetDefault; module.exports = presetDefault;
},{"../tools.js":383,"./cleanupAttrs.js":331,"./cleanupEnableBackground.js":332,"./cleanupIds.js":333,"./cleanupNumericValues.js":335,"./collapseGroups.js":336,"./convertColors.js":337,"./convertEllipseToCircle.js":338,"./convertPathData.js":339,"./convertShapeToPath.js":340,"./convertTransform.js":342,"./inlineStyles.js":344,"./mergePaths.js":345,"./mergeStyles.js":346,"./minifyStyles.js":347,"./moveElemsAttrsToGroup.js":348,"./moveGroupAttrsToElems.js":349,"./removeComments.js":353,"./removeDesc.js":354,"./removeDoctype.js":356,"./removeEditorsNSData.js":357,"./removeEmptyAttrs.js":359,"./removeEmptyContainers.js":360,"./removeEmptyText.js":361,"./removeHiddenElems.js":362,"./removeMetadata.js":363,"./removeNonInheritableGroupAttrs.js":364,"./removeTitle.js":369,"./removeUnknownsAndDefaults.js":370,"./removeUnusedNS.js":371,"./removeUselessDefs.js":372,"./removeUselessStrokeAndFill.js":373,"./removeViewBox.js":374,"./removeXMLProcInst.js":376,"./sortAttrs.js":379,"./sortDefsChildren.js":380}],344:[function(require,module,exports){ },{"../tools.js":384,"./cleanupAttrs.js":331,"./cleanupEnableBackground.js":332,"./cleanupIds.js":333,"./cleanupNumericValues.js":335,"./collapseGroups.js":336,"./convertColors.js":337,"./convertEllipseToCircle.js":338,"./convertPathData.js":339,"./convertShapeToPath.js":340,"./convertTransform.js":342,"./inlineStyles.js":344,"./mergePaths.js":345,"./mergeStyles.js":346,"./minifyStyles.js":347,"./moveElemsAttrsToGroup.js":348,"./moveGroupAttrsToElems.js":349,"./removeComments.js":353,"./removeDesc.js":354,"./removeDoctype.js":356,"./removeEditorsNSData.js":357,"./removeEmptyAttrs.js":359,"./removeEmptyContainers.js":360,"./removeEmptyText.js":361,"./removeHiddenElems.js":362,"./removeMetadata.js":363,"./removeNonInheritableGroupAttrs.js":364,"./removeTitle.js":369,"./removeUnknownsAndDefaults.js":370,"./removeUnusedNS.js":371,"./removeUselessDefs.js":372,"./removeUselessStrokeAndFill.js":373,"./removeViewBox.js":374,"./removeXMLProcInst.js":376,"./sortAttrs.js":380,"./sortDefsChildren.js":381}],344:[function(require,module,exports){
'use strict'; 'use strict';
const csstree = require('css-tree'); const csstree = require('css-tree');
@ -36937,7 +36938,7 @@ exports.fn = (root, params) => {
}; };
}; };
},{"../xast.js":384,"css-tree":25,"csso":138}],345:[function(require,module,exports){ },{"../xast.js":385,"css-tree":25,"csso":138}],345:[function(require,module,exports){
'use strict'; 'use strict';
const { detachNodeFromParent } = require('../xast.js'); const { detachNodeFromParent } = require('../xast.js');
@ -37035,7 +37036,7 @@ exports.fn = (root, params) => {
}; };
}; };
},{"../style.js":382,"../xast.js":384,"./_path.js":326}],346:[function(require,module,exports){ },{"../style.js":383,"../xast.js":385,"./_path.js":326}],346:[function(require,module,exports){
'use strict'; 'use strict';
const { visitSkip, detachNodeFromParent } = require('../xast.js'); const { visitSkip, detachNodeFromParent } = require('../xast.js');
@ -37119,7 +37120,7 @@ exports.fn = () => {
}; };
}; };
},{"../xast.js":384}],347:[function(require,module,exports){ },{"../xast.js":385}],347:[function(require,module,exports){
'use strict'; 'use strict';
const csso = require('csso'); const csso = require('csso');
@ -37364,7 +37365,7 @@ exports.fn = (root) => {
}; };
}; };
},{"../xast.js":384,"./_collections.js":325}],349:[function(require,module,exports){ },{"../xast.js":385,"./_collections.js":325}],349:[function(require,module,exports){
'use strict'; 'use strict';
const { pathElems, referencesProps } = require('./_collections.js'); const { pathElems, referencesProps } = require('./_collections.js');
@ -37742,7 +37743,7 @@ exports.fn = (root, params) => {
return {}; return {};
}; };
},{"../xast.js":384}],352:[function(require,module,exports){ },{"../xast.js":385}],352:[function(require,module,exports){
'use strict'; 'use strict';
exports.name = 'removeAttrs'; exports.name = 'removeAttrs';
@ -37924,7 +37925,7 @@ exports.fn = () => {
}; };
}; };
},{"../xast.js":384}],354:[function(require,module,exports){ },{"../xast.js":385}],354:[function(require,module,exports){
'use strict'; 'use strict';
const { detachNodeFromParent } = require('../xast.js'); const { detachNodeFromParent } = require('../xast.js');
@ -37963,7 +37964,7 @@ exports.fn = (root, params) => {
}; };
}; };
},{"../xast.js":384}],355:[function(require,module,exports){ },{"../xast.js":385}],355:[function(require,module,exports){
'use strict'; 'use strict';
exports.name = 'removeDimensions'; exports.name = 'removeDimensions';
@ -38046,7 +38047,7 @@ exports.fn = () => {
}; };
}; };
},{"../xast.js":384}],357:[function(require,module,exports){ },{"../xast.js":385}],357:[function(require,module,exports){
'use strict'; 'use strict';
const { detachNodeFromParent } = require('../xast.js'); const { detachNodeFromParent } = require('../xast.js');
@ -38110,7 +38111,7 @@ exports.fn = (_root, params) => {
}; };
}; };
},{"../xast.js":384,"./_collections.js":325}],358:[function(require,module,exports){ },{"../xast.js":385,"./_collections.js":325}],358:[function(require,module,exports){
'use strict'; 'use strict';
const { detachNodeFromParent } = require('../xast.js'); const { detachNodeFromParent } = require('../xast.js');
@ -38183,7 +38184,7 @@ exports.fn = (root, params) => {
}; };
}; };
},{"../xast.js":384}],359:[function(require,module,exports){ },{"../xast.js":385}],359:[function(require,module,exports){
'use strict'; 'use strict';
const { attrsGroups } = require('./_collections.js'); const { attrsGroups } = require('./_collections.js');
@ -38270,7 +38271,7 @@ exports.fn = () => {
}; };
}; };
},{"../xast.js":384,"./_collections.js":325}],361:[function(require,module,exports){ },{"../xast.js":385,"./_collections.js":325}],361:[function(require,module,exports){
'use strict'; 'use strict';
const { detachNodeFromParent } = require('../xast.js'); const { detachNodeFromParent } = require('../xast.js');
@ -38321,7 +38322,7 @@ exports.fn = (root, params) => {
}; };
}; };
},{"../xast.js":384}],362:[function(require,module,exports){ },{"../xast.js":385}],362:[function(require,module,exports){
'use strict'; 'use strict';
const { const {
@ -38631,7 +38632,7 @@ exports.fn = (root, params) => {
}; };
}; };
},{"../path.js":323,"../style.js":382,"../xast.js":384}],363:[function(require,module,exports){ },{"../path.js":323,"../style.js":383,"../xast.js":385}],363:[function(require,module,exports){
'use strict'; 'use strict';
const { detachNodeFromParent } = require('../xast.js'); const { detachNodeFromParent } = require('../xast.js');
@ -38658,7 +38659,7 @@ exports.fn = () => {
}; };
}; };
},{"../xast.js":384}],364:[function(require,module,exports){ },{"../xast.js":385}],364:[function(require,module,exports){
'use strict'; 'use strict';
const { const {
@ -38815,7 +38816,7 @@ exports.fn = () => {
}; };
}; };
},{"../path.js":323,"../xast.js":384,"./_path.js":326}],366:[function(require,module,exports){ },{"../path.js":323,"../xast.js":385,"./_path.js":326}],366:[function(require,module,exports){
'use strict'; 'use strict';
const { detachNodeFromParent } = require('../xast.js'); const { detachNodeFromParent } = require('../xast.js');
@ -38846,7 +38847,7 @@ exports.fn = () => {
}; };
}; };
},{"../xast.js":384}],367:[function(require,module,exports){ },{"../xast.js":385}],367:[function(require,module,exports){
'use strict'; 'use strict';
const { detachNodeFromParent } = require('../xast.js'); const { detachNodeFromParent } = require('../xast.js');
@ -38873,7 +38874,7 @@ exports.fn = () => {
}; };
}; };
},{"../xast.js":384}],368:[function(require,module,exports){ },{"../xast.js":385}],368:[function(require,module,exports){
'use strict'; 'use strict';
const { detachNodeFromParent } = require('../xast.js'); const { detachNodeFromParent } = require('../xast.js');
@ -38900,7 +38901,7 @@ exports.fn = () => {
}; };
}; };
},{"../xast.js":384}],369:[function(require,module,exports){ },{"../xast.js":385}],369:[function(require,module,exports){
'use strict'; 'use strict';
const { detachNodeFromParent } = require('../xast.js'); const { detachNodeFromParent } = require('../xast.js');
@ -38927,7 +38928,7 @@ exports.fn = () => {
}; };
}; };
},{"../xast.js":384}],370:[function(require,module,exports){ },{"../xast.js":385}],370:[function(require,module,exports){
'use strict'; 'use strict';
const { visitSkip, detachNodeFromParent } = require('../xast.js'); const { visitSkip, detachNodeFromParent } = require('../xast.js');
@ -39135,7 +39136,7 @@ exports.fn = (root, params) => {
}; };
}; };
},{"../style.js":382,"../xast.js":384,"./_collections":325}],371:[function(require,module,exports){ },{"../style.js":383,"../xast.js":385,"./_collections":325}],371:[function(require,module,exports){
'use strict'; 'use strict';
exports.name = 'removeUnusedNS'; exports.name = 'removeUnusedNS';
@ -39252,7 +39253,7 @@ const collectUsefulNodes = (node, usefulNodes) => {
} }
}; };
},{"../xast.js":384,"./_collections.js":325}],373:[function(require,module,exports){ },{"../xast.js":385,"./_collections.js":325}],373:[function(require,module,exports){
'use strict'; 'use strict';
const { visit, visitSkip, detachNodeFromParent } = require('../xast.js'); const { visit, visitSkip, detachNodeFromParent } = require('../xast.js');
@ -39390,7 +39391,7 @@ exports.fn = (root, params) => {
}; };
}; };
},{"../style.js":382,"../xast.js":384,"./_collections.js":325}],374:[function(require,module,exports){ },{"../style.js":383,"../xast.js":385,"./_collections.js":325}],374:[function(require,module,exports){
'use strict'; 'use strict';
exports.name = 'removeViewBox'; exports.name = 'removeViewBox';
@ -39497,7 +39498,7 @@ exports.fn = () => {
}; };
}; };
},{"../xast.js":384}],377:[function(require,module,exports){ },{"../xast.js":385}],377:[function(require,module,exports){
'use strict'; 'use strict';
exports.name = 'reusePaths'; exports.name = 'reusePaths';
@ -39678,7 +39679,72 @@ const presetSafe = createPreset({
module.exports = presetSafe; module.exports = presetSafe;
},{"../tools.js":383,"./cleanupAttrs.js":331,"./cleanupEnableBackground.js":332,"./cleanupIds.js":333,"./cleanupNumericValues.js":335,"./collapseGroups.js":336,"./convertColors.js":337,"./convertEllipseToCircle.js":338,"./convertPathData.js":339,"./convertTransform.js":342,"./inlineStyles.js":344,"./mergePaths.js":345,"./mergeStyles.js":346,"./minifyStyles.js":347,"./removeComments.js":353,"./removeDesc.js":354,"./removeDoctype.js":356,"./removeEditorsNSData.js":357,"./removeEmptyAttrs.js":359,"./removeEmptyContainers.js":360,"./removeEmptyText.js":361,"./removeHiddenElems.js":362,"./removeMetadata.js":363,"./removeNonInheritableGroupAttrs.js":364,"./removeTitle.js":369,"./removeUnknownsAndDefaults.js":370,"./removeUnusedNS.js":371,"./removeUselessDefs.js":372,"./removeUselessStrokeAndFill.js":373,"./removeViewBox.js":374,"./removeXMLProcInst.js":376,"./sortDefsChildren.js":380}],379:[function(require,module,exports){ },{"../tools.js":384,"./cleanupAttrs.js":331,"./cleanupEnableBackground.js":332,"./cleanupIds.js":333,"./cleanupNumericValues.js":335,"./collapseGroups.js":336,"./convertColors.js":337,"./convertEllipseToCircle.js":338,"./convertPathData.js":339,"./convertTransform.js":342,"./inlineStyles.js":344,"./mergePaths.js":345,"./mergeStyles.js":346,"./minifyStyles.js":347,"./removeComments.js":353,"./removeDesc.js":354,"./removeDoctype.js":356,"./removeEditorsNSData.js":357,"./removeEmptyAttrs.js":359,"./removeEmptyContainers.js":360,"./removeEmptyText.js":361,"./removeHiddenElems.js":362,"./removeMetadata.js":363,"./removeNonInheritableGroupAttrs.js":364,"./removeTitle.js":369,"./removeUnknownsAndDefaults.js":370,"./removeUnusedNS.js":371,"./removeUselessDefs.js":372,"./removeUselessStrokeAndFill.js":373,"./removeViewBox.js":374,"./removeXMLProcInst.js":376,"./sortDefsChildren.js":381}],379:[function(require,module,exports){
'use strict';
const { createPreset } = require('../tools.js');
const removeDoctype = require('./removeDoctype.js');
const removeXMLProcInst = require('./removeXMLProcInst.js');
const removeComments = require('./removeComments.js');
const removeMetadata = require('./removeMetadata.js');
const removeEditorsNSData = require('./removeEditorsNSData.js');
const cleanupAttrs = require('./cleanupAttrs.js');
const mergeStyles = require('./mergeStyles.js');
const minifyStyles = require('./minifyStyles.js');
const cleanupIds = require('./cleanupIds.js');
const removeUselessDefs = require('./removeUselessDefs.js');
const cleanupNumericValues = require('./cleanupNumericValues.js');
const convertColors = require('./convertColors.js');
const removeUnknownsAndDefaults = require('./removeUnknownsAndDefaults.js');
const removeNonInheritableGroupAttrs = require('./removeNonInheritableGroupAttrs.js');
const removeUselessStrokeAndFill = require('./removeUselessStrokeAndFill.js');
const removeViewBox = require('./removeViewBox.js');
const cleanupEnableBackground = require('./cleanupEnableBackground.js');
const removeHiddenElems = require('./removeHiddenElems.js');
const removeEmptyText = require('./removeEmptyText.js');
const collapseGroups = require('./collapseGroups.js');
const removeEmptyAttrs = require('./removeEmptyAttrs.js');
const removeEmptyContainers = require('./removeEmptyContainers.js');
const mergePaths = require('./mergePaths.js');
const removeUnusedNS = require('./removeUnusedNS.js');
const sortDefsChildren = require('./sortDefsChildren.js');
const removeTitle = require('./removeTitle.js');
const removeDesc = require('./removeDesc.js');
const presetSafe = createPreset({
name: 'safeAndFastPreset',
plugins: [
removeDoctype,
removeXMLProcInst,
removeComments,
removeMetadata,
removeEditorsNSData,
cleanupAttrs,
mergeStyles,
cleanupIds,
removeUselessDefs,
cleanupNumericValues,
convertColors,
removeUnknownsAndDefaults,
removeNonInheritableGroupAttrs,
removeUselessStrokeAndFill,
removeViewBox,
cleanupEnableBackground,
removeHiddenElems,
removeEmptyText,
collapseGroups,
removeEmptyAttrs,
removeEmptyContainers,
removeUnusedNS,
removeTitle,
removeDesc,
],
});
module.exports = presetSafe;
},{"../tools.js":384,"./cleanupAttrs.js":331,"./cleanupEnableBackground.js":332,"./cleanupIds.js":333,"./cleanupNumericValues.js":335,"./collapseGroups.js":336,"./convertColors.js":337,"./mergePaths.js":345,"./mergeStyles.js":346,"./minifyStyles.js":347,"./removeComments.js":353,"./removeDesc.js":354,"./removeDoctype.js":356,"./removeEditorsNSData.js":357,"./removeEmptyAttrs.js":359,"./removeEmptyContainers.js":360,"./removeEmptyText.js":361,"./removeHiddenElems.js":362,"./removeMetadata.js":363,"./removeNonInheritableGroupAttrs.js":364,"./removeTitle.js":369,"./removeUnknownsAndDefaults.js":370,"./removeUnusedNS.js":371,"./removeUselessDefs.js":372,"./removeUselessStrokeAndFill.js":373,"./removeViewBox.js":374,"./removeXMLProcInst.js":376,"./sortDefsChildren.js":381}],380:[function(require,module,exports){
'use strict'; 'use strict';
exports.name = 'sortAttrs'; exports.name = 'sortAttrs';
@ -39778,7 +39844,7 @@ exports.fn = (_root, params) => {
}; };
}; };
},{}],380:[function(require,module,exports){ },{}],381:[function(require,module,exports){
'use strict'; 'use strict';
exports.name = 'sortDefsChildren'; exports.name = 'sortDefsChildren';
@ -39836,7 +39902,7 @@ exports.fn = () => {
}; };
}; };
},{}],381:[function(require,module,exports){ },{}],382:[function(require,module,exports){
'use strict'; 'use strict';
const { textElems } = require('./plugins/_collections.js'); const { textElems } = require('./plugins/_collections.js');
@ -40070,7 +40136,7 @@ const stringifyText = (node, config, state) => {
); );
}; };
},{"./plugins/_collections.js":325}],382:[function(require,module,exports){ },{"./plugins/_collections.js":325}],383:[function(require,module,exports){
'use strict'; 'use strict';
const csstree = require('css-tree'); const csstree = require('css-tree');
@ -40300,7 +40366,7 @@ const computeStyle = (stylesheet, node) => {
}; };
exports.computeStyle = computeStyle; exports.computeStyle = computeStyle;
},{"./plugins/_collections.js":325,"./xast.js":384,"css-tree":25,"csso":138}],383:[function(require,module,exports){ },{"./plugins/_collections.js":325,"./xast.js":385,"css-tree":25,"csso":138}],384:[function(require,module,exports){
(function (Buffer){(function (){ (function (Buffer){(function (){
'use strict'; 'use strict';
@ -40455,7 +40521,7 @@ exports.invokePlugins = invokePlugins;
exports.createPreset = createPreset; exports.createPreset = createPreset;
}).call(this)}).call(this,require("buffer").Buffer) }).call(this)}).call(this,require("buffer").Buffer)
},{"./xast.js":384,"buffer":4}],384:[function(require,module,exports){ },{"./xast.js":385,"buffer":4}],385:[function(require,module,exports){
'use strict'; 'use strict';
const { selectAll, selectOne, is } = require('css-select'); const { selectAll, selectOne, is } = require('css-select');

23
common/src/app/common/types/container.cljc
View file

@ -255,7 +255,11 @@
(dissoc :component-root))) (dissoc :component-root)))
[new-root-shape new-shapes updated-shapes] [new-root-shape new-shapes updated-shapes]
(ctst/clone-object shape nil objects update-new-shape update-original-shape) (ctst/clone-shape shape
nil
objects
:update-new-shape update-new-shape
:update-original-shape update-original-shape)
;; If frame-id points to a shape inside the component, remap it to the ;; If frame-id points to a shape inside the component, remap it to the
;; corresponding new frame shape. If not, set it to nil. ;; corresponding new frame shape. If not, set it to nil.
@ -339,15 +343,14 @@
(dissoc :component-root)))) (dissoc :component-root))))
[new-shape new-shapes _] [new-shape new-shapes _]
(ctst/clone-object component-shape (ctst/clone-shape component-shape
frame-id frame-id
(if components-v2 (:objects component-page) (:objects component)) (if components-v2 (:objects component-page) (:objects component))
update-new-shape :update-new-shape update-new-shape
(fn [object _] object) :force-id force-id
force-id :keep-ids? keep-ids?
keep-ids? :frame-id frame-id
frame-id) :dest-objects (:objects container))
;; Fix empty parent-id and remap all grid cells to the new ids. ;; Fix empty parent-id and remap all grid cells to the new ids.
remap-ids remap-ids

130
common/src/app/common/types/shape_tree.cljc
View file

@ -342,91 +342,89 @@
[frame] [frame]
(not (mth/almost-zero? (:rotation frame 0)))) (not (mth/almost-zero? (:rotation frame 0))))
(defn clone-object (defn clone-shape
"Gets a copy of the object and all its children, with new ids and with "Gets a copy of the shape and all its children, with new ids and with
the parent-children links correctly set. Admits functions to make the parent-children links correctly set. Admits functions to make
more transformations to the cloned objects and the original ones. more transformations to the cloned shapes and the original ones.
Returns the cloned object, the list of all new objects (including Returns the cloned shape, the list of all new shapes (including
the cloned one), and possibly a list of original objects modified. the cloned one), and possibly a list of original shapes modified.
The list of objects are returned in tree traversal order, respecting The list of shapes are returned in tree traversal order, respecting
the order of the children of each parent." the order of the children of each parent."
[shape parent-id objects & {:keys [update-new-shape update-original-shape force-id keep-ids? frame-id dest-objects]
([object parent-id objects] :or {update-new-shape (fn [shape _] shape)
(clone-object object parent-id objects (fn [object _] object) (fn [object _] object) nil false nil)) update-original-shape (fn [shape _] shape)
force-id nil
([object parent-id objects update-new-object] keep-ids? false
(clone-object object parent-id objects update-new-object (fn [object _] object) nil false nil)) frame-id nil
dest-objects objects}}]
([object parent-id objects update-new-object update-original-object] (let [new-id (cond
(clone-object object parent-id objects update-new-object update-original-object nil false nil)) (some? force-id) force-id
keep-ids? (:id shape)
([object parent-id objects update-new-object update-original-object force-id] :else (uuid/next))
(clone-object object parent-id objects update-new-object update-original-object force-id false nil))
([object parent-id objects update-new-object update-original-object force-id keep-ids?]
(clone-object object parent-id objects update-new-object update-original-object force-id keep-ids? nil))
([object parent-id objects update-new-object update-original-object force-id keep-ids? frame-id]
(let [new-id (cond
(some? force-id) force-id
keep-ids? (:id object)
:else (uuid/next))
;; Assign the correct frame-id for the given parent. It's the parent-id (if parent is frame) ;; Assign the correct frame-id for the given parent. It's the parent-id (if parent is frame)
;; or the parent's frame-id otherwise. Only for the first cloned shapes. In recursive calls ;; or the parent's frame-id otherwise. Only for the first cloned shapes. In recursive calls
;; this is not needed. ;; this is not needed.
frame-id (cond frame-id (cond
(and (nil? frame-id) (cfh/frame-shape? objects parent-id)) (and (nil? frame-id) (cfh/frame-shape? dest-objects parent-id))
parent-id parent-id
(nil? frame-id) (nil? frame-id)
(dm/get-in objects [parent-id :frame-id]) (dm/get-in dest-objects [parent-id :frame-id] uuid/zero)
:else :else
frame-id)] frame-id)]
(loop [child-ids (seq (:shapes object)) (loop [child-ids (seq (:shapes shape))
new-direct-children [] new-direct-children []
new-children [] new-children []
updated-children []] updated-children []]
(if (empty? child-ids) (if (empty? child-ids)
(let [new-object (cond-> object (let [new-shape (cond-> shape
:always :always
(assoc :id new-id (assoc :id new-id
:parent-id parent-id :parent-id parent-id
:frame-id frame-id) :frame-id frame-id)
(some? (:shapes object)) (some? (:shapes shape))
(assoc :shapes (mapv :id new-direct-children))) (assoc :shapes (mapv :id new-direct-children)))
new-object (update-new-object new-object object) new-shape (update-new-shape new-shape shape)
new-objects (into [new-object] new-children) new-shapes (into [new-shape] new-children)
updated-object (update-original-object object new-object) updated-shape (update-original-shape shape new-shape)
updated-objects (if (identical? object updated-object) updated-shapes (if (identical? shape updated-shape)
updated-children updated-children
(into [updated-object] updated-children))] (into [updated-shape] updated-children))]
[new-object new-objects updated-objects]) [new-shape new-shapes updated-shapes])
(let [child-id (first child-ids) (let [child-id (first child-ids)
child (get objects child-id) child (get objects child-id)
_ (dm/assert! (some? child)) _ (dm/assert! (some? child))
frame-id-child (if (cfh/frame-shape? object) frame-id-child (if (cfh/frame-shape? shape)
new-id new-id
frame-id) frame-id)
[new-child new-child-objects updated-child-objects] [new-child new-child-shapes updated-child-shapes]
(clone-object child new-id objects update-new-object update-original-object nil keep-ids? frame-id-child)] (clone-shape child
new-id
objects
:update-new-shape update-new-shape
:update-original-shape update-original-shape
:force-id nil
:keep-ids? keep-ids?
:frame-id frame-id-child
:dest-objects dest-objects)]
(recur (recur
(next child-ids) (next child-ids)
(into new-direct-children [new-child]) (into new-direct-children [new-child])
(into new-children new-child-objects) (into new-children new-child-shapes)
(into updated-children updated-child-objects)))))))) (into updated-children updated-child-shapes)))))))
(defn generate-shape-grid (defn generate-shape-grid
"Generate a sequence of positions that lays out the list of "Generate a sequence of positions that lays out the list of

34
frontend/src/app/main/data/workspace/libraries_helpers.cljs
View file

@ -98,11 +98,11 @@
(gsh/move delta))) (gsh/move delta)))
[new-instance-shape new-instance-shapes _] [new-instance-shape new-instance-shapes _]
(ctst/clone-object main-instance-shape (ctst/clone-shape main-instance-shape
(:parent-id main-instance-shape) (:parent-id main-instance-shape)
(:objects main-instance-page) (:objects main-instance-page)
update-new-shape :update-new-shape update-new-shape
update-original-shape) :update-original-shape update-original-shape)
remap-frame remap-frame
(fn [shape] (fn [shape]
@ -134,10 +134,9 @@
(let [component-root (d/seek #(nil? (:parent-id %)) (vals (:objects component))) (let [component-root (d/seek #(nil? (:parent-id %)) (vals (:objects component)))
[new-component-shape new-component-shapes _] [new-component-shape new-component-shapes _]
(ctst/clone-object component-root (ctst/clone-shape component-root
nil nil
(get component :objects) (get component :objects))]
identity)]
[new-component-shape new-component-shapes nil nil])))) [new-component-shape new-component-shapes nil nil]))))
@ -976,11 +975,12 @@
original-shape) original-shape)
[_ new-shapes _] [_ new-shapes _]
(ctst/clone-object component-shape (ctst/clone-shape component-shape
(:id parent-shape) (:id parent-shape)
(get component-page :objects) (get component-page :objects)
update-new-shape :update-new-shape update-new-shape
update-original-shape) :update-original-shape update-original-shape
:dest-objects (get container :objects))
add-obj-change (fn [changes shape'] add-obj-change (fn [changes shape']
(update changes :redo-changes conj (update changes :redo-changes conj
@ -1038,11 +1038,11 @@
original-shape)) original-shape))
[_new-shape new-shapes updated-shapes] [_new-shape new-shapes updated-shapes]
(ctst/clone-object shape (ctst/clone-shape shape
(:id component-parent-shape) (:id component-parent-shape)
(get page :objects) (get page :objects)
update-new-shape :update-new-shape update-new-shape
update-original-shape) :update-original-shape update-original-shape)
add-obj-change (fn [changes shape'] add-obj-change (fn [changes shape']
(update changes :redo-changes conj (update changes :redo-changes conj