mirror of
https://github.com/penpot/penpot.git
synced 2025-05-19 00:06:14 +02:00
♻️ Simplify components-v2 migration functions impl
This commit is contained in:
parent
02d8208553
commit
0a5e15b916
2 changed files with 192 additions and 168 deletions
|
@ -20,6 +20,7 @@
|
||||||
[app.common.geom.rect :as grc]
|
[app.common.geom.rect :as grc]
|
||||||
[app.common.geom.shapes :as gsh]
|
[app.common.geom.shapes :as gsh]
|
||||||
[app.common.logging :as l]
|
[app.common.logging :as l]
|
||||||
|
[app.common.math :as mth]
|
||||||
[app.common.svg :as csvg]
|
[app.common.svg :as csvg]
|
||||||
[app.common.svg.shapes-builder :as sbuilder]
|
[app.common.svg.shapes-builder :as sbuilder]
|
||||||
[app.common.types.component :as ctk]
|
[app.common.types.component :as ctk]
|
||||||
|
@ -48,7 +49,8 @@
|
||||||
[buddy.core.codecs :as bc]
|
[buddy.core.codecs :as bc]
|
||||||
[cuerdas.core :as str]
|
[cuerdas.core :as str]
|
||||||
[datoteka.io :as io]
|
[datoteka.io :as io]
|
||||||
[promesa.core :as p]))
|
[promesa.exec :as px]
|
||||||
|
[promesa.util :as pu]))
|
||||||
|
|
||||||
(def ^:dynamic *stats*
|
(def ^:dynamic *stats*
|
||||||
"A dynamic var for setting up state for collect stats globally."
|
"A dynamic var for setting up state for collect stats globally."
|
||||||
|
@ -67,6 +69,10 @@
|
||||||
internal functions without the need to explicitly pass it top down."
|
internal functions without the need to explicitly pass it top down."
|
||||||
nil)
|
nil)
|
||||||
|
|
||||||
|
(def ^:dynamic ^:private *team-id*
|
||||||
|
"A dynamic var that holds the current processing team-id."
|
||||||
|
nil)
|
||||||
|
|
||||||
(def ^:dynamic ^:private *file-stats*
|
(def ^:dynamic ^:private *file-stats*
|
||||||
"An internal dynamic var for collect stats by file."
|
"An internal dynamic var for collect stats by file."
|
||||||
nil)
|
nil)
|
||||||
|
@ -575,32 +581,40 @@
|
||||||
(if (> ext-idx 0) (subs filename 0 ext-idx) filename)))
|
(if (> ext-idx 0) (subs filename 0 ext-idx) filename)))
|
||||||
|
|
||||||
(defn- collect-and-persist-images
|
(defn- collect-and-persist-images
|
||||||
[svg-data file-id]
|
[svg-data file-id media-id]
|
||||||
(letfn [(process-image [{:keys [href] :as item}]
|
(letfn [(process-image [{:keys [href] :as item}]
|
||||||
(let [item (if (str/starts-with? href "data:")
|
(try
|
||||||
(let [[mtype data] (parse-datauri href)
|
(let [item (if (str/starts-with? href "data:")
|
||||||
size (alength data)
|
(let [[mtype data] (parse-datauri href)
|
||||||
path (tmp/tempfile :prefix "penpot.media.download.")
|
size (alength data)
|
||||||
written (io/write-to-file! data path :size size)]
|
path (tmp/tempfile :prefix "penpot.media.download.")
|
||||||
|
written (io/write-to-file! data path :size size)]
|
||||||
|
|
||||||
(when (not= written size)
|
(when (not= written size)
|
||||||
(ex/raise :type :internal
|
(ex/raise :type :internal
|
||||||
:code :mismatch-write-size
|
:code :mismatch-write-size
|
||||||
:hint "unexpected state: unable to write to file"))
|
:hint "unexpected state: unable to write to file"))
|
||||||
|
|
||||||
(-> item
|
(-> item
|
||||||
(assoc :size size)
|
(assoc :size size)
|
||||||
(assoc :path path)
|
(assoc :path path)
|
||||||
(assoc :filename "tempfile")
|
(assoc :filename "tempfile")
|
||||||
(assoc :mtype mtype)))
|
(assoc :mtype mtype)))
|
||||||
|
|
||||||
(let [result (cmd.media/download-image *system* href)]
|
(let [result (cmd.media/download-image *system* href)]
|
||||||
(-> (merge item result)
|
(-> (merge item result)
|
||||||
(assoc :name (extract-name href)))))]
|
(assoc :name (extract-name href)))))]
|
||||||
|
|
||||||
;; The media processing adds the data to the
|
;; The media processing adds the data to the
|
||||||
;; input map and returns it.
|
;; input map and returns it.
|
||||||
(media/run {:cmd :info :input item})))
|
(media/run {:cmd :info :input item}))
|
||||||
|
(catch Throwable _
|
||||||
|
(let [team-id *team-id*]
|
||||||
|
(l/wrn :hint "unable to process embedded images on svg file"
|
||||||
|
:team-id (str team-id)
|
||||||
|
:file-id (str file-id)
|
||||||
|
:media-id (str media-id)))
|
||||||
|
nil)))
|
||||||
|
|
||||||
(persist-image [acc {:keys [path size width height mtype href] :as item}]
|
(persist-image [acc {:keys [path size width height mtype href] :as item}]
|
||||||
(let [storage (::sto/storage *system*)
|
(let [storage (::sto/storage *system*)
|
||||||
|
@ -639,9 +653,7 @@
|
||||||
(defn- resolve-sobject-id
|
(defn- resolve-sobject-id
|
||||||
[id]
|
[id]
|
||||||
(let [fmobject (db/get *system* :file-media-object {:id id}
|
(let [fmobject (db/get *system* :file-media-object {:id id}
|
||||||
{::db/check-deleted false
|
{::sql/columns [:media-id]})]
|
||||||
::db/remove-deleted false
|
|
||||||
::sql/columns [:media-id]})]
|
|
||||||
(:media-id fmobject)))
|
(:media-id fmobject)))
|
||||||
|
|
||||||
(defn- get-sobject-content
|
(defn- get-sobject-content
|
||||||
|
@ -660,12 +672,11 @@
|
||||||
(assoc :name (:name mobj)))))
|
(assoc :name (:name mobj)))))
|
||||||
|
|
||||||
sid (resolve-sobject-id id)
|
sid (resolve-sobject-id id)
|
||||||
|
|
||||||
svg-data (if (cache/cache? *cache*)
|
svg-data (if (cache/cache? *cache*)
|
||||||
(cache/get *cache* sid get-svg)
|
(cache/get *cache* sid (px/wrap-bindings get-svg))
|
||||||
(get-svg sid))
|
(get-svg sid))
|
||||||
|
|
||||||
svg-data (collect-and-persist-images file-id)]
|
svg-data (collect-and-persist-images svg-data file-id id)]
|
||||||
|
|
||||||
(sbuilder/create-svg-shapes svg-data position objects frame-id frame-id #{} false)))
|
(sbuilder/create-svg-shapes svg-data position objects frame-id frame-id #{} false)))
|
||||||
|
|
||||||
|
@ -724,25 +735,49 @@
|
||||||
|
|
||||||
(defn- create-media-grid
|
(defn- create-media-grid
|
||||||
[fdata page-id frame-id grid media-group]
|
[fdata page-id frame-id grid media-group]
|
||||||
(letfn [(process [fdata mobj position]
|
(letfn [(process-media-object [fdata mobj position]
|
||||||
(let [position (gpt/add position (gpt/point grid-gap grid-gap))
|
(let [position (gpt/add position (gpt/point grid-gap grid-gap))
|
||||||
tp (dt/tpoint)]
|
tp (dt/tpoint)
|
||||||
|
err (volatile! false)]
|
||||||
(try
|
(try
|
||||||
(let [changes (process-media-object fdata page-id frame-id mobj position)]
|
(let [changes (process-media-object fdata page-id frame-id mobj position)]
|
||||||
(cp/process-changes fdata changes false))
|
(cp/process-changes fdata changes false))
|
||||||
|
|
||||||
(catch Throwable cause
|
(catch Throwable cause
|
||||||
(if *skip-on-graphic-error*
|
(vreset! err true)
|
||||||
(l/wrn :hint "unable to process file media object (skiping)"
|
(let [cause (pu/unwrap-exception cause)
|
||||||
:file-id (str (:id fdata))
|
edata (ex-data cause)
|
||||||
:id (str (:id mobj))
|
team-id *team-id*]
|
||||||
:cause cause)
|
(cond
|
||||||
(throw cause))
|
(instance? org.xml.sax.SAXParseException cause)
|
||||||
nil)
|
(l/inf :hint "skip processing media object: invalid svg found"
|
||||||
|
:team-id (str team-id)
|
||||||
|
:file-id (str (:id fdata))
|
||||||
|
:id (str (:id mobj)))
|
||||||
|
|
||||||
|
(= (:type edata) :not-found)
|
||||||
|
(l/inf :hint "skip processing media object: underlying object does not exist"
|
||||||
|
:team-id (str team-id)
|
||||||
|
:file-id (str (:id fdata))
|
||||||
|
:id (str (:id mobj)))
|
||||||
|
|
||||||
|
:else
|
||||||
|
(let [skip? *skip-on-graphic-error*]
|
||||||
|
(l/wrn :hint "unable to process file media object"
|
||||||
|
:skiped skip?
|
||||||
|
:team-id (str team-id)
|
||||||
|
:file-id (str (:id fdata))
|
||||||
|
:id (str (:id mobj))
|
||||||
|
:cause cause)
|
||||||
|
(when-not skip?
|
||||||
|
(throw cause))))
|
||||||
|
nil))
|
||||||
(finally
|
(finally
|
||||||
(let [elapsed (tp)]
|
(let [elapsed (tp)]
|
||||||
(l/trc :hint "graphic processed"
|
(l/trc :hint "graphic processed"
|
||||||
:file-id (str (:id fdata))
|
:file-id (str (:id fdata))
|
||||||
:media-id (str (:id mobj))
|
:media-id (str (:id mobj))
|
||||||
|
:error @err
|
||||||
:elapsed (dt/format-duration elapsed)))))))]
|
:elapsed (dt/format-duration elapsed)))))))]
|
||||||
|
|
||||||
(->> (d/zip media-group grid)
|
(->> (d/zip media-group grid)
|
||||||
|
@ -750,7 +785,8 @@
|
||||||
(sse/tap {:type :migration-progress
|
(sse/tap {:type :migration-progress
|
||||||
:section :graphics
|
:section :graphics
|
||||||
:name (:name mobj)})
|
:name (:name mobj)})
|
||||||
(or (process fdata mobj position) fdata))
|
(or (process-media-object fdata mobj position)
|
||||||
|
fdata))
|
||||||
(assoc-in fdata [:options :components-v2] true)))))
|
(assoc-in fdata [:options :components-v2] true)))))
|
||||||
|
|
||||||
(defn- migrate-graphics
|
(defn- migrate-graphics
|
||||||
|
@ -822,7 +858,6 @@
|
||||||
(update :data fdata/process-pointers deref)
|
(update :data fdata/process-pointers deref)
|
||||||
(fmg/migrate-file))))
|
(fmg/migrate-file))))
|
||||||
|
|
||||||
|
|
||||||
(defn- get-team
|
(defn- get-team
|
||||||
[system team-id]
|
[system team-id]
|
||||||
(-> (db/get system :team {:id team-id}
|
(-> (db/get system :team {:id team-id}
|
||||||
|
@ -870,12 +905,13 @@
|
||||||
|
|
||||||
(dissoc file :data)))
|
(dissoc file :data)))
|
||||||
|
|
||||||
|
|
||||||
(def ^:private sql:get-and-lock-team-files
|
(def ^:private sql:get-and-lock-team-files
|
||||||
"SELECT f.id
|
"SELECT f.id
|
||||||
FROM file AS f
|
FROM file AS f
|
||||||
JOIN project AS p ON (p.id = f.project_id)
|
JOIN project AS p ON (p.id = f.project_id)
|
||||||
WHERE p.team_id = ?
|
WHERE p.team_id = ?
|
||||||
|
AND p.deleted_at IS NULL
|
||||||
|
AND f.deleted_at IS NULL
|
||||||
FOR UPDATE")
|
FOR UPDATE")
|
||||||
|
|
||||||
(defn- get-and-lock-files
|
(defn- get-and-lock-files
|
||||||
|
@ -900,14 +936,26 @@
|
||||||
(binding [*file-stats* (atom {})
|
(binding [*file-stats* (atom {})
|
||||||
*skip-on-graphic-error* skip-on-graphic-error?]
|
*skip-on-graphic-error* skip-on-graphic-error?]
|
||||||
(try
|
(try
|
||||||
(l/dbg :hint "migrate:file:start" :file-id (str file-id))
|
(l/dbg :hint "migrate:file:start"
|
||||||
|
:file-id (str file-id)
|
||||||
|
:validate validate?
|
||||||
|
:skip-on-graphics-error skip-on-graphic-error?)
|
||||||
|
|
||||||
(let [system (update system ::sto/storage media/configure-assets-storage)]
|
(let [system (update system ::sto/storage media/configure-assets-storage)]
|
||||||
(db/tx-run! system
|
(db/tx-run! system
|
||||||
(fn [system]
|
(fn [system]
|
||||||
(binding [*system* system]
|
(try
|
||||||
(fsnap/take-file-snapshot! system {:file-id file-id :label "migration/components-v2"})
|
(binding [*system* system]
|
||||||
(process-file system file-id :validate? validate?)))))
|
(fsnap/take-file-snapshot! system {:file-id file-id :label "migration/components-v2"})
|
||||||
|
(process-file system file-id :validate? validate?))
|
||||||
|
|
||||||
|
(catch Throwable cause
|
||||||
|
(let [team-id *team-id*]
|
||||||
|
(l/wrn :hint "error on processing file"
|
||||||
|
:team-id (str team-id)
|
||||||
|
:file-id (str file-id))
|
||||||
|
(throw cause)))))))
|
||||||
|
|
||||||
(finally
|
(finally
|
||||||
(let [elapsed (tpoint)
|
(let [elapsed (tpoint)
|
||||||
components (get @*file-stats* :processed/components 0)
|
components (get @*file-stats* :processed/components 0)
|
||||||
|
@ -917,6 +965,7 @@
|
||||||
:file-id (str file-id)
|
:file-id (str file-id)
|
||||||
:graphics graphics
|
:graphics graphics
|
||||||
:components components
|
:components components
|
||||||
|
:validate validate?
|
||||||
:elapsed (dt/format-duration elapsed))
|
:elapsed (dt/format-duration elapsed))
|
||||||
|
|
||||||
(some-> *stats* (swap! update :processed/files (fnil inc 0)))
|
(some-> *stats* (swap! update :processed/files (fnil inc 0)))
|
||||||
|
@ -929,6 +978,7 @@
|
||||||
:team-id (dm/str team-id))
|
:team-id (dm/str team-id))
|
||||||
|
|
||||||
(let [tpoint (dt/tpoint)
|
(let [tpoint (dt/tpoint)
|
||||||
|
err (volatile! false)
|
||||||
|
|
||||||
migrate-file
|
migrate-file
|
||||||
(fn [system file-id]
|
(fn [system file-id]
|
||||||
|
@ -948,7 +998,8 @@
|
||||||
|
|
||||||
(update-team-features! conn id features)))]
|
(update-team-features! conn id features)))]
|
||||||
|
|
||||||
(binding [*team-stats* (atom {})]
|
(binding [*team-stats* (atom {})
|
||||||
|
*team-id* team-id]
|
||||||
(try
|
(try
|
||||||
(db/tx-run! system (fn [system]
|
(db/tx-run! system (fn [system]
|
||||||
(db/exec-one! system ["SET idle_in_transaction_session_timeout = 0"])
|
(db/exec-one! system ["SET idle_in_transaction_session_timeout = 0"])
|
||||||
|
@ -956,6 +1007,10 @@
|
||||||
(if (contains? (:features team) "components/v2")
|
(if (contains? (:features team) "components/v2")
|
||||||
(l/inf :hint "team already migrated")
|
(l/inf :hint "team already migrated")
|
||||||
(migrate-team system team)))))
|
(migrate-team system team)))))
|
||||||
|
(catch Throwable cause
|
||||||
|
(vreset! err true)
|
||||||
|
(throw cause))
|
||||||
|
|
||||||
(finally
|
(finally
|
||||||
(let [elapsed (tpoint)
|
(let [elapsed (tpoint)
|
||||||
components (get @*team-stats* :processed/components 0)
|
components (get @*team-stats* :processed/components 0)
|
||||||
|
@ -964,9 +1019,21 @@
|
||||||
|
|
||||||
(some-> *stats* (swap! update :processed/teams (fnil inc 0)))
|
(some-> *stats* (swap! update :processed/teams (fnil inc 0)))
|
||||||
|
|
||||||
(l/dbg :hint "migrate:team:end"
|
(if (cache/cache? *cache*)
|
||||||
:team-id (dm/str team-id)
|
(let [cache-stats (cache/stats *cache*)]
|
||||||
:files files
|
(l/dbg :hint "migrate:team:end"
|
||||||
:components components
|
:team-id (dm/str team-id)
|
||||||
:graphics graphics
|
:files files
|
||||||
:elapsed (dt/format-duration elapsed))))))))
|
:components components
|
||||||
|
:graphics graphics
|
||||||
|
:crt (mth/to-fixed (:hit-rate cache-stats) 2)
|
||||||
|
:crq (str (:req-count cache-stats))
|
||||||
|
:error @err
|
||||||
|
:elapsed (dt/format-duration elapsed)))
|
||||||
|
|
||||||
|
(l/dbg :hint "migrate:team:end"
|
||||||
|
:team-id (dm/str team-id)
|
||||||
|
:files files
|
||||||
|
:components components
|
||||||
|
:graphics graphics
|
||||||
|
:elapsed (dt/format-duration elapsed)))))))))
|
||||||
|
|
|
@ -10,10 +10,12 @@
|
||||||
[app.common.pprint :as pp]
|
[app.common.pprint :as pp]
|
||||||
[app.db :as db]
|
[app.db :as db]
|
||||||
[app.features.components-v2 :as feat]
|
[app.features.components-v2 :as feat]
|
||||||
|
[app.main :as main]
|
||||||
[app.svgo :as svgo]
|
[app.svgo :as svgo]
|
||||||
|
[app.util.cache :as cache]
|
||||||
[app.util.time :as dt]
|
[app.util.time :as dt]
|
||||||
|
[app.worker :as-alias wrk]
|
||||||
[cuerdas.core :as str]
|
[cuerdas.core :as str]
|
||||||
[promesa.core :as p]
|
|
||||||
[promesa.exec :as px]
|
[promesa.exec :as px]
|
||||||
[promesa.exec.semaphore :as ps]
|
[promesa.exec.semaphore :as ps]
|
||||||
[promesa.util :as pu]))
|
[promesa.util :as pu]))
|
||||||
|
@ -36,8 +38,7 @@
|
||||||
(fn [_ _ oldv newv]
|
(fn [_ _ oldv newv]
|
||||||
(when (not= (:processed/files oldv)
|
(when (not= (:processed/files oldv)
|
||||||
(:processed/files newv))
|
(:processed/files newv))
|
||||||
(let [completed (:processed/files newv)
|
(let [elapsed (tpoint)]
|
||||||
elapsed (tpoint)]
|
|
||||||
(l/dbg :hint "progress"
|
(l/dbg :hint "progress"
|
||||||
:completed (:processed/files newv)
|
:completed (:processed/files newv)
|
||||||
:elapsed (dt/format-duration elapsed))))))
|
:elapsed (dt/format-duration elapsed))))))
|
||||||
|
@ -56,71 +57,13 @@
|
||||||
:completed completed
|
:completed completed
|
||||||
:elapsed elapsed)))))
|
:elapsed elapsed)))))
|
||||||
|
|
||||||
(defn- get-total-files
|
(def ^:private sql:get-teams-1
|
||||||
[pool & {:keys [team-id]}]
|
"SELECT id, features
|
||||||
(if (some? team-id)
|
FROM team
|
||||||
(let [sql (str/concat
|
WHERE deleted_at IS NULL
|
||||||
"SELECT count(f.id) AS count FROM file AS f "
|
ORDER BY created_at DESC")
|
||||||
" JOIN project AS p ON (p.id = f.project_id) "
|
|
||||||
" WHERE p.team_id = ? AND f.deleted_at IS NULL "
|
|
||||||
" AND p.deleted_at IS NULL")
|
|
||||||
res (db/exec-one! pool [sql team-id])]
|
|
||||||
(:count res))
|
|
||||||
|
|
||||||
(let [sql (str/concat
|
(def ^:private sql:get-teams-2
|
||||||
"SELECT count(id) AS count FROM file "
|
|
||||||
" WHERE deleted_at IS NULL")
|
|
||||||
res (db/exec-one! pool [sql])]
|
|
||||||
(:count res))))
|
|
||||||
|
|
||||||
(defn- get-total-teams
|
|
||||||
[pool]
|
|
||||||
(let [sql (str/concat
|
|
||||||
"SELECT count(id) AS count FROM team "
|
|
||||||
" WHERE deleted_at IS NULL")
|
|
||||||
res (db/exec-one! pool [sql])]
|
|
||||||
(:count res)))
|
|
||||||
|
|
||||||
(defn- mark-team-migration!
|
|
||||||
[{:keys [::db/pool]} team-id]
|
|
||||||
;; We execute this out of transaction because we want this
|
|
||||||
;; change to be visible to all other sessions before starting
|
|
||||||
;; the migration
|
|
||||||
(let [sql (str "UPDATE team SET features = "
|
|
||||||
" array_append(features, 'ephimeral/v2-migration') "
|
|
||||||
" WHERE id = ?")]
|
|
||||||
(db/exec-one! pool [sql team-id])))
|
|
||||||
|
|
||||||
(defn- unmark-team-migration!
|
|
||||||
[{:keys [::db/pool]} team-id]
|
|
||||||
;; We execute this out of transaction because we want this
|
|
||||||
;; change to be visible to all other sessions before starting
|
|
||||||
;; the migration
|
|
||||||
(let [sql (str "UPDATE team SET features = "
|
|
||||||
" array_remove(features, 'ephimeral/v2-migration') "
|
|
||||||
" WHERE id = ?")]
|
|
||||||
(db/exec-one! pool [sql team-id])))
|
|
||||||
|
|
||||||
;; (def ^:private sql:get-teams
|
|
||||||
;; "SELECT id, features
|
|
||||||
;; FROM team
|
|
||||||
;; WHERE deleted_at IS NULL
|
|
||||||
;; ORDER BY created_at DESC")
|
|
||||||
|
|
||||||
;; (def ^:private sql:get-teams
|
|
||||||
;; "SELECT t.id, t.features,
|
|
||||||
;; (SELECT count(*)
|
|
||||||
;; FROM file_media_object AS fmo
|
|
||||||
;; JOIN file AS f ON (f.id = fmo.file_id)
|
|
||||||
;; JOIN project AS p ON (p.id = f.project_id)
|
|
||||||
;; WHERE p.team_id = t.id
|
|
||||||
;; AND fmo.mtype = 'image/svg+xml'
|
|
||||||
;; AND fmo.is_local = false) AS graphics
|
|
||||||
;; FROM team AS t
|
|
||||||
;; ORDER BY t.created_at DESC")
|
|
||||||
|
|
||||||
|
|
||||||
(def ^:private sql:get-teams
|
|
||||||
"WITH teams AS (
|
"WITH teams AS (
|
||||||
SELECT t.id, t.features,
|
SELECT t.id, t.features,
|
||||||
(SELECT count(*)
|
(SELECT count(*)
|
||||||
|
@ -136,20 +79,37 @@
|
||||||
SELECT * FROM teams ")
|
SELECT * FROM teams ")
|
||||||
|
|
||||||
(defn- read-pred
|
(defn- read-pred
|
||||||
[[op val field]]
|
[entries]
|
||||||
(let [field (name field)]
|
(let [entries (if (and (vector? entries)
|
||||||
(case op
|
(keyword? (first entries)))
|
||||||
:lt [(str/ffmt "WHERE % < ?" field) val]
|
[entries]
|
||||||
:lte [(str/ffmt "WHERE % <= ?" field) val]
|
entries)]
|
||||||
:gt [(str/ffmt "WHERE % > ?" field) val]
|
(loop [params []
|
||||||
:gte [(str/ffmt "WHERE % >= ?" field) val]
|
queries []
|
||||||
:eq [(str/ffmt "WHERE % = ?" field) val]
|
entries (seq entries)]
|
||||||
[""])))
|
(if-let [[op val field] (first entries)]
|
||||||
|
(let [field (name field)
|
||||||
|
cond (case op
|
||||||
|
:lt (str/ffmt "% < ?" field)
|
||||||
|
:lte (str/ffmt "% <= ?" field)
|
||||||
|
:gt (str/ffmt "% > ?" field)
|
||||||
|
:gte (str/ffmt "% >= ?" field)
|
||||||
|
:eq (str/ffmt "% = ?" field))]
|
||||||
|
(recur (conj params val)
|
||||||
|
(conj queries cond)
|
||||||
|
(rest entries)))
|
||||||
|
|
||||||
|
(let [sql (apply str "WHERE " (str/join " AND " queries))]
|
||||||
|
(apply vector sql params))))))
|
||||||
|
|
||||||
(defn- get-teams
|
(defn- get-teams
|
||||||
[conn pred]
|
[conn pred]
|
||||||
(let [[sql & params] (read-pred pred)]
|
(let [sql (if pred
|
||||||
(->> (db/cursor conn (apply vector (str sql:get-teams sql) params))
|
(let [[sql & params] (read-pred pred)]
|
||||||
|
(apply vector (str sql:get-teams-2 sql) params))
|
||||||
|
[sql:get-teams-1])]
|
||||||
|
|
||||||
|
(->> (db/cursor conn sql)
|
||||||
(map feat/decode-row)
|
(map feat/decode-row)
|
||||||
(remove (fn [{:keys [features]}]
|
(remove (fn [{:keys [features]}]
|
||||||
(or (contains? features "ephimeral/v2-migration")
|
(or (contains? features "ephimeral/v2-migration")
|
||||||
|
@ -162,8 +122,7 @@
|
||||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||||
|
|
||||||
(defn migrate-file!
|
(defn migrate-file!
|
||||||
[system file-id & {:keys [rollback?] :or {rollback? true}}]
|
[file-id & {:keys [rollback? validate?] :or {rollback? true validate? false}}]
|
||||||
|
|
||||||
(l/dbg :hint "migrate:start" :rollback rollback?)
|
(l/dbg :hint "migrate:start" :rollback rollback?)
|
||||||
(let [tpoint (dt/tpoint)
|
(let [tpoint (dt/tpoint)
|
||||||
file-id (if (string? file-id)
|
file-id (if (string? file-id)
|
||||||
|
@ -171,8 +130,8 @@
|
||||||
file-id)]
|
file-id)]
|
||||||
(binding [feat/*stats* (atom {})]
|
(binding [feat/*stats* (atom {})]
|
||||||
(try
|
(try
|
||||||
(-> (assoc system ::db/rollback rollback?)
|
(-> (assoc main/system ::db/rollback rollback?)
|
||||||
(feat/migrate-file! file-id))
|
(feat/migrate-file! file-id :validate? validate?))
|
||||||
|
|
||||||
(-> (deref feat/*stats*)
|
(-> (deref feat/*stats*)
|
||||||
(assoc :elapsed (dt/format-duration (tpoint))))
|
(assoc :elapsed (dt/format-duration (tpoint))))
|
||||||
|
@ -185,12 +144,10 @@
|
||||||
(l/dbg :hint "migrate:end" :rollback rollback? :elapsed elapsed)))))))
|
(l/dbg :hint "migrate:end" :rollback rollback? :elapsed elapsed)))))))
|
||||||
|
|
||||||
(defn migrate-team!
|
(defn migrate-team!
|
||||||
[{:keys [::db/pool] :as system} team-id & {:keys [rollback? skip-on-graphic-error? validate? skip-mark?]
|
[team-id & {:keys [rollback? skip-on-graphic-error? validate?]
|
||||||
:or {rollback? true
|
:or {rollback? true
|
||||||
validate? true
|
validate? true
|
||||||
skip-on-graphic-error? false
|
skip-on-graphic-error? false}}]
|
||||||
skip-mark? false}
|
|
||||||
:as opts}]
|
|
||||||
|
|
||||||
(l/dbg :hint "migrate:start" :rollback rollback?)
|
(l/dbg :hint "migrate:start" :rollback rollback?)
|
||||||
|
|
||||||
|
@ -204,10 +161,7 @@
|
||||||
|
|
||||||
(binding [feat/*stats* stats]
|
(binding [feat/*stats* stats]
|
||||||
(try
|
(try
|
||||||
(when-not skip-mark?
|
(-> (assoc main/system ::db/rollback rollback?)
|
||||||
(mark-team-migration! system team-id))
|
|
||||||
|
|
||||||
(-> (assoc system ::db/rollback rollback?)
|
|
||||||
(feat/migrate-team! team-id
|
(feat/migrate-team! team-id
|
||||||
:validate? validate?
|
:validate? validate?
|
||||||
:skip-on-graphics-error? skip-on-graphic-error?))
|
:skip-on-graphics-error? skip-on-graphic-error?))
|
||||||
|
@ -219,9 +173,6 @@
|
||||||
(l/dbg :hint "migrate:error" :cause cause))
|
(l/dbg :hint "migrate:error" :cause cause))
|
||||||
|
|
||||||
(finally
|
(finally
|
||||||
(when-not skip-mark?
|
|
||||||
(unmark-team-migration! system team-id))
|
|
||||||
|
|
||||||
(let [elapsed (dt/format-duration (tpoint))]
|
(let [elapsed (dt/format-duration (tpoint))]
|
||||||
(l/dbg :hint "migrate:end" :rollback rollback? :elapsed elapsed)))))))
|
(l/dbg :hint "migrate:end" :rollback rollback? :elapsed elapsed)))))))
|
||||||
|
|
||||||
|
@ -232,28 +183,31 @@
|
||||||
until thw maximum number of jobs is reached which by default has the
|
until thw maximum number of jobs is reached which by default has the
|
||||||
value of `1`. This is controled with the `:max-jobs` option."
|
value of `1`. This is controled with the `:max-jobs` option."
|
||||||
|
|
||||||
[{:keys [::db/pool] :as system} & {:keys [max-jobs max-items max-time
|
[& {:keys [max-jobs max-items max-time rollback? validate?
|
||||||
rollback? validate? preset
|
pred max-procs cache on-start on-progress on-error on-end
|
||||||
pred max-procs skip-mark?
|
skip-on-graphic-error?]
|
||||||
on-start on-progress on-error on-end]
|
:or {validate? false
|
||||||
:or {validate? true
|
rollback? true
|
||||||
rollback? true
|
max-jobs 1
|
||||||
preset :shutdown-on-failure
|
skip-on-graphic-error? true
|
||||||
skip-mark? true
|
max-items Long/MAX_VALUE}}]
|
||||||
max-jobs 1
|
|
||||||
max-items Long/MAX_VALUE}
|
|
||||||
:as opts}]
|
|
||||||
|
|
||||||
(let [stats (atom {})
|
(let [stats (atom {})
|
||||||
tpoint (dt/tpoint)
|
tpoint (dt/tpoint)
|
||||||
mtime (some-> max-time dt/duration)
|
mtime (some-> max-time dt/duration)
|
||||||
|
|
||||||
factory (px/thread-factory :virtual false :prefix "penpot/migration/compv2/")
|
factory (px/thread-factory :virtual false :prefix "penpot/migration/")
|
||||||
executor (px/cached-executor :factory factory)
|
executor (px/cached-executor :factory factory)
|
||||||
|
|
||||||
max-procs (or max-procs max-jobs)
|
max-procs (or max-procs max-jobs)
|
||||||
sjobs (ps/create :permits max-jobs)
|
sjobs (ps/create :permits max-jobs)
|
||||||
sprocs (ps/create :permits max-procs)
|
sprocs (ps/create :permits max-procs)
|
||||||
|
|
||||||
|
cache (if (int? cache)
|
||||||
|
(cache/create :executor (::wrk/executor main/system)
|
||||||
|
:max-items cache)
|
||||||
|
nil)
|
||||||
|
|
||||||
migrate-team
|
migrate-team
|
||||||
(fn [team-id]
|
(fn [team-id]
|
||||||
(ps/acquire! sjobs)
|
(ps/acquire! sjobs)
|
||||||
|
@ -268,18 +222,18 @@
|
||||||
|
|
||||||
(px/run! executor (fn []
|
(px/run! executor (fn []
|
||||||
(try
|
(try
|
||||||
(when-not skip-mark?
|
(-> (assoc main/system ::db/rollback rollback?)
|
||||||
(mark-team-migration! system team-id))
|
(feat/migrate-team! team-id
|
||||||
(-> (assoc system ::db/rollback rollback?)
|
:validate? validate?
|
||||||
(feat/migrate-team! team-id :validate? validate?))
|
:skip-on-graphics-error? skip-on-graphic-error?))
|
||||||
|
|
||||||
(catch Throwable cause
|
(catch Throwable cause
|
||||||
(l/err :hint "unexpected error on processing team (skiping)"
|
(l/wrn :hint "unexpected error on processing team (skiping)"
|
||||||
:team-id (str team-id)
|
:team-id (str team-id)
|
||||||
:cause cause))
|
:cause cause))
|
||||||
|
|
||||||
(finally
|
(finally
|
||||||
(ps/release! sjobs)
|
(ps/release! sjobs))))))))]
|
||||||
(when-not skip-mark?
|
|
||||||
(unmark-team-migration! system team-id)))))))))]
|
|
||||||
|
|
||||||
(l/dbg :hint "migrate:start"
|
(l/dbg :hint "migrate:start"
|
||||||
:rollback rollback?
|
:rollback rollback?
|
||||||
|
@ -289,14 +243,17 @@
|
||||||
(add-watch stats :progress-report (report-progress-teams tpoint on-progress))
|
(add-watch stats :progress-report (report-progress-teams tpoint on-progress))
|
||||||
|
|
||||||
(binding [feat/*stats* stats
|
(binding [feat/*stats* stats
|
||||||
|
feat/*cache* cache
|
||||||
svgo/*semaphore* sprocs]
|
svgo/*semaphore* sprocs]
|
||||||
(try
|
(try
|
||||||
(when (fn? on-start)
|
(when (fn? on-start)
|
||||||
(on-start {:rollback rollback?}))
|
(on-start {:rollback rollback?}))
|
||||||
|
|
||||||
(db/tx-run! system
|
(db/tx-run! main/system
|
||||||
(fn [{:keys [::db/conn]}]
|
(fn [{:keys [::db/conn]}]
|
||||||
(run! (partial migrate-team)
|
(db/exec! conn ["SET statement_timeout = 0;"])
|
||||||
|
(db/exec! conn ["SET idle_in_transaction_session_timeout = 0;"])
|
||||||
|
(run! migrate-team
|
||||||
(->> (get-teams conn pred)
|
(->> (get-teams conn pred)
|
||||||
(take max-items)))))
|
(take max-items)))))
|
||||||
|
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue