3pp-mirror/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-05-22 16:06:11 +02:00
Code Issues Projects Releases Packages Wiki Activity

✨ Add optional validation when migrating files to components-v2

Browse source
This commit is contained in:
Andrés Moya 2023-11-13 17:24:43 +01:00 • committed by Andrey Antukh
parent 3dfd54d8e2
commit 451d6c1d7b
2 changed files with 20 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

15
backend/src/app/features/components_v2.clj
View file

@ -13,6 +13,7 @@
[app.common.files.libraries-helpers :as cflh] [app.common.files.libraries-helpers :as cflh]
[app.common.files.migrations :as pmg] [app.common.files.migrations :as pmg]
[app.common.files.shapes-helpers :as cfsh] [app.common.files.shapes-helpers :as cfsh]
[app.common.files.validate :as cfv]
[app.common.geom.point :as gpt] [app.common.geom.point :as gpt]
[app.common.geom.rect :as grc] [app.common.geom.rect :as grc]
[app.common.geom.shapes :as gsh] [app.common.geom.shapes :as gsh]
@ -623,7 +624,7 @@
(update fdata :options assoc :components-v2 true))))) (update fdata :options assoc :components-v2 true)))))
(defn- process-file (defn- process-file
[{:keys [id] :as file}] [{:keys [id] :as file} & {:keys [validate?]}]
(let [conn (::db/conn *system*)] (let [conn (::db/conn *system*)]
(binding [pmap/*tracked* (atom {}) (binding [pmap/*tracked* (atom {})
pmap/*load-fn* (partial files/load-pointer conn id) pmap/*load-fn* (partial files/load-pointer conn id)
@ -659,10 +660,13 @@
:revn (:revn file)} :revn (:revn file)}
{:id (:id file)}) {:id (:id file)})
(when validate?
(cfv/validate-file file libs :throw? true))
(dissoc file :data))))) (dissoc file :data)))))
(defn migrate-file! (defn migrate-file!
[system file-id] [system file-id & {:keys [validate?]}]
(let [tpoint (dt/tpoint) (let [tpoint (dt/tpoint)
file-id (if (string? file-id) file-id (if (string? file-id)
(parse-uuid file-id) (parse-uuid file-id)
@ -678,7 +682,7 @@
(binding [*system* system] (binding [*system* system]
(-> (db/get conn :file {:id file-id}) (-> (db/get conn :file {:id file-id})
(update :features db/decode-pgarray #{}) (update :features db/decode-pgarray #{})
(process-file)))))) (process-file :validate? validate?))))))
(finally (finally
(let [elapsed (tpoint) (let [elapsed (tpoint)
@ -701,9 +705,8 @@
(assoc :elapsed/total-by-file total) (assoc :elapsed/total-by-file total)
(assoc :processed/files completed))))))))))) (assoc :processed/files completed)))))))))))
(defn migrate-team! (defn migrate-team!
[system team-id] [system team-id & {:keys [validate?]}]
(let [tpoint (dt/tpoint) (let [tpoint (dt/tpoint)
team-id (if (string? team-id) team-id (if (string? team-id)
(parse-uuid team-id) (parse-uuid team-id)
@ -737,7 +740,7 @@
rows (->> (db/exec! conn [sql team-id]) rows (->> (db/exec! conn [sql team-id])
(map :id))] (map :id))]
(run! (partial migrate-file! system) rows) (run! #(migrate-file! system % :validate? validate?) rows)
(some-> *stats* (swap! assoc :current/files (count rows))) (some-> *stats* (swap! assoc :current/files (count rows)))
(let [features (-> features (let [features (-> features

20
backend/src/app/srepl/components_v2.clj
View file

@ -110,13 +110,14 @@
(l/dbg :hint "migrate:end" :elapsed elapsed)))))) (l/dbg :hint "migrate:end" :elapsed elapsed))))))
(defn migrate-files! (defn migrate-files!
[{:keys [::db/pool] :as system} & {:keys [chunk-size max-jobs max-items start-at preset rollback skip-on-error] [{:keys [::db/pool] :as system} & {:keys [chunk-size max-jobs max-items start-at preset rollback skip-on-error validate]
:or {chunk-size 10 :or {chunk-size 10
skip-on-error true skip-on-error true
max-jobs 10 max-jobs 10
max-items Long/MAX_VALUE max-items Long/MAX_VALUE
preset :shutdown-on-failure preset :shutdown-on-failure
rollback true}}] rollback true
validate false}}]
(letfn [(get-chunk [cursor] (letfn [(get-chunk [cursor]
(let [sql (str/concat (let [sql (str/concat
"SELECT id, created_at FROM file " "SELECT id, created_at FROM file "
@ -151,7 +152,7 @@
(ps/acquire! feat/*semaphore*) (ps/acquire! feat/*semaphore*)
(px/submit! scope (fn [] (px/submit! scope (fn []
(-> (assoc system ::db/rollback rollback) (-> (assoc system ::db/rollback rollback)
(feat/migrate-file! file-id))))) (feat/migrate-file! file-id :validate? validate)))))
(get-candidates)) (get-candidates))
(p/await! scope)) (p/await! scope))
@ -171,8 +172,8 @@
(defn migrate-team! (defn migrate-team!
[{:keys [::db/pool] :as system} team-id [{:keys [::db/pool] :as system} team-id
& {:keys [rollback skip-on-error] & {:keys [rollback skip-on-error validate]
:or {rollback true skip-on-error true}}] :or {rollback true skip-on-error true validate false}}]
(l/dbg :hint "migrate:start") (l/dbg :hint "migrate:start")
(let [total (get-total-files pool :team-id team-id) (let [total (get-total-files pool :team-id team-id)
@ -185,7 +186,7 @@
(binding [feat/*stats* stats (binding [feat/*stats* stats
feat/*skip-on-error* skip-on-error] feat/*skip-on-error* skip-on-error]
(-> (assoc system ::db/rollback rollback) (-> (assoc system ::db/rollback rollback)
(feat/migrate-team! team-id)) (feat/migrate-team! team-id :validate? validate))
(print-stats! (print-stats!
(-> (deref feat/*stats*) (-> (deref feat/*stats*)
@ -203,13 +204,14 @@
(defn migrate-teams! (defn migrate-teams!
[{:keys [::db/pool] :as system} [{:keys [::db/pool] :as system}
& {:keys [chunk-size max-jobs max-items start-at rollback preset skip-on-error max-time] & {:keys [chunk-size max-jobs max-items start-at rollback preset skip-on-error max-time validate]
:or {chunk-size 10000 :or {chunk-size 10000
rollback true rollback true
skip-on-error true skip-on-error true
preset :shutdown-on-failure preset :shutdown-on-failure
max-jobs Integer/MAX_VALUE max-jobs Integer/MAX_VALUE
max-items Long/MAX_VALUE}}] max-items Long/MAX_VALUE
validate false}}]
(letfn [(get-chunk [cursor] (letfn [(get-chunk [cursor]
(let [sql (str/concat (let [sql (str/concat
@ -232,7 +234,7 @@
(migrate-team [team-id] (migrate-team [team-id]
(try (try
(-> (assoc system ::db/rollback rollback) (-> (assoc system ::db/rollback rollback)
(feat/migrate-team! team-id)) (feat/migrate-team! team-id :validate? validate))
(catch Throwable cause (catch Throwable cause
(l/err :hint "unexpected error on processing team" :team-id (dm/str team-id) :cause cause)))) (l/err :hint "unexpected error on processing team" :team-id (dm/str team-id) :cause cause))))