3pp-mirror/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-05-26 01:36:13 +02:00
Code Issues Projects Releases Packages Wiki Activity

♻️ Add minor refactor to file migrations

Browse source 
Relevant changes:

- Add the ability to create migration in both directions, defaulting
  to identity if not provided
- Move the version attribute to file table column for to make it more
  accessible (previously it was on data blob)
- Reduce db update operations on file-update rpc method
This commit is contained in:
Andrey Antukh 2024-02-14 11:37:27 +01:00
parent 7ac4b89a0e
commit b718a282e0
16 changed files with 240 additions and 151 deletions
Download patch file Download diff file Expand all files Collapse all files

32
backend/src/app/binfile/common.clj
View file

@ -12,7 +12,6 @@
[app.common.data.macros :as dm]
[app.common.exceptions :as ex]
[app.common.features :as cfeat]
[app.common.files.defaults :as cfd]
[app.common.files.migrations :as fmg]
[app.common.files.validate :as fval]
[app.common.logging :as l]
@ -381,23 +380,24 @@
(d/group-by first rest)
(reduce (partial process-group-of-assets) data)))))
(defn- fix-version
[file]
(let [file (fmg/fix-version file)]
;; FIXME: We're temporarily activating all migrations because a
;; problem in the environments messed up with the version numbers
;; When this problem is fixed delete the following line
(if (> (:version file) 22)
(assoc file :version 22)
file)))
(defn process-file
[{:keys [id] :as file}]
(-> file
(fix-version)
(update :data (fn [fdata]
(-> fdata
(assoc :id id)
(dissoc :recent-colors)
(cond-> (> (:version fdata) cfd/version)
(assoc :version cfd/version))
;; FIXME: We're temporarily activating all
;; migrations because a problem in the
;; environments messed up with the version
;; numbers When this problem is fixed delete
;; the following line
(cond-> (> (:version fdata) 22)
(assoc :version 22)))))
(dissoc :recent-colors))))
(fmg/migrate-file)
(update :data (fn [fdata]
(-> fdata
@ -409,19 +409,21 @@
(defn- upsert-file!
[conn file]
(let [sql (str "INSERT INTO file (id, project_id, name, revn, is_shared, data, created_at, modified_at) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?) "
"ON CONFLICT (id) DO UPDATE SET data=?")]
(let [sql (str "INSERT INTO file (id, project_id, name, revn, version, is_shared, data, created_at, modified_at) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) "
"ON CONFLICT (id) DO UPDATE SET data=?, version=?")]
(db/exec-one! conn [sql
(:id file)
(:project-id file)
(:name file)
(:revn file)
(:version file)
(:is-shared file)
(:data file)
(:created-at file)
(:modified-at file)
(:data file)])))
(:data file)
(:version file)])))
(defn persist-file!
"Applies all the final validations and perist the file."

12
backend/src/app/features/components_v2.clj
View file

@ -1496,6 +1496,13 @@
fdata (migrate-graphics fdata)]
(update fdata :options assoc :components-v2 true)))))
(defn- fix-version
[file]
(let [file (fmg/fix-version file)]
(if (> (:version file) 22)
(assoc file :version 22)
file)))
(defn- get-file
[system id]
(binding [pmap/*load-fn* (partial fdata/load-pointer system id)]
@ -1506,10 +1513,7 @@
(update :data assoc :id id)
(update :data fdata/process-pointers deref)
(update :data fdata/process-objects (partial into {}))
(update :data (fn [data]
(if (> (:version data) 22)
(assoc data :version 22)
data)))
(fix-version)
(fmg/migrate-file))))
(defn get-team

8
backend/src/app/http/debug.clj
View file

@ -393,7 +393,7 @@
::rres/body (str/ffmt "PROFILE '%' ACTIVATED" (:email profile))}))))
(defn- reset-file-data-version
(defn- reset-file-version
[cfg {:keys [params] :as request}]
(let [file-id (some-> params :file-id d/parse-uuid)
version (some-> params :version d/parse-integer)]
@ -413,7 +413,7 @@
:code :invalid-version
:hint "provided invalid version"))
(db/tx-run! cfg srepl/process-file! file-id #(update % :data assoc :version version))
(db/tx-run! cfg srepl/process-file! file-id #(assoc % :version version))
{::rres/status 200
::rres/headers {"content-type" "text/plain"}
@ -486,8 +486,8 @@
["/error" {:handler (partial error-list-handler cfg)}]
["/actions/resend-email-verification"
{:handler (partial resend-email-notification cfg)}]
["/actions/reset-file-data-version"
{:handler (partial reset-file-data-version cfg)}]
["/actions/reset-file-version"
{:handler (partial reset-file-version cfg)}]
["/file/export" {:handler (partial export-handler cfg)}]
["/file/import" {:handler (partial import-handler cfg)}]
["/file/data" {:handler (partial file-data-handler cfg)}]

5
backend/src/app/migrations.clj
View file

@ -373,7 +373,10 @@
:fn (mg/resource "app/migrations/sql/0117-mod-file-object-thumbnail-table.sql")}
{:name "0118-mod-task-table"
:fn (mg/resource "app/migrations/sql/0118-mod-task-table.sql")}])
:fn (mg/resource "app/migrations/sql/0118-mod-task-table.sql")}
{:name "0119-mod-file-table"
:fn (mg/resource "app/migrations/sql/0119-mod-file-table.sql")}])
(defn apply-migrations!
[pool name migrations]

2
backend/src/app/migrations/sql/0119-mod-file-table.sql Normal file
View file

@ -0,0 +1,2 @@
ALTER TABLE file
ADD COLUMN version integer NULL;

10
backend/src/app/rpc/commands/files.clj
View file

@ -71,19 +71,14 @@
data (assoc :data (blob/decode data)))))
(defn check-version!
[{:keys [data] :as file}]
(dm/assert!
"expect data to be decoded"
(map? data))
(let [version (:version data 0)]
[file]
(let [version (:version file)]
(when (> version fmg/version)
(ex/raise :type :restriction
:code :file-version-not-supported
:hint "file version is greated that the maximum"
:file-version version
:max-version fmg/version))
file))
;; --- FILE PERMISSIONS
@ -252,6 +247,7 @@
(db/update! conn :file
{:data (blob/encode (:data file))
:version (:version file)
:features (db/create-array conn "text" (:features file))}
{:id id})

2
backend/src/app/rpc/commands/files_create.clj
View file

@ -9,6 +9,7 @@
[app.common.data :as d]
[app.common.data.macros :as dm]
[app.common.features :as cfeat]
[app.common.files.defaults :refer [version]]
[app.common.schema :as sm]
[app.common.types.file :as ctf]
[app.common.uuid :as uuid]
@ -61,6 +62,7 @@
:name name
:revn revn
:is-shared is-shared
:version version
:data data
:features features
:ignore-sync-until ignore-sync-until

15
backend/src/app/rpc/commands/files_update.clj
View file

@ -183,8 +183,8 @@
(l/trace :hint "update-file" :time (dt/format-duration elapsed))))))))))
(defn update-file
[{:keys [::db/conn ::mtx/metrics] :as cfg}
{:keys [id file features changes changes-with-metadata] :as params}]
[{:keys [::mtx/metrics] :as cfg}
{:keys [file features changes changes-with-metadata] :as params}]
(let [features (-> features
(set/difference cfeat/frontend-only-features)
(set/union (:features file)))
@ -207,12 +207,6 @@
(mtx/run! metrics {:id :update-file-changes :inc (count changes)})
(when (not= features (:features file))
(let [features (db/create-array conn "text" features)]
(db/update! conn :file
{:features features}
{:id id})))
(binding [cfeat/*current* features
cfeat/*previous* (:features file)]
(let [file (assoc file :features features)
@ -234,7 +228,8 @@
{:keys [profile-id file changes session-id ::created-at skip-validate] :as params}]
(let [;; Process the file data on separated thread for avoid to do
;; the CPU intensive operation on vthread.
file (px/invoke! executor (partial update-file-data cfg file changes skip-validate))]
file (px/invoke! executor (partial update-file-data cfg file changes skip-validate))
features (db/create-array conn "text" (:features file))]
(db/insert! conn :file-change
{:id (uuid/next)
@ -252,6 +247,8 @@
(db/update! conn :file
{:revn (:revn file)
:data (:data file)
:version (:version file)
:features features
:data-backend nil
:modified-at created-at
:has-media-trimmed false}

1
backend/src/app/srepl/helpers.clj
View file

@ -66,6 +66,7 @@
(db/update! conn :file
{:revn (:revn file)
:data (:data file)
:version (:version file)
:features (:features file)
:deleted-at (:deleted-at file)
:created-at (:created-at file)

5
backend/src/app/tasks/file_gc.clj
View file

@ -62,6 +62,8 @@
(db/update! conn :file
{:has-media-trimmed true
:features (:features file)
:version (:version file)
:data (:data file)}
{:id id})))
@ -82,6 +84,7 @@
"SELECT f.id,
f.data,
f.revn,
f.version,
f.features,
f.modified_at
FROM file AS f
@ -175,7 +178,7 @@
(def ^:private sql:get-files-for-library
"SELECT f.id, f.data, f.modified_at, f.features
"SELECT f.id, f.data, f.modified_at, f.features, f.version
FROM file AS f
LEFT JOIN file_library_rel AS fl ON (fl.file_id = f.id)
WHERE fl.library_file_id = ?