3pp-mirror/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-06-13 14:01:39 +02:00
Code Issues Projects Releases Packages Wiki Activity

✨ Add the ability to access libraries from file migrations

Browse source
This commit is contained in:
Pablo Alba 2025-05-27 13:12:34 +02:00 • committed by GitHub
parent c7c8e91183
commit 443cabe94e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 109 additions and 102 deletions
Download patch file Download diff file Expand all files Collapse all files

103
backend/src/app/binfile/common.clj
View file

@ -53,6 +53,7 @@
(* 1024 1024 100)) (* 1024 1024 100))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(declare get-resolved-file-libraries)
(def file-attrs (def file-attrs
#{:id #{:id
@ -143,11 +144,13 @@
(reduce #(index-object %1 %2 attr) index coll))) (reduce #(index-object %1 %2 attr) index coll)))
(defn decode-row (defn decode-row
"A generic decode row helper" [{:keys [data changes features] :as row}]
[{:keys [data features] :as row}] (when row
(cond-> row (cond-> row
features (assoc :features (db/decode-pgarray features #{})) features (assoc :features (db/decode-pgarray features #{}))
data (assoc :data (blob/decode data)))) changes (assoc :changes (blob/decode changes))
data (assoc :data (blob/decode data)))))
(defn decode-file (defn decode-file
"A general purpose file decoding function that resolves all external "A general purpose file decoding function that resolves all external
@ -156,7 +159,8 @@
(binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)] (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
(let [file (->> file (let [file (->> file
(feat.fmigr/resolve-applied-migrations cfg) (feat.fmigr/resolve-applied-migrations cfg)
(feat.fdata/resolve-file-data cfg))] (feat.fdata/resolve-file-data cfg))
libs (delay (get-resolved-file-libraries cfg file))]
(-> file (-> file
(update :features db/decode-pgarray #{}) (update :features db/decode-pgarray #{})
@ -164,7 +168,7 @@
(update :data feat.fdata/process-pointers deref) (update :data feat.fdata/process-pointers deref)
(update :data feat.fdata/process-objects (partial into {})) (update :data feat.fdata/process-objects (partial into {}))
(update :data assoc :id id) (update :data assoc :id id)
(fmg/migrate-file))))) (fmg/migrate-file libs)))))
(defn get-file (defn get-file
"Get file, resolve all features and apply migrations. "Get file, resolve all features and apply migrations.
@ -418,26 +422,27 @@
(db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"]))) (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])))
(defn process-file (defn process-file
[{:keys [id] :as file}] [cfg {:keys [id] :as file}]
(-> file (let [libs (delay (get-resolved-file-libraries cfg file))]
(update :data (fn [fdata] (-> file
(-> fdata (update :data (fn [fdata]
(assoc :id id) (-> fdata
(dissoc :recent-colors)))) (assoc :id id)
(fmg/migrate-file) (dissoc :recent-colors))))
(update :data (fn [fdata] (fmg/migrate-file libs)
(-> fdata (update :data (fn [fdata]
(update :pages-index relink-shapes) (-> fdata
(update :components relink-shapes) (update :pages-index relink-shapes)
(update :media relink-media) (update :components relink-shapes)
(update :colors relink-colors) (update :media relink-media)
(d/without-nils)))) (update :colors relink-colors)
(d/without-nils))))
;; NOTE: this is necessary because when we just creating a new ;; NOTE: this is necessary because when we just creating a new
;; file from imported artifact or cloned file there are no ;; file from imported artifact or cloned file there are no
;; migrations registered on the database, so we need to persist ;; migrations registered on the database, so we need to persist
;; all of them, not only the applied ;; all of them, not only the applied
(vary-meta dissoc ::fmg/migrated))) (vary-meta dissoc ::fmg/migrated))))
(defn encode-file (defn encode-file
[{:keys [::db/conn] :as cfg} {:keys [id features] :as file}] [{:keys [::db/conn] :as cfg} {:keys [id features] :as file}]
@ -528,3 +533,49 @@
(l/error :hint "file schema validation error" :cause result)))) (l/error :hint "file schema validation error" :cause result))))
(insert-file! cfg file opts))) (insert-file! cfg file opts)))
(def ^:private sql:get-file-libraries
"WITH RECURSIVE libs AS (
SELECT fl.*, flr.synced_at
FROM file AS fl
JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
WHERE flr.file_id = ?::uuid
UNION
SELECT fl.*, flr.synced_at
FROM file AS fl
JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
JOIN libs AS l ON (flr.file_id = l.id)
)
SELECT l.id,
l.features,
l.project_id,
p.team_id,
l.created_at,
l.modified_at,
l.deleted_at,
l.name,
l.revn,
l.vern,
l.synced_at,
l.is_shared
FROM libs AS l
INNER JOIN project AS p ON (p.id = l.project_id)
WHERE l.deleted_at IS NULL OR l.deleted_at > now();")
(defn get-file-libraries
[conn file-id]
(into []
(comp
;; FIXME: :is-indirect set to false to all rows looks
;; completly useless
(map #(assoc % :is-indirect false))
(map decode-row))
(db/exec! conn [sql:get-file-libraries file-id])))
(defn get-resolved-file-libraries
"A helper for preload file libraries"
[{:keys [::db/conn] :as cfg} file]
(->> (get-file-libraries conn (:id file))
(into [file] (map #(get-file cfg (:id %))))
(d/index-by :id)))

4
backend/src/app/binfile/v1.clj
View file

@ -551,8 +551,8 @@
(cond-> (and (= idx 0) (some? name)) (cond-> (and (= idx 0) (some? name))
(assoc :name name)) (assoc :name name))
(assoc :project-id project-id) (assoc :project-id project-id)
(dissoc :thumbnails) (dissoc :thumbnails))
(bfc/process-file))] file (bfc/process-file system file)]
;; All features that are enabled and requires explicit migration are ;; All features that are enabled and requires explicit migration are
;; added to the state for a posterior migration step. ;; added to the state for a posterior migration step.

4
backend/src/app/binfile/v2.clj
View file

@ -281,8 +281,8 @@
(let [file (-> (read-obj cfg :file file-id) (let [file (-> (read-obj cfg :file file-id)
(update :id bfc/lookup-index) (update :id bfc/lookup-index)
(update :project-id bfc/lookup-index) (update :project-id bfc/lookup-index))
(bfc/process-file))] file (bfc/process-file cfg file)]
(events/tap :progress (events/tap :progress
{:op :import {:op :import

5
backend/src/app/binfile/v3.clj
View file

@ -754,8 +754,9 @@
(assoc :data data) (assoc :data data)
(assoc :name file-name) (assoc :name file-name)
(assoc :project-id project-id) (assoc :project-id project-id)
(dissoc :options) (dissoc :options))
(bfc/process-file))]
file (bfc/process-file cfg file)]
(bfm/register-pending-migrations! cfg file) (bfm/register-pending-migrations! cfg file)
(bfc/save-file! cfg file ::db/return-keys false) (bfc/save-file! cfg file ::db/return-keys false)

46
backend/src/app/rpc/commands/files.clj
View file

@ -6,6 +6,7 @@
(ns app.rpc.commands.files (ns app.rpc.commands.files
(:require (:require
[app.binfile.common :as bfc]
[app.common.data :as d] [app.common.data :as d]
[app.common.data.macros :as dm] [app.common.data.macros :as dm]
[app.common.exceptions :as ex] [app.common.exceptions :as ex]
@ -211,7 +212,8 @@
[{:keys [::db/conn] :as cfg} {:keys [id] :as file} {:keys [read-only?]}] [{:keys [::db/conn] :as cfg} {:keys [id] :as file} {:keys [read-only?]}]
(binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id) (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)
pmap/*tracked* (pmap/create-tracked)] pmap/*tracked* (pmap/create-tracked)]
(let [;; For avoid unnecesary overhead of creating multiple pointers and (let [libs (delay (bfc/get-resolved-file-libraries cfg file))
;; For avoid unnecesary overhead of creating multiple pointers and
;; handly internally with objects map in their worst case (when ;; handly internally with objects map in their worst case (when
;; probably all shapes and all pointers will be readed in any ;; probably all shapes and all pointers will be readed in any
;; case), we just realize/resolve them before applying the ;; case), we just realize/resolve them before applying the
@ -219,7 +221,7 @@
file (-> file file (-> file
(update :data feat.fdata/process-pointers deref) (update :data feat.fdata/process-pointers deref)
(update :data feat.fdata/process-objects (partial into {})) (update :data feat.fdata/process-objects (partial into {}))
(fmg/migrate-file))] (fmg/migrate-file libs))]
(if (or read-only? (db/read-only? conn)) (if (or read-only? (db/read-only? conn))
file file
@ -615,44 +617,6 @@
;; --- COMMAND QUERY: get-file-libraries ;; --- COMMAND QUERY: get-file-libraries
(def ^:private sql:get-file-libraries
"WITH RECURSIVE libs AS (
SELECT fl.*, flr.synced_at
FROM file AS fl
JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
WHERE flr.file_id = ?::uuid
UNION
SELECT fl.*, flr.synced_at
FROM file AS fl
JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
JOIN libs AS l ON (flr.file_id = l.id)
)
SELECT l.id,
l.features,
l.project_id,
p.team_id,
l.created_at,
l.modified_at,
l.deleted_at,
l.name,
l.revn,
l.vern,
l.synced_at,
l.is_shared
FROM libs AS l
INNER JOIN project AS p ON (p.id = l.project_id)
WHERE l.deleted_at IS NULL OR l.deleted_at > now();")
(defn get-file-libraries
[conn file-id]
(into []
(comp
;; FIXME: :is-indirect set to false to all rows looks
;; completly useless
(map #(assoc % :is-indirect false))
(map decode-row))
(db/exec! conn [sql:get-file-libraries file-id])))
(def ^:private schema:get-file-libraries (def ^:private schema:get-file-libraries
[:map {:title "get-file-libraries"} [:map {:title "get-file-libraries"}
[:file-id ::sm/uuid]]) [:file-id ::sm/uuid]])
@ -664,7 +628,7 @@
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id]}] [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id]}]
(dm/with-open [conn (db/open pool)] (dm/with-open [conn (db/open pool)]
(check-read-permissions! conn profile-id file-id) (check-read-permissions! conn profile-id file-id)
(get-file-libraries conn file-id))) (bfc/get-file-libraries conn file-id)))
;; --- COMMAND QUERY: Files that use this File library ;; --- COMMAND QUERY: Files that use this File library

12
backend/src/app/rpc/commands/files_update.clj
View file

@ -340,6 +340,7 @@
(-> data (-> data
(blob/decode) (blob/decode)
(assoc :id (:id file))))) (assoc :id (:id file)))))
libs (delay (bfc/get-resolved-file-libraries cfg file))
;; For avoid unnecesary overhead of creating multiple pointers ;; For avoid unnecesary overhead of creating multiple pointers
;; and handly internally with objects map in their worst ;; and handly internally with objects map in their worst
@ -350,7 +351,7 @@
(-> file (-> file
(update :data feat.fdata/process-pointers deref) (update :data feat.fdata/process-pointers deref)
(update :data feat.fdata/process-objects (partial into {})) (update :data feat.fdata/process-objects (partial into {}))
(fmg/migrate-file)) (fmg/migrate-file libs))
file) file)
file (apply update-fn cfg file args) file (apply update-fn cfg file args)
@ -379,13 +380,6 @@
(bfc/encode-file cfg file)))) (bfc/encode-file cfg file))))
(defn- get-file-libraries
"A helper for preload file libraries, mainly used for perform file
semantical and structural validation"
[{:keys [::db/conn] :as cfg} file]
(->> (files/get-file-libraries conn (:id file))
(into [file] (map #(bfc/get-file cfg (:id %))))
(d/index-by :id)))
(defn- soft-validate-file-schema! (defn- soft-validate-file-schema!
[file] [file]
@ -411,7 +405,7 @@
(when (and (or (contains? cf/flags :file-validation) (when (and (or (contains? cf/flags :file-validation)
(contains? cf/flags :soft-file-validation)) (contains? cf/flags :soft-file-validation))
(not skip-validate)) (not skip-validate))
(get-file-libraries cfg file)) (bfc/get-resolved-file-libraries cfg file))
;; The main purpose of this atom is provide a contextual state ;; The main purpose of this atom is provide a contextual state

2
backend/src/app/rpc/commands/management.clj
View file

@ -56,7 +56,7 @@
(vswap! bfc/*state* update :index bfc/update-index fmeds :id) (vswap! bfc/*state* update :index bfc/update-index fmeds :id)
;; Process and persist file ;; Process and persist file
(let [file (bfc/process-file file)] (let [file (bfc/process-file cfg file)]
(bfc/insert-file! cfg file ::db/return-keys false) (bfc/insert-file! cfg file ::db/return-keys false)
;; The file profile creation is optional, so when no profile is ;; The file profile creation is optional, so when no profile is

3
backend/src/app/rpc/commands/viewer.clj
View file

@ -6,6 +6,7 @@
(ns app.rpc.commands.viewer (ns app.rpc.commands.viewer
(:require (:require
[app.binfile.common :as bfc]
[app.common.exceptions :as ex] [app.common.exceptions :as ex]
[app.common.features :as cfeat] [app.common.features :as cfeat]
[app.common.schema :as sm] [app.common.schema :as sm]
@ -78,7 +79,7 @@
:always :always
(update :data select-keys [:id :options :pages :pages-index :components])) (update :data select-keys [:id :options :pages :pages-index :components]))
libs (->> (files/get-file-libraries conn file-id) libs (->> (bfc/get-file-libraries conn file-id)
(mapv (fn [{:keys [id] :as lib}] (mapv (fn [{:keys [id] :as lib}]
(merge lib (files/get-file cfg id))))) (merge lib (files/get-file cfg id)))))

8
backend/src/app/srepl/helpers.clj
View file

@ -146,13 +146,9 @@
(defn process-file! (defn process-file!
[system file-id update-fn & {:keys [label validate? with-libraries?] :or {validate? true} :as opts}] [system file-id update-fn & {:keys [label validate? with-libraries?] :or {validate? true} :as opts}]
(let [conn (db/get-connection system) (let [file (bfc/get-file system file-id ::db/for-update true)
file (bfc/get-file system file-id ::db/for-update true)
libs (when with-libraries? libs (when with-libraries?
(->> (files/get-file-libraries conn file-id) (bfc/get-resolved-file-libraries system file))
(into [file] (map (fn [{:keys [id]}]
(bfc/get-file system id))))
(d/index-by :id)))
file' (when file file' (when file
(if with-libraries? (if with-libraries?

9
backend/src/app/srepl/main.clj
View file

@ -390,12 +390,9 @@
[file-id] [file-id]
(let [file-id (h/parse-uuid file-id)] (let [file-id (h/parse-uuid file-id)]
(db/tx-run! (assoc main/system ::db/rollback true) (db/tx-run! (assoc main/system ::db/rollback true)
(fn [{:keys [::db/conn] :as system}] (fn [system]
(let [file (h/get-file system file-id) (let [file (bfc/get-file system file-id)
libs (->> (files/get-file-libraries conn file-id) libs (bfc/get-resolved-file-libraries system file)]
(into [file] (map (fn [{:keys [id]}]
(h/get-file system id))))
(d/index-by :id))]
(cfv/validate-file file libs)))))) (cfv/validate-file file libs))))))
(defn repair-file! (defn repair-file!

13
common/src/app/common/files/migrations.cljc
View file

@ -58,18 +58,21 @@
(map :name)) (map :name))
(defn migrate (defn migrate
[{:keys [id] :as file}] [{:keys [id] :as file} libs]
(let [diff (let [diff
(set/difference available-migrations (:migrations file)) (set/difference available-migrations (:migrations file))
data (-> (:data file)
(assoc :libs libs))
data data
(reduce migrate-data (:data file) diff) (reduce migrate-data data diff)
data data
(-> data (-> data
(assoc :id id) (assoc :id id)
(dissoc :version))] (dissoc :version :libs))]
(-> file (-> file
(assoc :data data) (assoc :data data)
@ -88,7 +91,7 @@
result)) result))
(defn migrate-file (defn migrate-file
[file] [file libs]
(binding [cfeat/*new* (atom #{})] (binding [cfeat/*new* (atom #{})]
(let [version (or (:version file) (let [version (or (:version file)
(-> file :data :version))] (-> file :data :version))]
@ -104,7 +107,7 @@
;; this code from this function that executes on ;; this code from this function that executes on
;; each file migration operation ;; each file migration operation
(update :features cfeat/migrate-legacy-features) (update :features cfeat/migrate-legacy-features)
(migrate) (migrate libs)
(update :features (fnil into #{}) (deref cfeat/*new*)))))) (update :features (fnil into #{}) (deref cfeat/*new*))))))
(defn migrated? (defn migrated?

2
common/test/common_tests/files_migrations_test.cljc
View file

@ -21,6 +21,6 @@
(let [file {:data {:sum 1} (let [file {:data {:sum 1}
:id 1 :id 1
:migrations (d/ordered-set "test/1")} :migrations (d/ordered-set "test/1")}
file' (cfm/migrate file)] file' (cfm/migrate file nil)]
(t/is (= cfm/available-migrations (:migrations file'))) (t/is (= cfm/available-migrations (:migrations file')))
(t/is (= 3 (:sum (:data file')))))))) (t/is (= 3 (:sum (:data file'))))))))