3pp-mirror/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-05-22 20:06:11 +02:00
Code Issues Projects Releases Packages Wiki Activity

✨ Add general features handling improvements

Browse source
This commit is contained in:
Andrey Antukh 2023-12-13 21:17:28 +01:00
parent bdb1742d59
commit 611594a392
6 changed files with 149 additions and 126 deletions
Download patch file Download diff file Expand all files Collapse all files

65
backend/src/app/features/components_v2.clj
View file

@ -769,7 +769,7 @@
fdata (migrate-graphics fdata)] fdata (migrate-graphics fdata)]
(update fdata :options assoc :components-v2 true))))) (update fdata :options assoc :components-v2 true)))))
(defn- process-fdata (defn- prepare-fdata
[fdata id] [fdata id]
(-> fdata (-> fdata
(assoc :id id) (assoc :id id)
@ -788,43 +788,46 @@
(defn- process-file (defn- process-file
[{:keys [::db/conn] :as system} id & {:keys [validate? throw-on-validate?]}] [{:keys [::db/conn] :as system} id & {:keys [validate? throw-on-validate?]}]
(binding [pmap/*tracked* (pmap/create-tracked) (let [file (binding [cfeat/*new* (atom #{})
pmap/*load-fn* (partial fdata/load-pointer *system* id)] pmap/*load-fn* (partial fdata/load-pointer system id)]
(-> (files/get-file system id :migrate? false)
(update :data prepare-fdata id)
(update :features into (deref cfeat/*new*))
(update :features cfeat/migrate-legacy-features)))
(let [file (binding [cfeat/*new* (atom #{})] libs (->> (files/get-file-libraries conn id)
(-> (files/get-file system id :migrate? false) (into [file] (map (fn [{:keys [id]}]
(update :data process-fdata id) (binding [pmap/*load-fn* (partial fdata/load-pointer system id)]
(update :features into (deref cfeat/*new*)) (-> (files/get-file system id :migrate? false)
(update :features cfeat/migrate-legacy-features))) (update :data prepare-fdata id))))))
(d/index-by :id))
libs (->> (files/get-file-libraries conn id) file (-> file
(into [file] (map (fn [{:keys [id]}] (update :data migrate-fdata libs)
(binding [pmap/*load-fn* (partial fdata/load-pointer system id)] (update :features conj "components/v2"))
(-> (files/get-file system id :migrate? false)
(update :data process-fdata id))))))
(d/index-by :id))
pmap? (contains? (:features file) "fdata/pointer-map") _ (when validate?
(validate-file! file libs throw-on-validate?))
file (-> file file (if (contains? (:features file) "fdata/objects-map")
(update :data migrate-fdata libs) (fdata/enable-objects-map file)
(update :features conj "components/v2") file)
(cond-> pmap? (fdata/enable-pointer-map)))
]
(when validate? file (if (contains? (:features file) "fdata/pointer-map")
(validate-file! file libs throw-on-validate?)) (binding [pmap/*tracked* (pmap/create-tracked)]
(let [file (fdata/enable-pointer-map file)]
(fdata/persist-pointers! system id)
file))
file)]
(db/update! conn :file (db/update! conn :file
{:data (blob/encode (:data file)) {:data (blob/encode (:data file))
:features (db/create-array conn "text" (:features file)) :features (db/create-array conn "text" (:features file))
:revn (:revn file)} :revn (:revn file)}
{:id (:id file)}) {:id (:id file)}
{::db/return-keys? false})
(when pmap? (dissoc file :data)))
(fdata/persist-pointers! system id))
(dissoc file :data))))
(defn migrate-file! (defn migrate-file!
[system file-id & {:keys [validate? throw-on-validate?]}] [system file-id & {:keys [validate? throw-on-validate?]}]

85
backend/src/app/rpc/commands/binfile.clj
View file

@ -701,6 +701,25 @@
(update :object-id #(str/replace-first % #"^(.*?)/" (str file-id "/"))))) (update :object-id #(str/replace-first % #"^(.*?)/" (str file-id "/")))))
thumbnails)) thumbnails))
(defn- process-fdata
[fdata id]
(-> fdata
(dissoc :recent-colors)
(assoc :id id)
(cond-> (> (:version fdata) cfd/version)
(assoc :version cfd/version))
;; FIXME: We're temporarily activating all
;; migrations because a problem in the
;; environments messed up with the version
;; numbers When this problem is fixed delete
;; the following line
(assoc :version 22)
(pmg/migrate-data)
(update :pages-index relink-shapes)
(update :components relink-shapes)
(update :media relink-media)))
(defmethod read-section :v1/files (defmethod read-section :v1/files
[{:keys [::db/conn ::input ::project-id ::enabled-features ::timestamp ::overwrite?] :as system}] [{:keys [::db/conn ::input ::project-id ::enabled-features ::timestamp ::overwrite?] :as system}]
@ -754,58 +773,40 @@
(l/dbg :hint "update media references" ::l/sync? true) (l/dbg :hint "update media references" ::l/sync? true)
(vswap! *state* update :media into (map #(update % :id lookup-index)) media)) (vswap! *state* update :media into (map #(update % :id lookup-index)) media))
(binding [pmap/*tracked* (pmap/create-tracked) (let [file (binding [cfeat/*new* (atom #{})]
cfeat/*new* (atom #{})] (-> file
(let [file (-> file
(assoc :id file-id') (assoc :id file-id')
(assoc :features features) (assoc :features features)
(assoc :project-id project-id) (assoc :project-id project-id)
(assoc :created-at timestamp) (assoc :created-at timestamp)
(assoc :modified-at timestamp) (assoc :modified-at timestamp)
(dissoc :thumbnails) (dissoc :thumbnails)
(update :data (fn [data] (update :data process-fdata file-id')
(-> data (update :features into (deref cfeat/*new*))))
(dissoc :recent-colors)
(assoc :id file-id')
(cond-> (> (:version data) cfd/version)
(assoc :version cfd/version))
;; FIXME: We're temporarily activating all _ (when (contains? cf/flags :file-schema-validation)
;; migrations because a problem in the (fval/validate-file-schema! file))
;; environments messed up with the version
;; numbers When this problem is fixed delete
;; the following line
(assoc :version 22)
(update :pages-index relink-shapes)
(update :components relink-shapes)
(update :media relink-media)
(pmg/migrate-data)
(d/without-nils)))))
;; Add to file features all possible features added on _ (when (contains? cf/flags :soft-file-schema-validation)
;; migration process. (let [result (ex/try! (fval/validate-file-schema! file))]
file (update file :features into (deref cfeat/*new*)) (when (ex/exception? result)
(l/error :hint "file schema validation error" :cause result))))
file (if (contains? cf/flags :file-schema-validation) file (if (contains? (:features file) "fdata/objects-map")
(fval/validate-file-schema! file) (feat.fdata/enable-objects-map file)
file) file)
_ (when (contains? cf/flags :soft-file-schema-validation) file (if (contains? (:features file) "fdata/pointer-map")
(try (binding [pmap/*tracked* (pmap/create-tracked)]
(fval/validate-file-schema! file) (let [file (feat.fdata/enable-pointer-map file)]
(catch Throwable cause (feat.fdata/persist-pointers! system file-id')
(l/error :hint "file schema validation error" :cause cause)))) file))
file)
file (cond-> file
(contains? (:features file) "fdata/objects-map")
(feat.fdata/enable-objects-map)
(contains? (:features file) "fdata/pointer-map") file (-> file
(feat.fdata/enable-pointer-map)) (update :features #(db/create-array conn "text" %))
(update :data blob/encode))]
file (-> file
(update :features #(db/create-array conn "text" %))
(update :data blob/encode))]
(l/dbg :hint "create file" :id (str file-id') ::l/sync? true) (l/dbg :hint "create file" :id (str file-id') ::l/sync? true)
@ -813,12 +814,10 @@
(create-or-update-file! conn file) (create-or-update-file! conn file)
(db/insert! conn :file file)) (db/insert! conn :file file))
(feat.fdata/persist-pointers! system file-id')
(when overwrite? (when overwrite?
(db/delete! conn :file-thumbnail {:file-id file-id'})) (db/delete! conn :file-thumbnail {:file-id file-id'}))
file-id'))))) file-id'))))
(defmethod read-section :v1/rels (defmethod read-section :v1/rels
[{:keys [::db/conn ::input ::timestamp]}] [{:keys [::db/conn ::input ::timestamp]}]

27
backend/src/app/rpc/commands/files.clj
View file

@ -34,7 +34,6 @@
[app.util.pointer-map :as pmap] [app.util.pointer-map :as pmap]
[app.util.services :as sv] [app.util.services :as sv]
[app.util.time :as dt] [app.util.time :as dt]
[clojure.set :as set]
[clojure.spec.alpha :as s] [clojure.spec.alpha :as s]
[cuerdas.core :as str])) [cuerdas.core :as str]))
@ -227,7 +226,10 @@
(binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id) (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)
pmap/*tracked* (pmap/create-tracked) pmap/*tracked* (pmap/create-tracked)
cfeat/*new* (atom #{})] cfeat/*new* (atom #{})]
(let [file (fmg/migrate-file file)] (let [file (-> (fmg/migrate-file file)
(update :features into (deref cfeat/*new*))
(update :features cfeat/migrate-legacy-features))]
;; NOTE: when file is migrated, we break the rule of no perform ;; NOTE: when file is migrated, we break the rule of no perform
;; mutations on get operations and update the file with all ;; mutations on get operations and update the file with all
;; migrations applied ;; migrations applied
@ -235,16 +237,17 @@
;; NOTE: the following code will not work on read-only mode, it ;; NOTE: the following code will not work on read-only mode, it
;; is a known issue; we keep is not implemented until we really ;; is a known issue; we keep is not implemented until we really
;; need this ;; need this
(if (fmg/migrated? file) (when (fmg/migrated? file)
(let [file (update file :features cfeat/migrate-legacy-features) (db/update! conn :file
features (set/union (deref cfeat/*new*) (:features file))] {:data (blob/encode (:data file))
(db/update! conn :file :features (db/create-array conn "text" (:features file))}
{:data (blob/encode (:data file)) {:id id}
:features (db/create-array conn "text" features)} {::db/return-keys? false})
{:id id})
(feat.fdata/persist-pointers! cfg id) (when (contains? (:features file) "fdata/pointer-map")
(assoc file :features features)) (feat.fdata/persist-pointers! cfg id)))
file))))
file)))
(defn get-file (defn get-file
[{:keys [::db/conn] :as cfg} id & {:keys [project-id migrate? [{:keys [::db/conn] :as cfg} id & {:keys [project-id migrate?

53
backend/src/app/rpc/commands/files_update.clj
View file

@ -182,40 +182,39 @@
(defn update-file (defn update-file
[{:keys [::db/conn ::mtx/metrics] :as cfg} [{:keys [::db/conn ::mtx/metrics] :as cfg}
{:keys [id file features changes changes-with-metadata] :as params}] {:keys [id file features changes changes-with-metadata] :as params}]
(binding [cfeat/*current* features (let [features (-> features
cfeat/*previous* (:features file)] (set/difference cfeat/frontend-only-features)
(set/union (:features file)))
(let [features (-> features update-fn (cond-> update-file*
(set/difference cfeat/frontend-only-features) (contains? features "fdata/pointer-map")
(set/union (:features file))) (wrap-with-pointer-map-context)
update-fn (cond-> update-file* (contains? features "fdata/objects-map")
(contains? features "fdata/pointer-map") (wrap-with-objects-map-context))
(wrap-with-pointer-map-context)
(contains? features "fdata/objects-map") changes (if changes-with-metadata
(wrap-with-objects-map-context)) (->> changes-with-metadata (mapcat :changes) vec)
(vec changes))]
changes (if changes-with-metadata (when (> (:revn params)
(->> changes-with-metadata (mapcat :changes) vec) (:revn file))
(vec changes))] (ex/raise :type :validation
:code :revn-conflict
:hint "The incoming revision number is greater that stored version."
:context {:incoming-revn (:revn params)
:stored-revn (:revn file)}))
(when (> (:revn params) (mtx/run! metrics {:id :update-file-changes :inc (count changes)})
(:revn file))
(ex/raise :type :validation
:code :revn-conflict
:hint "The incoming revision number is greater that stored version."
:context {:incoming-revn (:revn params)
:stored-revn (:revn file)}))
(mtx/run! metrics {:id :update-file-changes :inc (count changes)}) (when (not= features (:features file))
(let [features (db/create-array conn "text" features)]
(when (not= features (:features file)) (db/update! conn :file
(let [features (db/create-array conn "text" features)] {:features features}
(db/update! conn :file {:id id})))
{:features features}
{:id id})))
(binding [cfeat/*current* features
cfeat/*previous* (:features file)]
(let [file (assoc file :features features) (let [file (assoc file :features features)
params (-> params params (-> params
(assoc :file file) (assoc :file file)

40
backend/src/app/rpc/commands/management.clj
View file

@ -109,23 +109,37 @@
(update-fdata [fdata new-id] (update-fdata [fdata new-id]
(-> fdata (-> fdata
(assoc :id new-id) (assoc :id new-id)
(feat.fdata/process-pointers deref)
(pmg/migrate-data) (pmg/migrate-data)
(update :pages-index relink-shapes) (update :pages-index relink-shapes)
(update :components relink-shapes) (update :components relink-shapes)
(update :media relink-media) (update :media relink-media)
(d/without-nils) (d/without-nils)))]
(feat.fdata/process-pointers pmap/clone)))]
(binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id) (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)
pmap/*tracked* (pmap/create-tracked) pmap/*tracked* (pmap/create-tracked)
cfeat/*new* (atom #{})] cfeat/*new* (atom #{})]
(let [new-id (get index id) (let [new-id (get index id)
file (-> file file (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)
(assoc :id new-id) cfeat/*new* (atom #{})]
(update :data update-fdata new-id) (-> file
(update :features into (deref cfeat/*new*)) (assoc :id new-id)
(update :features cfeat/migrate-legacy-features))] (update :data update-fdata new-id)
(feat.fdata/persist-pointers! cfg new-id) (update :features into (deref cfeat/*new*))
(update :features cfeat/migrate-legacy-features)))
file (if (contains? (:features file) "fdata/objects-map")
(feat.fdata/enable-objects-map file)
file)
file (if (contains? (:features file) "fdata/pointer-map")
(binding [pmap/*tracked* (pmap/create-tracked)]
(let [file (feat.fdata/enable-pointer-map file)]
(feat.fdata/persist-pointers! cfg (:id file))
file))
file)]
file)))) file))))
(def sql:get-used-libraries (def sql:get-used-libraries
@ -191,20 +205,22 @@
(db/insert! conn :file (db/insert! conn :file
(-> file (-> file
(update :features #(db/create-array conn "text" %)) (update :features #(db/create-array conn "text" %))
(update :data blob/encode))) (update :data blob/encode))
{::db/return-keys? false})
(db/insert! conn :file-profile-rel (db/insert! conn :file-profile-rel
{:file-id (:id file) {:file-id (:id file)
:profile-id profile-id :profile-id profile-id
:is-owner true :is-owner true
:is-admin true :is-admin true
:can-edit true}) :can-edit true}
{::db/return-keys? false})
(doseq [params flibs] (doseq [params flibs]
(db/insert! conn :file-library-rel params)) (db/insert! conn :file-library-rel params ::db/return-keys? false))
(doseq [params fmeds] (doseq [params fmeds]
(db/insert! conn :file-media-object params)) (db/insert! conn :file-media-object params ::db/return-keys? false))
file)) file))

5
common/src/app/common/features.cljc
View file

@ -51,7 +51,10 @@
"layout/grid"}) "layout/grid"})
;; A set of features enabled by default for each file, they are ;; A set of features enabled by default for each file, they are
;; implicit and are enabled by default and can't be disabled ;; implicit and are enabled by default and can't be disabled. The
;; features listed in this set are mainly freatures addedby file
;; migrations process, so all features referenced in migrations should
;; be here.
(def default-enabled-features (def default-enabled-features
#{"fdata/shape-data-type"}) #{"fdata/shape-data-type"})