diff --git a/backend/src/app/binfile/common.clj b/backend/src/app/binfile/common.clj index a893fe3a0..84f539e3a 100644 --- a/backend/src/app/binfile/common.clj +++ b/backend/src/app/binfile/common.clj @@ -53,6 +53,7 @@ (* 1024 1024 100)) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +(declare get-resolved-file-libraries) (def file-attrs #{:id @@ -143,11 +144,13 @@ (reduce #(index-object %1 %2 attr) index coll))) (defn decode-row - "A generic decode row helper" - [{:keys [data features] :as row}] - (cond-> row - features (assoc :features (db/decode-pgarray features #{})) - data (assoc :data (blob/decode data)))) + [{:keys [data changes features] :as row}] + (when row + (cond-> row + features (assoc :features (db/decode-pgarray features #{})) + changes (assoc :changes (blob/decode changes)) + data (assoc :data (blob/decode data))))) + (defn decode-file "A general purpose file decoding function that resolves all external @@ -156,7 +159,8 @@ (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)] (let [file (->> file (feat.fmigr/resolve-applied-migrations cfg) - (feat.fdata/resolve-file-data cfg))] + (feat.fdata/resolve-file-data cfg)) + libs (delay (get-resolved-file-libraries cfg file))] (-> file (update :features db/decode-pgarray #{}) @@ -164,7 +168,7 @@ (update :data feat.fdata/process-pointers deref) (update :data feat.fdata/process-objects (partial into {})) (update :data assoc :id id) - (fmg/migrate-file))))) + (fmg/migrate-file libs))))) (defn get-file "Get file, resolve all features and apply migrations. @@ -418,26 +422,27 @@ (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"]))) (defn process-file - [{:keys [id] :as file}] - (-> file - (update :data (fn [fdata] - (-> fdata - (assoc :id id) - (dissoc :recent-colors)))) - (fmg/migrate-file) - (update :data (fn [fdata] - (-> fdata - (update :pages-index relink-shapes) - (update :components relink-shapes) - (update :media relink-media) - (update :colors relink-colors) - (d/without-nils)))) + [cfg {:keys [id] :as file}] + (let [libs (delay (get-resolved-file-libraries cfg file))] + (-> file + (update :data (fn [fdata] + (-> fdata + (assoc :id id) + (dissoc :recent-colors)))) + (fmg/migrate-file libs) + (update :data (fn [fdata] + (-> fdata + (update :pages-index relink-shapes) + (update :components relink-shapes) + (update :media relink-media) + (update :colors relink-colors) + (d/without-nils)))) - ;; NOTE: this is necessary because when we just creating a new - ;; file from imported artifact or cloned file there are no - ;; migrations registered on the database, so we need to persist - ;; all of them, not only the applied - (vary-meta dissoc ::fmg/migrated))) + ;; NOTE: this is necessary because when we just creating a new + ;; file from imported artifact or cloned file there are no + ;; migrations registered on the database, so we need to persist + ;; all of them, not only the applied + (vary-meta dissoc ::fmg/migrated)))) (defn encode-file [{:keys [::db/conn] :as cfg} {:keys [id features] :as file}] @@ -528,3 +533,49 @@ (l/error :hint "file schema validation error" :cause result)))) (insert-file! cfg file opts))) + + +(def ^:private sql:get-file-libraries + "WITH RECURSIVE libs AS ( + SELECT fl.*, flr.synced_at + FROM file AS fl + JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id) + WHERE flr.file_id = ?::uuid + UNION + SELECT fl.*, flr.synced_at + FROM file AS fl + JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id) + JOIN libs AS l ON (flr.file_id = l.id) + ) + SELECT l.id, + l.features, + l.project_id, + p.team_id, + l.created_at, + l.modified_at, + l.deleted_at, + l.name, + l.revn, + l.vern, + l.synced_at, + l.is_shared + FROM libs AS l + INNER JOIN project AS p ON (p.id = l.project_id) + WHERE l.deleted_at IS NULL OR l.deleted_at > now();") + +(defn get-file-libraries + [conn file-id] + (into [] + (comp + ;; FIXME: :is-indirect set to false to all rows looks + ;; completly useless + (map #(assoc % :is-indirect false)) + (map decode-row)) + (db/exec! conn [sql:get-file-libraries file-id]))) + +(defn get-resolved-file-libraries + "A helper for preload file libraries" + [{:keys [::db/conn] :as cfg} file] + (->> (get-file-libraries conn (:id file)) + (into [file] (map #(get-file cfg (:id %)))) + (d/index-by :id))) diff --git a/backend/src/app/binfile/v1.clj b/backend/src/app/binfile/v1.clj index 2f70ed50d..e73f2c05a 100644 --- a/backend/src/app/binfile/v1.clj +++ b/backend/src/app/binfile/v1.clj @@ -551,8 +551,8 @@ (cond-> (and (= idx 0) (some? name)) (assoc :name name)) (assoc :project-id project-id) - (dissoc :thumbnails) - (bfc/process-file))] + (dissoc :thumbnails)) + file (bfc/process-file system file)] ;; All features that are enabled and requires explicit migration are ;; added to the state for a posterior migration step. diff --git a/backend/src/app/binfile/v2.clj b/backend/src/app/binfile/v2.clj index b4acf7b0b..06e22bf42 100644 --- a/backend/src/app/binfile/v2.clj +++ b/backend/src/app/binfile/v2.clj @@ -281,8 +281,8 @@ (let [file (-> (read-obj cfg :file file-id) (update :id bfc/lookup-index) - (update :project-id bfc/lookup-index) - (bfc/process-file))] + (update :project-id bfc/lookup-index)) + file (bfc/process-file cfg file)] (events/tap :progress {:op :import diff --git a/backend/src/app/binfile/v3.clj b/backend/src/app/binfile/v3.clj index 591d9a079..4763e1230 100644 --- a/backend/src/app/binfile/v3.clj +++ b/backend/src/app/binfile/v3.clj @@ -754,8 +754,9 @@ (assoc :data data) (assoc :name file-name) (assoc :project-id project-id) - (dissoc :options) - (bfc/process-file))] + (dissoc :options)) + + file (bfc/process-file cfg file)] (bfm/register-pending-migrations! cfg file) (bfc/save-file! cfg file ::db/return-keys false) diff --git a/backend/src/app/rpc/commands/files.clj b/backend/src/app/rpc/commands/files.clj index aff510d4c..d6e030c30 100644 --- a/backend/src/app/rpc/commands/files.clj +++ b/backend/src/app/rpc/commands/files.clj @@ -6,6 +6,7 @@ (ns app.rpc.commands.files (:require + [app.binfile.common :as bfc] [app.common.data :as d] [app.common.data.macros :as dm] [app.common.exceptions :as ex] @@ -211,7 +212,8 @@ [{:keys [::db/conn] :as cfg} {:keys [id] :as file} {:keys [read-only?]}] (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id) pmap/*tracked* (pmap/create-tracked)] - (let [;; For avoid unnecesary overhead of creating multiple pointers and + (let [libs (delay (bfc/get-resolved-file-libraries cfg file)) + ;; For avoid unnecesary overhead of creating multiple pointers and ;; handly internally with objects map in their worst case (when ;; probably all shapes and all pointers will be readed in any ;; case), we just realize/resolve them before applying the @@ -219,7 +221,7 @@ file (-> file (update :data feat.fdata/process-pointers deref) (update :data feat.fdata/process-objects (partial into {})) - (fmg/migrate-file))] + (fmg/migrate-file libs))] (if (or read-only? (db/read-only? conn)) file @@ -615,44 +617,6 @@ ;; --- COMMAND QUERY: get-file-libraries -(def ^:private sql:get-file-libraries - "WITH RECURSIVE libs AS ( - SELECT fl.*, flr.synced_at - FROM file AS fl - JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id) - WHERE flr.file_id = ?::uuid - UNION - SELECT fl.*, flr.synced_at - FROM file AS fl - JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id) - JOIN libs AS l ON (flr.file_id = l.id) - ) - SELECT l.id, - l.features, - l.project_id, - p.team_id, - l.created_at, - l.modified_at, - l.deleted_at, - l.name, - l.revn, - l.vern, - l.synced_at, - l.is_shared - FROM libs AS l - INNER JOIN project AS p ON (p.id = l.project_id) - WHERE l.deleted_at IS NULL OR l.deleted_at > now();") - -(defn get-file-libraries - [conn file-id] - (into [] - (comp - ;; FIXME: :is-indirect set to false to all rows looks - ;; completly useless - (map #(assoc % :is-indirect false)) - (map decode-row)) - (db/exec! conn [sql:get-file-libraries file-id]))) - (def ^:private schema:get-file-libraries [:map {:title "get-file-libraries"} [:file-id ::sm/uuid]]) @@ -664,7 +628,7 @@ [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id]}] (dm/with-open [conn (db/open pool)] (check-read-permissions! conn profile-id file-id) - (get-file-libraries conn file-id))) + (bfc/get-file-libraries conn file-id))) ;; --- COMMAND QUERY: Files that use this File library diff --git a/backend/src/app/rpc/commands/files_update.clj b/backend/src/app/rpc/commands/files_update.clj index 866e67b1f..04a0551b5 100644 --- a/backend/src/app/rpc/commands/files_update.clj +++ b/backend/src/app/rpc/commands/files_update.clj @@ -340,6 +340,7 @@ (-> data (blob/decode) (assoc :id (:id file))))) + libs (delay (bfc/get-resolved-file-libraries cfg file)) ;; For avoid unnecesary overhead of creating multiple pointers ;; and handly internally with objects map in their worst @@ -350,7 +351,7 @@ (-> file (update :data feat.fdata/process-pointers deref) (update :data feat.fdata/process-objects (partial into {})) - (fmg/migrate-file)) + (fmg/migrate-file libs)) file) file (apply update-fn cfg file args) @@ -379,13 +380,6 @@ (bfc/encode-file cfg file)))) -(defn- get-file-libraries - "A helper for preload file libraries, mainly used for perform file - semantical and structural validation" - [{:keys [::db/conn] :as cfg} file] - (->> (files/get-file-libraries conn (:id file)) - (into [file] (map #(bfc/get-file cfg (:id %)))) - (d/index-by :id))) (defn- soft-validate-file-schema! [file] @@ -411,7 +405,7 @@ (when (and (or (contains? cf/flags :file-validation) (contains? cf/flags :soft-file-validation)) (not skip-validate)) - (get-file-libraries cfg file)) + (bfc/get-resolved-file-libraries cfg file)) ;; The main purpose of this atom is provide a contextual state diff --git a/backend/src/app/rpc/commands/management.clj b/backend/src/app/rpc/commands/management.clj index 147b1fe24..cee172df3 100644 --- a/backend/src/app/rpc/commands/management.clj +++ b/backend/src/app/rpc/commands/management.clj @@ -56,7 +56,7 @@ (vswap! bfc/*state* update :index bfc/update-index fmeds :id) ;; Process and persist file - (let [file (bfc/process-file file)] + (let [file (bfc/process-file cfg file)] (bfc/insert-file! cfg file ::db/return-keys false) ;; The file profile creation is optional, so when no profile is diff --git a/backend/src/app/rpc/commands/viewer.clj b/backend/src/app/rpc/commands/viewer.clj index e86f13033..db8cb7a80 100644 --- a/backend/src/app/rpc/commands/viewer.clj +++ b/backend/src/app/rpc/commands/viewer.clj @@ -6,6 +6,7 @@ (ns app.rpc.commands.viewer (:require + [app.binfile.common :as bfc] [app.common.exceptions :as ex] [app.common.features :as cfeat] [app.common.schema :as sm] @@ -78,7 +79,7 @@ :always (update :data select-keys [:id :options :pages :pages-index :components])) - libs (->> (files/get-file-libraries conn file-id) + libs (->> (bfc/get-file-libraries conn file-id) (mapv (fn [{:keys [id] :as lib}] (merge lib (files/get-file cfg id))))) diff --git a/backend/src/app/srepl/helpers.clj b/backend/src/app/srepl/helpers.clj index 2ce9e58f3..6a67cd237 100644 --- a/backend/src/app/srepl/helpers.clj +++ b/backend/src/app/srepl/helpers.clj @@ -146,13 +146,9 @@ (defn process-file! [system file-id update-fn & {:keys [label validate? with-libraries?] :or {validate? true} :as opts}] - (let [conn (db/get-connection system) - file (bfc/get-file system file-id ::db/for-update true) + (let [file (bfc/get-file system file-id ::db/for-update true) libs (when with-libraries? - (->> (files/get-file-libraries conn file-id) - (into [file] (map (fn [{:keys [id]}] - (bfc/get-file system id)))) - (d/index-by :id))) + (bfc/get-resolved-file-libraries system file)) file' (when file (if with-libraries? diff --git a/backend/src/app/srepl/main.clj b/backend/src/app/srepl/main.clj index a9ebb1379..7a07f0671 100644 --- a/backend/src/app/srepl/main.clj +++ b/backend/src/app/srepl/main.clj @@ -390,12 +390,9 @@ [file-id] (let [file-id (h/parse-uuid file-id)] (db/tx-run! (assoc main/system ::db/rollback true) - (fn [{:keys [::db/conn] :as system}] - (let [file (h/get-file system file-id) - libs (->> (files/get-file-libraries conn file-id) - (into [file] (map (fn [{:keys [id]}] - (h/get-file system id)))) - (d/index-by :id))] + (fn [system] + (let [file (bfc/get-file system file-id) + libs (bfc/get-resolved-file-libraries system file)] (cfv/validate-file file libs)))))) (defn repair-file! diff --git a/common/src/app/common/files/migrations.cljc b/common/src/app/common/files/migrations.cljc index 23c7f04e9..8beef88d5 100644 --- a/common/src/app/common/files/migrations.cljc +++ b/common/src/app/common/files/migrations.cljc @@ -58,18 +58,21 @@ (map :name)) (defn migrate - [{:keys [id] :as file}] + [{:keys [id] :as file} libs] (let [diff (set/difference available-migrations (:migrations file)) + data (-> (:data file) + (assoc :libs libs)) + data - (reduce migrate-data (:data file) diff) + (reduce migrate-data data diff) data (-> data (assoc :id id) - (dissoc :version))] + (dissoc :version :libs))] (-> file (assoc :data data) @@ -88,7 +91,7 @@ result)) (defn migrate-file - [file] + [file libs] (binding [cfeat/*new* (atom #{})] (let [version (or (:version file) (-> file :data :version))] @@ -104,7 +107,7 @@ ;; this code from this function that executes on ;; each file migration operation (update :features cfeat/migrate-legacy-features) - (migrate) + (migrate libs) (update :features (fnil into #{}) (deref cfeat/*new*)))))) (defn migrated? diff --git a/common/test/common_tests/files_migrations_test.cljc b/common/test/common_tests/files_migrations_test.cljc index 7badb9902..f5fe884ba 100644 --- a/common/test/common_tests/files_migrations_test.cljc +++ b/common/test/common_tests/files_migrations_test.cljc @@ -21,6 +21,6 @@ (let [file {:data {:sum 1} :id 1 :migrations (d/ordered-set "test/1")} - file' (cfm/migrate file)] + file' (cfm/migrate file nil)] (t/is (= cfm/available-migrations (:migrations file'))) (t/is (= 3 (:sum (:data file'))))))))