diff --git a/backend/src/app/rpc/commands/binfile.clj b/backend/src/app/rpc/commands/binfile.clj index 76df7eafe..be386c29d 100644 --- a/backend/src/app/rpc/commands/binfile.clj +++ b/backend/src/app/rpc/commands/binfile.clj @@ -630,10 +630,12 @@ (pu/with-open [input (zstd-input-stream input) input (io/data-input-stream input)] (binding [*state* (volatile! {:media [] :index {}})] - (let [team (teams/get-team options + (let [team (teams/get-team options :profile-id profile-id :project-id project-id) - features (cfeat/get-team-enabled-features cf/flags team)] + + validate? (contains? cf/flags :file-validation) + features (cfeat/get-team-enabled-features cf/flags team)] ;; Process all sections (run! (fn [section] @@ -651,13 +653,13 @@ (doseq [[feature file-id] (-> *state* deref :pending-to-migrate)] (case feature "components/v2" - (features.components-v2/migrate-file! options file-id) + (features.components-v2/migrate-file! options file-id + :validate? validate? + :throw-on-validate? true) "fdata/shape-data-type" nil - ;; "fdata/shape-data-type" - ;; (features.fdata/enable-objects-map (ex/raise :type :internal :code :no-migration-defined :hint (str/ffmt "no migation for feature '%' on file importation" feature) @@ -774,17 +776,16 @@ ;; environments messed up with the version ;; numbers When this problem is fixed delete ;; the following line - (assoc :version 0) + (assoc :version 22) (update :pages-index relink-shapes) (update :components relink-shapes) (update :media relink-media) (pmg/migrate-data) (d/without-nils)))) - ;; Without providing all libs, here we just - ;; peform a structural file data validation, - ;; full referential check is omited. - (fval/validate-file!) + (cond-> (contains? cf/flags :file-validation) + (fval/validate-file-schema!)) + (postprocess-file) (update :features #(db/create-array conn "text" %)) (update :data blob/encode))] diff --git a/backend/src/app/rpc/commands/files_update.clj b/backend/src/app/rpc/commands/files_update.clj index 39aadab55..6da9e86e0 100644 --- a/backend/src/app/rpc/commands/files_update.clj +++ b/backend/src/app/rpc/commands/files_update.clj @@ -266,6 +266,7 @@ ;; Retrieve and return lagged data (get-lagged-changes conn params)))) + (defn- update-file-data [conn file changes skip-validate] (let [file (update file :data (fn [data] @@ -278,7 +279,6 @@ ;; some other way to do general validation libs (when (and (contains? cf/flags :file-validation) (not skip-validate)) - ;; FIXME: we need properly handle pointer-map here ???? (->> (files/get-file-libraries conn (:id file)) (into [file] (map (fn [{:keys [id]}] (binding [pmap/*load-fn* (partial files/load-pointer conn id)] @@ -293,7 +293,10 @@ (update :data cpc/process-changes changes) ;; If `libs` is defined, then full validation is performed - (val/validate-file! libs) + (cond-> (and (contains? cf/flags :file-validation) + (not skip-validate)) + (-> (val/validate-file! libs) + (val/validate-file-schema!))) (cond-> (and (contains? cfeat/*current* "fdata/objects-map") (not (contains? cfeat/*previous* "fdata/objects-map"))) diff --git a/backend/src/app/srepl/components_v2.clj b/backend/src/app/srepl/components_v2.clj index fc4725b57..8f24021c3 100644 --- a/backend/src/app/srepl/components_v2.clj +++ b/backend/src/app/srepl/components_v2.clj @@ -238,7 +238,7 @@ (migrate-team [team-id] (try (-> (assoc system ::db/rollback rollback?) - (feat/migrate-team! team-id :validate? validate?)) + (feat/migrate-team! team-id :validate? validate? :throw-on-validate? (not skip-on-error))) (catch Throwable cause (l/err :hint "unexpected error on processing team" :team-id (dm/str team-id) :cause cause)))) diff --git a/common/src/app/common/files/defaults.cljc b/common/src/app/common/files/defaults.cljc index c1d1ce741..ff8c0e829 100644 --- a/common/src/app/common/files/defaults.cljc +++ b/common/src/app/common/files/defaults.cljc @@ -6,4 +6,4 @@ (ns app.common.files.defaults) -(def version 35) +(def version 36) diff --git a/common/src/app/common/files/migrations.cljc b/common/src/app/common/files/migrations.cljc index c683e7826..2bb858420 100644 --- a/common/src/app/common/files/migrations.cljc +++ b/common/src/app/common/files/migrations.cljc @@ -651,3 +651,15 @@ (migrate) (assoc :version 35))) + +(defmethod migrate 36 + [data] + (letfn [(update-container [container] + (d/update-when container :objects (fn [objects] + (if (contains? objects nil) + (dissoc objects nil) + objects))))] + (-> data + (update :pages-index update-vals update-container) + (update :components update-vals update-container)))) + diff --git a/common/src/app/common/files/validate.cljc b/common/src/app/common/files/validate.cljc index 9474ca338..e6fa1a897 100644 --- a/common/src/app/common/files/validate.cljc +++ b/common/src/app/common/files/validate.cljc @@ -442,44 +442,40 @@ "Get schema explain data for file data" (sm/lazy-explainer ::ctf/data)) -(defn validate-file! - "Validate file data structure. +(defn validate-file-schema! + [{:keys [id data] :as file}] + (when-not (valid-fdata? data) + (if (some? *errors*) + (vswap! *errors* conj + {:code :invalid-file-data-structure + :hint (str/ffmt "invalid file data structure found on file '%'" id) + :file-id id}) + (ex/raise :type :validation + :code :data-validation + :hint (str/ffmt "invalid file data structure found on file '%'" id) + :file-id id + ::sm/explain (get-fdata-explain data)))) + file) - If libraries are provided, then a full referential integrity and - semantic coherence check will be performed on all content of the - file. +(defn validate-file! + "Validate full referential integrity and semantic coherence on file data. Raises a validation exception on first error found." + [{:keys [data] :as file} libraries] - ([file] (validate-file! file nil)) - ([{:keys [data] :as file} libraries] - ;; (when-not (valid-fdata? data) - ;; (if (some? *errors*) - ;; (vswap! *errors* conj - ;; {:code :invalid-file-data-structure - ;; :hint (str/ffmt "invalid file data structure found on file '%'" id) - ;; :file-id id}) - ;; (ex/raise :type :validation - ;; :code :data-validation - ;; :hint (str/ffmt "invalid file data found on file '%'" id) - ;; :file-id id - ;; ::sm/explain (get-fdata-explain data)))) + (doseq [page (filter :id (ctpl/pages-seq data))] + (validate-shape! uuid/zero file page libraries)) - ;; If `libraries` is provided, this means the full file - ;; validation is activated so we proceed to execute the - ;; validation - (when (some? libraries) - (doseq [page (filter :id (ctpl/pages-seq data))] - (validate-shape! uuid/zero file page libraries)) - (doseq [component (vals (:components data))] - (validate-component! component file))) + (doseq [component (vals (:components data))] + (validate-component! component file)) - file)) + file) (defn validate-file - "Validate referencial integrity and semantic coherence of + "Validate structure, referencial integrity and semantic coherence of all contents of a file. Returns a list of errors." [file libraries] (binding [*errors* (volatile! [])] + (validate-file-schema! file) (validate-file! file libraries) (deref *errors*)))