From ac3d7f00d5473491d45931bc025783bb5fa21cc5 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 20 Nov 2023 19:27:54 +0100 Subject: [PATCH] :sparkles: Make file schema validation configurable using flags --- backend/scripts/repl | 4 +++- backend/scripts/start-dev | 4 +++- backend/src/app/loggers/database.clj | 1 - backend/src/app/rpc/commands/binfile.clj | 16 ++++++++++++---- backend/src/app/rpc/commands/files_update.clj | 18 ++++++++++++++++-- backend/src/app/srepl/components_v2.clj | 12 +++++++++--- 6 files changed, 43 insertions(+), 12 deletions(-) diff --git a/backend/scripts/repl b/backend/scripts/repl index 61cee99fc..219e2c164 100755 --- a/backend/scripts/repl +++ b/backend/scripts/repl @@ -26,7 +26,9 @@ export PENPOT_FLAGS="\ enable-soft-rpc-rlimit \ enable-webhooks \ enable-access-tokens \ - enable-file-validation"; + disable-file-validation \ + disable-file-schema-validation \ + enable-soft-file-schema-validation"; # export PENPOT_DATABASE_URI="postgresql://172.17.0.1:5432/penpot" # export PENPOT_DATABASE_USERNAME="penpot" diff --git a/backend/scripts/start-dev b/backend/scripts/start-dev index 78fdd4ee6..f905fe3ad 100755 --- a/backend/scripts/start-dev +++ b/backend/scripts/start-dev @@ -17,7 +17,9 @@ export PENPOT_FLAGS="\ disable-secure-session-cookies \ enable-smtp \ enable-access-tokens \ - enable-file-validation"; + disable-file-validation \ + disable-file-schema-validation \ + enable-soft-file-schema-validation"; # Initialize MINIO config mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin diff --git a/backend/src/app/loggers/database.clj b/backend/src/app/loggers/database.clj index 9cdfdcf8c..8319e7566 100644 --- a/backend/src/app/loggers/database.clj +++ b/backend/src/app/loggers/database.clj @@ -39,7 +39,6 @@ (defn record->report [{:keys [::l/context ::l/message ::l/props ::l/logger ::l/level ::l/cause] :as record}] (us/assert! ::l/record record) - (if (or (instance? java.util.concurrent.CompletionException cause) (instance? java.util.concurrent.ExecutionException cause)) (-> record diff --git a/backend/src/app/rpc/commands/binfile.clj b/backend/src/app/rpc/commands/binfile.clj index be386c29d..07b31275d 100644 --- a/backend/src/app/rpc/commands/binfile.clj +++ b/backend/src/app/rpc/commands/binfile.clj @@ -781,11 +781,19 @@ (update :components relink-shapes) (update :media relink-media) (pmg/migrate-data) - (d/without-nils)))) + (d/without-nils))))) - (cond-> (contains? cf/flags :file-validation) - (fval/validate-file-schema!)) + params (if (contains? cf/flags :file-schema-validation) + (fval/validate-file-schema! params) + params) + _ (when (contains? cf/flags :soft-file-schema-validation) + (try + (fval/validate-file-schema! params) + (catch Throwable cause + (l/error :hint "file schema validation error" :cause cause)))) + + params (-> params (postprocess-file) (update :features #(db/create-array conn "text" %)) (update :data blob/encode))] @@ -878,7 +886,7 @@ {:on-conflict-do-nothing overwrite?})))) (doseq [item (:thumbnails @*state*)] - (l/dbg :hint "inserting file tagged object thumbnail" + (l/dbg :hint "inserting file object thumbnail" :file-id (:file-id item) :object-id (:object-id item) ::l/sync? true) diff --git a/backend/src/app/rpc/commands/files_update.clj b/backend/src/app/rpc/commands/files_update.clj index 6da9e86e0..8b8eb6bfd 100644 --- a/backend/src/app/rpc/commands/files_update.clj +++ b/backend/src/app/rpc/commands/files_update.clj @@ -266,6 +266,14 @@ ;; Retrieve and return lagged data (get-lagged-changes conn params)))) +(defn- soft-validate-file-schema! + [file] + (try + (val/validate-file-schema! file) + (catch Throwable cause + (l/error :hint "file schema validation error" :cause cause))) + + file) (defn- update-file-data [conn file changes skip-validate] @@ -295,8 +303,14 @@ ;; If `libs` is defined, then full validation is performed (cond-> (and (contains? cf/flags :file-validation) (not skip-validate)) - (-> (val/validate-file! libs) - (val/validate-file-schema!))) + (val/validate-file! libs)) + + (cond-> (and (contains? cf/flags :file-schema-validation) + (not skip-validate)) + (val/validate-file-schema!)) + + (cond-> (contains? cf/flags :soft-file-schema-validation) + (soft-validate-file-schema!)) (cond-> (and (contains? cfeat/*current* "fdata/objects-map") (not (contains? cfeat/*previous* "fdata/objects-map"))) diff --git a/backend/src/app/srepl/components_v2.clj b/backend/src/app/srepl/components_v2.clj index 8f24021c3..01a1bbeb5 100644 --- a/backend/src/app/srepl/components_v2.clj +++ b/backend/src/app/srepl/components_v2.clj @@ -151,7 +151,9 @@ (ps/acquire! feat/*semaphore*) (px/submit! scope (fn [] (-> (assoc system ::db/rollback rollback?) - (feat/migrate-file! file-id :validate? validate?))))) + (feat/migrate-file! file-id + :validate? validate? + :throw-on-validate? (not skip-on-error)))))) (get-candidates)) (p/await! scope)) @@ -182,7 +184,9 @@ (binding [feat/*stats* stats feat/*skip-on-error* skip-on-error] (-> (assoc system ::db/rollback rollback?) - (feat/migrate-team! team-id :validate? validate?)) + (feat/migrate-team! team-id + :validate? validate? + :throw-on-validate? (not skip-on-error))) (print-stats! (-> (deref feat/*stats*) @@ -238,7 +242,9 @@ (migrate-team [team-id] (try (-> (assoc system ::db/rollback rollback?) - (feat/migrate-team! team-id :validate? validate? :throw-on-validate? (not skip-on-error))) + (feat/migrate-team! team-id + :validate? validate? + :throw-on-validate? (not skip-on-error))) (catch Throwable cause (l/err :hint "unexpected error on processing team" :team-id (dm/str team-id) :cause cause))))