From 7d0c19fcc77595b651ae23f47e3ed799f263882b Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 19 Feb 2025 11:56:28 +0100 Subject: [PATCH 1/5] :bug: Add correct feature check on manifest reading Instead on the file save operation so we can raise exception if something does not match without processing the whole file --- backend/src/app/binfile/common.clj | 11 +++++------ backend/src/app/binfile/v3.clj | 7 +++++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/backend/src/app/binfile/common.clj b/backend/src/app/binfile/common.clj index a0b2566600..8cd4f8cd3c 100644 --- a/backend/src/app/binfile/common.clj +++ b/backend/src/app/binfile/common.clj @@ -513,12 +513,11 @@ (assoc :ignore-sync-until (dt/plus timestamp (dt/duration {:seconds 5}))) (update :features (fn [features] - (let [features (cfeat/check-supported-features! features)] - (-> (::features cfg #{}) - (set/union features) - ;; We never want to store - ;; frontend-only features on file - (set/difference cfeat/frontend-only-features))))))] + (-> (::features cfg #{}) + (set/union features) + ;; We never want to store + ;; frontend-only features on file + (set/difference cfeat/frontend-only-features)))))] (when (contains? cf/flags :file-schema-validation) (fval/validate-file-schema! file)) diff --git a/backend/src/app/binfile/v3.clj b/backend/src/app/binfile/v3.clj index 006fc5ff00..dcda354553 100644 --- a/backend/src/app/binfile/v3.clj +++ b/backend/src/app/binfile/v3.clj @@ -875,14 +875,17 @@ :manifest manifest)) ;; Check if all files referenced on manifest are present - (doseq [{file-id :id} (:files manifest)] + (doseq [{file-id :id features :features} (:files manifest)] (let [path (str "files/" file-id ".json")] + (when-not (get-zip-entry input path) (ex/raise :type :validation :code :invalid-binfile-v3 :hint "some files referenced on manifest not found" :path path - :file-id file-id)))) + :file-id file-id)) + + (cfeat/check-supported-features! features))) (events/tap :progress {:section :manifest}) From a391d71b6018ff99e36c8daa2c6b8674962c2634 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 19 Feb 2025 12:12:38 +0100 Subject: [PATCH 2/5] :bug: Add correct feature handling on binfile import --- backend/src/app/binfile/common.clj | 1 - backend/src/app/rpc/commands/binfile.clj | 54 ++++++++++-------------- 2 files changed, 23 insertions(+), 32 deletions(-) diff --git a/backend/src/app/binfile/common.clj b/backend/src/app/binfile/common.clj index 8cd4f8cd3c..f461ab7b68 100644 --- a/backend/src/app/binfile/common.clj +++ b/backend/src/app/binfile/common.clj @@ -541,7 +541,6 @@ file) - (defn apply-pending-migrations! "Apply alredy registered pending migrations to files" [cfg] diff --git a/backend/src/app/rpc/commands/binfile.clj b/backend/src/app/rpc/commands/binfile.clj index 358e23ea58..49ab841931 100644 --- a/backend/src/app/rpc/commands/binfile.clj +++ b/backend/src/app/rpc/commands/binfile.clj @@ -10,8 +10,10 @@ [app.binfile.common :as bfc] [app.binfile.v1 :as bf.v1] [app.binfile.v3 :as bf.v3] + [app.common.features :as cfeat] [app.common.logging :as l] [app.common.schema :as sm] + [app.config :as cf] [app.db :as db] [app.http.sse :as sse] [app.loggers.audit :as-alias audit] @@ -20,6 +22,7 @@ [app.rpc :as-alias rpc] [app.rpc.commands.files :as files] [app.rpc.commands.projects :as projects] + [app.rpc.commands.teams :as teams] [app.rpc.doc :as-alias doc] [app.tasks.file-gc] [app.util.services :as sv] @@ -91,41 +94,30 @@ ;; --- Command: import-binfile -(defn- import-binfile-v1 - [{:keys [::wrk/executor] :as cfg} {:keys [project-id profile-id name file]}] - (let [cfg (-> cfg - (assoc ::bfc/project-id project-id) - (assoc ::bfc/profile-id profile-id) - (assoc ::bfc/name name) - (assoc ::bfc/input (:path file)))] - - ;; NOTE: the importation process performs some operations that are - ;; not very friendly with virtual threads, and for avoid - ;; unexpected blocking of other concurrent operations we dispatch - ;; that operation to a dedicated executor. - (px/invoke! executor (partial bf.v1/import-files! cfg)))) - -(defn- import-binfile-v3 - [{:keys [::wrk/executor] :as cfg} {:keys [project-id profile-id name file]}] - (let [cfg (-> cfg - (assoc ::bfc/project-id project-id) - (assoc ::bfc/profile-id profile-id) - (assoc ::bfc/name name) - (assoc ::bfc/input (:path file)))] - ;; NOTE: the importation process performs some operations that are - ;; not very friendly with virtual threads, and for avoid - ;; unexpected blocking of other concurrent operations we dispatch - ;; that operation to a dedicated executor. - (px/invoke! executor (partial bf.v3/import-files! cfg)))) - (defn- import-binfile - [{:keys [::db/pool] :as cfg} {:keys [project-id version] :as params}] - (let [result (case (int version) - 1 (import-binfile-v1 cfg params) - 3 (import-binfile-v3 cfg params))] + [{:keys [::db/pool ::wrk/executor] :as cfg} {:keys [profile-id project-id version name file]}] + (let [team (teams/get-team pool + :profile-id profile-id + :project-id project-id) + cfg (-> cfg + (assoc ::bfc/features (cfeat/get-team-enabled-features cf/flags team)) + (assoc ::bfc/project-id project-id) + (assoc ::bfc/profile-id profile-id) + (assoc ::bfc/name name) + (assoc ::bfc/input (:path file))) + + ;; NOTE: the importation process performs some operations that are + ;; not very friendly with virtual threads, and for avoid + ;; unexpected blocking of other concurrent operations we dispatch + ;; that operation to a dedicated executor. + result (case (int version) + 1 (px/invoke! executor (partial bf.v1/import-files! cfg)) + 3 (px/invoke! executor (partial bf.v3/import-files! cfg)))] + (db/update! pool :project {:modified-at (dt/now)} {:id project-id}) + result)) (def ^:private schema:import-binfile From ca9b5b1b8a98e27925153a93b14332d9418dca7a Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 19 Feb 2025 12:13:40 +0100 Subject: [PATCH 3/5] :paperclip: Use standard asserts on binfile common ns --- backend/src/app/binfile/common.clj | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/backend/src/app/binfile/common.clj b/backend/src/app/binfile/common.clj index f461ab7b68..0db55f664b 100644 --- a/backend/src/app/binfile/common.clj +++ b/backend/src/app/binfile/common.clj @@ -9,7 +9,6 @@ binfile format implementations and management rpc methods." (:require [app.common.data :as d] - [app.common.data.macros :as dm] [app.common.exceptions :as ex] [app.common.features :as cfeat] [app.common.files.helpers :as cfh] @@ -219,10 +218,8 @@ "Given a set of file-id's, return all matching relations with the libraries" [cfg ids] - (dm/assert! - "expected a set of uuids" - (and (set? ids) - (every? uuid? ids))) + (assert (set? ids) "expected a set of uuids") + (assert (every? uuid? ids) "expected a set of uuids") (db/run! cfg (fn [{:keys [::db/conn]}] (let [ids (db/create-array conn "uuid" ids) @@ -503,9 +500,7 @@ specific, should not be used outside of binfile domain" [{:keys [::timestamp] :as cfg} file & {:as opts}] - (dm/assert! - "expected valid timestamp" - (dt/instant? timestamp)) + (assert (dt/instant? timestamp) "expected valid timestamp") (let [file (-> file (assoc :created-at timestamp) From cb8e31e7f8d45565ec6b2e72ae9e3ee23b5be301 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 19 Feb 2025 22:32:42 +0100 Subject: [PATCH 4/5] :bug: Add correct handling of features on clone-template --- backend/src/app/rpc/commands/management.clj | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/src/app/rpc/commands/management.clj b/backend/src/app/rpc/commands/management.clj index 73d832fe3b..c04660f6a8 100644 --- a/backend/src/app/rpc/commands/management.clj +++ b/backend/src/app/rpc/commands/management.clj @@ -406,12 +406,16 @@ :prefix "penpot.template." :suffix "" :min-age "30m") - format (bfc/parse-file-format template) + format (bfc/parse-file-format template) + team (teams/get-team conn + :profile-id profile-id + :project-id project-id) cfg (-> cfg (assoc ::bfc/project-id project-id) (assoc ::bfc/profile-id profile-id) - (assoc ::bfc/input template)) + (assoc ::bfc/input template) + (assoc ::bfc/features (cfeat/get-team-enabled-features cf/flags team))) result (if (= format :binfile-v3) (px/invoke! executor (partial bf.v3/import-files! cfg)) From dd6ae81e838b193b00dc8de0f70f626acfc6a833 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 19 Feb 2025 22:40:11 +0100 Subject: [PATCH 5/5] :bug: Add correct feature handling on dbg binfile import --- backend/src/app/http/debug.clj | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/backend/src/app/http/debug.clj b/backend/src/app/http/debug.clj index 4eed5a4d0e..5c6f5cce19 100644 --- a/backend/src/app/http/debug.clj +++ b/backend/src/app/http/debug.clj @@ -12,6 +12,7 @@ [app.binfile.v3 :as bf.v3] [app.common.data :as d] [app.common.exceptions :as ex] + [app.common.features :as cfeat] [app.common.logging :as l] [app.common.pprint :as pp] [app.common.uuid :as uuid] @@ -21,6 +22,7 @@ [app.rpc.commands.auth :as auth] [app.rpc.commands.files-create :refer [create-file]] [app.rpc.commands.profile :as profile] + [app.rpc.commands.teams :as teams] [app.setup :as-alias setup] [app.srepl.helpers :as srepl] [app.storage :as-alias sto] @@ -317,7 +319,10 @@ :hint "missing upload file")) (let [profile (profile/get-profile pool profile-id) - project-id (:default-project-id profile)] + project-id (:default-project-id profile) + team (teams/get-team pool + :profile-id profile-id + :project-id project-id)] (when-not project-id (ex/raise :type :validation @@ -329,7 +334,8 @@ cfg (assoc cfg ::bfc/profile-id profile-id ::bfc/project-id project-id - ::bfc/input path)] + ::bfc/input path + ::bfc/features (cfeat/get-team-enabled-features cf/flags team))] (if (= format :binfile-v3) (bf.v3/import-files! cfg)