diff --git a/backend/src/app/binfile/v3.clj b/backend/src/app/binfile/v3.clj index e28d07a76..70a9960e6 100644 --- a/backend/src/app/binfile/v3.clj +++ b/backend/src/app/binfile/v3.clj @@ -617,8 +617,7 @@ (let [object (->> (read-entry input entry) (clean-component-pre-decode) (decode-component) - (clean-component-post-decode) - (validate-component))] + (clean-component-post-decode))] (if (= id (:id object)) (assoc result id object) result))) @@ -652,8 +651,7 @@ (let [object (->> (read-entry input entry) (bfl/clean-shape-pre-decode) (decode-shape) - (bfl/clean-shape-post-decode) - (validate-shape))] + (bfl/clean-shape-post-decode))] (if (= id (:id object)) (assoc result id object) result))) @@ -699,7 +697,6 @@ components (read-file-components cfg) plugin-data (read-file-plugin-data cfg) pages (read-file-pages cfg)] - {:pages (-> pages keys vec) :pages-index (into {} pages) :colors colors @@ -756,7 +753,8 @@ (assoc :project-id project-id) (dissoc :options)) - file (bfc/process-file cfg file)] + file (bfc/process-file cfg file) + file (ctf/check-file file)] (bfm/register-pending-migrations! cfg file) (bfc/save-file! cfg file ::db/return-keys false) diff --git a/backend/test/backend_tests/rpc_file_test.clj b/backend/test/backend_tests/rpc_file_test.clj index 538cc82f0..9a46efd37 100644 --- a/backend/test/backend_tests/rpc_file_test.clj +++ b/backend/test/backend_tests/rpc_file_test.clj @@ -343,8 +343,9 @@ :name "image" :frame-id uuid/zero :parent-id uuid/zero - :type :image - :metadata {:id (:id fmo1) :width 100 :height 100 :mtype "image/jpeg"}})}]) + :type :rect + :fills [{:fill-opacity 1 + :fill-image {:id (:id fmo1) :width 100 :height 100 :mtype "image/jpeg"}}]})}]) ;; Check that reference storage objects on filemediaobjects ;; are the same because of deduplication feature. @@ -462,7 +463,8 @@ fmo3 (add-file-media-object :profile-id (:id profile) :file-id (:id file)) fmo4 (add-file-media-object :profile-id (:id profile) :file-id (:id file)) fmo5 (add-file-media-object :profile-id (:id profile) :file-id (:id file)) - s-shid (uuid/random) + s1-shid (uuid/random) + s2-shid (uuid/random) t-shid (uuid/random) page-id (first (get-in file [:data :pages]))] @@ -481,19 +483,31 @@ :changes [{:type :add-obj :page-id page-id - :id s-shid + :id s1-shid :parent-id uuid/zero :frame-id uuid/zero :components-v2 true :obj (cts/setup-shape - {:id s-shid + {:id s1-shid :name "image" :frame-id uuid/zero :parent-id uuid/zero - :type :image - :metadata {:id (:id fmo1) :width 100 :height 100 :mtype "image/jpeg"} - :fills [{:opacity 1 :fill-image {:id (:id fmo2) :width 100 :height 100 :mtype "image/jpeg"}}] - :strokes [{:opacity 1 :stroke-image {:id (:id fmo3) :width 100 :height 100 :mtype "image/jpeg"}}]})} + :type :rect + :fills [{:fill-opacity 1 :fill-image {:id (:id fmo2) :width 101 :height 100 :mtype "image/jpeg"}}] + :strokes [{:stroke-opacity 1 :stroke-image {:id (:id fmo3) :width 102 :height 100 :mtype "image/jpeg"}}]})} + {:type :add-obj + :page-id page-id + :id s2-shid + :parent-id uuid/zero + :frame-id uuid/zero + :components-v2 true + :obj (cts/setup-shape + {:id s2-shid + :name "image" + :frame-id uuid/zero + :parent-id uuid/zero + :type :rect + :fills [{:fill-opacity 1 :fill-image {:id (:id fmo1) :width 103 :height 100 :mtype "image/jpeg"}}]})} {:type :add-obj :page-id page-id :id t-shid @@ -519,7 +533,8 @@ {:fills [{:fill-opacity 1 :fill-color "#000000"}] :text "bye"}]}]}]} - :strokes [{:opacity 1 :stroke-image {:id (:id fmo5) :width 100 :height 100 :mtype "image/jpeg"}}]})}]) + :strokes [{:stroke-opacity 1 :stroke-image {:id (:id fmo5) :width 100 :height 100 :mtype "image/jpeg"}}]})}]) + ;; run the file-gc task immediately without forced min-age (t/is (false? (th/run-task! :file-gc {:file-id (:id file)}))) @@ -557,10 +572,13 @@ :vern 0 :changes [{:type :del-obj :page-id (first (get-in file [:data :pages])) - :id s-shid} + :id s1-shid} {:type :del-obj :page-id (first (get-in file [:data :pages])) - :id t-shid}]) + :id t-shid} + {:type :del-obj + :page-id (first (get-in file [:data :pages])) + :id s2-shid}]) ;; Now, we have deleted the usage of pointers to the ;; file-media-objects, if we paste file-gc, they should be marked diff --git a/common/src/app/common/files/migrations.cljc b/common/src/app/common/files/migrations.cljc index 038e28915..34c96cc64 100644 --- a/common/src/app/common/files/migrations.cljc +++ b/common/src/app/common/files/migrations.cljc @@ -22,7 +22,7 @@ [app.common.schema :as sm] [app.common.svg :as csvg] [app.common.text :as txt] - [app.common.types.color :as ctc] + [app.common.types.color :as types.color] [app.common.types.component :as ctk] [app.common.types.container :as ctn] [app.common.types.file :as ctf] @@ -1009,7 +1009,7 @@ [data _] (let [update-colors (fn [colors] - (into {} (filter #(-> % val ctc/valid-color?) colors)))] + (into {} (filter #(-> % val types.color/valid-color?) colors)))] (update data :colors update-colors))) (defmethod migrate-data "legacy-52" @@ -1351,59 +1351,6 @@ (update data :pages-index d/update-vals update-page))) -(defmethod migrate-data "0005-deprecate-image-type" - [data _] - (letfn [(update-object [object] - (if (cfh/image-shape? object) - (let [metadata (:metadata object) - fills (into [{:fill-image (assoc metadata :keep-aspect-ratio false) - :opacity 1}] - (:fills object))] - (-> object - (assoc :fills fills) - (dissoc :metadata) - (assoc :type :rect))) - object)) - - (update-container [container] - (d/update-when container :objects d/update-vals update-object))] - - (-> data - (update :pages-index d/update-vals update-container) - (d/update-when :components d/update-vals update-container)))) - -(defmethod migrate-data "0006-fix-old-texts-fills" - [data _] - (letfn [(fix-fills [node] - (let [fills (cond - (or (some? (:fill-color node)) - (some? (:fill-opacity node)) - (some? (:fill-color-gradient node))) - [(d/without-nils (select-keys node [:fill-color :fill-opacity :fill-color-gradient - :fill-color-ref-id :fill-color-ref-file]))] - - (nil? (:fills node)) - [{:fill-color "#000000" :fill-opacity 1}] - - :else - (:fills node))] - (-> node - (assoc :fills fills) - (dissoc :fill-color :fill-opacity :fill-color-gradient - :fill-color-ref-id :fill-color-ref-file)))) - - (update-object [object] - (if (cfh/text-shape? object) - (update object :content (partial txt/transform-nodes identity fix-fills)) - object)) - - (update-container [container] - (d/update-when container :objects d/update-vals update-object))] - - (-> data - (update :pages-index d/update-vals update-container) - (d/update-when :components d/update-vals update-container)))) - (defmethod migrate-data "0004-clean-shadow-and-colors" [data _] (letfn [(clean-shadow [shadow] @@ -1431,6 +1378,125 @@ (d/update-when :components d/update-vals update-container) (d/update-when :colors d/update-vals clean-library-color)))) +(defmethod migrate-data "0005-deprecate-image-type" + [data _] + (letfn [(update-object [object] + (if (cfh/image-shape? object) + (let [metadata (:metadata object) + fills (into [{:fill-image (assoc metadata :keep-aspect-ratio false) + :opacity 1}] + (:fills object))] + (-> object + (assoc :fills fills) + (dissoc :metadata) + (assoc :type :rect))) + object)) + + (update-container [container] + (d/update-when container :objects d/update-vals update-object))] + + (-> data + (update :pages-index d/update-vals update-container) + (d/update-when :components d/update-vals update-container)))) + +(defmethod migrate-data "0006-fix-old-texts-fills" + [data _] + (letfn [(fix-fills [node] + (let [fills (if (and (not (seq (:fills node))) + (or (some? (:fill-color node)) + (some? (:fill-opacity node)) + (some? (:fill-color-gradient node)))) + [(d/without-nils (select-keys node [:fill-color :fill-opacity :fill-color-gradient + :fill-color-ref-id :fill-color-ref-file]))] + (:fills node))] + (-> node + (assoc :fills fills) + (dissoc :fill-color :fill-opacity :fill-color-gradient + :fill-color-ref-id :fill-color-ref-file)))) + + (update-object [object] + (if (cfh/text-shape? object) + (update object :content (partial txt/transform-nodes identity fix-fills)) + object)) + + (update-container [container] + (d/update-when container :objects d/update-vals update-object))] + + (-> data + (update :pages-index d/update-vals update-container) + (d/update-when :components d/update-vals update-container)))) + +(def ^:private valid-stroke? + (sm/lazy-validator cts/schema:stroke)) + +(defmethod migrate-data "0007-clear-invalid-strokes-and-fills" + [data _] + (letfn [(clear-color-image [image] + (select-keys image types.color/image-attrs)) + + (clear-color-gradient [gradient] + (select-keys gradient types.color/gradient-attrs)) + + (clear-stroke [stroke] + (-> stroke + (select-keys cts/stroke-attrs) + (d/update-when :stroke-color-gradient clear-color-gradient) + (d/update-when :stroke-image clear-color-image))) + + (fix-strokes [strokes] + (->> (map clear-stroke strokes) + (filterv valid-stroke?))) + + ;; Fixes shapes with nested :fills in the :fills attribute + ;; introduced in a migration `0006-fix-old-texts-fills` when + ;; txt/transform-nodes with identity pred was broken + (remove-nested-fills [[fill :as fills]] + (if (and (= 1 (count fills)) + (contains? fill :fills)) + (:fills fill) + fills)) + + (clear-fill [fill] + (-> fill + (select-keys types.fill/fill-attrs) + (d/update-when :fill-image clear-color-image) + (d/update-when :fill-color-gradient clear-color-gradient))) + + (fix-fills [fills] + (->> fills + (remove-nested-fills) + (map clear-fill) + (filterv valid-fill?))) + + (fix-object [object] + (-> object + (d/update-when :strokes fix-strokes) + (d/update-when :fills fix-fills))) + + (update-shape [object] + (-> object + (fix-object) + ;; The text shape also can has strokes and fils on the + ;; text fragments so we need to fix them there + (cond-> (cfh/text-shape? object) + (update :content (partial txt/transform-nodes identity fix-object))))) + + (update-container [container] + (d/update-when container :objects d/update-vals update-shape))] + + (-> data + (update :pages-index d/update-vals update-container) + (d/update-when :components d/update-vals update-container)))) + +(defmethod migrate-data "0008-fix-library-colors-opacity" + [data _] + (letfn [(update-color [color] + (if (and (contains? color :opacity) + (nil? (get color :opacity))) + (assoc color :opacity 1) + color))] + (d/update-when data :colors d/update-vals update-color))) + (def available-migrations (into (d/ordered-set) ["legacy-2" @@ -1493,4 +1559,6 @@ "0004-add-partial-text-touched-flags" "0004-clean-shadow-and-colors" "0005-deprecate-image-type" - "0006-fix-old-texts-fills"])) + "0006-fix-old-texts-fills" + "0007-clear-invalid-strokes-and-fills" + "0008-fix-library-colors-opacity"])) diff --git a/common/src/app/common/text.cljc b/common/src/app/common/text.cljc index ad86914ac..5c6bcf313 100644 --- a/common/src/app/common/text.cljc +++ b/common/src/app/common/text.cljc @@ -121,30 +121,6 @@ {:name "Source Sans Pro Regular"} (select-keys default-text-attrs typography-fields))) -(defn transform-nodes - ([transform root] - (transform-nodes identity transform root)) - ([pred transform root] - (walk/postwalk - (fn [item] - (if (and (map? item) (pred item)) - (transform item) - item)) - root))) - -(defn xform-nodes - "The same as transform but instead of receiving a funcion, receives - a transducer." - [xf root] - (let [rf (fn [_ v] v)] - (walk/postwalk - (fn [item] - (let [rf (xf rf)] - (if (map? item) - (d/nilv (rf nil item) item) - item))) - root))) - (defn node-seq ([root] (node-seq identity root)) ([match? root] @@ -165,6 +141,34 @@ [node] (= "root" (:type node))) +(defn is-node? + [node] + (or (is-text-node? node) (is-paragraph-node? node) (is-root-node? node))) + +(defn transform-nodes + ([transform root] + (transform-nodes identity transform root)) + ([pred transform root] + (walk/postwalk + (fn [item] + (if (and (is-node? item) (pred item)) + (transform item) + item)) + root))) + +(defn xform-nodes + "The same as transform but instead of receiving a funcion, receives + a transducer." + [xf root] + (let [rf (fn [_ v] v)] + (walk/postwalk + (fn [item] + (let [rf (xf rf)] + (if (is-node? item) + (d/nilv (rf nil item) item) + item))) + root))) + (defn generate-shape-name [text] (subs text 0 (min 280 (count text)))) diff --git a/common/src/app/common/types/color.cljc b/common/src/app/common/types/color.cljc index 0b163dc15..9173af243 100644 --- a/common/src/app/common/types/color.cljc +++ b/common/src/app/common/types/color.cljc @@ -79,6 +79,10 @@ [:name {:optional true} ::sm/text] [:keep-aspect-ratio {:optional true} :boolean]]) +(def image-attrs + "A set of attrs that corresponds to image data type" + (sm/keys schema:image)) + (def schema:image-color [:map [:image schema:image]]) @@ -100,6 +104,10 @@ [:opacity {:optional true} [::sm/number {:min 0 :max 1}]] [:offset [::sm/number {:min 0 :max 1}]]]]]]) +(def gradient-attrs + "A set of attrs that corresponds to gradient data type" + (sm/keys schema:gradient)) + (def schema:gradient-color [:map [:gradient schema:gradient]]) diff --git a/common/src/app/common/types/fill.cljc b/common/src/app/common/types/fill.cljc index 2cfcda37c..827b7e24f 100644 --- a/common/src/app/common/types/fill.cljc +++ b/common/src/app/common/types/fill.cljc @@ -19,7 +19,7 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (def schema:fill-attrs - [:map {:title "FillAttrs"} + [:map {:title "FillAttrs" :closed true} [:fill-color-ref-file {:optional true} ::sm/uuid] [:fill-color-ref-id {:optional true} ::sm/uuid] [:fill-opacity {:optional true} [::sm/number {:min 0 :max 1}]] diff --git a/common/src/app/common/types/shape.cljc b/common/src/app/common/types/shape.cljc index 41d54c6ec..3e473df67 100644 --- a/common/src/app/common/types/shape.cljc +++ b/common/src/app/common/types/shape.cljc @@ -135,7 +135,7 @@ (= 1 (count result)))) (def schema:stroke-attrs - [:map {:title "StrokeAttrs"} + [:map {:title "StrokeAttrs" :closed true} [:stroke-color-ref-file {:optional true} ::sm/uuid] [:stroke-color-ref-id {:optional true} ::sm/uuid] [:stroke-opacity {:optional true} ::sm/safe-number] @@ -152,6 +152,10 @@ [:stroke-color-gradient {:optional true} types.color/schema:gradient] [:stroke-image {:optional true} types.color/schema:image]]) +(def stroke-attrs + "A set of attrs that corresponds to stroke data type" + (sm/keys schema:stroke-attrs)) + (def schema:stroke (sm/register! ^{::sm/type ::stroke}