🐛 Add migrations to fix colors

backend/src/app/binfile/v3.clj

@@ -617,8 +617,7 @@
                    (let [object (->> (read-entry input entry)
                                      (clean-component-pre-decode)
                                      (decode-component)
-                                     (clean-component-post-decode)
+                                     (clean-component-post-decode))]
-                                     (validate-component))]
                      (if (= id (:id object))
                        (assoc result id object)
                        result)))
@@ -652,8 +651,7 @@
                  (let [object (->> (read-entry input entry)
                                    (bfl/clean-shape-pre-decode)
                                    (decode-shape)
-                                   (bfl/clean-shape-post-decode)
+                                   (bfl/clean-shape-post-decode))]
-                                   (validate-shape))]
                    (if (= id (:id object))
                      (assoc result id object)
                      result)))
@@ -699,7 +697,6 @@
         components   (read-file-components cfg)
         plugin-data  (read-file-plugin-data cfg)
         pages        (read-file-pages cfg)]
-
     {:pages (-> pages keys vec)
      :pages-index (into {} pages)
      :colors colors
@@ -756,7 +753,8 @@
                    (assoc :project-id project-id)
                    (dissoc :options))
 
-          file  (bfc/process-file cfg file)]
+          file  (bfc/process-file cfg file)
+          file  (ctf/check-file file)]
 
       (bfm/register-pending-migrations! cfg file)
       (bfc/save-file! cfg file ::db/return-keys false)

backend/test/backend_tests/rpc_file_test.clj

@@ -343,8 +343,9 @@
                 :name "image"
                 :frame-id uuid/zero
                 :parent-id uuid/zero
-                :type :image
+                :type :rect
-                :metadata {:id (:id fmo1) :width 100 :height 100 :mtype "image/jpeg"}})}])
+                :fills [{:fill-opacity 1
+                         :fill-image {:id (:id fmo1) :width 100 :height 100 :mtype "image/jpeg"}}]})}])
 
       ;; Check that reference storage objects on filemediaobjects
       ;; are the same because of deduplication feature.
@@ -462,7 +463,8 @@
           fmo3    (add-file-media-object :profile-id (:id profile) :file-id (:id file))
           fmo4    (add-file-media-object :profile-id (:id profile) :file-id (:id file))
           fmo5    (add-file-media-object :profile-id (:id profile) :file-id (:id file))
-          s-shid  (uuid/random)
+          s1-shid  (uuid/random)
+          s2-shid  (uuid/random)
           t-shid  (uuid/random)
 
           page-id (first (get-in file [:data :pages]))]
@@ -481,19 +483,31 @@
        :changes
        [{:type :add-obj
          :page-id page-id
-         :id s-shid
+         :id s1-shid
          :parent-id uuid/zero
          :frame-id uuid/zero
          :components-v2 true
          :obj (cts/setup-shape
-               {:id s-shid
+               {:id s1-shid
                 :name "image"
                 :frame-id uuid/zero
                 :parent-id uuid/zero
-                :type :image
+                :type :rect
-                :metadata {:id (:id fmo1) :width 100 :height 100 :mtype "image/jpeg"}
+                :fills [{:fill-opacity 1 :fill-image {:id (:id fmo2) :width 101 :height 100 :mtype "image/jpeg"}}]
-                :fills [{:opacity 1 :fill-image {:id (:id fmo2) :width 100 :height 100 :mtype "image/jpeg"}}]
+                :strokes [{:stroke-opacity 1 :stroke-image {:id (:id fmo3) :width 102 :height 100 :mtype "image/jpeg"}}]})}
-                :strokes [{:opacity 1 :stroke-image {:id (:id fmo3) :width 100 :height 100 :mtype "image/jpeg"}}]})}
+        {:type :add-obj
+         :page-id page-id
+         :id s2-shid
+         :parent-id uuid/zero
+         :frame-id uuid/zero
+         :components-v2 true
+         :obj (cts/setup-shape
+               {:id s2-shid
+                :name "image"
+                :frame-id uuid/zero
+                :parent-id uuid/zero
+                :type :rect
+                :fills [{:fill-opacity 1 :fill-image {:id (:id fmo1) :width 103 :height 100 :mtype "image/jpeg"}}]})}
         {:type :add-obj
          :page-id page-id
          :id t-shid
@@ -519,7 +533,8 @@
                                                              {:fills [{:fill-opacity 1
                                                                        :fill-color "#000000"}]
                                                               :text "bye"}]}]}]}
-                :strokes [{:opacity 1 :stroke-image {:id (:id fmo5) :width 100 :height 100 :mtype "image/jpeg"}}]})}])
+                :strokes [{:stroke-opacity 1 :stroke-image {:id (:id fmo5) :width 100 :height 100 :mtype "image/jpeg"}}]})}])
+
 
       ;; run the file-gc task immediately without forced min-age
       (t/is (false? (th/run-task! :file-gc {:file-id (:id file)})))
@@ -557,10 +572,13 @@
        :vern 0
        :changes [{:type :del-obj
                   :page-id (first (get-in file [:data :pages]))
-                  :id s-shid}
+                  :id s1-shid}
                  {:type :del-obj
                   :page-id (first (get-in file [:data :pages]))
-                  :id t-shid}])
+                  :id t-shid}
+                 {:type :del-obj
+                  :page-id (first (get-in file [:data :pages]))
+                  :id s2-shid}])
 
       ;; Now, we have deleted the usage of pointers to the
       ;; file-media-objects, if we paste file-gc, they should be marked

common/src/app/common/files/migrations.cljc

@@ -22,7 +22,7 @@
    [app.common.schema :as sm]
    [app.common.svg :as csvg]
    [app.common.text :as txt]
-   [app.common.types.color :as ctc]
+   [app.common.types.color :as types.color]
    [app.common.types.component :as ctk]
    [app.common.types.container :as ctn]
    [app.common.types.file :as ctf]
@@ -1009,7 +1009,7 @@
   [data _]
   (let [update-colors
         (fn [colors]
-          (into {} (filter #(-> % val ctc/valid-color?) colors)))]
+          (into {} (filter #(-> % val types.color/valid-color?) colors)))]
     (update data :colors update-colors)))
 
 (defmethod migrate-data "legacy-52"
@@ -1351,59 +1351,6 @@
 
     (update data :pages-index d/update-vals update-page)))
 
-(defmethod migrate-data "0005-deprecate-image-type"
-  [data _]
-  (letfn [(update-object [object]
-            (if (cfh/image-shape? object)
-              (let [metadata (:metadata object)
-                    fills (into [{:fill-image (assoc metadata :keep-aspect-ratio false)
-                                  :opacity    1}]
-                                (:fills object))]
-                (-> object
-                    (assoc :fills fills)
-                    (dissoc :metadata)
-                    (assoc :type :rect)))
-              object))
-
-          (update-container [container]
-            (d/update-when container :objects d/update-vals update-object))]
-
-    (-> data
-        (update :pages-index d/update-vals update-container)
-        (d/update-when :components d/update-vals update-container))))
-
-(defmethod migrate-data "0006-fix-old-texts-fills"
-  [data _]
-  (letfn [(fix-fills [node]
-            (let [fills (cond
-                          (or (some? (:fill-color node))
-                              (some? (:fill-opacity node))
-                              (some? (:fill-color-gradient node)))
-                          [(d/without-nils (select-keys node [:fill-color :fill-opacity :fill-color-gradient
-                                                              :fill-color-ref-id :fill-color-ref-file]))]
-
-                          (nil? (:fills node))
-                          [{:fill-color "#000000" :fill-opacity 1}]
-
-                          :else
-                          (:fills node))]
-              (-> node
-                  (assoc :fills fills)
-                  (dissoc :fill-color :fill-opacity :fill-color-gradient
-                          :fill-color-ref-id :fill-color-ref-file))))
-
-          (update-object [object]
-            (if (cfh/text-shape? object)
-              (update object :content (partial txt/transform-nodes identity fix-fills))
-              object))
-
-          (update-container [container]
-            (d/update-when container :objects d/update-vals update-object))]
-
-    (-> data
-        (update :pages-index d/update-vals update-container)
-        (d/update-when :components d/update-vals update-container))))
-
 (defmethod migrate-data "0004-clean-shadow-and-colors"
   [data _]
   (letfn [(clean-shadow [shadow]
@@ -1431,6 +1378,125 @@
         (d/update-when :components d/update-vals update-container)
         (d/update-when :colors d/update-vals clean-library-color))))
 
+(defmethod migrate-data "0005-deprecate-image-type"
+  [data _]
+  (letfn [(update-object [object]
+            (if (cfh/image-shape? object)
+              (let [metadata (:metadata object)
+                    fills (into [{:fill-image (assoc metadata :keep-aspect-ratio false)
+                                  :opacity    1}]
+                                (:fills object))]
+                (-> object
+                    (assoc :fills fills)
+                    (dissoc :metadata)
+                    (assoc :type :rect)))
+              object))
+
+          (update-container [container]
+            (d/update-when container :objects d/update-vals update-object))]
+
+    (-> data
+        (update :pages-index d/update-vals update-container)
+        (d/update-when :components d/update-vals update-container))))
+
+(defmethod migrate-data "0006-fix-old-texts-fills"
+  [data _]
+  (letfn [(fix-fills [node]
+            (let [fills (if (and (not (seq (:fills node)))
+                                 (or (some? (:fill-color node))
+                                     (some? (:fill-opacity node))
+                                     (some? (:fill-color-gradient node))))
+                          [(d/without-nils (select-keys node [:fill-color :fill-opacity :fill-color-gradient
+                                                              :fill-color-ref-id :fill-color-ref-file]))]
+                          (:fills node))]
+              (-> node
+                  (assoc :fills fills)
+                  (dissoc :fill-color :fill-opacity :fill-color-gradient
+                          :fill-color-ref-id :fill-color-ref-file))))
+
+          (update-object [object]
+            (if (cfh/text-shape? object)
+              (update object :content (partial txt/transform-nodes identity fix-fills))
+              object))
+
+          (update-container [container]
+            (d/update-when container :objects d/update-vals update-object))]
+
+    (-> data
+        (update :pages-index d/update-vals update-container)
+        (d/update-when :components d/update-vals update-container))))
+
+(def ^:private valid-stroke?
+  (sm/lazy-validator cts/schema:stroke))
+
+(defmethod migrate-data "0007-clear-invalid-strokes-and-fills"
+  [data _]
+  (letfn [(clear-color-image [image]
+            (select-keys image types.color/image-attrs))
+
+          (clear-color-gradient [gradient]
+            (select-keys gradient types.color/gradient-attrs))
+
+          (clear-stroke [stroke]
+            (-> stroke
+                (select-keys cts/stroke-attrs)
+                (d/update-when :stroke-color-gradient clear-color-gradient)
+                (d/update-when :stroke-image clear-color-image)))
+
+          (fix-strokes [strokes]
+            (->> (map clear-stroke strokes)
+                 (filterv valid-stroke?)))
+
+          ;; Fixes shapes with nested :fills in the :fills attribute
+          ;; introduced in a migration `0006-fix-old-texts-fills` when
+          ;; txt/transform-nodes with identity pred was broken
+          (remove-nested-fills [[fill :as fills]]
+            (if (and (= 1 (count fills))
+                     (contains? fill :fills))
+              (:fills fill)
+              fills))
+
+          (clear-fill [fill]
+            (-> fill
+                (select-keys types.fill/fill-attrs)
+                (d/update-when :fill-image clear-color-image)
+                (d/update-when :fill-color-gradient clear-color-gradient)))
+
+          (fix-fills [fills]
+            (->> fills
+                 (remove-nested-fills)
+                 (map clear-fill)
+                 (filterv valid-fill?)))
+
+          (fix-object [object]
+            (-> object
+                (d/update-when :strokes fix-strokes)
+                (d/update-when :fills fix-fills)))
+
+          (update-shape [object]
+            (-> object
+                (fix-object)
+                ;; The text shape also can has strokes and fils on the
+                ;; text fragments so we need to fix them there
+                (cond-> (cfh/text-shape? object)
+                  (update :content (partial txt/transform-nodes identity fix-object)))))
+
+          (update-container [container]
+            (d/update-when container :objects d/update-vals update-shape))]
+
+    (-> data
+        (update :pages-index d/update-vals update-container)
+        (d/update-when :components d/update-vals update-container))))
+
+(defmethod migrate-data "0008-fix-library-colors-opacity"
+  [data _]
+  (letfn [(update-color [color]
+            (if (and (contains? color :opacity)
+                     (nil? (get color :opacity)))
+              (assoc color :opacity 1)
+              color))]
+    (d/update-when data :colors d/update-vals update-color)))
+
 (def available-migrations
   (into (d/ordered-set)
         ["legacy-2"
@@ -1493,4 +1559,6 @@
          "0004-add-partial-text-touched-flags"
          "0004-clean-shadow-and-colors"
          "0005-deprecate-image-type"
-         "0006-fix-old-texts-fills"]))
+         "0006-fix-old-texts-fills"
+         "0007-clear-invalid-strokes-and-fills"
+         "0008-fix-library-colors-opacity"]))

common/src/app/common/text.cljc

@@ -121,30 +121,6 @@
    {:name "Source Sans Pro Regular"}
    (select-keys default-text-attrs typography-fields)))
 
-(defn transform-nodes
-  ([transform root]
-   (transform-nodes identity transform root))
-  ([pred transform root]
-   (walk/postwalk
-    (fn [item]
-      (if (and (map? item) (pred item))
-        (transform item)
-        item))
-    root)))
-
-(defn xform-nodes
-  "The same as transform but instead of receiving a funcion, receives
-  a transducer."
-  [xf root]
-  (let [rf (fn [_ v] v)]
-    (walk/postwalk
-     (fn [item]
-       (let [rf (xf rf)]
-         (if (map? item)
-           (d/nilv (rf nil item) item)
-           item)))
-     root)))
-
 (defn node-seq
   ([root] (node-seq identity root))
   ([match? root]
@@ -165,6 +141,34 @@
   [node]
   (= "root" (:type node)))
 
+(defn is-node?
+  [node]
+  (or (is-text-node? node) (is-paragraph-node? node) (is-root-node? node)))
+
+(defn transform-nodes
+  ([transform root]
+   (transform-nodes identity transform root))
+  ([pred transform root]
+   (walk/postwalk
+    (fn [item]
+      (if (and (is-node? item) (pred item))
+        (transform item)
+        item))
+    root)))
+
+(defn xform-nodes
+  "The same as transform but instead of receiving a funcion, receives
+  a transducer."
+  [xf root]
+  (let [rf (fn [_ v] v)]
+    (walk/postwalk
+     (fn [item]
+       (let [rf (xf rf)]
+         (if (is-node? item)
+           (d/nilv (rf nil item) item)
+           item)))
+     root)))
+
 (defn generate-shape-name
   [text]
   (subs text 0 (min 280 (count text))))

common/src/app/common/types/color.cljc

@@ -79,6 +79,10 @@
    [:name {:optional true} ::sm/text]
    [:keep-aspect-ratio {:optional true} :boolean]])
 
+(def image-attrs
+  "A set of attrs that corresponds to image data type"
+  (sm/keys schema:image))
+
 (def schema:image-color
   [:map [:image schema:image]])
 
@@ -100,6 +104,10 @@
       [:opacity {:optional true} [::sm/number {:min 0 :max 1}]]
       [:offset [::sm/number {:min 0 :max 1}]]]]]])
 
+(def gradient-attrs
+  "A set of attrs that corresponds to gradient data type"
+  (sm/keys schema:gradient))
+
 (def schema:gradient-color
   [:map [:gradient schema:gradient]])
 

common/src/app/common/types/fill.cljc

@@ -19,7 +19,7 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (def schema:fill-attrs
-  [:map {:title "FillAttrs"}
+  [:map {:title "FillAttrs" :closed true}
    [:fill-color-ref-file {:optional true} ::sm/uuid]
    [:fill-color-ref-id {:optional true} ::sm/uuid]
    [:fill-opacity {:optional true} [::sm/number {:min 0 :max 1}]]

common/src/app/common/types/shape.cljc

@@ -135,7 +135,7 @@
     (= 1 (count result))))
 
 (def schema:stroke-attrs
-  [:map {:title "StrokeAttrs"}
+  [:map {:title "StrokeAttrs" :closed true}
    [:stroke-color-ref-file {:optional true} ::sm/uuid]
    [:stroke-color-ref-id {:optional true} ::sm/uuid]
    [:stroke-opacity {:optional true} ::sm/safe-number]
@@ -152,6 +152,10 @@
    [:stroke-color-gradient {:optional true} types.color/schema:gradient]
    [:stroke-image {:optional true} types.color/schema:image]])
 
+(def stroke-attrs
+  "A set of attrs that corresponds to stroke data type"
+  (sm/keys schema:stroke-attrs))
+
 (def schema:stroke
   (sm/register!
    ^{::sm/type ::stroke}