Merge pull request #6787 from penpot/niwinz-staging-hotfix-4

🐛 Several fixes

backend/src/app/rpc/commands/files.clj

@@ -560,6 +560,7 @@
           f.created_at,
           f.modified_at,
           f.name,
+          f.version,
           f.is_shared,
           ft.media_id,
           p.team_id

backend/src/app/srepl/fixes/lost_colors.clj

@@ -23,7 +23,7 @@
       AND label IS NOT NULL
       AND data IS NOT NULL
     ORDER BY created_at DESC
-    LIMIT 1")
+    LIMIT 2")
 
 (defn get-affected-migration
   [conn file-id]
@@ -33,11 +33,12 @@
 
 (defn get-last-valid-snapshot
   [conn migration]
-  (when-let [snapshot (db/exec-one! conn [sql:get-matching-snapshot
+  (let [[snapshot] (db/exec! conn [sql:get-matching-snapshot
-                                          (:file-id migration)
+                                   (:file-id migration)
-                                          (:created-at migration)])]
+                                   (:created-at migration)])]
-    (let [snapshot (assoc snapshot :id (:file-id snapshot))]
+    (when snapshot
-      (bfc/decode-file h/*system* snapshot))))
+      (let [snapshot (assoc snapshot :id (:file-id snapshot))]
+        (bfc/decode-file h/*system* snapshot)))))
 
 (defn restore-color
   [{:keys [data] :as snapshot} color]
@@ -47,25 +48,41 @@
 
 (defn restore-missing-colors
   [{:keys [id] :as file} & _opts]
-  (when-let [colors (-> file :data :colors not-empty)]
+  (l/inf :hint "process file" :file-id (str id) :name (:name file) :has-colors (-> file :data :colors not-empty boolean))
-    (when-let [migration (get-affected-migration h/*system* id)]
+  (if-let [colors (-> file :data :colors not-empty)]
-      (when-let [snapshot (get-last-valid-snapshot h/*system* migration)]
+    (let [migration (get-affected-migration h/*system* id)]
-        (let [colors (reduce-kv (fn [colors color-id color]
+      (if-let [snapshot (get-last-valid-snapshot h/*system* migration)]
-                                  (if-let [result (restore-color snapshot color)]
+        (do
-                                    (do
+          (l/inf :hint "using snapshot" :snapshot (:label snapshot))
-                                      (l/inf :hint "restored color" :file-id (str id) :color-id (str color-id))
+          (let [colors (reduce-kv (fn [colors color-id color]
-                                      (assoc colors color-id result))
+                                    (if-let [result (restore-color snapshot color)]
-                                    (do
+                                      (do
-                                      (l/wrn :hint "ignoring color" :file-id (str id) :color (pr-str color))
+                                        (l/inf :hint "restored color" :file-id (str id) :color-id (str color-id))
-                                      colors)))
+                                        (assoc colors color-id result))
-                                colors
+                                      (do
-                                colors)
+                                        (l/wrn :hint "ignoring color" :file-id (str id) :color (pr-str color))
-              file   (-> file
+                                        colors)))
-                         (update :data assoc :colors colors)
+                                  colors
-                         (update :migrations disj "0008-fix-library-colors-v2"))]
+                                  colors)
+                file   (-> file
+                           (update :data assoc :colors colors)
+                           (update :migrations disj "0008-fix-library-colors-v2"))]
 
+            (db/delete! h/*system* :file-migration
+                        {:name "0008-fix-library-colors-v2"
+                         :file-id (:id file)})
+            file))
+
+        (do
           (db/delete! h/*system* :file-migration
                       {:name "0008-fix-library-colors-v2"
                        :file-id (:id file)})
+          nil)))
+
+    (do
+      (db/delete! h/*system* :file-migration
+                  {:name "0008-fix-library-colors-v2"
+                   :file-id (:id file)})
+      nil)))
+
 
-          file)))))

backend/src/app/srepl/main.clj

@@ -17,6 +17,7 @@
    [app.common.files.validate :as cfv]
    [app.common.logging :as l]
    [app.common.pprint :as p]
+   [app.common.schema :as sm]
    [app.common.spec :as us]
    [app.common.uuid :as uuid]
    [app.config :as cf]
@@ -395,6 +396,22 @@
                         libs (bfc/get-resolved-file-libraries system file)]
                     (cfv/validate-file file libs))))))
 
+(defn validate-file-schema
+  "Validate structure, referencial integrity and semantic coherence of
+  all contents of a file. Returns a list of errors."
+  [file-id]
+  (let [file-id (h/parse-uuid file-id)]
+    (db/tx-run! (assoc main/system ::db/rollback true)
+                (fn [system]
+                  (try
+                    (let [file (bfc/get-file system file-id)]
+                      (cfv/validate-file-schema! file)
+                      (println "OK"))
+                    (catch Exception cause
+                      (if-let [explain (-> cause ex-data ::sm/explain)]
+                        (println (sm/humanize-explain explain))
+                        (ex/print-throwable cause))))))))
+
 (defn repair-file!
   "Repair the list of errors detected by validation."
   [file-id & {:keys [rollback?] :or {rollback? true} :as opts}]

common/src/app/common/files/migrations.cljc

@@ -855,7 +855,7 @@
 
           (update-object [object]
             (if (cfh/text-shape? object)
-              (update object :content #(txt/transform-nodes identity update-text-node %))
+              (update object :content #(txt/transform-nodes txt/is-content-node? update-text-node %))
               object))
 
           (update-container [container]
@@ -1105,7 +1105,7 @@
                 ;; The text shape also can has fills on the text
                 ;; fragments so we need to fix fills there
                 (cond-> (cfh/text-shape? object)
-                  (update :content (partial txt/transform-nodes identity fix-fills)))))
+                  (update :content (partial txt/transform-nodes txt/is-content-node? fix-fills)))))
 
           (update-container [container]
             (d/update-when container :objects d/update-vals update-object))]
@@ -1277,22 +1277,21 @@
         (update :pages-index d/update-vals update-container)
         (d/update-when :components d/update-vals update-container))))
 
-(defmethod migrate-data "0002-normalize-bool-content"
+(defmethod migrate-data "0002-normalize-bool-content-v2"
   [data _]
   (letfn [(update-object [object]
-            ;; NOTE: we still preserve the previous value for possible
+            (if (cfh/bool-shape? object)
-            ;; rollback, we still need to perform an other migration
+              (if (contains? object :content)
-            ;; for properly delete the bool-content prop from shapes
+                (dissoc object :bool-content)
-            ;; once the know the migration was OK
+                (let [content (:bool-content object)]
-            (if (and (cfh/bool-shape? object)
+                  (-> object
-                     (not (contains? object :content)))
+                      (assoc :content content)
-              (if-let [content (:bool-content object)]
+                      (dissoc :bool-content))))
-                (assoc object :content content)
+
-                object)
               (dissoc object :bool-content :bool-type)))
 
           (update-container [container]
-            (d/update-when container :objects update-vals update-object))]
+            (d/update-when container :objects d/update-vals update-object))]
 
     (-> data
         (update :pages-index d/update-vals update-container)
@@ -1331,38 +1330,43 @@
               object))
 
           (update-container [container]
-            (d/update-when container :objects update-vals update-object))]
+            (d/update-when container :objects d/update-vals update-object))]
 
     (-> data
         (update :pages-index d/update-vals update-container)
         (d/update-when :components d/update-vals update-container))))
 
-(defmethod migrate-data "0004-clean-shadow-and-colors"
+(defmethod migrate-data "0004-clean-shadow-color"
   [data _]
-  (letfn [(clean-shadow [shadow]
+  (let [decode-color (sm/decoder types.color/schema:color sm/json-transformer)
-            (update shadow :color (fn [color]
-                                    (let [ref-id   (get color :id)
-                                          ref-file (get color :file-id)]
-                                      (-> (d/without-qualified color)
-                                          (select-keys [:opacity :color :gradient :image :ref-id :ref-file])
-                                          (cond-> ref-id
-                                            (assoc :ref-id ref-id))
-                                          (cond-> ref-file
-                                            (assoc :ref-file ref-file)))))))
 
-          (update-object [object]
+        clean-shadow-color
-            (d/update-when object :shadow #(mapv clean-shadow %)))
+        (fn [color]
+          (let [ref-id   (get color :id)
+                ref-file (get color :file-id)]
+            (-> (d/without-qualified color)
+                (select-keys [:opacity :color :gradient :image :ref-id :ref-file])
+                (cond-> ref-id
+                  (assoc :ref-id ref-id))
+                (cond-> ref-file
+                  (assoc :ref-file ref-file))
+                (decode-color))))
 
-          (update-container [container]
+        clean-shadow
-            (d/update-when container :objects d/update-vals update-object))
+        (fn [shadow]
+          (update shadow :color clean-shadow-color))
 
-          (clean-library-color [color]
+        update-object
-            (dissoc color :file-id))]
+        (fn [object]
+          (d/update-when object :shadow #(mapv clean-shadow %)))
+
+        update-container
+        (fn [container]
+          (d/update-when container :objects d/update-vals update-object))]
 
     (-> data
         (update :pages-index d/update-vals update-container)
-        (d/update-when :components d/update-vals update-container)
+        (d/update-when :components d/update-vals update-container))))
-        (d/update-when :colors d/update-vals clean-library-color))))
 
 (defmethod migrate-data "0005-deprecate-image-type"
   [data _]
@@ -1402,7 +1406,7 @@
 
           (update-object [object]
             (if (cfh/text-shape? object)
-              (update object :content (partial txt/transform-nodes identity fix-fills))
+              (update object :content (partial txt/transform-nodes txt/is-content-node? fix-fills))
               object))
 
           (update-container [container]
@@ -1480,7 +1484,7 @@
         (update :pages-index d/update-vals update-container)
         (d/update-when :components d/update-vals update-container))))
 
-(defmethod migrate-data "0008-fix-library-colors-v3"
+(defmethod migrate-data "0008-fix-library-colors-v4"
   [data _]
   (letfn [(clear-color-opacity [color]
             (if (and (contains? color :opacity)
@@ -1491,7 +1495,8 @@
           (clear-color [color]
             (-> color
                 (select-keys types.color/library-color-attrs)
-                (clear-color-opacity)))]
+                (clear-color-opacity)
+                (d/without-nils)))]
 
     (d/update-when data :colors d/update-vals clear-color)))
 
@@ -1550,12 +1555,12 @@
          "legacy-66"
          "legacy-67"
          "0001-remove-tokens-from-groups"
-         "0002-normalize-bool-content"
+         "0002-normalize-bool-content-v2"
          "0002-clean-shape-interactions"
          "0003-fix-root-shape"
          "0003-convert-path-content"
-         "0004-clean-shadow-and-colors"
+         "0004-clean-shadow-color"
          "0005-deprecate-image-type"
          "0006-fix-old-texts-fills"
          "0007-clear-invalid-strokes-and-fills-v2"
-         "0008-fix-library-colors-v3"]))
+         "0008-fix-library-colors-v4"]))

common/src/app/common/text.cljc

@@ -130,8 +130,8 @@
 
 (defn is-text-node?
   [node]
-  (and (string? (:text node))
+  (and (nil? (:type node))
-       (not= (:text node) "")))
+       (string? (:text node))))
 
 (defn is-paragraph-set-node?
   [node]