✨ Add multi file import on tokens (#6444)

* ✨ Implement token multi-file import

* ♻️ Refactor import modal UI

* 🐛 Fix comments

---------

Co-authored-by: Florian Schroedl <flo.schroedl@gmail.com>

CHANGES.md

@@ -65,6 +65,7 @@ root.
 - Duplicate token sets [Taiga #10694](https://tree.taiga.io/project/penpot/issue/10694)
 - Add set selection in create Token themes flow [Taiga #10746](https://tree.taiga.io/project/penpot/issue/10746)
 - Display indicator on not active sets [Taiga #10668](https://tree.taiga.io/project/penpot/issue/10668)
+- Allow multi file token import [Github #27](https://github.com/tokens-studio/penpot/issues/27)
 - Create `input*` wrapper component, and `label*`, `input-field*` and `hint-message*` components [Taiga #10713](https://tree.taiga.io/project/penpot/us/10713)
 
 ### :bug: Bugs fixed

frontend/resources/images/icons/folder.svg

@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" stroke="currentColor" stroke-linejoin="round" >
+    <path d="m2 13.786 2.203-7.152a1 1 0 0 1 .956-.705h9.431a1 1 0 0 1 .944 1.33l-2.05 5.857a1 1 0 0 1-.944.67H2Zm0 0a1 1 0 0 1-1-1v-9.29c0-.396.15-.777.419-1.057A1.391 1.391 0 0 1 2.428 2h2.858l1.071 1.684h5.429a1 1 0 0 1 1 1v1.245" />
+</svg>

frontend/src/app/main/data/style_dictionary.cljs

@@ -284,6 +284,7 @@
     (when name-error?
       (wte/error-ex-info :error.import/invalid-token-name (:value schema-error) err))))
 
+
 (defn- group-by-value [m]
   (reduce (fn [acc [k v]]
             (update acc v conj k)) {} m))
@@ -297,46 +298,50 @@
                :type :toast
                :level :info})))
 
+(defn parse-json [data]
+  (try
+    (t/decode-str data)
+    (catch js/Error e
+      (throw (wte/error-ex-info :error.import/json-parse-error data e)))))
+
+(defn decode-json-data [data file-name]
+  (let [single-set? (ctob/single-set? data)
+        json-format (ctob/get-json-format data)
+        unknown-tokens (ctob/get-tokens-of-unknown-type
+                        data
+                        ""
+                        (= json-format :json-format/dtcg))]
+    {:tokens-lib
+     (try
+       (cond
+         (and single-set?
+              (= :json-format/legacy json-format))
+         (decode-single-set-legacy-json (ctob/ensure-tokens-lib nil) file-name data)
+
+         (and single-set?
+              (= :json-format/dtcg json-format))
+         (decode-single-set-json (ctob/ensure-tokens-lib nil) file-name data)
+
+         (= :json-format/legacy json-format)
+         (ctob/decode-legacy-json (ctob/ensure-tokens-lib nil) data)
+
+         :else
+         (ctob/decode-dtcg-json (ctob/ensure-tokens-lib nil) data))
+
+       (catch js/Error e
+         (let [err (or (name-error e)
+                       (wte/error-ex-info :error.import/invalid-json-data data e))]
+           (throw err))))
+     :unknown-tokens unknown-tokens}))
+
 (defn process-json-stream
   ([data-stream]
    (process-json-stream nil data-stream))
   ([params data-stream]
    (let [{:keys [file-name]} params]
      (->> data-stream
-          (rx/map (fn [data]
+          (rx/map parse-json)
-                    (try
+          (rx/map #(decode-json-data % file-name))
-                      (t/decode-str data)
-                      (catch js/Error e
-                        (throw (wte/error-ex-info :error.import/json-parse-error data e))))))
-          (rx/map (fn [json-data]
-                    (let [single-set? (ctob/single-set? json-data)
-                          json-format (ctob/get-json-format json-data)
-                          unknown-tokens (ctob/get-tokens-of-unknown-type
-                                          json-data
-                                          ""
-                                          (= json-format :json-format/dtcg))]
-                      {:tokens-lib
-                       (try
-                         (cond
-                           (and single-set?
-                                (= :json-format/legacy json-format))
-                           (decode-single-set-legacy-json (ctob/ensure-tokens-lib nil) file-name json-data)
-
-                           (and single-set?
-                                (= :json-format/dtcg json-format))
-                           (decode-single-set-json (ctob/ensure-tokens-lib nil) file-name json-data)
-
-                           (= :json-format/legacy json-format)
-                           (ctob/decode-legacy-json (ctob/ensure-tokens-lib nil) json-data)
-
-                           :else
-                           (ctob/decode-dtcg-json (ctob/ensure-tokens-lib nil) json-data))
-
-                         (catch js/Error e
-                           (let [err (or (name-error e)
-                                         (wte/error-ex-info :error.import/invalid-json-data json-data e))]
-                             (throw err))))
-                       :unknown-tokens unknown-tokens})))
           (rx/mapcat (fn [{:keys [tokens-lib unknown-tokens]}]
                        (when unknown-tokens
                          (st/emit! (tokens-of-unknown-type-warning unknown-tokens)))

frontend/src/app/main/ui/ds/foundations/assets/icon.cljs

@@ -147,6 +147,7 @@
 (def ^:icon-id flex-vertical "flex-vertical")
 (def ^:icon-id flip-horizontal "flip-horizontal")
 (def ^:icon-id flip-vertical "flip-vertical")
+(def ^:icon-id folder "folder")
 (def ^:icon-id gap-horizontal "gap-horizontal")
 (def ^:icon-id gap-vertical "gap-vertical")
 (def ^:icon-id graphics "graphics")

frontend/src/app/main/ui/ds/notifications/shared/notification_pill.cljs

@@ -18,7 +18,7 @@
   [level]
   (case level
     :info i/info
-    :default i/msg-neutral
+    :default i/info
     :warning i/msg-neutral
     :error i/delete-text
     :success i/status-tick

frontend/src/app/main/ui/icons.cljs

@@ -125,6 +125,7 @@
 (def ^:icon flex (icon-xref :flex))
 (def ^:icon flip-horizontal (icon-xref :flip-horizontal))
 (def ^:icon flip-vertical (icon-xref :flip-vertical))
+(def ^:icon folder (icon-xref :folder))
 (def ^:icon gap-horizontal (icon-xref :gap-horizontal))
 (def ^:icon gap-vertical (icon-xref :gap-vertical))
 (def ^:icon graphics (icon-xref :graphics))

frontend/src/app/main/ui/workspace.cljs

@@ -33,6 +33,7 @@
    [app.main.ui.workspace.sidebar.collapsable-button :refer [collapsed-button]]
    [app.main.ui.workspace.sidebar.history :refer [history-toolbox*]]
    [app.main.ui.workspace.tokens.modals]
+   [app.main.ui.workspace.tokens.modals.import]
    [app.main.ui.workspace.tokens.modals.themes]
    [app.main.ui.workspace.viewport :refer [viewport*]]
    [app.util.debug :as dbg]

frontend/src/app/main/ui/workspace/tokens/modals/import.cljs

@@ -0,0 +1,177 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.main.ui.workspace.tokens.modals.import
+  (:require-macros [app.main.style :as stl])
+  (:require
+   [app.common.files.helpers :as cfh]
+   [app.main.data.event :as ev]
+   [app.main.data.modal :as modal]
+   [app.main.data.notifications :as ntf]
+   [app.main.data.style-dictionary :as sd]
+   [app.main.data.workspace.tokens.errors :as wte]
+   [app.main.data.workspace.tokens.library-edit :as dwtl]
+   [app.main.store :as st]
+   [app.main.ui.ds.buttons.button :refer [button*]]
+   [app.main.ui.ds.buttons.icon-button :refer [icon-button*]]
+   [app.main.ui.ds.foundations.assets.icon :as i]
+   [app.main.ui.ds.foundations.typography.heading :refer [heading*]]
+   [app.main.ui.ds.foundations.typography.text :refer [text*]]
+   [app.main.ui.ds.notifications.context-notification :refer [context-notification*]]
+   [app.util.dom :as dom]
+   [app.util.i18n :refer [tr]]
+   [app.util.webapi :as wapi]
+   [beicon.v2.core :as rx]
+   [cuerdas.core :as str]
+   [potok.v2.core :as ptk]
+   [rumext.v2 :as mf]))
+
+(defn- drop-parent-directory [path]
+  (->> (cfh/split-path path)
+       (rest)
+       (str/join "/")))
+
+(defn- remove-path-extension [path]
+  (-> (str/split path ".")
+      (butlast)
+      (str/join)))
+
+(defn- file-path->set-name
+  [path]
+  (-> path
+      (drop-parent-directory)
+      (remove-path-extension)))
+
+(defn- on-import-stream [tokens-lib-stream]
+  (rx/sub!
+   tokens-lib-stream
+   (fn [lib]
+     (st/emit! (ptk/data-event ::ev/event {::ev/name "import-tokens"})
+               (dwtl/import-tokens-lib lib))
+     (modal/hide!))
+   (fn [err]
+     (js/console.error err)
+     (st/emit! (ntf/show {:content (wte/humanize-errors [(ex-data err)])
+                          :detail (wte/detail-errors [(ex-data err)])
+                          :type :toast
+                          :level :error})))))
+
+(mf/defc import-modal-body*
+  {::mf/private true}
+  []
+  (let [file-input-ref (mf/use-ref)
+        dir-input-ref (mf/use-ref)
+
+        on-display-file-explorer
+        (mf/use-fn #(dom/click (mf/ref-val file-input-ref)))
+
+        on-display-dir-explorer
+        (mf/use-fn #(dom/click (mf/ref-val dir-input-ref)))
+
+        on-import-directory
+        (mf/use-fn
+         (fn [event]
+           (let [files (->> (dom/get-target event)
+                            (dom/get-files)
+                            (filter (fn [file]
+                                      (let [name (.-name file)
+                                            type (.-type file)]
+                                        (or
+                                         (= type "application/json")
+                                         (str/ends-with? name ".json")))))
+                            ;; Read files as text, ignore files with json parse errors
+                            (map (fn [file]
+                                   (->> (wapi/read-file-as-text file)
+                                        (rx/mapcat (fn [json]
+                                                     (let [path (.-webkitRelativePath file)]
+                                                       (rx/of
+                                                        (try
+                                                          {(file-path->set-name path) (sd/parse-json json)}
+                                                          (catch js/Error e
+                                                            {:path path :error e}))))))))))]
+
+             (->> (apply rx/merge files)
+                  (rx/reduce (fn [acc cur]
+                               (if (:error cur)
+                                 acc
+                                 (conj acc cur)))
+                             {})
+                  (rx/map #(sd/decode-json-data (if (= 1 (count %))
+                                                  (val (first %))
+                                                  %)
+                                                (ffirst %)))
+                  (on-import-stream))
+
+             (-> (mf/ref-val dir-input-ref)
+                 (dom/set-value! "")))))
+
+        on-import
+        (mf/use-fn
+         (fn [event]
+           (let [file (-> (dom/get-target event)
+                          (dom/get-files)
+                          (first))
+                 file-name (remove-path-extension (.-name file))]
+             (->> (wapi/read-file-as-text file)
+                  (sd/process-json-stream {:file-name file-name})
+                  (on-import-stream))
+
+             (-> (mf/ref-val file-input-ref)
+                 (dom/set-value! "")))))]
+
+    [:div {:class (stl/css :import-modal-wrapper)}
+     [:> heading* {:level 2 :typography "headline-medium" :class (stl/css :import-modal-title)}
+      (tr "workspace.token.import-tokens")]
+
+     [:> text* {:as "ul" :typography "body-medium" :class (stl/css :import-description)}
+      [:li (tr "workspace.token.import-single-file")]
+      [:li (tr "workspace.token.import-multiple-files")]]
+
+     [:> context-notification* {:type :context
+                                :appearance "neutral"
+                                :level "default"
+                                :is-html true}
+      (tr "workspace.token.import-warning")]
+
+     [:div {:class (stl/css :import-actions)}
+      [:input {:type "file"
+               :ref file-input-ref
+               :style {:display "none"}
+               :accept ".json"
+               :on-change on-import}]
+      [:input {:type "file"
+               :ref dir-input-ref
+               :style {:display "none"}
+               :accept ""
+               :webkitdirectory "true"
+               :on-change on-import-directory}]
+      [:> button* {:variant "secondary"
+                   :type "button"
+                   :on-click modal/hide!}
+       (tr "labels.cancel")]
+      [:> button* {:variant "primary"
+                   :type "button"
+                   :icon i/document
+                   :on-click on-display-file-explorer}
+       (tr "workspace.token.choose-file")]
+      [:> button* {:variant "primary"
+                   :type "button"
+                   :icon i/folder
+                   :on-click on-display-dir-explorer}
+       (tr "workspace.token.choose-folder")]]]))
+
+(mf/defc import-modal*
+  {::mf/register modal/components
+   ::mf/register-as :tokens/import}
+  []
+  [:div {:class (stl/css :modal-overlay)}
+   [:div {:class (stl/css :modal-dialog)}
+    [:> icon-button* {:class (stl/css :close-btn)
+                      :on-click modal/hide!
+                      :aria-label (tr "labels.close")
+                      :variant "ghost"
+                      :icon "close"}]
+    [:> import-modal-body*]]])

frontend/src/app/main/ui/workspace/tokens/modals/import.scss

@@ -0,0 +1,50 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// Copyright (c) KALEIDOS INC
+
+@use "../../../ds/typography.scss" as t;
+@use "../../../ds/_sizes.scss" as *;
+@import "refactor/common-refactor.scss";
+
+.modal-overlay {
+  @extend .modal-overlay-base;
+}
+
+.modal-dialog {
+  @extend .modal-container-base;
+  user-select: none;
+}
+
+.import-modal-wrapper {
+  display: flex;
+  flex-direction: column;
+  gap: var(--sp-xxl);
+}
+
+.import-modal-title {
+  color: var(--color-foreground-primary);
+}
+
+.import-description {
+  display: flex;
+  flex-direction: column;
+  gap: var(--sp-s);
+  padding-inline-start: var(--sp-m);
+  margin: 0;
+  color: var(--color-foreground-secondary);
+  list-style: disc;
+}
+
+.import-actions {
+  display: flex;
+  justify-content: flex-end;
+  gap: var(--sp-s);
+}
+
+.close-btn {
+  position: absolute;
+  inset-block-start: var(--sp-s);
+  inset-inline-end: var(--sp-s);
+}

frontend/src/app/main/ui/workspace/tokens/sidebar.cljs

@@ -12,14 +12,13 @@
    [app.common.types.tokens-lib :as ctob]
    [app.main.data.event :as ev]
    [app.main.data.modal :as modal]
-   [app.main.data.notifications :as ntf]
    [app.main.data.style-dictionary :as sd]
    [app.main.data.workspace.tokens.application :as dwta]
-   [app.main.data.workspace.tokens.errors :as wte]
    [app.main.data.workspace.tokens.library-edit :as dwtl]
    [app.main.refs :as refs]
    [app.main.store :as st]
-   [app.main.ui.components.dropdown-menu :refer [dropdown-menu dropdown-menu-item*]]
+   [app.main.ui.components.dropdown-menu :refer [dropdown-menu
+                                                 dropdown-menu-item*]]
    [app.main.ui.components.title-bar :refer [title-bar]]
    [app.main.ui.context :as ctx]
    [app.main.ui.ds.buttons.button :refer [button*]]
@@ -38,8 +37,6 @@
    [app.util.dom :as dom]
    [app.util.i18n :refer [tr]]
    [app.util.webapi :as wapi]
-   [beicon.v2.core :as rx]
-   [cuerdas.core :as str]
    [okulary.core :as l]
    [potok.v2.core :as ptk]
    [rumext.v2 :as mf]
@@ -360,9 +357,7 @@
 
 (mf/defc import-export-button*
   []
-  (let [input-ref  (mf/use-ref)
+  (let [show-menu* (mf/use-state false)
-
-        show-menu* (mf/use-state false)
         show-menu? (deref show-menu*)
 
         can-edit?
@@ -380,30 +375,6 @@
            (dom/stop-propagation event)
            (reset! show-menu* false)))
 
-        on-display-file-explorer
-        (mf/use-fn #(dom/click (mf/ref-val input-ref)))
-
-        on-import
-        (mf/use-fn
-         (fn [event]
-           (let [file (-> (dom/get-target event)
-                          (dom/get-files)
-                          (first))
-                 file-name (str/replace (.-name file) ".json" "")]
-             (->> (wapi/read-file-as-text file)
-                  (sd/process-json-stream {:file-name file-name})
-                  (rx/subs! (fn [lib]
-                              (st/emit! (ptk/data-event ::ev/event {::ev/name "import-tokens"})
-                                        (dwtl/import-tokens-lib lib)))
-                            (fn [err]
-                              (js/console.error err)
-                              (st/emit! (ntf/show {:content (wte/humanize-errors [(ex-data err)])
-                                                   :detail (wte/detail-errors [(ex-data err)])
-                                                   :type :toast
-                                                   :level :error})))))
-             (-> (mf/ref-val input-ref)
-                 (dom/set-value! "")))))
-
         on-export
         (mf/use-fn
          (fn []
@@ -412,16 +383,13 @@
                                      (ctob/encode-dtcg)
                                      (json/encode :key-fn identity))]
              (->> (wapi/create-blob (or tokens-json "{}") "application/json")
-                  (dom/trigger-download "tokens.json")))))]
+                  (dom/trigger-download "tokens.json")))))
+        on-modal-show
+        (mf/use-fn
+         (fn []
+           (modal/show! :tokens/import {})))]
 
     [:div {:class (stl/css :import-export-button-wrapper)}
-     (when can-edit?
-       [:input {:type "file"
-                :ref input-ref
-                :style {:display "none"}
-                :id "file-input"
-                :accept ".json"
-                :on-change on-import}])
      [:> button* {:on-click open-menu
                   :icon "import-export"
                   :variant "secondary"}
@@ -431,11 +399,9 @@
                         :list-class (stl/css :import-export-menu)}
       (when can-edit?
         [:> dropdown-menu-item* {:class (stl/css :import-export-menu-item)
-                                 :on-click on-display-file-explorer}
+                                 :on-click on-modal-show}
          [:div {:class (stl/css :import-menu-item)}
-          [:div (tr "labels.import")]
+          [:div (tr "labels.import")]]])
-          [:div {:class (stl/css :import-export-menu-item-icon) :title (tr "workspace.token.import-tooltip")}
-           [:> i/icon* {:icon-id i/info :aria-label (tr "workspace.token.import-tooltip")}]]]])
       [:> dropdown-menu-item* {:class (stl/css :import-export-menu-item)
                                :on-click on-export}
        (tr "labels.export")]]]))

frontend/translations/en.po

@@ -2069,6 +2069,30 @@ msgstr "Hide resolved comments"
 msgid "labels.import"
 msgstr "Import"
 
+#: src/app/main/ui/workspace/tokens/modals/import.cljs:116
+msgid "workspace.token.import-tokens"
+msgstr "Import tokens"
+
+#: src/app/main/ui/workspace/tokens/modals/import.cljs:121
+msgid "workspace.token.import-single-file"
+msgstr "In a single JSON file, the first-level keys should be the token set names."
+
+#: src/app/main/ui/workspace/tokens/modals/import.cljs:122
+msgid "workspace.token.import-multiple-files"
+msgstr "In multiple files, the file name / path are the set names."
+
+#: src/app/main/ui/workspace/tokens/modals/import.cljs:127
+msgid "workspace.token.import-warning"
+msgstr "Importing tokens will override all your current tokens, sets and themes."
+
+#: src/app/main/ui/workspace/tokens/modals/import.cljs:149
+msgid "workspace.token.choose-file"
+msgstr "Choose file"
+
+#: src/app/main/ui/workspace/tokens/modals/import.cljs:155
+msgid "workspace.token.choose-folder"
+msgstr "Choose folder"
+
 #: src/app/main/ui/dashboard/team.cljs:1018
 msgid "labels.inactive"
 msgstr "Inactive"

frontend/translations/es.po

@@ -2092,6 +2092,30 @@ msgstr "Ocultar comentarios resueltos"
 msgid "labels.import"
 msgstr "Importar"
 
+#: src/app/main/ui/workspace/tokens/modals/import.cljs:116
+msgid "workspace.token.import-tokens"
+msgstr "Import tokens"
+
+#: src/app/main/ui/workspace/tokens/modals/import.cljs:121
+msgid "workspace.token.import-single-file"
+msgstr "En un archivo JSON único, las claves de primer nivel deben ser los nombres de los sets de tokens."
+
+#: src/app/main/ui/workspace/tokens/modals/import.cljs:122
+msgid "workspace.token.import-multiple-files"
+msgstr "En multiples archivos, el nombre o la ruta del archivo serán los nombres de los sets."
+
+#: src/app/main/ui/workspace/tokens/modals/import.cljs:127
+msgid "workspace.token.import-warning"
+msgstr "Al importar tokens sobreescribirás todos tus tokens, sets y themes."
+
+#: src/app/main/ui/workspace/tokens/modals/import.cljs:149
+msgid "workspace.token.choose-file"
+msgstr "Elige archivo"
+
+#: src/app/main/ui/workspace/tokens/modals/import.cljs:155
+msgid "workspace.token.choose-folder"
+msgstr "Elige carpeta"
+
 #: src/app/main/ui/dashboard/team.cljs:1018
 msgid "labels.inactive"
 msgstr "Inactivo"