diff --git a/common/src/app/common/types/tokens_lib.cljc b/common/src/app/common/types/tokens_lib.cljc index 6140c8b9a..d02baf137 100644 --- a/common/src/app/common/types/tokens_lib.cljc +++ b/common/src/app/common/types/tokens_lib.cljc @@ -662,63 +662,6 @@ (def valid-active-token-themes? (sm/validator schema:active-themes)) -;; === Import / Export from DTCG format - -(def ^:private legacy-node? - (sm/validator - [:or - [:map - ["value" :string] - ["type" :string]] - [:map - ["value" [:sequential [:map ["type" :string]]]] - ["type" :string]] - [:map - ["value" :map] - ["type" :string]]])) - -(def ^:private dtcg-node? - (sm/validator - [:or - [:map - ["$value" :string] - ["$type" :string]] - [:map - ["$value" [:sequential [:map ["$type" :string]]]] - ["$type" :string]] - [:map - ["$value" :map] - ["$type" :string]]])) - -(defn get-json-format - "Searches through parsed token file and returns: - - `:json-format/legacy` when first node satisfies `legacy-node?` predicate - - `:json-format/dtcg` when first node satisfies `dtcg-node?` predicate - - `nil` if neither combination is found" - ([data] - (get-json-format data legacy-node? dtcg-node?)) - ([data legacy-node? dtcg-node?] - (let [branch? map? - children (fn [node] (vals node)) - check-node (fn [node] - (cond - (legacy-node? node) :json-format/legacy - (dtcg-node? node) :json-format/dtcg - :else nil)) - walk (fn walk [node] - (lazy-seq - (cons - (check-node node) - (when (branch? node) - (mapcat walk (children node))))))] - (->> (walk data) - (filter some?) - first)))) - -(defn single-set? [data] - (and (not (contains? data "$metadata")) - (not (contains? data "$themes")))) - ;; DEPRECATED (defn walk-sets-tree-seq "Walk sets tree as a flat list. @@ -828,72 +771,10 @@ (map-indexed (fn [index item] (assoc item :index index)))))) -(defn get-tokens-of-unknown-type - "Recursively search the tokens for unknown types" - [tokens token-path dctg?] - (let [type-key (if dctg? "$type" "type")] - (reduce-kv - (fn [unknown-tokens k v] - (let [child-path (if (empty? token-path) - (name k) - (str token-path "." k))] - (if (and (map? v) - (not (contains? v type-key))) - (let [nested-unknown-tokens (get-tokens-of-unknown-type v child-path dctg?)] - (merge unknown-tokens nested-unknown-tokens)) - (let [token-type-str (get v type-key) - token-type (cto/dtcg-token-type->token-type token-type-str)] - (if (and (not (some? token-type)) (some? token-type-str)) - (assoc unknown-tokens child-path token-type-str) - unknown-tokens))))) - nil - tokens))) - -(defn flatten-nested-tokens-json - "Recursively flatten the dtcg token structure, joining keys with '.'." - [tokens token-path] - (reduce-kv - (fn [acc k v] - (let [child-path (if (empty? token-path) - (name k) - (str token-path "." k))] - (if (and (map? v) - (not (contains? v "$type"))) - (merge acc (flatten-nested-tokens-json v child-path)) - (let [token-type (cto/dtcg-token-type->token-type (get v "$type"))] - (if token-type - (assoc acc child-path (make-token - :name child-path - :type token-type - :value (get v "$value") - :description (get v "$description"))) - ;; Discard unknown tokens - acc))))) - {} - tokens)) - ;; === Tokens Lib (declare make-tokens-lib) -(defn legacy-nodes->dtcg-nodes [sets-data] - (walk/postwalk - (fn [node] - (cond-> node - (and (map? node) - (contains? node "value") - (sequential? (get node "value"))) - (update "value" - (fn [seq-value] - (map #(set/rename-keys % {"type" "$type"}) seq-value))) - - (and (map? node) - (and (contains? node "type") - (contains? node "value"))) - (set/rename-keys {"value" "$value" - "type" "$type"}))) - sets-data)) - (defprotocol ITokensLib "A library of tokens, sets and themes." (set-path-exists? [_ path] "if a set at `path` exists") @@ -910,12 +791,11 @@ Will return a value that matches this schema: `:all` All of the nested sets are active `:partial` Mixed active state of nested sets") (get-active-themes-set-tokens [_] "set of set names that are active in the the active themes") - (encode-dtcg [_] "Encodes library to a dtcg compatible json string") - (decode-dtcg-json [_ parsed-json] "Decodes parsed json containing tokens and converts to library") - (decode-legacy-json [_ parsed-json] "Decodes parsed legacy json containing tokens and converts to library") (get-all-tokens [_] "all tokens in the lib") (validate [_])) +(declare parse-multi-set-dtcg-json) +(declare export-dtcg-json) (deftype TokensLib [sets themes active-themes] ;; NOTE: This is only for debug purposes, pending to properly ;; implement the toString and alternative printing. @@ -932,12 +812,9 @@ Will return a value that matches this schema: (-clj->js [_] (js-obj "sets" (clj->js sets) "themes" (clj->js themes) "active-themes" (clj->js active-themes)))]) - - #?@(:clj [json/JSONWriter - (-write [this writter options] (json/-write (encode-dtcg this) writter options))]) - + (-write [this writter options] (json/-write (export-dtcg-json this) writter options))]) ITokenSets (add-set [_ token-set] @@ -1312,142 +1189,6 @@ Will return a value that matches this schema: active-set-names)] tokens)) - (encode-dtcg [this] - (let [themes-xform - (comp - (filter #(and (instance? TokenTheme %) - (not (hidden-temporary-theme? %)))) - (map (fn [token-theme] - (let [theme-map (->> token-theme - (into {}) - walk/stringify-keys)] - (-> theme-map - (set/rename-keys {"sets" "selectedTokenSets"}) - (update "selectedTokenSets" (fn [sets] - (->> (for [s sets] [s "enabled"]) - (into {}))))))))) - themes - (->> (tree-seq d/ordered-map? vals themes) - (into [] themes-xform)) - - ;; Active themes without exposing hidden penpot theme - active-themes-clear - (disj active-themes hidden-token-theme-path) - - update-token-fn - (fn [token] - (cond-> {"$value" (:value token) - "$type" (cto/token-type->dtcg-token-type (:type token))} - (:description token) (assoc "$description" (:description token)))) - - name-set-tuples - (->> sets - (tree-seq d/ordered-map? vals) - (filter (partial instance? TokenSet)) - (map (fn [{:keys [name tokens]}] - [name (tokens-tree tokens :update-token-fn update-token-fn)]))) - - ordered-set-names - (mapv first name-set-tuples) - - sets - (into {} name-set-tuples) - - active-sets - (get-active-themes-set-names this)] - - (-> sets - (assoc "$themes" themes) - (assoc-in ["$metadata" "tokenSetOrder"] ordered-set-names) - (assoc-in ["$metadata" "activeThemes"] active-themes-clear) - (assoc-in ["$metadata" "activeSets"] active-sets)))) - - (decode-dtcg-json [_ data] - (assert (map? data) "expected a map data structure for `data`") - - (let [metadata (get data "$metadata") - - xf-normalize-set-name - (map normalize-set-name) - - sets - (dissoc data "$themes" "$metadata") - - ordered-sets - (-> (d/ordered-set) - (into xf-normalize-set-name (get metadata "tokenSetOrder")) - (into xf-normalize-set-name (keys sets))) - - active-sets - (or (->> (get metadata "activeSets") - (into #{} xf-normalize-set-name) - (not-empty)) - #{}) - - active-themes - (or (->> (get metadata "activeThemes") - (into #{}) - (not-empty)) - #{hidden-token-theme-path}) - - themes - (->> (get data "$themes") - (map (fn [theme] - (make-token-theme - :name (get theme "name") - :group (get theme "group") - :is-source (get theme "is-source") - :id (get theme "id") - :modified-at (some-> (get theme "modified-at") - (dt/parse-instant)) - :sets (into #{} - (comp (map key) - xf-normalize-set-name - (filter #(contains? ordered-sets %))) - (get theme "selectedTokenSets"))))) - (not-empty)) - - library - (make-tokens-lib) - - sets - (reduce-kv (fn [result name tokens] - (assoc result - (normalize-set-name name) - (flatten-nested-tokens-json tokens ""))) - {} - sets) - - library - (reduce (fn [library name] - (if-let [tokens (get sets name)] - (add-set library (make-token-set :name name :tokens tokens)) - library)) - library - ordered-sets) - - library - (update-theme library hidden-token-theme-group hidden-token-theme-name - #(assoc % :sets active-sets)) - - library - (reduce add-theme library themes) - - library - (reduce (fn [library theme-path] - (let [[group name] (split-token-theme-path theme-path)] - (activate-theme library group name))) - library - active-themes)] - - library)) - - (decode-legacy-json [this parsed-legacy-json] - (let [other-data (select-keys parsed-legacy-json ["$themes" "$metadata"]) - sets-data (dissoc parsed-legacy-json "$themes" "$metadata") - dtcg-sets-data (legacy-nodes->dtcg-nodes sets-data)] - (decode-dtcg-json this (merge other-data - dtcg-sets-data)))) (get-all-tokens [this] (reduce (fn [tokens' set] @@ -1509,18 +1250,13 @@ Will return a value that matches this schema: [tokens-lib] (or tokens-lib (make-tokens-lib))) -(defn decode-dtcg - [encoded-json] - (-> (make-tokens-lib) - (decode-dtcg-json encoded-json))) - (def schema:tokens-lib (sm/register! {:type ::tokens-lib :pred valid-tokens-lib? :type-properties - {:encode/json encode-dtcg - :decode/json decode-dtcg}})) + {:encode/json export-dtcg-json + :decode/json parse-multi-set-dtcg-json}})) (defn duplicate-set [set-name lib & {:keys [suffix]}] (let [sets (get-sets lib) @@ -1530,6 +1266,336 @@ Will return a value that matches this schema: (assoc :name copy-name) (assoc :modified-at (dt/now))))) +;; === Import / Export from JSON format + +;; Supported formats: +;; - Legacy: for tokens files prior to DTCG second draft +;; - DTCG: for tokens files conforming to the DTCG second draft (current for now) +;; https://www.w3.org/community/design-tokens/2022/06/14/call-to-implement-the-second-editors-draft-and-share-feedback/ +;; +;; - Single set: for files that comply with the base DTCG format, that contain a single tree of tokens. +;; - Multi sets: for files with the Tokens Studio extension, that may contain several sets, and also themes and other $metadata. +;; +;; Small glossary: +;; * json data: a json-encoded string +;; * decode: convert a json string into a plain clojure nested map +;; * parse: build a TokensLib (or a fragment) from a decoded json data +;; * export: generate from a TokensLib a plain clojure nested map, suitable to be encoded as a json string + +(def ^:private legacy-node? + (sm/validator + [:or + [:map + ["value" :string] + ["type" :string]] + [:map + ["value" [:sequential [:map ["type" :string]]]] + ["type" :string]] + [:map + ["value" :map] + ["type" :string]]])) + +(def ^:private dtcg-node? + (sm/validator + [:or + [:map + ["$value" :string] + ["$type" :string]] + [:map + ["$value" [:sequential [:map ["$type" :string]]]] + ["$type" :string]] + [:map + ["$value" :map] + ["$type" :string]]])) + +(defn- get-json-format + "Searches through decoded token file and returns: + - `:json-format/legacy` when first node satisfies `legacy-node?` predicate + - `:json-format/dtcg` when first node satisfies `dtcg-node?` predicate + - `nil` if neither combination is found" + ([decoded-json] + (get-json-format decoded-json legacy-node? dtcg-node?)) + ([decoded-json legacy-node? dtcg-node?] + (assert (map? decoded-json) "expected a plain clojure map for `decoded-json`") + (let [branch? map? + children (fn [node] (vals node)) + check-node (fn [node] + (cond + (legacy-node? node) :json-format/legacy + (dtcg-node? node) :json-format/dtcg + :else nil)) + walk (fn walk [node] + (lazy-seq + (cons + (check-node node) + (when (branch? node) + (mapcat walk (children node))))))] + (->> (walk decoded-json) + (filter some?) + first)))) ;; TODO: throw error if format cannot be determined + +(defn- legacy-json->dtcg-json + "Converts a decoded json file in legacy format into DTCG format." + [decoded-json] + (assert (map? decoded-json) "expected a plain clojure map for `decoded-json`") + (walk/postwalk + (fn [node] + (cond-> node + (and (map? node) + (contains? node "value") + (sequential? (get node "value"))) + (update "value" + (fn [seq-value] + (map #(set/rename-keys % {"type" "$type"}) seq-value))) + + (and (map? node) + (and (contains? node "type") + (contains? node "value"))) + (set/rename-keys {"value" "$value" + "type" "$type"}))) + decoded-json)) + +(defn- single-set? + "Check if the decoded json file conforms to basic DTCG format with a single set." + [decoded-json] + (assert (map? decoded-json) "expected a plain clojure map for `decoded-json`") + (and (not (contains? decoded-json "$metadata")) + (not (contains? decoded-json "$themes")))) + +(defn- flatten-nested-tokens-json + "Convert a tokens tree in the decoded json fragment into a flat map, + being the keys the token paths after joining the keys with '.'." + [decoded-json-tokens parent-path] + (reduce-kv + (fn [tokens k v] + (let [child-path (if (empty? parent-path) + (name k) + (str parent-path "." k))] + (if (and (map? v) + (not (contains? v "$type"))) + (merge tokens (flatten-nested-tokens-json v child-path)) + (let [token-type (cto/dtcg-token-type->token-type (get v "$type"))] + (if token-type + (assoc tokens child-path (make-token + :name child-path + :type token-type + :value (get v "$value") + :description (get v "$description"))) + ;; Discard unknown type tokens + tokens))))) + {} + decoded-json-tokens)) + +(defn- parse-single-set-dtcg-json + "Parse a decoded json file with a single set of tokens in DTCG format into a TokensLib." + [set-name decoded-json-tokens] + (assert (map? decoded-json-tokens) "expected a plain clojure map for `decoded-json-tokens`") + (assert (= (get-json-format decoded-json-tokens) :json-format/dtcg) "expected a dtcg format for `decoded-json-tokens`") + (-> (make-tokens-lib) + (add-set (make-token-set :name (normalize-set-name set-name) + :tokens (flatten-nested-tokens-json decoded-json-tokens ""))))) + +(defn- parse-single-set-legacy-json + "Parse a decoded json file with a single set of tokens in legacy format into a TokensLib." + [set-name decoded-json-tokens] + (assert (map? decoded-json-tokens) "expected a plain clojure map for `decoded-json-tokens`") + (assert (= (get-json-format decoded-json-tokens) :json-format/legacy) "expected a legacy format for `decoded-json-tokens`") + (parse-single-set-dtcg-json set-name (legacy-json->dtcg-json decoded-json-tokens))) + +(defn- parse-multi-set-dtcg-json + "Parse a decoded json file with multi sets in DTCG format into a TokensLib." + [decoded-json] + (assert (map? decoded-json) "expected a plain clojure map for `decoded-json`") + (assert (= (get-json-format decoded-json) :json-format/dtcg) "expected a dtcg format for `decoded-json`") + + (let [metadata (get decoded-json "$metadata") + + xf-normalize-set-name + (map normalize-set-name) + + sets + (dissoc decoded-json "$themes" "$metadata") + + ordered-set-names + (-> (d/ordered-set) + (into xf-normalize-set-name (get metadata "tokenSetOrder")) + (into xf-normalize-set-name (keys sets))) + + active-set-names + (or (->> (get metadata "activeSets") + (into #{} xf-normalize-set-name) + (not-empty)) + #{}) + + active-theme-names + (or (->> (get metadata "activeThemes") + (into #{}) + (not-empty)) + #{hidden-token-theme-path}) + + themes + (->> (get decoded-json "$themes") + (map (fn [theme] + (make-token-theme + :name (get theme "name") + :group (get theme "group") + :is-source (get theme "is-source") + :id (get theme "id") + :modified-at (some-> (get theme "modified-at") + (dt/parse-instant)) + :sets (into #{} + (comp (map key) + xf-normalize-set-name + (filter #(contains? ordered-set-names %))) + (get theme "selectedTokenSets"))))) + (not-empty)) + + library + (make-tokens-lib) + + sets + (reduce-kv (fn [result name tokens] + (assoc result + (normalize-set-name name) + (flatten-nested-tokens-json tokens ""))) + {} + sets) + + library + (reduce (fn [library name] + (if-let [tokens (get sets name)] + (add-set library (make-token-set :name name :tokens tokens)) + library)) + library + ordered-set-names) + + library + (update-theme library hidden-token-theme-group hidden-token-theme-name + #(assoc % :sets active-set-names)) + + library + (reduce add-theme library themes) + + library + (reduce (fn [library theme-path] + (let [[group name] (split-token-theme-path theme-path)] + (activate-theme library group name))) + library + active-theme-names)] + + library)) + +(defn- parse-multi-set-legacy-json + "Parse a decoded json file with multi sets in legacy format into a TokensLib." + [decoded-json] + (assert (map? decoded-json) "expected a plain clojure map for `decoded-json`") + (assert (= (get-json-format decoded-json) :json-format/legacy) "expected a legacy format for `decoded-json`") + + (let [sets-data (dissoc decoded-json "$themes" "$metadata") + other-data (select-keys decoded-json ["$themes" "$metadata"]) + dtcg-sets-data (legacy-json->dtcg-json sets-data)] + (parse-multi-set-dtcg-json (merge other-data + dtcg-sets-data)))) + +(defn parse-decoded-json + "Guess the format and content type of the decoded json file and parse it into a TokensLib. + The `file-name` is used to determine the set name when the json file contains a single set." + [decoded-json file-name] + (let [single-set? (single-set? decoded-json) + json-format (get-json-format decoded-json)] + (cond + (and single-set? + (= :json-format/legacy json-format)) + (parse-single-set-legacy-json file-name decoded-json) + + (and single-set? + (= :json-format/dtcg json-format)) + (parse-single-set-dtcg-json file-name decoded-json) + + (= :json-format/legacy json-format) + (parse-multi-set-legacy-json decoded-json) + + :else + (parse-multi-set-dtcg-json decoded-json)))) + +(defn export-dtcg-json + "Convert a TokensLib into a plain clojure map, suitable to be encoded as a multi sets json string in DTCG format." + [tokens-lib] + (let [themes-xform + (comp + (filter #(and (instance? TokenTheme %) + (not (hidden-temporary-theme? %)))) + (map (fn [token-theme] + (let [theme-map (->> token-theme + (into {}) + walk/stringify-keys)] + (-> theme-map + (set/rename-keys {"sets" "selectedTokenSets"}) + (update "selectedTokenSets" (fn [sets] + (->> (for [s sets] [s "enabled"]) + (into {}))))))))) + themes + (->> (get-theme-tree tokens-lib) + (tree-seq d/ordered-map? vals) + (into [] themes-xform)) + + ;; Active themes without exposing hidden penpot theme + active-themes-clear + (-> (get-active-theme-paths tokens-lib) + (disj hidden-token-theme-path)) + + update-token-fn + (fn [token] + (cond-> {"$value" (:value token) + "$type" (cto/token-type->dtcg-token-type (:type token))} + (:description token) (assoc "$description" (:description token)))) + + name-set-tuples + (->> (get-set-tree tokens-lib) + (tree-seq d/ordered-map? vals) + (filter (partial instance? TokenSet)) + (map (fn [{:keys [name tokens]}] + [name (tokens-tree tokens :update-token-fn update-token-fn)]))) + + ordered-set-names + (mapv first name-set-tuples) + + sets + (into {} name-set-tuples) + + active-set-names + (get-active-themes-set-names tokens-lib)] + + (-> sets + (assoc "$themes" themes) + (assoc-in ["$metadata" "tokenSetOrder"] ordered-set-names) + (assoc-in ["$metadata" "activeThemes"] active-themes-clear) + (assoc-in ["$metadata" "activeSets"] active-set-names)))) + +(defn get-tokens-of-unknown-type + "Search for all tokens in the decoded json file that have a type that is not currently + supported by Penpot. Returns a map token-path -> token type." + ([decoded-json] + (get-tokens-of-unknown-type decoded-json "" (get-json-format decoded-json))) + ([decoded-json parent-path json-format] + (let [type-key (if (= json-format :json-format/dtcg) "$type" "type")] + (reduce-kv + (fn [unknown-tokens k v] + (let [child-path (if (empty? parent-path) + (name k) + (str parent-path "." k))] + (if (and (map? v) + (not (contains? v type-key))) + (let [nested-unknown-tokens (get-tokens-of-unknown-type v child-path json-format)] + (merge unknown-tokens nested-unknown-tokens)) + (let [token-type-str (get v type-key) + token-type (cto/dtcg-token-type->token-type token-type-str)] + (if (and (not (some? token-type)) (some? token-type-str)) + (assoc unknown-tokens child-path token-type-str) + unknown-tokens))))) + nil + decoded-json)))) + ;; === Serialization handlers for RPC API (transit) and database (fressian) (t/add-handlers! diff --git a/common/test/common_tests/types/data/tokens-single-set-dtcg-example.json b/common/test/common_tests/types/data/tokens-single-set-dtcg-example.json new file mode 100644 index 000000000..3b2d18531 --- /dev/null +++ b/common/test/common_tests/types/data/tokens-single-set-dtcg-example.json @@ -0,0 +1,11 @@ +{ + "color": { + "red": { + "100": { + "$value": "red", + "$type": "color", + "$description": "" + } + } + } +} \ No newline at end of file diff --git a/common/test/common_tests/types/data/tokens-single-set-legacy-example.json b/common/test/common_tests/types/data/tokens-single-set-legacy-example.json new file mode 100644 index 000000000..ef90d6d83 --- /dev/null +++ b/common/test/common_tests/types/data/tokens-single-set-legacy-example.json @@ -0,0 +1,11 @@ +{ + "color": { + "red": { + "100": { + "value": "red", + "type": "color", + "description": "" + } + } + } +} \ No newline at end of file diff --git a/common/test/common_tests/types/tokens_lib_test.cljc b/common/test/common_tests/types/tokens_lib_test.cljc index ab660fcba..9905a49ff 100644 --- a/common/test/common_tests/types/tokens_lib_test.cljc +++ b/common/test/common_tests/types/tokens_lib_test.cljc @@ -7,8 +7,9 @@ (ns common-tests.types.tokens-lib-test (:require #?(:clj [app.common.fressian :as fres]) + #?(:clj [app.common.json :as json]) + #?(:clj [app.common.test-helpers.tokens :as tht]) [app.common.data :as d] - [app.common.test-helpers.tokens :as tht] [app.common.time :as dt] [app.common.transit :as tr] [app.common.types.tokens-lib :as ctob] @@ -1387,47 +1388,28 @@ (t/is (nil? token-theme')))) #?(:clj - (t/deftest legacy-json-decoding - (let [json (-> (slurp "test/common_tests/types/data/tokens-multi-set-legacy-example.json") - (tr/decode-str)) - lib (ctob/decode-legacy-json (ctob/ensure-tokens-lib nil) json) - get-set-token (fn [set-name token-name] - (some-> (ctob/get-set lib set-name) - (ctob/get-token token-name) - (dissoc :modified-at))) - token-theme (ctob/get-theme lib "group-1" "theme-1")] - (t/is (= '("core" "light" "dark" "theme") (ctob/get-ordered-set-names lib))) - (t/testing "set exists in theme" - (t/is (= (:group token-theme) "group-1")) - (t/is (= (:name token-theme) "theme-1")) - (t/is (= (:sets token-theme) #{"light"}))) - (t/testing "tokens exist in core set" - (t/is (= (get-set-token "core" "colors.red.600") - {:name "colors.red.600" - :type :color - :value "#e53e3e" - :description ""})) - (t/is (= (get-set-token "core" "spacing.multi-value") - {:name "spacing.multi-value" - :type :spacing - :value "{dimension.sm} {dimension.xl}" - :description "You can have multiple values in a single spacing token"})) - (t/is (= (get-set-token "theme" "button.primary.background") - {:name "button.primary.background" - :type :color - :value "{accent.default}" - :description ""}))) - (t/testing "invalid tokens got discarded" - (t/is (nil? (get-set-token "typography" "H1.Bold"))))))) + (t/deftest parse-single-set-legacy-json + (let [json (-> (slurp "test/common_tests/types/data/tokens-single-set-legacy-example.json") + (json/decode {:key-fn identity})) + lib (ctob/parse-decoded-json json "single_set")] + (t/is (= '("single_set") (ctob/get-ordered-set-names lib))) + (t/testing "token added" + (t/is (some? (ctob/get-token-in-set lib "single_set" "color.red.100"))))))) #?(:clj - (t/deftest dtcg-encoding-decoding-json - (let [json (-> (slurp "test/common_tests/types/data/tokens-multi-set-example.json") - (tr/decode-str)) - lib (ctob/decode-dtcg-json (ctob/ensure-tokens-lib nil) json) - get-set-token (fn [set-name token-name] - (some-> (ctob/get-set lib set-name) - (ctob/get-token token-name))) + (t/deftest parse-single-set-dtcg-json + (let [json (-> (slurp "test/common_tests/types/data/tokens-single-set-dtcg-example.json") + (json/decode {:key-fn identity})) + lib (ctob/parse-decoded-json json "single_set")] + (t/is (= '("single_set") (ctob/get-ordered-set-names lib))) + (t/testing "token added" + (t/is (some? (ctob/get-token-in-set lib "single_set" "color.red.100"))))))) + +#?(:clj + (t/deftest parse-multi-set-legacy-json + (let [json (-> (slurp "test/common_tests/types/data/tokens-multi-set-legacy-example.json") + (json/decode {:key-fn identity})) + lib (ctob/parse-decoded-json json "") token-theme (ctob/get-theme lib "group-1" "theme-1")] (t/is (= '("core" "light" "dark" "theme") (ctob/get-ordered-set-names lib))) (t/testing "set exists in theme" @@ -1435,32 +1417,59 @@ (t/is (= (:name token-theme) "theme-1")) (t/is (= (:sets token-theme) #{"light"}))) (t/testing "tokens exist in core set" - (t/is (tht/token-data-eq? (get-set-token "core" "colors.red.600") + (t/is (tht/token-data-eq? (ctob/get-token-in-set lib "core" "colors.red.600") {:name "colors.red.600" :type :color :value "#e53e3e" :description ""})) - (t/is (tht/token-data-eq? (get-set-token "core" "spacing.multi-value") + (t/is (tht/token-data-eq? (ctob/get-token-in-set lib "core" "spacing.multi-value") {:name "spacing.multi-value" :type :spacing :value "{dimension.sm} {dimension.xl}" :description "You can have multiple values in a single spacing token"})) - (t/is (tht/token-data-eq? (get-set-token "theme" "button.primary.background") + (t/is (tht/token-data-eq? (ctob/get-token-in-set lib "theme" "button.primary.background") {:name "button.primary.background" :type :color :value "{accent.default}" :description ""}))) (t/testing "invalid tokens got discarded" - (t/is (nil? (get-set-token "typography" "H1.Bold"))))))) + (t/is (nil? (ctob/get-token-in-set lib "typography" "H1.Bold"))))))) #?(:clj - (t/deftest decode-dtcg-json-default-team + (t/deftest parse-multi-set-dtcg-json + (let [json (-> (slurp "test/common_tests/types/data/tokens-multi-set-example.json") + (json/decode {:key-fn identity})) + lib (ctob/parse-decoded-json json "") + token-theme (ctob/get-theme lib "group-1" "theme-1")] + (t/is (= '("core" "light" "dark" "theme") (ctob/get-ordered-set-names lib))) + (t/testing "set exists in theme" + (t/is (= (:group token-theme) "group-1")) + (t/is (= (:name token-theme) "theme-1")) + (t/is (= (:sets token-theme) #{"light"}))) + (t/testing "tokens exist in core set" + (t/is (tht/token-data-eq? (ctob/get-token-in-set lib "core" "colors.red.600") + {:name "colors.red.600" + :type :color + :value "#e53e3e" + :description ""})) + (t/is (tht/token-data-eq? (ctob/get-token-in-set lib "core" "spacing.multi-value") + {:name "spacing.multi-value" + :type :spacing + :value "{dimension.sm} {dimension.xl}" + :description "You can have multiple values in a single spacing token"})) + (t/is (tht/token-data-eq? (ctob/get-token-in-set lib "theme" "button.primary.background") + {:name "button.primary.background" + :type :color + :value "{accent.default}" + :description ""}))) + (t/testing "invalid tokens got discarded" + (t/is (nil? (ctob/get-token-in-set lib "typography" "H1.Bold"))))))) + +#?(:clj + (t/deftest parse-multi-set-dtcg-json-default-team (let [json (-> (slurp "test/common_tests/types/data/tokens-default-team-only.json") - (tr/decode-str)) - lib (ctob/decode-dtcg-json (ctob/ensure-tokens-lib nil) json) - get-set-token (fn [set-name token-name] - (some-> (ctob/get-set lib set-name) - (ctob/get-token token-name))) + (json/decode {:key-fn identity})) + lib (ctob/parse-decoded-json json "") themes (ctob/get-themes lib) first-theme (first themes)] (t/is (= '("dark") (ctob/get-ordered-set-names lib))) @@ -1469,15 +1478,14 @@ (t/is (= (:group first-theme) "")) (t/is (= (:name first-theme) ctob/hidden-token-theme-name))) (t/testing "token exist in dark set" - (t/is (tht/token-data-eq? (get-set-token "dark" "small") + (t/is (tht/token-data-eq? (ctob/get-token-in-set lib "dark" "small") {:name "small" :value "8" :type :border-radius :description ""})))))) - #?(:clj - (t/deftest encode-dtcg-json + (t/deftest export-dtcg-json (let [now (dt/now) tokens-lib (-> (ctob/make-tokens-lib) (ctob/add-set (ctob/make-token-set :name "core" @@ -1502,7 +1510,7 @@ :id "test-id-00" :modified-at now :sets #{"core"}))) - result (ctob/encode-dtcg tokens-lib) + result (ctob/export-dtcg-json tokens-lib) expected {"$themes" [{"description" "" "group" "group-1" "is-source" false @@ -1528,7 +1536,7 @@ (t/is (= expected result))))) #?(:clj - (t/deftest encode-decode-dtcg-json + (t/deftest export-parse-dtcg-json (with-redefs [dt/now (constantly #inst "2024-10-16T12:01:20.257840055-00:00")] (let [tokens-lib (-> (ctob/make-tokens-lib) (ctob/add-set (ctob/make-token-set :name "core" @@ -1549,17 +1557,14 @@ :type :color :value "{accent.default}"})}))) - encoded (ctob/encode-dtcg tokens-lib) - with-prev-tokens-lib (ctob/decode-dtcg-json tokens-lib encoded) - with-empty-tokens-lib (ctob/decode-dtcg-json (ctob/ensure-tokens-lib nil) encoded)] + encoded (ctob/export-dtcg-json tokens-lib) + tokens-lib' (ctob/parse-decoded-json encoded "")] (t/testing "library got updated but data is equal" - (t/is (not= with-prev-tokens-lib tokens-lib)) - (t/is (= @with-prev-tokens-lib @tokens-lib))) - (t/testing "fresh tokens library is also equal" - (= @with-empty-tokens-lib @tokens-lib)))))) + (t/is (not= tokens-lib' tokens-lib)) + (t/is (= @tokens-lib' @tokens-lib))))))) #?(:clj - (t/deftest encode-default-theme-json + (t/deftest export-dtcg-json-with-default-theme (let [tokens-lib (-> (ctob/make-tokens-lib) (ctob/add-set (ctob/make-token-set :name "core" :tokens {"colors.red.600" @@ -1578,7 +1583,7 @@ {:name "button.primary.background" :type :color :value "{accent.default}"})}))) - result (ctob/encode-dtcg tokens-lib) + result (ctob/export-dtcg-json tokens-lib) expected {"$themes" [] "$metadata" {"tokenSetOrder" ["core"] "activeSets" #{}, "activeThemes" #{}} @@ -1599,7 +1604,7 @@ (t/is (= expected result))))) #?(:clj - (t/deftest encode-dtcg-json-with-active-theme-and-set + (t/deftest export-dtcg-json-with-active-theme-and-set (let [now (dt/now) tokens-lib (-> (ctob/make-tokens-lib) (ctob/add-set (ctob/make-token-set :name "core" @@ -1625,7 +1630,7 @@ :modified-at now :sets #{"core"})) (ctob/toggle-theme-active? "group-1" "theme-1")) - result (ctob/encode-dtcg tokens-lib) + result (ctob/export-dtcg-json tokens-lib) expected {"$themes" [{"description" "" "group" "group-1" "is-source" false diff --git a/frontend/src/app/main/data/style_dictionary.cljs b/frontend/src/app/main/data/style_dictionary.cljs index af3eb30cd..696ed6538 100644 --- a/frontend/src/app/main/data/style_dictionary.cljs +++ b/frontend/src/app/main/data/style_dictionary.cljs @@ -11,14 +11,10 @@ [app.common.files.tokens :as cft] [app.common.logging :as l] [app.common.schema :as sm] - [app.common.transit :as t] [app.common.types.tokens-lib :as ctob] - [app.main.data.notifications :as ntf] [app.main.data.tinycolor :as tinycolor] [app.main.data.workspace.tokens.errors :as wte] [app.main.data.workspace.tokens.warnings :as wtw] - [app.main.store :as st] - [app.util.i18n :refer [tr]] [app.util.time :as dt] [beicon.v2.core :as rx] [cuerdas.core :as str] @@ -249,116 +245,6 @@ #(get tokens (sd-token-name %)) (StyleDictionary. (assoc default-config :log {:verbosity "verbose"})))) -;; === Import - -(defn- decode-single-set-json - "Decodes parsed json containing single token set and converts to library" - [this set-name tokens] - (assert (map? tokens) "expected a map data structure for `data`") - - (ctob/add-set this (ctob/make-token-set :name (ctob/normalize-set-name set-name) - :tokens (ctob/flatten-nested-tokens-json tokens "")))) - -(defn- decode-single-set-legacy-json - "Decodes parsed legacy json containing single token set and converts to library" - [this set-name tokens] - (assert (map? tokens) "expected a map data structure for `data`") - (decode-single-set-json this set-name (ctob/legacy-nodes->dtcg-nodes tokens))) - -(defn- reference-errors - "Extracts reference errors from StyleDictionary." - [err] - (let [[header-1 header-2 & errors] (str/split err "\n")] - (when (and - (= header-1 "Error: ") - (= header-2 "Reference Errors:")) - errors))) - -(defn name-error - "Extracts name error out of malli schema error during import." - [err] - (let [schema-error (some-> (ex-data err) - (get-in [:app.common.schema/explain :errors]) - (first)) - name-error? (= (:in schema-error) [:name])] - (when name-error? - (wte/error-ex-info :error.import/invalid-token-name (:value schema-error) err)))) - - -(defn- group-by-value [m] - (reduce (fn [acc [k v]] - (update acc v conj k)) {} m)) - -(defn- tokens-of-unknown-type-warning [unknown-tokens] - (let [type->tokens (group-by-value unknown-tokens)] - (ntf/show {:content (tr "workspace.tokens.unknown-token-type") - :detail (->> (for [[token-type tokens] type->tokens] - (tr "workspace.tokens.unknown-token-type-section" token-type (count tokens))) - (str/join "\n")) - :type :toast - :level :info}))) - -(defn parse-json [data] - (try - (t/decode-str data) - (catch js/Error e - (throw (wte/error-ex-info :error.import/json-parse-error data e))))) - -(defn decode-json-data [data file-name] - (let [single-set? (ctob/single-set? data) - json-format (ctob/get-json-format data) - unknown-tokens (ctob/get-tokens-of-unknown-type - data - "" - (= json-format :json-format/dtcg))] - {:tokens-lib - (try - (cond - (and single-set? - (= :json-format/legacy json-format)) - (decode-single-set-legacy-json (ctob/ensure-tokens-lib nil) file-name data) - - (and single-set? - (= :json-format/dtcg json-format)) - (decode-single-set-json (ctob/ensure-tokens-lib nil) file-name data) - - (= :json-format/legacy json-format) - (ctob/decode-legacy-json (ctob/ensure-tokens-lib nil) data) - - :else - (ctob/decode-dtcg-json (ctob/ensure-tokens-lib nil) data)) - - (catch js/Error e - (let [err (or (name-error e) - (wte/error-ex-info :error.import/invalid-json-data data e))] - (throw err)))) - :unknown-tokens unknown-tokens})) - -(defn process-json-stream - ([data-stream] - (process-json-stream nil data-stream)) - ([params data-stream] - (let [{:keys [file-name]} params] - (->> data-stream - (rx/map parse-json) - (rx/map #(decode-json-data % file-name)) - (rx/mapcat (fn [{:keys [tokens-lib unknown-tokens]}] - (when unknown-tokens - (st/emit! (tokens-of-unknown-type-warning unknown-tokens))) - (try - (->> (ctob/get-all-tokens tokens-lib) - (resolve-tokens-with-errors) - (rx/map (fn [_] - tokens-lib)) - (rx/catch (fn [sd-error] - (let [reference-errors (reference-errors sd-error)] - ;; We allow reference errors for the users to resolve in the ui and throw on any other errors - (if reference-errors - (rx/of tokens-lib) - (throw (wte/error-ex-info :error.import/style-dictionary-unknown-error sd-error sd-error))))))) - (catch js/Error e - (throw (wte/error-ex-info :error.import/style-dictionary-unknown-error "" e)))))))))) - ;; === Hooks (defonce !tokens-cache (atom nil)) diff --git a/frontend/src/app/main/data/workspace/tokens/import_export.cljs b/frontend/src/app/main/data/workspace/tokens/import_export.cljs new file mode 100644 index 000000000..f886446a9 --- /dev/null +++ b/frontend/src/app/main/data/workspace/tokens/import_export.cljs @@ -0,0 +1,134 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns app.main.data.workspace.tokens.import-export + (:require + [app.common.files.helpers :as cfh] + [app.common.json :as json] + [app.common.types.tokens-lib :as ctob] + [app.main.data.notifications :as ntf] + [app.main.data.style-dictionary :as sd] + [app.main.data.workspace.tokens.errors :as wte] + [app.main.store :as st] + [app.util.i18n :refer [tr]] + [beicon.v2.core :as rx] + [cuerdas.core :as str])) + +(defn- extract-reference-errors + "Extracts reference errors from errors produced by StyleDictionary." + [err] + (let [[header-1 header-2 & errors] (str/split err "\n")] + (when (and + (= header-1 "Error: ") + (= header-2 "Reference Errors:")) + errors))) + +(defn- extract-name-error + "Extracts name error out of malli schema error during import." + [err] + (let [schema-error (some-> (ex-data err) + (get-in [:app.common.schema/explain :errors]) + (first)) + name-error? (= (:in schema-error) [:name])] + (when name-error? + (wte/error-ex-info :error.import/invalid-token-name (:value schema-error) err)))) + +(defn- group-by-value [m] + (reduce (fn [acc [k v]] + (update acc v conj k)) {} m)) + +(defn- show-unknown-types-warning [unknown-tokens] + (let [type->tokens (group-by-value unknown-tokens)] + (ntf/show {:content (tr "workspace.tokens.unknown-token-type") + :detail (->> (for [[token-type tokens] type->tokens] + (tr "workspace.tokens.unknown-token-type-section" token-type (count tokens))) + (str/join "\n")) + :type :toast + :level :info}))) + +(defn- decode-json + [json-string] + (try + (json/decode json-string {:key-fn identity}) + (catch js/Error e + (throw (wte/error-ex-info :error.import/json-parse-error json-string e))))) + +(defn- parse-decoded-json + [decoded-json file-name] + (try + {:tokens-lib (ctob/parse-decoded-json decoded-json file-name) + :unknown-tokens (ctob/get-tokens-of-unknown-type decoded-json)} + (catch js/Error e + (let [err (or (extract-name-error e) + (wte/error-ex-info :error.import/invalid-json-data decoded-json e))] + (throw err))))) + +(defn- validate-library + "Resolve tokens in the library and search for errors. Reference errors are ignored, since + it can be resolved by the user in the UI. All the other errors are thrown as exceptions." + [{:keys [tokens-lib unknown-tokens]}] + (when unknown-tokens + (st/emit! (show-unknown-types-warning unknown-tokens))) + (try + (->> (ctob/get-all-tokens tokens-lib) + (sd/resolve-tokens-with-errors) + (rx/map (fn [_] + tokens-lib)) + (rx/catch (fn [sd-error] + (let [reference-errors (extract-reference-errors sd-error)] + (if reference-errors + (rx/of tokens-lib) + (throw (wte/error-ex-info :error.import/style-dictionary-unknown-error sd-error sd-error))))))) + (catch js/Error e + (throw (wte/error-ex-info :error.import/style-dictionary-unknown-error "" e))))) + +(defn- drop-parent-directory + [path] + (->> (cfh/split-path path) + (rest) + (str/join "/"))) + +(defn- remove-path-extension + [path] + (-> (str/split path ".") + (butlast) + (str/join))) + +(defn- file-path->set-name + [path] + (-> path + (drop-parent-directory) + (remove-path-extension))) + +(defn import-file-stream + [file-path file-text] + (let [file-name (remove-path-extension file-path)] + (->> file-text + (rx/map decode-json) + (rx/map #(parse-decoded-json % file-name)) + (rx/mapcat validate-library)))) + +(defn import-directory-stream + [file-stream] + (->> file-stream + (rx/map (fn [[file-path file-text]] + (let [set-name (file-path->set-name file-path)] + (try + {set-name (decode-json file-text)} + (catch js/Error e + ;; Ignore files with json parse errors + {:path file-path :error e}))))) + (rx/reduce (fn [merged-json decoded-json] + (if (:error decoded-json) + merged-json + (conj merged-json decoded-json))) + {}) + (rx/map (fn [merged-json] + (parse-decoded-json (if (= 1 (count merged-json)) + (val (first merged-json)) + merged-json) + (ffirst merged-json)))) + (rx/mapcat validate-library))) diff --git a/frontend/src/app/main/ui/workspace/tokens/modals/import.cljs b/frontend/src/app/main/ui/workspace/tokens/modals/import.cljs index 64ef9f314..d0130e188 100644 --- a/frontend/src/app/main/ui/workspace/tokens/modals/import.cljs +++ b/frontend/src/app/main/ui/workspace/tokens/modals/import.cljs @@ -7,12 +7,11 @@ (ns app.main.ui.workspace.tokens.modals.import (:require-macros [app.main.style :as stl]) (:require - [app.common.files.helpers :as cfh] [app.main.data.event :as ev] [app.main.data.modal :as modal] [app.main.data.notifications :as ntf] - [app.main.data.style-dictionary :as sd] - [app.main.data.workspace.tokens.errors :as wte] + [app.main.data.workspace.tokens.errors :as dwte] + [app.main.data.workspace.tokens.import-export :as dwti] [app.main.data.workspace.tokens.library-edit :as dwtl] [app.main.store :as st] [app.main.ui.ds.buttons.button :refer [button*]] @@ -29,23 +28,8 @@ [potok.v2.core :as ptk] [rumext.v2 :as mf])) -(defn- drop-parent-directory [path] - (->> (cfh/split-path path) - (rest) - (str/join "/"))) - -(defn- remove-path-extension [path] - (-> (str/split path ".") - (butlast) - (str/join))) - -(defn- file-path->set-name - [path] - (-> path - (drop-parent-directory) - (remove-path-extension))) - -(defn- on-import-stream [tokens-lib-stream] +(defn- on-stream-imported + [tokens-lib-stream] (rx/sub! tokens-lib-stream (fn [lib] @@ -53,9 +37,8 @@ (dwtl/import-tokens-lib lib)) (modal/hide!)) (fn [err] - (js/console.error err) - (st/emit! (ntf/show {:content (wte/humanize-errors [(ex-data err)]) - :detail (wte/detail-errors [(ex-data err)]) + (st/emit! (ntf/show {:content (dwte/humanize-errors [(ex-data err)]) + :detail (dwte/detail-errors [(ex-data err)]) :type :toast :level :error}))))) @@ -81,44 +64,28 @@ type (.-type file)] (or (= type "application/json") - (str/ends-with? name ".json"))))) - ;; Read files as text, ignore files with json parse errors - (map (fn [file] - (->> (wapi/read-file-as-text file) - (rx/mapcat (fn [json] - (let [path (.-webkitRelativePath file)] - (rx/of - (try - {(file-path->set-name path) (sd/parse-json json)} - (catch js/Error e - {:path path :error e}))))))))))] - - (->> (apply rx/merge files) - (rx/reduce (fn [acc cur] - (if (:error cur) - acc - (conj acc cur))) - {}) - (rx/map #(sd/decode-json-data (if (= 1 (count %)) - (val (first %)) - %) - (ffirst %))) - (rx/map :tokens-lib) - (on-import-stream)) + (str/ends-with? name ".json"))))))] + (->> (rx/from files) + (rx/mapcat (fn [file] + (->> (wapi/read-file-as-text file) + (rx/map (fn [file-text] + [(.-webkitRelativePath file) + file-text]))))) + (dwti/import-directory-stream) + (on-stream-imported)) (-> (mf/ref-val dir-input-ref) (dom/set-value! ""))))) - on-import + on-import-file (mf/use-fn (fn [event] (let [file (-> (dom/get-target event) (dom/get-files) - (first)) - file-name (remove-path-extension (.-name file))] + (first))] (->> (wapi/read-file-as-text file) - (sd/process-json-stream {:file-name file-name}) - (on-import-stream)) + (dwti/import-file-stream (.-name file)) + (on-stream-imported)) (-> (mf/ref-val file-input-ref) (dom/set-value! "")))))] @@ -142,7 +109,7 @@ :ref file-input-ref :style {:display "none"} :accept ".json" - :on-change on-import}] + :on-change on-import-file}] [:input {:type "file" :ref dir-input-ref :style {:display "none"} diff --git a/frontend/src/app/main/ui/workspace/tokens/sidebar.cljs b/frontend/src/app/main/ui/workspace/tokens/sidebar.cljs index ba7c101d5..a81381f3a 100644 --- a/frontend/src/app/main/ui/workspace/tokens/sidebar.cljs +++ b/frontend/src/app/main/ui/workspace/tokens/sidebar.cljs @@ -381,7 +381,7 @@ (fn [] (st/emit! (ptk/data-event ::ev/event {::ev/name "export-tokens"})) (let [tokens-json (some-> (deref refs/tokens-lib) - (ctob/encode-dtcg) + (ctob/export-dtcg-json) (json/encode :key-fn identity))] (->> (wapi/create-blob (or tokens-json "{}") "application/json") (dom/trigger-download "tokens.json"))))) diff --git a/frontend/test/frontend_tests/runner.cljs b/frontend/test/frontend_tests/runner.cljs index 9799fb6aa..b8bc7c705 100644 --- a/frontend/test/frontend_tests/runner.cljs +++ b/frontend/test/frontend_tests/runner.cljs @@ -9,6 +9,7 @@ [frontend-tests.logic.frame-guides-test] [frontend-tests.logic.groups-test] [frontend-tests.plugins.context-shapes-test] + [frontend-tests.tokens.import-export-test] [frontend-tests.tokens.logic.token-actions-test] [frontend-tests.tokens.logic.token-data-test] [frontend-tests.tokens.style-dictionary-test] @@ -40,5 +41,6 @@ 'frontend-tests.basic-shapes-test 'frontend-tests.tokens.logic.token-actions-test 'frontend-tests.tokens.logic.token-data-test + 'frontend-tests.tokens.import-export-test 'frontend-tests.tokens.style-dictionary-test 'frontend-tests.tokens.token-form-test)) diff --git a/frontend/test/frontend_tests/tokens/import_export_test.cljs b/frontend/test/frontend_tests/tokens/import_export_test.cljs new file mode 100644 index 000000000..5cf90c8fc --- /dev/null +++ b/frontend/test/frontend_tests/tokens/import_export_test.cljs @@ -0,0 +1,101 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns frontend-tests.tokens.import-export-test + (:require + [app.common.json :as json] + [app.common.types.tokens-lib :as ctob] + [app.main.data.workspace.tokens.import-export :as dwti] + [beicon.v2.core :as rx] + [cljs.test :as t :include-macros true])) + +(t/deftest import-file-stream-test + (t/async + done + (t/testing "import simple color token value" + (let [json (-> {"core" {"color" {"$value" "red" + "$type" "color"}} + "$metadata" {"tokenSetOrder" ["core"]}} + (json/encode {:type :json-verbose}))] + (->> (rx/of json) + (dwti/import-file-stream "core") + (rx/subs! (fn [tokens-lib] + (t/is (instance? ctob/TokensLib tokens-lib)) + (t/is (= "red" (-> (ctob/get-set tokens-lib "core") + (ctob/get-token "color") + (:value)))) + (done)))))))) + +(t/deftest reference-errors-test + (t/testing "Extracts reference errors from StyleDictionary errors" + ;; Using unicode for the white-space after "Error: " as some editors might remove it and its more visible + (t/is (= + ["Some token references (2) could not be found." + "" + "foo.value tries to reference missing, which is not defined." + "color.value tries to reference missing, which is not defined."] + (#'dwti/extract-reference-errors "Error:\u0020 +Reference Errors: +Some token references (2) could not be found. + +foo.value tries to reference missing, which is not defined. +color.value tries to reference missing, which is not defined."))) + (t/is (nil? (#'dwti/extract-reference-errors nil))) ;; #' is used to access private functions + (t/is (nil? (#'dwti/extract-reference-errors "none"))))) + +(t/deftest import-empty-json-stream-test + (t/async + done + (t/testing "fails on empty json string" + (->> (rx/of "{}") + (dwti/import-file-stream "") + (rx/subs! + (fn [_] + (throw (js/Error. "Should be an error"))) + (fn [err] + (t/is (= :error.import/invalid-json-data (:error/code (ex-data err)))) + (done))))))) + +(t/deftest import-invalid-json-stream-test + (t/async + done + (t/testing "fails on invalid json" + (->> (rx/of "{,}") + (dwti/import-file-stream "") + (rx/subs! + (fn [_] + (throw (js/Error. "Should be an error"))) + (fn [err] + (t/is (= :error.import/json-parse-error (:error/code (ex-data err)))) + (done))))))) + +(t/deftest import-non-token-json-stream-test + (t/async + done + (t/testing "fails on non-token json" + (->> (rx/of "{\"foo\": \"bar\"}") + (dwti/import-file-stream "") + (rx/subs! + (fn [] + (throw (js/Error. "Should be an error"))) + (fn [err] + (t/is (= :error.import/invalid-json-data (:error/code (ex-data err)))) + (done))))))) + +(t/deftest import-missing-references-json-test + (t/async + done + (t/testing "allows missing references in tokens" + (let [json (-> {"core" {"color" {"$value" "{missing}" + "$type" "color"}} + "$metadata" {"tokenSetOrder" ["core"]}} + (json/encode {:type :json-verbose}))] + (->> (rx/of json) + (dwti/import-file-stream "") + (rx/subs! (fn [tokens-lib] + (t/is (instance? ctob/TokensLib tokens-lib)) + (t/is (= "{missing}" (:value (ctob/get-token-in-set tokens-lib "core" "color")))) + (done)))))))) diff --git a/frontend/test/frontend_tests/tokens/style_dictionary_test.cljs b/frontend/test/frontend_tests/tokens/style_dictionary_test.cljs index 24a609960..745321d79 100644 --- a/frontend/test/frontend_tests/tokens/style_dictionary_test.cljs +++ b/frontend/test/frontend_tests/tokens/style_dictionary_test.cljs @@ -6,7 +6,6 @@ (ns frontend-tests.tokens.style-dictionary-test (:require - [app.common.transit :as tr] [app.common.types.tokens-lib :as ctob] [app.main.data.style-dictionary :as sd] [beicon.v2.core :as rx] @@ -51,114 +50,3 @@ (t/is (= :error.token/number-too-large (get-in resolved-tokens ["borderRadius.largeFn" :errors 0 :error/code]))) (done)))))))) - -(t/deftest process-json-stream-test - (t/async - done - (t/testing "process simple color token value" - (let [json (-> {"core" {"color" {"$value" "red" - "$type" "color"}} - "$metadata" {"tokenSetOrder" ["core"]}} - (tr/encode-str {:type :json-verbose}))] - (->> (rx/of json) - (sd/process-json-stream) - (rx/subs! (fn [tokens-lib] - (t/is (instance? ctob/TokensLib tokens-lib)) - (t/is (= "red" (-> (ctob/get-set tokens-lib "core") - (ctob/get-token "color") - (:value)))) - (done)))))))) - -(t/deftest reference-errros-test - (t/testing "Extracts reference errors from StyleDictionary errors" - ;; Using unicode for the white-space after "Error: " as some editors might remove it and its more visible - (t/is (= - ["Some token references (2) could not be found." - "" - "foo.value tries to reference missing, which is not defined." - "color.value tries to reference missing, which is not defined."] - (sd/reference-errors "Error:\u0020 -Reference Errors: -Some token references (2) could not be found. - -foo.value tries to reference missing, which is not defined. -color.value tries to reference missing, which is not defined."))) - (t/is (nil? (sd/reference-errors nil))) - (t/is (nil? (sd/reference-errors "none"))))) - -(t/deftest process-empty-json-stream-test - (t/async - done - (t/testing "processes empty json string" - (->> (rx/of "{}") - (sd/process-json-stream) - (rx/subs! (fn [tokens-lib] - (t/is (instance? ctob/TokensLib tokens-lib)) - (done))))))) - -(t/deftest process-invalid-json-stream-test - (t/async - done - (t/testing "fails on invalid json" - (->> (rx/of "{,}") - (sd/process-json-stream) - (rx/subs! - (fn [] - (throw (js/Error. "Should be an error"))) - (fn [err] - (t/is (= :error.import/json-parse-error (:error/code (ex-data err)))) - (done))))))) - -(t/deftest process-non-token-json-stream-test - (t/async - done - (t/testing "fails on non-token json" - (->> (rx/of "{\"foo\": \"bar\"}") - (sd/process-json-stream) - (rx/subs! - (fn [] - (throw (js/Error. "Should be an error"))) - (fn [err] - (t/is (= :error.import/invalid-json-data (:error/code (ex-data err)))) - (done))))))) - -(t/deftest process-missing-references-json-test - (t/async - done - (t/testing "allows missing references in tokens" - (let [json (-> {"core" {"color" {"$value" "{missing}" - "$type" "color"}} - "$metadata" {"tokenSetOrder" ["core"]}} - (tr/encode-str {:type :json-verbose}))] - (->> (rx/of json) - (sd/process-json-stream) - (rx/subs! (fn [tokens-lib] - (t/is (instance? ctob/TokensLib tokens-lib)) - (t/is (= "{missing}" (:value (ctob/get-token-in-set tokens-lib "core" "color")))) - (done)))))))) - -(t/deftest single-set-legacy-json-decoding - (let [decode-single-set-legacy-json #'sd/decode-single-set-legacy-json - json {"color" {"red" {"100" {"value" "red" - "type" "color" - "description" ""}}}} - lib (decode-single-set-legacy-json (ctob/ensure-tokens-lib nil) "single_set" json) - get-set-token (fn [set-name token-name] - (some-> (ctob/get-set lib set-name) - (ctob/get-token token-name)))] - (t/is (= '("single_set") (ctob/get-ordered-set-names lib))) - (t/testing "token added" - (t/is (some? (get-set-token "single_set" "color.red.100")))))) - -(t/deftest single-set-dtcg-json-decoding - (let [decode-single-set-json #'sd/decode-single-set-json - json (-> {"color" {"red" {"100" {"$value" "red" - "$type" "color" - "$description" ""}}}}) - lib (decode-single-set-json (ctob/ensure-tokens-lib nil) "single_set" json) - get-set-token (fn [set-name token-name] - (some-> (ctob/get-set lib set-name) - (ctob/get-token token-name)))] - (t/is (= '("single_set") (ctob/get-ordered-set-names lib))) - (t/testing "token added" - (t/is (some? (get-set-token "single_set" "color.red.100"))))))