✨ Detect reference errors when importing tokens

common/src/app/common/types/tokens_lib.cljc

@@ -197,6 +197,21 @@
        (assoc-in acc path (update-token-fn token))))
    {} tokens))
 
+(defn backtrace-tokens-tree
+  "Convert tokens into a nested tree with their `:name` as the path.
+  Generates a uuid per token to backtrace a token from an external source (StyleDictionary).
+  The backtrace can't be the name as the name might not exist when the user is creating a token."
+  [tokens]
+  (reduce
+   (fn [acc [_ token]]
+     (let [temp-id (random-uuid)
+           token (assoc token :temp/id temp-id)
+           path (split-token-path (:name token))]
+       (-> acc
+           (assoc-in (concat [:tokens-tree] path) token)
+           (assoc-in [:ids temp-id] token))))
+   {:tokens-tree {} :ids {}} tokens))
+
 (defprotocol ITokenSet
   (add-token [_ token] "add a token at the end of the list")
   (update-token [_ token-name f] "update a token in the list")
@@ -508,6 +523,7 @@ When `before-set-name` is nil, move set to bottom")
   (update-set-name [_ old-set-name new-set-name] "updates set name in themes")
   (encode-dtcg [_] "Encodes library to a dtcg compatible json string")
   (decode-dtcg-json [_ parsed-json] "Decodes parsed json containing tokens and converts to library")
+  (get-all-tokens [_] "all tokens in the lib")
   (validate [_]))
 
 (deftype TokensLib [sets set-groups themes active-themes]
@@ -800,6 +816,12 @@ When `before-set-name` is nil, move set to bottom")
                   themes
                   active-themes)))
 
+  (get-all-tokens [this]
+    (reduce
+     (fn [tokens' set]
+       (into tokens' (map (fn [x] [(:name x) x]) (get-tokens set))))
+     {} (get-sets this)))
+
   (validate [_]
     (and (valid-token-sets? sets)  ;; TODO: validate set-groups
          (valid-token-themes? themes)

frontend/src/app/main/ui/workspace/tokens/errors.cljs

@@ -3,7 +3,23 @@
    [cuerdas.core :as str]))
 
 (def error-codes
-  {:error.token/direct-self-reference
+  {:error.import/json-parse-error
+   {:error/code :error.import/json-parse-error
+    :error/message "Import Error: Could not parse json"}
+
+   :error.import/invalid-json-data
+   {:error/code :error.import/invalid-json-data
+    :error/message "Import Error: Invalid token data in json."}
+
+   :error.import/style-dictionary-reference-errors
+   {:error/code :error.import/style-dictionary-reference-errors
+    :error/fn #(str "Import Error:\n\n" (str/join "\n\n" %))}
+
+   :error.import/style-dictionary-unknown-error
+   {:error/code :error.import/style-dictionary-reference-errors
+    :error/message "Import Error:"}
+
+   :error.token/direct-self-reference
    {:error/code :error.token/direct-self-reference
     :error/message "Token has self reference"}
 
@@ -30,6 +46,11 @@
   (-> (get-error-code error-key)
       (assoc :error/value error-value)))
 
+(defn error-ex-info [error-key error-value exception]
+  (let [err (-> (error-with-value error-key error-value)
+                (assoc :error/exception exception))]
+    (ex-info (:error/code err) err)))
+
 (defn has-error-code? [error-key errors]
   (some #(= (:error/code %) error-key) errors))
 

frontend/src/app/main/ui/workspace/tokens/form.cljs

@@ -114,7 +114,7 @@ Token names should only contain letters and digits separated by . characters.")}
       (-> (update tokens token-name merge {:value value
                                            :name token-name
                                            :type (:type token)})
-          (sd/resolve-tokens+ {:names-map? true})
+          (sd/resolve-tokens+)
           (p/then
            (fn [resolved-tokens]
              (let [{:keys [errors resolved-value] :as resolved-token} (get resolved-tokens token-name)]
@@ -204,9 +204,8 @@ Token names should only contain letters and digits separated by . characters.")}
         color? (wtt/color-token? token)
         selected-set-tokens (mf/deref refs/workspace-selected-token-set-tokens)
         active-theme-tokens (mf/deref refs/workspace-active-theme-sets-tokens)
-        resolved-tokens (sd/use-resolved-tokens active-theme-tokens
+        resolved-tokens (sd/use-resolved-tokens active-theme-tokens {:cache-atom form-token-cache-atom
-                                                {:names-map? true
+                                                                     :interactive? true})
-                                                 :cache-atom form-token-cache-atom})
         token-path (mf/use-memo
                     (mf/deps (:name token))
                     #(wtt/token-name->path (:name token)))

frontend/src/app/main/ui/workspace/tokens/sidebar.cljs

@@ -25,6 +25,7 @@
    [app.main.ui.workspace.sidebar.assets.common :as cmm]
    [app.main.ui.workspace.tokens.changes :as wtch]
    [app.main.ui.workspace.tokens.context-menu :refer [token-context-menu]]
+   [app.main.ui.workspace.tokens.errors :as wte]
    [app.main.ui.workspace.tokens.sets :refer [sets-list]]
    [app.main.ui.workspace.tokens.sets-context :as sets-context]
    [app.main.ui.workspace.tokens.sets-context-menu :refer [sets-context-menu]]
@@ -38,6 +39,7 @@
    [beicon.v2.core :as rx]
    [cuerdas.core :as str]
    [okulary.core :as l]
+   [promesa.core :as p]
    [rumext.v2 :as mf]
    [shadow.resource]))
 
@@ -276,32 +278,15 @@
         (fn [event]
           (let [file (-> event .-target .-files (aget 0))]
             (->> (wapi/read-file-as-text file)
-                 (rx/map (fn [data]
+                 (sd/process-json-stream)
-                           (try
-                             (t/decode-str data)
-                             (catch js/Error e
-                               (throw (ex-info "Json parse error"
-                                               {:user-error "Import Error: Could not parse json"
-                                                :type :json-parse-error
-                                                :data data
-                                                :exception e}))))))
-                 (rx/map (fn [json-data]
-                           (try
-                             (ctob/decode-dtcg-json (ctob/ensure-tokens-lib nil) json-data)
-                             (catch js/Error e
-                               (throw (ex-info "invalid token data"
-                                               {:user-error "Import Error: Invalid token data in json."
-                                                :type :invalid-token-data
-                                                :data json-data
-                                                :exception e}))))))
                  (rx/subs! (fn [lib]
                              (st/emit! (dt/import-tokens-lib lib)))
                            (fn [err]
-                             (let [{:keys [user-error]} (ex-data err)]
+                             (js/console.error err)
-                               (st/emit! (ntf/show {:content user-error
+                             (st/emit! (ntf/show {:content (wte/humanize-errors [(ex-data err)])
-                                                    :type :toast
+                                                  :type :toast
-                                                    :level :warning
+                                                  :level :warning
-                                                    :timeout 3000}))))))
+                                                  :timeout 9000})))))
             (set! (.-value (mf/ref-val input-ref)) "")))
         on-export (fn []
                     (let [tokens-blob (some-> (deref refs/tokens-lib)

frontend/src/app/main/ui/workspace/tokens/style_dictionary.cljs

@@ -3,20 +3,25 @@
    ["@tokens-studio/sd-transforms" :as sd-transforms]
    ["style-dictionary$default" :as sd]
    [app.common.logging :as l]
+   [app.common.transit :as t]
    [app.common.types.tokens-lib :as ctob]
    [app.main.refs :as refs]
    [app.main.ui.workspace.tokens.errors :as wte]
    [app.main.ui.workspace.tokens.tinycolor :as tinycolor]
    [app.main.ui.workspace.tokens.token :as wtt]
+   [beicon.v2.core :as rx]
    [cuerdas.core :as str]
    [promesa.core :as p]
-   [rumext.v2 :as mf]))
+   [rumext.v2 :as mf]
+   [app.common.data :as d]))
 
 (l/set-level! "app.main.ui.workspace.tokens.style-dictionary" :warn)
 
-(def StyleDictionary
+;; === Style Dictionary
-  "Initiates the global StyleDictionary instance with transforms
+
-  from tokens-studio used to parse and resolved token values."
+(def setup-style-dictionary
+  "Initiates the StyleDictionary instance.
+  Setup transforms from tokens-studio used to parse and resolved token values."
   (do
     (sd-transforms/registerTransforms sd)
     (.registerFormat sd #js {:name "custom/json"
@@ -24,44 +29,137 @@
                                        (.-tokens (.-dictionary res)))})
     sd))
 
-;; Functions -------------------------------------------------------------------
+(def default-config
+  {:platforms {:json
+               {:transformGroup "tokens-studio"
+                ;; Required: The StyleDictionary API is focused on files even when working in the browser
+                :files [{:format "custom/json" :destination "penpot"}]}}
+   :preprocessors ["tokens-studio"]
+   ;; Silences style dictionary logs and errors
+   ;; We handle token errors in the UI
+   :log {:verbosity "silent"
+         :warnings "silent"
+         :errors {:brokenReferences "console"}}})
 
-(defn tokens->style-dictionary+
+(defn process-sd-tokens [sd-tokens get-origin-token]
-  "Resolves references and math expressions using StyleDictionary.
+  (reduce
-  Returns a promise with the resolved dictionary."
+   (fn [acc ^js sd-token]
-  [tokens]
+     (let [{:keys [type] :as origin-token} (get-origin-token sd-token)
-  (let [data (cond-> {:tokens tokens
+           value (.-value sd-token)
-                      :platforms {:json {:transformGroup "tokens-studio"
+           token-or-err (case type
-                                         :files [{:format "custom/json"
+                          :color (if-let [tc (tinycolor/valid-color value)]
-                                                  :destination "fake-filename"}]}}
+                                   {:value value :unit (tinycolor/color-format tc)}
-                      :log {:verbosity "silent"
+                                   {:errors [(wte/error-with-value :error.token/invalid-color value)]})
-                            :warnings "silent"
+                          (or (wtt/parse-token-value value)
-                            :errors {:brokenReferences "console"}}
+                              (if-let [references (-> (ctob/find-token-value-references value)
-                      :preprocessors ["tokens-studio"]}
+                                                      (seq))]
-               (l/enabled? "app.main.ui.workspace.tokens.style-dictionary" :debug)
+                                {:errors [(wte/error-with-value :error.style-dictionary/missing-reference references)]
-               (update :log merge {:verbosity "verbose"
+                                 :references references}
-                                   :warnings "warn"}))
+                                {:errors [(wte/error-with-value :error.style-dictionary/invalid-token-value value)]})))
-        js-data (clj->js data)]
+           output-token (if (:errors token-or-err)
-    (l/debug :hint "Input Data" :js/data js-data)
+                          (merge origin-token token-or-err)
-    (sd. js-data)))
+                          (assoc origin-token
+                                 :resolved-value (:value token-or-err)
+                                 :unit (:unit token-or-err)))]
+       (assoc acc (wtt/token-identifier output-token) output-token)))
+   {} sd-tokens))
 
-(defn resolve-sd-tokens+
+(defprotocol IStyleDictionary
-  "Resolves references and math expressions using StyleDictionary.
+  (add-tokens [_ tokens])
-  Returns a promise with the resolved dictionary."
+  (enable-debug [_])
-  [tokens]
+  (set-config [_])
-  (let [performance-start (js/performance.now)
+  (get-config [_])
-        sd (tokens->style-dictionary+ tokens)]
+  (build-dictionary [_]))
-    (l/debug :hint "StyleDictionary" :js/style-dictionary sd)
+
-    (-> sd
+(deftype StyleDictionary [config]
+  IStyleDictionary
+  (add-tokens [_ tokens]
+    (StyleDictionary. (assoc config :tokens tokens)))
+
+  (enable-debug [_]
+    (StyleDictionary. (update config :log merge {:verbosity "verbose"})))
+
+  (set-config [_]
+    (StyleDictionary. config))
+
+  (get-config [_]
+    config)
+
+  (build-dictionary [_]
+    (-> (sd. (clj->js config))
         (.buildAllPlatforms "json")
-        (.catch #(l/error :hint "Styledictionary build error" :js/error %))
+        (p/then #(.-allTokens ^js %)))))
-        (.then (fn [^js resp]
+
-                 (let [performance-end (js/performance.now)
+(defn resolve-tokens-tree+
-                       duration-ms (- performance-end performance-start)
+  ([tokens-tree get-token]
-                       resolved-tokens (.-allTokens resp)]
+   (resolve-tokens-tree+ tokens-tree get-token (StyleDictionary. default-config)))
-                   (l/debug :hint (str "Time elapsed" duration-ms "ms") :duration duration-ms)
+  ([tokens-tree get-token style-dictionary]
-                   (l/debug :hint "Resolved tokens" :js/tokens resolved-tokens)
+   (-> style-dictionary
-                   resolved-tokens))))))
+       (add-tokens tokens-tree)
+       (build-dictionary)
+       (p/then #(process-sd-tokens % get-token)))))
+
+(defn sd-token-name [^js sd-token]
+  (.. sd-token -original -name))
+
+(defn sd-token-uuid [^js sd-token]
+  (uuid (.-uuid (.-id ^js sd-token))))
+
+(defn resolve-tokens+ [tokens]
+  (resolve-tokens-tree+ (ctob/tokens-tree tokens) #(get tokens (sd-token-name %))))
+
+(defn resolve-tokens-interactive+
+  "Interactive check of resolving tokens.
+  Uses a ids map to backtrace the original token from the resolved StyleDictionary token.
+  This is necessary as the user might have removed/changed the token name but we still want to validate the value interactively."
+  [tokens]
+  (let [{:keys [tokens-tree ids]} (ctob/backtrace-tokens-tree tokens)]
+    (resolve-tokens-tree+ tokens-tree  #(get ids (sd-token-uuid %)))))
+
+(defn resolve-tokens-with-errors+ [tokens]
+  (resolve-tokens-tree+
+   (ctob/tokens-tree tokens)
+   #(get tokens (sd-token-name %))
+   (StyleDictionary. (assoc default-config :log {:verbosity "verbose"}))))
+
+;; === Import
+
+(defn reference-errors
+  "Extracts reference errors from StyleDictionary."
+  [err]
+  (let [[header-1 header-2 & errors] (str/split err "\n")]
+    (when (and
+           (= header-1 "Error: ")
+           (= header-2 "Reference Errors:"))
+      errors)))
+
+(defn process-json-stream [data-stream]
+  (->> data-stream
+       (rx/map (fn [data]
+                 (try
+                   (t/decode-str data)
+                   (catch js/Error e
+                     (throw (wte/error-ex-info :error.import/json-parse-error data e))))))
+       (rx/map (fn [json-data]
+                 (try
+                   (ctob/decode-dtcg-json (ctob/ensure-tokens-lib nil) json-data)
+                   (catch js/Error e
+                     (throw (wte/error-ex-info :error.import/invalid-json-data json-data e))))))
+       (rx/mapcat (fn [tokens-lib]
+                    (try
+                      (-> (ctob/get-all-tokens tokens-lib)
+                          (resolve-tokens-with-errors+)
+                          (p/then (fn [_] tokens-lib))
+                          (p/catch (fn [sd-error]
+                                     (let [reference-errors (reference-errors sd-error)
+                                           err (if reference-errors
+                                                 (wte/error-ex-info :error.import/style-dictionary-reference-errors reference-errors sd-error)
+                                                 (wte/error-ex-info :error.import/style-dictionary-unknown-error sd-error sd-error))]
+                                       (throw err)))))
+                      (catch js/Error e
+                        (p/rejected (wte/error-ex-info :error.import/style-dictionary-unknown-error "" e))))))))
+
+;; === Errors
 
 (defn humanize-errors [{:keys [errors value] :as _token}]
   (->> (map (fn [err]
@@ -71,51 +169,18 @@
             errors)
        (str/join "\n")))
 
-(defn resolve-tokens+
+;; === Hooks
-  [tokens & {:keys [names-map?] :as config}]
-  (let [{:keys [tree ids-map]} (wtt/token-names-tree-id-map tokens)]
-    (p/let [sd-tokens (resolve-sd-tokens+ tree)]
-      (let [resolved-tokens (reduce
-                             (fn [acc ^js cur]
-                               (let [{:keys [type] :as origin-token} (if names-map?
-                                                                       (get tokens (.. cur -original -name))
-                                                                       (get ids-map (uuid (.-uuid (.-id cur)))))
-                                     value (.-value cur)
-                                     token-or-err (case type
-                                                    :color (if-let [tc (tinycolor/valid-color value)]
-                                                             {:value value :unit (tinycolor/color-format tc)}
-                                                             {:errors [(wte/error-with-value :error.token/invalid-color value)]})
-                                                    (or (wtt/parse-token-value value)
-                                                        (if-let [references (-> (ctob/find-token-value-references value)
-                                                                                (seq))]
-                                                          {:errors [(wte/error-with-value :error.style-dictionary/missing-reference references)]
-                                                           :references references}
-                                                          {:errors [(wte/error-with-value :error.style-dictionary/invalid-token-value value)]})))
-                                     output-token (if (:errors token-or-err)
-                                                    (merge origin-token token-or-err)
-                                                    (assoc origin-token
-                                                           :resolved-value (:value token-or-err)
-                                                           :unit (:unit token-or-err)))]
-                                 (assoc acc (wtt/token-identifier output-token) output-token)))
-                             {} sd-tokens)]
-        (l/debug :hint "Resolved tokens" :js/tokens resolved-tokens)
-        resolved-tokens))))
-
-;; Hooks -----------------------------------------------------------------------
 
 (defonce !tokens-cache (atom nil))
 
 (defonce !theme-tokens-cache (atom nil))
 
-(defn get-cached-tokens [tokens]
-  (get @!tokens-cache tokens tokens))
-
 (defn use-resolved-tokens
   "The StyleDictionary process function is async, so we can't use resolved values directly.
 
   This hook will return the unresolved tokens as state until they are processed,
   then the state will be updated with the resolved tokens."
-  [tokens & {:keys [cache-atom names-map?]
+  [tokens & {:keys [cache-atom interactive?]
              :or {cache-atom !tokens-cache}
              :as config}]
   (let [tokens-state (mf/use-state (get @cache-atom tokens))]
@@ -124,7 +189,7 @@
      (fn []
        (let [cached (get @cache-atom tokens)]
          (cond
-           (nil? tokens) (if names-map? {} [])
+           (nil? tokens) nil
            ;; The tokens are already processing somewhere
            (p/promise? cached) (-> cached
                                    (p/then #(reset! tokens-state %))
@@ -132,19 +197,19 @@
            ;; Get the cached entry
            (some? cached) (reset! tokens-state cached)
            ;; No cached entry, start processing
-           :else (let [promise+ (resolve-tokens+ tokens config)]
+           :else (let [promise+ (if interactive?
+                                  (resolve-tokens-interactive+ tokens)
+                                  (resolve-tokens+ tokens))]
                    (swap! cache-atom assoc tokens promise+)
                    (p/then promise+ (fn [resolved-tokens]
                                       (swap! cache-atom assoc tokens resolved-tokens)
                                       (reset! tokens-state resolved-tokens))))))))
     @tokens-state))
 
-(defn use-resolved-workspace-tokens [& {:as config}]
+(defn use-resolved-workspace-tokens []
   (-> (mf/deref refs/workspace-selected-token-set-tokens)
-      (use-resolved-tokens config)))
+      (use-resolved-tokens)))
 
-(defn use-active-theme-sets-tokens [& {:as config}]
+(defn use-active-theme-sets-tokens []
   (-> (mf/deref refs/workspace-active-theme-sets-tokens)
-      (use-resolved-tokens (merge {:cache-atom !theme-tokens-cache
+      (use-resolved-tokens {:cache-atom !theme-tokens-cache})))
-                                   :names-map? true}
-                                  config))))

frontend/src/app/main/ui/workspace/tokens/update.cljs

@@ -125,7 +125,7 @@
        (rx/from
         (->
          (wtts/get-active-theme-sets-tokens-names-map state)
-         (wtsd/resolve-tokens+ {:names-map? true})))
+         (wtsd/resolve-tokens+)))
        (rx/mapcat
         (fn [sd-tokens]
           (let [undo-id (js/Symbol)]

frontend/test/token_tests/helpers/state.cljs

@@ -24,7 +24,7 @@
     (watch [_ state _]
       (->> (rx/from (-> (get-in state [:workspace-data :tokens-lib])
                         (ctob/get-active-themes-set-tokens)
-                        (sd/resolve-tokens+ {:names-map? true})))
+                        (sd/resolve-tokens+)))
            (rx/mapcat #(rx/of (end)))))))
 
 (defn stop-on

frontend/test/token_tests/style_dictionary_test.cljs

@@ -2,9 +2,11 @@
   (:require
    [app.common.data :as d]
    [app.main.ui.workspace.tokens.style-dictionary :as sd]
-   [app.main.ui.workspace.tokens.token :as wtt]
+   [beicon.v2.core :as rx]
+   [app.common.transit :as tr]
    [cljs.test :as t :include-macros true]
-   [promesa.core :as p]))
+   [promesa.core :as p]
+   [app.common.types.tokens-lib :as ctob]))
 
 (def border-radius-token
   {:value "12px"
@@ -19,6 +21,7 @@
 (def tokens (d/ordered-map
              (:name border-radius-token) border-radius-token
              (:name reference-border-radius-token) reference-border-radius-token))
+
 (t/deftest resolve-tokens-test
   (t/async
     done
@@ -26,16 +29,93 @@
       (-> (sd/resolve-tokens+ tokens)
           (p/finally
             (fn [resolved-tokens]
-              (let [expected-tokens {"borderRadius.sm"
+              (t/is (= 12 (get-in resolved-tokens ["borderRadius.sm" :resolved-value])))
-                                     (assoc border-radius-token
+              (t/is (= "px" (get-in resolved-tokens ["borderRadius.sm" :unit])))
-                                            :resolved-value 12
+              (t/is (= 24 (get-in resolved-tokens ["borderRadius.md-with-dashes" :resolved-value])))
-                                            :resolved-unit "px")
+              (t/is (= "px" (get-in resolved-tokens ["borderRadius.md-with-dashes" :unit])))
-                                     "borderRadius.md-with-dashes"
+              (done)))))))
-                                     (assoc reference-border-radius-token
+
-                                            :resolved-value 24
+(t/deftest process-json-stream-test
-                                            :resolved-unit "px")}]
+  (t/async
-                (t/is (= 12 (get-in resolved-tokens ["borderRadius.sm" :resolved-value])))
+   done
-                (t/is (= "px" (get-in resolved-tokens ["borderRadius.sm" :unit])))
+   (t/testing "processes empty json string"
-                (t/is (= 24 (get-in resolved-tokens ["borderRadius.md-with-dashes" :resolved-value])))
+     (let [json (-> {"core" {"color" {"$value" "red"
-                (t/is (= "px" (get-in resolved-tokens ["borderRadius.md-with-dashes" :unit])))
+                                      "$type" "color"}}}
-                (done))))))))
+                    (tr/encode-str {:type :json-verbose}))]
+       (->> (rx/of json)
+            (sd/process-json-stream)
+            (rx/subs! (fn [tokens-lib]
+                        (t/is (instance? ctob/TokensLib tokens-lib))
+                        (t/is (= "red" (-> (ctob/get-set tokens-lib "core")
+                                           (ctob/get-token "color")
+                                           (:value))))
+                        (done))))))))
+
+(t/deftest reference-errros-test
+  (t/testing "Extracts reference errors from StyleDictionary errors"
+    ;; Using unicode for the white-space after "Error: " as some editors might remove it and its more visible
+    (t/is (=
+           ["Some token references (2) could not be found."
+            ""
+            "foo.value tries to reference missing, which is not defined."
+            "color.value tries to reference missing, which is not defined."]
+           (sd/reference-errors "Error:\u0020
+Reference Errors:
+Some token references (2) could not be found.
+
+foo.value tries to reference missing, which is not defined.
+color.value tries to reference missing, which is not defined.")))
+    (t/is (nil? (sd/reference-errors nil)))
+    (t/is (nil? (sd/reference-errors "none")))))
+
+(t/deftest process-empty-json-stream-test
+  (t/async
+   done
+   (t/testing "processes empty json string"
+     (->> (rx/of "{}")
+          (sd/process-json-stream)
+          (rx/subs! (fn [tokens-lib]
+                      (t/is (instance? ctob/TokensLib tokens-lib))
+                      (done)))))))
+
+(t/deftest process-invalid-json-stream-test
+  (t/async
+   done
+   (t/testing "fails on invalid json"
+     (->> (rx/of "{,}")
+          (sd/process-json-stream)
+          (rx/subs!
+           (fn []
+             (throw (js/Error. "Should be an error")))
+           (fn [err]
+             (t/is (= :error.import/json-parse-error (:error/code (ex-data err))))
+             (done)))))))
+
+(t/deftest process-non-token-json-stream-test
+  (t/async
+   done
+   (t/testing "fails on non-token json"
+     (->> (rx/of "{\"foo\": \"bar\"}")
+          (sd/process-json-stream)
+          (rx/subs!
+           (fn []
+             (throw (js/Error. "Should be an error")))
+           (fn [err]
+             (t/is (= :error.import/invalid-json-data (:error/code (ex-data err))))
+             (done)))))))
+
+(t/deftest process-missing-references-json-test
+  (t/async
+   done
+   (t/testing "fails on missing references in tokens"
+     (let [json (-> {"core" {"color" {"$value" "{missing}"
+                                      "$type" "color"}}}
+                    (tr/encode-str {:type :json-verbose}))]
+       (->> (rx/of json)
+            (sd/process-json-stream)
+            (rx/subs!
+             (fn []
+               (throw (js/Error. "Should be an error")))
+             (fn [err]
+               (t/is (= :error.import/style-dictionary-reference-errors (:error/code (ex-data err))))
+               (done))))))))