Merge pull request #312 from tokens-studio/import-sd-2

✨ Import: Verify data with StyleDictionary

common/src/app/common/types/token.cljc

@@ -36,7 +36,6 @@
 (def token-type->dtcg-token-type
   {:boolean       "boolean"
    :border-radius "borderRadius"
-   :box-shadow    "boxShadow"
    :color         "color"
    :dimensions    "dimension"
    :numeric       "numeric"

common/src/app/common/types/tokens_lib.cljc

@@ -197,6 +197,21 @@
        (assoc-in acc path (update-token-fn token))))
    {} tokens))
 
+(defn backtrace-tokens-tree
+  "Convert tokens into a nested tree with their `:name` as the path.
+  Generates a uuid per token to backtrace a token from an external source (StyleDictionary).
+  The backtrace can't be the name as the name might not exist when the user is creating a token."
+  [tokens]
+  (reduce
+   (fn [acc [_ token]]
+     (let [temp-id (random-uuid)
+           token (assoc token :temp/id temp-id)
+           path (split-token-path (:name token))]
+       (-> acc
+           (assoc-in (concat [:tokens-tree] path) token)
+           (assoc-in [:ids temp-id] token))))
+   {:tokens-tree {} :ids {}} tokens))
+
 (defprotocol ITokenSet
   (add-token [_ token] "add a token at the end of the list")
   (update-token [_ token-name f] "update a token in the list")
@@ -508,6 +523,7 @@ When `before-set-name` is nil, move set to bottom")
   (update-set-name [_ old-set-name new-set-name] "updates set name in themes")
   (encode-dtcg [_] "Encodes library to a dtcg compatible json string")
   (decode-dtcg-json [_ parsed-json] "Decodes parsed json containing tokens and converts to library")
+  (get-all-tokens [_] "all tokens in the lib")
   (validate [_]))
 
 (deftype TokensLib [sets set-groups themes active-themes]
@@ -800,6 +816,12 @@ When `before-set-name` is nil, move set to bottom")
                   themes
                   active-themes)))
 
+  (get-all-tokens [this]
+    (reduce
+     (fn [tokens' set]
+       (into tokens' (map (fn [x] [(:name x) x]) (get-tokens set))))
+     {} (get-sets this)))
+
   (validate [_]
     (and (valid-token-sets? sets)  ;; TODO: validate set-groups
          (valid-token-themes? themes)

frontend/shadow-cljs.edn

@@ -177,6 +177,12 @@
 
    :devtools {:http-port 3460}
 
+   :js-options
+   {:entry-keys ["module" "browser" "main"]
+    :resolve {"penpot/vendor/text-editor-v2"
+              {:target :file
+               :file   "vendor/text_editor_v2.js"}}}
+
    :compiler-options
    {:output-feature-set :es2020
     :output-wrapper false

frontend/src/app/main/ui/workspace/tokens/errors.cljs

@@ -3,7 +3,23 @@
    [cuerdas.core :as str]))
 
 (def error-codes
-  {:error.token/direct-self-reference
+  {:error.import/json-parse-error
+   {:error/code :error.import/json-parse-error
+    :error/message "Import Error: Could not parse json"}
+
+   :error.import/invalid-json-data
+   {:error/code :error.import/invalid-json-data
+    :error/message "Import Error: Invalid token data in json."}
+
+   :error.import/style-dictionary-reference-errors
+   {:error/code :error.import/style-dictionary-reference-errors
+    :error/fn #(str "Import Error:\n\n" (str/join "\n\n" %))}
+
+   :error.import/style-dictionary-unknown-error
+   {:error/code :error.import/style-dictionary-reference-errors
+    :error/message "Import Error:"}
+
+   :error.token/direct-self-reference
    {:error/code :error.token/direct-self-reference
     :error/message "Token has self reference"}
 
@@ -30,6 +46,11 @@
   (-> (get-error-code error-key)
       (assoc :error/value error-value)))
 
+(defn error-ex-info [error-key error-value exception]
+  (let [err (-> (error-with-value error-key error-value)
+                (assoc :error/exception exception))]
+    (ex-info (:error/code err) err)))
+
 (defn has-error-code? [error-key errors]
   (some #(= (:error/code %) error-key) errors))
 

frontend/src/app/main/ui/workspace/tokens/form.cljs

@@ -115,7 +115,7 @@ Token names should only contain letters and digits separated by . characters.")}
       (-> (update tokens token-name merge {:value value
                                            :name token-name
                                            :type (:type token)})
-          (sd/resolve-tokens+ {:names-map? true})
+          (sd/resolve-tokens+)
           (p/then
            (fn [resolved-tokens]
              (let [{:keys [errors resolved-value] :as resolved-token} (get resolved-tokens token-name)]
@@ -205,9 +205,8 @@ Token names should only contain letters and digits separated by . characters.")}
         color? (wtt/color-token? token)
         selected-set-tokens (mf/deref refs/workspace-selected-token-set-tokens)
         active-theme-tokens (mf/deref refs/workspace-active-theme-sets-tokens)
-        resolved-tokens (sd/use-resolved-tokens active-theme-tokens
-                                                {:names-map? true
-                                                 :cache-atom form-token-cache-atom})
+        resolved-tokens (sd/use-resolved-tokens active-theme-tokens {:cache-atom form-token-cache-atom
+                                                                     :interactive? true})
         token-path (mf/use-memo
                     (mf/deps (:name token))
                     #(wtt/token-name->path (:name token)))

frontend/src/app/main/ui/workspace/tokens/sidebar.cljs

@@ -26,6 +26,7 @@
    [app.main.ui.workspace.sidebar.assets.common :as cmm]
    [app.main.ui.workspace.tokens.changes :as wtch]
    [app.main.ui.workspace.tokens.context-menu :refer [token-context-menu]]
+   [app.main.ui.workspace.tokens.errors :as wte]
    [app.main.ui.workspace.tokens.sets :refer [sets-list]]
    [app.main.ui.workspace.tokens.sets-context :as sets-context]
    [app.main.ui.workspace.tokens.sets-context-menu :refer [sets-context-menu]]
@@ -39,6 +40,7 @@
    [beicon.v2.core :as rx]
    [cuerdas.core :as str]
    [okulary.core :as l]
+   [promesa.core :as p]
    [rumext.v2 :as mf]
    [shadow.resource]))
 
@@ -281,32 +283,15 @@
         (fn [event]
           (let [file (-> event .-target .-files (aget 0))]
             (->> (wapi/read-file-as-text file)
-                 (rx/map (fn [data]
-                           (try
-                             (t/decode-str data)
-                             (catch js/Error e
-                               (throw (ex-info "Json parse error"
-                                               {:user-error "Import Error: Could not parse json"
-                                                :type :json-parse-error
-                                                :data data
-                                                :exception e}))))))
-                 (rx/map (fn [json-data]
-                           (try
-                             (ctob/decode-dtcg-json (ctob/ensure-tokens-lib nil) json-data)
-                             (catch js/Error e
-                               (throw (ex-info "invalid token data"
-                                               {:user-error "Import Error: Invalid token data in json."
-                                                :type :invalid-token-data
-                                                :data json-data
-                                                :exception e}))))))
+                 (sd/process-json-stream)
                  (rx/subs! (fn [lib]
                              (st/emit! (dt/import-tokens-lib lib)))
                            (fn [err]
-                             (let [{:keys [user-error]} (ex-data err)]
-                               (st/emit! (ntf/show {:content user-error
-                                                    :notification-type :toast
-                                                    :type :warning
-                                                    :timeout 3000}))))))
+                             (js/console.error err)
+                             (st/emit! (ntf/show {:content (wte/humanize-errors [(ex-data err)])
+                                                  :type :toast
+                                                  :level :warning
+                                                  :timeout 9000})))))
             (set! (.-value (mf/ref-val input-ref)) "")))
         on-export (fn []
                     (let [tokens-blob (some-> (deref refs/tokens-lib)

frontend/src/app/main/ui/workspace/tokens/style_dictionary.cljs

@@ -3,20 +3,25 @@
    ["@tokens-studio/sd-transforms" :as sd-transforms]
    ["style-dictionary$default" :as sd]
    [app.common.logging :as l]
+   [app.common.transit :as t]
    [app.common.types.tokens-lib :as ctob]
    [app.main.refs :as refs]
    [app.main.ui.workspace.tokens.errors :as wte]
    [app.main.ui.workspace.tokens.tinycolor :as tinycolor]
    [app.main.ui.workspace.tokens.token :as wtt]
+   [beicon.v2.core :as rx]
    [cuerdas.core :as str]
    [promesa.core :as p]
-   [rumext.v2 :as mf]))
+   [rumext.v2 :as mf]
+   [app.common.data :as d]))
 
 (l/set-level! "app.main.ui.workspace.tokens.style-dictionary" :warn)
 
-(def StyleDictionary
-  "Initiates the global StyleDictionary instance with transforms
-  from tokens-studio used to parse and resolved token values."
+;; === Style Dictionary
+
+(def setup-style-dictionary
+  "Initiates the StyleDictionary instance.
+  Setup transforms from tokens-studio used to parse and resolved token values."
   (do
     (sd-transforms/registerTransforms sd)
     (.registerFormat sd #js {:name "custom/json"
@@ -24,44 +29,182 @@
                                        (.-tokens (.-dictionary res)))})
     sd))
 
-;; Functions -------------------------------------------------------------------
+(def default-config
+  {:platforms {:json
+               {:transformGroup "tokens-studio"
+                ;; Required: The StyleDictionary API is focused on files even when working in the browser
+                :files [{:format "custom/json" :destination "penpot"}]}}
+   :preprocessors ["tokens-studio"]
+   ;; Silences style dictionary logs and errors
+   ;; We handle token errors in the UI
+   :log {:verbosity "silent"
+         :warnings "silent"
+         :errors {:brokenReferences "console"}}})
 
-(defn tokens->style-dictionary+
-  "Resolves references and math expressions using StyleDictionary.
-  Returns a promise with the resolved dictionary."
-  [tokens]
-  (let [data (cond-> {:tokens tokens
-                      :platforms {:json {:transformGroup "tokens-studio"
-                                         :files [{:format "custom/json"
-                                                  :destination "fake-filename"}]}}
-                      :log {:verbosity "silent"
-                            :warnings "silent"
-                            :errors {:brokenReferences "console"}}
-                      :preprocessors ["tokens-studio"]}
-               (l/enabled? "app.main.ui.workspace.tokens.style-dictionary" :debug)
-               (update :log merge {:verbosity "verbose"
-                                   :warnings "warn"}))
-        js-data (clj->js data)]
-    (l/debug :hint "Input Data" :js/data js-data)
-    (sd. js-data)))
+(defn parse-sd-token-color-value
+  "Parses `value` of a color `sd-token` into a map like `{:value 1 :unit \"px\"}`.
+  If the value is not parseable and/or has missing references returns a map with `:errors`."
+  [value]
+  (if-let [tc (tinycolor/valid-color value)]
+    {:value value :unit (tinycolor/color-format tc)}
+    {:errors [(wte/error-with-value :error.token/invalid-color value)]}))
 
-(defn resolve-sd-tokens+
-  "Resolves references and math expressions using StyleDictionary.
-  Returns a promise with the resolved dictionary."
-  [tokens]
-  (let [performance-start (js/performance.now)
-        sd (tokens->style-dictionary+ tokens)]
-    (l/debug :hint "StyleDictionary" :js/style-dictionary sd)
-    (-> sd
+(defn parse-sd-token-dimensions-value
+  "Parses `value` of a dimensions `sd-token` into a map like `{:value 1 :unit \"px\"}`.
+  If the `value` is not parseable and/or has missing references returns a map with `:errors`."
+  [value]
+  (or
+   (wtt/parse-token-value value)
+   (if-let [references (seq (ctob/find-token-value-references value))]
+     {:errors [(wte/error-with-value :error.style-dictionary/missing-reference references)]
+      :references references}
+     {:errors [(wte/error-with-value :error.style-dictionary/invalid-token-value value)]})))
+
+(defn process-sd-tokens
+  "Converts a StyleDictionary dictionary with resolved tokens (aka `sd-tokens`) back to clojure.
+  The `get-origin-token` argument should be a function that takes an `sd-token` and returns the original penpot token, so we can merge the resolved attributes back in.
+
+  The `sd-token` will have references in `value` replaced with the computed value as a string.
+  Here's an example for a `sd-token`:
+  ```js
+  {
+    name:  'token.with.reference',
+    value: '12px',
+    type:  'border-radius',
+    path: ['token', 'with', 'reference'],
+
+    // The penpot origin token converted to a js object
+    original: {
+        name:  'token.with.reference',
+        value: '{referenced.token}',
+        type:  'border-radius'
+    },
+  }
+  ```
+  
+  We also convert `sd-token` value string into a unit that can be used as penpot shape attributes.
+    - Dimensions like '12px' will be converted into numbers
+    - Colors will be validated & converted to hex
+
+  Lastly we check for errors in each token
+  `sd-token` will keep the missing references in the `value` (E.g \"{missing} + {existing}\" -> \"{missing} + 12px\")
+  So we parse out the missing references and add them to `:errors` in the final token."
+  [sd-tokens get-origin-token]
+  (reduce
+   (fn [acc ^js sd-token]
+     (let [origin-token (get-origin-token sd-token)
+           value (.-value sd-token)
+           parsed-token-value (case (:type origin-token)
+                                :color (parse-sd-token-color-value value)
+                                (parse-sd-token-dimensions-value value))
+           output-token (if (:errors parsed-token-value)
+                          (merge origin-token parsed-token-value)
+                          (assoc origin-token
+                                 :resolved-value (:value parsed-token-value)
+                                 :unit (:unit parsed-token-value)))]
+       (assoc acc (:name output-token) output-token)))
+   {} sd-tokens))
+
+(defprotocol IStyleDictionary
+  (add-tokens [_ tokens])
+  (enable-debug [_])
+  (get-config [_])
+  (build-dictionary [_]))
+
+(deftype StyleDictionary [config]
+  IStyleDictionary
+  (add-tokens [_ tokens]
+    (StyleDictionary. (assoc config :tokens tokens)))
+
+  (enable-debug [_]
+    (StyleDictionary. (update config :log merge {:verbosity "verbose"})))
+
+  (get-config [_]
+    config)
+
+  (build-dictionary [_]
+    (-> (sd. (clj->js config))
         (.buildAllPlatforms "json")
-        (.catch #(l/error :hint "Styledictionary build error" :js/error %))
-        (.then (fn [^js resp]
-                 (let [performance-end (js/performance.now)
-                       duration-ms (- performance-end performance-start)
-                       resolved-tokens (.-allTokens resp)]
-                   (l/debug :hint (str "Time elapsed" duration-ms "ms") :duration duration-ms)
-                   (l/debug :hint "Resolved tokens" :js/tokens resolved-tokens)
-                   resolved-tokens))))))
+        (p/then #(.-allTokens ^js %)))))
+
+(defn resolve-tokens-tree+
+  ([tokens-tree get-token]
+   (resolve-tokens-tree+ tokens-tree get-token (StyleDictionary. default-config)))
+  ([tokens-tree get-token style-dictionary]
+   (-> style-dictionary
+       (add-tokens tokens-tree)
+       (build-dictionary)
+       (p/then #(process-sd-tokens % get-token)))))
+
+(defn sd-token-name [^js sd-token]
+  (.. sd-token -original -name))
+
+(defn sd-token-uuid [^js sd-token]
+  (uuid (.-uuid (.-id ^js sd-token))))
+
+(defn resolve-tokens+ [tokens]
+  (resolve-tokens-tree+ (ctob/tokens-tree tokens) #(get tokens (sd-token-name %))))
+
+(defn resolve-tokens-interactive+
+  "Interactive check of resolving tokens.
+  Uses a ids map to backtrace the original token from the resolved StyleDictionary token.
+
+  We have to pass in all tokens from all sets in the entire library to style dictionary
+  so we know if references are missing / to resolve them and possibly show interactive previews (in the tokens form) to the user.
+
+  Since we're using the :name path as the identifier we might be throwing away or overriding tokens in the tree that we pass to StyleDictionary.
+
+  So to get back the original token from the resolved sd-token (see my updates for what an sd-token is) we include a temporary :id for the token that we pass to StyleDictionary,
+  this way after the resolving computation we can restore any token, even clashing ones with the same :name path by just looking up that :id in the ids map."
+  [tokens]
+  (let [{:keys [tokens-tree ids]} (ctob/backtrace-tokens-tree tokens)]
+    (resolve-tokens-tree+ tokens-tree  #(get ids (sd-token-uuid %)))))
+
+(defn resolve-tokens-with-errors+ [tokens]
+  (resolve-tokens-tree+
+   (ctob/tokens-tree tokens)
+   #(get tokens (sd-token-name %))
+   (StyleDictionary. (assoc default-config :log {:verbosity "verbose"}))))
+
+;; === Import
+
+(defn reference-errors
+  "Extracts reference errors from StyleDictionary."
+  [err]
+  (let [[header-1 header-2 & errors] (str/split err "\n")]
+    (when (and
+           (= header-1 "Error: ")
+           (= header-2 "Reference Errors:"))
+      errors)))
+
+(defn process-json-stream [data-stream]
+  (->> data-stream
+       (rx/map (fn [data]
+                 (try
+                   (-> (str/replace data "/" "-") ;; TODO Remove when token groups work
+                       (t/decode-str))
+                   (catch js/Error e
+                     (throw (wte/error-ex-info :error.import/json-parse-error data e))))))
+       (rx/map (fn [json-data]
+                 (try
+                   (ctob/decode-dtcg-json (ctob/ensure-tokens-lib nil) json-data)
+                   (catch js/Error e
+                     (throw (wte/error-ex-info :error.import/invalid-json-data json-data e))))))
+       (rx/mapcat (fn [tokens-lib]
+                    (try
+                      (-> (ctob/get-all-tokens tokens-lib)
+                          (resolve-tokens-with-errors+)
+                          (p/then (fn [_] tokens-lib))
+                          (p/catch (fn [sd-error]
+                                     (let [reference-errors (reference-errors sd-error)
+                                           err (if reference-errors
+                                                 (wte/error-ex-info :error.import/style-dictionary-reference-errors reference-errors sd-error)
+                                                 (wte/error-ex-info :error.import/style-dictionary-unknown-error sd-error sd-error))]
+                                       (throw err)))))
+                      (catch js/Error e
+                        (p/rejected (wte/error-ex-info :error.import/style-dictionary-unknown-error "" e))))))))
+
+;; === Errors
 
 (defn humanize-errors [{:keys [errors value] :as _token}]
   (->> (map (fn [err]
@@ -71,51 +214,18 @@
             errors)
        (str/join "\n")))
 
-(defn resolve-tokens+
-  [tokens & {:keys [names-map?] :as config}]
-  (let [{:keys [tree ids-map]} (wtt/token-names-tree-id-map tokens)]
-    (p/let [sd-tokens (resolve-sd-tokens+ tree)]
-      (let [resolved-tokens (reduce
-                             (fn [acc ^js cur]
-                               (let [{:keys [type] :as origin-token} (if names-map?
-                                                                       (get tokens (.. cur -original -name))
-                                                                       (get ids-map (uuid (.-uuid (.-id cur)))))
-                                     value (.-value cur)
-                                     token-or-err (case type
-                                                    :color (if-let [tc (tinycolor/valid-color value)]
-                                                             {:value value :unit (tinycolor/color-format tc)}
-                                                             {:errors [(wte/error-with-value :error.token/invalid-color value)]})
-                                                    (or (wtt/parse-token-value value)
-                                                        (if-let [references (-> (ctob/find-token-value-references value)
-                                                                                (seq))]
-                                                          {:errors [(wte/error-with-value :error.style-dictionary/missing-reference references)]
-                                                           :references references}
-                                                          {:errors [(wte/error-with-value :error.style-dictionary/invalid-token-value value)]})))
-                                     output-token (if (:errors token-or-err)
-                                                    (merge origin-token token-or-err)
-                                                    (assoc origin-token
-                                                           :resolved-value (:value token-or-err)
-                                                           :unit (:unit token-or-err)))]
-                                 (assoc acc (wtt/token-identifier output-token) output-token)))
-                             {} sd-tokens)]
-        (l/debug :hint "Resolved tokens" :js/tokens resolved-tokens)
-        resolved-tokens))))
-
-;; Hooks -----------------------------------------------------------------------
+;; === Hooks
 
 (defonce !tokens-cache (atom nil))
 
 (defonce !theme-tokens-cache (atom nil))
 
-(defn get-cached-tokens [tokens]
-  (get @!tokens-cache tokens tokens))
-
 (defn use-resolved-tokens
   "The StyleDictionary process function is async, so we can't use resolved values directly.
 
   This hook will return the unresolved tokens as state until they are processed,
   then the state will be updated with the resolved tokens."
-  [tokens & {:keys [cache-atom names-map?]
+  [tokens & {:keys [cache-atom interactive?]
              :or {cache-atom !tokens-cache}
              :as config}]
   (let [tokens-state (mf/use-state (get @cache-atom tokens))]
@@ -124,7 +234,7 @@
      (fn []
        (let [cached (get @cache-atom tokens)]
          (cond
-           (nil? tokens) (if names-map? {} [])
+           (nil? tokens) nil
            ;; The tokens are already processing somewhere
            (p/promise? cached) (-> cached
                                    (p/then #(reset! tokens-state %))
@@ -132,19 +242,19 @@
            ;; Get the cached entry
            (some? cached) (reset! tokens-state cached)
            ;; No cached entry, start processing
-           :else (let [promise+ (resolve-tokens+ tokens config)]
+           :else (let [promise+ (if interactive?
+                                  (resolve-tokens-interactive+ tokens)
+                                  (resolve-tokens+ tokens))]
                    (swap! cache-atom assoc tokens promise+)
                    (p/then promise+ (fn [resolved-tokens]
                                       (swap! cache-atom assoc tokens resolved-tokens)
                                       (reset! tokens-state resolved-tokens))))))))
     @tokens-state))
 
-(defn use-resolved-workspace-tokens [& {:as config}]
+(defn use-resolved-workspace-tokens []
   (-> (mf/deref refs/workspace-selected-token-set-tokens)
-      (use-resolved-tokens config)))
+      (use-resolved-tokens)))
 
-(defn use-active-theme-sets-tokens [& {:as config}]
+(defn use-active-theme-sets-tokens []
   (-> (mf/deref refs/workspace-active-theme-sets-tokens)
-      (use-resolved-tokens (merge {:cache-atom !theme-tokens-cache
-                                   :names-map? true}
-                                  config))))
+      (use-resolved-tokens {:cache-atom !theme-tokens-cache})))

frontend/src/app/main/ui/workspace/tokens/update.cljs

@@ -125,7 +125,7 @@
        (rx/from
         (->
          (wtts/get-active-theme-sets-tokens-names-map state)
-         (wtsd/resolve-tokens+ {:names-map? true})))
+         (wtsd/resolve-tokens+)))
        (rx/mapcat
         (fn [sd-tokens]
           (let [undo-id (js/Symbol)]

frontend/test/token_tests/helpers/state.cljs

@@ -24,7 +24,7 @@
     (watch [_ state _]
       (->> (rx/from (-> (get-in state [:workspace-data :tokens-lib])
                         (ctob/get-active-themes-set-tokens)
-                        (sd/resolve-tokens+ {:names-map? true})))
+                        (sd/resolve-tokens+)))
            (rx/mapcat #(rx/of (end)))))))
 
 (defn stop-on

frontend/test/token_tests/style_dictionary_test.cljs

@@ -1,41 +1,115 @@
 (ns token-tests.style-dictionary-test
   (:require
-   [app.common.data :as d]
+   [app.common.transit :as tr]
+   [app.common.types.tokens-lib :as ctob]
    [app.main.ui.workspace.tokens.style-dictionary :as sd]
-   [app.main.ui.workspace.tokens.token :as wtt]
+   [beicon.v2.core :as rx]
    [cljs.test :as t :include-macros true]
    [promesa.core :as p]))
 
-(def border-radius-token
-  {:value "12px"
-   :name "borderRadius.sm"
-   :type :border-radius})
-
-(def reference-border-radius-token
-  {:value "{borderRadius.sm} * 2"
-   :name "borderRadius.md-with-dashes"
-   :type :border-radius})
-
-(def tokens (d/ordered-map
-             (:name border-radius-token) border-radius-token
-             (:name reference-border-radius-token) reference-border-radius-token))
 (t/deftest resolve-tokens-test
   (t/async
-    done
-    (t/testing "resolves tokens using style-dictionary from a ids map"
-      (-> (sd/resolve-tokens+ tokens)
-          (p/finally
-            (fn [resolved-tokens]
-              (let [expected-tokens {"borderRadius.sm"
-                                     (assoc border-radius-token
-                                            :resolved-value 12
-                                            :resolved-unit "px")
-                                     "borderRadius.md-with-dashes"
-                                     (assoc reference-border-radius-token
-                                            :resolved-value 24
-                                            :resolved-unit "px")}]
-                (t/is (= 12 (get-in resolved-tokens ["borderRadius.sm" :resolved-value])))
-                (t/is (= "px" (get-in resolved-tokens ["borderRadius.sm" :unit])))
-                (t/is (= 24 (get-in resolved-tokens ["borderRadius.md-with-dashes" :resolved-value])))
-                (t/is (= "px" (get-in resolved-tokens ["borderRadius.md-with-dashes" :unit])))
-                (done))))))))
+   done
+   (t/testing "resolves tokens using style-dictionary from a ids map"
+     (let [tokens (-> (ctob/make-tokens-lib)
+                      (ctob/add-set (ctob/make-token-set :name "core"))
+                      (ctob/add-token-in-set "core" (ctob/make-token {:name "borderRadius.sm"
+                                                                      :value "12px"
+                                                                      :type :border-radius}))
+                      (ctob/add-token-in-set "core" (ctob/make-token {:value "{borderRadius.sm} * 2"
+                                                                      :name "borderRadius.md-with-dashes"
+                                                                      :type :border-radius}))
+                      (ctob/get-all-tokens))]
+       (-> (sd/resolve-tokens+ tokens)
+           (p/finally
+             (fn [resolved-tokens]
+               (t/is (= 12 (get-in resolved-tokens ["borderRadius.sm" :resolved-value])))
+               (t/is (= "px" (get-in resolved-tokens ["borderRadius.sm" :unit])))
+               (t/is (= 24 (get-in resolved-tokens ["borderRadius.md-with-dashes" :resolved-value])))
+               (t/is (= "px" (get-in resolved-tokens ["borderRadius.md-with-dashes" :unit])))
+               (done))))))))
+
+(t/deftest process-json-stream-test
+  (t/async
+   done
+   (t/testing "processes empty json string"
+     (let [json (-> {"core" {"color" {"$value" "red"
+                                      "$type" "color"}}}
+                    (tr/encode-str {:type :json-verbose}))]
+       (->> (rx/of json)
+            (sd/process-json-stream)
+            (rx/subs! (fn [tokens-lib]
+                        (t/is (instance? ctob/TokensLib tokens-lib))
+                        (t/is (= "red" (-> (ctob/get-set tokens-lib "core")
+                                           (ctob/get-token "color")
+                                           (:value))))
+                        (done))))))))
+
+(t/deftest reference-errros-test
+  (t/testing "Extracts reference errors from StyleDictionary errors"
+    ;; Using unicode for the white-space after "Error: " as some editors might remove it and its more visible
+    (t/is (=
+           ["Some token references (2) could not be found."
+            ""
+            "foo.value tries to reference missing, which is not defined."
+            "color.value tries to reference missing, which is not defined."]
+           (sd/reference-errors "Error:\u0020
+Reference Errors:
+Some token references (2) could not be found.
+
+foo.value tries to reference missing, which is not defined.
+color.value tries to reference missing, which is not defined.")))
+    (t/is (nil? (sd/reference-errors nil)))
+    (t/is (nil? (sd/reference-errors "none")))))
+
+(t/deftest process-empty-json-stream-test
+  (t/async
+   done
+   (t/testing "processes empty json string"
+     (->> (rx/of "{}")
+          (sd/process-json-stream)
+          (rx/subs! (fn [tokens-lib]
+                      (t/is (instance? ctob/TokensLib tokens-lib))
+                      (done)))))))
+
+(t/deftest process-invalid-json-stream-test
+  (t/async
+   done
+   (t/testing "fails on invalid json"
+     (->> (rx/of "{,}")
+          (sd/process-json-stream)
+          (rx/subs!
+           (fn []
+             (throw (js/Error. "Should be an error")))
+           (fn [err]
+             (t/is (= :error.import/json-parse-error (:error/code (ex-data err))))
+             (done)))))))
+
+(t/deftest process-non-token-json-stream-test
+  (t/async
+   done
+   (t/testing "fails on non-token json"
+     (->> (rx/of "{\"foo\": \"bar\"}")
+          (sd/process-json-stream)
+          (rx/subs!
+           (fn []
+             (throw (js/Error. "Should be an error")))
+           (fn [err]
+             (t/is (= :error.import/invalid-json-data (:error/code (ex-data err))))
+             (done)))))))
+
+(t/deftest process-missing-references-json-test
+  (t/async
+   done
+   (t/testing "fails on missing references in tokens"
+     (let [json (-> {"core" {"color" {"$value" "{missing}"
+                                      "$type" "color"}}}
+                    (tr/encode-str {:type :json-verbose}))]
+       (->> (rx/of json)
+            (sd/process-json-stream)
+            (rx/subs!
+             (fn []
+               (throw (js/Error. "Should be an error")))
+             (fn [err]
+               (t/is (= :error.import/style-dictionary-reference-errors (:error/code (ex-data err))))
+               (done))))))))