0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-01-21 06:02:32 -05:00

Detect reference errors when importing tokens

This commit is contained in:
Florian Schroedl 2024-10-22 10:14:47 +02:00
parent d3ded00bc6
commit 66dce0e795
8 changed files with 299 additions and 127 deletions

View file

@ -197,6 +197,21 @@
(assoc-in acc path (update-token-fn token))))
{} tokens))
(defn backtrace-tokens-tree
"Convert tokens into a nested tree with their `:name` as the path.
Generates a uuid per token to backtrace a token from an external source (StyleDictionary).
The backtrace can't be the name as the name might not exist when the user is creating a token."
[tokens]
(reduce
(fn [acc [_ token]]
(let [temp-id (random-uuid)
token (assoc token :temp/id temp-id)
path (split-token-path (:name token))]
(-> acc
(assoc-in (concat [:tokens-tree] path) token)
(assoc-in [:ids temp-id] token))))
{:tokens-tree {} :ids {}} tokens))
(defprotocol ITokenSet
(add-token [_ token] "add a token at the end of the list")
(update-token [_ token-name f] "update a token in the list")
@ -508,6 +523,7 @@ When `before-set-name` is nil, move set to bottom")
(update-set-name [_ old-set-name new-set-name] "updates set name in themes")
(encode-dtcg [_] "Encodes library to a dtcg compatible json string")
(decode-dtcg-json [_ parsed-json] "Decodes parsed json containing tokens and converts to library")
(get-all-tokens [_] "all tokens in the lib")
(validate [_]))
(deftype TokensLib [sets set-groups themes active-themes]
@ -800,6 +816,12 @@ When `before-set-name` is nil, move set to bottom")
themes
active-themes)))
(get-all-tokens [this]
(reduce
(fn [tokens' set]
(into tokens' (map (fn [x] [(:name x) x]) (get-tokens set))))
{} (get-sets this)))
(validate [_]
(and (valid-token-sets? sets) ;; TODO: validate set-groups
(valid-token-themes? themes)

View file

@ -3,7 +3,23 @@
[cuerdas.core :as str]))
(def error-codes
{:error.token/direct-self-reference
{:error.import/json-parse-error
{:error/code :error.import/json-parse-error
:error/message "Import Error: Could not parse json"}
:error.import/invalid-json-data
{:error/code :error.import/invalid-json-data
:error/message "Import Error: Invalid token data in json."}
:error.import/style-dictionary-reference-errors
{:error/code :error.import/style-dictionary-reference-errors
:error/fn #(str "Import Error:\n\n" (str/join "\n\n" %))}
:error.import/style-dictionary-unknown-error
{:error/code :error.import/style-dictionary-reference-errors
:error/message "Import Error:"}
:error.token/direct-self-reference
{:error/code :error.token/direct-self-reference
:error/message "Token has self reference"}
@ -30,6 +46,11 @@
(-> (get-error-code error-key)
(assoc :error/value error-value)))
(defn error-ex-info [error-key error-value exception]
(let [err (-> (error-with-value error-key error-value)
(assoc :error/exception exception))]
(ex-info (:error/code err) err)))
(defn has-error-code? [error-key errors]
(some #(= (:error/code %) error-key) errors))

View file

@ -114,7 +114,7 @@ Token names should only contain letters and digits separated by . characters.")}
(-> (update tokens token-name merge {:value value
:name token-name
:type (:type token)})
(sd/resolve-tokens+ {:names-map? true})
(sd/resolve-tokens+)
(p/then
(fn [resolved-tokens]
(let [{:keys [errors resolved-value] :as resolved-token} (get resolved-tokens token-name)]
@ -204,9 +204,8 @@ Token names should only contain letters and digits separated by . characters.")}
color? (wtt/color-token? token)
selected-set-tokens (mf/deref refs/workspace-selected-token-set-tokens)
active-theme-tokens (mf/deref refs/workspace-active-theme-sets-tokens)
resolved-tokens (sd/use-resolved-tokens active-theme-tokens
{:names-map? true
:cache-atom form-token-cache-atom})
resolved-tokens (sd/use-resolved-tokens active-theme-tokens {:cache-atom form-token-cache-atom
:interactive? true})
token-path (mf/use-memo
(mf/deps (:name token))
#(wtt/token-name->path (:name token)))

View file

@ -25,6 +25,7 @@
[app.main.ui.workspace.sidebar.assets.common :as cmm]
[app.main.ui.workspace.tokens.changes :as wtch]
[app.main.ui.workspace.tokens.context-menu :refer [token-context-menu]]
[app.main.ui.workspace.tokens.errors :as wte]
[app.main.ui.workspace.tokens.sets :refer [sets-list]]
[app.main.ui.workspace.tokens.sets-context :as sets-context]
[app.main.ui.workspace.tokens.sets-context-menu :refer [sets-context-menu]]
@ -38,6 +39,7 @@
[beicon.v2.core :as rx]
[cuerdas.core :as str]
[okulary.core :as l]
[promesa.core :as p]
[rumext.v2 :as mf]
[shadow.resource]))
@ -276,32 +278,15 @@
(fn [event]
(let [file (-> event .-target .-files (aget 0))]
(->> (wapi/read-file-as-text file)
(rx/map (fn [data]
(try
(t/decode-str data)
(catch js/Error e
(throw (ex-info "Json parse error"
{:user-error "Import Error: Could not parse json"
:type :json-parse-error
:data data
:exception e}))))))
(rx/map (fn [json-data]
(try
(ctob/decode-dtcg-json (ctob/ensure-tokens-lib nil) json-data)
(catch js/Error e
(throw (ex-info "invalid token data"
{:user-error "Import Error: Invalid token data in json."
:type :invalid-token-data
:data json-data
:exception e}))))))
(sd/process-json-stream)
(rx/subs! (fn [lib]
(st/emit! (dt/import-tokens-lib lib)))
(fn [err]
(let [{:keys [user-error]} (ex-data err)]
(st/emit! (ntf/show {:content user-error
:type :toast
:level :warning
:timeout 3000}))))))
(js/console.error err)
(st/emit! (ntf/show {:content (wte/humanize-errors [(ex-data err)])
:type :toast
:level :warning
:timeout 9000})))))
(set! (.-value (mf/ref-val input-ref)) "")))
on-export (fn []
(let [tokens-blob (some-> (deref refs/tokens-lib)

View file

@ -3,20 +3,25 @@
["@tokens-studio/sd-transforms" :as sd-transforms]
["style-dictionary$default" :as sd]
[app.common.logging :as l]
[app.common.transit :as t]
[app.common.types.tokens-lib :as ctob]
[app.main.refs :as refs]
[app.main.ui.workspace.tokens.errors :as wte]
[app.main.ui.workspace.tokens.tinycolor :as tinycolor]
[app.main.ui.workspace.tokens.token :as wtt]
[beicon.v2.core :as rx]
[cuerdas.core :as str]
[promesa.core :as p]
[rumext.v2 :as mf]))
[rumext.v2 :as mf]
[app.common.data :as d]))
(l/set-level! "app.main.ui.workspace.tokens.style-dictionary" :warn)
(def StyleDictionary
"Initiates the global StyleDictionary instance with transforms
from tokens-studio used to parse and resolved token values."
;; === Style Dictionary
(def setup-style-dictionary
"Initiates the StyleDictionary instance.
Setup transforms from tokens-studio used to parse and resolved token values."
(do
(sd-transforms/registerTransforms sd)
(.registerFormat sd #js {:name "custom/json"
@ -24,44 +29,137 @@
(.-tokens (.-dictionary res)))})
sd))
;; Functions -------------------------------------------------------------------
(def default-config
{:platforms {:json
{:transformGroup "tokens-studio"
;; Required: The StyleDictionary API is focused on files even when working in the browser
:files [{:format "custom/json" :destination "penpot"}]}}
:preprocessors ["tokens-studio"]
;; Silences style dictionary logs and errors
;; We handle token errors in the UI
:log {:verbosity "silent"
:warnings "silent"
:errors {:brokenReferences "console"}}})
(defn tokens->style-dictionary+
"Resolves references and math expressions using StyleDictionary.
Returns a promise with the resolved dictionary."
[tokens]
(let [data (cond-> {:tokens tokens
:platforms {:json {:transformGroup "tokens-studio"
:files [{:format "custom/json"
:destination "fake-filename"}]}}
:log {:verbosity "silent"
:warnings "silent"
:errors {:brokenReferences "console"}}
:preprocessors ["tokens-studio"]}
(l/enabled? "app.main.ui.workspace.tokens.style-dictionary" :debug)
(update :log merge {:verbosity "verbose"
:warnings "warn"}))
js-data (clj->js data)]
(l/debug :hint "Input Data" :js/data js-data)
(sd. js-data)))
(defn process-sd-tokens [sd-tokens get-origin-token]
(reduce
(fn [acc ^js sd-token]
(let [{:keys [type] :as origin-token} (get-origin-token sd-token)
value (.-value sd-token)
token-or-err (case type
:color (if-let [tc (tinycolor/valid-color value)]
{:value value :unit (tinycolor/color-format tc)}
{:errors [(wte/error-with-value :error.token/invalid-color value)]})
(or (wtt/parse-token-value value)
(if-let [references (-> (ctob/find-token-value-references value)
(seq))]
{:errors [(wte/error-with-value :error.style-dictionary/missing-reference references)]
:references references}
{:errors [(wte/error-with-value :error.style-dictionary/invalid-token-value value)]})))
output-token (if (:errors token-or-err)
(merge origin-token token-or-err)
(assoc origin-token
:resolved-value (:value token-or-err)
:unit (:unit token-or-err)))]
(assoc acc (wtt/token-identifier output-token) output-token)))
{} sd-tokens))
(defn resolve-sd-tokens+
"Resolves references and math expressions using StyleDictionary.
Returns a promise with the resolved dictionary."
[tokens]
(let [performance-start (js/performance.now)
sd (tokens->style-dictionary+ tokens)]
(l/debug :hint "StyleDictionary" :js/style-dictionary sd)
(-> sd
(defprotocol IStyleDictionary
(add-tokens [_ tokens])
(enable-debug [_])
(set-config [_])
(get-config [_])
(build-dictionary [_]))
(deftype StyleDictionary [config]
IStyleDictionary
(add-tokens [_ tokens]
(StyleDictionary. (assoc config :tokens tokens)))
(enable-debug [_]
(StyleDictionary. (update config :log merge {:verbosity "verbose"})))
(set-config [_]
(StyleDictionary. config))
(get-config [_]
config)
(build-dictionary [_]
(-> (sd. (clj->js config))
(.buildAllPlatforms "json")
(.catch #(l/error :hint "Styledictionary build error" :js/error %))
(.then (fn [^js resp]
(let [performance-end (js/performance.now)
duration-ms (- performance-end performance-start)
resolved-tokens (.-allTokens resp)]
(l/debug :hint (str "Time elapsed" duration-ms "ms") :duration duration-ms)
(l/debug :hint "Resolved tokens" :js/tokens resolved-tokens)
resolved-tokens))))))
(p/then #(.-allTokens ^js %)))))
(defn resolve-tokens-tree+
([tokens-tree get-token]
(resolve-tokens-tree+ tokens-tree get-token (StyleDictionary. default-config)))
([tokens-tree get-token style-dictionary]
(-> style-dictionary
(add-tokens tokens-tree)
(build-dictionary)
(p/then #(process-sd-tokens % get-token)))))
(defn sd-token-name [^js sd-token]
(.. sd-token -original -name))
(defn sd-token-uuid [^js sd-token]
(uuid (.-uuid (.-id ^js sd-token))))
(defn resolve-tokens+ [tokens]
(resolve-tokens-tree+ (ctob/tokens-tree tokens) #(get tokens (sd-token-name %))))
(defn resolve-tokens-interactive+
"Interactive check of resolving tokens.
Uses a ids map to backtrace the original token from the resolved StyleDictionary token.
This is necessary as the user might have removed/changed the token name but we still want to validate the value interactively."
[tokens]
(let [{:keys [tokens-tree ids]} (ctob/backtrace-tokens-tree tokens)]
(resolve-tokens-tree+ tokens-tree #(get ids (sd-token-uuid %)))))
(defn resolve-tokens-with-errors+ [tokens]
(resolve-tokens-tree+
(ctob/tokens-tree tokens)
#(get tokens (sd-token-name %))
(StyleDictionary. (assoc default-config :log {:verbosity "verbose"}))))
;; === Import
(defn reference-errors
"Extracts reference errors from StyleDictionary."
[err]
(let [[header-1 header-2 & errors] (str/split err "\n")]
(when (and
(= header-1 "Error: ")
(= header-2 "Reference Errors:"))
errors)))
(defn process-json-stream [data-stream]
(->> data-stream
(rx/map (fn [data]
(try
(t/decode-str data)
(catch js/Error e
(throw (wte/error-ex-info :error.import/json-parse-error data e))))))
(rx/map (fn [json-data]
(try
(ctob/decode-dtcg-json (ctob/ensure-tokens-lib nil) json-data)
(catch js/Error e
(throw (wte/error-ex-info :error.import/invalid-json-data json-data e))))))
(rx/mapcat (fn [tokens-lib]
(try
(-> (ctob/get-all-tokens tokens-lib)
(resolve-tokens-with-errors+)
(p/then (fn [_] tokens-lib))
(p/catch (fn [sd-error]
(let [reference-errors (reference-errors sd-error)
err (if reference-errors
(wte/error-ex-info :error.import/style-dictionary-reference-errors reference-errors sd-error)
(wte/error-ex-info :error.import/style-dictionary-unknown-error sd-error sd-error))]
(throw err)))))
(catch js/Error e
(p/rejected (wte/error-ex-info :error.import/style-dictionary-unknown-error "" e))))))))
;; === Errors
(defn humanize-errors [{:keys [errors value] :as _token}]
(->> (map (fn [err]
@ -71,51 +169,18 @@
errors)
(str/join "\n")))
(defn resolve-tokens+
[tokens & {:keys [names-map?] :as config}]
(let [{:keys [tree ids-map]} (wtt/token-names-tree-id-map tokens)]
(p/let [sd-tokens (resolve-sd-tokens+ tree)]
(let [resolved-tokens (reduce
(fn [acc ^js cur]
(let [{:keys [type] :as origin-token} (if names-map?
(get tokens (.. cur -original -name))
(get ids-map (uuid (.-uuid (.-id cur)))))
value (.-value cur)
token-or-err (case type
:color (if-let [tc (tinycolor/valid-color value)]
{:value value :unit (tinycolor/color-format tc)}
{:errors [(wte/error-with-value :error.token/invalid-color value)]})
(or (wtt/parse-token-value value)
(if-let [references (-> (ctob/find-token-value-references value)
(seq))]
{:errors [(wte/error-with-value :error.style-dictionary/missing-reference references)]
:references references}
{:errors [(wte/error-with-value :error.style-dictionary/invalid-token-value value)]})))
output-token (if (:errors token-or-err)
(merge origin-token token-or-err)
(assoc origin-token
:resolved-value (:value token-or-err)
:unit (:unit token-or-err)))]
(assoc acc (wtt/token-identifier output-token) output-token)))
{} sd-tokens)]
(l/debug :hint "Resolved tokens" :js/tokens resolved-tokens)
resolved-tokens))))
;; Hooks -----------------------------------------------------------------------
;; === Hooks
(defonce !tokens-cache (atom nil))
(defonce !theme-tokens-cache (atom nil))
(defn get-cached-tokens [tokens]
(get @!tokens-cache tokens tokens))
(defn use-resolved-tokens
"The StyleDictionary process function is async, so we can't use resolved values directly.
This hook will return the unresolved tokens as state until they are processed,
then the state will be updated with the resolved tokens."
[tokens & {:keys [cache-atom names-map?]
[tokens & {:keys [cache-atom interactive?]
:or {cache-atom !tokens-cache}
:as config}]
(let [tokens-state (mf/use-state (get @cache-atom tokens))]
@ -124,7 +189,7 @@
(fn []
(let [cached (get @cache-atom tokens)]
(cond
(nil? tokens) (if names-map? {} [])
(nil? tokens) nil
;; The tokens are already processing somewhere
(p/promise? cached) (-> cached
(p/then #(reset! tokens-state %))
@ -132,19 +197,19 @@
;; Get the cached entry
(some? cached) (reset! tokens-state cached)
;; No cached entry, start processing
:else (let [promise+ (resolve-tokens+ tokens config)]
:else (let [promise+ (if interactive?
(resolve-tokens-interactive+ tokens)
(resolve-tokens+ tokens))]
(swap! cache-atom assoc tokens promise+)
(p/then promise+ (fn [resolved-tokens]
(swap! cache-atom assoc tokens resolved-tokens)
(reset! tokens-state resolved-tokens))))))))
@tokens-state))
(defn use-resolved-workspace-tokens [& {:as config}]
(defn use-resolved-workspace-tokens []
(-> (mf/deref refs/workspace-selected-token-set-tokens)
(use-resolved-tokens config)))
(use-resolved-tokens)))
(defn use-active-theme-sets-tokens [& {:as config}]
(defn use-active-theme-sets-tokens []
(-> (mf/deref refs/workspace-active-theme-sets-tokens)
(use-resolved-tokens (merge {:cache-atom !theme-tokens-cache
:names-map? true}
config))))
(use-resolved-tokens {:cache-atom !theme-tokens-cache})))

View file

@ -125,7 +125,7 @@
(rx/from
(->
(wtts/get-active-theme-sets-tokens-names-map state)
(wtsd/resolve-tokens+ {:names-map? true})))
(wtsd/resolve-tokens+)))
(rx/mapcat
(fn [sd-tokens]
(let [undo-id (js/Symbol)]

View file

@ -24,7 +24,7 @@
(watch [_ state _]
(->> (rx/from (-> (get-in state [:workspace-data :tokens-lib])
(ctob/get-active-themes-set-tokens)
(sd/resolve-tokens+ {:names-map? true})))
(sd/resolve-tokens+)))
(rx/mapcat #(rx/of (end)))))))
(defn stop-on

View file

@ -2,9 +2,11 @@
(:require
[app.common.data :as d]
[app.main.ui.workspace.tokens.style-dictionary :as sd]
[app.main.ui.workspace.tokens.token :as wtt]
[beicon.v2.core :as rx]
[app.common.transit :as tr]
[cljs.test :as t :include-macros true]
[promesa.core :as p]))
[promesa.core :as p]
[app.common.types.tokens-lib :as ctob]))
(def border-radius-token
{:value "12px"
@ -19,6 +21,7 @@
(def tokens (d/ordered-map
(:name border-radius-token) border-radius-token
(:name reference-border-radius-token) reference-border-radius-token))
(t/deftest resolve-tokens-test
(t/async
done
@ -26,16 +29,93 @@
(-> (sd/resolve-tokens+ tokens)
(p/finally
(fn [resolved-tokens]
(let [expected-tokens {"borderRadius.sm"
(assoc border-radius-token
:resolved-value 12
:resolved-unit "px")
"borderRadius.md-with-dashes"
(assoc reference-border-radius-token
:resolved-value 24
:resolved-unit "px")}]
(t/is (= 12 (get-in resolved-tokens ["borderRadius.sm" :resolved-value])))
(t/is (= "px" (get-in resolved-tokens ["borderRadius.sm" :unit])))
(t/is (= 24 (get-in resolved-tokens ["borderRadius.md-with-dashes" :resolved-value])))
(t/is (= "px" (get-in resolved-tokens ["borderRadius.md-with-dashes" :unit])))
(done))))))))
(t/is (= 12 (get-in resolved-tokens ["borderRadius.sm" :resolved-value])))
(t/is (= "px" (get-in resolved-tokens ["borderRadius.sm" :unit])))
(t/is (= 24 (get-in resolved-tokens ["borderRadius.md-with-dashes" :resolved-value])))
(t/is (= "px" (get-in resolved-tokens ["borderRadius.md-with-dashes" :unit])))
(done)))))))
(t/deftest process-json-stream-test
(t/async
done
(t/testing "processes empty json string"
(let [json (-> {"core" {"color" {"$value" "red"
"$type" "color"}}}
(tr/encode-str {:type :json-verbose}))]
(->> (rx/of json)
(sd/process-json-stream)
(rx/subs! (fn [tokens-lib]
(t/is (instance? ctob/TokensLib tokens-lib))
(t/is (= "red" (-> (ctob/get-set tokens-lib "core")
(ctob/get-token "color")
(:value))))
(done))))))))
(t/deftest reference-errros-test
(t/testing "Extracts reference errors from StyleDictionary errors"
;; Using unicode for the white-space after "Error: " as some editors might remove it and its more visible
(t/is (=
["Some token references (2) could not be found."
""
"foo.value tries to reference missing, which is not defined."
"color.value tries to reference missing, which is not defined."]
(sd/reference-errors "Error:\u0020
Reference Errors:
Some token references (2) could not be found.
foo.value tries to reference missing, which is not defined.
color.value tries to reference missing, which is not defined.")))
(t/is (nil? (sd/reference-errors nil)))
(t/is (nil? (sd/reference-errors "none")))))
(t/deftest process-empty-json-stream-test
(t/async
done
(t/testing "processes empty json string"
(->> (rx/of "{}")
(sd/process-json-stream)
(rx/subs! (fn [tokens-lib]
(t/is (instance? ctob/TokensLib tokens-lib))
(done)))))))
(t/deftest process-invalid-json-stream-test
(t/async
done
(t/testing "fails on invalid json"
(->> (rx/of "{,}")
(sd/process-json-stream)
(rx/subs!
(fn []
(throw (js/Error. "Should be an error")))
(fn [err]
(t/is (= :error.import/json-parse-error (:error/code (ex-data err))))
(done)))))))
(t/deftest process-non-token-json-stream-test
(t/async
done
(t/testing "fails on non-token json"
(->> (rx/of "{\"foo\": \"bar\"}")
(sd/process-json-stream)
(rx/subs!
(fn []
(throw (js/Error. "Should be an error")))
(fn [err]
(t/is (= :error.import/invalid-json-data (:error/code (ex-data err))))
(done)))))))
(t/deftest process-missing-references-json-test
(t/async
done
(t/testing "fails on missing references in tokens"
(let [json (-> {"core" {"color" {"$value" "{missing}"
"$type" "color"}}}
(tr/encode-str {:type :json-verbose}))]
(->> (rx/of json)
(sd/process-json-stream)
(rx/subs!
(fn []
(throw (js/Error. "Should be an error")))
(fn [err]
(t/is (= :error.import/style-dictionary-reference-errors (:error/code (ex-data err))))
(done))))))))