Merge pull request #324 from tokens-studio/named-set-groups-2

Allow sets and set groups at the same level

common/src/app/common/files/changes.cljc

@@ -1049,13 +1049,10 @@
 (defmethod process-change :mod-token-set
   [data {:keys [name token-set]}]
   (update data :tokens-lib (fn [lib]
-                             (let [path-changed? (not= name (:name token-set))
-                                   lib' (-> lib
-                                            (ctob/ensure-tokens-lib)
-                                            (ctob/update-set name (fn [prev-set]
-                                                                    (merge prev-set (dissoc token-set :tokens)))))]
-                               (cond-> lib'
-                                 path-changed? (ctob/update-set-name name (:name token-set)))))))
+                             (-> lib
+                                 (ctob/ensure-tokens-lib)
+                                 (ctob/update-set name (fn [prev-set]
+                                                         (merge prev-set (dissoc token-set :tokens))))))))
 
 (defmethod process-change :move-token-set-before
   [data {:keys [set-name before-set-name]}]

common/src/app/common/types/tokens_lib.cljc

@@ -59,7 +59,7 @@
                           (join-path separator))))
 
 (defn get-path
-  "Get the groups part of the name as a vector. E.g. group.subgroup.name -> ['group' 'subrgoup']"
+  "Get the groups part of the name as a vector. E.g. group.subgroup.name -> ['group' 'subgroup']"
   [item separator]
   (dm/assert!
    "expected groupable item"
@@ -67,7 +67,7 @@
   (split-path (:name item) separator))
 
 (defn get-groups-str
-  "Get the groups part of the name. E.g. group.subgroup.name -> group.subrgoup"
+  "Get the groups part of the name. E.g. group.subgroup.name -> group.subgroup"
   [item separator]
   (-> (get-path item separator)
       (butlast)
@@ -177,13 +177,35 @@
 
 ;; === Token Set
 
+(def set-prefix "S-")
+
+(def set-group-prefix "G-")
+
 (def set-separator "/")
 
-(defn get-token-set-path [path]
-  (get-path path set-separator))
+(defn add-set-prefix [set-name]
+  (str set-prefix set-name))
 
-(defn split-token-set-path [path]
-  (split-path path set-separator))
+(defn add-set-group-prefix [group-path]
+  (str set-group-prefix group-path))
+
+(defn add-token-set-paths-prefix
+  "Returns token-set paths with prefixes to differentiate between sets and set-groups.
+
+  Sets will be prefixed with `set-prefix` (S-).
+  Set groups will be prefixed with `set-group-prefix` (G-)."
+  [paths]
+  (let [set-path (mapv add-set-group-prefix (butlast paths))
+        set-name (add-set-prefix (last paths))]
+    (conj set-path set-name)))
+
+(defn split-token-set-name [token-set-name]
+  (-> (split-path token-set-name set-separator)
+      (add-token-set-paths-prefix)))
+
+(defn get-token-set-path [token-set]
+  (let [path (get-path token-set set-separator)]
+    (add-token-set-paths-prefix path)))
 
 (defn tokens-tree
   "Convert tokens into a nested tree with their `:name` as the path.
@@ -300,6 +322,7 @@
 
 (defprotocol ITokenSets
   (add-set [_ token-set] "add a set to the library, at the end")
+  (add-sets [_ token-set] "add a collection of sets to the library, at the end")
   (update-set [_ set-name f] "modify a set in the ilbrary")
   (delete-set [_ set-name] "delete a set in the library")
   (move-set-before [_ set-name before-set-name] "move a set with `set-name` before a set with `before-set-name` in the library.
@@ -504,6 +527,8 @@ When `before-set-name` is nil, move set to bottom")
 
 ;; === Tokens Lib
 
+(declare make-tokens-lib)
+
 (defprotocol ITokensLib
   "A library of tokens, sets and themes."
   (add-token-in-set [_ set-name token] "add token to a set")
@@ -512,7 +537,6 @@ When `before-set-name` is nil, move set to bottom")
   (toggle-set-in-theme [_ group-name theme-name set-name] "toggle a set used / not used in a theme")
   (get-active-themes-set-names [_] "set of set names that are active in the the active themes")
   (get-active-themes-set-tokens [_] "set of set names that are active in the the active themes")
-  (update-set-name [_ old-set-name new-set-name] "updates set name in themes")
   (encode-dtcg [_] "Encodes library to a dtcg compatible json string")
   (decode-dtcg-json [_ parsed-json] "Decodes parsed json containing tokens and converts to library")
   (get-all-tokens [_] "all tokens in the lib")
@@ -543,13 +567,19 @@ When `before-set-name` is nil, move set to bottom")
                   themes
                   active-themes)))
 
+  (add-sets [this token-sets]
+    (reduce
+       (fn [lib set]
+         (add-set lib set))
+       this token-sets))
+
   (update-set [this set-name f]
-    (let [path (split-token-set-path set-name)
+    (let [path (split-token-set-name set-name)
           set  (get-in sets path)]
       (if set
         (let [set'  (-> (make-token-set (f set))
                         (assoc :modified-at (dt/now)))
-              path' (get-path set' "/")]
+              path' (get-token-set-path set')]
           (check-token-set! set')
           (TokensLib. (if (= (:name set) (:name set'))
                         (d/oassoc-in sets path set')
@@ -561,7 +591,7 @@ When `before-set-name` is nil, move set to bottom")
         this)))
 
   (delete-set [_ set-name]
-    (let [path (split-token-set-path set-name)]
+    (let [path (split-token-set-name set-name)]
       (TokensLib. (d/dissoc-in sets path)
                   (walk/postwalk
                    (fn [form]
@@ -573,10 +603,10 @@ When `before-set-name` is nil, move set to bottom")
 
   ;; TODO Handle groups and nesting
   (move-set-before [this set-name before-set-name]
-    (let [source-path (split-token-set-path set-name)
+    (let [source-path (split-token-set-name set-name)
           token-set (-> (get-set this set-name)
                         (assoc :modified-at (dt/now)))
-          target-path (split-token-set-path before-set-name)]
+          target-path (split-token-set-name before-set-name)]
       (if before-set-name
         (TokensLib. (d/oassoc-in-before sets target-path source-path token-set)
                     themes
@@ -601,7 +631,7 @@ When `before-set-name` is nil, move set to bottom")
     (count (get-sets this)))
 
   (get-set [_ set-name]
-    (let [path (split-path set-name "/")]
+    (let [path (split-token-set-name set-name)]
       (get-in sets path)))
 
   (get-neighbor-set-name [this set-name index-offset]
@@ -708,27 +738,13 @@ When `before-set-name` is nil, move set to bottom")
   ITokensLib
   (add-token-in-set [this set-name token]
     (dm/assert! "expected valid token instance" (check-token! token))
-    (if (contains? sets set-name)
-      (TokensLib. (update sets set-name add-token token)
-                  themes
-                  active-themes)
-      this))
+    (update-set this set-name #(add-token % token)))
 
   (update-token-in-set [this set-name token-name f]
-    (if (contains? sets set-name)
-      (TokensLib. (update sets set-name
-                          #(update-token % token-name f))
-                  themes
-                  active-themes)
-      this))
+    (update-set this set-name #(update-token % token-name f)))
 
   (delete-token-from-set [this set-name token-name]
-    (if (contains? sets set-name)
-      (TokensLib. (update sets set-name
-                          #(delete-token % token-name))
-                  themes
-                  active-themes)
-      this))
+    (update-set this set-name #(delete-token % token-name)))
 
   (toggle-set-in-theme [this theme-group theme-name set-name]
     (if-let [_theme (get-in themes theme-group theme-name)]
@@ -756,36 +772,24 @@ When `before-set-name` is nil, move set to bottom")
           tokens (order-theme-set theme)))
        (d/ordered-map) active-themes)))
 
-  ;; TODO Move to `update-set`
-  (update-set-name [_ old-set-name new-set-name]
-    (TokensLib. sets
-                (walk/postwalk
-                 (fn [form]
-                   (if (instance? TokenTheme form)
-                     (-> form
-                         (update :sets disj old-set-name)
-                         (update :sets conj new-set-name))
-                     form))
-                 themes)
-                active-themes))
-
   (encode-dtcg [_]
-    (into {} (map (fn [[k v]]
-                    [k (get-dtcg-tokens-tree v)])
-                  sets)))
+    (into {} (comp
+              (filter (partial instance? TokenSet))
+              (map (fn [token-set]
+                     [(:name token-set) (get-dtcg-tokens-tree token-set)])))
+          (tree-seq d/ordered-map? vals sets)))
 
   (decode-dtcg-json [_ parsed-json]
-    (let [token-sets (into (d/ordered-map)
-                           (map (fn [[set-name tokens]]
-                                  [set-name (make-token-set
-                                             :name set-name
-                                             :tokens (flatten-nested-tokens-json tokens ""))]))
-                           (-> parsed-json
-                               ;; tokens-studio/plugin will add these meta properties, remove them for now
-                               (dissoc "$themes" "$metadata")))]
-      (TokensLib. token-sets
-                  themes
-                  active-themes)))
+    (let [;; tokens-studio/plugin will add these meta properties, remove them for now
+          sets-data (dissoc parsed-json "$themes" "$metadata")
+          lib (make-tokens-lib)
+          lib' (reduce
+                (fn [lib [set-name tokens]]
+                  (add-set lib (make-token-set
+                                :name set-name
+                                :tokens (flatten-nested-tokens-json tokens ""))))
+                lib sets-data)]
+      lib'))
 
   (get-all-tokens [this]
     (reduce
@@ -894,7 +898,14 @@ When `before-set-name` is nil, move set to bottom")
 
     {:name "penpot/tokens-lib/v1"
      :rfn (fn [r]
-            (let [sets          (fres/read-object! r)
+            (let [;; Migrate sets tree without prefix to new format
+                  prev-sets (->> (fres/read-object! r)
+                                 (tree-seq d/ordered-map? vals)
+                                 (filter (partial instance? TokenSet)))
+                  sets (-> (make-tokens-lib)
+                           (add-sets prev-sets)
+                           (deref)
+                           :sets)
                   _set-groups   (fres/read-object! r)
                   themes        (fres/read-object! r)
                   active-themes (fres/read-object! r)]

common/test/common_tests/types/tokens_lib_test.cljc

@@ -748,31 +748,31 @@
         (t/is (= (:name (nth sets-list 3)) "group1/subgroup11/token-set-4"))
         (t/is (= (:name (nth sets-list 4)) "group2/token-set-5"))
 
-        (t/is (= (first node-set1) "token-set-1"))
+        (t/is (= (first node-set1) "S-token-set-1"))
         (t/is (= (ctob/group? (second node-set1)) false))
         (t/is (= (:name (second node-set1)) "token-set-1"))
 
-        (t/is (= (first node-group1) "group1"))
+        (t/is (= (first node-group1) "G-group1"))
         (t/is (= (ctob/group? (second node-group1)) true))
         (t/is (= (count (second node-group1)) 3))
 
-        (t/is (= (first node-set2) "token-set-2"))
+        (t/is (= (first node-set2) "S-token-set-2"))
         (t/is (= (ctob/group? (second node-set2)) false))
         (t/is (= (:name (second node-set2)) "group1/token-set-2"))
 
-        (t/is (= (first node-set3) "token-set-3"))
+        (t/is (= (first node-set3) "S-token-set-3"))
         (t/is (= (ctob/group? (second node-set3)) false))
         (t/is (= (:name (second node-set3)) "group1/token-set-3"))
 
-        (t/is (= (first node-subgroup11) "subgroup11"))
+        (t/is (= (first node-subgroup11) "G-subgroup11"))
         (t/is (= (ctob/group? (second node-subgroup11)) true))
         (t/is (= (count (second node-subgroup11)) 1))
 
-        (t/is (= (first node-set4) "token-set-4"))
+        (t/is (= (first node-set4) "S-token-set-4"))
         (t/is (= (ctob/group? (second node-set4)) false))
         (t/is (= (:name (second node-set4)) "group1/subgroup11/token-set-4"))
 
-        (t/is (= (first node-set5) "token-set-5"))
+        (t/is (= (first node-set5) "S-token-set-5"))
         (t/is (= (ctob/group? (second node-set5)) false))
         (t/is (= (:name (second node-set5)) "group2/token-set-5"))))
 
@@ -791,13 +791,13 @@
 
             sets-tree   (ctob/get-set-tree tokens-lib)
             sets-tree'  (ctob/get-set-tree tokens-lib')
-            group1'     (get sets-tree' "group1")
-            token-set   (get-in sets-tree ["group1" "token-set-2"])
-            token-set'  (get-in sets-tree' ["group1" "token-set-2"])]
+            group1'     (get sets-tree' "G-group1")
+            token-set   (get-in sets-tree ["G-group1" "S-token-set-2"])
+            token-set'  (get-in sets-tree' ["G-group1" "S-token-set-2"])]
 
         (t/is (= (ctob/set-count tokens-lib') 5))
         (t/is (= (count group1') 3))
-        (t/is (= (d/index-of (keys group1') "token-set-2") 0))
+        (t/is (= (d/index-of (keys group1') "S-token-set-2") 0))
         (t/is (= (:name token-set') "group1/token-set-2"))
         (t/is (= (:description token-set') "some description"))
         (t/is (dt/is-after? (:modified-at token-set') (:modified-at token-set)))))
@@ -818,16 +818,18 @@
 
             sets-tree   (ctob/get-set-tree tokens-lib)
             sets-tree'  (ctob/get-set-tree tokens-lib')
-            group1'     (get sets-tree' "group1")
-            token-set   (get-in sets-tree ["group1" "token-set-2"])
-            token-set'  (get-in sets-tree' ["group1" "updated-name"])]
+            group1'     (get sets-tree' "G-group1")
+            token-set   (get-in sets-tree ["G-group1" "S-token-set-2"])
+            token-set'  (get-in sets-tree' ["G-group1" "S-updated-name"])]
 
         (t/is (= (ctob/set-count tokens-lib') 5))
         (t/is (= (count group1') 3))
-        (t/is (= (d/index-of (keys group1') "updated-name") 0))
+        (t/is (= (d/index-of (keys group1') "S-updated-name") 0))
         (t/is (= (:name token-set') "group1/updated-name"))
         (t/is (= (:description token-set') nil))
-        (t/is (dt/is-after? (:modified-at token-set') (:modified-at token-set)))))
+        (t/is (dt/is-after? (:modified-at token-set') (:modified-at token-set)))
+        sets-tree'))
+
 
     (t/deftest move-set-of-group
       (let [tokens-lib  (-> (ctob/make-tokens-lib)
@@ -845,15 +847,15 @@
 
             sets-tree   (ctob/get-set-tree tokens-lib)
             sets-tree'  (ctob/get-set-tree tokens-lib')
-            group1'     (get sets-tree' "group1")
-            group2'     (get sets-tree' "group2")
-            token-set   (get-in sets-tree ["group1" "token-set-2"])
-            token-set'  (get-in sets-tree' ["group2" "updated-name"])]
+            group1'     (get sets-tree' "G-group1")
+            group2'     (get sets-tree' "G-group2")
+            token-set   (get-in sets-tree ["G-group1" "S-token-set-2"])
+            token-set'  (get-in sets-tree' ["G-group2" "S-updated-name"])]
 
         (t/is (= (ctob/set-count tokens-lib') 4))
         (t/is (= (count group1') 2))
         (t/is (= (count group2') 1))
-        (t/is (= (d/index-of (keys group2') "updated-name") 0))
+        (t/is (nil? (get group1' "S-updated-name")))
         (t/is (= (:name token-set') "group2/updated-name"))
         (t/is (= (:description token-set') nil))
         (t/is (dt/is-after? (:modified-at token-set') (:modified-at token-set)))))
@@ -1129,4 +1131,3 @@
              (t/is (= @with-prev-tokens-lib @tokens-lib)))
            (t/testing "fresh tokens library is also equal"
              (= @with-empty-tokens-lib @tokens-lib)))))))
-

frontend/src/app/main/ui/workspace/tokens/sets.cljs

@@ -200,13 +200,9 @@
            context]
     :as _props}]
   (let [{:keys [editing? new? on-edit on-create on-reset] :as ctx} (or context (sets-context/use-context))
-        avoid-token-set-grouping #(str/replace % "/" "-")
         submit-token
         #(do
-           ;; TODO: We don't support set grouping for now so we rename sets for now
-           (when (str/includes? (:name %) "/")
-             (warn-on-try-create-token-set-group!))
-           (on-create-token-set (update % :name avoid-token-set-grouping))
+           (on-create-token-set %)
            (on-reset))]
     [:ul {:class (stl/css :sets-list)}
      (if (and
@@ -218,10 +214,7 @@
          (when token-set
            (let [update-token
                  #(do
-                  ;; TODO: We don't support set grouping for now so we rename sets for now
-                    (when (str/includes? (:name %) "/")
-                      (warn-on-try-create-token-set-group!))
-                    (on-update-token-set (avoid-token-set-grouping (:name token-set)) (update % :name avoid-token-set-grouping))
+                    (on-update-token-set (:name token-set) %)
                     (on-reset))]
              [:& sets-tree
               {:key (:name token-set)

frontend/src/app/main/ui/workspace/tokens/style_dictionary.cljs

@@ -182,8 +182,7 @@
   (->> data-stream
        (rx/map (fn [data]
                  (try
-                   (-> (str/replace data "/" "-") ;; TODO Remove when token groups work
-                       (t/decode-str))
+                   (t/decode-str data)
                    (catch js/Error e
                      (throw (wte/error-ex-info :error.import/json-parse-error data e))))))
        (rx/map (fn [json-data]