Attempt to use Queues

package-lock.json

@@ -12,6 +12,7 @@
         "@create-figma-plugin/ui": "^3.2",
         "base64-js": "^1.5",
         "classnames": "^2.5",
+        "fastq": "^1.17.1",
         "lru-cache": "^10.2",
         "preact": "^10.22",
         "react-hook-form": "^7.51",
@@ -4821,7 +4822,6 @@
       "version": "1.17.1",
       "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz",
       "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==",
-      "dev": true,
       "dependencies": {
         "reusify": "^1.0.4"
       }
@@ -7129,7 +7129,6 @@
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz",
       "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==",
-      "dev": true,
       "engines": {
         "iojs": ">=1.0.0",
         "node": ">=0.10.0"

package.json

@@ -25,6 +25,7 @@
     "@create-figma-plugin/ui": "^3.2",
     "base64-js": "^1.5",
     "classnames": "^2.5",
+    "fastq": "^1.17.1",
     "lru-cache": "^10.2",
     "preact": "^10.22",
     "react-hook-form": "^7.51",

plugin-src/Queue.ts

@@ -0,0 +1,30 @@
+import * as fastq from 'fastq';
+import type { queueAsPromised } from 'fastq';
+
+import { transformSceneNode } from '@plugin/transformers';
+
+import { PenpotNode } from '@ui/types';
+
+class Queue<T, R> {
+  private queue: queueAsPromised<T, R>;
+
+  constructor(worker: fastq.asyncWorker<unknown, T, R>, concurrency: number) {
+    this.queue = fastq.promise(worker, concurrency);
+  }
+
+  public enqueue(task: T): Promise<R> {
+    return this.queue.push(task);
+  }
+
+  public async waitIdle() {
+    await this.queue.drain();
+  }
+}
+
+export const nodeQueue = new Queue(
+  async ([sceneNode, position]: [SceneNode, number]): Promise<[PenpotNode | undefined, number]> => [
+    await transformSceneNode(sceneNode),
+    position
+  ],
+  1
+);

plugin-src/RemoteComponentLibrary.ts

@@ -1,33 +1,27 @@
+import { PenpotNode } from '@ui/types';
+
 class RemoteComponentsLibrary {
-  private components: Record<string, ComponentNode | ComponentSetNode> = {};
+  private components: PenpotNode[] = [];
-  private queue: string[] = [];
+  private registry: Record<string, null> = {};
 
-  public register(id: string, component: ComponentNode | ComponentSetNode) {
+  public add(component: PenpotNode) {
-    if (!Object.prototype.hasOwnProperty.call(this.components, id)) {
+    this.components.push(component);
-      this.queue.push(id);
   }
 
-    this.components[id] = component;
+  public register(id: string) {
+    this.registry[id] = null;
   }
 
-  public get(id: string): ComponentNode | ComponentSetNode | undefined {
+  public has(id: string): boolean {
-    return this.components[id];
+    return this.registry[id] === null;
-  }
-
-  public next(): ComponentNode | ComponentSetNode {
-    const lastKey = this.queue.pop();
-
-    if (!lastKey) throw new Error('No components to pop');
-
-    return this.components[lastKey];
-  }
-
-  public remaining(): number {
-    return this.queue.length;
   }
 
   public total(): number {
-    return Object.keys(this.components).length;
+    return this.components.length;
+  }
+
+  public all(): PenpotNode[] {
+    return this.components;
   }
 }
 

plugin-src/transformers/transformDocumentNode.ts

@@ -1,5 +1,6 @@
 import { componentsLibrary } from '@plugin/ComponentLibrary';
 import { imagesLibrary } from '@plugin/ImageLibrary';
+import { nodeQueue } from '@plugin/Queue';
 import { remoteComponentLibrary } from '@plugin/RemoteComponentLibrary';
 import { translateRemoteChildren } from '@plugin/translators';
 import { sleep } from '@plugin/utils';
@@ -75,10 +76,12 @@ const processPages = async (node: DocumentNode): Promise<PenpotPage[]> => {
 export const transformDocumentNode = async (node: DocumentNode): Promise<PenpotDocument> => {
   const children = await processPages(node);
 
-  if (remoteComponentLibrary.remaining() > 0) {
+  await nodeQueue.waitIdle();
+
+  if (remoteComponentLibrary.total() > 0) {
     children.push({
       name: 'External Components',
-      children: translateRemoteChildren()
+      children: remoteComponentLibrary.all()
     });
   }
 

plugin-src/transformers/transformInstanceNode.ts

@@ -1,4 +1,5 @@
 import { overridesLibrary } from '@plugin/OverridesLibrary';
+import { nodeQueue } from '@plugin/Queue';
 import { remoteComponentLibrary } from '@plugin/RemoteComponentLibrary';
 import {
   transformAutoLayout,
@@ -78,11 +79,15 @@ const getPrimaryComponent = (mainComponent: ComponentNode): ComponentNode | Comp
 };
 
 const registerExternalComponents = (primaryComponent: ComponentNode | ComponentSetNode): void => {
-  if (remoteComponentLibrary.get(primaryComponent.id) !== undefined) {
+  if (remoteComponentLibrary.has(primaryComponent.id)) {
     return;
   }
 
-  remoteComponentLibrary.register(primaryComponent.id, primaryComponent);
+  remoteComponentLibrary.register(primaryComponent.id);
+
+  nodeQueue.enqueue([primaryComponent, 0]).then(([penpotNode, _]) => {
+    if (penpotNode) remoteComponentLibrary.add(penpotNode);
+  });
 };
 
 const getComponentTextPropertyOverrides = (

plugin-src/translators/translateChildren.ts

@@ -1,8 +1,7 @@
-import { remoteComponentLibrary } from '@plugin/RemoteComponentLibrary';
+import { nodeQueue } from '@plugin/Queue';
-import { transformGroupNodeLike, transformSceneNode } from '@plugin/transformers';
+import { transformGroupNodeLike } from '@plugin/transformers';
 import { transformMaskFigmaIds } from '@plugin/transformers/partials';
 
-import { GroupShape } from '@ui/lib/types/shapes/groupShape';
 import { PenpotNode } from '@ui/types';
 
 /**
@@ -17,7 +16,7 @@ import { PenpotNode } from '@ui/types';
 export const translateMaskChildren = (
   children: readonly SceneNode[],
   maskIndex: number
-): Promise<PenpotNode | undefined>[] => {
+): PenpotNode[] => {
   const maskChild = children[maskIndex];
 
   const unmaskedChildren = translateChildren(children.slice(0, maskIndex));
@@ -38,50 +37,23 @@ export const translateMaskChildren = (
     return [...unmaskedChildren, ...maskedChildren];
   }
 
-  const maskGroup = Promise.resolve<GroupShape>({
+  const maskGroup = {
     ...transformMaskFigmaIds(maskChild),
     ...transformGroupNodeLike(maskChild),
     children: maskedChildren,
     maskedGroup: true
-  });
+  };
 
   return [...unmaskedChildren, maskGroup];
 };
 
-export const translateChildren = (
+export const translateChildren = (children: readonly SceneNode[]): PenpotNode[] => {
-  children: readonly SceneNode[]
+  const transformedChildren: PenpotNode[] = [];
-): Promise<PenpotNode | undefined>[] => {
+  let count = 0;
-  const transformedChildren: Promise<PenpotNode | undefined>[] = [];
 
   for (const child of children) {
-    transformedChildren.push(transformSceneNode(child));
+    nodeQueue.enqueue([child, count++]).then(([penpotNode, position]) => {
-  }
+      if (penpotNode) transformedChildren[position] = penpotNode;
-
-  return transformedChildren;
-};
-
-export const translateRemoteChildren = (): Promise<PenpotNode | undefined>[] => {
-  const transformedChildren: Promise<PenpotNode | undefined>[] = [];
-  let currentRemote = 1;
-
-  figma.ui.postMessage({
-    type: 'PROGRESS_STEP',
-    data: 'remote'
-  });
-
-  while (remoteComponentLibrary.remaining() > 0) {
-    figma.ui.postMessage({
-      type: 'PROGRESS_TOTAL_ITEMS',
-      data: remoteComponentLibrary.total()
-    });
-
-    const child = remoteComponentLibrary.next();
-
-    transformedChildren.push(transformSceneNode(child));
-
-    figma.ui.postMessage({
-      type: 'PROGRESS_PROCESSED_ITEMS',
-      data: currentRemote++
     });
   }
 

ui-src/lib/types/utils/children.ts

@@ -1,3 +1,3 @@
 import { PenpotNode } from '@ui/types';
 
-export type Children = { children: Promise<PenpotNode | undefined>[] | (PenpotNode | undefined)[] };
+export type Children = { children?: PenpotNode[] };