refactor(cli): improved connector add command (#2764)

packages/cli/package.json

@@ -55,6 +55,7 @@
     "inquirer": "^8.2.2",
     "nanoid": "^4.0.0",
     "ora": "^6.1.2",
+    "p-limit": "^4.0.0",
     "p-retry": "^5.1.2",
     "roarr": "^7.11.0",
     "semver": "^7.3.8",

packages/cli/src/commands/connector/list.ts

@@ -1,16 +1,17 @@
 import chalk from 'chalk';
 import type { CommandModule } from 'yargs';
 
+import type { ConnectorPackage } from '../../utilities.js';
 import { getConnectorPackagesFrom, isOfficialConnector } from './utils.js';
 
-const logConnectorNames = (type: string, names: string[]) => {
-  if (names.length === 0) {
+const logConnectorNames = (type: string, packages: ConnectorPackage[]) => {
+  if (packages.length === 0) {
     return;
   }
 
   console.log();
   console.log(chalk.blue(type));
-  console.log(names.map((value) => '  ' + value).join('\n'));
+  console.log(packages.map(({ name }) => '  ' + name).join('\n'));
 };
 
 const list: CommandModule<{ path?: string }, { path?: string }> = {
@@ -18,9 +19,8 @@ const list: CommandModule<{ path?: string }, { path?: string }> = {
   describe: 'List added Logto connectors',
   handler: async ({ path: inputPath }) => {
     const packages = await getConnectorPackagesFrom(inputPath);
-    const packageNames = packages.map(({ name }) => name);
-    const officialPackages = packageNames.filter((name) => isOfficialConnector(name));
-    const thirdPartyPackages = packageNames.filter((name) => !isOfficialConnector(name));
+    const officialPackages = packages.filter(({ name }) => isOfficialConnector(name));
+    const thirdPartyPackages = packages.filter(({ name }) => !isOfficialConnector(name));
 
     logConnectorNames('official'.toUpperCase(), officialPackages);
     logConnectorNames('3rd-party'.toUpperCase(), thirdPartyPackages);

packages/cli/src/commands/connector/utils.ts

@@ -6,7 +6,9 @@ import { promisify } from 'util';
 
 import { assert, conditionalString } from '@silverhand/essentials';
 import chalk from 'chalk';
+import { got } from 'got';
 import inquirer from 'inquirer';
+import pLimit from 'p-limit';
 import pRetry from 'p-retry';
 import tar from 'tar';
 import { z } from 'zod';
@@ -116,6 +118,7 @@ export const addConnectors = async (instancePath: string, packageNames: string[]
 
   log.info('Fetch connector metadata');
 
+  const limit = pLimit(10);
   const results = await Promise.all(
     packageNames
       .map((name) => normalizePackageName(name))
@@ -130,7 +133,7 @@ export const addConnectors = async (instancePath: string, packageNames: string[]
             );
           }
 
-          const { filename, name } = result[0];
+          const { filename, name, version } = result[0];
           const escapedFilename = filename.replace(/\//g, '-').replace(/@/g, '');
           const tarPath = path.join(cwd, escapedFilename);
           const packageDirectory = path.join(cwd, name.replace(/\//g, '-'));
@@ -140,16 +143,18 @@ export const addConnectors = async (instancePath: string, packageNames: string[]
           await tar.extract({ cwd: packageDirectory, file: tarPath, strip: 1 });
           await fs.unlink(tarPath);
 
-          log.succeed(`Added ${chalk.green(name)}`);
+          log.succeed(`Added ${chalk.green(name)} v${version}`);
         };
 
-        try {
-          await pRetry(run, { retries: 2 });
-        } catch (error: unknown) {
-          console.warn(`[${packageName}]`, error);
+        return limit(async () => {
+          try {
+            await pRetry(run, { retries: 2 });
+          } catch (error: unknown) {
+            console.warn(`[${packageName}]`, error);
 
-          return packageName;
-        }
+            return packageName;
+          }
+        });
       })
   );
 
@@ -169,19 +174,57 @@ export const addConnectors = async (instancePath: string, packageNames: string[]
 
 const officialConnectorPrefix = '@logto/connector-';
 
-const fetchOfficialConnectorList = async () => {
-  const { stdout } = await execPromise(`npm search ${officialConnectorPrefix} --json`);
-  const packages = z
-    .object({ name: z.string() })
-    .transform(({ name }) => name)
-    .array()
-    .parse(JSON.parse(stdout));
+type PackageMeta = { name: string; scope: string; version: string };
 
-  return packages.filter((name) =>
-    ['mock', 'kit'].every(
-      (excluded) => !name.slice(officialConnectorPrefix.length).startsWith(excluded)
-    )
-  );
+const fetchOfficialConnectorList = async () => {
+  // See https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#get-v1search
+  type FetchResult = {
+    objects: Array<{
+      package: PackageMeta;
+      flags?: { unstable?: boolean };
+    }>;
+    total: number;
+  };
+
+  const fetchList = async (from = 0, size = 20) => {
+    const parameters = new URLSearchParams({
+      text: officialConnectorPrefix,
+      from: String(from),
+      size: String(size),
+    });
+
+    return got(
+      `https://registry.npmjs.org/-/v1/search?${parameters.toString()}`
+    ).json<FetchResult>();
+  };
+
+  const packages: PackageMeta[] = [];
+
+  // Disable lint rules for business need
+  // eslint-disable-next-line @silverhand/fp/no-let, @silverhand/fp/no-mutation
+  for (let page = 0; ; ++page) {
+    // eslint-disable-next-line no-await-in-loop
+    const { objects } = await fetchList(page * 20, 20);
+
+    // eslint-disable-next-line @silverhand/fp/no-mutating-methods
+    packages.push(
+      ...objects
+        .filter(
+          ({ package: { name, scope } }) =>
+            scope === 'logto' &&
+            ['mock', 'kit'].every(
+              (excluded) => !name.slice(officialConnectorPrefix.length).startsWith(excluded)
+            )
+        )
+        .map(({ package: data }) => data)
+    );
+
+    if (objects.length < 20) {
+      break;
+    }
+  }
+
+  return packages;
 };
 
 export const addOfficialConnectors = async (instancePath: string) => {
@@ -189,5 +232,11 @@ export const addOfficialConnectors = async (instancePath: string) => {
     text: 'Fetch official connector list',
     prefixText: chalk.blue('[info]'),
   });
-  await addConnectors(instancePath, packages);
+
+  log.info(`Found ${packages.length} official connectors`);
+
+  await addConnectors(
+    instancePath,
+    packages.map(({ name }) => name)
+  );
 };

pnpm-lock.yaml

@@ -48,6 +48,7 @@ importers:
       lint-staged: ^13.0.0
       nanoid: ^4.0.0
       ora: ^6.1.2
+      p-limit: ^4.0.0
       p-retry: ^5.1.2
       prettier: ^2.8.1
       rimraf: ^3.0.2
@@ -74,6 +75,7 @@ importers:
       inquirer: 8.2.2
       nanoid: 4.0.0
       ora: 6.1.2
+      p-limit: 4.0.0
       p-retry: 5.1.2
       roarr: 7.11.0
       semver: 7.3.8