Merge pull request #2121 from logto-io/gao-remove-connector-command-in-core

refactor: remove connector commands in core

packages/cli/package.json

@@ -41,7 +41,6 @@
     "chalk": "^4.1.2",
     "decamelize": "^5.0.0",
     "dotenv": "^16.0.0",
-    "find-up": "^5.0.0",
     "fs-extra": "^10.1.0",
     "got": "^11.8.2",
     "hpagent": "^1.0.0",

packages/cli/src/commands/connector/utils.ts

@@ -12,6 +12,7 @@ import pRetry from 'p-retry';
 import tar from 'tar';
 import { z } from 'zod';
 
+import { connectorDirectory } from '../../constants';
 import { log } from '../../utilities';
 import { defaultPath } from '../install/utils';
 
@@ -86,7 +87,7 @@ export const normalizePackageName = (name: string) =>
     .join('/');
 
 export const addConnectors = async (instancePath: string, packageNames: string[]) => {
-  const cwd = path.join(instancePath, coreDirectory, 'connectors');
+  const cwd = path.join(instancePath, coreDirectory, connectorDirectory);
 
   if (!existsSync(cwd)) {
     await mkdir(cwd);

packages/cli/src/commands/database/alteration.ts

@@ -1,9 +1,9 @@
 import path from 'path';
 
 import { AlterationScript } from '@logto/schemas/lib/types/alteration';
-import { conditional, conditionalString } from '@silverhand/essentials';
+import { findPackage } from '@logto/shared';
+import { conditionalString } from '@silverhand/essentials';
 import chalk from 'chalk';
-import findUp, { exists } from 'find-up';
 import { copy, existsSync, remove, readdir } from 'fs-extra';
 import { DatabasePool } from 'slonik';
 import { CommandModule } from 'yargs';
@@ -45,18 +45,10 @@ export const getAlterationFiles = async (): Promise<AlterationFile[]> => {
    * since they need a proper context that includes required dependencies (such as slonik) in `node_modules/`.
    * While the original `@logto/schemas` may remove them in production.
    */
-  const packageDirectory = await findUp(
-    async (directory) => {
-      const hasPackageJson = await exists(path.join(directory, 'package.json'));
-
-      return conditional(hasPackageJson && directory);
-    },
-    {
-      // Until we migrate to ESM
-      // eslint-disable-next-line unicorn/prefer-module
-      cwd: __dirname,
-      type: 'directory',
-    }
+  const packageDirectory = await findPackage(
+    // Until we migrate to ESM
+    // eslint-disable-next-line unicorn/prefer-module
+    __dirname
   );
 
   const localAlterationDirectory = path.resolve(

packages/cli/src/constants.ts

@@ -0,0 +1 @@
+export const connectorDirectory = 'connectors';

packages/core/package.json

@@ -14,8 +14,8 @@
     "lint:report": "pnpm lint --format json --output-file report.json",
     "dev": "rm -rf build/ && pnpm run copyfiles && nodemon",
     "start": "NODE_ENV=production node build/index.js",
-    "add-connector": "node build/cli/add-connector.js",
-    "add-official-connectors": "node build/cli/add-official-connectors.js",
+    "add-connector": "logto connector add -p ../../",
+    "add-official-connectors": "logto connector add -o -p ../../",
     "alteration": "logto db alt",
     "cli": "logto",
     "test": "jest",
@@ -45,7 +45,6 @@
     "hash-wasm": "^4.9.0",
     "i18next": "^21.8.16",
     "iconv-lite": "0.6.3",
-    "inquirer": "^8.2.2",
     "jose": "^4.0.0",
     "js-yaml": "^4.1.0",
     "koa": "^2.13.1",
@@ -62,14 +61,12 @@
     "oidc-provider": "^7.11.3",
     "p-retry": "^4.6.1",
     "query-string": "^7.0.1",
-    "rimraf": "^3.0.2",
     "roarr": "^7.11.0",
     "slonik": "^30.0.0",
     "slonik-interceptor-preset": "^1.2.10",
     "slonik-sql-tag-raw": "^1.1.4",
     "snake-case": "^3.0.4",
     "snakecase-keys": "^5.1.0",
-    "tar": "^6.1.11",
     "zod": "^3.18.0"
   },
   "devDependencies": {
@@ -81,7 +78,6 @@
     "@types/etag": "^1.8.1",
     "@types/fs-extra": "^9.0.13",
     "@types/http-errors": "^1.8.2",
-    "@types/inquirer": "^8.2.1",
     "@types/jest": "^28.1.6",
     "@types/js-yaml": "^4.0.5",
     "@types/koa": "^2.13.3",
@@ -92,9 +88,7 @@
     "@types/lodash.pick": "^4.4.6",
     "@types/node": "^16.3.1",
     "@types/oidc-provider": "^7.11.1",
-    "@types/rimraf": "^3.0.2",
     "@types/supertest": "^2.0.11",
-    "@types/tar": "^6.1.2",
     "copyfiles": "^2.4.1",
     "eslint": "^8.21.0",
     "http-errors": "^1.6.3",

packages/core/src/cli/add-connector.ts

@@ -1,24 +0,0 @@
-import 'module-alias/register';
-import { getEnv } from '@silverhand/essentials';
-import chalk from 'chalk';
-
-import { addConnector } from '@/connectors/add-connectors';
-import { defaultConnectorDirectory } from '@/env-set';
-import { configDotEnv } from '@/env-set/dot-env';
-
-configDotEnv();
-
-const addConnectorCli = async (packageName: string) => {
-  const connectorDirectory = getEnv('CONNECTOR_DIRECTORY', defaultConnectorDirectory);
-
-  await addConnector(packageName, connectorDirectory);
-  console.log(`${chalk.blue(packageName)} added successfully.`);
-};
-
-const packageName = process.argv[2];
-
-if (!packageName) {
-  throw new Error('Please provide a package name');
-}
-
-void addConnectorCli(packageName);

packages/core/src/cli/add-official-connectors.ts

@@ -1,15 +0,0 @@
-import 'module-alias/register';
-import { getEnv } from '@silverhand/essentials';
-
-import { addOfficialConnectors } from '@/connectors/add-connectors';
-import { defaultConnectorDirectory } from '@/env-set';
-import { configDotEnv } from '@/env-set/dot-env';
-
-configDotEnv();
-
-const addOfficialConnectorsCli = async () => {
-  const connectorDirectory = getEnv('CONNECTOR_DIRECTORY', defaultConnectorDirectory);
-  await addOfficialConnectors(connectorDirectory);
-};
-
-void addOfficialConnectorsCli();

packages/core/src/connectors/add-connectors.ts

@@ -1,67 +0,0 @@
-import { exec } from 'child_process';
-import { existsSync } from 'fs';
-import { mkdir, rename, unlink } from 'fs/promises';
-import path from 'path';
-import { promisify } from 'util';
-
-import chalk from 'chalk';
-import rimraf from 'rimraf';
-import tar from 'tar';
-import { z } from 'zod';
-
-import { npmPackResultGuard } from './types';
-
-const execPromise = promisify(exec);
-const npmConnectorFilter = '@logto/connector-';
-const excludedPackages = new Set([
-  '@logto/connector-kit',
-  '@logto/connector-mock-sms',
-  '@logto/connector-mock-social',
-  '@logto/connector-mock-email',
-]);
-
-const fetchOfficialConnectorList = async () => {
-  const { stdout } = await execPromise(`npm search ${npmConnectorFilter} --json`);
-  const packages = z.object({ name: z.string() }).array().parse(JSON.parse(stdout));
-
-  return packages.filter(({ name }) => !excludedPackages.has(name));
-};
-
-export const addConnector = async (packageName: string, cwd: string) => {
-  if (!existsSync(cwd)) {
-    await mkdir(cwd);
-  }
-  const { stdout } = await execPromise(`npm pack ${packageName} --json`, { cwd });
-  const result = npmPackResultGuard.parse(JSON.parse(stdout));
-
-  if (!result[0]) {
-    throw new Error(`Failed to download package: ${packageName}`);
-  }
-
-  const { filename, name } = result[0];
-  const escapedFilename = filename.replace(/\//g, '-').replace(/@/g, '');
-  const filePath = path.join(cwd, escapedFilename);
-  await tar.extract({ cwd, file: filePath });
-  await unlink(filePath);
-
-  const packageFolder = path.join(cwd, name.replace(/\//g, '-').replace(/@/g, ''));
-  await promisify(rimraf)(packageFolder);
-
-  await rename(path.join(cwd, 'package'), packageFolder);
-};
-
-export const addOfficialConnectors = async (directory: string) => {
-  console.log(`${chalk.blue('[add-connectors]')} Fetching official connectors list`);
-  const packages = await fetchOfficialConnectorList();
-
-  // The await inside the loop is intended for better debugging experience and rate limitation.
-  for (const [index, { name }] of packages.entries()) {
-    console.log(
-      `${chalk.blue('[add-connectors]')} ${index + 1}/${
-        packages.length
-      } Adding connector package: ${name}`
-    );
-    // eslint-disable-next-line no-await-in-loop
-    await addConnector(name, directory);
-  }
-};

packages/core/src/connectors/index.ts

@@ -2,10 +2,11 @@ import { existsSync } from 'fs';
 import { readdir } from 'fs/promises';
 import path from 'path';
 
+import { connectorDirectory } from '@logto/cli/lib/constants';
 import { AllConnector, CreateConnector, validateConfig } from '@logto/connector-kit';
+import { findPackage } from '@logto/shared';
 import chalk from 'chalk';
 
-import envSet from '@/env-set';
 import RequestError from '@/errors/RequestError';
 import { findAllConnectors, insertConnector } from '@/queries/connector';
 
@@ -21,20 +22,21 @@ const loadConnectors = async () => {
     return cachedConnectors;
   }
 
-  const {
-    values: { connectorDirectory },
-  } = envSet;
+  // Until we migrate to ESM
+  // eslint-disable-next-line unicorn/prefer-module
+  const coreDirectory = await findPackage(__dirname);
+  const directory = coreDirectory && path.join(coreDirectory, connectorDirectory);
 
-  if (!existsSync(connectorDirectory)) {
+  if (!directory || !existsSync(directory)) {
     return [];
   }
 
-  const connectorFolders = await readdir(connectorDirectory);
+  const connectorFolders = await readdir(directory);
 
   const connectors = await Promise.all(
     connectorFolders.map(async (folder) => {
       try {
-        const packagePath = path.join(connectorDirectory, folder);
+        const packagePath = path.join(directory, folder);
         // eslint-disable-next-line no-restricted-syntax
         const { default: createConnector } = (await import(packagePath)) as {
           default: CreateConnector<AllConnector>;

packages/core/src/connectors/types.ts

@@ -29,11 +29,3 @@ export type LoadConnector<T extends AllConnector = AllConnector> = T & {
 export type LogtoConnector<T extends AllConnector = AllConnector> = LoadConnector<T> & {
   dbEntry: Connector;
 };
-
-export const npmPackResultGuard = z
-  .object({
-    name: z.string(),
-    version: z.string(),
-    filename: z.string(),
-  })
-  .array();

packages/core/src/env-set/add-connectors.ts

@@ -1,30 +0,0 @@
-import { existsSync } from 'fs';
-import { mkdir } from 'fs/promises';
-
-import inquirer from 'inquirer';
-
-import { addOfficialConnectors } from '@/connectors/add-connectors';
-
-import { allYes } from './parameters';
-
-export const addConnectors = async (directory: string) => {
-  if (existsSync(directory)) {
-    return;
-  }
-
-  if (!allYes) {
-    const add = await inquirer.prompt({
-      type: 'confirm',
-      name: 'value',
-      message: `Would you like to add built-in connectors?`,
-    });
-
-    if (!add.value) {
-      await mkdir(directory);
-
-      return;
-    }
-  }
-
-  await addOfficialConnectors(directory);
-};

packages/core/src/env-set/index.ts

@@ -1,12 +1,9 @@
-import path from 'path';
-
 import { getEnv, getEnvAsStringArray, Optional } from '@silverhand/essentials';
 import { DatabasePool } from 'slonik';
 
 import { getOidcConfigs } from '@/lib/logto-config';
 import { appendPath } from '@/utils/url';
 
-import { addConnectors } from './add-connectors';
 import { checkAlterationState } from './check-alteration-state';
 import createPoolByEnv from './create-pool-by-env';
 import loadOidcValues from './oidc';
@@ -20,9 +17,6 @@ export enum MountedApps {
   Welcome = 'welcome',
 }
 
-// eslint-disable-next-line unicorn/prefer-module
-export const defaultConnectorDirectory = path.join(__dirname, '../../connectors');
-
 const loadEnvValues = async () => {
   const isProduction = getEnv('NODE_ENV') === 'production';
   const isTest = getEnv('NODE_ENV') === 'test';
@@ -46,7 +40,6 @@ const loadEnvValues = async () => {
     developmentUserId: getEnv('DEVELOPMENT_USER_ID'),
     trustProxyHeader: isTrue(getEnv('TRUST_PROXY_HEADER')),
     adminConsoleUrl: appendPath(endpoint, '/console'),
-    connectorDirectory: getEnv('CONNECTOR_DIRECTORY', defaultConnectorDirectory),
   });
 };
 
@@ -93,8 +86,6 @@ function createEnvSet() {
 
       const [, oidcConfigs] = await Promise.all([checkAlterationState(pool), getOidcConfigs(pool)]);
       oidc = await loadOidcValues(appendPath(values.endpoint, '/oidc').toString(), oidcConfigs);
-
-      await addConnectors(values.connectorDirectory);
     },
   };
 }

packages/shared/package.json

@@ -43,6 +43,7 @@
     "@logto/schemas": "^1.0.0-beta.10",
     "@silverhand/essentials": "^1.3.0",
     "dayjs": "^1.10.5",
+    "find-up": "^5.0.0",
     "nanoid": "^3.3.4",
     "slonik": "^30.0.0"
   }

packages/shared/src/utils/find-package.ts

@@ -0,0 +1,18 @@
+import path from 'path';
+
+import { conditional } from '@silverhand/essentials';
+import findUp, { exists } from 'find-up';
+
+const findPackage = async (cwd: string) =>
+  findUp(
+    async (directory) => {
+      const hasPackageJson = await exists(path.join(directory, 'package.json'));
+
+      return conditional(hasPackageJson && directory);
+    },
+    {
+      cwd,
+      type: 'directory',
+    }
+  );
+export default findPackage;

packages/shared/src/utils/index.ts

@@ -1 +1,2 @@
 export * from './id';
+export { default as findPackage } from './find-package';

pnpm-lock.yaml

@@ -37,7 +37,6 @@ importers:
       decamelize: ^5.0.0
       dotenv: ^16.0.0
       eslint: ^8.21.0
-      find-up: ^5.0.0
       fs-extra: ^10.1.0
       got: ^11.8.2
       hpagent: ^1.0.0
@@ -66,7 +65,6 @@ importers:
       chalk: 4.1.2
       decamelize: 5.0.1
       dotenv: 16.0.0
-      find-up: 5.0.0
       fs-extra: 10.1.0
       got: 11.8.3
       hpagent: 1.0.0
@@ -256,7 +254,6 @@ importers:
       '@types/etag': ^1.8.1
       '@types/fs-extra': ^9.0.13
       '@types/http-errors': ^1.8.2
-      '@types/inquirer': ^8.2.1
       '@types/jest': ^28.1.6
       '@types/js-yaml': ^4.0.5
       '@types/koa': ^2.13.3
@@ -267,9 +264,7 @@ importers:
       '@types/lodash.pick': ^4.4.6
       '@types/node': ^16.3.1
       '@types/oidc-provider': ^7.11.1
-      '@types/rimraf': ^3.0.2
       '@types/supertest': ^2.0.11
-      '@types/tar': ^6.1.2
       chalk: ^4
       clean-deep: ^3.4.0
       copyfiles: ^2.4.1
@@ -286,7 +281,6 @@ importers:
       http-errors: ^1.6.3
       i18next: ^21.8.16
       iconv-lite: 0.6.3
-      inquirer: ^8.2.2
       jest: ^28.1.3
       jest-matcher-specific-error: ^1.0.0
       jose: ^4.0.0
@@ -310,7 +304,6 @@ importers:
       p-retry: ^4.6.1
       prettier: ^2.7.1
       query-string: ^7.0.1
-      rimraf: ^3.0.2
       roarr: ^7.11.0
       slonik: ^30.0.0
       slonik-interceptor-preset: ^1.2.10
@@ -318,7 +311,6 @@ importers:
       snake-case: ^3.0.4
       snakecase-keys: ^5.1.0
       supertest: ^6.2.2
-      tar: ^6.1.11
       typescript: ^4.7.4
       zod: ^3.18.0
     dependencies:
@@ -344,7 +336,6 @@ importers:
       hash-wasm: 4.9.0
       i18next: 21.8.16
       iconv-lite: 0.6.3
-      inquirer: 8.2.2
       jose: 4.6.0
       js-yaml: 4.1.0
       koa: 2.13.4
@@ -361,14 +352,12 @@ importers:
       oidc-provider: 7.11.3
       p-retry: 4.6.1
       query-string: 7.0.1
-      rimraf: 3.0.2
       roarr: 7.11.0
       slonik: 30.1.2
       slonik-interceptor-preset: 1.2.10
       slonik-sql-tag-raw: 1.1.4_roarr@7.11.0+slonik@30.1.2
       snake-case: 3.0.4
       snakecase-keys: 5.1.2
-      tar: 6.1.11
       zod: 3.18.0
     devDependencies:
       '@shopify/jest-koa-mocks': 5.0.0
@@ -379,7 +368,6 @@ importers:
       '@types/etag': 1.8.1
       '@types/fs-extra': 9.0.13
       '@types/http-errors': 1.8.2
-      '@types/inquirer': 8.2.1
       '@types/jest': 28.1.6
       '@types/js-yaml': 4.0.5
       '@types/koa': 2.13.4
@@ -390,9 +378,7 @@ importers:
       '@types/lodash.pick': 4.4.6
       '@types/node': 16.11.12
       '@types/oidc-provider': 7.11.1
-      '@types/rimraf': 3.0.2
       '@types/supertest': 2.0.11
-      '@types/tar': 6.1.2
       copyfiles: 2.4.1
       eslint: 8.21.0
       http-errors: 1.8.1
@@ -633,6 +619,7 @@ importers:
       '@types/jest': ^28.1.6
       dayjs: ^1.10.5
       eslint: ^8.21.0
+      find-up: ^5.0.0
       jest: ^28.1.3
       lint-staged: ^13.0.0
       nanoid: ^3.3.4
@@ -643,6 +630,7 @@ importers:
       '@logto/schemas': link:../schemas
       '@silverhand/essentials': 1.3.0
       dayjs: 1.10.7
+      find-up: 5.0.0
       nanoid: 3.3.4
       slonik: 30.1.2
     devDependencies:
@@ -4451,13 +4439,6 @@ packages:
       '@types/node': 17.0.23
     dev: true
 
-  /@types/glob/8.0.0:
-    resolution: {integrity: sha512-l6NQsDDyQUVeoTynNpC9uRvCUint/gSUXQA2euwmTuWGvPY5LSDUu6tkCtJB2SvGQlJQzLaKqcGZP4//7EDveA==}
-    dependencies:
-      '@types/minimatch': 3.0.5
-      '@types/node': 17.0.23
-    dev: true
-
   /@types/graceful-fs/4.1.5:
     resolution: {integrity: sha512-anKkLmZZ+xm4p8JWBf4hElkM4XR+EZeA2M9BAkkTldmcyDY4mbdIJnRghDJH3Ov5ooY7/UAoENtmdMSkaAd7Cw==}
     dependencies:
@@ -4755,13 +4736,6 @@ packages:
     resolution: {integrity: sha512-xoDlM2S4ortawSWORYqsdU+2rxdh4LRW9ytc3zmT37RIKQh6IHyKwwtKhKis9ah8ol07DCkZxPt8BBvPjC6v4g==}
     dev: false
 
-  /@types/rimraf/3.0.2:
-    resolution: {integrity: sha512-F3OznnSLAUxFrCEu/L5PY8+ny8DtcFRjx7fZZ9bycvXRi3KPTRS9HOitGZwvPg0juRhXFWIeKX58cnX5YqLohQ==}
-    dependencies:
-      '@types/glob': 8.0.0
-      '@types/node': 17.0.23
-    dev: true
-
   /@types/scheduler/0.16.2:
     resolution: {integrity: sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==}
     dev: true
@@ -7862,6 +7836,7 @@ packages:
 
   /fs.realpath/1.0.0:
     resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==}
+    dev: true
 
   /fsevents/2.3.2:
     resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==}
@@ -8085,6 +8060,7 @@ packages:
       minimatch: 3.1.2
       once: 1.4.0
       path-is-absolute: 1.0.1
+    dev: true
 
   /glob/8.0.3:
     resolution: {integrity: sha512-ull455NHSHI/Y1FqGaaYFaLGkNMMJbavMrEGFXG/PGrg6y7sutWHUHrz6gy6WEBH6akM1M414dWKCNs+IhKdiQ==}
@@ -8766,6 +8742,7 @@ packages:
     dependencies:
       once: 1.4.0
       wrappy: 1.0.2
+    dev: true
 
   /inherits/2.0.1:
     resolution: {integrity: sha512-8nWq2nLTAwd02jTqJExUYFSD/fKq6VH9Y/oG2accc/kdI0V98Bag8d5a4gi3XHz73rDWa2PvTtvcWYquKqSENA==}
@@ -13963,6 +13940,7 @@ packages:
     hasBin: true
     dependencies:
       glob: 7.2.0
+    dev: true
 
   /roarr/2.15.4:
     resolution: {integrity: sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==}