feat(core)!: load connectors by folder (#1879)

.github/workflows/integration-test.yml

@@ -4,7 +4,7 @@ on:
   push:
     branches:
       - master
-      - 'push-action/**'
+      - "push-action/**"
   pull_request:
 
 concurrency:
@@ -21,13 +21,15 @@ jobs:
       - name: Setup Node and pnpm
         uses: silverhand-io/actions-node-pnpm-run-steps@v1.2.3
 
+      - name: Build
+        run: pnpm -- lerna run build --stream
+
       - name: Add the mock connectors for integration tests only
         run: |
-          unset CI
+          pnpm add-connector @logto/connector-mock-sms
-          unset GITHUB_ACTIONS
+          pnpm add-connector @logto/connector-mock-email
-          lerna add @logto/connector-mock-sms --scope=@logto/core
+          pnpm add-connector @logto/connector-mock-social
-          lerna add @logto/connector-mock-email --scope=@logto/core
+        working-directory: packages/core
-          lerna add @logto/connector-mock-social --scope=@logto/core
 
       - name: Package
         run: ./package.sh
@@ -88,7 +90,6 @@ jobs:
           INTEGRATION_TEST: true
           NODE_ENV: production
           DB_URL_DEFAULT: postgres://postgres:postgres@localhost:5432
-          ADDITIONAL_CONNECTOR_PACKAGES: '@logto/connector-mock-sms,@logto/connector-mock-email,@logto/connector-mock-social'
 
       - name: Sleep for 5 seconds
         run: sleep 5

.github/workflows/release.yml

@@ -79,6 +79,9 @@ jobs:
           cat -s \
           > /tmp/changelog.txt
 
+      - name: Build
+        run: pnpm -- lerna run build --stream
+
       - name: Package
         run: ./package.sh
 

.gitignore

@@ -29,3 +29,6 @@ cache
 .idea/
 *.pem
 .history
+
+# connectors
+/packages/core/connectors

package.sh

@@ -1,8 +1,5 @@
 set -eo pipefail
 
-echo Building packages
-pnpm -- lerna run build --stream
-
 echo Prune dependencies
 rm -rf node_modules packages/*/node_modules
 

packages/core/package.json

@@ -14,27 +14,14 @@
     "lint:report": "pnpm lint --format json --output-file report.json",
     "dev": "rm -rf build/ && pnpm run copyfiles && nodemon",
     "start": "NODE_ENV=production node build/index.js",
+    "add-connector": "node build/cli/add-connector.js",
+    "add-official-connectors": "node build/cli/add-official-connectors.js",
     "test": "jest",
     "test:coverage": "jest --coverage --silent",
     "test:report": "codecov -F core"
   },
   "dependencies": {
-    "@logto/connector-alipay-native": "^1.0.0-beta.8",
-    "@logto/connector-alipay-web": "^1.0.0-beta.8",
-    "@logto/connector-aliyun-dm": "^1.0.0-beta.8",
-    "@logto/connector-aliyun-sms": "^1.0.0-beta.8",
-    "@logto/connector-apple": "^1.0.0-beta.8",
-    "@logto/connector-azuread": "^1.0.0-beta.8",
     "@logto/connector-core": "^1.0.0-beta.8",
-    "@logto/connector-facebook": "^1.0.0-beta.8",
-    "@logto/connector-github": "^1.0.0-beta.8",
-    "@logto/connector-google": "^1.0.0-beta.8",
-    "@logto/connector-kakao": "^1.0.0-beta.8",
-    "@logto/connector-sendgrid-email": "^1.0.0-beta.8",
-    "@logto/connector-smtp": "^1.0.0-beta.8",
-    "@logto/connector-twilio-sms": "^1.0.0-beta.8",
-    "@logto/connector-wechat-native": "^1.0.0-beta.8",
-    "@logto/connector-wechat-web": "^1.0.0-beta.8",
     "@logto/phrases": "^1.0.0-beta.8",
     "@logto/schemas": "^1.0.0-beta.8",
     "@logto/shared": "^1.0.0-beta.8",
@@ -67,13 +54,14 @@
     "oidc-provider": "^7.11.3",
     "p-retry": "^4.6.1",
     "query-string": "^7.0.1",
-    "resolve-package-path": "^4.0.3",
+    "rimraf": "^3.0.2",
     "roarr": "^7.11.0",
     "slonik": "^30.0.0",
     "slonik-interceptor-preset": "^1.2.10",
     "slonik-sql-tag-raw": "^1.1.4",
     "snake-case": "^3.0.4",
     "snakecase-keys": "^5.1.0",
+    "tar": "^6.1.11",
     "zod": "^3.14.3"
   },
   "devDependencies": {
@@ -94,7 +82,9 @@
     "@types/lodash.pick": "^4.4.6",
     "@types/node": "^16.3.1",
     "@types/oidc-provider": "^7.11.1",
+    "@types/rimraf": "^3.0.2",
     "@types/supertest": "^2.0.11",
+    "@types/tar": "^6.1.2",
     "copyfiles": "^2.4.1",
     "eslint": "^8.21.0",
     "jest": "^28.1.3",

packages/core/src/cli/add-connector.ts

@@ -0,0 +1,25 @@
+import 'module-alias/register';
+import { getEnv } from '@silverhand/essentials';
+import chalk from 'chalk';
+
+import { addConnector } from '@/connectors/add-connectors';
+import { defaultConnectorDirectory } from '@/env-set';
+
+import { configDotEnv } from '../env-set/dot-env';
+
+configDotEnv();
+
+const addConnectorCli = async (packageName: string) => {
+  const connectorDirectory = getEnv('CONNECTOR_DIRECTORY', defaultConnectorDirectory);
+
+  await addConnector(packageName, connectorDirectory);
+  console.log(`${chalk.blue(packageName)} added successfully.`);
+};
+
+const packageName = process.argv[2];
+
+if (!packageName) {
+  throw new Error('Please provide a package name');
+}
+
+void addConnectorCli(packageName);

packages/core/src/cli/add-official-connectors.ts

@@ -0,0 +1,16 @@
+import 'module-alias/register';
+import { getEnv } from '@silverhand/essentials';
+
+import { addOfficialConnectors } from '@/connectors/add-connectors';
+import { defaultConnectorDirectory } from '@/env-set';
+
+import { configDotEnv } from '../env-set/dot-env';
+
+configDotEnv();
+
+const addOfficialConnectorsCli = async () => {
+  const connectorDirectory = getEnv('CONNECTOR_DIRECTORY', defaultConnectorDirectory);
+  await addOfficialConnectors(connectorDirectory);
+};
+
+void addOfficialConnectorsCli();

packages/core/src/connectors/add-connectors.ts

@@ -0,0 +1,72 @@
+import { exec } from 'child_process';
+import { existsSync } from 'fs';
+import { mkdir, rename, unlink } from 'fs/promises';
+import path from 'path';
+import { promisify } from 'util';
+
+import chalk from 'chalk';
+import got from 'got';
+import rimraf from 'rimraf';
+import tar from 'tar';
+
+import { npmPackResultGuard } from './types';
+
+const execPromise = promisify(exec);
+
+const fetchOfficialConnectorList = async () => {
+  // Will change to "logto-io/connectors" once the new repo is ready.
+  const directories = await got
+    .get('https://api.github.com/repos/logto-io/logto/contents/packages')
+    .json<Array<{ name: string }>>();
+
+  return (
+    directories
+      // Will be removed once the new repo is ready.
+      .filter(
+        ({ name }) =>
+          name.startsWith('connector-') &&
+          name !== 'connector-core' &&
+          name !== 'connector-sendgrid-mail'
+      )
+      .map(({ name }) => `@logto/${name}`)
+  );
+};
+
+export const addConnector = async (packageName: string, cwd: string) => {
+  if (!existsSync(cwd)) {
+    await mkdir(cwd);
+  }
+  const { stdout } = await execPromise(`npm pack ${packageName} --json`, { cwd });
+  const result = npmPackResultGuard.parse(JSON.parse(stdout));
+
+  if (!result[0]) {
+    throw new Error(`Failed to download package: ${packageName}`);
+  }
+
+  const { filename, name } = result[0];
+  const escapedFilename = filename.replace(/\//g, '-').replace(/@/g, '');
+  const filePath = path.join(cwd, escapedFilename);
+  await tar.extract({ cwd, file: filePath });
+  await unlink(filePath);
+
+  const packageFolder = path.join(cwd, name.replace(/\//g, '-').replace(/@/g, ''));
+  await promisify(rimraf)(packageFolder);
+
+  await rename(path.join(cwd, 'package'), packageFolder);
+};
+
+export const addOfficialConnectors = async (directory: string) => {
+  console.log(`${chalk.blue('[add-connectors]')} Fetching official connectors list`);
+  const packages = await fetchOfficialConnectorList();
+
+  // The await inside the loop is intended for better debugging experience and rate limitation.
+  for (const [index, packageName] of packages.entries()) {
+    console.log(
+      `${chalk.blue('[add-connectors]')} ${index + 1}/${
+        packages.length
+      } Adding connector package: ${packageName}`
+    );
+    // eslint-disable-next-line no-await-in-loop
+    await addConnector(packageName, directory);
+  }
+};

packages/core/src/connectors/consts.ts

@@ -1,23 +1,5 @@
 import { ConnectorError, ConnectorErrorCodes } from '@logto/connector-core';
 
-export const defaultConnectorPackages = [
-  '@logto/connector-alipay-web',
-  '@logto/connector-alipay-native',
-  '@logto/connector-aliyun-dm',
-  '@logto/connector-aliyun-sms',
-  '@logto/connector-apple',
-  '@logto/connector-facebook',
-  '@logto/connector-github',
-  '@logto/connector-google',
-  '@logto/connector-azuread',
-  '@logto/connector-sendgrid-email',
-  '@logto/connector-smtp',
-  '@logto/connector-twilio-sms',
-  '@logto/connector-wechat-web',
-  '@logto/connector-wechat-native',
-  '@logto/connector-kakao',
-];
-
 const notImplemented = () => {
   throw new ConnectorError(ConnectorErrorCodes.NotImplemented);
 };

packages/core/src/connectors/index.test.ts

@@ -1,207 +0,0 @@
-import { ConnectorPlatform } from '@logto/connector-core';
-import { Connector } from '@logto/schemas';
-
-import { getLogtoConnectorById, getLogtoConnectors, initConnectors } from '@/connectors';
-import RequestError from '@/errors/RequestError';
-
-const alipayConnector = {
-  id: 'alipay-web',
-  enabled: true,
-  config: {},
-  createdAt: 1_646_382_233_911,
-};
-const alipayNativeConnector = {
-  id: 'alipay-native',
-  enabled: false,
-  config: {},
-  createdAt: 1_646_382_233_911,
-};
-const aliyunDmConnector = {
-  id: 'aliyun-direct-mail',
-  enabled: true,
-  config: {},
-  createdAt: 1_646_382_233_911,
-};
-const aliyunSmsConnector = {
-  id: 'aliyun-short-message-service',
-  enabled: false,
-  config: {},
-  createdAt: 1_646_382_233_666,
-};
-const appleConnector = {
-  id: 'apple-universal',
-  enabled: false,
-  config: {},
-  createdAt: 1_646_382_233_666,
-};
-const facebookConnector = {
-  id: 'facebook-universal',
-  enabled: true,
-  config: {},
-  createdAt: 1_646_382_233_333,
-};
-const githubConnector = {
-  id: 'github-universal',
-  enabled: true,
-  config: {},
-  createdAt: 1_646_382_233_555,
-};
-const googleConnector = {
-  id: 'google-universal',
-  enabled: false,
-  config: {},
-  createdAt: 1_646_382_233_000,
-};
-const azureADConnector = {
-  id: 'azuread-universal',
-  enabled: false,
-  config: {},
-  createdAt: 1_646_382_233_000,
-};
-const sendGridMailConnector = {
-  id: 'sendgrid-email-service',
-  enabled: false,
-  config: {},
-  createdAt: 1_646_382_233_111,
-};
-const smtpConnector = {
-  id: 'simple-mail-transfer-protocol',
-  enabled: false,
-  config: {},
-  createdAt: 1_646_382_233_111,
-};
-const twilioSmsConnector = {
-  id: 'twilio-short-message-service',
-  enabled: false,
-  config: {},
-  createdAt: 1_646_382_233_000,
-};
-const wechatConnector = {
-  id: 'wechat-web',
-  enabled: false,
-  config: {},
-  createdAt: 1_646_382_233_000,
-};
-const wechatNativeConnector = {
-  id: 'wechat-native',
-  enabled: false,
-  config: {},
-  createdAt: 1_646_382_233_000,
-};
-const kakaoConnector = {
-  id: 'kakao-universal',
-  enabled: false,
-  config: {},
-  createdAt: 1_646_382_233_000,
-};
-
-const connectors = [
-  alipayConnector,
-  alipayNativeConnector,
-  aliyunDmConnector,
-  aliyunSmsConnector,
-  appleConnector,
-  facebookConnector,
-  githubConnector,
-  googleConnector,
-  azureADConnector,
-  sendGridMailConnector,
-  smtpConnector,
-  twilioSmsConnector,
-  wechatConnector,
-  wechatNativeConnector,
-  kakaoConnector,
-];
-
-const findAllConnectors = jest.fn(async () => connectors);
-const insertConnector = jest.fn(async (connector: Connector) => connector);
-
-jest.mock('@/queries/connector', () => ({
-  ...jest.requireActual('@/queries/connector'),
-  findAllConnectors: async () => findAllConnectors(),
-  insertConnector: async (connector: Connector) => insertConnector(connector),
-}));
-
-describe('getLogtoConnectors', () => {
-  test('should return the connectors existing in DB', async () => {
-    const logtoConnectors = await getLogtoConnectors();
-    expect(logtoConnectors).toHaveLength(connectors.length);
-
-    for (const [index, connector] of connectors.entries()) {
-      expect(logtoConnectors[index]).toHaveProperty('dbEntry', connector);
-    }
-  });
-
-  test('should throw if any required connector does not exist in DB', async () => {
-    const id = 'aliyun-dm';
-    findAllConnectors.mockImplementationOnce(async () => []);
-    await expect(getLogtoConnectors()).rejects.toMatchError(
-      new RequestError({ code: 'entity.not_found', id, status: 404 })
-    );
-  });
-
-  test('should access DB only once and should not throw', async () => {
-    await expect(getLogtoConnectors()).resolves.not.toThrow();
-    expect(findAllConnectors).toHaveBeenCalled();
-  });
-
-  afterEach(() => {
-    findAllConnectors.mockClear();
-  });
-});
-
-describe('getLogtoConnectorBy', () => {
-  afterEach(() => {
-    jest.clearAllMocks();
-  });
-
-  test('should return the connector existing in DB', async () => {
-    const connector = await getLogtoConnectorById('github-universal');
-    expect(connector).toHaveProperty('dbEntry', githubConnector);
-  });
-
-  test('should throw on invalid id (on DB query)', async () => {
-    const id = 'invalid_id';
-    await expect(getLogtoConnectorById(id)).rejects.toThrow();
-  });
-
-  test('should throw on invalid id (on finding metadata)', async () => {
-    const id = 'invalid_id';
-    await expect(getLogtoConnectorById(id)).rejects.toMatchError(
-      new RequestError({
-        code: 'entity.not_found',
-        target: 'invalid_target',
-        platfrom: ConnectorPlatform.Web,
-        status: 404,
-      })
-    );
-  });
-});
-
-describe('initConnectors', () => {
-  test('should insert the necessary connector if it does not exist in DB', async () => {
-    findAllConnectors.mockImplementationOnce(async () => []);
-    await expect(initConnectors()).resolves.not.toThrow();
-    expect(insertConnector).toHaveBeenCalledTimes(connectors.length);
-
-    for (const [i, connector] of connectors.entries()) {
-      const { id } = connector;
-      expect(insertConnector).toHaveBeenNthCalledWith(
-        i + 1,
-        expect.objectContaining({
-          id,
-        })
-      );
-    }
-  });
-
-  test('should not insert the connector if it exists in DB', async () => {
-    await expect(initConnectors()).resolves.not.toThrow();
-    expect(insertConnector).not.toHaveBeenCalled();
-  });
-
-  afterEach(() => {
-    findAllConnectors.mockClear();
-    insertConnector.mockClear();
-  });
-});

packages/core/src/connectors/index.ts

@@ -1,16 +1,17 @@
-import { existsSync, readFileSync } from 'fs';
+import { existsSync } from 'fs';
+import { readdir } from 'fs/promises';
 import path from 'path';
 
 import { AllConnector, CreateConnector, validateConfig } from '@logto/connector-core';
-import resolvePackagePath from 'resolve-package-path';
+import chalk from 'chalk';
 
 import envSet from '@/env-set';
 import RequestError from '@/errors/RequestError';
 import { findAllConnectors, insertConnector } from '@/queries/connector';
 
-import { defaultConnectorMethods, defaultConnectorPackages } from './consts';
+import { defaultConnectorMethods } from './consts';
 import { LoadConnector, LogtoConnector } from './types';
-import { getConnectorConfig, validateConnectorModule } from './utilities';
+import { getConnectorConfig, readUrl, validateConnectorModule } from './utilities';
 
 // eslint-disable-next-line @silverhand/fp/no-let
 let cachedConnectors: LoadConnector[] | undefined;
@@ -21,16 +22,21 @@ const loadConnectors = async () => {
   }
 
   const {
-    values: { additionalConnectorPackages },
+    values: { connectorDirectory },
   } = envSet;
 
-  const connectorPackages = [...defaultConnectorPackages, ...additionalConnectorPackages];
+  if (!existsSync(connectorDirectory)) {
+    return [];
+  }
 
-  // eslint-disable-next-line @silverhand/fp/no-mutation
+  const connectorFolders = await readdir(connectorDirectory);
-  cachedConnectors = await Promise.all(
+
-    connectorPackages.map(async (packageName) => {
+  const connectors = await Promise.all(
+    connectorFolders.map(async (folder) => {
+      try {
+        const packagePath = path.join(connectorDirectory, folder);
         // eslint-disable-next-line no-restricted-syntax
-      const { default: createConnector } = (await import(packageName)) as {
+        const { default: createConnector } = (await import(packagePath)) as {
           default: CreateConnector<AllConnector>;
         };
         const rawConnector = await createConnector({ getConfig: getConnectorConfig });
@@ -39,63 +45,46 @@ const loadConnectors = async () => {
         const connector: LoadConnector = {
           ...defaultConnectorMethods,
           ...rawConnector,
+          metadata: {
+            ...rawConnector.metadata,
+            logo: await readUrl(rawConnector.metadata.logo, packagePath, 'svg'),
+            logoDark:
+              rawConnector.metadata.logoDark &&
+              (await readUrl(rawConnector.metadata.logoDark, packagePath, 'svg')),
+            readme: await readUrl(rawConnector.metadata.readme, packagePath, 'text'),
+            configTemplate: await readUrl(
+              rawConnector.metadata.configTemplate,
+              packagePath,
+              'text'
+            ),
+          },
           validateConfig: (config: unknown) => {
             validateConfig(config, rawConnector.configGuard);
           },
         };
-      // eslint-disable-next-line unicorn/prefer-module
-      const packagePath = resolvePackagePath(packageName, __dirname);
-
-      // For relative path logo url, try to read local asset.
-      if (
-        packagePath &&
-        !connector.metadata.logo.startsWith('http') &&
-        existsSync(path.join(packagePath, '..', connector.metadata.logo))
-      ) {
-        const data = readFileSync(path.join(packagePath, '..', connector.metadata.logo));
-        // eslint-disable-next-line @silverhand/fp/no-mutation
-        connector.metadata.logo = `data:image/svg+xml;base64,${data.toString('base64')}`;
-      }
-
-      if (
-        packagePath &&
-        connector.metadata.logoDark &&
-        !connector.metadata.logoDark.startsWith('http') &&
-        existsSync(path.join(packagePath, '..', connector.metadata.logoDark))
-      ) {
-        const data = readFileSync(path.join(packagePath, '..', connector.metadata.logoDark));
-        // eslint-disable-next-line @silverhand/fp/no-mutation
-        connector.metadata.logoDark = `data:image/svg+xml;base64,${data.toString('base64')}`;
-      }
-
-      if (
-        packagePath &&
-        connector.metadata.readme &&
-        existsSync(path.join(packagePath, '..', connector.metadata.readme))
-      ) {
-        // eslint-disable-next-line @silverhand/fp/no-mutation
-        connector.metadata.readme = readFileSync(
-          path.join(packagePath, '..', connector.metadata.readme),
-          'utf8'
-        );
-      }
-
-      if (
-        packagePath &&
-        connector.metadata.configTemplate &&
-        existsSync(path.join(packagePath, '..', connector.metadata.configTemplate))
-      ) {
-        // eslint-disable-next-line @silverhand/fp/no-mutation
-        connector.metadata.configTemplate = readFileSync(
-          path.join(packagePath, '..', connector.metadata.configTemplate),
-          'utf8'
-        );
-      }
 
         return connector;
+      } catch (error: unknown) {
+        if (error instanceof Error) {
+          console.log(
+            `${chalk.red(
+              `[load-connector] skip ${chalk.bold(folder)} due to error: ${error.message}`
+            )}`
+          );
+
+          return;
+        }
+
+        throw error;
+      }
     })
   );
 
+  // eslint-disable-next-line @silverhand/fp/no-mutation
+  cachedConnectors = connectors.filter(
+    (connector): connector is LoadConnector => connector !== undefined
+  );
+
   return cachedConnectors;
 };
 

packages/core/src/connectors/types.ts

@@ -29,3 +29,11 @@ export type LoadConnector<T extends AllConnector = AllConnector> = T & {
 export type LogtoConnector<T extends AllConnector = AllConnector> = LoadConnector<T> & {
   dbEntry: Connector;
 };
+
+export const npmPackResultGuard = z
+  .object({
+    name: z.string(),
+    version: z.string(),
+    filename: z.string(),
+  })
+  .array();

packages/core/src/connectors/utilities/index.ts

@@ -1,3 +1,7 @@
+import { existsSync } from 'fs';
+import { readFile } from 'fs/promises';
+import path from 'path';
+
 import {
   BaseConnector,
   ConnectorError,
@@ -33,3 +37,29 @@ export function validateConnectorModule(
     throw new ConnectorError(ConnectorErrorCodes.UnexpectedType);
   }
 }
+
+export const readUrl = async (
+  url: string,
+  baseUrl: string,
+  type: 'text' | 'svg'
+): Promise<string> => {
+  if (!url) {
+    return url;
+  }
+
+  if (type !== 'text' && url.startsWith('http')) {
+    return url;
+  }
+
+  if (!existsSync(path.join(baseUrl, url))) {
+    return url;
+  }
+
+  if (type === 'svg') {
+    const data = await readFile(path.join(baseUrl, url));
+
+    return `data:image/svg+xml;base64,${data.toString('base64')}`;
+  }
+
+  return readFile(path.join(baseUrl, url), 'utf8');
+};

packages/core/src/env-set/add-connectors.ts

@@ -0,0 +1,27 @@
+import { existsSync } from 'fs';
+
+import inquirer from 'inquirer';
+
+import { addOfficialConnectors } from '@/connectors/add-connectors';
+
+import { allYes } from './parameters';
+
+export const addConnectors = async (directory: string) => {
+  if (existsSync(directory)) {
+    return;
+  }
+
+  if (!allYes) {
+    const add = await inquirer.prompt({
+      type: 'confirm',
+      name: 'value',
+      message: `Would you like to add built-in connectors?`,
+    });
+
+    if (!add.value) {
+      return;
+    }
+  }
+
+  await addOfficialConnectors(directory);
+};

packages/core/src/env-set/index.ts

@@ -1,8 +1,11 @@
+import path from 'path';
+
 import { getEnv, getEnvAsStringArray, Optional } from '@silverhand/essentials';
 import { DatabasePool } from 'slonik';
 
 import { appendPath } from '@/utils/url';
 
+import { addConnectors } from './add-connectors';
 import createPoolByEnv from './create-pool-by-env';
 import loadOidcValues from './oidc';
 import { isTrue } from './parameters';
@@ -15,6 +18,9 @@ export enum MountedApps {
   Welcome = 'welcome',
 }
 
+// eslint-disable-next-line unicorn/prefer-module
+export const defaultConnectorDirectory = path.join(__dirname, '../../connectors');
+
 const loadEnvValues = async () => {
   const isProduction = getEnv('NODE_ENV') === 'production';
   const isTest = getEnv('NODE_ENV') === 'test';
@@ -34,12 +40,12 @@ const loadEnvValues = async () => {
     port,
     localhostUrl,
     endpoint,
-    additionalConnectorPackages: getEnvAsStringArray('ADDITIONAL_CONNECTOR_PACKAGES'),
     userDefaultRoleNames: getEnvAsStringArray('USER_DEFAULT_ROLE_NAMES'),
     developmentUserId: getEnv('DEVELOPMENT_USER_ID'),
     trustProxyHeader: isTrue(getEnv('TRUST_PROXY_HEADER')),
     oidc: await loadOidcValues(appendPath(endpoint, '/oidc').toString()),
     adminConsoleUrl: appendPath(endpoint, '/console'),
+    connectorDirectory: getEnv('CONNECTOR_DIRECTORY', defaultConnectorDirectory),
   });
 };
 
@@ -73,6 +79,7 @@ function createEnvSet() {
     load: async () => {
       values = await loadEnvValues();
       pool = await createPoolByEnv(values.isTest);
+      await addConnectors(values.connectorDirectory);
     },
   };
 }

packages/integration-tests/tests/api/connector.test.ts

@@ -2,10 +2,6 @@ import { ConnectorType } from '@logto/schemas';
 import { HTTPError } from 'got';
 
 import {
-  aliyunEmailConnectorConfig,
-  aliyunEmailConnectorId,
-  aliyunSmsConnectorConfig,
-  aliyunSmsConnectorId,
   mockEmailConnectorConfig,
   mockEmailConnectorId,
   mockSmsConnectorConfig,
@@ -34,8 +30,8 @@ test('connector set-up flow', async () => {
    */
   await Promise.all(
     [
-      { id: aliyunSmsConnectorId, config: aliyunSmsConnectorConfig },
+      { id: mockSmsConnectorId, config: mockSmsConnectorConfig },
-      { id: aliyunEmailConnectorId, config: aliyunEmailConnectorConfig },
+      { id: mockEmailConnectorId, config: mockEmailConnectorConfig },
       { id: mockSocialConnectorId, config: mockSocialConnectorConfig },
     ].map(async ({ id, config }) => {
       const updatedConnector = await updateConnectorConfig(id, config);
@@ -55,7 +51,7 @@ test('connector set-up flow', async () => {
    * We will test updating to the invalid connector config, that is the case not covered above.
    */
   await expect(
-    updateConnectorConfig(mockSocialConnectorId, aliyunSmsConnectorConfig)
+    updateConnectorConfig(mockSocialConnectorId, mockSmsConnectorConfig)
   ).rejects.toThrow(HTTPError);
   // To confirm the failed updating request above did not modify the original config,
   // we check: the mock connector config should stay the same.
@@ -103,13 +99,8 @@ test('connector set-up flow', async () => {
   expect(await listConnectors()).toEqual(
     expect.arrayContaining([
       expect.objectContaining({
-        id: mockSocialConnectorId,
+        id: mockEmailConnectorId,
-        config: mockSocialConnectorConfig,
+        config: mockEmailConnectorConfig,
-        enabled: true,
-      }),
-      expect.objectContaining({
-        id: aliyunSmsConnectorId,
-        config: aliyunSmsConnectorConfig,
         enabled: false,
       }),
       expect.objectContaining({
@@ -118,14 +109,9 @@ test('connector set-up flow', async () => {
         enabled: true,
       }),
       expect.objectContaining({
-        id: aliyunEmailConnectorId,
+        id: mockSocialConnectorId,
-        config: aliyunEmailConnectorConfig,
+        config: mockSocialConnectorConfig,
-        enabled: false,
+        enabled: true,
-      }),
-      expect.objectContaining({
-        id: mockEmailConnectorId,
-        config: mockEmailConnectorConfig,
-        enabled: false,
       }),
     ])
   );

packages/integration-tests/tests/api/sign-in-experience.test.ts

@@ -1,13 +1,13 @@
 import { BrandingStyle, SignInMethodState } from '@logto/schemas';
 
 import {
+  mockEmailConnectorConfig,
+  mockEmailConnectorId,
+  mockSmsConnectorConfig,
+  mockSmsConnectorId,
   mockSocialConnectorConfig,
   mockSocialConnectorId,
   mockSocialConnectorTarget,
-  twilioSmsConnectorConfig,
-  twilioSmsConnectorId,
-  sendgridEmailConnectorConfig,
-  sendgridEmailConnectorId,
 } from '@/__mocks__/connectors-mock';
 import { getSignInExperience, updateSignInExperience } from '@/api';
 import { updateConnectorConfig, enableConnector, disableConnector } from '@/api/connector';
@@ -48,11 +48,11 @@ describe('admin console sign-in experience', () => {
       updateConnectorConfig(mockSocialConnectorId, mockSocialConnectorConfig).then(async () =>
         enableConnector(mockSocialConnectorId)
       ),
-      updateConnectorConfig(twilioSmsConnectorId, twilioSmsConnectorConfig).then(async () =>
+      updateConnectorConfig(mockSmsConnectorId, mockSmsConnectorConfig).then(async () =>
-        enableConnector(twilioSmsConnectorId)
+        enableConnector(mockSmsConnectorId)
       ),
-      updateConnectorConfig(sendgridEmailConnectorId, sendgridEmailConnectorConfig).then(async () =>
+      updateConnectorConfig(mockEmailConnectorId, mockEmailConnectorConfig).then(async () =>
-        enableConnector(sendgridEmailConnectorId)
+        enableConnector(mockEmailConnectorId)
       ),
     ]);
 
@@ -74,8 +74,8 @@ describe('admin console sign-in experience', () => {
     // Reset connectors
     await Promise.all([
       disableConnector(mockSocialConnectorId),
-      disableConnector(twilioSmsConnectorId),
+      disableConnector(mockSmsConnectorId),
-      disableConnector(sendgridEmailConnectorId),
+      disableConnector(mockEmailConnectorId),
     ]);
   });
 });

pnpm-lock.yaml

@@ -893,22 +893,7 @@ importers:
 
   packages/core:
     specifiers:
-      '@logto/connector-alipay-native': ^1.0.0-beta.8
-      '@logto/connector-alipay-web': ^1.0.0-beta.8
-      '@logto/connector-aliyun-dm': ^1.0.0-beta.8
-      '@logto/connector-aliyun-sms': ^1.0.0-beta.8
-      '@logto/connector-apple': ^1.0.0-beta.8
-      '@logto/connector-azuread': ^1.0.0-beta.8
       '@logto/connector-core': ^1.0.0-beta.8
-      '@logto/connector-facebook': ^1.0.0-beta.8
-      '@logto/connector-github': ^1.0.0-beta.8
-      '@logto/connector-google': ^1.0.0-beta.8
-      '@logto/connector-kakao': ^1.0.0-beta.8
-      '@logto/connector-sendgrid-email': ^1.0.0-beta.8
-      '@logto/connector-smtp': ^1.0.0-beta.8
-      '@logto/connector-twilio-sms': ^1.0.0-beta.8
-      '@logto/connector-wechat-native': ^1.0.0-beta.8
-      '@logto/connector-wechat-web': ^1.0.0-beta.8
       '@logto/phrases': ^1.0.0-beta.8
       '@logto/schemas': ^1.0.0-beta.8
       '@logto/shared': ^1.0.0-beta.8
@@ -930,7 +915,9 @@ importers:
       '@types/lodash.pick': ^4.4.6
       '@types/node': ^16.3.1
       '@types/oidc-provider': ^7.11.1
+      '@types/rimraf': ^3.0.2
       '@types/supertest': ^2.0.11
+      '@types/tar': ^6.1.2
       chalk: ^4
       copyfiles: ^2.4.1
       dayjs: ^1.10.5
@@ -968,7 +955,7 @@ importers:
       p-retry: ^4.6.1
       prettier: ^2.7.1
       query-string: ^7.0.1
-      resolve-package-path: ^4.0.3
+      rimraf: ^3.0.2
       roarr: ^7.11.0
       slonik: ^30.0.0
       slonik-interceptor-preset: ^1.2.10
@@ -976,25 +963,11 @@ importers:
       snake-case: ^3.0.4
       snakecase-keys: ^5.1.0
       supertest: ^6.2.2
+      tar: ^6.1.11
       typescript: ^4.7.4
       zod: ^3.14.3
     dependencies:
-      '@logto/connector-alipay-native': link:../connector-alipay-native
-      '@logto/connector-alipay-web': link:../connector-alipay-web
-      '@logto/connector-aliyun-dm': link:../connector-aliyun-dm
-      '@logto/connector-aliyun-sms': link:../connector-aliyun-sms
-      '@logto/connector-apple': link:../connector-apple
-      '@logto/connector-azuread': link:../connector-azuread
       '@logto/connector-core': link:../connector-core
-      '@logto/connector-facebook': link:../connector-facebook
-      '@logto/connector-github': link:../connector-github
-      '@logto/connector-google': link:../connector-google
-      '@logto/connector-kakao': link:../connector-kakao
-      '@logto/connector-sendgrid-email': link:../connector-sendgrid-mail
-      '@logto/connector-smtp': link:../connector-smtp
-      '@logto/connector-twilio-sms': link:../connector-twilio-sms
-      '@logto/connector-wechat-native': link:../connector-wechat-native
-      '@logto/connector-wechat-web': link:../connector-wechat-web
       '@logto/phrases': link:../phrases
       '@logto/schemas': link:../schemas
       '@logto/shared': link:../shared
@@ -1027,13 +1000,14 @@ importers:
       oidc-provider: 7.11.3
       p-retry: 4.6.1
       query-string: 7.0.1
-      resolve-package-path: 4.0.3
+      rimraf: 3.0.2
       roarr: 7.11.0
       slonik: 30.1.2
       slonik-interceptor-preset: 1.2.10
       slonik-sql-tag-raw: 1.1.4_roarr@7.11.0+slonik@30.1.2
       snake-case: 3.0.4
       snakecase-keys: 5.1.2
+      tar: 6.1.11
       zod: 3.14.3
     devDependencies:
       '@shopify/jest-koa-mocks': 5.0.0
@@ -1053,7 +1027,9 @@ importers:
       '@types/lodash.pick': 4.4.6
       '@types/node': 16.11.12
       '@types/oidc-provider': 7.11.1
+      '@types/rimraf': 3.0.2
       '@types/supertest': 2.0.11
+      '@types/tar': 6.1.2
       copyfiles: 2.4.1
       eslint: 8.21.0
       jest: 28.1.3_@types+node@16.11.12
@@ -4962,6 +4938,13 @@ packages:
       '@types/node': 17.0.23
     dev: false
 
+  /@types/glob/8.0.0:
+    resolution: {integrity: sha512-l6NQsDDyQUVeoTynNpC9uRvCUint/gSUXQA2euwmTuWGvPY5LSDUu6tkCtJB2SvGQlJQzLaKqcGZP4//7EDveA==}
+    dependencies:
+      '@types/minimatch': 3.0.5
+      '@types/node': 17.0.23
+    dev: true
+
   /@types/graceful-fs/4.1.5:
     resolution: {integrity: sha512-anKkLmZZ+xm4p8JWBf4hElkM4XR+EZeA2M9BAkkTldmcyDY4mbdIJnRghDJH3Ov5ooY7/UAoENtmdMSkaAd7Cw==}
     dependencies:
@@ -5259,6 +5242,13 @@ packages:
     resolution: {integrity: sha512-xoDlM2S4ortawSWORYqsdU+2rxdh4LRW9ytc3zmT37RIKQh6IHyKwwtKhKis9ah8ol07DCkZxPt8BBvPjC6v4g==}
     dev: false
 
+  /@types/rimraf/3.0.2:
+    resolution: {integrity: sha512-F3OznnSLAUxFrCEu/L5PY8+ny8DtcFRjx7fZZ9bycvXRi3KPTRS9HOitGZwvPg0juRhXFWIeKX58cnX5YqLohQ==}
+    dependencies:
+      '@types/glob': 8.0.0
+      '@types/node': 17.0.23
+    dev: true
+
   /@types/scheduler/0.16.2:
     resolution: {integrity: sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==}
     dev: true
@@ -5286,6 +5276,13 @@ packages:
       '@types/superagent': 4.1.15
     dev: true
 
+  /@types/tar/6.1.2:
+    resolution: {integrity: sha512-bnX3RRm70/n1WMwmevdOAeDU4YP7f5JSubgnuU+yrO+xQQjwDboJj3u2NTJI5ngCQhXihqVVAH5h5J8YpdpEvg==}
+    dependencies:
+      '@types/node': 17.0.23
+      minipass: 3.3.5
+    dev: true
+
   /@types/through/0.0.30:
     resolution: {integrity: sha512-FvnCJljyxhPM3gkRgWmxmDZyAQSiBQQWLI0A0VFL0K7W1oRUrPJSqNO0NvTnLkBcotdlp3lKvaT0JrnyRDkzOg==}
     dependencies:
@@ -6254,7 +6251,6 @@ packages:
   /chownr/2.0.0:
     resolution: {integrity: sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==}
     engines: {node: '>=10'}
-    dev: true
 
   /chrome-trace-event/1.0.3:
     resolution: {integrity: sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg==}
@@ -6612,8 +6608,8 @@ packages:
     engines: {node: '>=10'}
     hasBin: true
     dependencies:
-      JSONStream: 1.3.5
       is-text-path: 1.0.1
+      JSONStream: 1.3.5
       lodash: 4.17.21
       meow: 8.1.2
       split2: 3.2.2
@@ -8215,11 +8211,10 @@ packages:
     resolution: {integrity: sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==}
     engines: {node: '>= 8'}
     dependencies:
-      minipass: 3.1.6
+      minipass: 3.3.5
-    dev: true
 
   /fs.realpath/1.0.0:
-    resolution: {integrity: sha1-FQStJSMVjKpA20onh8sBQRmU6k8=}
+    resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==}
 
   /fsevents/2.3.2:
     resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==}
@@ -9086,13 +9081,13 @@ packages:
     dev: false
 
   /inflight/1.0.6:
-    resolution: {integrity: sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=}
+    resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==}
     dependencies:
       once: 1.4.0
       wrappy: 1.0.2
 
   /inherits/2.0.1:
-    resolution: {integrity: sha1-sX0I0ya0Qj5Wjv9xn5GwscvfafE=}
+    resolution: {integrity: sha512-8nWq2nLTAwd02jTqJExUYFSD/fKq6VH9Y/oG2accc/kdI0V98Bag8d5a4gi3XHz73rDWa2PvTtvcWYquKqSENA==}
     dev: false
 
   /inherits/2.0.3:
@@ -11776,15 +11771,19 @@ packages:
     engines: {node: '>=8'}
     dependencies:
       yallist: 4.0.0
-    dev: true
+
+  /minipass/3.3.5:
+    resolution: {integrity: sha512-rQ/p+KfKBkeNwo04U15i+hOwoVBVmekmm/HcfTkTN2t9pbQKCMm4eN5gFeqgrrSp/kH/7BYYhTIHOxGqzbBPaA==}
+    engines: {node: '>=8'}
+    dependencies:
+      yallist: 4.0.0
 
   /minizlib/2.1.2:
     resolution: {integrity: sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==}
     engines: {node: '>= 8'}
     dependencies:
-      minipass: 3.1.6
+      minipass: 3.3.5
       yallist: 4.0.0
-    dev: true
 
   /mixin-object/2.0.1:
     resolution: {integrity: sha512-ALGF1Jt9ouehcaXaHhn6t1yGWRqGaHkPFndtFVHfZXOvkIZ/yoGaSi0AHVTafb3ZBGg4dr/bDwnaEKqCXzchMA==}
@@ -11811,7 +11810,6 @@ packages:
     resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==}
     engines: {node: '>=10'}
     hasBin: true
-    dev: true
 
   /modify-values/1.0.1:
     resolution: {integrity: sha512-xV2bxeN6F7oYjZWTe/YPAy6MN2M+sL4u/Rlm2AHCIVGfo2p1yGmBHQ6vHehl4bRTZBdHu3TSkWdYgkwpYzAGSw==}
@@ -12409,7 +12407,7 @@ packages:
       ee-first: 1.1.1
 
   /once/1.4.0:
-    resolution: {integrity: sha1-WDsap3WWHUsROsF9nFC6753Xa9E=}
+    resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==}
     dependencies:
       wrappy: 1.0.2
 
@@ -12821,7 +12819,7 @@ packages:
     engines: {node: '>=8'}
 
   /path-is-absolute/1.0.1:
-    resolution: {integrity: sha1-F0uSaHNVNP+8es5r9TpanhtcX18=}
+    resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==}
     engines: {node: '>=0.10.0'}
 
   /path-key/3.1.1:
@@ -12843,18 +12841,6 @@ packages:
   /path-parse/1.0.7:
     resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==}
 
-  /path-root-regex/0.1.2:
-    resolution: {integrity: sha1-v8zcjfWxLcUsi0PsONGNcsBLqW0=}
-    engines: {node: '>=0.10.0'}
-    dev: false
-
-  /path-root/0.1.1:
-    resolution: {integrity: sha1-mkpoFMrBwM1zNgqV8yCDyOpHRbc=}
-    engines: {node: '>=0.10.0'}
-    dependencies:
-      path-root-regex: 0.1.2
-    dev: false
-
   /path-to-regexp/1.8.0:
     resolution: {integrity: sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==}
     dependencies:
@@ -14213,13 +14199,6 @@ packages:
       global-dirs: 0.1.1
     dev: true
 
-  /resolve-package-path/4.0.3:
-    resolution: {integrity: sha512-SRpNAPW4kewOaNUt8VPqhJ0UMxawMwzJD8V7m1cJfdSTK9ieZwS6K7Dabsm4bmLFM96Z5Y/UznrpG5kt1im8yA==}
-    engines: {node: '>= 12'}
-    dependencies:
-      path-root: 0.1.1
-    dev: false
-
   /resolve-path/1.4.0:
     resolution: {integrity: sha1-xL2p9e+y/OZSR4c6s2u02DT+Fvc=}
     engines: {node: '>= 0.8'}
@@ -15320,7 +15299,6 @@ packages:
       minizlib: 2.1.2
       mkdirp: 1.0.4
       yallist: 4.0.0
-    dev: true
 
   /temp-dir/1.0.0:
     resolution: {integrity: sha1-CnwOom06Oa+n4OvqnB/AvE2qAR0=}
@@ -16233,7 +16211,7 @@ packages:
       strip-ansi: 6.0.1
 
   /wrappy/1.0.2:
-    resolution: {integrity: sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=}
+    resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==}
 
   /write-file-atomic/2.4.3:
     resolution: {integrity: sha512-GaETH5wwsX+GcnzhPgKcKjJ6M2Cq3/iZp1WyY/X1CSqrW+jVNM9Y7D8EC2sM4ZG/V8wZlSniJnCKWPmBYAucRQ==}