From 52b9dd8569017ad7fda97a847c95ca1e391aabae Mon Sep 17 00:00:00 2001 From: Wang Sijie Date: Mon, 5 Sep 2022 17:49:11 +0800 Subject: [PATCH] feat(core)!: load connectors by folder (#1879) --- .github/workflows/integration-test.yml | 15 +- .github/workflows/release.yml | 7 +- .gitignore | 3 + package.sh | 3 - packages/core/package.json | 22 +- packages/core/src/cli/add-connector.ts | 25 +++ .../core/src/cli/add-official-connectors.ts | 16 ++ .../core/src/connectors/add-connectors.ts | 72 ++++++ packages/core/src/connectors/consts.ts | 18 -- packages/core/src/connectors/index.test.ts | 207 ------------------ packages/core/src/connectors/index.ts | 135 ++++++------ packages/core/src/connectors/types.ts | 8 + .../core/src/connectors/utilities/index.ts | 30 +++ packages/core/src/env-set/add-connectors.ts | 27 +++ packages/core/src/env-set/index.ts | 9 +- .../tests/api/connector.test.ts | 30 +-- .../tests/api/sign-in-experience.test.ts | 20 +- pnpm-lock.yaml | 110 ++++------ 18 files changed, 332 insertions(+), 425 deletions(-) create mode 100644 packages/core/src/cli/add-connector.ts create mode 100644 packages/core/src/cli/add-official-connectors.ts create mode 100644 packages/core/src/connectors/add-connectors.ts delete mode 100644 packages/core/src/connectors/index.test.ts create mode 100644 packages/core/src/env-set/add-connectors.ts diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 05e8ec980..ac9b6d6e5 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -4,7 +4,7 @@ on: push: branches: - master - - 'push-action/**' + - "push-action/**" pull_request: concurrency: @@ -21,13 +21,15 @@ jobs: - name: Setup Node and pnpm uses: silverhand-io/actions-node-pnpm-run-steps@v1.2.3 + - name: Build + run: pnpm -- lerna run build --stream + - name: Add the mock connectors for integration tests only run: | - unset CI - unset GITHUB_ACTIONS - lerna add @logto/connector-mock-sms --scope=@logto/core - lerna add @logto/connector-mock-email --scope=@logto/core - lerna add @logto/connector-mock-social --scope=@logto/core + pnpm add-connector @logto/connector-mock-sms + pnpm add-connector @logto/connector-mock-email + pnpm add-connector @logto/connector-mock-social + working-directory: packages/core - name: Package run: ./package.sh @@ -88,7 +90,6 @@ jobs: INTEGRATION_TEST: true NODE_ENV: production DB_URL_DEFAULT: postgres://postgres:postgres@localhost:5432 - ADDITIONAL_CONNECTOR_PACKAGES: '@logto/connector-mock-sms,@logto/connector-mock-email,@logto/connector-mock-social' - name: Sleep for 5 seconds run: sleep 5 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7899bf748..8f519bce1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,7 +2,7 @@ name: Release on: push: - branches: + branches: - master tags: - v*.*.* @@ -36,7 +36,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - + - name: Login to DockerHub uses: docker/login-action@v2 with: @@ -79,6 +79,9 @@ jobs: cat -s \ > /tmp/changelog.txt + - name: Build + run: pnpm -- lerna run build --stream + - name: Package run: ./package.sh diff --git a/.gitignore b/.gitignore index 9132715ca..0a6e52864 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,6 @@ cache .idea/ *.pem .history + +# connectors +/packages/core/connectors diff --git a/package.sh b/package.sh index 05580352e..940e2a6bd 100755 --- a/package.sh +++ b/package.sh @@ -1,8 +1,5 @@ set -eo pipefail -echo Building packages -pnpm -- lerna run build --stream - echo Prune dependencies rm -rf node_modules packages/*/node_modules diff --git a/packages/core/package.json b/packages/core/package.json index 64be72cce..a17df3861 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -14,27 +14,14 @@ "lint:report": "pnpm lint --format json --output-file report.json", "dev": "rm -rf build/ && pnpm run copyfiles && nodemon", "start": "NODE_ENV=production node build/index.js", + "add-connector": "node build/cli/add-connector.js", + "add-official-connectors": "node build/cli/add-official-connectors.js", "test": "jest", "test:coverage": "jest --coverage --silent", "test:report": "codecov -F core" }, "dependencies": { - "@logto/connector-alipay-native": "^1.0.0-beta.8", - "@logto/connector-alipay-web": "^1.0.0-beta.8", - "@logto/connector-aliyun-dm": "^1.0.0-beta.8", - "@logto/connector-aliyun-sms": "^1.0.0-beta.8", - "@logto/connector-apple": "^1.0.0-beta.8", - "@logto/connector-azuread": "^1.0.0-beta.8", "@logto/connector-core": "^1.0.0-beta.8", - "@logto/connector-facebook": "^1.0.0-beta.8", - "@logto/connector-github": "^1.0.0-beta.8", - "@logto/connector-google": "^1.0.0-beta.8", - "@logto/connector-kakao": "^1.0.0-beta.8", - "@logto/connector-sendgrid-email": "^1.0.0-beta.8", - "@logto/connector-smtp": "^1.0.0-beta.8", - "@logto/connector-twilio-sms": "^1.0.0-beta.8", - "@logto/connector-wechat-native": "^1.0.0-beta.8", - "@logto/connector-wechat-web": "^1.0.0-beta.8", "@logto/phrases": "^1.0.0-beta.8", "@logto/schemas": "^1.0.0-beta.8", "@logto/shared": "^1.0.0-beta.8", @@ -67,13 +54,14 @@ "oidc-provider": "^7.11.3", "p-retry": "^4.6.1", "query-string": "^7.0.1", - "resolve-package-path": "^4.0.3", + "rimraf": "^3.0.2", "roarr": "^7.11.0", "slonik": "^30.0.0", "slonik-interceptor-preset": "^1.2.10", "slonik-sql-tag-raw": "^1.1.4", "snake-case": "^3.0.4", "snakecase-keys": "^5.1.0", + "tar": "^6.1.11", "zod": "^3.14.3" }, "devDependencies": { @@ -94,7 +82,9 @@ "@types/lodash.pick": "^4.4.6", "@types/node": "^16.3.1", "@types/oidc-provider": "^7.11.1", + "@types/rimraf": "^3.0.2", "@types/supertest": "^2.0.11", + "@types/tar": "^6.1.2", "copyfiles": "^2.4.1", "eslint": "^8.21.0", "jest": "^28.1.3", diff --git a/packages/core/src/cli/add-connector.ts b/packages/core/src/cli/add-connector.ts new file mode 100644 index 000000000..213b4bf82 --- /dev/null +++ b/packages/core/src/cli/add-connector.ts @@ -0,0 +1,25 @@ +import 'module-alias/register'; +import { getEnv } from '@silverhand/essentials'; +import chalk from 'chalk'; + +import { addConnector } from '@/connectors/add-connectors'; +import { defaultConnectorDirectory } from '@/env-set'; + +import { configDotEnv } from '../env-set/dot-env'; + +configDotEnv(); + +const addConnectorCli = async (packageName: string) => { + const connectorDirectory = getEnv('CONNECTOR_DIRECTORY', defaultConnectorDirectory); + + await addConnector(packageName, connectorDirectory); + console.log(`${chalk.blue(packageName)} added successfully.`); +}; + +const packageName = process.argv[2]; + +if (!packageName) { + throw new Error('Please provide a package name'); +} + +void addConnectorCli(packageName); diff --git a/packages/core/src/cli/add-official-connectors.ts b/packages/core/src/cli/add-official-connectors.ts new file mode 100644 index 000000000..bb22a97c2 --- /dev/null +++ b/packages/core/src/cli/add-official-connectors.ts @@ -0,0 +1,16 @@ +import 'module-alias/register'; +import { getEnv } from '@silverhand/essentials'; + +import { addOfficialConnectors } from '@/connectors/add-connectors'; +import { defaultConnectorDirectory } from '@/env-set'; + +import { configDotEnv } from '../env-set/dot-env'; + +configDotEnv(); + +const addOfficialConnectorsCli = async () => { + const connectorDirectory = getEnv('CONNECTOR_DIRECTORY', defaultConnectorDirectory); + await addOfficialConnectors(connectorDirectory); +}; + +void addOfficialConnectorsCli(); diff --git a/packages/core/src/connectors/add-connectors.ts b/packages/core/src/connectors/add-connectors.ts new file mode 100644 index 000000000..7f4dbca34 --- /dev/null +++ b/packages/core/src/connectors/add-connectors.ts @@ -0,0 +1,72 @@ +import { exec } from 'child_process'; +import { existsSync } from 'fs'; +import { mkdir, rename, unlink } from 'fs/promises'; +import path from 'path'; +import { promisify } from 'util'; + +import chalk from 'chalk'; +import got from 'got'; +import rimraf from 'rimraf'; +import tar from 'tar'; + +import { npmPackResultGuard } from './types'; + +const execPromise = promisify(exec); + +const fetchOfficialConnectorList = async () => { + // Will change to "logto-io/connectors" once the new repo is ready. + const directories = await got + .get('https://api.github.com/repos/logto-io/logto/contents/packages') + .json>(); + + return ( + directories + // Will be removed once the new repo is ready. + .filter( + ({ name }) => + name.startsWith('connector-') && + name !== 'connector-core' && + name !== 'connector-sendgrid-mail' + ) + .map(({ name }) => `@logto/${name}`) + ); +}; + +export const addConnector = async (packageName: string, cwd: string) => { + if (!existsSync(cwd)) { + await mkdir(cwd); + } + const { stdout } = await execPromise(`npm pack ${packageName} --json`, { cwd }); + const result = npmPackResultGuard.parse(JSON.parse(stdout)); + + if (!result[0]) { + throw new Error(`Failed to download package: ${packageName}`); + } + + const { filename, name } = result[0]; + const escapedFilename = filename.replace(/\//g, '-').replace(/@/g, ''); + const filePath = path.join(cwd, escapedFilename); + await tar.extract({ cwd, file: filePath }); + await unlink(filePath); + + const packageFolder = path.join(cwd, name.replace(/\//g, '-').replace(/@/g, '')); + await promisify(rimraf)(packageFolder); + + await rename(path.join(cwd, 'package'), packageFolder); +}; + +export const addOfficialConnectors = async (directory: string) => { + console.log(`${chalk.blue('[add-connectors]')} Fetching official connectors list`); + const packages = await fetchOfficialConnectorList(); + + // The await inside the loop is intended for better debugging experience and rate limitation. + for (const [index, packageName] of packages.entries()) { + console.log( + `${chalk.blue('[add-connectors]')} ${index + 1}/${ + packages.length + } Adding connector package: ${packageName}` + ); + // eslint-disable-next-line no-await-in-loop + await addConnector(packageName, directory); + } +}; diff --git a/packages/core/src/connectors/consts.ts b/packages/core/src/connectors/consts.ts index 49a61a053..4aaf9f747 100644 --- a/packages/core/src/connectors/consts.ts +++ b/packages/core/src/connectors/consts.ts @@ -1,23 +1,5 @@ import { ConnectorError, ConnectorErrorCodes } from '@logto/connector-core'; -export const defaultConnectorPackages = [ - '@logto/connector-alipay-web', - '@logto/connector-alipay-native', - '@logto/connector-aliyun-dm', - '@logto/connector-aliyun-sms', - '@logto/connector-apple', - '@logto/connector-facebook', - '@logto/connector-github', - '@logto/connector-google', - '@logto/connector-azuread', - '@logto/connector-sendgrid-email', - '@logto/connector-smtp', - '@logto/connector-twilio-sms', - '@logto/connector-wechat-web', - '@logto/connector-wechat-native', - '@logto/connector-kakao', -]; - const notImplemented = () => { throw new ConnectorError(ConnectorErrorCodes.NotImplemented); }; diff --git a/packages/core/src/connectors/index.test.ts b/packages/core/src/connectors/index.test.ts deleted file mode 100644 index 5d38676b3..000000000 --- a/packages/core/src/connectors/index.test.ts +++ /dev/null @@ -1,207 +0,0 @@ -import { ConnectorPlatform } from '@logto/connector-core'; -import { Connector } from '@logto/schemas'; - -import { getLogtoConnectorById, getLogtoConnectors, initConnectors } from '@/connectors'; -import RequestError from '@/errors/RequestError'; - -const alipayConnector = { - id: 'alipay-web', - enabled: true, - config: {}, - createdAt: 1_646_382_233_911, -}; -const alipayNativeConnector = { - id: 'alipay-native', - enabled: false, - config: {}, - createdAt: 1_646_382_233_911, -}; -const aliyunDmConnector = { - id: 'aliyun-direct-mail', - enabled: true, - config: {}, - createdAt: 1_646_382_233_911, -}; -const aliyunSmsConnector = { - id: 'aliyun-short-message-service', - enabled: false, - config: {}, - createdAt: 1_646_382_233_666, -}; -const appleConnector = { - id: 'apple-universal', - enabled: false, - config: {}, - createdAt: 1_646_382_233_666, -}; -const facebookConnector = { - id: 'facebook-universal', - enabled: true, - config: {}, - createdAt: 1_646_382_233_333, -}; -const githubConnector = { - id: 'github-universal', - enabled: true, - config: {}, - createdAt: 1_646_382_233_555, -}; -const googleConnector = { - id: 'google-universal', - enabled: false, - config: {}, - createdAt: 1_646_382_233_000, -}; -const azureADConnector = { - id: 'azuread-universal', - enabled: false, - config: {}, - createdAt: 1_646_382_233_000, -}; -const sendGridMailConnector = { - id: 'sendgrid-email-service', - enabled: false, - config: {}, - createdAt: 1_646_382_233_111, -}; -const smtpConnector = { - id: 'simple-mail-transfer-protocol', - enabled: false, - config: {}, - createdAt: 1_646_382_233_111, -}; -const twilioSmsConnector = { - id: 'twilio-short-message-service', - enabled: false, - config: {}, - createdAt: 1_646_382_233_000, -}; -const wechatConnector = { - id: 'wechat-web', - enabled: false, - config: {}, - createdAt: 1_646_382_233_000, -}; -const wechatNativeConnector = { - id: 'wechat-native', - enabled: false, - config: {}, - createdAt: 1_646_382_233_000, -}; -const kakaoConnector = { - id: 'kakao-universal', - enabled: false, - config: {}, - createdAt: 1_646_382_233_000, -}; - -const connectors = [ - alipayConnector, - alipayNativeConnector, - aliyunDmConnector, - aliyunSmsConnector, - appleConnector, - facebookConnector, - githubConnector, - googleConnector, - azureADConnector, - sendGridMailConnector, - smtpConnector, - twilioSmsConnector, - wechatConnector, - wechatNativeConnector, - kakaoConnector, -]; - -const findAllConnectors = jest.fn(async () => connectors); -const insertConnector = jest.fn(async (connector: Connector) => connector); - -jest.mock('@/queries/connector', () => ({ - ...jest.requireActual('@/queries/connector'), - findAllConnectors: async () => findAllConnectors(), - insertConnector: async (connector: Connector) => insertConnector(connector), -})); - -describe('getLogtoConnectors', () => { - test('should return the connectors existing in DB', async () => { - const logtoConnectors = await getLogtoConnectors(); - expect(logtoConnectors).toHaveLength(connectors.length); - - for (const [index, connector] of connectors.entries()) { - expect(logtoConnectors[index]).toHaveProperty('dbEntry', connector); - } - }); - - test('should throw if any required connector does not exist in DB', async () => { - const id = 'aliyun-dm'; - findAllConnectors.mockImplementationOnce(async () => []); - await expect(getLogtoConnectors()).rejects.toMatchError( - new RequestError({ code: 'entity.not_found', id, status: 404 }) - ); - }); - - test('should access DB only once and should not throw', async () => { - await expect(getLogtoConnectors()).resolves.not.toThrow(); - expect(findAllConnectors).toHaveBeenCalled(); - }); - - afterEach(() => { - findAllConnectors.mockClear(); - }); -}); - -describe('getLogtoConnectorBy', () => { - afterEach(() => { - jest.clearAllMocks(); - }); - - test('should return the connector existing in DB', async () => { - const connector = await getLogtoConnectorById('github-universal'); - expect(connector).toHaveProperty('dbEntry', githubConnector); - }); - - test('should throw on invalid id (on DB query)', async () => { - const id = 'invalid_id'; - await expect(getLogtoConnectorById(id)).rejects.toThrow(); - }); - - test('should throw on invalid id (on finding metadata)', async () => { - const id = 'invalid_id'; - await expect(getLogtoConnectorById(id)).rejects.toMatchError( - new RequestError({ - code: 'entity.not_found', - target: 'invalid_target', - platfrom: ConnectorPlatform.Web, - status: 404, - }) - ); - }); -}); - -describe('initConnectors', () => { - test('should insert the necessary connector if it does not exist in DB', async () => { - findAllConnectors.mockImplementationOnce(async () => []); - await expect(initConnectors()).resolves.not.toThrow(); - expect(insertConnector).toHaveBeenCalledTimes(connectors.length); - - for (const [i, connector] of connectors.entries()) { - const { id } = connector; - expect(insertConnector).toHaveBeenNthCalledWith( - i + 1, - expect.objectContaining({ - id, - }) - ); - } - }); - - test('should not insert the connector if it exists in DB', async () => { - await expect(initConnectors()).resolves.not.toThrow(); - expect(insertConnector).not.toHaveBeenCalled(); - }); - - afterEach(() => { - findAllConnectors.mockClear(); - insertConnector.mockClear(); - }); -}); diff --git a/packages/core/src/connectors/index.ts b/packages/core/src/connectors/index.ts index 775f969ba..a347c0d3d 100644 --- a/packages/core/src/connectors/index.ts +++ b/packages/core/src/connectors/index.ts @@ -1,16 +1,17 @@ -import { existsSync, readFileSync } from 'fs'; +import { existsSync } from 'fs'; +import { readdir } from 'fs/promises'; import path from 'path'; import { AllConnector, CreateConnector, validateConfig } from '@logto/connector-core'; -import resolvePackagePath from 'resolve-package-path'; +import chalk from 'chalk'; import envSet from '@/env-set'; import RequestError from '@/errors/RequestError'; import { findAllConnectors, insertConnector } from '@/queries/connector'; -import { defaultConnectorMethods, defaultConnectorPackages } from './consts'; +import { defaultConnectorMethods } from './consts'; import { LoadConnector, LogtoConnector } from './types'; -import { getConnectorConfig, validateConnectorModule } from './utilities'; +import { getConnectorConfig, readUrl, validateConnectorModule } from './utilities'; // eslint-disable-next-line @silverhand/fp/no-let let cachedConnectors: LoadConnector[] | undefined; @@ -21,79 +22,67 @@ const loadConnectors = async () => { } const { - values: { additionalConnectorPackages }, + values: { connectorDirectory }, } = envSet; - const connectorPackages = [...defaultConnectorPackages, ...additionalConnectorPackages]; + if (!existsSync(connectorDirectory)) { + return []; + } + + const connectorFolders = await readdir(connectorDirectory); + + const connectors = await Promise.all( + connectorFolders.map(async (folder) => { + try { + const packagePath = path.join(connectorDirectory, folder); + // eslint-disable-next-line no-restricted-syntax + const { default: createConnector } = (await import(packagePath)) as { + default: CreateConnector; + }; + const rawConnector = await createConnector({ getConfig: getConnectorConfig }); + validateConnectorModule(rawConnector); + + const connector: LoadConnector = { + ...defaultConnectorMethods, + ...rawConnector, + metadata: { + ...rawConnector.metadata, + logo: await readUrl(rawConnector.metadata.logo, packagePath, 'svg'), + logoDark: + rawConnector.metadata.logoDark && + (await readUrl(rawConnector.metadata.logoDark, packagePath, 'svg')), + readme: await readUrl(rawConnector.metadata.readme, packagePath, 'text'), + configTemplate: await readUrl( + rawConnector.metadata.configTemplate, + packagePath, + 'text' + ), + }, + validateConfig: (config: unknown) => { + validateConfig(config, rawConnector.configGuard); + }, + }; + + return connector; + } catch (error: unknown) { + if (error instanceof Error) { + console.log( + `${chalk.red( + `[load-connector] skip ${chalk.bold(folder)} due to error: ${error.message}` + )}` + ); + + return; + } + + throw error; + } + }) + ); // eslint-disable-next-line @silverhand/fp/no-mutation - cachedConnectors = await Promise.all( - connectorPackages.map(async (packageName) => { - // eslint-disable-next-line no-restricted-syntax - const { default: createConnector } = (await import(packageName)) as { - default: CreateConnector; - }; - const rawConnector = await createConnector({ getConfig: getConnectorConfig }); - validateConnectorModule(rawConnector); - - const connector: LoadConnector = { - ...defaultConnectorMethods, - ...rawConnector, - validateConfig: (config: unknown) => { - validateConfig(config, rawConnector.configGuard); - }, - }; - // eslint-disable-next-line unicorn/prefer-module - const packagePath = resolvePackagePath(packageName, __dirname); - - // For relative path logo url, try to read local asset. - if ( - packagePath && - !connector.metadata.logo.startsWith('http') && - existsSync(path.join(packagePath, '..', connector.metadata.logo)) - ) { - const data = readFileSync(path.join(packagePath, '..', connector.metadata.logo)); - // eslint-disable-next-line @silverhand/fp/no-mutation - connector.metadata.logo = `data:image/svg+xml;base64,${data.toString('base64')}`; - } - - if ( - packagePath && - connector.metadata.logoDark && - !connector.metadata.logoDark.startsWith('http') && - existsSync(path.join(packagePath, '..', connector.metadata.logoDark)) - ) { - const data = readFileSync(path.join(packagePath, '..', connector.metadata.logoDark)); - // eslint-disable-next-line @silverhand/fp/no-mutation - connector.metadata.logoDark = `data:image/svg+xml;base64,${data.toString('base64')}`; - } - - if ( - packagePath && - connector.metadata.readme && - existsSync(path.join(packagePath, '..', connector.metadata.readme)) - ) { - // eslint-disable-next-line @silverhand/fp/no-mutation - connector.metadata.readme = readFileSync( - path.join(packagePath, '..', connector.metadata.readme), - 'utf8' - ); - } - - if ( - packagePath && - connector.metadata.configTemplate && - existsSync(path.join(packagePath, '..', connector.metadata.configTemplate)) - ) { - // eslint-disable-next-line @silverhand/fp/no-mutation - connector.metadata.configTemplate = readFileSync( - path.join(packagePath, '..', connector.metadata.configTemplate), - 'utf8' - ); - } - - return connector; - }) + cachedConnectors = connectors.filter( + (connector): connector is LoadConnector => connector !== undefined ); return cachedConnectors; diff --git a/packages/core/src/connectors/types.ts b/packages/core/src/connectors/types.ts index 09d3c888c..cd9669402 100644 --- a/packages/core/src/connectors/types.ts +++ b/packages/core/src/connectors/types.ts @@ -29,3 +29,11 @@ export type LoadConnector = T & { export type LogtoConnector = LoadConnector & { dbEntry: Connector; }; + +export const npmPackResultGuard = z + .object({ + name: z.string(), + version: z.string(), + filename: z.string(), + }) + .array(); diff --git a/packages/core/src/connectors/utilities/index.ts b/packages/core/src/connectors/utilities/index.ts index b86cdac87..c8e4e7c25 100644 --- a/packages/core/src/connectors/utilities/index.ts +++ b/packages/core/src/connectors/utilities/index.ts @@ -1,3 +1,7 @@ +import { existsSync } from 'fs'; +import { readFile } from 'fs/promises'; +import path from 'path'; + import { BaseConnector, ConnectorError, @@ -33,3 +37,29 @@ export function validateConnectorModule( throw new ConnectorError(ConnectorErrorCodes.UnexpectedType); } } + +export const readUrl = async ( + url: string, + baseUrl: string, + type: 'text' | 'svg' +): Promise => { + if (!url) { + return url; + } + + if (type !== 'text' && url.startsWith('http')) { + return url; + } + + if (!existsSync(path.join(baseUrl, url))) { + return url; + } + + if (type === 'svg') { + const data = await readFile(path.join(baseUrl, url)); + + return `data:image/svg+xml;base64,${data.toString('base64')}`; + } + + return readFile(path.join(baseUrl, url), 'utf8'); +}; diff --git a/packages/core/src/env-set/add-connectors.ts b/packages/core/src/env-set/add-connectors.ts new file mode 100644 index 000000000..524045fe8 --- /dev/null +++ b/packages/core/src/env-set/add-connectors.ts @@ -0,0 +1,27 @@ +import { existsSync } from 'fs'; + +import inquirer from 'inquirer'; + +import { addOfficialConnectors } from '@/connectors/add-connectors'; + +import { allYes } from './parameters'; + +export const addConnectors = async (directory: string) => { + if (existsSync(directory)) { + return; + } + + if (!allYes) { + const add = await inquirer.prompt({ + type: 'confirm', + name: 'value', + message: `Would you like to add built-in connectors?`, + }); + + if (!add.value) { + return; + } + } + + await addOfficialConnectors(directory); +}; diff --git a/packages/core/src/env-set/index.ts b/packages/core/src/env-set/index.ts index f560beb42..2f7487375 100644 --- a/packages/core/src/env-set/index.ts +++ b/packages/core/src/env-set/index.ts @@ -1,8 +1,11 @@ +import path from 'path'; + import { getEnv, getEnvAsStringArray, Optional } from '@silverhand/essentials'; import { DatabasePool } from 'slonik'; import { appendPath } from '@/utils/url'; +import { addConnectors } from './add-connectors'; import createPoolByEnv from './create-pool-by-env'; import loadOidcValues from './oidc'; import { isTrue } from './parameters'; @@ -15,6 +18,9 @@ export enum MountedApps { Welcome = 'welcome', } +// eslint-disable-next-line unicorn/prefer-module +export const defaultConnectorDirectory = path.join(__dirname, '../../connectors'); + const loadEnvValues = async () => { const isProduction = getEnv('NODE_ENV') === 'production'; const isTest = getEnv('NODE_ENV') === 'test'; @@ -34,12 +40,12 @@ const loadEnvValues = async () => { port, localhostUrl, endpoint, - additionalConnectorPackages: getEnvAsStringArray('ADDITIONAL_CONNECTOR_PACKAGES'), userDefaultRoleNames: getEnvAsStringArray('USER_DEFAULT_ROLE_NAMES'), developmentUserId: getEnv('DEVELOPMENT_USER_ID'), trustProxyHeader: isTrue(getEnv('TRUST_PROXY_HEADER')), oidc: await loadOidcValues(appendPath(endpoint, '/oidc').toString()), adminConsoleUrl: appendPath(endpoint, '/console'), + connectorDirectory: getEnv('CONNECTOR_DIRECTORY', defaultConnectorDirectory), }); }; @@ -73,6 +79,7 @@ function createEnvSet() { load: async () => { values = await loadEnvValues(); pool = await createPoolByEnv(values.isTest); + await addConnectors(values.connectorDirectory); }, }; } diff --git a/packages/integration-tests/tests/api/connector.test.ts b/packages/integration-tests/tests/api/connector.test.ts index ab60ecdc6..47f0030fb 100644 --- a/packages/integration-tests/tests/api/connector.test.ts +++ b/packages/integration-tests/tests/api/connector.test.ts @@ -2,10 +2,6 @@ import { ConnectorType } from '@logto/schemas'; import { HTTPError } from 'got'; import { - aliyunEmailConnectorConfig, - aliyunEmailConnectorId, - aliyunSmsConnectorConfig, - aliyunSmsConnectorId, mockEmailConnectorConfig, mockEmailConnectorId, mockSmsConnectorConfig, @@ -34,8 +30,8 @@ test('connector set-up flow', async () => { */ await Promise.all( [ - { id: aliyunSmsConnectorId, config: aliyunSmsConnectorConfig }, - { id: aliyunEmailConnectorId, config: aliyunEmailConnectorConfig }, + { id: mockSmsConnectorId, config: mockSmsConnectorConfig }, + { id: mockEmailConnectorId, config: mockEmailConnectorConfig }, { id: mockSocialConnectorId, config: mockSocialConnectorConfig }, ].map(async ({ id, config }) => { const updatedConnector = await updateConnectorConfig(id, config); @@ -55,7 +51,7 @@ test('connector set-up flow', async () => { * We will test updating to the invalid connector config, that is the case not covered above. */ await expect( - updateConnectorConfig(mockSocialConnectorId, aliyunSmsConnectorConfig) + updateConnectorConfig(mockSocialConnectorId, mockSmsConnectorConfig) ).rejects.toThrow(HTTPError); // To confirm the failed updating request above did not modify the original config, // we check: the mock connector config should stay the same. @@ -103,13 +99,8 @@ test('connector set-up flow', async () => { expect(await listConnectors()).toEqual( expect.arrayContaining([ expect.objectContaining({ - id: mockSocialConnectorId, - config: mockSocialConnectorConfig, - enabled: true, - }), - expect.objectContaining({ - id: aliyunSmsConnectorId, - config: aliyunSmsConnectorConfig, + id: mockEmailConnectorId, + config: mockEmailConnectorConfig, enabled: false, }), expect.objectContaining({ @@ -118,14 +109,9 @@ test('connector set-up flow', async () => { enabled: true, }), expect.objectContaining({ - id: aliyunEmailConnectorId, - config: aliyunEmailConnectorConfig, - enabled: false, - }), - expect.objectContaining({ - id: mockEmailConnectorId, - config: mockEmailConnectorConfig, - enabled: false, + id: mockSocialConnectorId, + config: mockSocialConnectorConfig, + enabled: true, }), ]) ); diff --git a/packages/integration-tests/tests/api/sign-in-experience.test.ts b/packages/integration-tests/tests/api/sign-in-experience.test.ts index b8e0283f0..6540527d3 100644 --- a/packages/integration-tests/tests/api/sign-in-experience.test.ts +++ b/packages/integration-tests/tests/api/sign-in-experience.test.ts @@ -1,13 +1,13 @@ import { BrandingStyle, SignInMethodState } from '@logto/schemas'; import { + mockEmailConnectorConfig, + mockEmailConnectorId, + mockSmsConnectorConfig, + mockSmsConnectorId, mockSocialConnectorConfig, mockSocialConnectorId, mockSocialConnectorTarget, - twilioSmsConnectorConfig, - twilioSmsConnectorId, - sendgridEmailConnectorConfig, - sendgridEmailConnectorId, } from '@/__mocks__/connectors-mock'; import { getSignInExperience, updateSignInExperience } from '@/api'; import { updateConnectorConfig, enableConnector, disableConnector } from '@/api/connector'; @@ -48,11 +48,11 @@ describe('admin console sign-in experience', () => { updateConnectorConfig(mockSocialConnectorId, mockSocialConnectorConfig).then(async () => enableConnector(mockSocialConnectorId) ), - updateConnectorConfig(twilioSmsConnectorId, twilioSmsConnectorConfig).then(async () => - enableConnector(twilioSmsConnectorId) + updateConnectorConfig(mockSmsConnectorId, mockSmsConnectorConfig).then(async () => + enableConnector(mockSmsConnectorId) ), - updateConnectorConfig(sendgridEmailConnectorId, sendgridEmailConnectorConfig).then(async () => - enableConnector(sendgridEmailConnectorId) + updateConnectorConfig(mockEmailConnectorId, mockEmailConnectorConfig).then(async () => + enableConnector(mockEmailConnectorId) ), ]); @@ -74,8 +74,8 @@ describe('admin console sign-in experience', () => { // Reset connectors await Promise.all([ disableConnector(mockSocialConnectorId), - disableConnector(twilioSmsConnectorId), - disableConnector(sendgridEmailConnectorId), + disableConnector(mockSmsConnectorId), + disableConnector(mockEmailConnectorId), ]); }); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2a361ffcb..403c7e807 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -893,22 +893,7 @@ importers: packages/core: specifiers: - '@logto/connector-alipay-native': ^1.0.0-beta.8 - '@logto/connector-alipay-web': ^1.0.0-beta.8 - '@logto/connector-aliyun-dm': ^1.0.0-beta.8 - '@logto/connector-aliyun-sms': ^1.0.0-beta.8 - '@logto/connector-apple': ^1.0.0-beta.8 - '@logto/connector-azuread': ^1.0.0-beta.8 '@logto/connector-core': ^1.0.0-beta.8 - '@logto/connector-facebook': ^1.0.0-beta.8 - '@logto/connector-github': ^1.0.0-beta.8 - '@logto/connector-google': ^1.0.0-beta.8 - '@logto/connector-kakao': ^1.0.0-beta.8 - '@logto/connector-sendgrid-email': ^1.0.0-beta.8 - '@logto/connector-smtp': ^1.0.0-beta.8 - '@logto/connector-twilio-sms': ^1.0.0-beta.8 - '@logto/connector-wechat-native': ^1.0.0-beta.8 - '@logto/connector-wechat-web': ^1.0.0-beta.8 '@logto/phrases': ^1.0.0-beta.8 '@logto/schemas': ^1.0.0-beta.8 '@logto/shared': ^1.0.0-beta.8 @@ -930,7 +915,9 @@ importers: '@types/lodash.pick': ^4.4.6 '@types/node': ^16.3.1 '@types/oidc-provider': ^7.11.1 + '@types/rimraf': ^3.0.2 '@types/supertest': ^2.0.11 + '@types/tar': ^6.1.2 chalk: ^4 copyfiles: ^2.4.1 dayjs: ^1.10.5 @@ -968,7 +955,7 @@ importers: p-retry: ^4.6.1 prettier: ^2.7.1 query-string: ^7.0.1 - resolve-package-path: ^4.0.3 + rimraf: ^3.0.2 roarr: ^7.11.0 slonik: ^30.0.0 slonik-interceptor-preset: ^1.2.10 @@ -976,25 +963,11 @@ importers: snake-case: ^3.0.4 snakecase-keys: ^5.1.0 supertest: ^6.2.2 + tar: ^6.1.11 typescript: ^4.7.4 zod: ^3.14.3 dependencies: - '@logto/connector-alipay-native': link:../connector-alipay-native - '@logto/connector-alipay-web': link:../connector-alipay-web - '@logto/connector-aliyun-dm': link:../connector-aliyun-dm - '@logto/connector-aliyun-sms': link:../connector-aliyun-sms - '@logto/connector-apple': link:../connector-apple - '@logto/connector-azuread': link:../connector-azuread '@logto/connector-core': link:../connector-core - '@logto/connector-facebook': link:../connector-facebook - '@logto/connector-github': link:../connector-github - '@logto/connector-google': link:../connector-google - '@logto/connector-kakao': link:../connector-kakao - '@logto/connector-sendgrid-email': link:../connector-sendgrid-mail - '@logto/connector-smtp': link:../connector-smtp - '@logto/connector-twilio-sms': link:../connector-twilio-sms - '@logto/connector-wechat-native': link:../connector-wechat-native - '@logto/connector-wechat-web': link:../connector-wechat-web '@logto/phrases': link:../phrases '@logto/schemas': link:../schemas '@logto/shared': link:../shared @@ -1027,13 +1000,14 @@ importers: oidc-provider: 7.11.3 p-retry: 4.6.1 query-string: 7.0.1 - resolve-package-path: 4.0.3 + rimraf: 3.0.2 roarr: 7.11.0 slonik: 30.1.2 slonik-interceptor-preset: 1.2.10 slonik-sql-tag-raw: 1.1.4_roarr@7.11.0+slonik@30.1.2 snake-case: 3.0.4 snakecase-keys: 5.1.2 + tar: 6.1.11 zod: 3.14.3 devDependencies: '@shopify/jest-koa-mocks': 5.0.0 @@ -1053,7 +1027,9 @@ importers: '@types/lodash.pick': 4.4.6 '@types/node': 16.11.12 '@types/oidc-provider': 7.11.1 + '@types/rimraf': 3.0.2 '@types/supertest': 2.0.11 + '@types/tar': 6.1.2 copyfiles: 2.4.1 eslint: 8.21.0 jest: 28.1.3_@types+node@16.11.12 @@ -4962,6 +4938,13 @@ packages: '@types/node': 17.0.23 dev: false + /@types/glob/8.0.0: + resolution: {integrity: sha512-l6NQsDDyQUVeoTynNpC9uRvCUint/gSUXQA2euwmTuWGvPY5LSDUu6tkCtJB2SvGQlJQzLaKqcGZP4//7EDveA==} + dependencies: + '@types/minimatch': 3.0.5 + '@types/node': 17.0.23 + dev: true + /@types/graceful-fs/4.1.5: resolution: {integrity: sha512-anKkLmZZ+xm4p8JWBf4hElkM4XR+EZeA2M9BAkkTldmcyDY4mbdIJnRghDJH3Ov5ooY7/UAoENtmdMSkaAd7Cw==} dependencies: @@ -5259,6 +5242,13 @@ packages: resolution: {integrity: sha512-xoDlM2S4ortawSWORYqsdU+2rxdh4LRW9ytc3zmT37RIKQh6IHyKwwtKhKis9ah8ol07DCkZxPt8BBvPjC6v4g==} dev: false + /@types/rimraf/3.0.2: + resolution: {integrity: sha512-F3OznnSLAUxFrCEu/L5PY8+ny8DtcFRjx7fZZ9bycvXRi3KPTRS9HOitGZwvPg0juRhXFWIeKX58cnX5YqLohQ==} + dependencies: + '@types/glob': 8.0.0 + '@types/node': 17.0.23 + dev: true + /@types/scheduler/0.16.2: resolution: {integrity: sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==} dev: true @@ -5286,6 +5276,13 @@ packages: '@types/superagent': 4.1.15 dev: true + /@types/tar/6.1.2: + resolution: {integrity: sha512-bnX3RRm70/n1WMwmevdOAeDU4YP7f5JSubgnuU+yrO+xQQjwDboJj3u2NTJI5ngCQhXihqVVAH5h5J8YpdpEvg==} + dependencies: + '@types/node': 17.0.23 + minipass: 3.3.5 + dev: true + /@types/through/0.0.30: resolution: {integrity: sha512-FvnCJljyxhPM3gkRgWmxmDZyAQSiBQQWLI0A0VFL0K7W1oRUrPJSqNO0NvTnLkBcotdlp3lKvaT0JrnyRDkzOg==} dependencies: @@ -6254,7 +6251,6 @@ packages: /chownr/2.0.0: resolution: {integrity: sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==} engines: {node: '>=10'} - dev: true /chrome-trace-event/1.0.3: resolution: {integrity: sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg==} @@ -6612,8 +6608,8 @@ packages: engines: {node: '>=10'} hasBin: true dependencies: - JSONStream: 1.3.5 is-text-path: 1.0.1 + JSONStream: 1.3.5 lodash: 4.17.21 meow: 8.1.2 split2: 3.2.2 @@ -8215,11 +8211,10 @@ packages: resolution: {integrity: sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==} engines: {node: '>= 8'} dependencies: - minipass: 3.1.6 - dev: true + minipass: 3.3.5 /fs.realpath/1.0.0: - resolution: {integrity: sha1-FQStJSMVjKpA20onh8sBQRmU6k8=} + resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} /fsevents/2.3.2: resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} @@ -9086,13 +9081,13 @@ packages: dev: false /inflight/1.0.6: - resolution: {integrity: sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=} + resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} dependencies: once: 1.4.0 wrappy: 1.0.2 /inherits/2.0.1: - resolution: {integrity: sha1-sX0I0ya0Qj5Wjv9xn5GwscvfafE=} + resolution: {integrity: sha512-8nWq2nLTAwd02jTqJExUYFSD/fKq6VH9Y/oG2accc/kdI0V98Bag8d5a4gi3XHz73rDWa2PvTtvcWYquKqSENA==} dev: false /inherits/2.0.3: @@ -11776,15 +11771,19 @@ packages: engines: {node: '>=8'} dependencies: yallist: 4.0.0 - dev: true + + /minipass/3.3.5: + resolution: {integrity: sha512-rQ/p+KfKBkeNwo04U15i+hOwoVBVmekmm/HcfTkTN2t9pbQKCMm4eN5gFeqgrrSp/kH/7BYYhTIHOxGqzbBPaA==} + engines: {node: '>=8'} + dependencies: + yallist: 4.0.0 /minizlib/2.1.2: resolution: {integrity: sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==} engines: {node: '>= 8'} dependencies: - minipass: 3.1.6 + minipass: 3.3.5 yallist: 4.0.0 - dev: true /mixin-object/2.0.1: resolution: {integrity: sha512-ALGF1Jt9ouehcaXaHhn6t1yGWRqGaHkPFndtFVHfZXOvkIZ/yoGaSi0AHVTafb3ZBGg4dr/bDwnaEKqCXzchMA==} @@ -11811,7 +11810,6 @@ packages: resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==} engines: {node: '>=10'} hasBin: true - dev: true /modify-values/1.0.1: resolution: {integrity: sha512-xV2bxeN6F7oYjZWTe/YPAy6MN2M+sL4u/Rlm2AHCIVGfo2p1yGmBHQ6vHehl4bRTZBdHu3TSkWdYgkwpYzAGSw==} @@ -12409,7 +12407,7 @@ packages: ee-first: 1.1.1 /once/1.4.0: - resolution: {integrity: sha1-WDsap3WWHUsROsF9nFC6753Xa9E=} + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} dependencies: wrappy: 1.0.2 @@ -12821,7 +12819,7 @@ packages: engines: {node: '>=8'} /path-is-absolute/1.0.1: - resolution: {integrity: sha1-F0uSaHNVNP+8es5r9TpanhtcX18=} + resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} engines: {node: '>=0.10.0'} /path-key/3.1.1: @@ -12843,18 +12841,6 @@ packages: /path-parse/1.0.7: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} - /path-root-regex/0.1.2: - resolution: {integrity: sha1-v8zcjfWxLcUsi0PsONGNcsBLqW0=} - engines: {node: '>=0.10.0'} - dev: false - - /path-root/0.1.1: - resolution: {integrity: sha1-mkpoFMrBwM1zNgqV8yCDyOpHRbc=} - engines: {node: '>=0.10.0'} - dependencies: - path-root-regex: 0.1.2 - dev: false - /path-to-regexp/1.8.0: resolution: {integrity: sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==} dependencies: @@ -14213,13 +14199,6 @@ packages: global-dirs: 0.1.1 dev: true - /resolve-package-path/4.0.3: - resolution: {integrity: sha512-SRpNAPW4kewOaNUt8VPqhJ0UMxawMwzJD8V7m1cJfdSTK9ieZwS6K7Dabsm4bmLFM96Z5Y/UznrpG5kt1im8yA==} - engines: {node: '>= 12'} - dependencies: - path-root: 0.1.1 - dev: false - /resolve-path/1.4.0: resolution: {integrity: sha1-xL2p9e+y/OZSR4c6s2u02DT+Fvc=} engines: {node: '>= 0.8'} @@ -15320,7 +15299,6 @@ packages: minizlib: 2.1.2 mkdirp: 1.0.4 yallist: 4.0.0 - dev: true /temp-dir/1.0.0: resolution: {integrity: sha1-CnwOom06Oa+n4OvqnB/AvE2qAR0=} @@ -16233,7 +16211,7 @@ packages: strip-ansi: 6.0.1 /wrappy/1.0.2: - resolution: {integrity: sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=} + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} /write-file-atomic/2.4.3: resolution: {integrity: sha512-GaETH5wwsX+GcnzhPgKcKjJ6M2Cq3/iZp1WyY/X1CSqrW+jVNM9Y7D8EC2sM4ZG/V8wZlSniJnCKWPmBYAucRQ==}