Merge pull request #3659 from logto-io/gao-use-local-connectors

refactor: use local connectors

.changeset/fifty-dolls-divide.md

@@ -0,0 +1,5 @@
+---
+"@logto/cli": minor
+---
+
+add `logto connector link` command to link local connectors without downloading from the remote

.devcontainer/devcontainer.json

@@ -3,7 +3,7 @@
   "features": {
     "ghcr.io/devcontainers/features/node:1": {}
   },
-  "updateContentCommand": "npm i -g pnpm && pnpm i && pnpm prepack && pnpm cli connector add --official -p .",
+  "updateContentCommand": "npm i -g pnpm && pnpm i && pnpm prepack && pnpm connectors:build && pnpm cli connector link -p .",
   "postStartCommand": "docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=p0stgr3s postgres:14-alpine",
   "postAttachCommand": "pnpm cli db seed && [[ ! -z $CODESPACES  ]] && export ENDPOINT=https://$CODESPACE_NAME-3001.preview.app.github.dev",
   "containerEnv": {

.github/CONTRIBUTING.md

@@ -101,7 +101,9 @@ If you are developing something with database alterations, see [packages/schemas
 
 ### Add connectors (optional)
 
-Run `pnpm cli connector add --official -p .` to add all Logto official connectors. See [Manage connectors](https://docs.logto.io/docs/tutorials/using-cli/manage-connectors) for details about managing connectors via CLI.
+Run `logto connector link -p .` to link all local connectors. You can also use `logto connector add <name> -p .` to install connector from NPM.
+
+See [Manage connectors](https://docs.logto.io/docs/tutorials/using-cli/manage-connectors) for details about managing connectors via CLI.
 
 ## Start dev
 

.github/workflows/integration-test.yml

@@ -107,7 +107,7 @@ jobs:
         working-directory: tests
         run: |
           npm run cli connector list -- -p ../logto | grep OFFICIAL
-          npm run cli connector add @logto/connector-mock-sms @logto/connector-mock-email @logto/connector-mock-standard-email @logto/connector-mock-social -- -p ../logto
+          npm run cli connector link -- --mock -p ../logto
 
       - name: Run Logto
         working-directory: logto/

.gitpod.yml

@@ -12,7 +12,8 @@ tasks:
       cd packages/core
       pnpm build
       cd -
-      pnpm cli connector add --official -p .
+      pnpm connectors:build
+      pnpm cli connector link
     command: |
       gp ports await 5432
       sleep 3

.scripts/package.sh

@@ -26,7 +26,7 @@ packages/**/*.config.js packages/**/*.config.ts packages/**/tsconfig*.json \
 
 # Add official connectors
 cloud_option=$( [[ "$IS_CLOUD" =~ ^(1|true|y|yes|yep|yeah)$ ]] && echo "--cloud" || echo "" )
-pnpm cli connector add --official $cloud_option -p .
+pnpm cli connector link $cloud_option -p .
 
 echo Tar
 cd ..

Dockerfile

@@ -21,7 +21,7 @@ RUN pnpm -r build
 ### Add official connectors ###
 ARG additional_connector_args
 ENV ADDITIONAL_CONNECTOR_ARGS=${additional_connector_args}
-RUN pnpm cli connector add --official $ADDITIONAL_CONNECTOR_ARGS -p .
+RUN pnpm cli connector link $ADDITIONAL_CONNECTOR_ARGS -p .
 
 ### Prune dependencies for production ###
 RUN rm -rf node_modules packages/**/node_modules

Dockerfile.cloud

@@ -24,10 +24,10 @@ ENV IS_CLOUD=1
 ARG applicationinsights_connection_string
 ENV APPLICATIONINSIGHTS_CONNECTION_STRING=${applicationinsights_connection_string}
 RUN pnpm prepack
-RUN pnpm -r --filter @logto/console --filter @logto/cloud build
+RUN pnpm connectors:build && pnpm -r --filter @logto/console --filter @logto/cloud build
 
 ### Add official connectors ###
-RUN pnpm cli connector add --official -p .
+RUN pnpm cli connector link -p .
 
 ### Prune dependencies for production ###
 RUN rm -rf node_modules packages/**/node_modules

package.json

@@ -10,12 +10,13 @@
     "prepack": "pnpm -r prepack",
     "dev": "pnpm -r prepack && pnpm start:dev",
     "dev:cloud": "pnpm -r prepack && pnpm start:dev:cloud",
-    "start:dev": "pnpm -r --parallel --filter=!@logto/integration-tests --filter=!@logto/cloud dev",
-    "start:dev:cloud": "CONSOLE_PUBLIC_URL=/ IS_CLOUD=1 pnpm -r --parallel --filter=!@logto/integration-tests dev",
+    "start:dev": "pnpm -r --parallel --filter=!@logto/integration-tests --filter \"!./packages/connectors/connector-*\" --filter=!@logto/cloud dev",
+    "start:dev:cloud": "CONSOLE_PUBLIC_URL=/ IS_CLOUD=1 pnpm -r --parallel --filter=!@logto/integration-tests --filter \"!./packages/connectors/connector-*\" dev",
     "start": "cd packages/core && NODE_ENV=production node .",
     "start:cloud": "cd packages/cloud && NODE_ENV=production node .",
     "cli": "logto",
     "alteration": "logto db alt",
+    "connectors:build": "pnpm -r --filter \"./packages/connectors/connector-*\" build",
     "//": "# `changeset version` won't run version lifecycle scripts, see https://github.com/changesets/changesets/issues/860",
     "ci:version": "changeset version && pnpm -r version",
     "ci:build": "pnpm -r build",

packages/cli/src/commands/connector/index.ts

@@ -2,6 +2,7 @@ import { noop } from '@silverhand/essentials';
 import type { CommandModule } from 'yargs';
 
 import add from './add.js';
+import link from './link.js';
 import list from './list.js';
 import remove from './remove.js';
 
@@ -18,6 +19,7 @@ const connector: CommandModule = {
       .command(add)
       .command(list)
       .command(remove)
+      .command(link)
       .demandCommand(1),
   handler: noop,
 };

packages/cli/src/commands/connector/link.ts

@@ -0,0 +1,61 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+import type { CommandModule } from 'yargs';
+
+import { log } from '../../utils.js';
+
+import { getConnectorDirectory, getLocalConnectorPackages, inquireInstancePath } from './utils.js';
+
+const link: CommandModule<{ path?: string }, { path?: string; cloud: boolean; mock: boolean }> = {
+  command: ['link', 'ln'],
+  describe: 'Link all connectors in `packages/connectors`, useful for adding local connectors.',
+  builder: (yargs) =>
+    yargs
+      .option('cloud', {
+        describe: 'Add additional connectors for Logto Cloud',
+        type: 'boolean',
+        default: false,
+        hidden: true,
+      })
+      .option('mock', {
+        describe: 'Add mock connectors',
+        type: 'boolean',
+        default: false,
+        hidden: true,
+      }),
+  handler: async ({ path: inputPath, cloud, mock }) => {
+    const instancePath = await inquireInstancePath(inputPath);
+    const packages = await getLocalConnectorPackages(instancePath);
+    const connectorDirectory = getConnectorDirectory(instancePath);
+
+    await fs.mkdir(connectorDirectory, { recursive: true });
+
+    await Promise.all(
+      packages.map(async ([packageName, packagePath]) => {
+        if (!cloud && packageName.startsWith('connector-logto-')) {
+          return;
+        }
+
+        if (!mock && packageName.startsWith('connector-mock-')) {
+          return;
+        }
+
+        try {
+          const targetPath = path.join(connectorDirectory, '@logto-' + packageName);
+
+          await fs.rm(targetPath, { recursive: true, force: true });
+          await fs.symlink(path.relative(connectorDirectory, packagePath), targetPath);
+        } catch (error) {
+          log.warn(error);
+
+          return;
+        }
+
+        log.succeed('Linked', packagePath);
+      })
+    );
+  },
+};
+
+export default link;

packages/cli/src/commands/connector/utils.ts

@@ -97,7 +97,7 @@ export const normalizePackageName = (name: string) =>
     )
     .join('/');
 
-const getConnectorDirectory = (instancePath: string) =>
+export const getConnectorDirectory = (instancePath: string) =>
   path.join(instancePath, coreDirectory, connectorDirectory);
 
 export const isOfficialConnector = (packageName: string) =>
@@ -108,12 +108,26 @@ export const getConnectorPackagesFrom = async (instancePath?: string) => {
   const packages = await trySafe(getConnectorPackagesFromDirectory(directory));
 
   if (!packages || packages.length === 0) {
-    throw new Error('No connector found');
+    throw new Error('No connector found in ' + directory);
   }
 
   return packages;
 };
 
+export const getLocalConnectorPackages = async (instancePath: string) => {
+  const directory = path.join(instancePath, 'packages', connectorDirectory);
+  const files = await fs.readdir(directory, { withFileTypes: true });
+  const packages = files.filter(
+    (entity) => entity.isDirectory() && entity.name.startsWith('connector-')
+  );
+
+  if (packages.length === 0) {
+    throw new Error('No connector found in ' + directory);
+  }
+
+  return packages.map(({ name }) => [name, path.join(directory, name)] as const);
+};
+
 export const addConnectors = async (instancePath: string, packageNames: string[]) => {
   const cwd = getConnectorDirectory(instancePath);
 

packages/cli/src/commands/database/system.ts

@@ -60,7 +60,7 @@ const setConfig: CommandModule<unknown, { key: string; value: string }> = {
         demandOption: true,
       })
       .positional('value', {
-        describe: 'The value to set, should be a valid JSON string',
+        describe: 'The value to set, should be a valid JSON string.',
         type: 'string',
         demandOption: true,
       }),

packages/cli/src/commands/install/index.ts

@@ -87,7 +87,7 @@ const install: CommandModule<
       },
       du: {
         alias: 'download-url',
-        describe: 'URL for downloading Logto, can be a local path to tar',
+        describe: 'URL for downloading Logto, can be a local path to tar.',
         type: 'string',
         hidden: true,
       },

packages/cli/src/connector/loader.ts

@@ -14,7 +14,7 @@ const checkConnectorKitVersion = (dependencies: unknown, ignoreVersionMismatch:
     const value = dependencies[connectorKit];
 
     if (typeof value === 'string') {
-      if (satisfies(currentVersion, value)) {
+      if (value.startsWith('workspace:') || satisfies(currentVersion, value)) {
         return;
       }