Merge pull request #2078 from logto-io/gao-log-4330-core-read-oidc-keys-and-config-from

refactor(core): read OIDC configs from database

packages/core/jest.config.ts

@@ -3,7 +3,6 @@ import { merge, Config } from '@silverhand/jest-config';
 const config: Config.InitialOptions = merge({
   testPathIgnorePatterns: ['/core/connectors/'],
   setupFilesAfterEnv: ['jest-matcher-specific-error', './jest.setup.ts'],
-  globalSetup: './jest.global-setup.ts',
 });
 
 export default config;

packages/core/jest.global-setup.ts

@@ -1,25 +0,0 @@
-/**
- * Generate private key for tests
- */
-import { generateKeyPairSync } from 'crypto';
-import { writeFileSync } from 'fs';
-
-export const privateKeyPath = 'oidc-private-key.test.pem';
-
-const globalSetup = () => {
-  const { privateKey } = generateKeyPairSync('rsa', {
-    modulusLength: 4096,
-    publicKeyEncoding: {
-      type: 'spki',
-      format: 'pem',
-    },
-    privateKeyEncoding: {
-      type: 'pkcs8',
-      format: 'pem',
-    },
-  });
-
-  writeFileSync(privateKeyPath, privateKey);
-};
-
-export default globalSetup;

packages/core/jest.setup.ts

@@ -4,14 +4,10 @@
 
 import envSet from '@/env-set';
 
-import { privateKeyPath } from './jest.global-setup';
+jest.mock('@/lib/logto-config');
+jest.mock('@/env-set/check-alteration-state');
 
 // eslint-disable-next-line unicorn/prefer-top-level-await
 (async () => {
-  process.env = {
-    ...process.env,
-    OIDC_PRIVATE_KEY_PATHS: privateKeyPath,
-    OIDC_COOKIE_KEYS: '["LOGTOSEKRIT1"]',
-  };
   await envSet.load();
 })();

packages/core/src/env-set/__mocks__/check-alteration-state.ts

@@ -0,0 +1,2 @@
+// eslint-disable-next-line @typescript-eslint/no-empty-function
+export const checkAlterationState = async () => {};

packages/core/src/env-set/create-pool-by-env.ts

@@ -1,12 +1,12 @@
 import { assertEnv } from '@silverhand/essentials';
 import chalk from 'chalk';
-import { createPool } from 'slonik';
+import { createMockPool, createMockQueryResult, createPool } from 'slonik';
 import { createInterceptors } from 'slonik-interceptor-preset';
 
 const createPoolByEnv = async (isTest: boolean) => {
   // Database connection is disabled in unit test environment
   if (isTest) {
-    return;
+    return createMockPool({ query: async () => createMockQueryResult([]) });
   }
 
   const key = 'DB_URL';

packages/core/src/env-set/index.ts

@@ -3,6 +3,7 @@ import path from 'path';
 import { getEnv, getEnvAsStringArray, Optional } from '@silverhand/essentials';
 import { DatabasePool } from 'slonik';
 
+import { getOidcConfigs } from '@/lib/logto-config';
 import { appendPath } from '@/utils/url';
 
 import { addConnectors } from './add-connectors';
@@ -44,7 +45,6 @@ const loadEnvValues = async () => {
     userDefaultRoleNames: getEnvAsStringArray('USER_DEFAULT_ROLE_NAMES'),
     developmentUserId: getEnv('DEVELOPMENT_USER_ID'),
     trustProxyHeader: isTrue(getEnv('TRUST_PROXY_HEADER')),
-    oidc: await loadOidcValues(appendPath(endpoint, '/oidc').toString()),
     adminConsoleUrl: appendPath(endpoint, '/console'),
     connectorDirectory: getEnv('CONNECTOR_DIRECTORY', defaultConnectorDirectory),
   });
@@ -60,6 +60,7 @@ const throwNotLoadedError = () => {
 function createEnvSet() {
   let values: Optional<Awaited<ReturnType<typeof loadEnvValues>>>;
   let pool: Optional<DatabasePool>;
+  let oidc: Optional<Awaited<ReturnType<typeof loadOidcValues>>>;
 
   return {
     get values() {
@@ -79,14 +80,21 @@ function createEnvSet() {
     get poolSafe() {
       return pool;
     },
+    get oidc() {
+      if (!oidc) {
+        return throwNotLoadedError();
+      }
+
+      return oidc;
+    },
     load: async () => {
       values = await loadEnvValues();
       pool = await createPoolByEnv(values.isTest);
-      await addConnectors(values.connectorDirectory);
 
-      if (pool) {
+      const [, oidcConfigs] = await Promise.all([checkAlterationState(pool), getOidcConfigs(pool)]);
-        await checkAlterationState(pool);
+      oidc = await loadOidcValues(appendPath(values.endpoint, '/oidc').toString(), oidcConfigs);
-      }
+
+      await addConnectors(values.connectorDirectory);
     },
   };
 }

packages/core/src/env-set/oidc.test.ts

@@ -1,123 +0,0 @@
-import crypto from 'crypto';
-import fs, { PathOrFileDescriptor } from 'fs';
-
-import inquirer from 'inquirer';
-
-import { readCookieKeys, readPrivateKeys } from './oidc';
-
-describe('oidc env-set', () => {
-  const envBackup = process.env;
-
-  beforeEach(() => {
-    process.env = { ...envBackup };
-  });
-
-  afterEach(() => {
-    jest.clearAllMocks();
-    jest.resetModules();
-  });
-
-  it('should read OIDC private keys if raw `OIDC_PRIVATE_KEYS` is provided', async () => {
-    const rawKeys = [
-      '-----BEGIN PRIVATE KEY-----\nFOO\n-----END PRIVATE KEY-----',
-      '-----BEGIN PRIVATE KEY-----\nBAR\n-----END PRIVATE KEY-----',
-    ];
-    process.env.OIDC_PRIVATE_KEYS = rawKeys.join(',');
-
-    const privateKeys = await readPrivateKeys();
-
-    expect(privateKeys).toEqual([
-      '-----BEGIN PRIVATE KEY-----\nFOO\n-----END PRIVATE KEY-----',
-      '-----BEGIN PRIVATE KEY-----\nBAR\n-----END PRIVATE KEY-----',
-    ]);
-  });
-
-  it('should transpile and read OIDC private keys if base64-formatted `OIDC_PRIVATE_KEYS` is provided', async () => {
-    const base64Keys = ['foo', 'bar'].map((key) => Buffer.from(key, 'utf8').toString('base64'));
-    process.env.OIDC_PRIVATE_KEYS = base64Keys.join(',');
-
-    const privateKeys = await readPrivateKeys();
-
-    expect(privateKeys).toEqual(['foo', 'bar']);
-  });
-
-  it('should read OIDC private keys if `OIDC_PRIVATE_KEY_PATHS` is provided', async () => {
-    process.env.OIDC_PRIVATE_KEY_PATHS = 'foo.pem, bar.pem';
-    const existsSyncSpy = jest.spyOn(fs, 'existsSync').mockReturnValue(true);
-    const readFileSyncSpy = jest
-      .spyOn(fs, 'readFileSync')
-      .mockImplementation((path: PathOrFileDescriptor) => path.toString());
-
-    const privateKeys = await readPrivateKeys();
-
-    expect(privateKeys).toEqual(['foo.pem', 'bar.pem']);
-
-    existsSyncSpy.mockRestore();
-    readFileSyncSpy.mockRestore();
-  });
-
-  it('should generate a default OIDC private key if neither `OIDC_PRIVATE_KEY_PATHS` nor `OIDC_PRIVATE_KEYS` is provided', async () => {
-    process.env.OIDC_PRIVATE_KEYS = '';
-    process.env.OIDC_PRIVATE_KEY_PATHS = '';
-
-    const readFileSyncSpy = jest.spyOn(fs, 'readFileSync').mockImplementation(() => {
-      throw new Error('Dummy read file error');
-    });
-
-    // eslint-disable-next-line @typescript-eslint/no-empty-function
-    const writeFileSyncSpy = jest.spyOn(fs, 'writeFileSync').mockImplementation(() => {});
-
-    const promptMock = jest.spyOn(inquirer, 'prompt').mockResolvedValue({ confirm: true });
-
-    const generateKeyPairSyncSpy = jest.spyOn(crypto, 'generateKeyPairSync');
-
-    const privateKeys = await readPrivateKeys();
-
-    expect(privateKeys.length).toBe(1);
-    expect(generateKeyPairSyncSpy).toHaveBeenCalled();
-    expect(writeFileSyncSpy).toHaveBeenCalled();
-
-    readFileSyncSpy.mockRestore();
-    promptMock.mockRestore();
-    generateKeyPairSyncSpy.mockRestore();
-    writeFileSyncSpy.mockRestore();
-  });
-
-  it('should throw if private keys configured in `OIDC_PRIVATE_KEY_PATHS` are not found', async () => {
-    process.env.OIDC_PRIVATE_KEY_PATHS = 'foo.pem, bar.pem, baz.pem';
-    const existsSyncSpy = jest.spyOn(fs, 'existsSync').mockReturnValue(false);
-
-    await expect(readPrivateKeys()).rejects.toMatchError(
-      new Error(
-        `Private keys foo.pem, bar.pem, and baz.pem configured in env \`OIDC_PRIVATE_KEY_PATHS\` not found.`
-      )
-    );
-
-    existsSyncSpy.mockRestore();
-  });
-
-  it('should read OIDC cookie keys if `OIDC_COOKIE_KEYS` is provided', async () => {
-    process.env.OIDC_COOKIE_KEYS = 'foo, bar';
-
-    const cookieKeys = await readCookieKeys();
-
-    expect(cookieKeys).toEqual(['foo', 'bar']);
-  });
-
-  it('should generate a default OIDC cookie key if `OIDC_COOKIE_KEYS` is not provided', async () => {
-    process.env.OIDC_COOKIE_KEYS = '';
-
-    const promptMock = jest.spyOn(inquirer, 'prompt').mockResolvedValue({ confirm: true });
-
-    // eslint-disable-next-line @typescript-eslint/no-empty-function
-    const appendFileSyncSpy = jest.spyOn(fs, 'appendFileSync').mockImplementation(() => {});
-
-    const cookieKeys = await readCookieKeys();
-
-    expect(cookieKeys.length).toBe(1);
-    expect(appendFileSyncSpy).toHaveBeenCalled();
-
-    promptMock.mockRestore();
-    appendFileSyncSpy.mockRestore();
-  });
-});

packages/core/src/env-set/oidc.ts

@@ -1,168 +1,27 @@
-import crypto, { generateKeyPairSync } from 'crypto';
+import crypto from 'crypto';
-import { existsSync, readFileSync, writeFileSync } from 'fs';
 
-import { getEnv, getEnvAsStringArray } from '@silverhand/essentials';
+import { LogtoOidcConfigKey, LogtoOidcConfigType } from '@logto/schemas';
-import inquirer from 'inquirer';
 import { createLocalJWKSet } from 'jose';
-import { nanoid } from 'nanoid';
 
 import { exportJWK } from '@/utils/jwks';
 
-import { appendDotEnv } from './dot-env';
+const loadOidcValues = async (issuer: string, configs: LogtoOidcConfigType) => {
-import { allYes, noInquiry } from './parameters';
+  const cookieKeys = configs[LogtoOidcConfigKey.CookieKeys];
-
+  const privateKeys = configs[LogtoOidcConfigKey.PrivateKeys].map((key) =>
-const defaultLogtoOidcPrivateKeyPath = './oidc-private-key.pem';
+    crypto.createPrivateKey(key)
-
-const listFormatter = new Intl.ListFormat('en', { style: 'long', type: 'conjunction' });
-
-const isBase64FormatPrivateKey = (key: string) => !key.includes('-');
-
-/**
- * Try to read private keys with the following order:
- *
- * 1. From `process.env.OIDC_PRIVATE_KEYS`.
- * 2. Fetch path from `process.env.OIDC_PRIVATE_KEY_PATHS` then read from that path.
- *
- * If none of above succeed, then inquire user to generate a new key if no `--no-inquiry` presents in argv.
- *
- * @returns The private keys for OIDC provider.
- * @throws An error when failed to read a private key.
- */
-export const readPrivateKeys = async (): Promise<string[]> => {
-  const privateKeys = getEnvAsStringArray('OIDC_PRIVATE_KEYS');
-
-  if (privateKeys.length > 0) {
-    return privateKeys.map((key) => {
-      if (isBase64FormatPrivateKey(key)) {
-        return Buffer.from(key, 'base64').toString('utf8');
-      }
-
-      return key;
-    });
-  }
-
-  const privateKeyPaths = getEnvAsStringArray('OIDC_PRIVATE_KEY_PATHS');
-
-  /**
-   * If neither `OIDC_PRIVATE_KEYS` nor `OIDC_PRIVATE_KEY_PATHS` is provided:
-   *
-   * 1. Try to read the private key from `defaultLogtoOidcPrivateKeyPath`
-   * 2. If the `defaultLogtoOidcPrivateKeyPath` doesn't exist, then ask user to generate a new key.
-   */
-  if (privateKeyPaths.length === 0) {
-    try {
-      return [readFileSync(defaultLogtoOidcPrivateKeyPath, 'utf8')];
-    } catch (error: unknown) {
-      if (noInquiry) {
-        throw error;
-      }
-
-      if (!allYes) {
-        const answer = await inquirer.prompt({
-          type: 'confirm',
-          name: 'confirm',
-          message: `No private key found in env \`OIDC_PRIVATE_KEYS\` nor \`${defaultLogtoOidcPrivateKeyPath}\`, would you like to generate a new one?`,
-        });
-
-        if (!answer.confirm) {
-          throw error;
-        }
-      }
-
-      const { privateKey } = generateKeyPairSync('rsa', {
-        modulusLength: 4096,
-        publicKeyEncoding: {
-          type: 'spki',
-          format: 'pem',
-        },
-        privateKeyEncoding: {
-          type: 'pkcs8',
-          format: 'pem',
-        },
-      });
-      writeFileSync(defaultLogtoOidcPrivateKeyPath, privateKey);
-
-      return [privateKey];
-    }
-  }
-
-  const nonExistentPrivateKeys = privateKeyPaths.filter((path): boolean => !existsSync(path));
-
-  if (nonExistentPrivateKeys.length > 0) {
-    throw new Error(
-      `Private keys ${listFormatter.format(
-        nonExistentPrivateKeys
-      )} configured in env \`OIDC_PRIVATE_KEY_PATHS\` not found.`
-    );
-  }
-
-  return privateKeyPaths.map((path): string => readFileSync(path, 'utf8'));
-};
-
-/**
- * Try to read the [signing cookie keys](https://github.com/panva/node-oidc-provider/blob/main/docs/README.md#cookieskeys) from env.
- *
- * If failed, then inquire user to generate a new keys array if no `--no-inquiry` presents in argv.
- *
- * @returns The cookie keys in array.
- */
-export const readCookieKeys = async (): Promise<string[]> => {
-  const envKey = 'OIDC_COOKIE_KEYS';
-  const keys = getEnvAsStringArray(envKey);
-
-  if (keys.length > 0) {
-    return keys;
-  }
-
-  const cookieKeysMissingError = new Error(
-    `The OIDC cookie keys are not found. Please check the value of env \`${envKey}\`.`
   );
-
-  if (noInquiry) {
-    throw cookieKeysMissingError;
-  }
-
-  if (!allYes) {
-    const answer = await inquirer.prompt({
-      type: 'confirm',
-      name: 'confirm',
-      message: `No cookie keys array found in env \`${envKey}\`, would you like to generate a new one?`,
-    });
-
-    if (!answer.confirm) {
-      throw cookieKeysMissingError;
-    }
-  }
-
-  const generated = nanoid();
-  appendDotEnv(envKey, generated);
-
-  return [generated];
-};
-
-const loadOidcValues = async (issuer: string) => {
-  const cookieKeys = await readCookieKeys();
-
-  const configPrivateKeys = await readPrivateKeys();
-  const privateKeys = configPrivateKeys.map((key) => crypto.createPrivateKey(key));
   const publicKeys = privateKeys.map((key) => crypto.createPublicKey(key));
   const privateJwks = await Promise.all(privateKeys.map(async (key) => exportJWK(key)));
   const publicJwks = await Promise.all(publicKeys.map(async (key) => exportJWK(key)));
   const localJWKSet = createLocalJWKSet({ keys: publicJwks });
-
+  const refreshTokenReuseInterval = configs[LogtoOidcConfigKey.RefreshTokenReuseInterval];
-  /**
-   * This interval helps to avoid concurrency issues when exchanging the rotating refresh token multiple times within a given timeframe.
-   * During the leeway window (in seconds), the consumed refresh token will be considered as valid.
-   * This is useful for distributed apps and serverless apps like Next.js, in which there is no shared memory.
-   */
-  const refreshTokenReuseInterval = getEnv('OIDC_REFRESH_TOKEN_REUSE_INTERVAL', '3');
 
   return Object.freeze({
     cookieKeys,
     privateJwks,
     localJWKSet,
     issuer,
-    refreshTokenReuseInterval: Number(refreshTokenReuseInterval),
+    refreshTokenReuseInterval,
     defaultIdTokenTtl: 60 * 60,
     defaultRefreshTokenTtl: 14 * 24 * 60 * 60,
   });

packages/core/src/lib/__mocks__/logto-config.ts

@@ -0,0 +1,21 @@
+import { generateKeyPairSync } from 'crypto';
+
+import { LogtoOidcConfigKey, LogtoOidcConfigType } from '@logto/schemas';
+
+const { privateKey } = generateKeyPairSync('rsa', {
+  modulusLength: 4096,
+  publicKeyEncoding: {
+    type: 'spki',
+    format: 'pem',
+  },
+  privateKeyEncoding: {
+    type: 'pkcs8',
+    format: 'pem',
+  },
+});
+
+export const getOidcConfigs = async (): Promise<LogtoOidcConfigType> => ({
+  [LogtoOidcConfigKey.PrivateKeys]: [privateKey],
+  [LogtoOidcConfigKey.CookieKeys]: ['LOGTOSEKRIT1'],
+  [LogtoOidcConfigKey.RefreshTokenReuseInterval]: 3,
+});

packages/core/src/lib/logto-config.ts

@@ -0,0 +1,35 @@
+import { getRowsByKeys } from '@logto/cli/lib/queries/logto-config';
+import { logtoOidcConfigGuard, LogtoOidcConfigKey, LogtoOidcConfigType } from '@logto/schemas';
+import chalk from 'chalk';
+import { DatabasePool, DatabaseTransactionConnection } from 'slonik';
+import { z, ZodError } from 'zod';
+
+export const getOidcConfigs = async (
+  pool: DatabasePool | DatabaseTransactionConnection
+): Promise<LogtoOidcConfigType> => {
+  try {
+    const { rows } = await getRowsByKeys(pool, Object.values(LogtoOidcConfigKey));
+
+    return z
+      .object(logtoOidcConfigGuard)
+      .parse(Object.fromEntries(rows.map(({ key, value }) => [key, value])));
+  } catch (error: unknown) {
+    if (error instanceof ZodError) {
+      console.error(
+        error.issues
+          .map(({ message, path }) => `${message} at ${chalk.green(path.join('.'))}`)
+          .join('\n')
+      );
+    } else {
+      console.error(error);
+    }
+
+    console.error(
+      `\n${chalk.red('[error]')} Failed to get OIDC configs from your Logto database.` +
+        ' Did you forget to seed your database?\n\n' +
+        `  Use ${chalk.green('npm run cli db seed')} to seed your Logto database;\n` +
+        `  Or use ${chalk.green('npm run cli db seed oidc')} to seed OIDC configs alone.\n`
+    );
+    throw new Error('Failed to get configs');
+  }
+};

packages/core/src/middleware/koa-auth.ts

@@ -49,7 +49,7 @@ export const verifyBearerTokenFromRequest = async (
   request: Request,
   resourceIndicator = managementResource.indicator
 ): Promise<TokenInfo> => {
-  const { isProduction, isIntegrationTest, developmentUserId, oidc } = envSet.values;
+  const { isProduction, isIntegrationTest, developmentUserId } = envSet.values;
   const userId = request.headers['development-user-id']?.toString() ?? developmentUserId;
 
   if ((!isProduction || isIntegrationTest) && userId) {
@@ -57,7 +57,7 @@ export const verifyBearerTokenFromRequest = async (
   }
 
   try {
-    const { localJWKSet, issuer } = oidc;
+    const { localJWKSet, issuer } = envSet.oidc;
     const {
       payload: { sub, client_id: clientId, role_names: roleNames },
     } = await jwtVerify(extractBearerTokenFromHeaders(request.headers), localJWKSet, {

packages/core/src/oidc/init.ts

@@ -23,7 +23,7 @@ import { claimToUserKey, getUserClaims } from './scope';
 
 export default async function initOidc(app: Koa): Promise<Provider> {
   const { issuer, cookieKeys, privateJwks, defaultIdTokenTtl, defaultRefreshTokenTtl } =
-    envSet.values.oidc;
+    envSet.oidc;
   const logoutSource = readFileSync('static/html/logout.html', 'utf8');
 
   const cookieConfig = Object.freeze({

packages/core/src/queries/oidc-model-instance.ts

@@ -22,7 +22,7 @@ const isConsumed = (modelName: string, consumedAt: Nullable<number>): boolean =>
     return false;
   }
 
-  const { refreshTokenReuseInterval } = envSet.values.oidc;
+  const { refreshTokenReuseInterval } = envSet.oidc;
 
   if (modelName !== 'RefreshToken' || !refreshTokenReuseInterval) {
     return Boolean(consumedAt);

packages/core/tsconfig.build.json

@@ -5,6 +5,6 @@
   ],
   "exclude": [
     "src/**/*.test.ts",
-    "src/__mocks__/",
+    "src/**/__mocks__/",
   ]
 }

packages/schemas/src/types/logto-config.ts

@@ -11,7 +11,7 @@ export type AlterationStateType = {
   [AlterationStateKey.AlterationState]: AlterationState;
 };
 
-const alterationStateGuard: Readonly<{
+export const alterationStateGuard: Readonly<{
   [key in AlterationStateKey]: ZodType<AlterationStateType[key]>;
 }> = Object.freeze({
   [AlterationStateKey.AlterationState]: z.object({
@@ -33,7 +33,7 @@ export type LogtoOidcConfigType = {
   [LogtoOidcConfigKey.RefreshTokenReuseInterval]: number;
 };
 
-const logtoOidcConfigGuard: Readonly<{
+export const logtoOidcConfigGuard: Readonly<{
   [key in LogtoOidcConfigKey]: ZodType<LogtoOidcConfigType[key]>;
 }> = Object.freeze({
   [LogtoOidcConfigKey.PrivateKeys]: z.string().array(),

pnpm-lock.yaml

@@ -3957,7 +3957,7 @@ packages:
       '@jest/types': 28.1.3
       deepmerge: 4.2.2
       identity-obj-proxy: 3.0.0
-      jest: 28.1.3_k5ytkvaprncdyzidqqws5bqksq
+      jest: 28.1.3_@types+node@16.11.12
       jest-matcher-specific-error: 1.0.0
       jest-transform-stub: 2.0.0
       ts-jest: 28.0.7_lhw3xkmzugq5tscs3x2ndm4sby
@@ -15126,7 +15126,7 @@ packages:
       '@jest/types': 28.1.3
       bs-logger: 0.2.6
       fast-json-stable-stringify: 2.1.0
-      jest: 28.1.3_k5ytkvaprncdyzidqqws5bqksq
+      jest: 28.1.3_@types+node@16.11.12
       jest-util: 28.1.3
       json5: 2.2.1
       lodash.memoize: 4.1.2