Infrastructure/logto
0
Fork 0
mirror of https://github.com/logto-io/logto.git synced 2025-01-06 20:40:08 -05:00
Code Issues Activity

Merge pull request #2067 from logto-io/gao-log-4329-cli-generate-and-save-keys-into-database

Browse source 
feat(cli): `db seed oidc` command
This commit is contained in:
Gao Sun 2022-10-09 13:58:19 +08:00 committed by GitHub
commit 50ae65e674
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 318 additions and 123 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

88
packages/cli/src/commands/database/seed.ts
View file

@ -1,88 +0,0 @@
import { readdir, readFile } from 'fs/promises';
import path from 'path';
import { seeds } from '@logto/schemas';
import chalk from 'chalk';
import { DatabasePool, DatabaseTransactionConnection, sql } from 'slonik';
import { raw } from 'slonik-sql-tag-raw';
import { CommandModule } from 'yargs';
import { createPoolAndDatabaseIfNeeded, insertInto } from '../../database';
import { updateDatabaseTimestamp } from '../../queries/logto-config';
import { buildApplicationSecret, getPathInModule, log, oraPromise } from '../../utilities';
import { getLatestAlterationTimestamp } from './alteration';
const createTables = async (connection: DatabaseTransactionConnection) => {
const tableDirectory = getPathInModule('@logto/schemas', 'tables');
const directoryFiles = await readdir(tableDirectory);
const tableFiles = directoryFiles.filter((file) => file.endsWith('.sql'));
const queries = await Promise.all(
tableFiles.map<Promise<[string, string]>>(async (file) => [
file,
await readFile(path.join(tableDirectory, file), 'utf8'),
])
);
// Await in loop is intended for better error handling
for (const [, query] of queries) {
// eslint-disable-next-line no-await-in-loop
await connection.query(sql`${raw(query)}`);
}
};
const seedTables = async (connection: DatabaseTransactionConnection) => {
const {
managementResource,
defaultSignInExperience,
createDefaultSetting,
createDemoAppApplication,
defaultRole,
} = seeds;
await Promise.all([
connection.query(insertInto(managementResource, 'resources')),
connection.query(insertInto(createDefaultSetting(), 'settings')),
connection.query(insertInto(defaultSignInExperience, 'sign_in_experiences')),
connection.query(
insertInto(createDemoAppApplication(buildApplicationSecret()), 'applications')
),
connection.query(insertInto(defaultRole, 'roles')),
updateDatabaseTimestamp(connection, await getLatestAlterationTimestamp()),
]);
};
export const seedByPool = async (pool: DatabasePool) => {
await pool.transaction(async (connection) => {
await oraPromise(createTables(connection), {
text: 'Create tables',
prefixText: chalk.blue('[info]'),
});
await oraPromise(seedTables(connection), {
text: 'Seed data',
prefixText: chalk.blue('[info]'),
});
});
};
const seed: CommandModule = {
command: 'seed',
describe: 'Create database and seed tables and data',
handler: async () => {
const pool = await createPoolAndDatabaseIfNeeded();
try {
await seedByPool(pool);
} catch (error: unknown) {
console.error(error);
console.log();
log.warn(
'Error ocurred during seeding your database.\n\n' +
' Nothing has changed since the seeding process was in a transaction.\n' +
' Try to fix the error and seed again.'
);
}
await pool.end();
},
};
export default seed;

153
packages/cli/src/commands/database/seed/index.ts Normal file
View file

@ -0,0 +1,153 @@
import { readdir, readFile } from 'fs/promises';
import path from 'path';
import { LogtoConfigKey, LogtoOidcConfig, logtoOidcConfigGuard, seeds } from '@logto/schemas';
import chalk from 'chalk';
import { DatabasePool, DatabaseTransactionConnection, sql } from 'slonik';
import { raw } from 'slonik-sql-tag-raw';
import { CommandModule } from 'yargs';
import { createPoolAndDatabaseIfNeeded, insertInto } from '../../../database';
import {
getRowsByKeys,
updateDatabaseTimestamp,
updateValueByKey,
} from '../../../queries/logto-config';
import { buildApplicationSecret, getPathInModule, log, oraPromise } from '../../../utilities';
import { getLatestAlterationTimestamp } from '../alteration';
import { OidcConfigKey, oidcConfigReaders } from './oidc-config';
const createTables = async (connection: DatabaseTransactionConnection) => {
const tableDirectory = getPathInModule('@logto/schemas', 'tables');
const directoryFiles = await readdir(tableDirectory);
const tableFiles = directoryFiles.filter((file) => file.endsWith('.sql'));
const queries = await Promise.all(
tableFiles.map<Promise<[string, string]>>(async (file) => [
file,
await readFile(path.join(tableDirectory, file), 'utf8'),
])
);
// Await in loop is intended for better error handling
for (const [, query] of queries) {
// eslint-disable-next-line no-await-in-loop
await connection.query(sql`${raw(query)}`);
}
};
const seedTables = async (connection: DatabaseTransactionConnection) => {
const {
managementResource,
defaultSignInExperience,
createDefaultSetting,
createDemoAppApplication,
defaultRole,
} = seeds;
await Promise.all([
connection.query(insertInto(managementResource, 'resources')),
connection.query(insertInto(createDefaultSetting(), 'settings')),
connection.query(insertInto(defaultSignInExperience, 'sign_in_experiences')),
connection.query(
insertInto(createDemoAppApplication(buildApplicationSecret()), 'applications')
),
connection.query(insertInto(defaultRole, 'roles')),
updateDatabaseTimestamp(connection, await getLatestAlterationTimestamp()),
]);
};
const seedOidcConfigs = async (pool: DatabaseTransactionConnection) => {
const { rows } = await getRowsByKeys(pool, [LogtoConfigKey.OidcConfig]);
const existingConfig = await logtoOidcConfigGuard
.parseAsync(rows[0]?.value)
// It's useful!
// eslint-disable-next-line unicorn/no-useless-undefined
.catch(() => undefined);
const existingKeys = existingConfig ? Object.keys(existingConfig) : [];
const validOptions = OidcConfigKey.options.filter((key) => {
const included = existingKeys.includes(key);
if (included) {
log.info(`Key ${chalk.green(key)} exists, skipping`);
}
return !included;
});
const entries: Array<[keyof LogtoOidcConfig, LogtoOidcConfig[keyof LogtoOidcConfig]]> = [];
// Both await in loop and `.push()` are intended since we'd like to log info in sequence
for (const key of validOptions) {
// eslint-disable-next-line no-await-in-loop
const { value, fromEnv } = await oidcConfigReaders[key]();
if (fromEnv) {
log.info(`Read config ${chalk.green(key)} from env`);
} else {
log.info(`Generated config ${chalk.green(key)}`);
}
// eslint-disable-next-line @silverhand/fp/no-mutating-methods
entries.push([key, value]);
}
await updateValueByKey(pool, LogtoConfigKey.OidcConfig, {
...existingConfig,
...Object.fromEntries(entries),
});
log.succeed('Seed OIDC config');
};
const seedChoices = Object.freeze(['all', 'oidc'] as const);
type SeedChoice = typeof seedChoices[number];
export const seedByPool = async (pool: DatabasePool, type: SeedChoice) => {
await pool.transaction(async (connection) => {
if (type !== 'oidc') {
await oraPromise(createTables(connection), {
text: 'Create tables',
prefixText: chalk.blue('[info]'),
});
await oraPromise(seedTables(connection), {
text: 'Seed data',
prefixText: chalk.blue('[info]'),
});
}
await seedOidcConfigs(connection);
});
};
const seed: CommandModule<Record<string, unknown>, { type: string }> = {
command: 'seed [type]',
describe: 'Create database then seed tables and data',
builder: (yargs) =>
yargs.positional('type', {
describe: 'Optional seed type',
type: 'string',
choices: seedChoices,
default: 'all',
}),
handler: async ({ type }) => {
const pool = await createPoolAndDatabaseIfNeeded();
try {
// Cannot avoid `as` since the official type definition of `yargs` doesn't work.
// The value of `type` can be ensured, so it's safe to use `as` here.
// eslint-disable-next-line no-restricted-syntax
await seedByPool(pool, type as SeedChoice);
} catch (error: unknown) {
console.error(error);
console.log();
log.warn(
'Error ocurred during seeding your database.\n\n' +
' Nothing has changed since the seeding process was in a transaction.\n' +
' Try to fix the error and seed again.'
);
}
await pool.end();
},
};
export default seed;

93
packages/cli/src/commands/database/seed/oidc-config.ts Normal file
View file

@ -0,0 +1,93 @@
import { generateKeyPair } from 'crypto';
import { readFile } from 'fs/promises';
import { promisify } from 'util';
import { LogtoOidcConfig, logtoOidcConfigGuard } from '@logto/schemas';
import { getEnv, getEnvAsStringArray } from '@silverhand/essentials';
import { nanoid } from 'nanoid';
import { z } from 'zod';
const isBase64FormatPrivateKey = (key: string) => !key.includes('-');
export const OidcConfigKey = logtoOidcConfigGuard.keyof();
/**
* Each config reader will do the following things in order:
* 1. Try to read value from env (mimic the behavior from the original core)
* 2. Generate value if #1 doesn't work
*/
export const oidcConfigReaders: {
[key in z.infer<typeof OidcConfigKey>]: () => Promise<{
value: NonNullable<LogtoOidcConfig[key]>;
fromEnv: boolean;
}>;
} = {
/**
* Try to read private keys with the following order:
*
* 1. From `process.env.OIDC_PRIVATE_KEYS`.
* 2. Fetch path from `process.env.OIDC_PRIVATE_KEY_PATHS` then read from that path.
*
*
* @returns The private keys for OIDC provider.
* @throws An error when failed to read a private key.
*/
privateKeys: async () => {
// Direct keys in env
const privateKeys = getEnvAsStringArray('OIDC_PRIVATE_KEYS');
if (privateKeys.length > 0) {
return {
value: privateKeys.map((key) => {
if (isBase64FormatPrivateKey(key)) {
return Buffer.from(key, 'base64').toString('utf8');
}
return key;
}),
fromEnv: true,
};
}
// Read keys from files
const privateKeyPaths = getEnvAsStringArray('OIDC_PRIVATE_KEY_PATHS');
if (privateKeyPaths.length > 0) {
return {
value: await Promise.all(privateKeyPaths.map(async (path) => readFile(path, 'utf8'))),
fromEnv: true,
};
}
// Generate a new key
const { privateKey } = await promisify(generateKeyPair)('rsa', {
modulusLength: 4096,
publicKeyEncoding: {
type: 'spki',
format: 'pem',
},
privateKeyEncoding: {
type: 'pkcs8',
format: 'pem',
},
});
return {
value: [privateKey],
fromEnv: false,
};
},
cookieKeys: async () => {
const envKey = 'OIDC_COOKIE_KEYS';
const keys = getEnvAsStringArray(envKey);
return { value: keys.length > 0 ? keys : [nanoid()], fromEnv: keys.length > 0 };
},
refreshTokenReuseInterval: async () => {
const envKey = 'OIDC_REFRESH_TOKEN_REUSE_INTERVAL';
const raw = Number(getEnv(envKey));
const value = Math.max(3, raw || 0);
return { value, fromEnv: raw === value };
},
};

5
packages/cli/src/commands/install.ts
View file

@ -122,7 +122,7 @@ const installLogto = async ({ path: pathArgument = defaultPath, silent = false }
try {
// Seed database
const pool = await createPoolAndDatabaseIfNeeded(); // It will ask for database URL and save to config
await seedByPool(pool);
await seedByPool(pool, 'all');
await pool.end();
} catch (error: unknown) {
console.error(error);
@ -131,7 +131,8 @@ const installLogto = async ({ path: pathArgument = defaultPath, silent = false }
name: 'value',
type: 'confirm',
message:
'Error occurred during seeding your Logto database. Would you like to continue without seed?',
'Error occurred during seeding your Logto database. Nothing has changed since the seeding process was in a transaction.\n' +
' Would you like to continue without seed?',
default: false,
});

38
packages/cli/src/database.ts
View file

@ -1,41 +1,19 @@
import { SchemaLike, SchemaValue, SchemaValuePrimitive } from '@logto/schemas';
import chalk from 'chalk';
import decamelize from 'decamelize';
import inquirer from 'inquirer';
import { createPool, IdentifierSqlToken, parseDsn, sql, SqlToken, stringifyDsn } from 'slonik';
import { createInterceptors } from 'slonik-interceptor-preset';
import { z } from 'zod';
import { log } from './utilities';
import { getCliConfig, log } from './utilities';
export const defaultDatabaseUrl = 'postgresql://localhost:5432/logto';
export const getDatabaseUrlFromEnv = async () => {
const { DB_URL: databaseUrl } = process.env;
if (!databaseUrl) {
const { value } = await inquirer
.prompt<{ value: string }>({
type: 'input',
name: 'value',
message: 'Enter your Logto database URL',
default: defaultDatabaseUrl,
})
.catch(async (error) => {
if (error.isTtyError) {
log.error('No database URL configured in env');
}
// The type definition does not give us type except `any`, throw it directly will honor the original behavior.
// eslint-disable-next-line @typescript-eslint/no-throw-literal
throw error;
});
return value;
}
return databaseUrl;
};
export const getDatabaseUrlFromEnv = async () =>
(await getCliConfig({
key: 'DB_URL',
readableKey: 'Logto database URL',
defaultValue: defaultDatabaseUrl,
})) ?? '';
export const createPoolFromEnv = async () => {
const databaseUrl = await getDatabaseUrlFromEnv();
@ -78,7 +56,7 @@ export const createPoolAndDatabaseIfNeeded = async () => {
`);
await maintenancePool.end();
log.info(`${chalk.green('✔')} Created database ${databaseName}`);
log.succeed(`Created database ${databaseName}`);
return createPoolFromEnv();
}

5
packages/cli/src/queries/logto-config.ts
View file

@ -13,7 +13,10 @@ import { convertToIdentifiers } from '../database';
const { table, fields } = convertToIdentifiers(LogtoConfigs);
export const getRowsByKeys = async (pool: DatabasePool, keys: LogtoConfigKey[]) =>
export const getRowsByKeys = async (
pool: DatabasePool | DatabaseTransactionConnection,
keys: LogtoConfigKey[]
) =>
pool.query<LogtoConfig>(sql`
select ${sql.join([fields.key, fields.value], sql`,`)} from ${table}
where ${fields.key} in (${sql.join(keys, sql`,`)})

54
packages/cli/src/utilities.ts
View file

@ -2,9 +2,11 @@ import { execSync } from 'child_process';
import { createWriteStream } from 'fs';
import path from 'path';
import { conditionalString, Optional } from '@silverhand/essentials';
import chalk from 'chalk';
import got, { Progress } from 'got';
import { HttpsProxyAgent } from 'hpagent';
import inquirer from 'inquirer';
import { customAlphabet } from 'nanoid';
import ora from 'ora';
@ -16,6 +18,7 @@ export const safeExecSync = (command: string) => {
type Log = Readonly<{
info: typeof console.log;
succeed: typeof console.log;
warn: typeof console.log;
error: (...args: Parameters<typeof console.log>) => never;
}>;
@ -24,11 +27,14 @@ export const log: Log = Object.freeze({
info: (...args) => {
console.log(chalk.blue('[info]'), ...args);
},
succeed: (...args) => {
log.info(chalk.green('✔'), ...args);
},
warn: (...args) => {
console.log(chalk.yellow('[warn]'), ...args);
console.warn(chalk.yellow('[warn]'), ...args);
},
error: (...args) => {
console.log(chalk.red('[error]'), ...args);
console.error(chalk.red('[error]'), ...args);
// eslint-disable-next-line unicorn/no-process-exit
process.exit(1);
},
@ -103,6 +109,50 @@ export const oraPromise = async <T>(
}
};
const cliConfig = new Map<string, Optional<string>>();
export type GetCliConfig = {
key: string;
readableKey: string;
comments?: string;
defaultValue?: string;
};
export const getCliConfig = async ({ key, readableKey, comments, defaultValue }: GetCliConfig) => {
if (cliConfig.has(key)) {
return cliConfig.get(key);
}
const { [key]: value } = process.env;
if (!value) {
const { input } = await inquirer
.prompt<{ input?: string }>({
type: 'input',
name: 'input',
message: `Enter your ${readableKey}${conditionalString(comments && ' ' + comments)}`,
default: defaultValue,
})
.catch(async (error) => {
if (error.isTtyError) {
log.error(`No ${readableKey} (${chalk.green(key)}) configured in env`);
}
// The type definition does not give us type except `any`, throw it directly will honor the original behavior.
// eslint-disable-next-line @typescript-eslint/no-throw-literal
throw error;
});
cliConfig.set(key, input);
return input;
}
cliConfig.set(key, value);
return value;
};
// TODO: Move to `@silverhand/essentials`
// Intended
// eslint-disable-next-line @typescript-eslint/no-empty-function

5
packages/schemas/src/types/logto-config.ts
View file

@ -12,6 +12,11 @@ export type AlterationState = z.infer<typeof alterationStateGuard>;
export const logtoOidcConfigGuard = z.object({
privateKeys: z.string().array().optional(),
cookieKeys: z.string().array().optional(),
/**
* This interval helps to avoid concurrency issues when exchanging the rotating refresh token multiple times within a given timeframe.
* During the leeway window (in seconds), the consumed refresh token will be considered as valid.
* This is useful for distributed apps and serverless apps like Next.js, in which there is no shared memory.
*/
refreshTokenReuseInterval: z.number().gte(3).optional(),
});