Merge pull request #2048 from logto-io/gao-log-4309-cli-seed-command

feat(cli): `db seed` command

packages/cli/package.json

@@ -36,14 +36,18 @@
   "dependencies": {
     "@logto/schemas": "^1.0.0-beta.10",
     "chalk": "^4.1.2",
+    "decamelize": "^5.0.0",
     "find-up": "^5.0.0",
     "got": "^11.8.2",
     "hpagent": "^1.0.0",
     "inquirer": "^8.2.2",
+    "nanoid": "^3.3.4",
     "ora": "^5.0.0",
+    "roarr": "^7.11.0",
     "semver": "^7.3.7",
     "slonik": "^30.0.0",
     "slonik-interceptor-preset": "^1.2.10",
+    "slonik-sql-tag-raw": "^1.1.4",
     "tar": "^6.1.11",
     "yargs": "^17.6.0",
     "zod": "^3.18.0"
@@ -65,7 +69,13 @@
     "typescript": "^4.7.4"
   },
   "eslintConfig": {
-    "extends": "@silverhand"
+    "extends": "@silverhand",
+    "rules": {
+      "complexity": [
+        "error",
+        7
+      ]
+    }
   },
   "prettier": "@silverhand/eslint-config/.prettierrc"
 }

packages/cli/src/commands/database/index.ts

@@ -2,13 +2,20 @@ import { CommandModule } from 'yargs';
 
 import { noop } from '../../utilities';
 import { getKey, setKey } from './key';
+import seed from './seed';
 import { getUrl, setUrl } from './url';
 
 const database: CommandModule = {
   command: ['database', 'db'],
   describe: 'Commands for Logto database',
   builder: (yargs) =>
-    yargs.command(getUrl).command(setUrl).command(getKey).command(setKey).demandCommand(1),
+    yargs
+      .command(getUrl)
+      .command(setUrl)
+      .command(getKey)
+      .command(setKey)
+      .command(seed)
+      .demandCommand(1),
   handler: noop,
 };
 

packages/cli/src/commands/database/seed.ts

@@ -0,0 +1,135 @@
+import { readdir, readFile } from 'fs/promises';
+import path from 'path';
+
+import { seeds } from '@logto/schemas';
+import {
+  createPool,
+  DatabasePool,
+  DatabaseTransactionConnection,
+  parseDsn,
+  sql,
+  stringifyDsn,
+} from 'slonik';
+import { raw } from 'slonik-sql-tag-raw';
+import { CommandModule } from 'yargs';
+import { z } from 'zod';
+
+import { createPoolFromConfig, getDatabaseUrlFromConfig, insertInto } from '../../database';
+import { buildApplicationSecret, log } from '../../utilities';
+
+/**
+ * Create a database pool with the database URL in config.
+ * If the given database does not exists, it will try to create a new database by connecting to the maintenance database `postgres`.
+ *
+ * @returns A new database pool with the database URL in config.
+ */
+const createDatabasePool = async () => {
+  try {
+    return await createPoolFromConfig();
+  } catch (error: unknown) {
+    const result = z.object({ code: z.string() }).safeParse(error);
+
+    // Database does not exist, try to create one
+    // https://www.postgresql.org/docs/14/errcodes-appendix.html
+    if (!(result.success && result.data.code === '3D000')) {
+      log.error(error);
+    }
+
+    const databaseUrl = await getDatabaseUrlFromConfig();
+    const dsn = parseDsn(databaseUrl);
+    // It's ok to fall back to '?' since:
+    // - Database name is required to connect in the previous pool
+    // - It will throw error when creating database using '?'
+    const databaseName = dsn.databaseName ?? '?';
+    const maintenancePool = await createPool(stringifyDsn({ ...dsn, databaseName: 'postgres' }));
+    await maintenancePool.query(sql`
+      create database ${sql.identifier([databaseName])}
+        with
+        encoding = 'UTF8'
+        connection_limit = -1;
+    `);
+    await maintenancePool.end();
+
+    log.info(`Database ${databaseName} successfully created.`);
+
+    return createPoolFromConfig();
+  }
+};
+
+const createTables = async (connection: DatabaseTransactionConnection) => {
+  // https://stackoverflow.com/a/49455609/12514940
+  const tableDirectory = path.join(
+    // Until we migrate to ESM
+    // eslint-disable-next-line unicorn/prefer-module
+    path.dirname(require.resolve('@logto/schemas/package.json')),
+    'tables'
+  );
+  const directoryFiles = await readdir(tableDirectory);
+  const tableFiles = directoryFiles.filter((file) => file.endsWith('.sql'));
+  const queries = await Promise.all(
+    tableFiles.map<Promise<[string, string]>>(async (file) => [
+      file,
+      await readFile(path.join(tableDirectory, file), 'utf8'),
+    ])
+  );
+
+  // Await in loop is intended for better error handling
+  for (const [file, query] of queries) {
+    // eslint-disable-next-line no-await-in-loop
+    await connection.query(sql`${raw(query)}`);
+    log.info(`Run ${file} succeeded.`);
+  }
+};
+
+const seedTables = async (connection: DatabaseTransactionConnection) => {
+  const {
+    managementResource,
+    defaultSignInExperience,
+    createDefaultSetting,
+    createDemoAppApplication,
+    defaultRole,
+  } = seeds;
+
+  // TODO: update database alteration timestamp when migrate alteration process from core
+
+  await Promise.all([
+    connection.query(insertInto(managementResource, 'resources')),
+    connection.query(insertInto(createDefaultSetting(), 'settings')),
+    connection.query(insertInto(defaultSignInExperience, 'sign_in_experiences')),
+    connection.query(
+      insertInto(createDemoAppApplication(buildApplicationSecret()), 'applications')
+    ),
+    connection.query(insertInto(defaultRole, 'roles')),
+  ]);
+  log.info('Seed tables succeeded.');
+};
+
+export const seedByPool = async (pool: DatabasePool) => {
+  await pool.transaction(async (connection) => {
+    await createTables(connection);
+    await seedTables(connection);
+  });
+};
+
+const seed: CommandModule = {
+  command: 'seed',
+  describe: 'Create database and seed tables and data',
+  handler: async () => {
+    const pool = await createDatabasePool();
+
+    try {
+      await seedByPool(pool);
+    } catch (error: unknown) {
+      console.error(error);
+      console.log();
+      log.warn(
+        'Error ocurred during seeding your database.\n\n' +
+          '  Nothing has changed since the seeding process was in a transaction.\n' +
+          '  Try to fix the error and seed again.'
+      );
+    }
+    await pool.end();
+  },
+};
+
+export default seed;

packages/cli/src/database.ts

@@ -1,19 +1,27 @@
+import { SchemaLike, SchemaValue, SchemaValuePrimitive } from '@logto/schemas';
 import chalk from 'chalk';
-import { createPool, IdentifierSqlToken, sql } from 'slonik';
+import decamelize from 'decamelize';
+import { createPool, IdentifierSqlToken, sql, SqlToken } from 'slonik';
 import { createInterceptors } from 'slonik-interceptor-preset';
 
 import { getConfig } from './config';
 import { log } from './utilities';
 
-export const createPoolFromConfig = async () => {
+export const getDatabaseUrlFromConfig = async () => {
   const { databaseUrl } = await getConfig();
 
   if (!databaseUrl) {
     log.error(
-      `No database URL configured. Set one via ${chalk.green('database set-url')} command first.`
+      `No database URL configured. Set it via ${chalk.green('database set-url')} command first.`
     );
   }
 
+  return databaseUrl;
+};
+
+export const createPoolFromConfig = async () => {
+  const databaseUrl = await getDatabaseUrlFromConfig();
+
   return createPool(databaseUrl, {
     interceptors: createInterceptors(),
   });
@@ -37,3 +45,61 @@ export const convertToIdentifiers = <T extends Table>({ table, fields }: T, with
     fields: Object.fromEntries(fieldsIdentifiers) as FieldIdentifiers<keyof T['fields']>,
   };
 };
+
+/**
+ * Note `undefined` is removed from the acceptable list,
+ * since you should NOT call this function if ignoring the field is the desired behavior.
+ * Calling this function with `null` means an explicit `null` setting in database is expected.
+ * @param key The key of value. Will treat as `timestamp` if it ends with `_at` or 'At' AND value is a number;
+ * @param value The value to convert.
+ * @returns A primitive that can be saved into database.
+ */
+// eslint-disable-next-line complexity
+export const convertToPrimitiveOrSql = (
+  key: string,
+  // eslint-disable-next-line @typescript-eslint/ban-types
+  value: NonNullable<SchemaValue> | null
+  // eslint-disable-next-line @typescript-eslint/ban-types
+): NonNullable<SchemaValuePrimitive> | SqlToken | null => {
+  if (value === null) {
+    return null;
+  }
+
+  if (typeof value === 'object') {
+    return JSON.stringify(value);
+  }
+
+  if (['_at', 'At'].some((value) => key.endsWith(value)) && typeof value === 'number') {
+    return sql`to_timestamp(${value}::double precision / 1000)`;
+  }
+
+  if (typeof value === 'number' || typeof value === 'boolean') {
+    return value;
+  }
+
+  if (typeof value === 'string') {
+    if (value === '') {
+      return null;
+    }
+
+    return value;
+  }
+
+  throw new Error(`Cannot convert ${key} to primitive`);
+};
+
+export const insertInto = <T extends SchemaLike>(object: T, table: string) => {
+  const keys = Object.keys(object);
+
+  return sql`
+    insert into ${sql.identifier([table])}
+    (${sql.join(
+      keys.map((key) => sql.identifier([decamelize(key)])),
+      sql`, `
+    )})
+    values (${sql.join(
+      keys.map((key) => convertToPrimitiveOrSql(key, object[key] ?? null)),
+      sql`, `
+    )})
+  `;
+};

packages/cli/src/utilities.ts

@@ -4,6 +4,7 @@ import { createWriteStream } from 'fs';
 import chalk from 'chalk';
 import got, { Progress } from 'got';
 import { HttpsProxyAgent } from 'hpagent';
+import { customAlphabet } from 'nanoid';
 import ora from 'ora';
 
 export const safeExecSync = (command: string) => {
@@ -69,8 +70,15 @@ export const downloadFile = async (url: string, destination: string) => {
   });
 };
 
+// TODO: Move to `@silverhand/essentials`
 // Intended
 // eslint-disable-next-line @typescript-eslint/no-empty-function
 export const noop = () => {};
 
 export const deduplicate = <T>(array: T[]) => [...new Set(array)];
+
+export const alphabet = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz';
+
+export const buildIdGenerator = (size: number) => customAlphabet(alphabet, size);
+
+export const buildApplicationSecret = buildIdGenerator(21);

pnpm-lock.yaml

@@ -30,18 +30,22 @@ importers:
       '@types/tar': ^6.1.2
       '@types/yargs': ^17.0.13
       chalk: ^4.1.2
+      decamelize: ^5.0.0
       eslint: ^8.21.0
       find-up: ^5.0.0
       got: ^11.8.2
       hpagent: ^1.0.0
       inquirer: ^8.2.2
       lint-staged: ^13.0.0
+      nanoid: ^3.3.4
       ora: ^5.0.0
       prettier: ^2.7.1
       rimraf: ^3.0.2
+      roarr: ^7.11.0
       semver: ^7.3.7
       slonik: ^30.0.0
       slonik-interceptor-preset: ^1.2.10
+      slonik-sql-tag-raw: ^1.1.4
       tar: ^6.1.11
       ts-node: ^10.9.1
       typescript: ^4.7.4
@@ -50,14 +54,18 @@ importers:
     dependencies:
       '@logto/schemas': link:../schemas
       chalk: 4.1.2
+      decamelize: 5.0.1
       find-up: 5.0.0
       got: 11.8.3
       hpagent: 1.0.0
       inquirer: 8.2.2
+      nanoid: 3.3.4
       ora: 5.4.1
+      roarr: 7.11.0
       semver: 7.3.7
       slonik: 30.1.2
       slonik-interceptor-preset: 1.2.10
+      slonik-sql-tag-raw: 1.1.4_roarr@7.11.0+slonik@30.1.2
       tar: 6.1.11
       yargs: 17.6.0
       zod: 3.18.0