feat(core,schemas): migration deploy cli (#1966)

packages/core/package.json

@@ -16,15 +16,16 @@
     "start": "NODE_ENV=production node build/index.js",
     "add-connector": "node build/cli/add-connector.js",
     "add-official-connectors": "node build/cli/add-official-connectors.js",
+    "migration-deploy": "node build/cli/migration-deploy.js",
     "test": "jest",
     "test:coverage": "jest --coverage --silent",
     "test:report": "codecov -F core"
   },
   "dependencies": {
     "@logto/connector-kit": "^1.0.0-beta.13",
+    "@logto/core-kit": "^1.0.0-beta.13",
     "@logto/phrases": "^1.0.0-beta.9",
     "@logto/schemas": "^1.0.0-beta.9",
-    "@logto/core-kit": "^1.0.0-beta.13",
     "@silverhand/essentials": "^1.2.1",
     "chalk": "^4",
     "dayjs": "^1.10.5",
@@ -56,6 +57,7 @@
     "query-string": "^7.0.1",
     "rimraf": "^3.0.2",
     "roarr": "^7.11.0",
+    "semver": "^7.3.7",
     "slonik": "^30.0.0",
     "slonik-interceptor-preset": "^1.2.10",
     "slonik-sql-tag-raw": "^1.1.4",
@@ -84,6 +86,7 @@
     "@types/node": "^16.3.1",
     "@types/oidc-provider": "^7.11.1",
     "@types/rimraf": "^3.0.2",
+    "@types/semver": "^7.3.12",
     "@types/supertest": "^2.0.11",
     "@types/tar": "^6.1.2",
     "copyfiles": "^2.4.1",

packages/core/src/cli/add-connector.ts

@@ -4,8 +4,7 @@ import chalk from 'chalk';
 
 import { addConnector } from '@/connectors/add-connectors';
 import { defaultConnectorDirectory } from '@/env-set';
-
-import { configDotEnv } from '../env-set/dot-env';
+import { configDotEnv } from '@/env-set/dot-env';
 
 configDotEnv();
 

packages/core/src/cli/add-official-connectors.ts

@@ -3,8 +3,7 @@ import { getEnv } from '@silverhand/essentials';
 
 import { addOfficialConnectors } from '@/connectors/add-connectors';
 import { defaultConnectorDirectory } from '@/env-set';
-
-import { configDotEnv } from '../env-set/dot-env';
+import { configDotEnv } from '@/env-set/dot-env';
 
 configDotEnv();
 

packages/core/src/cli/migration-deploy.ts

@@ -0,0 +1,17 @@
+import 'module-alias/register';
+import { assertEnv } from '@silverhand/essentials';
+import { createPool } from 'slonik';
+
+import { configDotEnv } from '@/env-set/dot-env';
+import { runMigrations } from '@/migration';
+
+configDotEnv();
+
+const deploy = async () => {
+  const databaseUrl = assertEnv('DB_URL');
+  const pool = await createPool(databaseUrl);
+  await runMigrations(pool);
+  await pool.end();
+};
+
+void deploy();

packages/core/src/migration/constants.ts

@@ -0,0 +1,3 @@
+export const databaseVersionKey = 'databaseVersion';
+export const logtoConfigsTableFilePath = 'node_modules/@logto/schemas/tables/logto_configs.sql';
+export const migrationFilesDirectory = 'node_modules/@logto/schemas/lib/migrations';

packages/core/src/migration/index.test.ts

@@ -0,0 +1,210 @@
+import { LogtoConfigs } from '@logto/schemas';
+import { createMockPool, createMockQueryResult, sql } from 'slonik';
+
+import { convertToIdentifiers } from '@/database/utils';
+import { QueryType, expectSqlAssert } from '@/utils/test-utils';
+
+import * as functions from '.';
+import { databaseVersionKey } from './constants';
+
+const mockQuery: jest.MockedFunction<QueryType> = jest.fn();
+const {
+  createLogtoConfigsTable,
+  getCurrentDatabaseVersion,
+  isLogtoConfigsTableExists,
+  updateDatabaseVersion,
+  getMigrationFiles,
+  getUndeployedMigrations,
+} = functions;
+const pool = createMockPool({
+  query: async (sql, values) => {
+    return mockQuery(sql, values);
+  },
+});
+const { table, fields } = convertToIdentifiers(LogtoConfigs);
+const existsSync = jest.fn();
+const readdir = jest.fn();
+
+jest.mock('fs', () => ({
+  existsSync: () => existsSync(),
+}));
+
+jest.mock('fs/promises', () => ({
+  ...jest.requireActual('fs/promises'),
+  readdir: async () => readdir(),
+}));
+
+describe('isLogtoConfigsTableExists()', () => {
+  it('generates "select exists" sql and query for result', async () => {
+    const expectSql = sql`
+      select exists (
+        select from 
+          pg_tables
+        where 
+          tablename = $1
+        );
+    `;
+
+    mockQuery.mockImplementationOnce(async (sql, values) => {
+      expectSqlAssert(sql, expectSql.sql);
+      expect(values).toEqual([LogtoConfigs.table]);
+
+      return createMockQueryResult([{ exists: true }]);
+    });
+
+    await expect(isLogtoConfigsTableExists(pool)).resolves.toEqual(true);
+  });
+});
+
+describe('getCurrentDatabaseVersion()', () => {
+  it('returns null if query failed (table not found)', async () => {
+    mockQuery.mockRejectedValueOnce(new Error('table not found'));
+
+    await expect(getCurrentDatabaseVersion(pool)).resolves.toBeNull();
+  });
+
+  it('returns null if the row is not found', async () => {
+    const expectSql = sql`
+      select * from ${table} where ${fields.key}=$1
+    `;
+
+    mockQuery.mockImplementationOnce(async (sql, values) => {
+      expectSqlAssert(sql, expectSql.sql);
+      expect(values).toEqual([databaseVersionKey]);
+
+      return createMockQueryResult([]);
+    });
+
+    await expect(getCurrentDatabaseVersion(pool)).resolves.toBeNull();
+  });
+
+  it('returns null if the value is in bad format', async () => {
+    const expectSql = sql`
+      select * from ${table} where ${fields.key}=$1
+    `;
+
+    mockQuery.mockImplementationOnce(async (sql, values) => {
+      expectSqlAssert(sql, expectSql.sql);
+      expect(values).toEqual([databaseVersionKey]);
+
+      return createMockQueryResult([{ value: 'some_version' }]);
+    });
+
+    await expect(getCurrentDatabaseVersion(pool)).resolves.toBeNull();
+  });
+
+  it('returns the version from database', async () => {
+    const expectSql = sql`
+      select * from ${table} where ${fields.key}=$1
+    `;
+    const version = 'version';
+
+    mockQuery.mockImplementationOnce(async (sql, values) => {
+      expectSqlAssert(sql, expectSql.sql);
+      expect(values).toEqual([databaseVersionKey]);
+
+      // @ts-expect-error createMockQueryResult doesn't support jsonb
+      return createMockQueryResult([{ value: { version, updatedAt: 'now' } }]);
+    });
+
+    await expect(getCurrentDatabaseVersion(pool)).resolves.toEqual(version);
+  });
+});
+
+describe('createLogtoConfigsTable()', () => {
+  it('sends sql to create target table', async () => {
+    mockQuery.mockImplementationOnce(async (sql, values) => {
+      expect(sql).toContain(LogtoConfigs.table);
+      expect(sql).toContain('create table');
+
+      return createMockQueryResult([]);
+    });
+
+    await createLogtoConfigsTable(pool);
+  });
+});
+
+describe('updateDatabaseVersion()', () => {
+  const expectSql = sql`
+    insert into ${table} (${fields.key}, ${fields.value}) 
+      values ($1, $2)
+      on conflict (${fields.key}) do update set ${fields.value}=excluded.${fields.value}
+  `;
+  const version = 'version';
+  const updatedAt = '2022-09-21T06:32:46.583Z';
+
+  beforeAll(() => {
+    jest.useFakeTimers();
+    jest.setSystemTime(new Date(updatedAt));
+  });
+
+  afterAll(() => {
+    jest.useRealTimers();
+  });
+
+  it('calls createLogtoConfigsTable() if table does not exist', async () => {
+    mockQuery.mockImplementationOnce(async (sql, values) => {
+      expectSqlAssert(sql, expectSql.sql);
+
+      return createMockQueryResult([]);
+    });
+
+    const mockCreateLogtoConfigsTable = jest
+      .spyOn(functions, 'createLogtoConfigsTable')
+      .mockImplementationOnce(jest.fn());
+    jest.spyOn(functions, 'isLogtoConfigsTableExists').mockResolvedValueOnce(false);
+
+    await updateDatabaseVersion(pool, version);
+    expect(mockCreateLogtoConfigsTable).toHaveBeenCalled();
+  });
+
+  it('sends upsert sql with version and updatedAt', async () => {
+    mockQuery.mockImplementationOnce(async (sql, values) => {
+      expectSqlAssert(sql, expectSql.sql);
+      expect(values).toEqual([databaseVersionKey, JSON.stringify({ version, updatedAt })]);
+
+      return createMockQueryResult([]);
+    });
+    jest.spyOn(functions, 'isLogtoConfigsTableExists').mockResolvedValueOnce(true);
+
+    await updateDatabaseVersion(pool, version);
+  });
+});
+
+describe('getMigrationFiles()', () => {
+  it('returns [] if directory does not exist', async () => {
+    existsSync.mockReturnValueOnce(false);
+    await expect(getMigrationFiles()).resolves.toEqual([]);
+  });
+
+  it('returns files without "next"', async () => {
+    existsSync.mockReturnValueOnce(true);
+    readdir.mockResolvedValueOnce(['next.js', '1.0.0.js', '1.0.2.js', '1.0.1.js']);
+
+    await expect(getMigrationFiles()).resolves.toEqual(['1.0.0.js', '1.0.2.js', '1.0.1.js']);
+  });
+});
+
+describe('getUndeployedMigrations()', () => {
+  beforeEach(() => {
+    jest
+      .spyOn(functions, 'getMigrationFiles')
+      .mockResolvedValueOnce(['1.0.0.js', '1.0.2.js', '1.0.1.js']);
+  });
+
+  it('returns all files with right order if database version is null', async () => {
+    jest.spyOn(functions, 'getCurrentDatabaseVersion').mockResolvedValueOnce(null);
+
+    await expect(getUndeployedMigrations(pool)).resolves.toEqual([
+      '1.0.0.js',
+      '1.0.1.js',
+      '1.0.2.js',
+    ]);
+  });
+
+  it('returns files whose version is greater then database version', async () => {
+    jest.spyOn(functions, 'getCurrentDatabaseVersion').mockResolvedValueOnce('1.0.0');
+
+    await expect(getUndeployedMigrations(pool)).resolves.toEqual(['1.0.1.js', '1.0.2.js']);
+  });
+});

packages/core/src/migration/index.ts

@@ -0,0 +1,141 @@
+import { existsSync } from 'fs';
+import { readdir, readFile } from 'fs/promises';
+import path from 'path';
+
+import { LogtoConfig, LogtoConfigs } from '@logto/schemas';
+import { conditionalString } from '@silverhand/essentials';
+import chalk from 'chalk';
+import { DatabasePool, sql } from 'slonik';
+import { raw } from 'slonik-sql-tag-raw';
+
+import { convertToIdentifiers } from '@/database/utils';
+
+import {
+  databaseVersionKey,
+  logtoConfigsTableFilePath,
+  migrationFilesDirectory,
+} from './constants';
+import { DatabaseVersion, databaseVersionGuard, MigrationScript } from './types';
+import { compareVersion, getVersionFromFileName, migrationFileNameRegex } from './utils';
+
+const { table, fields } = convertToIdentifiers(LogtoConfigs);
+
+export const isLogtoConfigsTableExists = async (pool: DatabasePool) => {
+  const { exists } = await pool.one<{ exists: boolean }>(sql`
+    select exists (
+      select from 
+        pg_tables
+      where 
+        tablename = ${LogtoConfigs.table}
+      );
+  `);
+
+  return exists;
+};
+
+export const getCurrentDatabaseVersion = async (pool: DatabasePool) => {
+  try {
+    const query = await pool.maybeOne<LogtoConfig>(
+      sql`select * from ${table} where ${fields.key}=${databaseVersionKey}`
+    );
+    const databaseVersion = databaseVersionGuard.parse(query?.value);
+
+    return databaseVersion.version;
+  } catch {
+    return null;
+  }
+};
+
+export const createLogtoConfigsTable = async (pool: DatabasePool) => {
+  const tableQuery = await readFile(logtoConfigsTableFilePath, 'utf8');
+  await pool.query(sql`${raw(tableQuery)}`);
+};
+
+export const updateDatabaseVersion = async (pool: DatabasePool, version: string) => {
+  if (!(await isLogtoConfigsTableExists(pool))) {
+    await createLogtoConfigsTable(pool);
+  }
+
+  const value: DatabaseVersion = {
+    version,
+    updatedAt: new Date().toISOString(),
+  };
+
+  await pool.query(
+    sql`
+      insert into ${table} (${fields.key}, ${fields.value}) 
+        values (${databaseVersionKey}, ${JSON.stringify(value)})
+        on conflict (${fields.key}) do update set ${fields.value}=excluded.${fields.value}
+    `
+  );
+};
+
+export const getMigrationFiles = async () => {
+  if (!existsSync(migrationFilesDirectory)) {
+    return [];
+  }
+
+  const directory = await readdir(migrationFilesDirectory);
+  const files = directory.filter((file) => migrationFileNameRegex.test(file));
+
+  return files;
+};
+
+export const getUndeployedMigrations = async (pool: DatabasePool) => {
+  const databaseVersion = await getCurrentDatabaseVersion(pool);
+  const files = await getMigrationFiles();
+
+  return files
+    .filter(
+      (file) =>
+        !databaseVersion || compareVersion(getVersionFromFileName(file), databaseVersion) > 0
+    )
+    .slice()
+    .sort((file1, file2) =>
+      compareVersion(getVersionFromFileName(file1), getVersionFromFileName(file2))
+    );
+};
+
+const importMigration = async (file: string): Promise<MigrationScript> => {
+  // eslint-disable-next-line no-restricted-syntax
+  const module = (await import(
+    path.join(migrationFilesDirectory, file).replace('node_modules/', '')
+  )) as MigrationScript;
+
+  return module;
+};
+
+const runMigration = async (pool: DatabasePool, file: string) => {
+  const { up } = await importMigration(file);
+
+  try {
+    await up(pool);
+  } catch (error: unknown) {
+    if (error instanceof Error) {
+      console.log(`${chalk.red('[migration]')} run ${file} failed: ${error.message}.`);
+
+      return;
+    }
+
+    throw error;
+  }
+
+  await updateDatabaseVersion(pool, getVersionFromFileName(file));
+  console.log(`${chalk.blue('[migration]')} run ${file} succeeded.`);
+};
+
+export const runMigrations = async (pool: DatabasePool) => {
+  const migrations = await getUndeployedMigrations(pool);
+
+  console.log(
+    `${chalk.blue('[migration]')} found ${migrations.length} migration${conditionalString(
+      migrations.length > 1 && 's'
+    )}`
+  );
+
+  // The await inside the loop is intended, migrations should run in order
+  for (const migration of migrations) {
+    // eslint-disable-next-line no-await-in-loop
+    await runMigration(pool, migration);
+  }
+};

packages/core/src/migration/types.ts

@@ -0,0 +1,14 @@
+import { DatabasePool } from 'slonik';
+import { z } from 'zod';
+
+export const databaseVersionGuard = z.object({
+  version: z.string(),
+  updatedAt: z.string().optional(),
+});
+
+export type DatabaseVersion = z.infer<typeof databaseVersionGuard>;
+
+export type MigrationScript = {
+  up: (pool: DatabasePool) => Promise<void>;
+  down: (pool: DatabasePool) => Promise<void>;
+};

packages/core/src/migration/utils.test.ts

@@ -0,0 +1,25 @@
+import { compareVersion, getVersionFromFileName } from './utils';
+
+describe('compareVersion', () => {
+  it('should return 1 for 1.0.0 and 1.0.0-beta.9', () => {
+    expect(compareVersion('1.0.0', '1.0.0-beta.9')).toBe(1);
+  });
+
+  it('should return 1 for 1.0.0-beta.10 and 1.0.0-beta.9', () => {
+    expect(compareVersion('1.0.0-beta.10', '1.0.0-beta.9')).toBe(1);
+  });
+
+  it('should return 1 for 1.0.0 and 0.0.8', () => {
+    expect(compareVersion('1.0.0', '0.0.8')).toBe(1);
+  });
+});
+
+describe('getVersionFromFileName', () => {
+  it('should get version for 1.0.2.js', () => {
+    expect(getVersionFromFileName('1.0.2.js')).toEqual('1.0.2');
+  });
+
+  it('should throw for next.js', () => {
+    expect(() => getVersionFromFileName('next.js')).toThrowError();
+  });
+});

packages/core/src/migration/utils.ts

@@ -0,0 +1,21 @@
+import semver from 'semver';
+
+export const migrationFileNameRegex = /^(((?!next).)*)\.js$/;
+
+export const getVersionFromFileName = (fileName: string) => {
+  const match = migrationFileNameRegex.exec(fileName);
+
+  if (!match?.[1]) {
+    throw new Error(`Can not find version name: ${fileName}`);
+  }
+
+  return match[1];
+};
+
+export const compareVersion = (version1: string, version2: string) => {
+  if (semver.eq(version1, version2)) {
+    return 0;
+  }
+
+  return semver.gt(version1, version2) ? 1 : -1;
+};

packages/schemas/package.json

@@ -49,9 +49,10 @@
   "prettier": "@silverhand/eslint-config/.prettierrc",
   "dependencies": {
     "@logto/connector-kit": "^1.0.0-beta.13",
+    "@logto/core-kit": "^1.0.0-beta.13",
     "@logto/phrases": "^1.0.0-beta.9",
     "@logto/phrases-ui": "^1.0.0-beta.9",
-    "@logto/core-kit": "^1.0.0-beta.13",
+    "slonik": "^30.0.0",
     "zod": "^3.18.0"
   }
 }

packages/schemas/src/foundations/schemas.ts

@@ -23,7 +23,7 @@ export type GeneratedSchema<Schema extends SchemaLike> = keyof Schema extends st
       table: string;
       tableSingular: string;
       fields: {
-        [key in keyof Schema]: string;
+        [key in keyof Required<Schema>]: string;
       };
       fieldKeys: ReadonlyArray<keyof Schema>;
       createGuard: CreateGuard<Schema>;

packages/schemas/src/migrations/README.md

@@ -0,0 +1,3 @@
+# Database Migrations
+
+The folder for all migration files.

pnpm-lock.yaml

@@ -174,6 +174,7 @@ importers:
       '@types/node': ^16.3.1
       '@types/oidc-provider': ^7.11.1
       '@types/rimraf': ^3.0.2
+      '@types/semver': ^7.3.12
       '@types/supertest': ^2.0.11
       '@types/tar': ^6.1.2
       chalk: ^4
@@ -216,6 +217,7 @@ importers:
       query-string: ^7.0.1
       rimraf: ^3.0.2
       roarr: ^7.11.0
+      semver: ^7.3.7
       slonik: ^30.0.0
       slonik-interceptor-preset: ^1.2.10
       slonik-sql-tag-raw: ^1.1.4
@@ -261,6 +263,7 @@ importers:
       query-string: 7.0.1
       rimraf: 3.0.2
       roarr: 7.11.0
+      semver: 7.3.7
       slonik: 30.1.2
       slonik-interceptor-preset: 1.2.10
       slonik-sql-tag-raw: 1.1.4_roarr@7.11.0+slonik@30.1.2
@@ -288,6 +291,7 @@ importers:
       '@types/node': 16.11.12
       '@types/oidc-provider': 7.11.1
       '@types/rimraf': 3.0.2
+      '@types/semver': 7.3.12
       '@types/supertest': 2.0.11
       '@types/tar': 6.1.2
       copyfiles: 2.4.1
@@ -472,6 +476,7 @@ importers:
       lodash.uniq: ^4.5.0
       pluralize: ^8.0.0
       prettier: ^2.7.1
+      slonik: ^30.0.0
       ts-node: ^10.9.1
       typescript: ^4.7.4
       zod: ^3.18.0
@@ -480,6 +485,7 @@ importers:
       '@logto/core-kit': 1.0.0-beta.13
       '@logto/phrases': link:../phrases
       '@logto/phrases-ui': link:../phrases-ui
+      slonik: 30.1.2
       zod: 3.18.0
     devDependencies:
       '@silverhand/eslint-config': 1.0.0_swk2g7ygmfleszo5c33j4vooni
@@ -4542,6 +4548,10 @@ packages:
     resolution: {integrity: sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==}
     dev: true
 
+  /@types/semver/7.3.12:
+    resolution: {integrity: sha512-WwA1MW0++RfXmCr12xeYOOC5baSC9mSb0ZqCquFzKhcoF4TvHu5MKOuXsncgZcpVFhB1pXd5hZmM0ryAoCp12A==}
+    dev: true
+
   /@types/serve-static/1.13.10:
     resolution: {integrity: sha512-nCkHGI4w7ZgAdNkrEu0bv+4xNV/XDqW+DydknebMOQwkpDGx8G+HTlj7R7ABI8i8nKxVw0wtKPi1D+lPOkh4YQ==}
     dependencies:
@@ -10341,7 +10351,6 @@ packages:
     engines: {node: '>=10'}
     dependencies:
       yallist: 4.0.0
-    dev: true
 
   /lru-cache/7.10.1:
     resolution: {integrity: sha512-BQuhQxPuRl79J5zSXRP+uNzPOyZw2oFI9JLRQ80XswSvg21KMKNtQza9eF42rfI/3Z40RvzBdXgziEkudzjo8A==}
@@ -13687,7 +13696,7 @@ packages:
     dev: true
 
   /semver-compare/1.0.0:
-    resolution: {integrity: sha1-De4hahyUGrN+nvsXiPavxf9VN/w=}
+    resolution: {integrity: sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==}
     dev: false
 
   /semver/5.7.1:
@@ -13705,21 +13714,12 @@ packages:
     hasBin: true
     dev: true
 
-  /semver/7.3.5:
-    resolution: {integrity: sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==}
-    engines: {node: '>=10'}
-    hasBin: true
-    dependencies:
-      lru-cache: 6.0.0
-    dev: true
-
   /semver/7.3.7:
     resolution: {integrity: sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==}
     engines: {node: '>=10'}
     hasBin: true
     dependencies:
       lru-cache: 6.0.0
-    dev: true
 
   /serialize-error/7.0.1:
     resolution: {integrity: sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==}
@@ -14502,7 +14502,7 @@ packages:
       mime: 2.6.0
       qs: 6.10.2
       readable-stream: 3.6.0
-      semver: 7.3.5
+      semver: 7.3.7
     transitivePeerDependencies:
       - supports-color
     dev: true