feat(cli): rollback command for alteration (#2975)

.changeset-staged/breezy-socks-joke.md

@@ -0,0 +1,5 @@
+---
+"@logto/cli": minor
+---
+
+Add `logto database alteration rollback` command for running `down()` alteration scripts

.github/workflows/main.yml

@@ -119,10 +119,6 @@ jobs:
         run: pnpm i && pnpm prepack
       # ** End **
 
-      - name: Check alteration sequence
-        working-directory: ./fresh
-        run: node .scripts/check-alterations-sequence.js
-
       - name: Setup Postgres
         uses: ikalnytskyi/action-setup-postgres@v4
 
@@ -153,5 +149,26 @@ jobs:
       # ** End **
 
       # ** Setup old databases and compare (test `down`) **
-      # TBD
+      - name: Setup old database
+        working-directory: ./alteration
+        run: |
+          cd packages/cli
+          pnpm start db seed
+        env:
+          DB_URL: postgres://postgres:postgres@localhost:5432/old
+
+      - name: Revert fresh database to old
+        working-directory: ./fresh
+        run: pnpm cli db alt r v1.0.0-beta.11
+        env:
+          DB_URL: postgres://postgres:postgres@localhost:5432/fresh
+
+      - name: Compare manifests
+        working-directory: ./fresh
+        run: node .scripts/compare-database.js fresh old
       # ** End **
+
+      - name: Check alteration sequence
+        continue-on-error: true # Temporarily skip this as we are fixing previous down scripts
+        working-directory: ./fresh
+        run: node .scripts/check-alterations-sequence.js

.scripts/check-alterations-sequence.js

@@ -29,7 +29,7 @@ for (const alteration of committedAlterations) {
 
   if (index < allAlterations.length - committedAlterations.length) {
     throw new Error(
-      `Wrong alteration sequence for commited file: ${alteration}\nAll timestamps of committed alteration files should be greater than the biggest one in the base branch.`
+      `Wrong alteration sequence for committed file: ${alteration}\nAll timestamps of committed alteration files should be greater than the biggest one in the base branch.`
     );
   }
 }

.scripts/compare-database.js

@@ -49,6 +49,7 @@ const queryDatabaseManifest = async (database) => {
     order by tablename, indexname asc;
   `);
 
+  // Omit generated ids and values
   return {
     tables: omitArray(tables, 'table_catalog'),
     columns: omitArray(columns, 'table_catalog', 'udt_catalog', 'ordinal_position', 'dtd_identifier'),

packages/cli/package.json

@@ -86,6 +86,9 @@
   },
   "eslintConfig": {
     "extends": "@silverhand",
+    "rules": {
+      "complexity": ["error", 11]
+    },
     "ignorePatterns": [
       "src/package-json.ts"
     ]

packages/cli/src/commands/database/alteration/index.test.ts

@@ -28,19 +28,19 @@ const { getCurrentDatabaseAlterationTimestamp } = await mockEsmWithActual(
   })
 );
 
-const { getUndeployedAlterations } = await import('./index.js');
+const { getAvailableAlterations } = await import('./index.js');
 
-describe('getUndeployedAlterations()', () => {
+describe('getAvailableAlterations()', () => {
   it('returns all files if database timestamp is 0', async () => {
     getCurrentDatabaseAlterationTimestamp.mockResolvedValue(0);
 
-    await expect(getUndeployedAlterations(pool)).resolves.toEqual(files);
+    await expect(getAvailableAlterations(pool)).resolves.toEqual(files);
   });
 
   it('returns files whose timestamp is greater then database timestamp', async () => {
     getCurrentDatabaseAlterationTimestamp.mockResolvedValue(1_663_923_770);
 
-    await expect(getUndeployedAlterations(pool)).resolves.toEqual([files[1], files[2]]);
+    await expect(getAvailableAlterations(pool)).resolves.toEqual([files[1], files[2]]);
   });
 });
 

packages/cli/src/commands/database/alteration/index.ts

@@ -12,7 +12,7 @@ import {
 import { log } from '../../../utilities.js';
 import type { AlterationFile } from './type.js';
 import { getAlterationFiles, getTimestampFromFilename } from './utils.js';
-import { chooseAlterationsByVersion } from './version.js';
+import { chooseAlterationsByVersion, chooseRevertAlterationsByVersion } from './version.js';
 
 const importAlterationScript = async (filePath: string): Promise<AlterationScript> => {
   // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
@@ -33,24 +33,44 @@ export const getLatestAlterationTimestamp = async () => {
   return getTimestampFromFilename(lastFile.filename);
 };
 
-export const getUndeployedAlterations = async (pool: DatabasePool) => {
+export const getAvailableAlterations = async (
+  pool: DatabasePool,
+  compareMode: 'gt' | 'lte' = 'gt'
+) => {
   const databaseTimestamp = await getCurrentDatabaseAlterationTimestamp(pool);
 
   const files = await getAlterationFiles();
 
-  return files.filter(({ filename }) => getTimestampFromFilename(filename) > databaseTimestamp);
+  return files.filter(({ filename }) =>
+    compareMode === 'gt'
+      ? getTimestampFromFilename(filename) > databaseTimestamp
+      : getTimestampFromFilename(filename) <= databaseTimestamp
+  );
 };
 
 const deployAlteration = async (
   pool: DatabasePool,
-  { path: filePath, filename }: AlterationFile
+  { path: filePath, filename }: AlterationFile,
+  action: 'up' | 'down' = 'up'
 ) => {
-  const { up } = await importAlterationScript(filePath);
+  const { up, down } = await importAlterationScript(filePath);
 
   try {
     await pool.transaction(async (connection) => {
-      await up(connection);
+      if (action === 'up') {
-      await updateDatabaseTimestamp(connection, getTimestampFromFilename(filename));
+        await up(connection);
+        await updateDatabaseTimestamp(connection, getTimestampFromFilename(filename));
+      }
+
+      if (action === 'down') {
+        await down(connection);
+
+        const newTimestamp = getTimestampFromFilename(filename) - 1;
+
+        if (newTimestamp > 0) {
+          await updateDatabaseTimestamp(connection, newTimestamp);
+        }
+      }
     });
   } catch (error: unknown) {
     console.error(error);
@@ -63,7 +83,7 @@ const deployAlteration = async (
     );
   }
 
-  log.info(`Run alteration ${filename} succeeded`);
+  log.info(`Run alteration ${filename} \`${action}()\` function succeeded`);
 };
 
 const alteration: CommandModule<unknown, { action: string; target?: string }> = {
@@ -72,7 +92,7 @@ const alteration: CommandModule<unknown, { action: string; target?: string }> =
   builder: (yargs) =>
     yargs
       .positional('action', {
-        describe: 'The action to perform, now it only accepts `deploy` and `list`',
+        describe: 'The action to perform, accepts `list`, `deploy`, and `rollback` (or `r`).',
         type: 'string',
         demandOption: true,
       })
@@ -90,7 +110,7 @@ const alteration: CommandModule<unknown, { action: string; target?: string }> =
     } else if (action === 'deploy') {
       const pool = await createPoolFromConfig();
       const alterations = await chooseAlterationsByVersion(
-        await getUndeployedAlterations(pool),
+        await getAvailableAlterations(pool),
         target
       );
 
@@ -106,6 +126,27 @@ const alteration: CommandModule<unknown, { action: string; target?: string }> =
         await deployAlteration(pool, alteration);
       }
 
+      await pool.end();
+    } else if (['rollback', 'r'].includes(action)) {
+      const pool = await createPoolFromConfig();
+      const alterations = await chooseRevertAlterationsByVersion(
+        await getAvailableAlterations(pool, 'lte'),
+        target ?? ''
+      );
+
+      log.info(
+        `Found ${alterations.length} alteration${conditionalString(
+          alterations.length > 1 && 's'
+        )} to revert`
+      );
+
+      // The await inside the loop is intended, alterations should run in order
+      // eslint-disable-next-line @silverhand/fp/no-mutating-methods
+      for (const alteration of alterations.slice().reverse()) {
+        // eslint-disable-next-line no-await-in-loop
+        await deployAlteration(pool, alteration, 'down');
+      }
+
       await pool.end();
     } else {
       log.error('Unsupported action');

packages/cli/src/commands/database/alteration/version.ts

@@ -6,12 +6,24 @@ import { SemVer, compare, eq, gt } from 'semver';
 import { findLastIndex, isTty, log } from '../../../utilities.js';
 import type { AlterationFile } from './type.js';
 
+const getVersionStringFromFilename = (filename: string) =>
+  filename.split('-')[0]?.replaceAll('_', '-') ?? 'unknown';
+
 const getVersionFromFilename = (filename: string) => {
   try {
-    return new SemVer(filename.split('-')[0]?.replaceAll('_', '-') ?? 'unknown');
+    return new SemVer(getVersionStringFromFilename(filename));
   } catch {}
 };
 
+const getAlterationVersions = (alterations: readonly AlterationFile[]) =>
+  alterations
+    .map(({ filename }) => getVersionFromFilename(filename))
+    .filter((version): version is SemVer => version instanceof SemVer)
+    // Cannot use `Set` to deduplicate since it's a class
+    .filter((version, index, self) => index === self.findIndex((another) => eq(version, another)))
+    .slice()
+    .sort((i, j) => compare(j, i));
+
 const latestTag = 'latest';
 const nextTag = 'next';
 
@@ -35,14 +47,7 @@ export const chooseAlterationsByVersion = async (
     return alterations.slice(0, endIndex + 1);
   }
 
-  const versions = alterations
+  const versions = getAlterationVersions(alterations);
-    .map(({ filename }) => getVersionFromFilename(filename))
-    .filter((version): version is SemVer => version instanceof SemVer)
-    // Cannot use `Set` to deduplicate since it's a class
-    .filter((version, index, self) => index === self.findIndex((another) => eq(version, another)))
-    .slice()
-    .sort((i, j) => compare(j, i));
-
   const initialSemVersion = conditional(
     initialVersion && initialVersion !== latestTag && new SemVer(initialVersion)
   );
@@ -91,3 +96,22 @@ export const chooseAlterationsByVersion = async (
     return version && !gt(version, targetVersion);
   });
 };
+
+export const chooseRevertAlterationsByVersion = async (
+  alterations: readonly AlterationFile[],
+  version: string
+) => {
+  const semVersion = new SemVer(version);
+
+  log.info(`Revert target ${chalk.green(semVersion.version)}`);
+
+  return alterations.filter(({ filename }) => {
+    if (getVersionStringFromFilename(filename) === nextTag) {
+      return true;
+    }
+
+    const version = getVersionFromFilename(filename);
+
+    return version && gt(version, semVersion);
+  });
+};

packages/core/src/env-set/check-alteration-state.ts

@@ -1,9 +1,9 @@
-import { getUndeployedAlterations } from '@logto/cli/lib/commands/database/alteration/index.js';
+import { getAvailableAlterations } from '@logto/cli/lib/commands/database/alteration/index.js';
 import chalk from 'chalk';
 import type { DatabasePool } from 'slonik';
 
 export const checkAlterationState = async (pool: DatabasePool) => {
-  const alterations = await getUndeployedAlterations(pool);
+  const alterations = await getAvailableAlterations(pool);
 
   if (alterations.length === 0) {
     return;

packages/schemas/alterations/1.0.0_beta.14-1667292082-remove-sign-in-method.ts

@@ -5,12 +5,13 @@ import type { AlterationScript } from '../lib/types/alteration.js';
 const alteration: AlterationScript = {
   up: async (pool) => {
     await pool.query(sql`
-      alter table sign_in_experiences drop column sign_in_methods
+      alter table sign_in_experiences drop column sign_in_methods;
     `);
   },
   down: async (pool) => {
     await pool.query(sql`
-      alter table sign_in_experiences add column sign_in_methods jsonb not null default '{}'::jsonb,
+      alter table sign_in_experiences add column sign_in_methods jsonb not null default '{}'::jsonb;
+      alter table sign_in_experiences alter column sign_in_methods drop default;
     `);
   },
 };

packages/schemas/alterations/1.0.0_beta.14-1667900481-add-passcode-type-continue.ts

@@ -5,13 +5,18 @@ import type { AlterationScript } from '../lib/types/alteration.js';
 const alteration: AlterationScript = {
   up: async (pool) => {
     await pool.query(sql`
-      alter type passcode_type add value 'Continue'
+      alter type passcode_type add value 'Continue';
     `);
   },
   down: async (pool) => {
     await pool.query(sql`
-      drop type passcode_type
+      create type passcode_type_new as enum ('SignIn', 'Register', 'ForgotPassword');
-      create type passcode_type as enum ('SignIn', 'Register', 'ForgotPassword');
+      delete from passcodes where "type"='Continue';
+      alter table passcodes
+        alter column "type" type passcode_type_new
+          using ("type"::text::passcode_type_new);
+      drop type passcode_type;
+      alter type passcode_type_new rename to passcode_type;
     `);
   },
 };

packages/schemas/alterations/1.0.0_beta.18-1669091623-roles-and-scopes.ts

@@ -31,11 +31,13 @@ const alteration: AlterationScript = {
   },
   down: async (pool) => {
     await pool.query(sql`
-      drop table permissions;
+      drop table roles_scopes cascade;
-      alter index roles_pkey rename to roles_pkey_1;
+      drop table scopes cascade;
-      create unique index roles_pkey on roles using btree(name)
+      alter table roles
-      drop index roles_pkey_1;
+        drop constraint if exists roles_pkey,
-      alter table roles drop column id;
+        drop column id,
+        add primary key (name);
+      drop index roles__name;
     `);
   },
 };

packages/schemas/alterations/1.0.0_beta.18-1671080370-terms-of-use.ts

@@ -72,7 +72,8 @@ const alteration: AlterationScript = {
     const rows = await pool.many<SignInExperience>(sql`select * from sign_in_experiences`);
 
     await pool.query(sql`
-      alter table sign_in_experiences add column terms_of_use jsonb not null default '{}'::jsonb
+      alter table sign_in_experiences add column terms_of_use jsonb not null default '{}'::jsonb;
+      alter table sign_in_experiences alter column terms_of_use drop default;
     `);
 
     await Promise.all(rows.map(async (row) => rollbackTermsOfUse(row, pool)));

packages/schemas/alterations/next-1672815959-user-roles.ts

@@ -62,7 +62,9 @@ const alteration: AlterationScript = {
 
       // eslint-disable-next-line no-await-in-loop
       await pool.query(sql`
-        update users set role_names = role_names || '[${role.name}]'::jsonb where id = ${relation.userId}
+        update users 
+        set role_names = role_names || ${sql.jsonb([role.name])}
+        where id = ${relation.userId}
       `);
     }
 

packages/schemas/alterations/next-1672901841-roles-and-scopes-not-null.ts

@@ -5,14 +5,18 @@ import type { AlterationScript } from '../lib/types/alteration.js';
 const alteration: AlterationScript = {
   up: async (pool) => {
     await pool.query(sql`
-      ALTER TABLE roles_scopes ALTER COLUMN role_id SET NOT NULL;
+      alter table roles_scopes alter column role_id set not null;
-      ALTER TABLE roles_scopes ALTER COLUMN scope_id SET NOT NULL;
+      alter table roles_scopes alter column scope_id set not null;
     `);
   },
   down: async (pool) => {
     await pool.query(sql`
-      ALTER TABLE roles_scopes ALTER COLUMN role_id DROP NOT NULL;
+      alter table roles_scopes
-      ALTER TABLE roles_scopes ALTER COLUMN scope_id DROP NOT NULL;
+        drop constraint if exists roles_permissison_pkey,
+        drop constraint if exists roles_scopes_pkey;
+      alter table roles_scopes alter column role_id drop not null;
+      alter table roles_scopes alter column scope_id drop not null;
+      alter table roles_scopes add primary key (role_id, scope_id)
     `);
   },
 };

packages/schemas/alterations/next-1673882867-fix-alteration-issues.ts

@@ -6,35 +6,63 @@ const alteration: AlterationScript = {
   up: async (pool) => {
     await pool.query(sql`
       alter table roles_scopes
-      drop constraint roles_scopes_role_id_fkey;
+        drop constraint roles_scopes_role_id_fkey;
     `);
     await pool.query(sql`
       alter table users_roles
-      drop constraint users_roles_role_id_fkey;
+        drop constraint users_roles_role_id_fkey;
     `);
-    await pool.query(sql`drop index roles_pkey;`);
     await pool.query(sql`
       alter table roles
-      add primary key (id);
+        drop constraint if exists roles_pkey;
+    `);
+    await pool.query(sql`drop index if exists roles_pkey;`);
+    await pool.query(sql`
+      alter table roles
+        add primary key (id);
     `);
 
     await pool.query(sql`
       alter table roles_scopes
-      drop constraint roles_permissison_pkey,
+        drop constraint if exists roles_permissison_pkey,
-      add primary key (role_id, scope_id);
+        drop constraint if exists roles_scopes_pkey,
+        add primary key (role_id, scope_id);
     `);
 
     await pool.query(sql`
       alter table users_roles
-      add foreign key (role_id) references roles (id) on update cascade on delete cascade;
+        add foreign key (role_id) references roles (id) on update cascade on delete cascade;
     `);
     await pool.query(sql`
       alter table roles_scopes
-      add foreign key (role_id) references roles (id) on update cascade on delete cascade;
+        add foreign key (role_id) references roles (id) on update cascade on delete cascade;
     `);
   },
   down: async (pool) => {
-    throw new Error('Not implemented');
+    await pool.query(sql`
+      alter table roles_scopes
+        drop constraint roles_scopes_role_id_fkey;
+    `);
+    await pool.query(sql`
+      alter table users_roles
+        drop constraint users_roles_role_id_fkey;
+    `);
+
+    await pool.query(sql`
+      alter table roles_scopes
+        drop constraint if exists roles_permissison_pkey,
+        drop constraint if exists roles_scopes_pkey,
+        add primary key (role_id, scope_id);
+    `);
+
+    await pool.query(sql`
+      alter table users_roles
+        add foreign key (role_id) references roles (id) on update cascade on delete cascade;
+    `);
+    await pool.query(sql`
+      alter table roles_scopes
+        add foreign key (role_id) references roles (id) on update cascade on delete cascade;
+    `);
   },
 };