From b8fd594feabc70478dcd257bf3fefbc80470df8c Mon Sep 17 00:00:00 2001 From: Gao Sun Date: Fri, 15 Dec 2023 16:50:14 +0800 Subject: [PATCH] refactor: fix workflow --- .github/workflows/main.yml | 20 +++++- .scripts/compare-database.js | 16 +++-- .../cli/src/commands/database/seed/index.ts | 37 ++++++---- .../cli/src/commands/database/seed/tables.ts | 70 ++++++++++++++++--- .../database/seed/tenant-organizations.ts | 16 ++--- .../next-1702544178-sync-tenant-orgs.ts | 59 +++++++--------- 6 files changed, 145 insertions(+), 73 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6b684a01a..811ddfd42 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -140,7 +140,15 @@ jobs: working-directory: ./alteration run: | cd packages/cli - pnpm start db seed --test + pnpm start db seed + env: + DB_URL: postgres://postgres:postgres@localhost:5432/alteration + + # FIXME: Last version of CLI doesn't support test data seeding. Here's a temporary workaround. + # We will remove this step when a new version of CLI is released. + - name: Setup alteration database test data + working-directory: ./fresh + run: pnpm cli db seed --testOnly env: DB_URL: postgres://postgres:postgres@localhost:5432/alteration @@ -161,7 +169,15 @@ jobs: working-directory: ./alteration run: | cd packages/cli - pnpm start db seed --test + pnpm start db seed + env: + DB_URL: postgres://postgres:postgres@localhost:5432/old + + # FIXME: Last version of CLI doesn't support test data seeding. Here's a temporary workaround. + # We will remove this step when a new version of CLI is released. + - name: Setup old database test data + working-directory: ./fresh + run: pnpm cli db seed --testOnly env: DB_URL: postgres://postgres:postgres@localhost:5432/old diff --git a/.scripts/compare-database.js b/.scripts/compare-database.js index 200b7ca13..5a44b4f2f 100644 --- a/.scripts/compare-database.js +++ b/.scripts/compare-database.js @@ -7,6 +7,15 @@ const omitArray = (arrayOfObjects, ...keys) => arrayOfObjects.map((value) => omi const schemas = ['cloud', 'public']; const schemasArray = `(${schemas.map((schema) => `'${schema}'`).join(', ')})`; +const tryCompare = (a, b) => { + try { + assert.deepStrictEqual(a, b); + } catch (error) { + console.error(error.toString()); + process.exit(1); + } +}; + const queryDatabaseManifest = async (database) => { const pool = new pg.Pool({ database, user: 'postgres', password: 'postgres' }); @@ -150,7 +159,7 @@ const manifests = [ await queryDatabaseManifest(database2), ]; -assert.deepStrictEqual(...manifests); +tryCompare(...manifests); const autoCompare = (a, b) => { if (typeof a !== typeof b) { @@ -200,7 +209,4 @@ const queryDatabaseData = async (database) => { console.log('Compare database data between', database1, 'and', database2); -assert.deepStrictEqual( - await queryDatabaseData(database1), - await queryDatabaseData(database2), -); +tryCompare(await queryDatabaseData(database1), await queryDatabaseData(database2)); diff --git a/packages/cli/src/commands/database/seed/index.ts b/packages/cli/src/commands/database/seed/index.ts index c718fda5f..7cff7ac9c 100644 --- a/packages/cli/src/commands/database/seed/index.ts +++ b/packages/cli/src/commands/database/seed/index.ts @@ -9,7 +9,12 @@ import { getAlterationDirectory } from '../alteration/utils.js'; import { createTables, seedCloud, seedTables, seedTest } from './tables.js'; -export const seedByPool = async (pool: DatabasePool, cloud = false, test = false) => { +export const seedByPool = async ( + pool: DatabasePool, + cloud = false, + test = false, + testOnly = false +) => { await pool.transaction(async (connection) => { // Check alteration scripts available in order to insert correct timestamp const latestTimestamp = await getLatestAlterationTimestamp(); @@ -21,16 +26,18 @@ export const seedByPool = async (pool: DatabasePool, cloud = false, test = false ); } - await oraPromise(createTables(connection), { - text: 'Create tables', - }); - await seedTables(connection, latestTimestamp, cloud); + if (!testOnly) { + await oraPromise(createTables(connection), { + text: 'Create tables', + }); + await seedTables(connection, latestTimestamp, cloud); - if (cloud) { - await seedCloud(connection); + if (cloud) { + await seedCloud(connection); + } } - if (test) { + if (test || testOnly) { await seedTest(connection); } }); @@ -38,7 +45,7 @@ export const seedByPool = async (pool: DatabasePool, cloud = false, test = false const seed: CommandModule< Record, - { swe?: boolean; cloud?: boolean; test?: boolean } + { swe?: boolean; cloud?: boolean; test?: boolean; 'test-only'?: boolean } > = { command: 'seed [type]', describe: 'Create database then seed tables and data', @@ -56,8 +63,12 @@ const seed: CommandModule< .option('test', { describe: 'Seed additional test data', type: 'boolean', + }) + .option('test-only', { + describe: 'Seed test data only, this option conflicts with `--cloud`', + type: 'boolean', }), - handler: async ({ swe, cloud, test }) => { + handler: async ({ swe, cloud, test, testOnly }) => { const pool = await createPoolAndDatabaseIfNeeded(); if (swe && (await doesConfigsTableExist(pool))) { @@ -68,7 +79,7 @@ const seed: CommandModule< } try { - await seedByPool(pool, cloud, test); + await seedByPool(pool, cloud, test, testOnly); } catch (error: unknown) { consoleLog.error(error); consoleLog.error( @@ -76,8 +87,10 @@ const seed: CommandModule< ' Nothing has changed since the seeding process was in a transaction.\n' + ' Try to fix the error and seed again.' ); + throw error; + } finally { + await pool.end(); } - await pool.end(); }, }; diff --git a/packages/cli/src/commands/database/seed/tables.ts b/packages/cli/src/commands/database/seed/tables.ts index 27231d10c..1e50e1ed5 100644 --- a/packages/cli/src/commands/database/seed/tables.ts +++ b/packages/cli/src/commands/database/seed/tables.ts @@ -20,7 +20,14 @@ import { LogtoConfigs, SignInExperiences, Applications, + OrganizationUserRelations, + getTenantOrganizationId, + Users, + OrganizationRoleUserRelations, + TenantRole, + Organizations, } from '@logto/schemas'; +import { getTenantRole } from '@logto/schemas'; import { Tenants } from '@logto/schemas/models'; import { convertToIdentifiers, generateStandardId } from '@logto/shared'; import type { DatabaseTransactionConnection } from 'slonik'; @@ -179,7 +186,7 @@ export const seedTables = async ( updateDatabaseTimestamp(connection, latestTimestamp), ]); - await seedTenantOrganizations(connection, isCloud); + await seedTenantOrganizations(connection); consoleLog.succeed('Seed data'); }; @@ -217,25 +224,72 @@ export const seedTest = async (connection: DatabaseTransactionConnection) => { ) ); + const userIds = Object.freeze(['test-1', 'test-2'] as const); await Promise.all([ connection.query( - insertInto({ id: 'test-1', username: 'test1', tenantId: adminTenantId }, 'users') + insertInto({ id: userIds[0], username: 'test1', tenantId: adminTenantId }, Users.table) ), connection.query( - insertInto({ id: 'test-2', username: 'test2', tenantId: adminTenantId }, 'users') + insertInto({ id: userIds[1], username: 'test2', tenantId: adminTenantId }, Users.table) ), ]); - consoleLog.succeed('Created test users'); const adminTenantRole = await getManagementRole(adminTenantId); const defaultTenantRole = await getManagementRole(defaultTenantId); await Promise.all([ - assignRoleToUser('test-1', adminTenantRole.id), - assignRoleToUser('test-1', defaultTenantRole.id), - assignRoleToUser('test-2', defaultTenantRole.id), + assignRoleToUser(userIds[0], adminTenantRole.id), + assignRoleToUser(userIds[0], defaultTenantRole.id), + assignRoleToUser(userIds[1], defaultTenantRole.id), + ]); + consoleLog.succeed('Assigned tenant management roles to the test users'); + + const organizations = convertToIdentifiers(Organizations); + const isTenantOrganizationInitialized = await connection.exists( + sql` + select 1 + from ${organizations.table} + where ${organizations.fields.tenantId} = ${getTenantOrganizationId(adminTenantId)} + ` + ); + + // This check is for older versions (<=v.12.0) that don't have tenant organization initialized. + if (!isTenantOrganizationInitialized) { + consoleLog.warn('Tenant organization is not enabled, skip seeding tenant organization data'); + return; + } + + const addOrganizationMembership = async (userId: string, tenantId: string) => + connection.query( + insertInto( + { userId, organizationId: getTenantOrganizationId(tenantId), tenantId: adminTenantId }, + OrganizationUserRelations.table + ) + ); + + await Promise.all([ + addOrganizationMembership(userIds[0], adminTenantId), + addOrganizationMembership(userIds[0], defaultTenantId), + addOrganizationMembership(userIds[1], defaultTenantId), ]); - consoleLog.succeed('Assigned tenant management roles to the test users'); + const assignOrganizationRole = async (userId: string, tenantId: string, tenantRole: TenantRole) => + connection.query( + insertInto( + { + userId, + organizationRoleId: getTenantRole(tenantRole).id, + organizationId: getTenantOrganizationId(tenantId), + tenantId: adminTenantId, + }, + OrganizationRoleUserRelations.table + ) + ); + + await Promise.all([ + assignOrganizationRole(userIds[0], adminTenantId, TenantRole.Owner), + assignOrganizationRole(userIds[0], defaultTenantId, TenantRole.Owner), + assignOrganizationRole(userIds[1], defaultTenantId, TenantRole.Owner), + ]); }; diff --git a/packages/cli/src/commands/database/seed/tenant-organizations.ts b/packages/cli/src/commands/database/seed/tenant-organizations.ts index eab428ff9..66bcf3510 100644 --- a/packages/cli/src/commands/database/seed/tenant-organizations.ts +++ b/packages/cli/src/commands/database/seed/tenant-organizations.ts @@ -33,17 +33,14 @@ import { consoleLog } from '../../../utils.js'; * - Organization roles and scopes for tenant management. * - Create tenant organizations for the initial tenants (`default` and `admin`). * - * If it is a cloud deployment, it will also seed the following: + * The following data are used for Logto Cloud, we seed them anyway for the sake of simplicity: * * - Machine-to-machine roles for Management API proxy. * - Assign the corresponding Management API scopes to the machine-to-machine roles. * - Machine-to-machine applications for Management API proxy. * - Assign the roles to the corresponding applications. */ -export const seedTenantOrganizations = async ( - connection: DatabaseTransactionConnection, - isCloud: boolean -) => { +export const seedTenantOrganizations = async (connection: DatabaseTransactionConnection) => { const tenantIds = [defaultTenantId, adminTenantId]; // Init organization template @@ -87,10 +84,7 @@ export const seedTenantOrganizations = async ( ); consoleLog.succeed('Created tenant organizations'); - if (!isCloud) { - return; - } - + /* === Cloud-specific data === */ // Init Management API proxy roles await connection.query( insertInto( @@ -101,8 +95,8 @@ export const seedTenantOrganizations = async ( consoleLog.succeed('Created Management API proxy roles'); // Prepare Management API scopes - const scopes = convertToIdentifiers(Scopes); - const resources = convertToIdentifiers(Resources); + const scopes = convertToIdentifiers(Scopes, true); + const resources = convertToIdentifiers(Resources, true); /** Scopes with the name {@link PredefinedScope.All} in all Management API resources. */ const allScopes = await connection.any<{ id: string; indicator: string }>(sql` select ${scopes.fields.id}, ${resources.fields.indicator} diff --git a/packages/schemas/alterations/next-1702544178-sync-tenant-orgs.ts b/packages/schemas/alterations/next-1702544178-sync-tenant-orgs.ts index 2991c5481..77e95b5ea 100644 --- a/packages/schemas/alterations/next-1702544178-sync-tenant-orgs.ts +++ b/packages/schemas/alterations/next-1702544178-sync-tenant-orgs.ts @@ -2,14 +2,16 @@ * @fileoverview A preparation for the update of using organizations to manage tenants in the admin * tenant. This script will do the following in the admin tenant: * - * 1. Disable registration. - * 2. Create organization template roles and scopes. - * 3. Create organizations for existing tenants. - * 4. Add membership records and assign organization roles to existing users. - * 5. Create machine-to-machine Management API role for each tenant. - * 6. Create the corresponding machine-to-machine app for each tenant, and assign the Management API role to it. + * 1. Create organization template roles and scopes. + * 2. Create organizations for existing tenants. + * 3. Add membership records and assign organization roles to existing users. + * 4. Create machine-to-machine Management API role for each tenant. + * 5. Create the corresponding machine-to-machine app for each tenant, and assign the Management API role to it. * * The `down` script will revert the changes. + * + * NOTE: In order to avoid unnecessary dirty data, it's recommended disabling the registration of + * new tenants before running this script and deploying the changes. */ import { ConsoleLog, generateStandardId } from '@logto/shared'; @@ -23,27 +25,21 @@ const consoleLog = new ConsoleLog(); const alteration: AlterationScript = { up: async (transaction) => { consoleLog.info('=== Sync tenant organizations ==='); - consoleLog.info('Disable registration'); - await transaction.query(sql` - update public.sign_in_experiences - set sign_in_mode = 'SignIn' - where tenant_id = ${adminTenantId}; - `); consoleLog.info('Create organization template roles and scopes'); await transaction.query(sql` insert into public.organization_roles (id, tenant_id, name, description) values ('owner', ${adminTenantId}, 'owner', 'Owner of the tenant, who has all permissions.'), - ('admin', ${adminTenantId}, 'admin', 'Admin of the tenant, who has all permissions except deleting the tenant.'), - ('member', ${adminTenantId}, 'member', 'Member of the tenant, who has limited permissions.'); + ('admin', ${adminTenantId}, 'admin', 'Admin of the tenant, who has all permissions except managing the tenant settings.'), + ('member', ${adminTenantId}, 'member', 'Member of the tenant, who has limited permissions on reading and writing the tenant data.'); `); await transaction.query(sql` insert into public.organization_scopes (id, tenant_id, name, description) values - ('read-tenant', ${adminTenantId}, 'read:tenant', 'Read the tenant data.'), - ('write-tenant', ${adminTenantId}, 'write:tenant', 'Write the tenant data, including creating and updating the tenant.'), - ('delete-tenant', ${adminTenantId}, 'delete:tenant', 'Delete data of the tenant.'), + ('read-data', ${adminTenantId}, 'read:data', 'Read the tenant data.'), + ('write-data', ${adminTenantId}, 'write:data', 'Write the tenant data, including creating and updating the tenant.'), + ('delete-data', ${adminTenantId}, 'delete:data', 'Delete data of the tenant.'), ('invite-member', ${adminTenantId}, 'invite:member', 'Invite members to the tenant.'), ('remove-member', ${adminTenantId}, 'remove:member', 'Remove members from the tenant.'), ('update-member-role', ${adminTenantId}, 'update:member:role', 'Update the role of a member in the tenant.'), @@ -52,21 +48,21 @@ const alteration: AlterationScript = { await transaction.query(sql` insert into public.organization_role_scope_relations (tenant_id, organization_role_id, organization_scope_id) values - (${adminTenantId}, 'owner', 'read-tenant'), - (${adminTenantId}, 'owner', 'write-tenant'), - (${adminTenantId}, 'owner', 'delete-tenant'), + (${adminTenantId}, 'owner', 'read-data'), + (${adminTenantId}, 'owner', 'write-data'), + (${adminTenantId}, 'owner', 'delete-data'), (${adminTenantId}, 'owner', 'invite-member'), (${adminTenantId}, 'owner', 'remove-member'), (${adminTenantId}, 'owner', 'update-member-role'), (${adminTenantId}, 'owner', 'manage-tenant'), - (${adminTenantId}, 'admin', 'read-tenant'), - (${adminTenantId}, 'admin', 'write-tenant'), - (${adminTenantId}, 'admin', 'delete-tenant'), + (${adminTenantId}, 'admin', 'read-data'), + (${adminTenantId}, 'admin', 'write-data'), + (${adminTenantId}, 'admin', 'delete-data'), (${adminTenantId}, 'admin', 'invite-member'), (${adminTenantId}, 'admin', 'remove-member'), (${adminTenantId}, 'admin', 'update-member-role'), - (${adminTenantId}, 'member', 'read-tenant'), - (${adminTenantId}, 'member', 'write-tenant'), + (${adminTenantId}, 'member', 'read-data'), + (${adminTenantId}, 'member', 'write-data'), (${adminTenantId}, 'member', 'invite-member') `); @@ -274,9 +270,9 @@ const alteration: AlterationScript = { delete from public.organization_scopes where public.organization_scopes.tenant_id = ${adminTenantId} and public.organization_scopes.id in ( - 'read-tenant', - 'write-tenant', - 'delete-tenant', + 'read-data', + 'write-data', + 'delete-data', 'invite-member', 'remove-member', 'update-member-role', @@ -284,13 +280,6 @@ const alteration: AlterationScript = { ); `); - consoleLog.info('Enable registration'); - await transaction.query(sql` - update public.sign_in_experiences - set sign_in_mode = 'SignInAndRegister' - where tenant_id = ${adminTenantId}; - `); - consoleLog.info('=== Revert sync tenant organizations done ==='); }, };