refactor: seed data for multi-tenancy 2

.scripts/compare-database.js

@@ -114,6 +114,19 @@ const manifests = [
 
 assert.deepStrictEqual(...manifests);
 
+const autoCompare = (a, b) => {
+  if (typeof a !== typeof b) {
+    return (typeof a).localeCompare(typeof b);
+  }
+
+  return String(a).localeCompare(String(b));
+};
+
+const buildSortByKeys = (keys) => (a, b) => {
+  const found = keys.find((key) => a[key] !== b[key]);
+  return found ? autoCompare(a[found], b[found]) : 0;
+};
+
 const queryDatabaseData = async (database) => {
   const pool = new pg.Pool({ database, user: 'postgres', password: 'postgres' });
   const result = await Promise.all(manifests[0].tables
@@ -122,16 +135,11 @@ const queryDatabaseData = async (database) => {
 
       // check config rows except the value column
       if (['logto_configs', '_logto_configs', 'systems'].includes(table_name)) {
-        return [table_name, omitArray(rows, 'value').sort((a, b) => {
+        const data = omitArray(rows, 'value');
-          if (a.tenant_id === b.tenant_id) {
+        return [table_name, data.sort(buildSortByKeys(Object.keys(data[0] ?? {})))];
-            return a.key.localeCompare(b.key);
-          }
-
-          return a.tenant_id.localeCompare(b.tenant_id);
-        })];
       }
 
-      return [table_name, omitArray(
+      const data = omitArray(
         rows,
         'id',
         'resource_id',
@@ -142,7 +150,9 @@ const queryDatabaseData = async (database) => {
         'secret',
         'db_user',
         'db_user_password'
-      )];
+      );
+
+      return [table_name, data.sort(buildSortByKeys(Object.keys(data[0] ?? {})))];
     })
   );
 

packages/cli/src/commands/database/seed/cloud.ts

@@ -0,0 +1,31 @@
+import { adminConsoleApplicationId, adminTenantId, defaultTenantId } from '@logto/schemas';
+import { appendPath, GlobalValues } from '@logto/shared';
+import type { CommonQueryMethods } from 'slonik';
+import { sql } from 'slonik';
+
+import { log } from '../../../utils.js';
+
+export const appendAdminConsoleRedirectUris = async (pool: CommonQueryMethods) => {
+  const redirectUris = new GlobalValues().cloudUrlSet
+    .deduplicated()
+    .map((endpoint) => appendPath(endpoint, defaultTenantId, 'callback'));
+
+  const metadataKey = sql.identifier(['oidc_client_metadata']);
+
+  // Copied from packages/cloud/src/queries/tenants.ts
+  // Can be merged into the original once we remove slonik
+  await pool.query(sql`
+    update applications
+    set ${metadataKey} = jsonb_set(
+      ${metadataKey},
+      '{redirectUris}',
+      (select jsonb_agg(distinct value) from jsonb_array_elements(
+        ${metadataKey}->'redirectUris' || ${sql.jsonb(redirectUris.map(String))}
+      ))
+    )
+    where id = ${adminConsoleApplicationId}
+    and tenant_id = ${adminTenantId}
+  `);
+
+  log.succeed('Appended initial Redirect URIs to Admin Console:', redirectUris.map(String));
+};

packages/cli/src/commands/database/seed/index.ts

@@ -7,38 +7,33 @@ import { doesConfigsTableExist } from '../../../queries/logto-config.js';
 import { log, oraPromise } from '../../../utils.js';
 import { getLatestAlterationTimestamp } from '../alteration/index.js';
 import { getAlterationDirectory } from '../alteration/utils.js';
-import { createTables, seedTables } from './tables.js';
+import { createTables, seedCloud, seedTables } from './tables.js';
 
-const seedChoices = Object.freeze(['all', 'oidc'] as const);
+export const seedByPool = async (pool: DatabasePool, cloud = false) => {
-
-type SeedChoice = (typeof seedChoices)[number];
-
-export const seedByPool = async (pool: DatabasePool, type: SeedChoice) => {
   await pool.transaction(async (connection) => {
-    if (type !== 'oidc') {
+    // Check alteration scripts available in order to insert correct timestamp
-      // Check alteration scripts available in order to insert correct timestamp
+    const latestTimestamp = await getLatestAlterationTimestamp();
-      const latestTimestamp = await getLatestAlterationTimestamp();
 
-      if (latestTimestamp < 1) {
+    if (latestTimestamp < 1) {
-        throw new Error(
+      throw new Error(
-          `No alteration script found when seeding the database.\n` +
+        `No alteration script found when seeding the database.\n` +
-            `Please check \`${getAlterationDirectory()}\` to see if there are alteration scripts available.\n`
+          `Please check \`${getAlterationDirectory()}\` to see if there are alteration scripts available.\n`
-        );
+      );
-      }
+    }
 
-      await oraPromise(createTables(connection), {
+    await oraPromise(createTables(connection), {
-        text: 'Create tables',
+      text: 'Create tables',
-        prefixText: chalk.blue('[info]'),
+      prefixText: chalk.blue('[info]'),
-      });
+    });
-      await oraPromise(seedTables(connection, latestTimestamp), {
+    await seedTables(connection, latestTimestamp);
-        text: 'Seed data',
+
-        prefixText: chalk.blue('[info]'),
+    if (cloud) {
-      });
+      await seedCloud(connection);
     }
   });
 };
 
-const seed: CommandModule<Record<string, unknown>, { type: string; swe?: boolean }> = {
+const seed: CommandModule<Record<string, unknown>, { swe?: boolean; cloud?: boolean }> = {
   command: 'seed [type]',
   describe: 'Create database then seed tables and data',
   builder: (yargs) =>
@@ -48,13 +43,12 @@ const seed: CommandModule<Record<string, unknown>, { type: string; swe?: boolean
         alias: 'skip-when-exists',
         type: 'boolean',
       })
-      .positional('type', {
+      .option('cloud', {
-        describe: 'Optional seed type',
+        describe: 'Seed additional cloud data',
-        type: 'string',
+        type: 'boolean',
-        choices: seedChoices,
+        hidden: true,
-        default: 'all',
       }),
-  handler: async ({ type, swe }) => {
+  handler: async ({ swe, cloud }) => {
     const pool = await createPoolAndDatabaseIfNeeded();
 
     if (swe && (await doesConfigsTableExist(pool))) {
@@ -65,10 +59,7 @@ const seed: CommandModule<Record<string, unknown>, { type: string; swe?: boolean
     }
 
     try {
-      // Cannot avoid `as` since the official type definition of `yargs` doesn't work.
+      await seedByPool(pool, cloud);
-      // The value of `type` can be ensured, so it's safe to use `as` here.
-      // eslint-disable-next-line no-restricted-syntax
-      await seedByPool(pool, type as SeedChoice);
     } catch (error: unknown) {
       console.error(error);
       console.log();

packages/cli/src/commands/database/seed/tables.ts

@@ -21,7 +21,8 @@ import { raw } from 'slonik-sql-tag-raw';
 import { insertInto } from '../../../database.js';
 import { getDatabaseName } from '../../../queries/database.js';
 import { updateDatabaseTimestamp } from '../../../queries/system.js';
-import { getPathInModule } from '../../../utils.js';
+import { getPathInModule, log } from '../../../utils.js';
+import { appendAdminConsoleRedirectUris } from './cloud.js';
 import { seedOidcConfigs } from './oidc-config.js';
 import { createTenant, seedAdminData } from './tenant.js';
 
@@ -136,4 +137,10 @@ export const seedTables = async (
     connection.query(insertInto(createDefaultAdminConsoleApplication(), 'applications')),
     updateDatabaseTimestamp(connection, latestTimestamp),
   ]);
+
+  log.succeed('Seed data');
+};
+
+export const seedCloud = async (connection: DatabaseTransactionConnection) => {
+  await appendAdminConsoleRedirectUris(connection);
 };

packages/cli/src/commands/install/utils.ts

@@ -136,7 +136,7 @@ export const decompress = async (toPath: string, tarPath: string) => {
 export const seedDatabase = async (instancePath: string) => {
   try {
     const pool = await createPoolAndDatabaseIfNeeded();
-    await seedByPool(pool, 'all');
+    await seedByPool(pool);
     await pool.end();
   } catch (error: unknown) {
     console.error(error);

packages/cloud/nodemon.json

@@ -6,9 +6,10 @@
   ],
   "watch": [
     "./src/",
+    "../core/src/",
     "./node_modules/",
     "../../.env"
   ],
   "ext": "json,js,jsx,ts,tsx",
-  "delay": 500
+  "delay": 1000
 }

packages/cloud/package.json

@@ -29,6 +29,7 @@
     "chalk": "^5.0.0",
     "decamelize": "^6.0.0",
     "dotenv": "^16.0.0",
+    "fetch-retry": "^5.0.4",
     "find-up": "^6.3.0",
     "http-proxy": "^1.18.1",
     "jose": "^4.11.0",

packages/cloud/src/middleware/with-auth.ts

@@ -5,6 +5,7 @@ import path from 'node:path/posix';
 import { tryThat } from '@logto/shared';
 import type { NextFunction, RequestContext } from '@withtyped/server';
 import { RequestError } from '@withtyped/server';
+import fetchRetry from 'fetch-retry';
 import { createRemoteJWKSet, jwtVerify } from 'jose';
 import { z } from 'zod';
 
@@ -46,9 +47,13 @@ export default function withAuth<InputContext extends RequestContext>({
   audience,
   scopes: expectScopes = [],
 }: WithAuthConfig) {
+  const fetch = fetchRetry(global.fetch);
   const getJwkSet = (async () => {
     const fetched = await fetch(
-      new URL(path.join(endpoint.pathname, 'oidc/.well-known/openid-configuration'), endpoint)
+      new Request(
+        new URL(path.join(endpoint.pathname, 'oidc/.well-known/openid-configuration'), endpoint)
+      ),
+      { retries: 5, retryDelay: (attempt) => 2 ** attempt * 1000 }
     );
     const { jwks_uri: jwksUri, issuer } = z
       .object({ jwks_uri: z.string(), issuer: z.string() })

packages/cloud/src/queries/tenants.ts

@@ -81,7 +81,9 @@ export const createTenantsQueries = (client: Queryable<PostgreSql>) => {
       set ${metadataKey} = jsonb_set(
         ${metadataKey},
         '{redirectUris}',
-        ${metadataKey}->'redirectUris' || ${jsonb(urls.map(String))}
+        (select jsonb_agg(distinct value) from jsonb_array_elements(
+          ${metadataKey}->'redirectUris' || ${jsonb(urls.map(String))}
+        ))
       )
       where id = ${adminConsoleApplicationId}
       and tenant_id = ${adminTenantId}

packages/core/src/env-set/index.ts

@@ -1,3 +1,4 @@
+import { GlobalValues, appendPath } from '@logto/shared';
 import type { Optional } from '@silverhand/essentials';
 import type { PostgreSql } from '@withtyped/postgres';
 import type { QueryClient } from '@withtyped/server';
@@ -5,9 +6,7 @@ import type { DatabasePool } from 'slonik';
 
 import { createLogtoConfigLibrary } from '#src/libraries/logto-config.js';
 import { createLogtoConfigQueries } from '#src/queries/logto-config.js';
-import { appendPath } from '#src/utils/url.js';
 
-import GlobalValues from './GlobalValues.js';
 import createPool from './create-pool.js';
 import createQueryClient from './create-query-client.js';
 import loadOidcValues from './oidc.js';

packages/core/src/env-set/throw-errors.ts

@@ -1,28 +1,5 @@
-import chalk from 'chalk';
-
 export const throwNotLoadedError = () => {
   throw new Error(
     'The env set is not loaded. Make sure to call `await envSet.load()` before using it.'
   );
 };
-
-export const throwErrorWithDsnMessage = (error: unknown) => {
-  const key = 'DB_URL';
-
-  if (error instanceof Error && error.message === `env variable ${key} not found`) {
-    console.error(
-      `${chalk.red('[error]')} No Postgres DSN (${chalk.green(key)}) found in env variables.\n\n` +
-        `  Either provide it in your env, or add it to the ${chalk.blue(
-          '.env'
-        )} file in the Logto project root.\n\n` +
-        `  If you want to set up a new Logto database, run ${chalk.green(
-          'npm run cli db seed'
-        )} before setting env ${chalk.green(key)}.\n\n` +
-        `  Visit ${chalk.blue(
-          'https://docs.logto.io/docs/references/core/configuration'
-        )} for more info about setting up env.\n`
-    );
-  }
-
-  throw error;
-};

packages/core/src/env-set/utils.ts

@@ -1,11 +1,10 @@
 import path from 'path';
 
 import { adminTenantId } from '@logto/schemas';
+import type { GlobalValues } from '@logto/shared';
 import type { Optional } from '@silverhand/essentials';
 import { deduplicate, trySafe } from '@silverhand/essentials';
 
-import type GlobalValues from './GlobalValues.js';
-
 export const getTenantEndpoint = (
   id: string,
   { urlSet, adminUrlSet, isDomainBasedMultiTenancy, isPathBasedMultiTenancy }: GlobalValues

packages/core/src/middleware/koa-auth/utils.ts

@@ -7,13 +7,12 @@ import {
   LogtoOidcConfigKey,
   LogtoConfigs,
 } from '@logto/schemas';
-import { convertToIdentifiers } from '@logto/shared';
+import { convertToIdentifiers, appendPath } from '@logto/shared';
 import type { JWK } from 'jose';
 import { sql } from 'slonik';
 
 import { EnvSet, getTenantEndpoint } from '#src/env-set/index.js';
 import { exportJWK } from '#src/utils/jwks.js';
-import { appendPath } from '#src/utils/url.js';
 
 const { table, fields } = convertToIdentifiers(LogtoConfigs);
 

packages/core/src/middleware/koa-cors.test.ts

@@ -1,8 +1,7 @@
+import { GlobalValues, UrlSet } from '@logto/shared';
 import { createMockUtils } from '@logto/shared/esm';
 import type { RequestMethod } from 'node-mocks-http';
 
-import GlobalValues from '#src/env-set/GlobalValues.js';
-import UrlSet from '#src/env-set/UrlSet.js';
 import createMockContext from '#src/test-utils/jest-koa-mocks/create-mock-context.js';
 
 const { jest } = import.meta;

packages/core/src/middleware/koa-cors.ts

@@ -1,7 +1,7 @@
 import cors from '@koa/cors';
+import type { UrlSet } from '@logto/shared';
 import type { MiddlewareType } from 'koa';
 
-import type UrlSet from '#src/env-set/UrlSet.js';
 import { EnvSet } from '#src/env-set/index.js';
 
 export default function koaCors<StateT, ContextT, ResponseBodyT>(

packages/core/src/middleware/koa-spa-session-guard.ts

@@ -1,9 +1,9 @@
+import { appendPath } from '@logto/shared';
 import type { MiddlewareType } from 'koa';
 import type { IRouterParamContext } from 'koa-router';
 import type Provider from 'oidc-provider';
 
 import { EnvSet } from '#src/env-set/index.js';
-import { appendPath } from '#src/utils/url.js';
 
 // Need To Align With UI
 export const sessionNotFoundPath = '/unknown-session';

packages/core/src/oidc/adapter.ts

@@ -1,6 +1,6 @@
 import type { CreateApplication } from '@logto/schemas';
 import { ApplicationType, adminConsoleApplicationId, demoAppApplicationId } from '@logto/schemas';
-import { tryThat } from '@logto/shared';
+import { tryThat, appendPath } from '@logto/shared';
 import { addSeconds } from 'date-fns';
 import type { AdapterFactory, AllClientMetadata } from 'oidc-provider';
 import { errors } from 'oidc-provider';
@@ -9,7 +9,6 @@ import snakecaseKeys from 'snakecase-keys';
 import { EnvSet } from '#src/env-set/index.js';
 import { getTenantUrls } from '#src/env-set/utils.js';
 import type Queries from '#src/tenants/Queries.js';
-import { appendPath } from '#src/utils/url.js';
 
 import { getConstantClientMetadata } from './utils.js';
 

packages/core/src/utils/tenant.test.ts

@@ -1,8 +1,7 @@
 import { adminTenantId, defaultTenantId } from '@logto/schemas';
+import { GlobalValues } from '@logto/shared';
 import { createMockUtils } from '@logto/shared/esm';
 
-import GlobalValues from '#src/env-set/GlobalValues.js';
-
 const { jest } = import.meta;
 
 const { mockEsmWithActual } = createMockUtils(jest);

packages/core/src/utils/tenant.ts

@@ -1,7 +1,7 @@
 import { adminTenantId, defaultTenantId } from '@logto/schemas';
+import type { UrlSet } from '@logto/shared';
 import { conditionalString } from '@silverhand/essentials';
 
-import type UrlSet from '#src/env-set/UrlSet.js';
 import { EnvSet, getTenantEndpoint } from '#src/env-set/index.js';
 
 const normalizePathname = (pathname: string) =>

packages/schemas/alterations/next-1677765137-seed-for-admin-tenant.ts

@@ -15,6 +15,11 @@ const addApiData = async (pool: CommonQueryMethods) => {
     resourceId: generateStandardId(),
     scopeId: generateStandardId(),
   };
+  const adminRole = {
+    id: generateStandardId(),
+    name: 'admin:admin',
+    description: 'Admin role for Logto.',
+  };
 
   await pool.query(sql`
     insert into resources (tenant_id, id, indicator, name)
@@ -27,7 +32,7 @@ const addApiData = async (pool: CommonQueryMethods) => {
         ${adminTenantId},
         ${cloudApi.resourceId},
         'https://cloud.logto.io/api',
-        'Logto Management API for tenant admin'
+        'Logto Cloud API'
       );
   `);
   await pool.query(sql`
@@ -37,17 +42,26 @@ const addApiData = async (pool: CommonQueryMethods) => {
         ${adminApi.scopeId},
         'all',
         'Default scope for Management API, allows all permissions.',
-        ${adminApi.scopeId}
+        ${adminApi.resourceId}
       ), (
         ${adminTenantId},
         ${cloudApi.scopeId},
         'create:tenant',
         'Allow creating new tenants.',
-        ${cloudApi.scopeId}
+        ${cloudApi.resourceId}
+      );
+  `);
+  await pool.query(sql`
+    insert into roles (tenant_id, id, name, description)
+      values (
+        ${adminTenantId},
+        ${adminRole.id},
+        ${adminRole.name},
+        ${adminRole.description}
       );
   `);
 
-  const { id: roleId } = await pool.one<{ id: string }>(sql`
+  const { id: userRoleId } = await pool.one<{ id: string }>(sql`
     select id from roles
     where tenant_id = ${adminTenantId}
     and name = 'user'
@@ -58,13 +72,13 @@ const addApiData = async (pool: CommonQueryMethods) => {
       values (
         ${adminTenantId},
         ${generateStandardId()},
-        ${roleId},
+        ${userRoleId},
-        ${adminApi.scopeId}
+        ${cloudApi.scopeId}
       ), (
         ${adminTenantId},
         ${generateStandardId()},
-        ${roleId},
+        ${adminRole.id},
-        ${cloudApi.scopeId}
+        ${adminApi.scopeId}
       );
   `);
 };
@@ -93,9 +107,19 @@ const alteration: AlterationScript = {
   },
   down: async (pool) => {
     await pool.query(sql`
-      delete from applications
+      delete from resources
-        where tenant_id = 'admin'
+        where tenant_id = ${adminTenantId}
-        and id = 'admin-console';
+        and indicator in ('https://admin.logto.app/api', 'https://cloud.logto.io/api');
+    `);
+    await pool.query(sql`
+      delete from roles
+        where tenant_id = ${adminTenantId}
+        and name = 'admin:admin';
+    `);
+    await pool.query(sql`
+      delete from logto_configs
+        where tenant_id = ${adminTenantId}
+        and key = 'adminConsole';
     `);
   },
 };

packages/shared/package.json

@@ -57,6 +57,7 @@
     "@logto/core-kit": "workspace:*",
     "@logto/schemas": "workspace:*",
     "@silverhand/essentials": "2.2.0",
+    "chalk": "^5.0.0",
     "find-up": "^6.3.0",
     "nanoid": "^4.0.0",
     "slonik": "^30.0.0"

packages/shared/src/env/GlobalValues.ts

@@ -1,6 +1,6 @@
-import { tryThat } from '@logto/shared';
 import { assertEnv, getEnv, getEnvAsStringArray, yes } from '@silverhand/essentials';
 
+import { tryThat } from '../utils/index.js';
 import UrlSet from './UrlSet.js';
 import { throwErrorWithDsnMessage } from './throw-errors.js';
 

packages/shared/src/env/throw-errors.ts

@@ -0,0 +1,28 @@
+import chalk from 'chalk';
+
+export const throwNotLoadedError = () => {
+  throw new Error(
+    'The env set is not loaded. Make sure to call `await envSet.load()` before using it.'
+  );
+};
+
+export const throwErrorWithDsnMessage = (error: unknown) => {
+  const key = 'DB_URL';
+
+  if (error instanceof Error && error.message === `env variable ${key} not found`) {
+    console.error(
+      `${chalk.red('[error]')} No Postgres DSN (${chalk.green(key)}) found in env variables.\n\n` +
+        `  Either provide it in your env, or add it to the ${chalk.blue(
+          '.env'
+        )} file in the Logto project root.\n\n` +
+        `  If you want to set up a new Logto database, run ${chalk.green(
+          'npm run cli db seed'
+        )} before setting env ${chalk.green(key)}.\n\n` +
+        `  Visit ${chalk.blue(
+          'https://docs.logto.io/docs/references/core/configuration'
+        )} for more info about setting up env.\n`
+    );
+  }
+
+  throw error;
+};

packages/shared/src/index.ts

@@ -1,3 +1,5 @@
 export * from './database/index.js';
 export * from './utils/index.js';
 export * from './models/index.js';
+export { default as UrlSet } from './env/UrlSet.js';
+export { default as GlobalValues } from './env/GlobalValues.js';

packages/shared/src/utils/index.ts

@@ -1,3 +1,4 @@
 export * from './function.js';
 export * from './object.js';
 export { default as findPackage } from './find-package.js';
+export * from './url.js';

pnpm-lock.yaml

@@ -122,6 +122,7 @@ importers:
       decamelize: ^6.0.0
       dotenv: ^16.0.0
       eslint: ^8.21.0
+      fetch-retry: ^5.0.4
       find-up: ^6.3.0
       http-proxy: ^1.18.1
       jose: ^4.11.0
@@ -142,6 +143,7 @@ importers:
       chalk: 5.1.2
       decamelize: 6.0.0
       dotenv: 16.0.0
+      fetch-retry: 5.0.4
       find-up: 6.3.0
       http-proxy: 1.18.1
       jose: 4.11.1
@@ -702,6 +704,7 @@ importers:
       '@silverhand/ts-config': 2.0.3
       '@types/jest': ^29.1.2
       '@types/node': ^18.11.18
+      chalk: ^5.0.0
       eslint: ^8.34.0
       find-up: ^6.3.0
       jest: ^29.1.2
@@ -714,6 +717,7 @@ importers:
       '@logto/core-kit': link:../toolkit/core-kit
       '@logto/schemas': link:../schemas
       '@silverhand/essentials': 2.2.0
+      chalk: 5.1.2
       find-up: 6.3.0
       nanoid: 4.0.0
       slonik: 30.1.2
@@ -7125,6 +7129,10 @@ packages:
       web-streams-polyfill: 3.2.1
     dev: true
 
+  /fetch-retry/5.0.4:
+    resolution: {integrity: sha512-LXcdgpdcVedccGg0AZqg+S8lX/FCdwXD92WNZ5k5qsb0irRhSFsBOpcJt7oevyqT2/C2nEE0zSFNdBEpj3YOSw==}
+    dev: false
+
   /figures/5.0.0:
     resolution: {integrity: sha512-ej8ksPF4x6e5wvK9yevct0UCXh8TTFlWGVLlgjZuoBH1HwjIfKE/IdL5mq89sFA7zELi1VhKpmtDnrs7zWyeyg==}
     engines: {node: '>=14'}