Merge pull request #3391 from logto-io/gao-use-env-to-set-pg-pool

refactor: support setting pool sizes by env variable

.github/workflows/release.yml

@@ -141,6 +141,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Setup Node and pnpm
+        if: ${{ (inputs.target || 'dev') == 'dev' }}
         uses: silverhand-io/actions-node-pnpm-run-steps@v2
 
       - name: Deploy database alteration

packages/core/src/env-set/create-pool.ts

@@ -2,7 +2,7 @@ import { assert } from '@silverhand/essentials';
 import { createMockPool, createMockQueryResult, createPool, parseDsn } from 'slonik';
 import { createInterceptors } from 'slonik-interceptor-preset';
 
-const createPoolByEnv = async (databaseDsn: string, isTest: boolean) => {
+const createPoolByEnv = async (databaseDsn: string, isTest: boolean, poolSize?: number) => {
   // Database connection is disabled in unit test environment
   if (isTest) {
     return createMockPool({ query: async () => createMockQueryResult([]) });
@@ -12,7 +12,7 @@ const createPoolByEnv = async (databaseDsn: string, isTest: boolean) => {
 
   assert(parseDsn(databaseDsn).databaseName, new Error('Database name is required'));
 
-  return createPool(databaseDsn, { interceptors });
+  return createPool(databaseDsn, { interceptors, maximumPoolSize: poolSize });
 };
 
 export default createPoolByEnv;

packages/core/src/env-set/index.ts

@@ -6,7 +6,7 @@ import type { DatabasePool } from 'slonik';
 import { createLogtoConfigLibrary } from '#src/libraries/logto-config.js';
 import { createLogtoConfigQueries } from '#src/queries/logto-config.js';
 
-import createPool from './create-pool.js';
+import createPoolByEnv from './create-pool.js';
 import loadOidcValues from './oidc.js';
 import { throwNotLoadedError } from './throw-errors.js';
 import { getTenantEndpoint } from './utils.js';
@@ -37,7 +37,7 @@ export class EnvSet {
     return this.values.dbUrl;
   }
 
-  static sharedPool = createPool(this.dbUrl, this.isTest);
+  static sharedPool = createPoolByEnv(this.dbUrl, this.isTest, this.values.databasePoolSize);
 
   #pool: Optional<DatabasePool>;
   #oidc: Optional<Awaited<ReturnType<typeof loadOidcValues>>>;
@@ -61,7 +61,11 @@ export class EnvSet {
   }
 
   async load() {
-    const pool = await createPool(this.databaseUrl, EnvSet.isTest);
+    const pool = await createPoolByEnv(
+      this.databaseUrl,
+      EnvSet.isTest,
+      EnvSet.values.databasePoolSize
+    );
 
     this.#pool = pool;
 

packages/core/src/libraries/hook.ts

@@ -52,7 +52,12 @@ export const createHookLibrary = (queries: Queries) => {
     }
 
     const hookEvent = eventToHook[event];
-    const rows = await findAllHooks();
+    const found = await findAllHooks();
+    const rows = found.filter(({ event }) => event === hookEvent);
+
+    if (rows.length === 0) {
+      return;
+    }
 
     const [user, application] = await Promise.all([
       trySafe(findUserById(userId)),
@@ -73,46 +78,44 @@ export const createHookLibrary = (queries: Queries) => {
     } satisfies Omit<HookEventPayload, 'hookId'>;
 
     await Promise.all(
-      rows
-        .filter(({ event }) => event === hookEvent)
-        .map(async ({ config: { url, headers, retries }, id }) => {
-          console.log(`\tTriggering hook ${id} due to ${hookEvent} event`);
-          const json: HookEventPayload = { hookId: id, ...payload };
-          const logEntry = new LogEntry(`TriggerHook.${hookEvent}`);
+      rows.map(async ({ config: { url, headers, retries }, id }) => {
+        console.log(`\tTriggering hook ${id} due to ${hookEvent} event`);
+        const json: HookEventPayload = { hookId: id, ...payload };
+        const logEntry = new LogEntry(`TriggerHook.${hookEvent}`);
 
-          logEntry.append({ json, hookId: id });
+        logEntry.append({ json, hookId: id });
 
-          // Trigger web hook and log response
-          await got
-            .post(url, {
-              headers: { 'user-agent': 'Logto (https://logto.io)', ...headers },
-              json,
-              retry: { limit: retries },
-              timeout: { request: 10_000 },
-            })
-            .then(async (response) => {
-              logEntry.append({
-                response: parseResponse(response),
-              });
-            })
-            .catch(async (error) => {
-              logEntry.append({
-                result: LogResult.Error,
-                response: conditional(error instanceof HTTPError && parseResponse(error.response)),
-                error: conditional(error instanceof Error && String(error)),
-              });
+        // Trigger web hook and log response
+        await got
+          .post(url, {
+            headers: { 'user-agent': 'Logto (https://logto.io)', ...headers },
+            json,
+            retry: { limit: retries },
+            timeout: { request: 10_000 },
+          })
+          .then(async (response) => {
+            logEntry.append({
+              response: parseResponse(response),
+            });
+          })
+          .catch(async (error) => {
+            logEntry.append({
+              result: LogResult.Error,
+              response: conditional(error instanceof HTTPError && parseResponse(error.response)),
+              error: conditional(error instanceof Error && String(error)),
             });
-
-          console.log(
-            `\tHook ${id} ${logEntry.payload.result === LogResult.Success ? 'succeeded' : 'failed'}`
-          );
-
-          await insertLog({
-            id: generateStandardId(),
-            key: logEntry.key,
-            payload: logEntry.payload,
           });
-        })
+
+        console.log(
+          `\tHook ${id} ${logEntry.payload.result === LogResult.Success ? 'succeeded' : 'failed'}`
+        );
+
+        await insertLog({
+          id: generateStandardId(),
+          key: logEntry.key,
+          payload: logEntry.payload,
+        });
+      })
     );
   };
 

packages/core/src/tenants/index.ts

@@ -1,10 +1,12 @@
 import LRUCache from 'lru-cache';
 
+import { EnvSet } from '#src/env-set/index.js';
+
 import Tenant from './Tenant.js';
 
 export class TenantPool {
   protected cache = new LRUCache<string, Promise<Tenant>>({
-    max: 100,
+    max: EnvSet.values.tenantPoolSize,
     dispose: async (entry) => {
       const tenant = await entry;
       void tenant.dispose();

packages/shared/src/env/GlobalValues.ts

@@ -98,6 +98,11 @@ export default class GlobalValues {
   public readonly trustProxyHeader = yes(getEnv('TRUST_PROXY_HEADER'));
   public readonly ignoreConnectorVersionCheck = yes(getEnv('IGNORE_CONNECTOR_VERSION_CHECK'));
 
+  /** Maximum number of tenants to keep in the tenant pool. */
+  public readonly tenantPoolSize = Number(getEnv('TENANT_POOL_SIZE', '100'));
+  /** Maximum number of clients to keep in a single database pool (i.e. per `Tenant` class). */
+  public readonly databasePoolSize = Number(getEnv('DATABASE_POOL_SIZE', '20'));
+
   public get dbUrl(): string {
     return this.databaseUrl;
   }