feat: implement central cache

with Redis as the default choice.

.dockerignore

@@ -27,6 +27,7 @@ cache
 .history
 .git
 .gitignore
+dump.rdb
 
 .changeset
 .devcontainer

.gitignore

@@ -30,6 +30,7 @@ cache
 *.pem
 .history
 fly.toml
+dump.rdb
 
 # connectors
 /packages/core/connectors

packages/core/package.json

@@ -71,6 +71,7 @@
     "p-memoize": "^7.1.1",
     "p-retry": "^5.1.2",
     "pg-protocol": "^1.6.0",
+    "redis": "^4.6.5",
     "roarr": "^7.11.0",
     "semver": "^7.3.8",
     "slonik": "^30.0.0",

packages/core/src/caches/index.ts

@@ -0,0 +1,56 @@
+import { appInsights } from '@logto/app-insights/node';
+import { type Optional, conditional, yes } from '@silverhand/essentials';
+import { createClient, type RedisClientType } from 'redis';
+
+import { EnvSet } from '#src/env-set/index.js';
+
+import { type CacheStore } from './types.js';
+
+export class RedisCache implements CacheStore {
+  readonly client?: RedisClientType;
+
+  constructor() {
+    const { redisUrl } = EnvSet.values;
+
+    if (redisUrl) {
+      this.client = createClient({
+        url: conditional(!yes(redisUrl) && redisUrl),
+      });
+      this.client.on('error', (error) => {
+        appInsights.trackException(error);
+      });
+    }
+  }
+
+  async set(key: string, value: string) {
+    await this.client?.set(key, value, {
+      EX: 30 * 60 /* 30 minutes */,
+    });
+  }
+
+  async get(key: string): Promise<Optional<string>> {
+    return conditional(await this.client?.get(key));
+  }
+
+  async delete(key: string) {
+    await this.client?.del(key);
+  }
+
+  async connect() {
+    if (this.client) {
+      await this.client.connect();
+      console.log('[CACHE] Connected to Redis');
+    } else {
+      console.warn('[CACHE] No Redis client initialized, skipping');
+    }
+  }
+
+  async disconnect() {
+    if (this.client) {
+      await this.client.disconnect();
+      console.log('[CACHE] Disconnected from Redis');
+    }
+  }
+}
+
+export const redisCache = new RedisCache();

packages/core/src/caches/types.ts

@@ -0,0 +1,7 @@
+import { type Optional } from '@silverhand/essentials';
+
+export type CacheStore<Key = string, Value = string> = {
+  get(key: Key): Promise<Optional<Value>> | Optional<Value>;
+  set(key: Key, value: Value): Promise<void | boolean> | void | boolean;
+  delete(key: Key): Promise<void | boolean> | void | boolean;
+};

packages/core/src/caches/well-known.ts

@@ -1,61 +1,144 @@
-import { TtlCache } from '@logto/shared';
-import type { AnyAsyncFunction } from 'p-memoize';
-import pMemoize from 'p-memoize';
+import { type SignInExperience, SignInExperiences } from '@logto/schemas';
+import { type Optional, trySafe } from '@silverhand/essentials';
+import { type ZodType, z } from 'zod';
 
-const cacheKeys = Object.freeze(['sie', 'sie-full', 'phrases', 'phrases-lng-tags'] as const);
+import { type ConnectorWellKnown, connectorWellKnownGuard } from '#src/utils/connectors/types.js';
 
-/** Well-known data type key for cache. */
-export type WellKnownCacheKey = (typeof cacheKeys)[number];
+import { type CacheStore } from './types.js';
 
-const buildKey = (tenantId: string, key: WellKnownCacheKey) => `${tenantId}:${key}` as const;
+type WellKnownMap = {
+  sie: SignInExperience;
+  'connectors-well-known': ConnectorWellKnown[];
+  'custom-phrases': Record<string, unknown>;
+  'custom-phrases-tags': string[];
+};
 
-class WellKnownCache {
-  // This TTL should be very small since the sign-in experiences will be unusable
-  // if requests hit different instances during the cache-hit period.
-  // We need to use a real central cache like Redis with invalidation mechanism for it.
-  #cache = new TtlCache<string, unknown>(5000);
+const defaultCacheKey = '#';
 
-  /**
-   * Use for centralized well-known data caching.
-   *
-   * WARN:
-   * - You should store only well-known (public) data since it's a central cache.
-   * - The cache does not guard types.
-   */
-  use<FunctionToMemoize extends AnyAsyncFunction>(
-    tenantId: string,
-    key: WellKnownCacheKey,
-    run: FunctionToMemoize
-  ) {
-    return pMemoize(run, {
-      cacheKey: () => buildKey(tenantId, key),
-      // Trust cache value type
-      // eslint-disable-next-line no-restricted-syntax
-      cache: this.#cache as TtlCache<string, Awaited<ReturnType<FunctionToMemoize>>>,
-    });
-  }
+export type WellKnownCacheType = keyof WellKnownMap;
 
-  invalidate(tenantId: string, keys: readonly WellKnownCacheKey[]) {
-    for (const key of keys) {
-      this.#cache.delete(buildKey(tenantId, key));
+type CacheKeyConfig<Args extends unknown[], Type = WellKnownCacheType> =
+  | [Type]
+  | [Type, (...args: Args) => string];
+
+// Cannot use generic type here, but direct type works.
+// See https://github.com/microsoft/TypeScript/issues/27808#issuecomment-1207161877
+// WARN: You should carefully check key and return type mapping since the implementation signature doesn't do this.
+function getValueGuard<Type extends WellKnownCacheType>(type: Type): ZodType<WellKnownMap[Type]>;
+
+function getValueGuard(type: WellKnownCacheType): ZodType<WellKnownMap[typeof type]> {
+  switch (type) {
+    case 'sie': {
+      return SignInExperiences.guard;
+    }
+    case 'connectors-well-known': {
+      return connectorWellKnownGuard.array();
+    }
+    case 'custom-phrases-tags': {
+      return z.string().array();
+    }
+    case 'custom-phrases': {
+      return z.record(z.unknown());
+    }
+    default: {
+      throw new Error(`No proper value guard found for cache key "${String(type)}".`);
     }
-  }
-
-  invalidateAll(tenantId: string) {
-    this.invalidate(tenantId, cacheKeys);
-  }
-
-  set(tenantId: string, key: WellKnownCacheKey, value: unknown) {
-    this.#cache.set(buildKey(tenantId, key), value);
   }
 }
 
-/**
- * The central TTL cache for well-known data. The default TTL is 3 minutes.
- *
- * This cache is intended for public APIs that are tolerant for data freshness.
- * For Management APIs, you should use uncached functions instead.
- *
- * WARN: You should store only well-known (public) data since it's a central cache.
- */
-export const wellKnownCache = new WellKnownCache();
+export class WellKnownCache {
+  constructor(public tenantId: string, protected cacheStore: CacheStore) {}
+
+  async get<Type extends WellKnownCacheType>(
+    type: Type,
+    key: string
+  ): Promise<Optional<WellKnownMap[Type]>> {
+    const data = await this.cacheStore.get(this.cacheKey(type, key));
+
+    return trySafe(() => getValueGuard(type).parse(JSON.parse(data ?? '')));
+  }
+
+  async set<Type extends WellKnownCacheType>(
+    type: Type,
+    key: string,
+    value: Readonly<WellKnownMap[Type]>
+  ) {
+    return this.cacheStore.set(this.cacheKey(type, key), JSON.stringify(value));
+  }
+
+  async delete(type: WellKnownCacheType, key: string) {
+    return this.cacheStore.delete(this.cacheKey(type, key));
+  }
+
+  mutate<Args extends unknown[], Return>(
+    run: (...args: Args) => Promise<Return>,
+    ...types: Array<CacheKeyConfig<Args>>
+  ) {
+    // Intended. We're going to use `this` cache inside another closure.
+    // eslint-disable-next-line @typescript-eslint/no-this-alias, unicorn/no-this-assignment
+    const kvCache = this;
+
+    const mutated = async function (this: unknown, ...args: Args): Promise<Return> {
+      const value = await run.apply(this, args);
+
+      // We don't leverage `finally` here since we want to ensure cache deleting
+      // only happens when the original function executed successfully
+      void Promise.all(
+        types.map(async ([type, cacheKey]) =>
+          trySafe(kvCache.delete(type, cacheKey?.(...args) ?? defaultCacheKey))
+        )
+      );
+
+      return value;
+    };
+
+    return mutated;
+  }
+
+  memoize<Type extends WellKnownCacheType, Args extends unknown[]>(
+    run: (...args: Args) => Promise<Readonly<WellKnownMap[Type]>>,
+    [type, cacheKey]: CacheKeyConfig<Args, Type>
+  ) {
+    const promiseCache = new Map<unknown, Promise<Readonly<WellKnownMap[Type]>>>();
+    // Intended. We're going to use `this` cache inside another closure.
+    // eslint-disable-next-line @typescript-eslint/no-this-alias, unicorn/no-this-assignment
+    const kvCache = this;
+
+    const memoized = async function (
+      this: unknown,
+      ...args: Args
+    ): Promise<Readonly<WellKnownMap[Type]>> {
+      const promiseKey = cacheKey?.(...args) ?? defaultCacheKey;
+      const cachedPromise = promiseCache.get(promiseKey);
+
+      if (cachedPromise) {
+        return cachedPromise;
+      }
+
+      const promise = (async () => {
+        // Wrap with `trySafe()` here to ignore Redis errors
+        const cachedValue = await trySafe(kvCache.get(type, promiseKey));
+
+        if (cachedValue) {
+          return cachedValue;
+        }
+
+        const value = await run.apply(this, args);
+        await trySafe(kvCache.set(type, promiseKey, value));
+        promiseCache.delete(promiseKey);
+
+        return value;
+      })();
+
+      promiseCache.set(promiseKey, promise);
+
+      return promise;
+    };
+
+    return memoized;
+  }
+
+  protected cacheKey(type: WellKnownCacheType, key: string) {
+    return `${this.tenantId}:${type}:${key}`;
+  }
+}

packages/core/src/include.d/array.d.ts

@@ -0,0 +1,12 @@
+// Cannot import from "@silverhand/essentials" in this file.
+// See https://www.karltarvas.com/2021/03/11/typescript-array-filter-boolean.html
+
+type Falsy = false | 0 | '' | undefined | undefined;
+
+interface Array<T> {
+  filter<S extends T>(predicate: BooleanConstructor, thisArg?: unknown): Array<Exclude<S, Falsy>>;
+}
+
+interface ReadonlyArray<T> {
+  filter<S extends T>(predicate: BooleanConstructor, thisArg?: unknown): Array<Exclude<S, Falsy>>;
+}

packages/core/src/index.ts

@@ -1,4 +1,4 @@
-import { noop } from '@silverhand/essentials';
+import { trySafe } from '@silverhand/essentials';
 import dotenv from 'dotenv';
 import { findUp } from 'find-up';
 import Koa from 'koa';
@@ -17,6 +17,7 @@ if (await appInsights.setup('logto')) {
 // Import after env has been configured
 const { loadConnectorFactories } = await import('./utils/connectors/index.js');
 const { EnvSet } = await import('./env-set/index.js');
+const { redisCache } = await import('./caches/index.js');
 const { default: initI18n } = await import('./i18n/init.js');
 const { tenantPool, checkRowLevelSecurity } = await import('./tenants/index.js');
 
@@ -25,13 +26,15 @@ try {
     proxy: EnvSet.values.trustProxyHeader,
   });
   const sharedAdminPool = await EnvSet.sharedPool;
-  await initI18n();
-  await loadConnectorFactories();
+
   await Promise.all([
+    initI18n(),
+    redisCache.connect(),
+    loadConnectorFactories(),
     checkRowLevelSecurity(sharedAdminPool),
     checkAlterationState(sharedAdminPool),
+    SystemContext.shared.loadStorageProviderConfig(sharedAdminPool),
   ]);
-  await SystemContext.shared.loadStorageProviderConfig(sharedAdminPool);
 
   // Import last until init completed
   const { default: initApp } = await import('./app/init.js');
@@ -40,5 +43,5 @@ try {
   console.error('Error while initializing app:');
   console.error(error);
 
-  await tenantPool.endAll().catch(noop);
+  await Promise.all([trySafe(tenantPool.endAll()), trySafe(redisCache.disconnect())]);
 }

packages/core/src/libraries/connector.ts

@@ -1,17 +1,18 @@
 import { buildRawConnector, defaultConnectorMethods } from '@logto/cli/lib/connector/index.js';
 import type { AllConnector } from '@logto/connector-kit';
 import { validateConfig } from '@logto/connector-kit';
+import { pick, trySafe } from '@silverhand/essentials';
 
 import RequestError from '#src/errors/RequestError/index.js';
 import type Queries from '#src/tenants/Queries.js';
 import assertThat from '#src/utils/assert-that.js';
 import { loadConnectorFactories } from '#src/utils/connectors/index.js';
-import type { LogtoConnector } from '#src/utils/connectors/types.js';
+import type { LogtoConnector, LogtoConnectorWellKnown } from '#src/utils/connectors/types.js';
 
 export type ConnectorLibrary = ReturnType<typeof createConnectorLibrary>;
 
 export const createConnectorLibrary = (queries: Queries) => {
-  const { findAllConnectors } = queries.connectors;
+  const { findAllConnectors, findAllConnectorsWellKnown } = queries.connectors;
 
   const getConnectorConfig = async (id: string): Promise<unknown> => {
     const connectors = await findAllConnectors();
@@ -22,14 +23,43 @@ export const createConnectorLibrary = (queries: Queries) => {
     return connector.config;
   };
 
+  const getLogtoConnectorsWellKnown = async (): Promise<LogtoConnectorWellKnown[]> => {
+    const databaseConnectors = await findAllConnectorsWellKnown();
+    const connectorFactories = await loadConnectorFactories();
+
+    const logtoConnectors = await Promise.all(
+      databaseConnectors.map(async (databaseEntry) => {
+        const { metadata, connectorId } = databaseEntry;
+        const connectorFactory = connectorFactories.find(
+          ({ metadata }) => metadata.id === connectorId
+        );
+
+        if (!connectorFactory) {
+          return;
+        }
+
+        return trySafe(async () => {
+          const { rawConnector, rawMetadata } = await buildRawConnector(connectorFactory);
+
+          return {
+            ...pick(rawConnector, 'type', 'metadata'),
+            metadata: { ...rawMetadata, ...metadata },
+            dbEntry: databaseEntry,
+          };
+        });
+      })
+    );
+
+    return logtoConnectors.filter(Boolean);
+  };
+
   const getLogtoConnectors = async (): Promise<LogtoConnector[]> => {
     const databaseConnectors = await findAllConnectors();
+    const connectorFactories = await loadConnectorFactories();
 
     const logtoConnectors = await Promise.all(
       databaseConnectors.map(async (databaseConnector) => {
         const { id, metadata, connectorId } = databaseConnector;
-
-        const connectorFactories = await loadConnectorFactories();
         const connectorFactory = connectorFactories.find(
           ({ metadata }) => metadata.id === connectorId
         );
@@ -64,9 +94,7 @@ export const createConnectorLibrary = (queries: Queries) => {
       })
     );
 
-    return logtoConnectors.filter(
-      (logtoConnector): logtoConnector is LogtoConnector => logtoConnector !== undefined
-    );
+    return logtoConnectors.filter(Boolean);
   };
 
   const getLogtoConnectorById = async (id: string): Promise<LogtoConnector> => {
@@ -84,5 +112,10 @@ export const createConnectorLibrary = (queries: Queries) => {
     return pickedConnector;
   };
 
-  return { getConnectorConfig, getLogtoConnectors, getLogtoConnectorById };
+  return {
+    getConnectorConfig,
+    getLogtoConnectors,
+    getLogtoConnectorsWellKnown,
+    getLogtoConnectorById,
+  };
 };

packages/core/src/libraries/phrase.test.ts

@@ -11,7 +11,6 @@ import {
   zhCnTag,
   mockTag,
 } from '#src/__mocks__/custom-phrase.js';
-import { wellKnownCache } from '#src/caches/well-known.js';
 import RequestError from '#src/errors/RequestError/index.js';
 import { MockQueries } from '#src/test-utils/tenant.js';
 
@@ -42,15 +41,12 @@ const findCustomPhraseByLanguageTag = jest.fn(async (languageTag: string) => {
   return mockCustomPhrase;
 });
 
-const tenantId = 'mock_id';
 const { createPhraseLibrary } = await import('#src/libraries/phrase.js');
 const { getPhrases } = createPhraseLibrary(
-  new MockQueries({ customPhrases: { findCustomPhraseByLanguageTag } }),
-  tenantId
+  new MockQueries({ customPhrases: { findCustomPhraseByLanguageTag } })
 );
 
 afterEach(() => {
-  wellKnownCache.invalidateAll(tenantId);
   jest.clearAllMocks();
 });
 
@@ -75,7 +71,7 @@ it('should ignore empty string values from the custom phrase', async () => {
   } satisfies CustomPhrase;
 
   findCustomPhraseByLanguageTag.mockResolvedValueOnce(mockEnCustomPhraseWithEmptyStringValues);
-  await expect(getPhrases(enTag, [enTag])).resolves.toEqual(
+  await expect(getPhrases(enTag)).resolves.toEqual(
     deepmerge(englishBuiltInPhrase, {
       id: 'fake_id',
       tenantId: 'fake_tenant',
@@ -92,19 +88,20 @@ it('should ignore empty string values from the custom phrase', async () => {
 
 describe('when the language is English', () => {
   it('should be English custom phrase merged with its built-in phrase when its custom phrase exists', async () => {
-    await expect(getPhrases(enTag, [enTag])).resolves.toEqual(
+    await expect(getPhrases(enTag)).resolves.toEqual(
       deepmerge(englishBuiltInPhrase, mockEnCustomPhrase)
     );
   });
 
   it('should be English built-in phrase when its custom phrase does not exist', async () => {
-    await expect(getPhrases(enTag, [])).resolves.toEqual(englishBuiltInPhrase);
+    findCustomPhraseByLanguageTag.mockRejectedValueOnce(new Error('not found'));
+    await expect(getPhrases(enTag)).resolves.toEqual(englishBuiltInPhrase);
   });
 });
 
 describe('when the language is not English', () => {
   it('should be custom phrase merged with built-in phrase when both of them exist', async () => {
-    await expect(getPhrases(customizedLanguage, [customizedLanguage])).resolves.toEqual(
+    await expect(getPhrases(customizedLanguage)).resolves.toEqual(
       deepmerge(customizedBuiltInPhrase, customizedCustomPhrase)
     );
   });
@@ -112,11 +109,11 @@ describe('when the language is not English', () => {
   it('should be built-in phrase when there is built-in phrase and no custom phrase', async () => {
     const builtInOnlyLanguage = trTrTag;
     const builtInOnlyPhrase = resource[trTrTag];
-    await expect(getPhrases(builtInOnlyLanguage, [])).resolves.toEqual(builtInOnlyPhrase);
+    await expect(getPhrases(builtInOnlyLanguage)).resolves.toEqual(builtInOnlyPhrase);
   });
 
   it('should be built-in phrase when there is custom phrase and no built-in phrase', async () => {
-    await expect(getPhrases(customOnlyLanguage, [customOnlyLanguage])).resolves.toEqual(
+    await expect(getPhrases(customOnlyLanguage)).resolves.toEqual(
       deepmerge(englishBuiltInPhrase, customOnlyCustomPhrase)
     );
   });

packages/core/src/libraries/phrase.ts

@@ -1,63 +1,23 @@
 import type { LocalePhrase } from '@logto/phrases-ui';
 import resource, { isBuiltInLanguageTag } from '@logto/phrases-ui';
-import type { CustomPhrase } from '@logto/schemas';
+import { trySafe } from '@silverhand/essentials';
 import cleanDeep from 'clean-deep';
 import deepmerge from 'deepmerge';
 
-import { wellKnownCache } from '#src/caches/well-known.js';
 import type Queries from '#src/tenants/Queries.js';
 
-export const createPhraseLibrary = (queries: Queries, tenantId: string) => {
+export const createPhraseLibrary = (queries: Queries) => {
   const { findCustomPhraseByLanguageTag, findAllCustomLanguageTags } = queries.customPhrases;
 
-  const _getPhrases = async (
-    supportedLanguage: string,
-    customLanguages: string[]
-  ): Promise<LocalePhrase> => {
-    if (!isBuiltInLanguageTag(supportedLanguage)) {
-      return deepmerge<LocalePhrase, CustomPhrase>(
-        resource.en,
-        cleanDeep(await findCustomPhraseByLanguageTag(supportedLanguage))
-      );
-    }
-
-    if (!customLanguages.includes(supportedLanguage)) {
-      return resource[supportedLanguage];
-    }
-
-    return deepmerge<LocalePhrase, CustomPhrase>(
-      resource[supportedLanguage],
-      cleanDeep(await findCustomPhraseByLanguageTag(supportedLanguage))
+  const getPhrases = async (forLanguage: string): Promise<LocalePhrase> => {
+    return deepmerge<LocalePhrase>(
+      resource[isBuiltInLanguageTag(forLanguage) ? forLanguage : 'en'],
+      cleanDeep((await trySafe(findCustomPhraseByLanguageTag(forLanguage))) ?? {})
     );
   };
 
-  const getPhrases = wellKnownCache.use(tenantId, 'phrases', _getPhrases);
-
-  const getAllCustomLanguageTags = wellKnownCache.use(
-    tenantId,
-    'phrases-lng-tags',
-    findAllCustomLanguageTags
-  );
-
   return {
-    /**
-     * NOTE: This function is cached by the first parameter.
-     * **Cache Invalidation**
-     *
-     * ```ts
-     * wellKnownCache.invalidate(tenantId, ['phrases']);
-     * ```
-     */
     getPhrases,
-    /**
-     * NOTE: This function is cached.
-     *
-     * **Cache Invalidation**
-     *
-     * ```ts
-     * wellKnownCache.invalidate(tenantId, ['phrases-lng-tags']);
-     * ```
-     */
-    getAllCustomLanguageTags,
+    findAllCustomLanguageTags,
   };
 };

packages/core/src/libraries/sign-in-experience/index.test.ts

@@ -42,7 +42,7 @@ const getLogtoConnectors = jest.spyOn(connectorLibrary, 'getLogtoConnectors');
 
 const { createSignInExperienceLibrary } = await import('./index.js');
 const { validateLanguageInfo, removeUnavailableSocialConnectorTargets } =
-  createSignInExperienceLibrary(queries, connectorLibrary, 'mock_id');
+  createSignInExperienceLibrary(queries, connectorLibrary);
 
 beforeEach(() => {
   jest.clearAllMocks();

packages/core/src/libraries/sign-in-experience/index.ts

@@ -1,16 +1,15 @@
-import { connectorMetadataGuard } from '@logto/connector-kit';
 import { builtInLanguages } from '@logto/phrases-ui';
-import type { ConnectorMetadata, LanguageInfo, SignInExperience } from '@logto/schemas';
-import { SignInExperiences, ConnectorType } from '@logto/schemas';
+import type { ConnectorMetadata, LanguageInfo } from '@logto/schemas';
+import { ConnectorType } from '@logto/schemas';
 import { deduplicate } from '@silverhand/essentials';
-import { z } from 'zod';
 
-import { wellKnownCache } from '#src/caches/well-known.js';
 import RequestError from '#src/errors/RequestError/index.js';
 import type { ConnectorLibrary } from '#src/libraries/connector.js';
 import type Queries from '#src/tenants/Queries.js';
 import assertThat from '#src/utils/assert-that.js';
 
+import { type FullSignInExperience } from './types.js';
+
 export * from './sign-up.js';
 export * from './sign-in.js';
 
@@ -18,8 +17,7 @@ export type SignInExperienceLibrary = ReturnType<typeof createSignInExperienceLi
 
 export const createSignInExperienceLibrary = (
   queries: Queries,
-  { getLogtoConnectors }: ConnectorLibrary,
-  tenantId: string
+  { getLogtoConnectors }: ConnectorLibrary
 ) => {
   const {
     customPhrases: { findAllCustomLanguageTags },
@@ -55,9 +53,9 @@ export const createSignInExperienceLibrary = (
     });
   };
 
-  const getSignInExperience = wellKnownCache.use(tenantId, 'sie', findDefaultSignInExperience);
+  const getSignInExperience = findDefaultSignInExperience;
 
-  const _getFullSignInExperience = async (): Promise<FullSignInExperience> => {
+  const getFullSignInExperience = async (): Promise<FullSignInExperience> => {
     const [signInExperience, logtoConnectors] = await Promise.all([
       getSignInExperience(),
       getLogtoConnectors(),
@@ -88,52 +86,10 @@ export const createSignInExperienceLibrary = (
     };
   };
 
-  const getFullSignInExperience = wellKnownCache.use(
-    tenantId,
-    'sie-full',
-    _getFullSignInExperience
-  );
-
   return {
     validateLanguageInfo,
     removeUnavailableSocialConnectorTargets,
-    /**
-     * NOTE: This function is cached.
-     *
-     * **Cache Invalidation**
-     *
-     * ```ts
-     * wellKnownCache.invalidate(tenantId, ['sie']);
-     * ```
-     */
     getSignInExperience,
-    /**
-     * NOTE: This function is cached.
-     *
-     * **Cache Invalidation**
-     *
-     * ```ts
-     * wellKnownCache.invalidate(tenantId, ['sie', 'sie-full']);
-     * ```
-     */
     getFullSignInExperience,
   };
 };
-
-export type ForgotPassword = {
-  phone: boolean;
-  email: boolean;
-};
-
-export type ConnectorMetadataWithId = ConnectorMetadata & { id: string };
-
-export type FullSignInExperience = SignInExperience & {
-  socialConnectors: ConnectorMetadataWithId[];
-  forgotPassword: ForgotPassword;
-};
-
-export const guardFullSignInExperience: z.ZodType<FullSignInExperience> =
-  SignInExperiences.guard.extend({
-    socialConnectors: connectorMetadataGuard.extend({ id: z.string() }).array(),
-    forgotPassword: z.object({ phone: z.boolean(), email: z.boolean() }),
-  });

packages/core/src/libraries/sign-in-experience/types.ts

@@ -0,0 +1,21 @@
+import { connectorMetadataGuard, type ConnectorMetadata } from '@logto/connector-kit';
+import { type SignInExperience, SignInExperiences } from '@logto/schemas';
+import { z } from 'zod';
+
+export type ForgotPassword = {
+  phone: boolean;
+  email: boolean;
+};
+
+export type ConnectorMetadataWithId = ConnectorMetadata & { id: string };
+
+export type FullSignInExperience = SignInExperience & {
+  socialConnectors: ConnectorMetadataWithId[];
+  forgotPassword: ForgotPassword;
+};
+
+export const guardFullSignInExperience: z.ZodType<FullSignInExperience> =
+  SignInExperiences.guard.extend({
+    socialConnectors: connectorMetadataGuard.extend({ id: z.string() }).array(),
+    forgotPassword: z.object({ phone: z.boolean(), email: z.boolean() }),
+  });

packages/core/src/queries/connector.test.ts

@@ -4,6 +4,7 @@ import { createMockPool, createMockQueryResult, sql } from 'slonik';
 
 import { mockConnector } from '#src/__mocks__/index.js';
 import { DeletionError } from '#src/errors/SlonikError/index.js';
+import { MockWellKnownCache } from '#src/test-utils/tenant.js';
 import type { QueryType } from '#src/utils/test-utils.js';
 import { expectSqlAssert } from '#src/utils/test-utils.js';
 
@@ -26,7 +27,7 @@ const {
   deleteConnectorByIds,
   insertConnector,
   updateConnector,
-} = createConnectorQueries(pool);
+} = createConnectorQueries(pool, new MockWellKnownCache());
 
 describe('connector queries', () => {
   const { table, fields } = convertToIdentifiers(Connectors);

packages/core/src/queries/connector.ts

@@ -4,13 +4,18 @@ import { manyRows, convertToIdentifiers } from '@logto/shared';
 import type { CommonQueryMethods } from 'slonik';
 import { sql } from 'slonik';
 
+import { type WellKnownCache } from '#src/caches/well-known.js';
 import { buildInsertIntoWithPool } from '#src/database/insert-into.js';
 import { buildUpdateWhereWithPool } from '#src/database/update-where.js';
 import { DeletionError } from '#src/errors/SlonikError/index.js';
+import { type ConnectorWellKnown } from '#src/utils/connectors/types.js';
 
 const { table, fields } = convertToIdentifiers(Connectors);
 
-export const createConnectorQueries = (pool: CommonQueryMethods) => {
+export const createConnectorQueries = (
+  pool: CommonQueryMethods,
+  wellKnownCache: WellKnownCache
+) => {
   const findAllConnectors = async () =>
     manyRows(
       pool.query<Connector>(sql`
@@ -19,6 +24,17 @@ export const createConnectorQueries = (pool: CommonQueryMethods) => {
         order by ${fields.id} asc
       `)
     );
+  const findAllConnectorsWellKnown = wellKnownCache.memoize(
+    async () =>
+      manyRows(
+        pool.query<ConnectorWellKnown>(sql`
+          select ${sql.join([fields.id, fields.metadata, fields.connectorId], sql`, `)}
+          from ${table}
+          order by ${fields.id} asc
+        `)
+      ),
+    ['connectors-well-known']
+  );
   const findConnectorById = async (id: string) =>
     pool.one<Connector>(sql`
       select ${sql.join(Object.values(fields), sql`,`)}
@@ -31,35 +47,46 @@ export const createConnectorQueries = (pool: CommonQueryMethods) => {
       from ${table}
       where ${fields.connectorId}=${connectorId}
     `);
+  const deleteConnectorById = wellKnownCache.mutate(
+    async (id: string) => {
+      const { rowCount } = await pool.query(sql`
+        delete from ${table}
+        where ${fields.id}=${id}
+      `);
 
-  const deleteConnectorById = async (id: string) => {
-    const { rowCount } = await pool.query(sql`
-      delete from ${table}
-      where ${fields.id}=${id}
-    `);
+      if (rowCount < 1) {
+        throw new DeletionError(Connectors.table, id);
+      }
+    },
+    ['connectors-well-known']
+  );
+  const deleteConnectorByIds = wellKnownCache.mutate(
+    async (ids: string[]) => {
+      const { rowCount } = await pool.query(sql`
+        delete from ${table}
+        where ${fields.id} in (${sql.join(ids, sql`, `)})
+      `);
 
-    if (rowCount < 1) {
-      throw new DeletionError(Connectors.table, id);
-    }
-  };
-
-  const deleteConnectorByIds = async (ids: string[]) => {
-    const { rowCount } = await pool.query(sql`
-      delete from ${table}
-      where ${fields.id} in (${sql.join(ids, sql`, `)})
-    `);
-
-    if (rowCount !== ids.length) {
-      throw new DeletionError(Connectors.table, JSON.stringify({ ids }));
-    }
-  };
-  const insertConnector = buildInsertIntoWithPool(pool)(Connectors, {
-    returning: true,
-  });
-  const updateConnector = buildUpdateWhereWithPool(pool)(Connectors, true);
+      if (rowCount !== ids.length) {
+        throw new DeletionError(Connectors.table, JSON.stringify({ ids }));
+      }
+    },
+    ['connectors-well-known']
+  );
+  const insertConnector = wellKnownCache.mutate(
+    buildInsertIntoWithPool(pool)(Connectors, {
+      returning: true,
+    }),
+    ['connectors-well-known']
+  );
+  const updateConnector = wellKnownCache.mutate(buildUpdateWhereWithPool(pool)(Connectors, true), [
+    'connectors-well-known',
+  ]);
 
   return {
     findAllConnectors,
+    /** Find all connectors from database with no sensitive info. */
+    findAllConnectorsWellKnown,
     findConnectorById,
     countConnectorByConnectorId,
     deleteConnectorById,

packages/core/src/queries/custom-phrase.ts

@@ -1,16 +1,20 @@
-import type { CustomPhrase } from '@logto/schemas';
+import type { CustomPhrase, Translation } from '@logto/schemas';
 import { CustomPhrases } from '@logto/schemas';
-import { convertToIdentifiers, manyRows } from '@logto/shared';
+import { convertToIdentifiers, generateStandardId, manyRows } from '@logto/shared';
 import type { CommonQueryMethods } from 'slonik';
 import { sql } from 'slonik';
 
+import { type WellKnownCache } from '#src/caches/well-known.js';
 import { buildInsertIntoWithPool } from '#src/database/insert-into.js';
 import { DeletionError } from '#src/errors/SlonikError/index.js';
 
 const { table, fields } = convertToIdentifiers(CustomPhrases);
 
-export const createCustomPhraseQueries = (pool: CommonQueryMethods) => {
-  const findAllCustomLanguageTags = async () => {
+export const createCustomPhraseQueries = (
+  pool: CommonQueryMethods,
+  wellKnownCache: WellKnownCache
+) => {
+  const findAllCustomLanguageTags = wellKnownCache.memoize(async () => {
     const rows = await manyRows<{ languageTag: string }>(
       pool.query(sql`
         select ${fields.languageTag}
@@ -20,7 +24,7 @@ export const createCustomPhraseQueries = (pool: CommonQueryMethods) => {
     );
 
     return rows.map((row) => row.languageTag);
-  };
+  }, ['custom-phrases-tags']);
 
   const findAllCustomPhrases = async () =>
     manyRows(
@@ -31,14 +35,17 @@ export const createCustomPhraseQueries = (pool: CommonQueryMethods) => {
       `)
     );
 
-  const findCustomPhraseByLanguageTag = async (languageTag: string): Promise<CustomPhrase> =>
-    pool.one<CustomPhrase>(sql`
-      select ${sql.join(Object.values(fields), sql`, `)}
-      from ${table}
-      where ${fields.languageTag} = ${languageTag}
-    `);
+  const findCustomPhraseByLanguageTag = wellKnownCache.memoize(
+    async (languageTag: string): Promise<CustomPhrase> =>
+      pool.one<CustomPhrase>(sql`
+        select ${sql.join(Object.values(fields), sql`, `)}
+        from ${table}
+        where ${fields.languageTag} = ${languageTag}
+      `),
+    ['custom-phrases', (languageTag) => languageTag]
+  );
 
-  const upsertCustomPhrase = buildInsertIntoWithPool(pool)(CustomPhrases, {
+  const _upsertCustomPhrase = buildInsertIntoWithPool(pool)(CustomPhrases, {
     returning: true,
     onConflict: {
       fields: [fields.tenantId, fields.languageTag],
@@ -46,22 +53,30 @@ export const createCustomPhraseQueries = (pool: CommonQueryMethods) => {
     },
   });
 
-  const deleteCustomPhraseByLanguageTag = async (languageTag: string) => {
-    const { rowCount } = await pool.query(sql`
-      delete from ${table}
-      where ${fields.languageTag}=${languageTag}
-    `);
+  const upsertCustomPhrase = wellKnownCache.mutate(
+    async (languageTag: string, translation: Translation) =>
+      // LOG-5915 Remove `id` in custom phrases
+      _upsertCustomPhrase({ id: generateStandardId(), languageTag, translation }),
+    ['custom-phrases', (languageTag) => languageTag],
+    ['custom-phrases-tags'] // Invalidate tags cache as well since it may add a new language tag
+  );
 
-    if (rowCount < 1) {
-      throw new DeletionError(CustomPhrases.table, languageTag);
-    }
-  };
+  const deleteCustomPhraseByLanguageTag = wellKnownCache.mutate(
+    async (languageTag: string) => {
+      const { rowCount } = await pool.query(sql`
+        delete from ${table}
+        where ${fields.languageTag}=${languageTag}
+      `);
+
+      if (rowCount < 1) {
+        throw new DeletionError(CustomPhrases.table, languageTag);
+      }
+    },
+    ['custom-phrases', (languageTag) => languageTag],
+    ['custom-phrases-tags']
+  );
 
   return {
-    /**
-     * NOTE: Use `getAllCustomLanguageTags()` from phrase library
-     * if possible since that function leverages cache.
-     */
     findAllCustomLanguageTags,
     findAllCustomPhrases,
     findCustomPhraseByLanguageTag,

packages/core/src/queries/sign-in-experience.test.ts

@@ -1,6 +1,7 @@
 import { createMockPool, createMockQueryResult } from 'slonik';
 
 import { mockSignInExperience } from '#src/__mocks__/index.js';
+import { MockWellKnownCache } from '#src/test-utils/tenant.js';
 import type { QueryType } from '#src/utils/test-utils.js';
 import { expectSqlAssert } from '#src/utils/test-utils.js';
 
@@ -16,7 +17,7 @@ const pool = createMockPool({
 
 const { createSignInExperienceQueries } = await import('./sign-in-experience.js');
 const { findDefaultSignInExperience, updateDefaultSignInExperience } =
-  createSignInExperienceQueries(pool);
+  createSignInExperienceQueries(pool, new MockWellKnownCache());
 
 describe('sign-in-experience query', () => {
   const id = 'default';

packages/core/src/queries/sign-in-experience.ts

@@ -2,19 +2,29 @@ import type { CreateSignInExperience } from '@logto/schemas';
 import { SignInExperiences } from '@logto/schemas';
 import type { CommonQueryMethods } from 'slonik';
 
+import { type WellKnownCache } from '#src/caches/well-known.js';
 import { buildFindEntityByIdWithPool } from '#src/database/find-entity-by-id.js';
 import { buildUpdateWhereWithPool } from '#src/database/update-where.js';
 
 const id = 'default';
 
-export const createSignInExperienceQueries = (pool: CommonQueryMethods) => {
+export const createSignInExperienceQueries = (
+  pool: CommonQueryMethods,
+  wellKnownCache: WellKnownCache
+) => {
   const updateSignInExperience = buildUpdateWhereWithPool(pool)(SignInExperiences, true);
+  const findSignInExperienceById = buildFindEntityByIdWithPool(pool)(SignInExperiences);
 
-  const updateDefaultSignInExperience = async (set: Partial<CreateSignInExperience>) =>
-    updateSignInExperience({ set, where: { id }, jsonbMode: 'replace' });
+  const updateDefaultSignInExperience = wellKnownCache.mutate(
+    async (set: Partial<CreateSignInExperience>) =>
+      updateSignInExperience({ set, where: { id }, jsonbMode: 'replace' }),
+    ['sie']
+  );
 
-  const findDefaultSignInExperience = async () =>
-    buildFindEntityByIdWithPool(pool)(SignInExperiences)(id);
+  const findDefaultSignInExperience = wellKnownCache.memoize(
+    async () => findSignInExperienceById(id),
+    ['sie']
+  );
 
   return {
     updateDefaultSignInExperience,

packages/core/src/routes/custom-phrase.test.ts

@@ -5,7 +5,7 @@ import { pickDefault, createMockUtils } from '@logto/shared/esm';
 import { mockZhCnCustomPhrase, trTrTag, zhCnTag } from '#src/__mocks__/custom-phrase.js';
 import { mockSignInExperience } from '#src/__mocks__/index.js';
 import RequestError from '#src/errors/RequestError/index.js';
-import { mockId, mockStandardId } from '#src/test-utils/nanoid.js';
+import { mockStandardId } from '#src/test-utils/nanoid.js';
 import { MockTenant } from '#src/test-utils/tenant.js';
 import { createRequester } from '#src/utils/test-utils.js';
 
@@ -127,11 +127,7 @@ describe('customPhraseRoutes', () => {
       await customPhraseRequest.put(`/custom-phrases/${mockLanguageTag}`).send({
         input: { ...inputTranslation, password: '' },
       });
-      expect(upsertCustomPhrase).toBeCalledWith({
-        id: mockId,
-        languageTag: mockLanguageTag,
-        translation: { input: inputTranslation },
-      });
+      expect(upsertCustomPhrase).toBeCalledWith(mockLanguageTag, { input: inputTranslation });
     });
 
     it('should call isStrictlyPartial', async () => {
@@ -151,7 +147,7 @@ describe('customPhraseRoutes', () => {
       await customPhraseRequest.put(`/custom-phrases/${mockLanguageTag}`).send(translation);
 
       const { tenantId, ...phrase } = mockCustomPhrases[mockLanguageTag]!;
-      expect(upsertCustomPhrase).toBeCalledWith(phrase);
+      expect(upsertCustomPhrase).toBeCalledWith(phrase.languageTag, phrase.translation);
     });
 
     it('should return custom phrase after upserting', async () => {

packages/core/src/routes/custom-phrase.ts

@@ -2,7 +2,6 @@ import { languageTagGuard } from '@logto/language-kit';
 import resource from '@logto/phrases-ui';
 import type { Translation } from '@logto/schemas';
 import { CustomPhrases, translationGuard } from '@logto/schemas';
-import { generateStandardId } from '@logto/shared';
 import cleanDeep from 'clean-deep';
 import { object } from 'zod';
 
@@ -80,7 +79,7 @@ export default function customPhraseRoutes<T extends AuthedRouter>(
         new RequestError('localization.invalid_translation_structure')
       );
 
-      ctx.body = await upsertCustomPhrase({ id: generateStandardId(), languageTag, translation });
+      ctx.body = await upsertCustomPhrase(languageTag, translation);
 
       return next();
     }

packages/core/src/routes/interaction/actions/submit-interaction.ts

@@ -10,7 +10,6 @@ import {
 } from '@logto/schemas';
 import { conditional, conditionalArray } from '@silverhand/essentials';
 
-import { wellKnownCache } from '#src/caches/well-known.js';
 import { EnvSet } from '#src/env-set/index.js';
 import type { ConnectorLibrary } from '#src/libraries/connector.js';
 import { assignInteractionResults } from '#src/libraries/session.js';
@@ -192,10 +191,6 @@ export default async function submitInteraction(
       await updateDefaultSignInExperience({
         signInMode: isCloud ? SignInMode.SignInAndRegister : SignInMode.SignIn,
       });
-
-      // Normally we don't need to manually invalidate TTL cache.
-      // This is for better OSS onboarding experience.
-      wellKnownCache.invalidate(tenantId, ['sie', 'sie-full']);
     }
 
     await assignInteractionResults(ctx, provider, { login: { accountId: id } });

packages/core/src/routes/interaction/index.ts

@@ -69,7 +69,7 @@ export default function interactionRoutes<T extends AnonymousRouter>(
         profile: profileGuard.optional(),
       }),
     }),
-    koaInteractionSie(libraries.signInExperiences, tenantId),
+    koaInteractionSie(libraries.signInExperiences),
     async (ctx, next) => {
       const { event, identifier, profile } = ctx.guard.body;
       const { signInExperience, createLog } = ctx;
@@ -119,7 +119,7 @@ export default function interactionRoutes<T extends AnonymousRouter>(
   router.put(
     `${interactionPrefix}/event`,
     koaGuard({ body: z.object({ event: eventGuard }) }),
-    koaInteractionSie(libraries.signInExperiences, tenantId),
+    koaInteractionSie(libraries.signInExperiences),
     async (ctx, next) => {
       const { event } = ctx.guard.body;
       const { signInExperience, interactionDetails, createLog } = ctx;
@@ -157,7 +157,7 @@ export default function interactionRoutes<T extends AnonymousRouter>(
     koaGuard({
       body: identifierPayloadGuard,
     }),
-    koaInteractionSie(libraries.signInExperiences, tenantId),
+    koaInteractionSie(libraries.signInExperiences),
     async (ctx, next) => {
       const identifierPayload = ctx.guard.body;
       const { signInExperience, interactionDetails, createLog } = ctx;
@@ -194,7 +194,7 @@ export default function interactionRoutes<T extends AnonymousRouter>(
     koaGuard({
       body: profileGuard,
     }),
-    koaInteractionSie(libraries.signInExperiences, tenantId),
+    koaInteractionSie(libraries.signInExperiences),
     async (ctx, next) => {
       const profilePayload = ctx.guard.body;
       const { signInExperience, interactionDetails, createLog } = ctx;
@@ -231,7 +231,7 @@ export default function interactionRoutes<T extends AnonymousRouter>(
     koaGuard({
       body: profileGuard,
     }),
-    koaInteractionSie(libraries.signInExperiences, tenantId),
+    koaInteractionSie(libraries.signInExperiences),
     async (ctx, next) => {
       const profilePayload = ctx.guard.body;
       const { signInExperience, interactionDetails, createLog } = ctx;
@@ -284,7 +284,7 @@ export default function interactionRoutes<T extends AnonymousRouter>(
   // Submit Interaction
   router.post(
     `${interactionPrefix}/submit`,
-    koaInteractionSie(libraries.signInExperiences, tenantId),
+    koaInteractionSie(libraries.signInExperiences),
     koaInteractionHooks(tenant),
     async (ctx, next) => {
       const { interactionDetails, createLog } = ctx;

packages/core/src/routes/interaction/middleware/koa-interaction-sie.ts

@@ -1,9 +1,7 @@
 import type { SignInExperience } from '@logto/schemas';
 import type { MiddlewareType } from 'koa';
 
-import { wellKnownCache } from '#src/caches/well-known.js';
 import type { SignInExperienceLibrary } from '#src/libraries/sign-in-experience/index.js';
-import { noCache } from '#src/utils/request.js';
 
 import type { WithInteractionDetailsContext } from './koa-interaction-details.js';
 
@@ -11,15 +9,14 @@ export type WithInteractionSieContext<ContextT> = WithInteractionDetailsContext<
   signInExperience: SignInExperience;
 };
 
-export default function koaInteractionSie<StateT, ContextT, ResponseT>(
-  { getSignInExperience }: SignInExperienceLibrary,
-  tenantId: string
-): MiddlewareType<StateT, WithInteractionSieContext<ContextT>, ResponseT> {
+export default function koaInteractionSie<StateT, ContextT, ResponseT>({
+  getSignInExperience,
+}: SignInExperienceLibrary): MiddlewareType<
+  StateT,
+  WithInteractionSieContext<ContextT>,
+  ResponseT
+> {
   return async (ctx, next) => {
-    if (noCache(ctx.request)) {
-      wellKnownCache.invalidate(tenantId, ['sie']);
-    }
-
     const signInExperience = await getSignInExperience();
 
     ctx.signInExperience = signInExperience;

packages/core/src/routes/well-known.phrases.content-language.test.ts

@@ -4,7 +4,6 @@ import { pickDefault } from '@logto/shared/esm';
 
 import { trTrTag, zhCnTag, mockTag } from '#src/__mocks__/custom-phrase.js';
 import { mockSignInExperience } from '#src/__mocks__/index.js';
-import { wellKnownCache } from '#src/caches/well-known.js';
 import { MockTenant } from '#src/test-utils/tenant.js';
 import { createRequester } from '#src/utils/test-utils.js';
 
@@ -42,7 +41,6 @@ const phraseRequest = createRequester({
 });
 
 afterEach(() => {
-  wellKnownCache.invalidateAll(tenantContext.id);
   jest.clearAllMocks();
 });
 

packages/core/src/routes/well-known.phrases.test.ts

@@ -4,18 +4,22 @@ import { pickDefault, createMockUtils } from '@logto/shared/esm';
 
 import { zhCnTag } from '#src/__mocks__/custom-phrase.js';
 import { mockSignInExperience } from '#src/__mocks__/index.js';
-import { wellKnownCache } from '#src/caches/well-known.js';
 import type Queries from '#src/tenants/Queries.js';
 import { createMockProvider } from '#src/test-utils/oidc-provider.js';
-import { MockTenant } from '#src/test-utils/tenant.js';
+import { MockTenant, MockWellKnownCache } from '#src/test-utils/tenant.js';
 
 const { jest } = import.meta;
 
 const { mockEsm } = createMockUtils(jest);
 
-const customizedLanguage = zhCnTag;
+const { default: detectLanguageSpy } = mockEsm('#src/i18n/detect-language.js', () => ({
+  default: jest.fn().mockReturnValue([]),
+}));
 
-const findDefaultSignInExperience = jest.fn(
+const customizedLanguage = zhCnTag;
+const mockCache = new MockWellKnownCache();
+
+const rawFindDefaultSignInExperience = jest.fn(
   async (): Promise<SignInExperience> => ({
     ...mockSignInExperience,
     languageInfo: {
@@ -24,13 +28,17 @@ const findDefaultSignInExperience = jest.fn(
     },
   })
 );
+const findDefaultSignInExperience = jest.fn(
+  mockCache.memoize(rawFindDefaultSignInExperience, ['sie'])
+);
 
-const { default: detectLanguageSpy } = mockEsm('#src/i18n/detect-language.js', () => ({
-  default: jest.fn().mockReturnValue([]),
-}));
+const rawFindAllCustomLanguageTags = jest.fn(async () => [customizedLanguage]);
+const findAllCustomLanguageTags = jest.fn(
+  mockCache.memoize(rawFindAllCustomLanguageTags, ['custom-phrases-tags'])
+);
 
 const customPhrases = {
-  findAllCustomLanguageTags: jest.fn(async () => [customizedLanguage]),
+  findAllCustomLanguageTags,
   findCustomPhraseByLanguageTag: jest.fn(
     async (tag: string): Promise<CustomPhrase> => ({
       tenantId: 'fake_tenant',
@@ -40,7 +48,6 @@ const customPhrases = {
     })
   ),
 } satisfies Partial<Queries['customPhrases']>;
-const { findAllCustomLanguageTags } = customPhrases;
 
 const getPhrases = jest.fn(async () => zhCN);
 
@@ -60,7 +67,7 @@ const phraseRequest = createRequester({
 });
 
 afterEach(() => {
-  wellKnownCache.invalidateAll(tenantContext.id);
+  mockCache.ttlCache.clear();
   jest.clearAllMocks();
 });
 
@@ -71,7 +78,7 @@ describe('when the application is not admin-console', () => {
   });
 
   it('should call detectLanguage when auto-detect is enabled', async () => {
-    findDefaultSignInExperience.mockResolvedValueOnce({
+    rawFindDefaultSignInExperience.mockResolvedValueOnce({
       ...mockSignInExperience,
       languageInfo: {
         ...mockSignInExperience.languageInfo,
@@ -83,7 +90,7 @@ describe('when the application is not admin-console', () => {
   });
 
   it('should not call detectLanguage when auto-detect is not enabled', async () => {
-    findDefaultSignInExperience.mockResolvedValueOnce({
+    rawFindDefaultSignInExperience.mockResolvedValueOnce({
       ...mockSignInExperience,
       languageInfo: {
         ...mockSignInExperience.languageInfo,
@@ -100,7 +107,7 @@ describe('when the application is not admin-console', () => {
   });
 
   it('should call getPhrases with fallback language from default sign-in experience', async () => {
-    findDefaultSignInExperience.mockResolvedValueOnce({
+    rawFindDefaultSignInExperience.mockResolvedValueOnce({
       ...mockSignInExperience,
       languageInfo: {
         autoDetect: false,
@@ -109,11 +116,11 @@ describe('when the application is not admin-console', () => {
     });
     await expect(phraseRequest.get('/.well-known/phrases')).resolves.toHaveProperty('status', 200);
     expect(getPhrases).toBeCalledTimes(1);
-    expect(getPhrases).toBeCalledWith(customizedLanguage, [customizedLanguage]);
+    expect(getPhrases).toBeCalledWith(customizedLanguage);
   });
 
   it('should call getPhrases with specific language is provided in params', async () => {
-    findDefaultSignInExperience.mockResolvedValueOnce({
+    rawFindDefaultSignInExperience.mockResolvedValueOnce({
       ...mockSignInExperience,
       languageInfo: {
         autoDetect: true,
@@ -124,7 +131,7 @@ describe('when the application is not admin-console', () => {
       'status',
       200
     );
-    expect(getPhrases).toBeCalledWith('fr', [customizedLanguage]);
+    expect(getPhrases).toBeCalledWith('fr');
   });
 
   it('should use cache for continuous requests', async () => {
@@ -133,8 +140,8 @@ describe('when the application is not admin-console', () => {
       phraseRequest.get('/.well-known/phrases'),
       phraseRequest.get('/.well-known/phrases'),
     ]);
-    expect(findDefaultSignInExperience).toHaveBeenCalledTimes(1);
-    expect(findAllCustomLanguageTags).toHaveBeenCalledTimes(1);
+    expect(rawFindDefaultSignInExperience).toHaveBeenCalledTimes(1);
+    expect(rawFindAllCustomLanguageTags).toHaveBeenCalledTimes(1);
     expect(response1.body).toStrictEqual(response2.body);
     expect(response1.body).toStrictEqual(response3.body);
   });

packages/core/src/routes/well-known.test.ts

@@ -10,7 +10,6 @@ import {
   mockWechatConnector,
   mockWechatNativeConnector,
 } from '#src/__mocks__/index.js';
-import { wellKnownCache } from '#src/caches/well-known.js';
 
 const { jest } = import.meta;
 const { mockEsm } = createMockUtils(jest);
@@ -56,7 +55,6 @@ const tenantContext = new MockTenant(
 
 describe('GET /.well-known/sign-in-exp', () => {
   afterEach(() => {
-    wellKnownCache.invalidateAll(tenantContext.id);
     jest.clearAllMocks();
   });
 
@@ -99,16 +97,4 @@ describe('GET /.well-known/sign-in-exp', () => {
       ],
     });
   });
-
-  it('should use cache for continuous requests', async () => {
-    const [response1, response2, response3] = await Promise.all([
-      sessionRequest.get('/.well-known/sign-in-exp'),
-      sessionRequest.get('/.well-known/sign-in-exp'),
-      sessionRequest.get('/.well-known/sign-in-exp'),
-    ]);
-    expect(findDefaultSignInExperience).toHaveBeenCalledTimes(1);
-    expect(getLogtoConnectors).toHaveBeenCalledTimes(1);
-    expect(response1.body).toStrictEqual(response2.body);
-    expect(response2.body).toStrictEqual(response3.body);
-  });
 });

packages/core/src/routes/well-known.ts

@@ -3,23 +3,22 @@ import { adminTenantId } from '@logto/schemas';
 import { conditionalArray } from '@silverhand/essentials';
 import { z } from 'zod';
 
-import { wellKnownCache } from '#src/caches/well-known.js';
 import { EnvSet, getTenantEndpoint } from '#src/env-set/index.js';
 import RequestError from '#src/errors/RequestError/index.js';
 import detectLanguage from '#src/i18n/detect-language.js';
-import { guardFullSignInExperience } from '#src/libraries/sign-in-experience/index.js';
+import { guardFullSignInExperience } from '#src/libraries/sign-in-experience/types.js';
 import koaGuard from '#src/middleware/koa-guard.js';
-import { noCache } from '#src/utils/request.js';
 
 import type { AnonymousRouter, RouterInitArgs } from './types.js';
 
 export default function wellKnownRoutes<T extends AnonymousRouter>(
-  ...[router, { libraries, id: tenantId }]: RouterInitArgs<T>
+  ...[router, { libraries, queries, id: tenantId }]: RouterInitArgs<T>
 ) {
   const {
     signInExperiences: { getSignInExperience, getFullSignInExperience },
-    phrases: { getPhrases, getAllCustomLanguageTags },
+    phrases: { getPhrases },
   } = libraries;
+  const { findAllCustomLanguageTags } = queries.customPhrases;
 
   if (tenantId === adminTenantId) {
     router.get('/.well-known/endpoints/:tenantId', async (ctx, next) => {
@@ -39,11 +38,6 @@ export default function wellKnownRoutes<T extends AnonymousRouter>(
     '/.well-known/sign-in-exp',
     koaGuard({ response: guardFullSignInExperience, status: 200 }),
     async (ctx, next) => {
-      if (noCache(ctx.request)) {
-        wellKnownCache.invalidate(tenantId, ['sie', 'sie-full']);
-        console.log('invalidated');
-      }
-
       ctx.body = await getFullSignInExperience();
 
       return next();
@@ -60,10 +54,6 @@ export default function wellKnownRoutes<T extends AnonymousRouter>(
       status: 200,
     }),
     async (ctx, next) => {
-      if (noCache(ctx.request)) {
-        wellKnownCache.invalidate(tenantId, ['sie', 'phrases-lng-tags', 'phrases']);
-      }
-
       const {
         query: { lng },
       } = ctx.guard;
@@ -77,14 +67,14 @@ export default function wellKnownRoutes<T extends AnonymousRouter>(
         autoDetect && detectLanguage(ctx),
         fallbackLanguage
       );
-      const customLanguages = await getAllCustomLanguageTags();
+      const customLanguages = await findAllCustomLanguageTags();
       const language =
         acceptableLanguages.find(
           (tag) => isBuiltInLanguageTag(tag) || customLanguages.includes(tag)
         ) ?? 'en';
 
       ctx.set('Content-Language', language);
-      ctx.body = await getPhrases(language, customLanguages);
+      ctx.body = await getPhrases(language);
 
       return next();
     }

packages/core/src/tenants/Libraries.ts

@@ -13,8 +13,8 @@ import type Queries from './Queries.js';
 
 export default class Libraries {
   users = createUserLibrary(this.queries);
-  signInExperiences = createSignInExperienceLibrary(this.queries, this.connectors, this.tenantId);
-  phrases = createPhraseLibrary(this.queries, this.tenantId);
+  signInExperiences = createSignInExperienceLibrary(this.queries, this.connectors);
+  phrases = createPhraseLibrary(this.queries);
   resources = createResourceLibrary(this.queries);
   hooks = createHookLibrary(this.queries);
   socials = createSocialLibrary(this.queries, this.connectors);

packages/core/src/tenants/Queries.ts

@@ -1,5 +1,6 @@
 import type { CommonQueryMethods } from 'slonik';
 
+import { type WellKnownCache } from '#src/caches/well-known.js';
 import { createApplicationQueries } from '#src/queries/application.js';
 import { createApplicationsRolesQueries } from '#src/queries/applications-roles.js';
 import { createConnectorQueries } from '#src/queries/connector.js';
@@ -20,8 +21,8 @@ import { createVerificationStatusQueries } from '#src/queries/verification-statu
 
 export default class Queries {
   applications = createApplicationQueries(this.pool);
-  connectors = createConnectorQueries(this.pool);
-  customPhrases = createCustomPhraseQueries(this.pool);
+  connectors = createConnectorQueries(this.pool, this.wellKnownCache);
+  customPhrases = createCustomPhraseQueries(this.pool, this.wellKnownCache);
   logs = createLogQueries(this.pool);
   oidcModelInstances = createOidcModelInstanceQueries(this.pool);
   passcodes = createPasscodeQueries(this.pool);
@@ -30,12 +31,15 @@ export default class Queries {
   roles = createRolesQueries(this.pool);
   scopes = createScopeQueries(this.pool);
   logtoConfigs = createLogtoConfigQueries(this.pool);
-  signInExperiences = createSignInExperienceQueries(this.pool);
+  signInExperiences = createSignInExperienceQueries(this.pool, this.wellKnownCache);
   users = createUserQueries(this.pool);
   usersRoles = createUsersRolesQueries(this.pool);
   applicationsRoles = createApplicationsRolesQueries(this.pool);
   verificationStatuses = createVerificationStatusQueries(this.pool);
   hooks = createHooksQueries(this.pool);
 
-  constructor(public readonly pool: CommonQueryMethods) {}
+  constructor(
+    public readonly pool: CommonQueryMethods,
+    public readonly wellKnownCache: WellKnownCache
+  ) {}
 }

packages/core/src/tenants/Tenant.test.ts

@@ -1,6 +1,7 @@
 import { adminTenantId, defaultTenantId } from '@logto/schemas';
 import { createMockUtils, pickDefault } from '@logto/shared/esm';
 
+import { RedisCache } from '#src/caches/index.js';
 import { createMockProvider } from '#src/test-utils/oidc-provider.js';
 import { emptyMiddleware } from '#src/utils/test-utils.js';
 
@@ -51,7 +52,7 @@ describe('Tenant', () => {
   });
 
   it('should call middleware factories for user tenants', async () => {
-    await Tenant.create(defaultTenantId);
+    await Tenant.create(defaultTenantId, new RedisCache());
 
     for (const [, middleware, shouldCall] of userMiddlewareList) {
       if (shouldCall) {
@@ -63,7 +64,7 @@ describe('Tenant', () => {
   });
 
   it('should call middleware factories for the admin tenant', async () => {
-    await Tenant.create(adminTenantId);
+    await Tenant.create(adminTenantId, new RedisCache());
 
     for (const [, middleware, shouldCall] of adminMiddlewareList) {
       if (shouldCall) {
@@ -77,7 +78,7 @@ describe('Tenant', () => {
 
 describe('Tenant `.run()`', () => {
   it('should return a function ', async () => {
-    const tenant = await Tenant.create(defaultTenantId);
+    const tenant = await Tenant.create(defaultTenantId, new RedisCache());
     expect(typeof tenant.run).toBe('function');
   });
 });

packages/core/src/tenants/Tenant.ts

@@ -7,6 +7,8 @@ import koaLogger from 'koa-logger';
 import mount from 'koa-mount';
 import type Provider from 'oidc-provider';
 
+import { type RedisCache } from '#src/caches/index.js';
+import { WellKnownCache } from '#src/caches/well-known.js';
 import { AdminApps, EnvSet, UserApps } from '#src/env-set/index.js';
 import { createConnectorLibrary } from '#src/libraries/connector.js';
 import koaConnectorErrorHandler from '#src/middleware/koa-connector-error-handler.js';
@@ -28,12 +30,12 @@ import type TenantContext from './TenantContext.js';
 import { getTenantDatabaseDsn } from './utils.js';
 
 export default class Tenant implements TenantContext {
-  static async create(id: string): Promise<Tenant> {
+  static async create(id: string, redisCache: RedisCache): Promise<Tenant> {
     // Treat the default database URL as the management URL
     const envSet = new EnvSet(id, await getTenantDatabaseDsn(id));
     await envSet.load();
 
-    return new Tenant(envSet, id);
+    return new Tenant(envSet, id, new WellKnownCache(id, redisCache));
   }
 
   public readonly provider: Provider;
@@ -48,7 +50,8 @@ export default class Tenant implements TenantContext {
   private constructor(
     public readonly envSet: EnvSet,
     public readonly id: string,
-    public readonly queries = new Queries(envSet.pool),
+    public readonly wellKnownCache: WellKnownCache,
+    public readonly queries = new Queries(envSet.pool, wellKnownCache),
     public readonly connectors = createConnectorLibrary(queries),
     public readonly libraries = new Libraries(id, queries, connectors)
   ) {

packages/core/src/tenants/index.ts

@@ -1,5 +1,6 @@
 import { LRUCache } from 'lru-cache';
 
+import { redisCache } from '#src/caches/index.js';
 import { EnvSet } from '#src/env-set/index.js';
 
 import Tenant from './Tenant.js';
@@ -21,7 +22,7 @@ export class TenantPool {
     }
 
     console.log('Init tenant:', tenantId);
-    const newTenant = Tenant.create(tenantId);
+    const newTenant = Tenant.create(tenantId, redisCache);
     this.cache.set(tenantId, newTenant);
 
     return newTenant;

packages/core/src/test-utils/tenant.ts

@@ -1,5 +1,7 @@
+import { TtlCache } from '@logto/shared';
 import { createMockPool, createMockQueryResult } from 'slonik';
 
+import { WellKnownCache } from '#src/caches/well-known.js';
 import type { ConnectorLibrary } from '#src/libraries/connector.js';
 import { createConnectorLibrary } from '#src/libraries/connector.js';
 import Libraries from '#src/tenants/Libraries.js';
@@ -10,6 +12,12 @@ import { mockEnvSet } from './env-set.js';
 import type { GrantMock } from './oidc-provider.js';
 import { createMockProvider } from './oidc-provider.js';
 
+export class MockWellKnownCache extends WellKnownCache {
+  constructor(public ttlCache = new TtlCache<string, string>(60_000)) {
+    super('mock_id', ttlCache);
+  }
+}
+
 export class MockQueries extends Queries {
   constructor(queriesOverride?: Partial2<Queries>) {
     super(
@@ -17,7 +25,8 @@ export class MockQueries extends Queries {
         query: async (sql, values) => {
           return createMockQueryResult([]);
         },
-      })
+      }),
+      new MockWellKnownCache()
     );
 
     if (!queriesOverride) {

packages/core/src/utils/connectors/types.ts

@@ -1,5 +1,6 @@
 import type { AllConnector, VerificationCodeType } from '@logto/connector-kit';
-import type { Connector } from '@logto/schemas';
+import { type Connector, Connectors } from '@logto/schemas';
+import { type z } from 'zod';
 
 export { ConnectorType } from '@logto/schemas';
 
@@ -11,3 +12,20 @@ export type TemplateType = VerificationCodeType;
 export type LogtoConnector<T extends AllConnector = AllConnector> = T & {
   validateConfig: (config: unknown) => void;
 } & { dbEntry: Connector };
+
+export const connectorWellKnownGuard = Connectors.guard.pick({
+  id: true,
+  metadata: true,
+  connectorId: true,
+});
+export type ConnectorWellKnown = z.infer<typeof connectorWellKnownGuard>;
+
+/**
+ * The connector type with full context but no sensitive info.
+ */
+export type LogtoConnectorWellKnown<T extends AllConnector = AllConnector> = Pick<
+  T,
+  'type' | 'metadata'
+> & {
+  dbEntry: ConnectorWellKnown;
+};

packages/core/src/utils/request.ts

@@ -1,8 +0,0 @@
-import type { Request } from 'koa';
-
-export const noCache = (request: Request): boolean =>
-  Boolean(
-    request.headers['cache-control']
-      ?.split(',')
-      .some((value) => ['no-cache', 'no-store'].includes(value.trim().toLowerCase()))
-  ) || request.URL.searchParams.get('no_cache') !== null;

packages/integration-tests/src/tests/api/well-known.test.ts

@@ -1,8 +1,7 @@
 import type { SignInExperience } from '@logto/schemas';
 
-import { adminTenantApi, authedAdminApi } from '#src/api/api.js';
+import { adminTenantApi } from '#src/api/api.js';
 import { api } from '#src/api/index.js';
-import { generateUserId } from '#src/utils.js';
 
 describe('.well-known api', () => {
   it('get /.well-known/sign-in-exp for console', async () => {
@@ -34,19 +33,4 @@ describe('.well-known api', () => {
     // Should support sign-in and register
     expect(response).toMatchObject({ signInMode: 'SignInAndRegister' });
   });
-
-  it('should use cached version if no-cache header is not present', async () => {
-    const response1 = await api.get('.well-known/sign-in-exp').json<SignInExperience>();
-
-    const randomId = generateUserId();
-    const customContent = { foo: randomId };
-    await authedAdminApi.patch('sign-in-exp', { json: { customContent } }).json<SignInExperience>();
-
-    const response2 = await api
-      .get('.well-known/sign-in-exp', { headers: { 'cache-control': '' } })
-      .json<SignInExperience>();
-
-    expect(response2.customContent.foo).not.toBe(randomId);
-    expect(response2).toStrictEqual(response1);
-  });
 });

packages/shared/src/node/env/GlobalValues.ts

@@ -102,6 +102,13 @@ export default class GlobalValues {
   /** Maximum number of clients to keep in a single database pool (i.e. per `Tenant` class). */
   public readonly databasePoolSize = Number(getEnv('DATABASE_POOL_SIZE', '20'));
 
+  /**
+   * The Redis endpoint (optional). If it's set, the central cache mechanism will be automatically enabled.
+   *
+   * You can set it to a truthy value like `true` or `1` to enable cache with the default Redis URL.
+   */
+  public readonly redisUrl = getEnv('REDIS_URL');
+
   public get dbUrl(): string {
     return this.databaseUrl;
   }

pnpm-lock.yaml

@@ -3140,6 +3140,9 @@ importers:
       pg-protocol:
         specifier: ^1.6.0
         version: 1.6.0
+      redis:
+        specifier: ^4.6.5
+        version: 4.6.5
       roarr:
         specifier: ^7.11.0
         version: 7.11.0
@@ -7426,6 +7429,55 @@ packages:
       react-dom: 18.2.0(react@18.2.0)
     dev: true
 
+  /@redis/bloom@1.2.0(@redis/client@1.5.6):
+    resolution: {integrity: sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+    dependencies:
+      '@redis/client': 1.5.6
+    dev: false
+
+  /@redis/client@1.5.6:
+    resolution: {integrity: sha512-dFD1S6je+A47Lj22jN/upVU2fj4huR7S9APd7/ziUXsIXDL+11GPYti4Suv5y8FuXaN+0ZG4JF+y1houEJ7ToA==}
+    engines: {node: '>=14'}
+    dependencies:
+      cluster-key-slot: 1.1.2
+      generic-pool: 3.9.0
+      yallist: 4.0.0
+    dev: false
+
+  /@redis/graph@1.1.0(@redis/client@1.5.6):
+    resolution: {integrity: sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+    dependencies:
+      '@redis/client': 1.5.6
+    dev: false
+
+  /@redis/json@1.0.4(@redis/client@1.5.6):
+    resolution: {integrity: sha512-LUZE2Gdrhg0Rx7AN+cZkb1e6HjoSKaeeW8rYnt89Tly13GBI5eP4CwDVr+MY8BAYfCg4/N15OUrtLoona9uSgw==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+    dependencies:
+      '@redis/client': 1.5.6
+    dev: false
+
+  /@redis/search@1.1.2(@redis/client@1.5.6):
+    resolution: {integrity: sha512-/cMfstG/fOh/SsE+4/BQGeuH/JJloeWuH+qJzM8dbxuWvdWibWAOAHHCZTMPhV3xIlH4/cUEIA8OV5QnYpaVoA==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+    dependencies:
+      '@redis/client': 1.5.6
+    dev: false
+
+  /@redis/time-series@1.0.4(@redis/client@1.5.6):
+    resolution: {integrity: sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+    dependencies:
+      '@redis/client': 1.5.6
+    dev: false
+
   /@rollup/plugin-commonjs@24.0.0(rollup@3.8.0):
     resolution: {integrity: sha512-0w0wyykzdyRRPHOb0cQt14mIBLujfAv6GgP6g8nvg/iBxEm112t3YPPq+Buqe2+imvElTka+bjNlJ/gB56TD8g==}
     engines: {node: '>=14.0.0'}
@@ -9621,6 +9673,11 @@ packages:
       semver: 5.7.1
     dev: false
 
+  /cluster-key-slot@1.1.2:
+    resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==}
+    engines: {node: '>=0.10.0'}
+    dev: false
+
   /co-body@5.2.0:
     resolution: {integrity: sha512-sX/LQ7LqUhgyaxzbe7IqwPeTr2yfpfUIQ/dgpKo6ZI4y4lpQA0YxAomWIY+7I7rHWcG02PG+OuPREzMW/5tszQ==}
     dependencies:
@@ -11591,6 +11648,11 @@ packages:
       loader-utils: 3.2.0
     dev: true
 
+  /generic-pool@3.9.0:
+    resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==}
+    engines: {node: '>= 4'}
+    dev: false
+
   /gensync@1.0.0-beta.2:
     resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==}
     engines: {node: '>=6.9.0'}
@@ -16793,6 +16855,17 @@ packages:
       strip-indent: 3.0.0
     dev: true
 
+  /redis@4.6.5:
+    resolution: {integrity: sha512-O0OWA36gDQbswOdUuAhRL6mTZpHFN525HlgZgDaVNgCJIAZR3ya06NTESb0R+TUZ+BFaDpz6NnnVvoMx9meUFg==}
+    dependencies:
+      '@redis/bloom': 1.2.0(@redis/client@1.5.6)
+      '@redis/client': 1.5.6
+      '@redis/graph': 1.1.0(@redis/client@1.5.6)
+      '@redis/json': 1.0.4(@redis/client@1.5.6)
+      '@redis/search': 1.1.2(@redis/client@1.5.6)
+      '@redis/time-series': 1.0.4(@redis/client@1.5.6)
+    dev: false
+
   /reduce-css-calc@2.1.8:
     resolution: {integrity: sha512-8liAVezDmUcH+tdzoEGrhfbGcP7nOV4NkGE3a74+qqvE7nt9i4sKLGBuZNOnpI4WiGksiNPklZxva80061QiPg==}
     dependencies: