feat: enabled connector db read and write (#3127)

packages/core/src/__mocks__/connector-base-data.ts

@@ -82,6 +82,7 @@ export const mockConnector0: Connector = {
   createdAt: 1_234_567_890_123,
   syncProfile: false,
   metadata: {},
+  storage: {},
   connectorId: 'id0',
 };
 
@@ -92,6 +93,7 @@ export const mockConnector1: Connector = {
   createdAt: 1_234_567_890_234,
   syncProfile: false,
   metadata: {},
+  storage: {},
   connectorId: 'id1',
 };
 
@@ -102,6 +104,7 @@ export const mockConnector2: Connector = {
   createdAt: 1_234_567_890_345,
   syncProfile: false,
   metadata: {},
+  storage: {},
   connectorId: 'id2',
 };
 
@@ -112,6 +115,7 @@ export const mockConnector3: Connector = {
   createdAt: 1_234_567_890_456,
   syncProfile: false,
   metadata: {},
+  storage: {},
   connectorId: 'id3',
 };
 
@@ -122,6 +126,7 @@ export const mockConnector4: Connector = {
   createdAt: 1_234_567_890_567,
   syncProfile: false,
   metadata: {},
+  storage: {},
   connectorId: 'id4',
 };
 
@@ -132,6 +137,7 @@ export const mockConnector5: Connector = {
   createdAt: 1_234_567_890_567,
   syncProfile: false,
   metadata: {},
+  storage: {},
   connectorId: 'id5',
 };
 
@@ -142,5 +148,6 @@ export const mockConnector6: Connector = {
   createdAt: 1_234_567_890_567,
   syncProfile: false,
   metadata: {},
+  storage: {},
   connectorId: 'id6',
 };

packages/core/src/__mocks__/connector.ts

@@ -42,6 +42,7 @@ export const mockConnector: Connector = {
   createdAt: 1_234_567_890_123,
   syncProfile: false,
   metadata: {},
+  storage: {},
   connectorId: 'id',
 };
 
@@ -242,6 +243,7 @@ export const mockSocialConnectors: LogtoConnector[] = [
       createdAt: 1_234_567_890_123,
       syncProfile: false,
       metadata: {},
+      storage: {},
       connectorId: 'id0',
     },
     metadata: {
@@ -259,6 +261,7 @@ export const mockSocialConnectors: LogtoConnector[] = [
       createdAt: 1_234_567_890_123,
       syncProfile: false,
       metadata: {},
+      storage: {},
       connectorId: 'id1',
     },
     metadata: {

packages/core/src/libraries/connector.test.ts

@@ -12,6 +12,7 @@ const connectors: Connector[] = [
     syncProfile: false,
     connectorId: 'id',
     metadata: {},
+    storage: {},
   },
 ];
 

packages/core/src/libraries/social.ts

@@ -1,4 +1,9 @@
-import type { GetSession, SocialUserInfo } from '@logto/connector-kit';
+import type {
+  GetSession,
+  SocialUserInfo,
+  SetStorageValue,
+  GetStorageValue,
+} from '@logto/connector-kit';
 import { socialUserInfoGuard } from '@logto/connector-kit';
 import type { User } from '@logto/schemas';
 import { ConnectorType } from '@logto/schemas';
@@ -65,7 +70,8 @@ export const createSocialLibrary = (queries: Queries, connectorLibrary: Connecto
   const getUserInfoByAuthCode = async (
     connectorId: string,
     data: unknown,
-    getConnectorSession: GetSession
+    getConnectorSession: GetSession,
+    storage: { set: SetStorageValue; get: GetStorageValue }
   ): Promise<SocialUserInfo> => {
     const connector = await getConnector(connectorId);
 
@@ -78,7 +84,7 @@ export const createSocialLibrary = (queries: Queries, connectorLibrary: Connecto
       })
     );
 
-    return connector.getUserInfo(data, getConnectorSession);
+    return connector.getUserInfo(data, getConnectorSession, storage);
   };
 
   /**

packages/core/src/queries/connector.test.ts

@@ -22,6 +22,8 @@ const {
   findAllConnectors,
   findConnectorById,
   countConnectorByConnectorId,
+  setValueByIdAndKey,
+  getValueByIdAndKey,
   deleteConnectorById,
   deleteConnectorByIds,
   insertConnector,
@@ -54,6 +56,7 @@ describe('connector queries', () => {
       ...mockConnector,
       config: JSON.stringify(mockConnector.config),
       metadata: JSON.stringify(mockConnector.metadata),
+      storage: JSON.stringify(mockConnector.storage),
     };
     const expectSql = sql`
       select ${sql.join(Object.values(fields), sql`,`)}
@@ -90,6 +93,62 @@ describe('connector queries', () => {
     await expect(countConnectorByConnectorId(rowData.connectorId)).resolves.toEqual(rowData);
   });
 
+  it('setValueByIdAndKey', async () => {
+    const id = 'foo';
+    const key = 'bar';
+    const value = {
+      foo: 'foo',
+      bar: 1,
+      baz: { key1: [1, 2, 3], key2: ['a', 'b', 'c'], key3: false },
+    };
+    const rowData = { id, storage: { [key]: value } };
+    const expectSql = sql`
+      update ${table}
+      set
+      ${fields.storage}=
+      coalesce(${fields.storage},'{}'::jsonb) || ${JSON.stringify({
+      [key]: value,
+    })}
+      where ${fields.id}=$2
+      returning *
+    `;
+
+    mockQuery.mockImplementationOnce(async (sql, values) => {
+      expectSqlAssert(sql, expectSql.sql);
+      expect(values).toEqual([JSON.stringify({ [key]: value }), id]);
+
+      // @ts-expect-error createMockQueryResult doesn't support jsonb
+      return createMockQueryResult([rowData]);
+    });
+
+    await expect(setValueByIdAndKey(id, key, value)).resolves.toEqual(undefined);
+  });
+
+  it('getValueByIdAndKey', async () => {
+    const id = 'foo';
+    const key = 'bar';
+    const value = {
+      foo: 'foo',
+      bar: 1,
+      baz: { key1: [1, 2, 3], key2: ['a', 'b', 'c'], key3: false },
+    };
+    const expectSql = sql`
+      select ${fields.storage}->$1 as value
+      from ${table}
+      where ${fields.id} = $2;
+    `;
+
+    mockQuery.mockImplementationOnce(async (sql, values) => {
+      expectSqlAssert(sql, expectSql.sql);
+      expect(values).toEqual([key, id]);
+
+      // @ts-expect-error createMockQueryResult doesn't support jsonb
+      return createMockQueryResult([{ value }]);
+    });
+
+    await expect(getValueByIdAndKey(id, key)).resolves.toEqual(value);
+  });
+
   it('deleteConnectorById', async () => {
     const rowData = { id: 'foo' };
     const id = 'foo';
@@ -169,11 +228,12 @@ describe('connector queries', () => {
       ...mockConnector,
       config: JSON.stringify(mockConnector.config),
       metadata: JSON.stringify(mockConnector.metadata),
+      storage: JSON.stringify(mockConnector.storage),
     };
 
     const expectSql = `
-      insert into "connectors" ("id", "sync_profile", "connector_id", "config", "metadata")
-      values ($1, $2, $3, $4, $5)
+      insert into "connectors" ("id", "sync_profile", "connector_id", "config", "metadata", "storage")
+      values ($1, $2, $3, $4, $5, $6)
       returning *
     `;
 
@@ -186,6 +246,7 @@ describe('connector queries', () => {
         connector.connectorId,
         connector.config,
         connector.metadata,
+        connector.storage,
       ]);
 
       return createMockQueryResult([connector]);

packages/core/src/queries/connector.ts

@@ -32,6 +32,24 @@ export const createConnectorQueries = (pool: CommonQueryMethods) => {
       where ${fields.connectorId}=${connectorId}
     `);
 
+  const setValueByIdAndKey = async (id: string, key: string, value: unknown): Promise<void> => {
+    await updateConnector({
+      set: { storage: { [key]: value } },
+      where: { id },
+      jsonbMode: 'merge',
+    });
+  };
+
+  const getValueByIdAndKey = async <T = unknown>(id: string, key: string): Promise<T> => {
+    const { value } = await pool.one<{ value: T }>(sql`
+      select ${fields.storage}->${key} as value
+      from ${table}
+      where ${fields.id} = ${id};
+    `);
+
+    return value;
+  };
+
   const deleteConnectorById = async (id: string) => {
     const { rowCount } = await pool.query(sql`
       delete from ${table}
@@ -65,6 +83,8 @@ export const createConnectorQueries = (pool: CommonQueryMethods) => {
     findAllConnectors,
     findConnectorById,
     countConnectorByConnectorId,
+    setValueByIdAndKey,
+    getValueByIdAndKey,
     deleteConnectorById,
     deleteConnectorByIds,
     insertConnector,

packages/core/src/routes/connector.ts

@@ -1,7 +1,9 @@
+/* eslint-disable max-lines */
 import { VerificationCodeType, validateConfig } from '@logto/connector-kit';
 import { emailRegEx, phoneRegEx, buildIdGenerator } from '@logto/core-kit';
 import type { ConnectorResponse, ConnectorFactoryResponse } from '@logto/schemas';
 import { arbitraryObjectGuard, Connectors, ConnectorType } from '@logto/schemas';
+import { pick } from '@silverhand/essentials';
 import cleanDeep from 'clean-deep';
 import { string, object } from 'zod';
 
@@ -19,11 +21,13 @@ const transpileLogtoConnector = ({
   dbEntry,
   metadata,
   type,
-}: LogtoConnector): ConnectorResponse => ({
-  type,
-  ...metadata,
-  ...dbEntry,
-});
+}: LogtoConnector): ConnectorResponse => {
+  return {
+    type,
+    ...metadata,
+    ...pick(dbEntry, 'id', 'connectorId', 'syncProfile', 'config', 'metadata'),
+  };
+};
 
 const generateConnectorId = buildIdGenerator(12);
 
@@ -355,3 +359,5 @@ export default function connectorRoutes<T extends AuthedRouter>(
     }
   );
 }
+// TODO: @darcy refactor this file, exceed max-lines
+/* eslint-enable max-lines */

packages/core/src/routes/interaction/utils/social-verification.test.ts

@@ -36,7 +36,12 @@ describe('social-verification', () => {
     const connectorData = { authCode: 'code' };
     const userInfo = await verifySocialIdentity({ connectorId, connectorData }, ctx, tenant);
 
-    expect(getUserInfoByAuthCode).toBeCalledWith(connectorId, connectorData, expect.anything());
+    expect(getUserInfoByAuthCode).toBeCalledWith(
+      connectorId,
+      connectorData,
+      expect.anything(),
+      expect.anything()
+    );
     expect(userInfo).toEqual({ id: 'foo' });
   });
 });

packages/core/src/routes/interaction/utils/social-verification.ts

@@ -39,7 +39,7 @@ export const createSocialAuthorizationUrl = async (
       state,
       redirectUri,
       /**
-       * For POST /saml-assertion-handler/:connectorId API, we need to block requests
+       * For POST /authn/saml/:connectorId API, we need to block requests
        * for non-SAML connector (relies on connectorFactoryId) and use `connectorId`
        * to find correct connector config.
        */
@@ -56,17 +56,26 @@ export const createSocialAuthorizationUrl = async (
 export const verifySocialIdentity = async (
   { connectorId, connectorData }: SocialConnectorPayload,
   ctx: WithLogContext,
-  { provider, libraries }: TenantContext
+  { provider, queries, libraries }: TenantContext
 ): Promise<SocialUserInfo> => {
   const {
     socials: { getUserInfoByAuthCode },
   } = libraries;
+  const {
+    connectors: { setValueByIdAndKey, getValueByIdAndKey },
+  } = queries;
 
   const log = ctx.createLog('Interaction.SignIn.Identifier.Social.Submit');
   log.append({ connectorId, connectorData });
 
-  const userInfo = await getUserInfoByAuthCode(connectorId, connectorData, async () =>
-    getConnectorSessionResult(ctx, provider)
+  const userInfo = await getUserInfoByAuthCode(
+    connectorId,
+    connectorData,
+    async () => getConnectorSessionResult(ctx, provider),
+    {
+      set: async (key: string, value: unknown) => setValueByIdAndKey(connectorId, key, value),
+      get: async (key: string) => getValueByIdAndKey(connectorId, key),
+    }
   );
 
   log.append(userInfo);

packages/schemas/alterations/next-1676886855-connector-database-read-write.ts

@@ -0,0 +1,18 @@
+import { sql } from 'slonik';
+
+import type { AlterationScript } from '../lib/types/alteration.js';
+
+const alteration: AlterationScript = {
+  up: async (pool) => {
+    await pool.query(sql`
+      alter table connectors add storage jsonb not null default '{}'::jsonb;
+    `);
+  },
+  down: async (pool) => {
+    await pool.query(sql`
+      alter table connectors drop storage;
+    `);
+  },
+};
+
+export default alteration;

packages/schemas/src/foundations/jsonb-types.ts

@@ -4,7 +4,9 @@ import { z } from 'zod';
 
 export {
   configurableConnectorMetadataGuard,
+  storageGuard,
   type ConfigurableConnectorMetadata,
+  type Storage,
 } from '@logto/connector-kit';
 
 /**

packages/schemas/src/types/connector.ts

@@ -5,7 +5,10 @@ import type { Connector } from '../db-entries/index.js';
 export type { ConnectorMetadata } from '@logto/connector-kit';
 export { ConnectorType, ConnectorPlatform } from '@logto/connector-kit';
 
-export type ConnectorResponse = Connector &
+export type ConnectorResponse = Pick<
+  Connector,
+  'id' | 'syncProfile' | 'config' | 'metadata' | 'connectorId'
+> &
   Omit<BaseConnector<ConnectorType>, 'configGuard' | 'metadata'> &
   ConnectorMetadata;
 

packages/schemas/tables/connectors.sql

@@ -6,6 +6,7 @@ create table connectors (
   connector_id varchar(128) not null,
   config jsonb /* @use ArbitraryObject */ not null default '{}'::jsonb,
   metadata jsonb /* @use ConfigurableConnectorMetadata */ not null default '{}'::jsonb,
+  storage jsonb /* @use Storage */ not null default '{}'::jsonb,
   created_at timestamptz not null default(now()),
   primary key (id)
 );

packages/toolkit/connector-kit/src/types.ts

@@ -1,7 +1,7 @@
 import type { LanguageTag } from '@logto/language-kit';
 import { isLanguageTag } from '@logto/language-kit';
 import type { ZodType } from 'zod';
-import { z } from 'zod';
+import { z, unknown } from 'zod';
 
 // MARK: Foundation
 export enum ConnectorType {
@@ -181,6 +181,14 @@ export type GetSession = () => Promise<ConnectorSession>;
 
 export type SetSession = (storage: ConnectorSession) => Promise<void>;
 
+export const storageGuard = z.record(unknown());
+
+export type Storage = z.infer<typeof storageGuard>;
+
+export type GetStorageValue = (key: string) => Promise<unknown>;
+
+export type SetStorageValue = (key: string, value: unknown) => Promise<void>;
+
 export type BaseConnector<Type extends ConnectorType> = {
   type: Type;
   metadata: ConnectorMetadata;
@@ -247,5 +255,6 @@ export type SocialUserInfo = z.infer<typeof socialUserInfoGuard>;
 
 export type GetUserInfo = (
   data: unknown,
-  getSession: GetSession
+  getSession: GetSession,
+  storage: { set: SetStorageValue; get: GetStorageValue }
 ) => Promise<SocialUserInfo & Record<string, string | boolean | number | undefined>>;