refactor(schemas,core,toolkit): remove connector db storage and disable access (#3505)

.changeset/thin-balloons-brush.md

@@ -0,0 +1,5 @@
+---
+"@logto/connector-kit": patch
+---
+
+Remove connector database `storage` column and its corresponding access.

packages/core/src/__mocks__/connector-base-data.ts

@@ -82,7 +82,6 @@ export const mockConnector0: Connector = {
   createdAt: 1_234_567_890_123,
   syncProfile: false,
   metadata: {},
-  storage: {},
   connectorId: 'id0',
 };
 
@@ -93,7 +92,6 @@ export const mockConnector1: Connector = {
   createdAt: 1_234_567_890_234,
   syncProfile: false,
   metadata: {},
-  storage: {},
   connectorId: 'id1',
 };
 
@@ -104,7 +102,6 @@ export const mockConnector2: Connector = {
   createdAt: 1_234_567_890_345,
   syncProfile: false,
   metadata: {},
-  storage: {},
   connectorId: 'id2',
 };
 
@@ -115,7 +112,6 @@ export const mockConnector3: Connector = {
   createdAt: 1_234_567_890_456,
   syncProfile: false,
   metadata: {},
-  storage: {},
   connectorId: 'id3',
 };
 
@@ -126,7 +122,6 @@ export const mockConnector4: Connector = {
   createdAt: 1_234_567_890_567,
   syncProfile: false,
   metadata: {},
-  storage: {},
   connectorId: 'id4',
 };
 
@@ -137,7 +132,6 @@ export const mockConnector5: Connector = {
   createdAt: 1_234_567_890_567,
   syncProfile: false,
   metadata: {},
-  storage: {},
   connectorId: 'id5',
 };
 
@@ -148,6 +142,5 @@ export const mockConnector6: Connector = {
   createdAt: 1_234_567_890_567,
   syncProfile: false,
   metadata: {},
-  storage: {},
   connectorId: 'id6',
 };

packages/core/src/__mocks__/connector.ts

@@ -44,7 +44,6 @@ export const mockConnector: Connector = {
   createdAt: 1_234_567_890_123,
   syncProfile: false,
   metadata: {},
-  storage: {},
   connectorId: 'id',
 };
 
@@ -260,7 +259,6 @@ export const mockSocialConnectors: LogtoConnector[] = [
       createdAt: 1_234_567_890_123,
       syncProfile: false,
       metadata: {},
-      storage: {},
       connectorId: 'id0',
     },
     metadata: {
@@ -278,7 +276,6 @@ export const mockSocialConnectors: LogtoConnector[] = [
       createdAt: 1_234_567_890_123,
       syncProfile: false,
       metadata: {},
-      storage: {},
       connectorId: 'id1',
     },
     metadata: {

packages/core/src/libraries/connector.test.ts

@@ -12,7 +12,6 @@ const connectors: Connector[] = [
     syncProfile: false,
     connectorId: 'id',
     metadata: {},
-    storage: {},
   },
 ];
 

packages/core/src/libraries/social.ts

@@ -1,9 +1,4 @@
-import type {
-  GetSession,
-  SocialUserInfo,
-  SetStorageValue,
-  GetStorageValue,
-} from '@logto/connector-kit';
+import type { GetSession, SocialUserInfo } from '@logto/connector-kit';
 import { socialUserInfoGuard } from '@logto/connector-kit';
 import type { User } from '@logto/schemas';
 import { ConnectorType } from '@logto/schemas';
@@ -70,8 +65,7 @@ export const createSocialLibrary = (queries: Queries, connectorLibrary: Connecto
   const getUserInfoByAuthCode = async (
     connectorId: string,
     data: unknown,
-    getConnectorSession: GetSession,
-    storage: { set: SetStorageValue; get: GetStorageValue }
+    getConnectorSession: GetSession
   ): Promise<SocialUserInfo> => {
     const connector = await getConnector(connectorId);
 
@@ -84,7 +78,7 @@ export const createSocialLibrary = (queries: Queries, connectorLibrary: Connecto
       })
     );
 
-    return connector.getUserInfo(data, getConnectorSession, storage);
+    return connector.getUserInfo(data, getConnectorSession);
   };
 
   /**

packages/core/src/queries/connector.test.ts

@@ -22,8 +22,6 @@ const {
   findAllConnectors,
   findConnectorById,
   countConnectorByConnectorId,
-  setValueByIdAndKey,
-  getValueByIdAndKey,
   deleteConnectorById,
   deleteConnectorByIds,
   insertConnector,
@@ -56,7 +54,6 @@ describe('connector queries', () => {
       ...mockConnector,
       config: JSON.stringify(mockConnector.config),
       metadata: JSON.stringify(mockConnector.metadata),
-      storage: JSON.stringify(mockConnector.storage),
     };
     const expectSql = sql`
       select ${sql.join(Object.values(fields), sql`,`)}
@@ -93,62 +90,6 @@ describe('connector queries', () => {
     await expect(countConnectorByConnectorId(rowData.connectorId)).resolves.toEqual(rowData);
   });
 
-  it('setValueByIdAndKey', async () => {
-    const id = 'foo';
-    const key = 'bar';
-    const value = {
-      foo: 'foo',
-      bar: 1,
-      baz: { key1: [1, 2, 3], key2: ['a', 'b', 'c'], key3: false },
-    };
-    const rowData = { id, storage: { [key]: value } };
-    const expectSql = sql`
-      update ${table}
-      set
-      ${fields.storage}=
-      coalesce(${fields.storage},'{}'::jsonb) || ${JSON.stringify({
-      [key]: value,
-    })}
-      where ${fields.id}=$2
-      returning *
-    `;
-
-    mockQuery.mockImplementationOnce(async (sql, values) => {
-      expectSqlAssert(sql, expectSql.sql);
-      expect(values).toEqual([JSON.stringify({ [key]: value }), id]);
-
-      // @ts-expect-error createMockQueryResult doesn't support jsonb
-      return createMockQueryResult([rowData]);
-    });
-
-    await expect(setValueByIdAndKey(id, key, value)).resolves.toEqual(undefined);
-  });
-
-  it('getValueByIdAndKey', async () => {
-    const id = 'foo';
-    const key = 'bar';
-    const value = {
-      foo: 'foo',
-      bar: 1,
-      baz: { key1: [1, 2, 3], key2: ['a', 'b', 'c'], key3: false },
-    };
-    const expectSql = sql`
-      select ${fields.storage}->$1 as value
-      from ${table}
-      where ${fields.id} = $2;
-    `;
-
-    mockQuery.mockImplementationOnce(async (sql, values) => {
-      expectSqlAssert(sql, expectSql.sql);
-      expect(values).toEqual([key, id]);
-
-      // @ts-expect-error createMockQueryResult doesn't support jsonb
-      return createMockQueryResult([{ value }]);
-    });
-
-    await expect(getValueByIdAndKey(id, key)).resolves.toEqual(value);
-  });
-
   it('deleteConnectorById', async () => {
     const rowData = { id: 'foo' };
     const id = 'foo';
@@ -228,12 +169,11 @@ describe('connector queries', () => {
       ...mockConnector,
       config: JSON.stringify(mockConnector.config),
       metadata: JSON.stringify(mockConnector.metadata),
-      storage: JSON.stringify(mockConnector.storage),
     };
 
     const expectSql = `
-      insert into "connectors" ("id", "sync_profile", "connector_id", "config", "metadata", "storage")
-      values ($1, $2, $3, $4, $5, $6)
+      insert into "connectors" ("id", "sync_profile", "connector_id", "config", "metadata")
+      values ($1, $2, $3, $4, $5)
       returning *
     `;
 
@@ -246,7 +186,6 @@ describe('connector queries', () => {
         connector.connectorId,
         connector.config,
         connector.metadata,
-        connector.storage,
       ]);
 
       return createMockQueryResult([connector]);

packages/core/src/queries/connector.ts

@@ -32,24 +32,6 @@ export const createConnectorQueries = (pool: CommonQueryMethods) => {
       where ${fields.connectorId}=${connectorId}
     `);
 
-  const setValueByIdAndKey = async (id: string, key: string, value: unknown): Promise<void> => {
-    await updateConnector({
-      set: { storage: { [key]: value } },
-      where: { id },
-      jsonbMode: 'merge',
-    });
-  };
-
-  const getValueByIdAndKey = async <T = unknown>(id: string, key: string): Promise<T> => {
-    const { value } = await pool.one<{ value: T }>(sql`
-      select ${fields.storage}->${key} as value
-      from ${table}
-      where ${fields.id} = ${id};
-    `);
-
-    return value;
-  };
-
   const deleteConnectorById = async (id: string) => {
     const { rowCount } = await pool.query(sql`
       delete from ${table}
@@ -80,8 +62,6 @@ export const createConnectorQueries = (pool: CommonQueryMethods) => {
     findAllConnectors,
     findConnectorById,
     countConnectorByConnectorId,
-    setValueByIdAndKey,
-    getValueByIdAndKey,
     deleteConnectorById,
     deleteConnectorByIds,
     insertConnector,

packages/core/src/routes-me/social.ts

@@ -115,10 +115,7 @@ export default function socialRoutes<T extends AuthedMeRouter>(
        * Same as above, passing `notImplemented` only works for connectors not relying on session storage.
        * E.g. Google and GitHub
        */
-      const socialUserInfo = await connector.getUserInfo(connectorData, notImplemented, {
-        get: notImplemented,
-        set: notImplemented,
-      });
+      const socialUserInfo = await connector.getUserInfo(connectorData, notImplemented);
 
       assertThat(
         !(await hasUserWithIdentity(target, socialUserInfo.id, userId)),

packages/core/src/routes/interaction/utils/social-verification.test.ts

@@ -38,12 +38,7 @@ describe('social-verification', () => {
     const connectorData = { authCode: 'code' };
     const userInfo = await verifySocialIdentity({ connectorId, connectorData }, ctx, tenant);
 
-    expect(getUserInfoByAuthCode).toBeCalledWith(
-      connectorId,
-      connectorData,
-      expect.anything(),
-      expect.anything()
-    );
+    expect(getUserInfoByAuthCode).toBeCalledWith(connectorId, connectorData, expect.anything());
     expect(userInfo).toEqual({ id: 'foo' });
   });
 });

packages/core/src/routes/interaction/utils/social-verification.ts

@@ -54,26 +54,17 @@ export const createSocialAuthorizationUrl = async (
 export const verifySocialIdentity = async (
   { connectorId, connectorData }: SocialConnectorPayload,
   ctx: WithLogContext,
-  { provider, queries, libraries }: TenantContext
+  { provider, libraries }: TenantContext
 ): Promise<SocialUserInfo> => {
   const {
     socials: { getUserInfoByAuthCode },
   } = libraries;
-  const {
-    connectors: { setValueByIdAndKey, getValueByIdAndKey },
-  } = queries;
 
   const log = ctx.createLog('Interaction.SignIn.Identifier.Social.Submit');
   log.append({ connectorId, connectorData });
 
-  const userInfo = await getUserInfoByAuthCode(
-    connectorId,
-    connectorData,
-    async () => getConnectorSessionResult(ctx, provider),
-    {
-      set: async (key: string, value: unknown) => setValueByIdAndKey(connectorId, key, value),
-      get: async (key: string) => getValueByIdAndKey(connectorId, key),
-    }
+  const userInfo = await getUserInfoByAuthCode(connectorId, connectorData, async () =>
+    getConnectorSessionResult(ctx, provider)
   );
 
   log.append(userInfo);

packages/schemas/alterations/next-1679209413-drop-connector-database-storage.ts

@@ -0,0 +1,18 @@
+import { sql } from 'slonik';
+
+import type { AlterationScript } from '../lib/types/alteration.js';
+
+const alteration: AlterationScript = {
+  up: async (pool) => {
+    await pool.query(sql`
+      alter table connectors drop storage;
+    `);
+  },
+  down: async (pool) => {
+    await pool.query(sql`
+      alter table connectors add storage jsonb not null default '{}'::jsonb;
+    `);
+  },
+};
+
+export default alteration;

packages/schemas/src/foundations/jsonb-types.ts

@@ -4,9 +4,7 @@ import { z } from 'zod';
 
 export {
   configurableConnectorMetadataGuard,
-  storageGuard,
   type ConfigurableConnectorMetadata,
-  type Storage,
 } from '@logto/connector-kit';
 
 /* === Commonly Used === */

packages/schemas/tables/connectors.sql

@@ -6,7 +6,6 @@ create table connectors (
   connector_id varchar(128) not null,
   config jsonb /* @use ArbitraryObject */ not null default '{}'::jsonb,
   metadata jsonb /* @use ConfigurableConnectorMetadata */ not null default '{}'::jsonb,
-  storage jsonb /* @use Storage */ not null default '{}'::jsonb,
   created_at timestamptz not null default(now()),
   primary key (id)
 );

packages/toolkit/connector-kit/src/types.ts

@@ -1,7 +1,7 @@
 import type { LanguageTag } from '@logto/language-kit';
 import { isLanguageTag } from '@logto/language-kit';
 import type { ZodType } from 'zod';
-import { z, unknown } from 'zod';
+import { z } from 'zod';
 
 // MARK: Foundation
 export enum ConnectorType {
@@ -184,14 +184,6 @@ export type GetSession = () => Promise<ConnectorSession>;
 
 export type SetSession = (storage: ConnectorSession) => Promise<void>;
 
-export const storageGuard = z.record(unknown());
-
-export type Storage = z.infer<typeof storageGuard>;
-
-export type GetStorageValue = (key: string) => Promise<unknown>;
-
-export type SetStorageValue = (key: string, value: unknown) => Promise<void>;
-
 export type BaseConnector<Type extends ConnectorType> = {
   type: Type;
   metadata: ConnectorMetadata;
@@ -265,6 +257,5 @@ export type SocialUserInfo = z.infer<typeof socialUserInfoGuard>;
 
 export type GetUserInfo = (
   data: unknown,
-  getSession: GetSession,
-  storage: { set: SetStorageValue; get: GetStorageValue }
+  getSession: GetSession
 ) => Promise<SocialUserInfo & Record<string, string | boolean | number | undefined>>;