feat(schemas): guard string max length (#1737)

packages/schemas/jest.config.ts

@@ -0,0 +1 @@
+export { default } from '@silverhand/jest-config';

packages/schemas/package.json

@@ -14,7 +14,8 @@
     "build": "pnpm generate && rm -rf lib/ && tsc --p tsconfig.build.json",
     "lint": "eslint --ext .ts src",
     "lint:report": "pnpm lint --format json --output-file report.json",
-    "prepack": "pnpm build"
+    "prepack": "pnpm build",
+    "test": "jest"
   },
   "engines": {
     "node": "^16.0.0"
@@ -22,12 +23,15 @@
   "devDependencies": {
     "@silverhand/eslint-config": "1.0.0-rc.2",
     "@silverhand/essentials": "^1.1.6",
+    "@silverhand/jest-config": "1.0.0-rc.3",
     "@silverhand/ts-config": "1.0.0-rc.2",
+    "@types/jest": "^27.4.1",
     "@types/lodash.uniq": "^4.5.6",
     "@types/node": "16",
     "@types/pluralize": "^0.0.29",
     "camelcase": "^6.2.0",
     "eslint": "^8.21.0",
+    "jest": "^28.1.3",
     "lint-staged": "^13.0.0",
     "lodash.uniq": "^4.5.0",
     "pluralize": "^8.0.0",

packages/schemas/src/gen/index.ts

@@ -14,27 +14,41 @@ import { generateSchema } from './schema';
 import { FileData, Table, Field, Type, GeneratedType, TableWithType } from './types';
 import {
   findFirstParentheses,
-  getType,
   normalizeWhitespaces,
-  removeParentheses,
+  parseType,
   removeUnrecognizedComments,
+  splitTableFieldDefinitions,
 } from './utils';
 
 const directory = 'tables';
+const constrainedKeywords = [
+  'primary',
+  'foreign',
+  'unique',
+  'exclude',
+  'check',
+  'constraint',
+  'references',
+];
 
 const getOutputFileName = (file: string) => pluralize(file.slice(0, -4).replace(/_/g, '-'), 1);
 
 const generate = async () => {
   const files = await fs.readdir(directory);
+
   const generated = await Promise.all(
     files
       .filter((file) => file.endsWith('.sql'))
       .map<Promise<[string, FileData]>>(async (file) => {
         const paragraph = await fs.readFile(path.join(directory, file), { encoding: 'utf8' });
+
+        // Get statements
         const statements = paragraph
           .split(';')
           .map((value) => normalizeWhitespaces(value))
           .map((value) => removeUnrecognizedComments(value));
+
+        // Parse Table statements
         const tables = statements
           .filter((value) => value.toLowerCase().startsWith('create table'))
           .map((value) => findFirstParentheses(value))
@@ -44,55 +58,19 @@ const generate = async () => {
             const name = normalizeWhitespaces(prefix).split(' ')[2];
             assert(name, 'Missing table name: ' + prefix);
 
-            const fields = removeParentheses(body)
+            const fields = splitTableFieldDefinitions(body)
-              .split(',')
               .map((value) => normalizeWhitespaces(value))
               .filter((value) =>
-                [
+                constrainedKeywords.every(
-                  'primary',
+                  (constraint) => !value.toLowerCase().startsWith(constraint + ' ')
-                  'foreign',
+                )
-                  'unique',
-                  'exclude',
-                  'check',
-                  'constraint',
-                  'references',
-                ].every((constraint) => !value.toLowerCase().startsWith(constraint + ' '))
               )
-              // eslint-disable-next-line complexity
+              .map<Field>((value) => parseType(value));
-              .map<Field>((value) => {
-                const [nameRaw, typeRaw, ...rest] = value.split(' ');
-                assert(nameRaw && typeRaw, 'Missing column name or type: ' + value);
-
-                const name = nameRaw.toLowerCase();
-                const type = typeRaw.toLowerCase();
-                const restJoined = rest.join(' ');
-                const restLowercased = restJoined.toLowerCase();
-                // CAUTION: Only works for single dimension arrays
-                const isArray = Boolean(/\[.*]/.test(type)) || restLowercased.includes('array');
-                const hasDefaultValue = restLowercased.includes('default');
-                const nullable = !restLowercased.includes('not null');
-                const primitiveType = getType(type);
-                const tsType = /\/\* @use (.*) \*\//.exec(restJoined)?.[1];
-                assert(
-                  !(!primitiveType && tsType),
-                  `TS type can only be applied on primitive types, found ${
-                    tsType ?? 'N/A'
-                  } over ${type}`
-                );
-
-                return {
-                  name,
-                  type: primitiveType,
-                  customType: conditional(!primitiveType && type),
-                  tsType,
-                  isArray,
-                  hasDefaultValue,
-                  nullable,
-                };
-              });
 
             return { name, fields };
           });
+
+        // Parse enum statements
         const types = statements
           .filter((value) => value.toLowerCase().startsWith('create type'))
           .map<Type>((value) => {
@@ -152,7 +130,6 @@ const generate = async () => {
   // Generate DB entry types
   await Promise.all(
     generated.map(async ([file, { tables }]) => {
-      // LOG-88 Need refactor, disable mutation rules for now.
       /* eslint-disable @silverhand/fp/no-mutating-methods */
       const tsTypes: string[] = [];
       const customTypes: string[] = [];
@@ -215,6 +192,7 @@ const generate = async () => {
       await fs.writeFile(path.join(generatedDirectory, getOutputFileName(file) + '.ts'), content);
     })
   );
+
   await fs.writeFile(
     path.join(generatedDirectory, 'index.ts'),
     header +

packages/schemas/src/gen/schema.ts

@@ -33,35 +33,45 @@ export const generateSchema = ({ name, fields }: TableWithType) => {
     '};',
     '',
     `const createGuard: CreateGuard<${databaseEntryType}> = z.object({`,
-    // eslint-disable-next-line complexity
-    ...fields.map(({ name, type, isArray, isEnum, nullable, hasDefaultValue, tsType }) => {
-      if (tsType) {
-        return `  ${camelcase(name)}: ${camelcase(tsType)}Guard${conditionalString(
-          nullable && '.nullable()'
-        )}${conditionalString((nullable || hasDefaultValue) && '.optional()')},`;
-      }
 
-      return `  ${camelcase(name)}: z.${
+    ...fields.map(
-        isEnum ? `nativeEnum(${type})` : `${type}()`
+      // eslint-disable-next-line complexity
-      }${conditionalString(isArray && '.array()')}${conditionalString(
+      ({ name, type, isArray, isEnum, nullable, hasDefaultValue, tsType, isString, maxLength }) => {
-        nullable && '.nullable()'
+        if (tsType) {
-      )}${conditionalString((nullable || hasDefaultValue) && '.optional()')},`;
+          return `  ${camelcase(name)}: ${camelcase(tsType)}Guard${conditionalString(
-    }),
+            nullable && '.nullable()'
+          )}${conditionalString((nullable || hasDefaultValue) && '.optional()')},`;
+        }
+
+        return `  ${camelcase(name)}: z.${
+          isEnum ? `nativeEnum(${type})` : `${type}()`
+        }${conditionalString(isString && maxLength && `.max(${maxLength})`)}${conditionalString(
+          isArray && '.array()'
+        )}${conditionalString(nullable && '.nullable()')}${conditionalString(
+          (nullable || hasDefaultValue) && '.optional()'
+        )},`;
+      }
+    ),
     '  });',
     '',
     `const guard: Guard<${modelName}> = z.object({`,
-    // eslint-disable-next-line complexity
-    ...fields.map(({ name, type, isArray, isEnum, nullable, tsType }) => {
-      if (tsType) {
-        return `  ${camelcase(name)}: ${camelcase(tsType)}Guard${conditionalString(
-          nullable && '.nullable()'
-        )},`;
-      }
 
-      return `  ${camelcase(name)}: z.${
+    ...fields.map(
-        isEnum ? `nativeEnum(${type})` : `${type}()`
+      // eslint-disable-next-line complexity
-      }${conditionalString(isArray && '.array()')}${conditionalString(nullable && '.nullable()')},`;
+      ({ name, type, isArray, isEnum, nullable, tsType, isString, maxLength }) => {
-    }),
+        if (tsType) {
+          return `  ${camelcase(name)}: ${camelcase(tsType)}Guard${conditionalString(
+            nullable && '.nullable()'
+          )},`;
+        }
+
+        return `  ${camelcase(name)}: z.${
+          isEnum ? `nativeEnum(${type})` : `${type}()`
+        }${conditionalString(isString && maxLength && `.max(${maxLength})`)}${conditionalString(
+          isArray && '.array()'
+        )}${conditionalString(nullable && '.nullable()')},`;
+      }
+    ),
     '  });',
     '',
     `export const ${camelcase(name, {

packages/schemas/src/gen/types.ts

@@ -3,6 +3,8 @@ export type Field = {
   type?: string;
   customType?: string;
   tsType?: string;
+  isString: boolean;
+  maxLength?: number;
   hasDefaultValue: boolean;
   nullable: boolean;
   isArray: boolean;

packages/schemas/src/gen/utils.test.ts

@@ -0,0 +1,96 @@
+import { parseType, getType, splitTableFieldDefinitions } from './utils';
+
+describe('splitTableFieldDefinitions', () => {
+  it('splitTableFieldDefinitions should split at each comma that is not in the parentheses', () => {
+    const segments = ['a', 'b(1)', 'c(2,3)', 'd(4,(5,6))'];
+    expect(splitTableFieldDefinitions(segments.join(','))).toEqual(segments);
+    const oneSegment = 'id bigint';
+    expect(splitTableFieldDefinitions(oneSegment)).toEqual([oneSegment]);
+  });
+});
+
+describe('getType', () => {
+  it.each(['varchar(32)[]', 'char', 'text'])('getStringType', (type) => {
+    expect(getType(type)).toBe('string');
+  });
+
+  it.each(['int2', 'float4', 'timestamp'])('should return number', (type) => {
+    expect(getType(type)).toBe('number');
+  });
+});
+
+describe('parseType', () => {
+  const length = 128;
+
+  it('should throw without column name', () => {
+    expect(() => parseType('varchar')).toThrow();
+  });
+
+  it.each([`foo bpchar(${length})`, `foo char(${length})`, `foo varchar(${length})`])(
+    'should return the string max length of %s',
+    (type) => {
+      expect(parseType(type)).toMatchObject({
+        name: 'foo',
+        type: 'string',
+        isArray: false,
+        isString: true,
+        maxLength: length,
+        hasDefaultValue: false,
+        nullable: true,
+        tsType: undefined,
+        customType: undefined,
+      });
+    }
+  );
+
+  it.each([
+    ['foo text', 'string'],
+    ['foo timestamp(6)', 'number'],
+    ['foo numeric(4,2)', 'number'],
+    ['foo jsonb', 'Record<string, unknown>'],
+  ])(
+    'should not return the max length since %s is not the character type with length limit',
+    (value, type) => {
+      expect(parseType(value)).toMatchObject({
+        name: 'foo',
+        type,
+        isArray: false,
+        maxLength: undefined,
+        hasDefaultValue: false,
+        nullable: true,
+        tsType: undefined,
+        customType: undefined,
+      });
+    }
+  );
+
+  it('should return isArray', () => {
+    expect(parseType(`foo varchar(${length})[]`)).toMatchObject({
+      name: 'foo',
+      type: 'string',
+      maxLength: length,
+      isArray: true,
+    });
+
+    expect(parseType(`foo varchar(${length}) array`)).toMatchObject({
+      name: 'foo',
+      type: 'string',
+      maxLength: length,
+      isArray: true,
+    });
+  });
+
+  it('should return tsType', () => {
+    expect(
+      parseType(
+        `custom_client_metadata jsonb /* @use CustomClientMetadata */ not null default '{}'::jsonb,`
+      )
+    ).toMatchObject({
+      name: 'custom_client_metadata',
+      type: 'Record<string, unknown>',
+      tsType: 'CustomClientMetadata',
+      nullable: false,
+      hasDefaultValue: true,
+    });
+  });
+});

packages/schemas/src/gen/utils.ts

@@ -1,4 +1,6 @@
-import { Optional } from '@silverhand/essentials';
+import { conditional, Optional, assert } from '@silverhand/essentials';
+
+import { Field } from './types';
 
 export const normalizeWhitespaces = (string: string): string => string.replace(/\s+/g, ' ').trim();
 
@@ -18,21 +20,6 @@ const getCountDelta = (value: string): number => {
   return 0;
 };
 
-export const removeParentheses = (value: string) =>
-  Object.values(value).reduce<{ result: string; count: number }>(
-    (previous, current) => {
-      const count = previous.count + getCountDelta(current);
-
-      return count === 0 && current !== ')'
-        ? { result: previous.result + current, count }
-        : { result: previous.result, count };
-    },
-    {
-      result: '',
-      count: 0,
-    }
-  ).result;
-
 export type ParenthesesMatch = { body: string; prefix: string };
 
 export const findFirstParentheses = (value: string): Optional<ParenthesesMatch> => {
@@ -78,10 +65,42 @@ export const findFirstParentheses = (value: string): Optional<ParenthesesMatch>
   return matched ? rest : undefined;
 };
 
-const getRawType = (value: string): string => {
+export const splitTableFieldDefinitions = (value: string) =>
-  const bracketIndex = value.indexOf('[');
+  // Split at each comma that is not in parentheses
+  Object.values(value).reduce<{ result: string[]; count: number }>(
+    ({ result, count: previousCount }, current) => {
+      const count = previousCount + getCountDelta(current);
 
-  return bracketIndex === -1 ? value : value.slice(0, bracketIndex);
+      if (count === 0 && current === ',') {
+        return {
+          result: [...result, ''],
+          count,
+        };
+      }
+
+      const rest = result.slice(0, -1);
+      const last = result.at(-1) ?? '';
+
+      return {
+        result: [...rest, `${last}${current}`],
+        count,
+      };
+    },
+    {
+      result: [''],
+      count: 0,
+    }
+  ).result;
+
+const getRawType = (value: string): string => {
+  const squareBracketIndex = value.indexOf('[');
+  const parenthesesIndex = value.indexOf('(');
+
+  if (parenthesesIndex !== -1) {
+    return value.slice(0, parenthesesIndex);
+  }
+
+  return squareBracketIndex === -1 ? value : value.slice(0, squareBracketIndex);
 };
 
 // Reference: https://github.com/SweetIQ/schemats/blob/7c3d3e16b5d507b4d9bd246794e7463b05d20e75/src/schemaPostgres.ts
@@ -90,7 +109,7 @@ export const getType = (
   value: string
 ): 'string' | 'number' | 'boolean' | 'Record<string, unknown>' | undefined => {
   switch (getRawType(value)) {
-    case 'bpchar':
+    case 'bpchar': // https://www.postgresql.org/docs/current/typeconv-query.html
     case 'char':
     case 'varchar':
     case 'text':
@@ -124,3 +143,59 @@ export const getType = (
     default:
   }
 };
+
+const parseStringMaxLength = (rawType: string) => {
+  const squareBracketIndex = rawType.indexOf('[');
+
+  const parenthesesMatch = findFirstParentheses(
+    squareBracketIndex === -1 ? rawType : rawType.slice(0, squareBracketIndex)
+  );
+
+  return conditional(
+    parenthesesMatch &&
+      ['bpchar', 'char', 'varchar'].includes(parenthesesMatch.prefix) &&
+      Number(parenthesesMatch.body)
+  );
+};
+
+// eslint-disable-next-line complexity
+export const parseType = (tableFieldDefinition: string): Field => {
+  const [nameRaw, typeRaw, ...rest] = tableFieldDefinition.split(' ');
+
+  assert(nameRaw && typeRaw, new Error('Missing field name or type: ' + tableFieldDefinition));
+
+  const name = nameRaw.toLowerCase();
+  const type = typeRaw.toLowerCase();
+
+  const restJoined = rest.join(' ');
+  const restLowercased = restJoined.toLowerCase();
+
+  const primitiveType = getType(type);
+
+  const isString = primitiveType === 'string';
+  // CAUTION: Only works for single dimension arrays
+  const isArray = Boolean(/\[.*]/.test(type)) || restLowercased.includes('array');
+
+  const hasDefaultValue = restLowercased.includes('default');
+  const nullable = !restLowercased.includes('not null');
+  const tsType = /\/\* @use (.*) \*\//.exec(restJoined)?.[1];
+
+  assert(
+    !(!primitiveType && tsType),
+    new Error(
+      `TS type can only be applied on primitive types, found ${tsType ?? 'N/A'} over ${type}`
+    )
+  );
+
+  return {
+    name,
+    type: primitiveType,
+    isString,
+    isArray,
+    maxLength: conditional(isString && parseStringMaxLength(type)),
+    customType: conditional(!primitiveType && type),
+    tsType,
+    hasDefaultValue,
+    nullable,
+  };
+};

packages/schemas/tsconfig.build.json

@@ -1,4 +1,5 @@
 {
   "extends": "./tsconfig",
+  "include": ["src"],
   "exclude": ["src/gen"]
 }

packages/schemas/tsconfig.json

@@ -5,6 +5,7 @@
     "declaration": true
   },
   "include": [
-    "src"
+    "src",
+    "jest.config.ts",
   ]
 }

packages/schemas/tsconfig.test.json

@@ -0,0 +1,3 @@
+{
+  "extends": "./tsconfig"
+}

pnpm-lock.yaml

@@ -1225,12 +1225,15 @@ importers:
       '@logto/shared': ^1.0.0-beta.3
       '@silverhand/eslint-config': 1.0.0-rc.2
       '@silverhand/essentials': ^1.1.6
+      '@silverhand/jest-config': 1.0.0-rc.3
       '@silverhand/ts-config': 1.0.0-rc.2
+      '@types/jest': ^27.4.1
       '@types/lodash.uniq': ^4.5.6
       '@types/node': '16'
       '@types/pluralize': ^0.0.29
       camelcase: ^6.2.0
       eslint: ^8.21.0
+      jest: ^28.1.3
       lint-staged: ^13.0.0
       lodash.uniq: ^4.5.0
       pluralize: ^8.0.0
@@ -1247,12 +1250,15 @@ importers:
     devDependencies:
       '@silverhand/eslint-config': 1.0.0-rc.2_swk2g7ygmfleszo5c33j4vooni
       '@silverhand/essentials': 1.1.7
+      '@silverhand/jest-config': 1.0.0-rc.3_bi2kohzqnxavgozw3csgny5hju
       '@silverhand/ts-config': 1.0.0-rc.2_typescript@4.7.4
+      '@types/jest': 27.5.2
       '@types/lodash.uniq': 4.5.6
       '@types/node': 16.11.12
       '@types/pluralize': 0.0.29
       camelcase: 6.2.1
       eslint: 8.21.0
+      jest: 28.1.3_k5ytkvaprncdyzidqqws5bqksq
       lint-staged: 13.0.0
       lodash.uniq: 4.5.0
       pluralize: 8.0.0
@@ -4982,6 +4988,13 @@ packages:
     dependencies:
       '@types/istanbul-lib-report': 3.0.0
 
+  /@types/jest/27.5.2:
+    resolution: {integrity: sha512-mpT8LJJ4CMeeahobofYWIjFo0xonRS/HfxnVEPMPFSQdGUt1uHCnoPT7Zhb+sjDU2wz0oKV0OLUR0WzrHNgfeA==}
+    dependencies:
+      jest-matcher-utils: 27.5.1
+      pretty-format: 27.5.1
+    dev: true
+
   /@types/jest/28.1.6:
     resolution: {integrity: sha512-0RbGAFMfcBJKOmqRazM8L98uokwuwD5F8rHrv/ZMbrZBwVOWZUyPG6VFNscjYr/vjM3Vu4fRrCPbOs42AfemaQ==}
     dependencies:
@@ -6999,6 +7012,11 @@ packages:
       asap: 2.0.6
       wrappy: 1.0.2
 
+  /diff-sequences/27.5.1:
+    resolution: {integrity: sha512-k1gCAXAsNgLwEL+Y8Wvl+M6oEFj5bgazfZULpS5CneoPPXRaCCW7dm+q21Ky2VEE5X+VeRDBVg1Pcvvsr4TtNQ==}
+    engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0}
+    dev: true
+
   /diff-sequences/28.1.1:
     resolution: {integrity: sha512-FU0iFaH/E23a+a718l8Qa/19bF9p06kgE0KipMOMadwa3SjnaElKzPaUC0vnibs6/B/9ni97s61mcejk8W1fQw==}
     engines: {node: ^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0}
@@ -9663,6 +9681,16 @@ packages:
       - supports-color
     dev: true
 
+  /jest-diff/27.5.1:
+    resolution: {integrity: sha512-m0NvkX55LDt9T4mctTEgnZk3fmEg3NRYutvMPWM/0iPnkFj2wIeF45O1718cMSOFO1vINkqmxqD8vE37uTEbqw==}
+    engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0}
+    dependencies:
+      chalk: 4.1.2
+      diff-sequences: 27.5.1
+      jest-get-type: 27.5.1
+      pretty-format: 27.5.1
+    dev: true
+
   /jest-diff/28.1.3:
     resolution: {integrity: sha512-8RqP1B/OXzjjTWkqMX67iqgwBVJRgCyKD3L9nq+6ZqJMdvjE8RgHktqZ6jNrkdMT+dJuYNI3rhQpxaz7drJHfw==}
     engines: {node: ^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0}
@@ -9718,6 +9746,11 @@ packages:
       jest-mock: 28.1.3
       jest-util: 28.1.3
 
+  /jest-get-type/27.5.1:
+    resolution: {integrity: sha512-2KY95ksYSaK7DMBWQn6dQz3kqAf3BB64y2udeG+hv4KfSOb9qwcYQstTJc1KCbsix+wLZWZYN8t7nwX3GOBLRw==}
+    engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0}
+    dev: true
+
   /jest-get-type/28.0.2:
     resolution: {integrity: sha512-ioj2w9/DxSYHfOm5lJKCdcAmPJzQXmbM/Url3rhlghrPvT3tt+7a/+oXc9azkKmLvoiXjtV83bEWqi+vs5nlPA==}
     engines: {node: ^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0}
@@ -9750,6 +9783,16 @@ packages:
   /jest-matcher-specific-error/1.0.0:
     resolution: {integrity: sha512-thJdy9ibhDo8k+0arFalNCQBJ0u7eqTfpTzS2MzL3iCLmbRCkI+yhhKSiAxEi55e5ZUyf01ySa0fMqzF+sblAw==}
 
+  /jest-matcher-utils/27.5.1:
+    resolution: {integrity: sha512-z2uTx/T6LBaCoNWNFWwChLBKYxTMcGBRjAt+2SbP929/Fflb9aa5LGma654Rz8z9HLxsrUaYzxE9T/EFIL/PAw==}
+    engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0}
+    dependencies:
+      chalk: 4.1.2
+      jest-diff: 27.5.1
+      jest-get-type: 27.5.1
+      pretty-format: 27.5.1
+    dev: true
+
   /jest-matcher-utils/28.1.3:
     resolution: {integrity: sha512-kQeJ7qHemKfbzKoGjHHrRKH6atgxMk8Enkk2iPQ3XwO6oE/KYD8lMYOziCkeSB9G4adPM4nR1DE8Tf5JeWH6Bw==}
     engines: {node: ^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0}