perf: config validation improvements (#192)

* perf: improve config validation

* chore: remove extra space in package.json

* fix: actually update file

* fix: `datasource.local` not providing a default value

* fix: small oversight in readConfig & better error

Co-authored-by: diced <pranaco2@gmail.com>

package.json

@@ -32,6 +32,7 @@
     "@prisma/client": "^4.1.0",
     "@prisma/internals": "^4.1.0",
     "@prisma/migrate": "^4.1.0",
+    "@sapphire/shapeshift": "^3.7.0",
     "@tanstack/react-query": "^4.2.3",
     "argon2": "^0.28.5",
     "chart.js": "^3.9.1",
@@ -54,8 +55,7 @@
     "react-dom": "^18.2.0",
     "react-feather": "^2.0.10",
     "recoil": "^0.7.5",
-    "sharp": "^0.30.7",
+    "sharp": "^0.30.7"
-    "yup": "^0.32.11"
   },
   "devDependencies": {
     "@types/cookie": "^0.5.1",

src/lib/config/readConfig.ts

@@ -148,8 +148,10 @@ export default function readConfig() {
           } catch (e) {
             parsed = [];
           }
+          break;
         default:
           parsed = value;
+          break;
       };
 
       set(config, map.path, parsed);

src/lib/config/validateConfig.ts

@@ -1,109 +1,148 @@
 import { Config } from 'lib/config/Config';
-import { object, string, number, boolean, array } from 'yup';
+import { CombinedError, s, ValidationError } from '@sapphire/shapeshift';
+import { inspect } from 'util';
+import Logger from 'lib/logger';
 
-const discord_content = object({
+const discord_content = s.object({
-  content: string().nullable(),
+  content: s.string.nullish.default(null),
-  embed: object({
+  embed: s.object({
-    title: string().nullable().default(null),
+    title: s.string.nullish.default(null),
-    description: string().nullable().default(null),
+    description: s.string.nullish.default(null),
-    footer: string().nullable().default(null),
+    footer: s.string.nullish.default(null),
-    color: string().nullable().default(null),
+    color: s.number.notEqual(NaN).nullish.default(null),
-    thumbnail: boolean().default(false),
+    thumbnail: s.boolean.default(false),
-    image: boolean().default(true),
+    image: s.boolean.default(true),
-    timestamp: boolean().default(true),
+    timestamp: s.boolean.default(true),
-  }).nullable().default(null),
+  }).default(null),
-}).nullable().default(null);
+}).default(null);
 
-const validator = object({
+const validator = s.object({
-  core: object({
+  core: s.object({
-    https: boolean().default(false),
+    https: s.boolean.default(false),
-    secret: string().min(8).required(),
+    secret: s.string.lengthGreaterThanOrEqual(8),
-    host: string().default('0.0.0.0'),
+    host: s.string.default('0.0.0.0'),
-    port: number().default(3000),
+    port: s.number.default(3000),
-    database_url: string().required(),
+    database_url: s.string,
-    logger: boolean().default(false),
+    logger: s.boolean.default(false),
-    stats_interval: number().default(1800),
+    stats_interval: s.number.default(1800),
-    invites_interval: number().default(1800),
+    invites_interval: s.number.default(1800),
-  }).required(),
-  datasource: object({
-    type: string().oneOf(['local', 's3', 'swift']).default('local'),
-    local: object({
-      directory: string().default('./uploads'),
   }),
-    s3: object({
+  datasource: s.object({
-      access_key_id: string(),
+    type: s.enum('local', 's3', 'swift').default('local'),
-      secret_access_key: string(),
+    local: s.object({
-      endpoint: string(),
+      directory: s.string.default('./uploads'),
-      bucket: string(),
+    }).default({
-      force_s3_path: boolean().default(false),
+      directory: './uploads',
-      region: string().default('us-east-1'),
-      use_ssl: boolean().default(false),
-    }).nullable().notRequired(),
-    swift: object({
-      username: string(),
-      password: string(),
-      auth_endpoint: string(),
-      container: string(),
-      project_id: string(),
-      domain_id: string().default('default'),
-      region_id: string().nullable(),
-    }).nullable().notRequired(),
-  }).required(),
-  uploader: object({
-    route: string().default('/u'),
-    embed_route: string().default('/a'),
-    length: number().default(6),
-    admin_limit: number().default(104900000),
-    user_limit: number().default(104900000),
-    disabled_extensions: array().default([]),
-    format_date: string().default('YYYY-MM-DD_HH:mm:ss'),
-  }).required(),
-  urls: object({
-    route: string().default('/go'),
-    length: number().default(6),
-  }).required(),
-  ratelimit: object({
-    user: number().default(0),
-    admin: number().default(0),
     }),
-  website: object({
+    s3: s.object({
-    title: string().default('Zipline'),
+      access_key_id: s.string,
-    show_files_per_user: boolean().default(true),
+      secret_access_key: s.string,
-    show_version: boolean().default(true),
+      endpoint: s.string,
-    disable_media_preview: boolean().default(false),
+      bucket: s.string,
+      force_s3_path: s.boolean.default(false),
+      region: s.string.default('us-east-1'),
+      use_ssl: s.boolean.default(false),
+    }).optional,
+    swift: s.object({
+      username: s.string,
+      password: s.string,
+      auth_endpoint: s.string,
+      container: s.string,
+      project_id: s.string,
+      domain_id: s.string.default('default'),
+      region_id: s.string.nullable,
+    }).optional,
+  }).default({
+    type: 'local',
+    local: {
+      directory: './uploads',
+    },
+    s3: {
+      region: 'us-east-1',
+      force_s3_path: false,
+    },
+    swift: {
+      domain_id: 'default',
+    },
+  }),
+  uploader: s.object({
+    route: s.string.default('/u'),
+    embed_route: s.string.default('/a'),
+    length: s.number.default(6),
+    admin_limit: s.number.default(104900000),
+    user_limit: s.number.default(104900000),
+    disabled_extensions: s.string.array.default([]),
+    format_date: s.string.default('YYYY-MM-DD_HH:mm:ss'),
+  }).default({
+    route: '/u',
+    embed_route: '/a',
+    length: 6,
+    admin_limit: 104900000,
+    user_limit: 104900000,
+    disabled_extensions: [],
+    format_date: 'YYYY-MM-DD_HH:mm:ss',
+  }),
+  urls: s.object({
+    route: s.string.default('/go'),
+    length: s.number.default(6),
+  }).default({
+    route: '/go',
+    length: 6,
+  }),
+  ratelimit: s.object({
+    user: s.number.default(0),
+    admin: s.number.default(0),
+  }).default({
+    user: 0,
+    admin: 0,
+  }),
+  website: s.object({
+    title: s.string.default('Zipline'),
+    show_files_per_user: s.boolean.default(true),
+    show_version: s.boolean.default(true),
+    disable_media_preview: s.boolean.default(false),
 
-    external_links: array(object({
+    external_links: s.array(s.object({
-      label: string(),
+      label: s.string,
-      link: string(),
+      link: s.string,
     })).default([
       { label: 'Zipline', link: 'https://github.com/diced/zipline' },
       { label: 'Documentation', link: 'https://zipline.diced.tech/' },
     ]),
+  }).default({
+    title: 'Zipline',
+    show_files_per_user: true,
+    show_version: true,
+    disable_media_preview: false,
+
+    external_links: [
+      { label: 'Zipline', link: 'https://github.com/diced/zipline' },
+      { label: 'Documentation', link: 'https://zipline.diced.tech/' },
+    ],
   }),
-  discord: object({
+  discord: s.object({
-    url: string(),
+    url: s.string,
-    username: string().default('Zipline'),
+    username: s.string.default('Zipline'),
-    avatar_url: string().default('https://raw.githubusercontent.com/diced/zipline/9b60147e112ec5b70170500b85c75ea621f41d03/public/zipline.png'),
+    avatar_url: s.string.default('https://raw.githubusercontent.com/diced/zipline/9b60147e112ec5b70170500b85c75ea621f41d03/public/zipline.png'),
     upload: discord_content,
     shorten: discord_content,
-  }).optional().nullable().default(null),
+  }),
-  oauth: object({
+  oauth: s.object({
-    github_client_id: string().nullable().default(null),
+    github_client_id: s.string.nullable.default(null),
-    github_client_secret: string().nullable().default(null),
+    github_client_secret: s.string.nullable.default(null),
-
-    discord_client_id: string().nullable().default(null),
-    discord_client_secret: string().nullable().default(null),
-  }).optional().nullable().default(null),
-  features: object({
-    invites: boolean().default(true),
-    oauth_registration: boolean().default(false),
-  }).required(),
 
+    discord_client_id: s.string.nullable.default(null),
+    discord_client_secret: s.string.nullable.default(null),
+  }).nullish.default(null),
+  features: s.object({
+    invites: s.boolean.default(true),
+    oauth_registration: s.boolean.default(false),
+  }).default({ invites: true, oauth_registration: false }),
 });
 
 export default function validate(config): Config {
   try {
-    const validated = validator.validateSync(config, { abortEarly: false });
+    const validated = validator.parse(config);
     switch (validated.datasource.type) {
       case 's3': {
         const errors = [];
@@ -138,6 +177,12 @@ export default function validate(config): Config {
     return validated as unknown as Config;
   } catch (e) {
     if (process.env.ZIPLINE_DOCKER_BUILD) return null;
-    throw `${e.errors.length} errors occured\n${e.errors.map((x) => '\t' + x).join('\n')}`;
+
+    e.stack = '';
+
+    Logger.get('config').error('Config is invalid, see below:');
+    Logger.get('config').error(inspect(e, { depth: Infinity, colors: true }));
+
+    process.exit(1);
   }
 }

yarn.lock

@@ -272,7 +272,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@babel/runtime@npm:^7.10.2, @babel/runtime@npm:^7.12.5, @babel/runtime@npm:^7.13.10, @babel/runtime@npm:^7.15.4, @babel/runtime@npm:^7.18.3, @babel/runtime@npm:^7.18.9, @babel/runtime@npm:^7.5.5, @babel/runtime@npm:^7.8.7":
+"@babel/runtime@npm:^7.10.2, @babel/runtime@npm:^7.12.5, @babel/runtime@npm:^7.13.10, @babel/runtime@npm:^7.18.3, @babel/runtime@npm:^7.18.9, @babel/runtime@npm:^7.5.5, @babel/runtime@npm:^7.8.7":
   version: 7.18.9
   resolution: "@babel/runtime@npm:7.18.9"
   dependencies:
@@ -1401,6 +1401,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@sapphire/shapeshift@npm:^3.7.0":
+  version: 3.7.0
+  resolution: "@sapphire/shapeshift@npm:3.7.0"
+  dependencies:
+    fast-deep-equal: ^3.1.3
+    lodash.uniqwith: ^4.5.0
+  checksum: 4fed0865abcf3653406cfa1f4a2a7d1c51103cee1c13ec4fd8fbc84bd32d20b2949e2266531c2d81b9b1e3af32787cd1f5d66a3d6146d6afb553ca2c6377beb1
+  languageName: node
+  linkType: hard
+
 "@sindresorhus/slugify@npm:1.1.2":
   version: 1.1.2
   resolution: "@sindresorhus/slugify@npm:1.1.2"
@@ -1608,13 +1618,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@types/lodash@npm:^4.14.175":
-  version: 4.14.183
-  resolution: "@types/lodash@npm:4.14.183"
-  checksum: 9c754dc7a2e5f26f9c67e494cffbe5447135a4e30eb2fcbc9da05dd5fa5fbf8579059bcf15014307c1c5d1c6d1b7870860618990d96abee9389d8cb79b3ac93c
-  languageName: node
-  linkType: hard
-
 "@types/mime@npm:*":
   version: 3.0.1
   resolution: "@types/mime@npm:3.0.1"
@@ -5586,13 +5589,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"lodash-es@npm:^4.17.21":
-  version: 4.17.21
-  resolution: "lodash-es@npm:4.17.21"
-  checksum: 05cbffad6e2adbb331a4e16fbd826e7faee403a1a04873b82b42c0f22090f280839f85b95393f487c1303c8a3d2a010048bf06151a6cbe03eee4d388fb0a12d2
-  languageName: node
-  linkType: hard
-
 "lodash.deburr@npm:^4.1.0":
   version: 4.1.0
   resolution: "lodash.deburr@npm:4.1.0"
@@ -5691,6 +5687,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"lodash.uniqwith@npm:^4.5.0":
+  version: 4.5.0
+  resolution: "lodash.uniqwith@npm:4.5.0"
+  checksum: d49a4565ed64efd86674127d321622673c29cde3e060baebc0f30372f22886c61b2ead44709db8c890053db1b9660e8ed689689812c1a485eb5703caa94d1150
+  languageName: node
+  linkType: hard
+
 "lodash@npm:^4.17.21":
   version: 4.17.21
   resolution: "lodash@npm:4.17.21"
@@ -6146,13 +6149,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"nanoclone@npm:^0.2.1":
-  version: 0.2.1
-  resolution: "nanoclone@npm:0.2.1"
-  checksum: 96b2954e22f70561f41e20d69856266c65583c2a441dae108f1dc71b716785d2c8038dac5f1d5e92b117aed3825f526b53139e2e5d6e6db8a77cfa35b3b8bf40
-  languageName: node
-  linkType: hard
-
 "nanoid@npm:^3.3.4":
   version: 3.3.4
   resolution: "nanoid@npm:3.3.4"
@@ -7059,13 +7055,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"property-expr@npm:^2.0.4":
-  version: 2.0.5
-  resolution: "property-expr@npm:2.0.5"
-  checksum: 4ebe82ce45aaf1527e96e2ab84d75d25217167ec3ff6378cf83a84fb4abc746e7c65768a79d275881602ae82f168f9a6dfaa7f5e331d0fcc83d692770bcce5f1
-  languageName: node
-  linkType: hard
-
 "public-encrypt@npm:^4.0.0":
   version: 4.0.3
   resolution: "public-encrypt@npm:4.0.3"
@@ -8326,13 +8315,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"toposort@npm:^2.0.2":
-  version: 2.0.2
-  resolution: "toposort@npm:2.0.2"
-  checksum: d64c74b570391c9432873f48e231b439ee56bc49f7cb9780b505cfdf5cb832f808d0bae072515d93834dd6bceca5bb34448b5b4b408335e4d4716eaf68195dcb
-  languageName: node
-  linkType: hard
-
 "tr46@npm:~0.0.3":
   version: 0.0.3
   resolution: "tr46@npm:0.0.3"
@@ -8888,21 +8870,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"yup@npm:^0.32.11":
-  version: 0.32.11
-  resolution: "yup@npm:0.32.11"
-  dependencies:
-    "@babel/runtime": ^7.15.4
-    "@types/lodash": ^4.14.175
-    lodash: ^4.17.21
-    lodash-es: ^4.17.21
-    nanoclone: ^0.2.1
-    property-expr: ^2.0.4
-    toposort: ^2.0.2
-  checksum: 43a16786b47cc910fed4891cebdd89df6d6e31702e9462e8f969c73eac88551ce750732608012201ea6b93802c8847cb0aa27b5d57370640f4ecf30f9f97d4b0
-  languageName: node
-  linkType: hard
-
 "zip-stream@npm:^4.1.0":
   version: 4.1.0
   resolution: "zip-stream@npm:4.1.0"
@@ -8933,6 +8900,7 @@ __metadata:
     "@prisma/client": ^4.1.0
     "@prisma/internals": ^4.1.0
     "@prisma/migrate": ^4.1.0
+    "@sapphire/shapeshift": ^3.7.0
     "@tanstack/react-query": ^4.2.3
     "@types/cookie": ^0.5.1
     "@types/minio": ^7.0.13
@@ -8972,6 +8940,5 @@ __metadata:
     ts-node: ^10.8.1
     tsx: ^3.8.0
     typescript: ^4.7.3
-    yup: ^0.32.11
   languageName: unknown
   linkType: soft