Infrastructure/zipline
1
Fork 0
Code Issues Pull requests Activity

fix(config): make s3/swift optional and not error out

Browse source
This commit is contained in:
dicedtomato 2022-07-09 20:35:21 +00:00 • committed by GitHub
parent 678ea20004
commit 964199f8a9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 42 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
src/lib/config/readConfig.ts
View file

@ -24,6 +24,14 @@ const envValues = [
e('DATASOURCE_S3_BUCKET', 'string', (c, v) => c.datasource.s3.bucket = v), e('DATASOURCE_S3_BUCKET', 'string', (c, v) => c.datasource.s3.bucket = v),
e('DATASOURCE_S3_REGION', 'string', (c, v) => c.datasource.s3.region = v ?? 'us-east-1'), e('DATASOURCE_S3_REGION', 'string', (c, v) => c.datasource.s3.region = v ?? 'us-east-1'),
e('DATASOURCE_SWIFT_CONTAINER', 'string', (c, v) => c.datasource.swift.container = v),
e('DATASOURCE_SWIFT_USERNAME', 'string', (c, v) => c.datasource.swift.username = v),
e('DATASOURCE_SWIFT_PASSWORD', 'string', (c, v) => c.datasource.swift.password = v),
e('DATASOURCE_SWIFT_AUTH_ENDPOINT', 'string', (c, v) => c.datasource.swift.auth_endpoint = v),
e('DATASOURCE_SWIFT_PROJECT_ID', 'string', (c, v) => c.datasource.swift.project_id = v),
e('DATASOURCE_SWIFT_DOMAIN_ID', 'string', (c, v) => c.datasource.swift.domain_id = v),
e('DATASOURCE_SWIFT_REGION_ID', 'string', (c, v) => c.datasource.swift.region_id = v),
e('UPLOADER_ROUTE', 'string', (c, v) => c.uploader.route = v), e('UPLOADER_ROUTE', 'string', (c, v) => c.uploader.route = v),
e('UPLOADER_LENGTH', 'number', (c, v) => c.uploader.length = v), e('UPLOADER_LENGTH', 'number', (c, v) => c.uploader.length = v),
e('UPLOADER_ADMIN_LIMIT', 'number', (c, v) => c.uploader.admin_limit = v), e('UPLOADER_ADMIN_LIMIT', 'number', (c, v) => c.uploader.admin_limit = v),
@ -76,6 +84,15 @@ function tryReadEnv(): Config {
force_s3_path: undefined, force_s3_path: undefined,
region: undefined, region: undefined,
}, },
swift: {
username: undefined,
password: undefined,
auth_endpoint: undefined,
container: undefined,
project_id: undefined,
domain_id: undefined,
region_id: undefined,
},
}, },
uploader: { uploader: {
route: undefined, route: undefined,
@ -108,6 +125,28 @@ function tryReadEnv(): Config {
envValues[i].fn(config, value); envValues[i].fn(config, value);
} }
} }
switch (config.datasource.type) {
case 's3':
config.datasource.swift = undefined;
break;
case 'swift':
config.datasource.s3 = undefined;
break;
case 'local':
config.datasource.s3 = undefined;
config.datasource.swift = undefined;
break;
default:
config.datasource.local = {
directory: null,
};
config.datasource.s3 = undefined;
config.datasource.swift = undefined;
break;
}
return config; return config;
} }

6
src/lib/config/validateConfig.ts
View file

@ -23,7 +23,7 @@ const validator = object({
bucket: string(), bucket: string(),
force_s3_path: boolean().default(false), force_s3_path: boolean().default(false),
region: string().default('us-east-1'), region: string().default('us-east-1'),
}).notRequired(), }).nullable().notRequired(),
swift: object({ swift: object({
username: string(), username: string(),
password: string(), password: string(),
@ -32,7 +32,7 @@ const validator = object({
project_id: string(), project_id: string(),
domain_id: string().default('default'), domain_id: string().default('default'),
region_id: string().nullable(), region_id: string().nullable(),
}), }).notRequired(),
}).required(), }).required(),
uploader: object({ uploader: object({
route: string().default('/u'), route: string().default('/u'),
@ -85,7 +85,7 @@ export default function validate(config): Config {
break; break;
} }
} }
return validated as unknown as Config; return validated as unknown as Config;
} catch (e) { } catch (e) {
if (process.env.ZIPLINE_DOCKER_BUILD) return null; if (process.env.ZIPLINE_DOCKER_BUILD) return null;