Infrastructure/zipline
1
Fork 0
Code Issues Pull requests Activity

refactor: remove config file in favor for env variables

Browse source
This commit is contained in:
dicedtomato 2022-07-13 02:50:25 +00:00 committed by GitHub
parent 56ff86db44
commit 54158c5dbe
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 201 additions and 203 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
.env.local.example Normal file
View file

@ -0,0 +1,50 @@
# every field in here is optional except, CORE_SECRET and CORE_DATABASE_URL.
# if CORE_SECRET is still "changethis" then zipline will exit and tell you to change it.
# if using s3/swift make sure to comment out the other datasources
CORE_SECURE=true
CORE_SECRET="changethis"
CORE_HOST=0.0.0.0
CORE_PORT=3000
CORE_DATABASE_URL="postgres://postgres:postgres@localhost/zip10"
CORE_LOGGER=false
CORE_STATS_INTERVAL=1800
# default
DATASOURCE_TYPE=local
DATASOURCE_LOCAL_DIRECTORY=./uploads
# or you can choose to use s3
DATASOURCE_TYPE=s3
DATASOURCE_S3_ACCESS_KEY_ID=key
DATASOURCE_S3_SECRET_ACCESS_KEY=secret
DATASOURCE_S3_BUCKET=bucket
DATASOURCE_S3_ENDPOINT=s3.amazonaws.com
DATASOURCE_S3_REGION=us-west-2
DATASOURCE_S3_FORCE_S3_PATH=false
# or you can use swift
DATASOURCE_TYPE=swift
DATASOURCE_SWIFT_CONTAINER=container
DATASOURCE_SWIFT_AUTH_ENDPOINT="https://something/v3"
DATASOURCE_SWIFT_USERNAME=username
DATASOURCE_SWIFT_PASSWORD=password
DATASOURCE_SWIFT_PROJECT_ID=project_id
DATASOURCE_SWIFT_DOMAIN_ID=domain_id
UPLOADER_ROUTE=/u
UPLOADER_LENGTH=6
UPLOADER_ADMIN_LIMIT=104900000
UPLOADER_USER_LIMIT=104900000
UPLOADER_DISABLED_EXTENSIONS=someext
URLS_ROUTE=/go
URLS_LENGTH=6
RATELIMIT_USER = 5
RATELIMIT_ADMIN = 3
META_DESCRIPTION="zipline cool"
META_KEYWORDS="zipline,cool,image,uploads,sharing"
META_THEME_COLOR="#000000"

44
config.example.toml
View file

@ -1,44 +0,0 @@
[core]
secure = true # whether to return https or http in links
secret = 'changethis' # change this or zipline will not work
host = '0.0.0.0'
port = 3000
database_url = 'postgres://postgres:postgres@postgres/postgres'
[datasource]
type = 'local' # s3, local, or swift
[datasource.local]
directory = './uploads' # directory to store uploads in
[datasource.s3]
access_key_id = 'AKIAEXAMPLEKEY'
secret_access_key = 'somethingsomethingsomething'
bucket = 'zipline-storage'
endpoint = 's3.amazonaws.com'
region = 'us-west-2' # not required, defaults to us-east-1 if not specified
force_s3_path = false
[datasource.swift]
container = 'default'
auth_endpoint = 'http://127.0.0.1:49155/v3' # only supports v3 swift endpoints at the moment.
username = 'swift'
password = 'fingertips'
project_id = 'Default'
domain_id = 'Default'
[urls]
route = '/go'
length = 6
[uploader]
route = '/u'
embed_route = '/a'
length = 6
user_limit = 104900000 # 100mb
admin_limit = 104900000 # 100mb
disabled_extensions = ['jpg']
[ratelimit]
user = 5 # 5 seconds
admin = 0 # 0 seconds, disabled

3
package.json
View file

@ -31,6 +31,9 @@
"argon2": "^0.28.5",
"colorette": "^2.0.19",
"cookie": "^0.5.0",
"dot-prop": "^7.2.0",
"dotenv": "^16.0.1",
"dotenv-expand": "^8.0.3",
"fecha": "^4.2.3",
"fflate": "^0.7.3",
"find-my-way": "^6.3.0",

10
src/lib/config/Config.ts
View file

@ -1,6 +1,6 @@
export interface ConfigCore {
// Whether to return http or https links
secure: boolean;
https: boolean;
// Used for signing of cookies and other stuff
secret: string;
@ -105,10 +105,18 @@ export interface ConfigRatelimit {
admin: number;
}
// Metadata for the site
export interface ConfigMeta {
description: string;
theme_color: string;
keywords: string;
}
export interface Config {
core: ConfigCore;
uploader: ConfigUploader;
urls: ConfigUrls;
ratelimit: ConfigRatelimit;
datasource: ConfigDatasource;
meta: ConfigMeta;
}

241
src/lib/config/readConfig.ts
View file

@ -1,168 +1,117 @@
import { parse } from 'dotenv';
import { expand } from 'dotenv-expand';
import { existsSync, readFileSync } from 'fs';
import { join } from 'path';
import parse from '@iarna/toml/parse-string';
import Logger from '../logger';
import { Config } from './Config';
const e = (val, type, fn: (c: Config, v: any) => void) => ({ val, type, fn });
function isObject(value: any): value is Record<string, any> {
return typeof value === 'object' && value !== null;
}
const envValues = [
e('SECURE', 'boolean', (c, v) => c.core.secure = v),
e('SECRET', 'string', (c, v) => c.core.secret = v),
e('HOST', 'string', (c, v) => c.core.host = v),
e('PORT', 'number', (c, v) => c.core.port = v),
e('DATABASE_URL', 'string', (c, v) => c.core.database_url = v),
e('LOGGER', 'boolean', (c, v) => c.core.logger = v ?? true),
e('STATS_INTERVAL', 'number', (c, v) => c.core.stats_interval = v),
function set(object: Record<string, any>, property: string, value: any) {
const parts = property.split('.');
e('DATASOURCE_TYPE', 'string', (c, v) => c.datasource.type = v),
e('DATASOURCE_LOCAL_DIRECTORY', 'string', (c, v) => c.datasource.local.directory = v),
e('DATASOURCE_S3_ACCESS_KEY_ID', 'string', (c, v) => c.datasource.s3.access_key_id = v),
e('DATASOURCE_S3_SECRET_ACCESS_KEY', 'string', (c, v) => c.datasource.s3.secret_access_key = v),
e('DATASOURCE_S3_ENDPOINT', 'string', (c, v) => c.datasource.s3.endpoint = v ?? null),
e('DATASOURCE_S3_FORCE_S3_PATH', 'boolean', (c, v) => c.datasource.s3.force_s3_path = v ?? false),
e('DATASOURCE_S3_BUCKET', 'string', (c, v) => c.datasource.s3.bucket = v),
e('DATASOURCE_S3_REGION', 'string', (c, v) => c.datasource.s3.region = v ?? 'us-east-1'),
for (let i = 0; i < parts.length; ++i) {
const key = parts[i];
e('DATASOURCE_SWIFT_CONTAINER', 'string', (c, v) => c.datasource.swift.container = v),
e('DATASOURCE_SWIFT_USERNAME', 'string', (c, v) => c.datasource.swift.username = v),
e('DATASOURCE_SWIFT_PASSWORD', 'string', (c, v) => c.datasource.swift.password = v),
e('DATASOURCE_SWIFT_AUTH_ENDPOINT', 'string', (c, v) => c.datasource.swift.auth_endpoint = v),
e('DATASOURCE_SWIFT_PROJECT_ID', 'string', (c, v) => c.datasource.swift.project_id = v),
e('DATASOURCE_SWIFT_DOMAIN_ID', 'string', (c, v) => c.datasource.swift.domain_id = v),
e('DATASOURCE_SWIFT_REGION_ID', 'string', (c, v) => c.datasource.swift.region_id = v),
if (i === parts.length - 1) {
object[key] = value;
} else if (!isObject(object[key])) {
object[key] = typeof parts[i + 1] === 'number' ? [] : {};
}
e('UPLOADER_ROUTE', 'string', (c, v) => c.uploader.route = v),
e('UPLOADER_LENGTH', 'number', (c, v) => c.uploader.length = v),
e('UPLOADER_ADMIN_LIMIT', 'number', (c, v) => c.uploader.admin_limit = v),
e('UPLOADER_USER_LIMIT', 'number', (c, v) => c.uploader.user_limit = v),
e('UPLOADER_DISABLED_EXTS', 'array', (c, v) => v ? c.uploader.disabled_extensions = v : c.uploader.disabled_extensions = []),
object = object[key];
}
e('URLS_ROUTE', 'string', (c, v) => c.urls.route = v),
e('URLS_LENGTH', 'number', (c, v) => c.urls.length = v),
return object;
}
e('RATELIMIT_USER', 'number', (c, v) => c.ratelimit.user = v ?? 0),
e('RATELIMIT_ADMIN', 'number', (c, v) => c.ratelimit.user = v ?? 0),
function map(env: string, type: 'string' | 'number' | 'boolean' | 'array', path: string) {
return {
env,
type,
path,
};
}
export default function readConfig() {
if (existsSync('.env.local')) {
const contents = readFileSync('.env.local');
expand({
parsed: parse(contents),
});
}
const maps = [
map('CORE_HTTPS', 'boolean', 'core.secure'),
map('CORE_SECRET', 'string', 'core.secret'),
map('CORE_HOST', 'string', 'core.host'),
map('CORE_PORT', 'number', 'core.port'),
map('CORE_DATABASE_URL', 'string', 'core.database_url'),
map('CORE_LOGGER', 'boolean', 'core.logger'),
map('CORE_STATS_INTERVAL', 'number', 'core.stats_interval'),
map('DATASOURCE_TYPE', 'string', 'datasource.type'),
map('DATASOURCE_LOCAL_DIRECTORY', 'string', 'datasource.local.directory'),
map('DATASOURCE_S3_ACCESS_KEY_ID', 'string', 'datasource.s3.access_key_id'),
map('DATASOURCE_S3_SECRET_ACCESS_KEY', 'string', 'datasource.s3.secret_access_key'),
map('DATASOURCE_S3_ENDPOINT', 'string', 'datasource.s3.endpoint'),
map('DATASOURCE_S3_BUCKET', 'string', 'datasource.s3.bucket'),
map('DATASOURCE_S3_FORCE_S3_PATH', 'boolean', 'datasource.s3.force_s3_path'),
map('DATASOURCE_S3_REGION', 'string', 'datasource.s3.region'),
map('DATASOURCE_SWIFT_USERNAME', 'string', 'datasource.swift.username'),
map('DATASOURCE_SWIFT_PASSWORD', 'string', 'datasource.swift.password'),
map('DATASOURCE_SWIFT_AUTH_ENDPOINT', 'string', 'datasource.swift.auth_endpoint'),
map('DATASOURCE_SWIFT_CONTAINER', 'string', 'datasource.swift.container'),
map('DATASOURCE_SWIFT_PROJECT_ID', 'string', 'datasource.swift.project_id'),
map('DATASOURCE_SWIFT_DOMAIN_ID', 'string', 'datasource.swift.domain_id'),
map('DATASOURCE_SWIFT_REGION_ID', 'string', 'datasource.swift.region_id'),
map('UPLOADER_ROUTE', 'string', 'uploader.route'),
map('UPLOADER_LENGTH', 'number', 'uploader.length'),
map('UPLOADER_ADMIN_LIMIT', 'number', 'uploader.admin_limit'),
map('UPLOADER_USER_LIMIT', 'number', 'uploader.user_limit'),
map('UPLOADER_DISABLED_EXTENSIONS', 'array', 'uploader.disabled_extensions'),
map('URLS_ROUTE', 'string', 'urls.route'),
map('URLS_LENGTH', 'number', 'urls.length'),
map('RATELIMIT_USER', 'number', 'ratelimit.user'),
map('RATELIMIT_ADMIN', 'number', 'ratelimit.admin'),
map('META_DESCRITPION', 'string', 'meta.description'),
map('META_KEYWORDS', 'string', 'meta.keywords'),
map('META_THEME_COLOR', 'string', 'meta.theme_color'),
];
export default function readConfig(): Config {
if (!existsSync(join(process.cwd(), 'config.toml'))) {
if (!process.env.ZIPLINE_DOCKER_BUILD) Logger.get('config').info('reading environment');
return tryReadEnv();
} else {
if (process.env.ZIPLINE_DOCKER_BUILD) return;
const config = {};
Logger.get('config').info('reading config file');
const str = readFileSync(join(process.cwd(), 'config.toml'), 'utf8');
const parsed = parse(str);
for (let i = 0; i !== maps.length; ++i) {
const map = maps[i];
return parsed;
}
};
const value = process.env[map.env];
function tryReadEnv(): Config {
const config = {
core: {
secure: undefined,
secret: undefined,
host: undefined,
port: undefined,
database_url: undefined,
logger: undefined,
stats_interval: undefined,
},
datasource: {
type: undefined,
local: {
directory: undefined,
},
s3: {
access_key_id: undefined,
secret_access_key: undefined,
endpoint: undefined,
bucket: undefined,
force_s3_path: undefined,
region: undefined,
},
swift: {
username: undefined,
password: undefined,
auth_endpoint: undefined,
container: undefined,
project_id: undefined,
domain_id: undefined,
region_id: undefined,
},
},
uploader: {
route: undefined,
length: undefined,
admin_limit: undefined,
user_limit: undefined,
disabled_extensions: undefined,
},
urls: {
route: undefined,
length: undefined,
},
ratelimit: {
user: undefined,
admin: undefined,
},
};
for (let i = 0, L = envValues.length; i !== L; ++i) {
const envValue = envValues[i];
let value: any = process.env[envValue.val];
if (!value) {
envValues[i].fn(config, undefined);
} else {
envValues[i].fn(config, value);
if (envValue.type === 'number') value = parseToNumber(value);
else if (envValue.type === 'boolean') value = parseToBoolean(value);
else if (envValue.type === 'array') value = parseToArray(value);
envValues[i].fn(config, value);
}
}
switch (config.datasource.type) {
case 's3':
config.datasource.swift = undefined;
if (value) {
let parsed: any;
switch (map.type) {
case 'array':
parsed = value.split(',');
break;
case 'swift':
config.datasource.s3 = undefined;
case 'number':
parsed = Number(value);
break;
case 'local':
config.datasource.s3 = undefined;
config.datasource.swift = undefined;
case 'boolean':
parsed = value === 'true';
break;
default:
config.datasource.local = {
directory: null,
parsed = value;
};
config.datasource.s3 = undefined;
config.datasource.swift = undefined;
break;
set(config, map.path, parsed);
}
}
return config;
}
function parseToNumber(value) {
// infer that it is a string since env values are only strings
const number = Number(value);
if (isNaN(number)) return undefined;
return number;
}
function parseToBoolean(value) {
// infer that it is a string since env values are only strings
if (!value || value === 'false') return false;
else return true;
}
function parseToArray(value) {
return value.split(',');
}

9
src/lib/config/validateConfig.ts
View file

@ -3,7 +3,7 @@ import { object, bool, string, number, boolean, array } from 'yup';
const validator = object({
core: object({
secure: bool().default(false),
https: bool().default(false),
secret: string().min(8).required(),
host: string().default('0.0.0.0'),
port: number().default(3000),
@ -32,7 +32,7 @@ const validator = object({
project_id: string(),
domain_id: string().default('default'),
region_id: string().nullable(),
}).notRequired(),
}).nullable().notRequired(),
}).required(),
uploader: object({
route: string().default('/u'),
@ -50,6 +50,11 @@ const validator = object({
user: number().default(0),
admin: number().default(0),
}),
meta: object({
description: string().nullable().notRequired(),
theme_color: string().nullable().notRequired(),
keywords: string().nullable().notRequired(),
}).notRequired(),
});
export default function validate(config): Config {

5
src/server/index.ts
View file

@ -4,7 +4,7 @@ import { NextServer, RequestHandler } from 'next/dist/server/next';
import { Image, PrismaClient } from '@prisma/client';
import { createServer, IncomingMessage, OutgoingMessage, ServerResponse } from 'http';
import { extname } from 'path';
import { mkdir } from 'fs/promises';
import { mkdir, readFile } from 'fs/promises';
import { getStats, log, migrations } from './util';
import Logger from '../lib/logger';
import mimes from '../lib/mimes';
@ -34,7 +34,8 @@ async function start() {
logger.error('Running Zipline as is, without a randomized secret is not recommended and leaves your instance at risk!');
logger.error('Please change your secret in the config file or environment variables.');
logger.error('The config file is located at `config.toml`, or if using docker-compose you can change the variables in the `docker-compose.yml` file.');
logger.error('It is recomended to use a secret that is alphanumeric and randomized. A way you can generate this is through a password manager you may have.');
logger.error('It is recomended to use a secret that is alphanumeric and randomized.');
logger.error('A way you can generate this is through a password manager you may have.');
process.exit(1);
};

28
yarn.lock
View file

@ -2874,7 +2874,23 @@ __metadata:
languageName: node
linkType: hard
"dotenv@npm:16.0.1":
"dot-prop@npm:^7.2.0":
version: 7.2.0
resolution: "dot-prop@npm:7.2.0"
dependencies:
type-fest: ^2.11.2
checksum: 08e4ff14f7305ffb5fda7e4c88f3cdbeb3cd97bd27efa4f47503869a2ee7f96938b4c21b0a2abf9c37d891a1bdb3994a09d219b0dcfd6130da4eaeb44c6f067e
languageName: node
linkType: hard
"dotenv-expand@npm:^8.0.3":
version: 8.0.3
resolution: "dotenv-expand@npm:8.0.3"
checksum: 128ce90ac825b543de3ece0154a51b056ab0dc36bb26d97a68cd0b8707327ecd3c182fb6ac63b26a0fcdfa85064419906a1065cb634f1f9dc08ad311375f1fc0
languageName: node
linkType: hard
"dotenv@npm:16.0.1, dotenv@npm:^16.0.1":
version: 16.0.1
resolution: "dotenv@npm:16.0.1"
checksum: f459ffce07b977b7f15d8cc4ee69cdff77d4dd8c5dc8c85d2d485ee84655352c2415f9dd09d42b5b5985ced3be186130871b34e2f3e2569ebc72fbc2e8096792
@ -7779,6 +7795,13 @@ __metadata:
languageName: node
linkType: hard
"type-fest@npm:^2.11.2":
version: 2.16.0
resolution: "type-fest@npm:2.16.0"
checksum: 897fc5f6833de5ade5c4841d034bdfb6aaa168f24f725354ad13320b2a463b9df03a7a664b836b4c3bc7d9f92b22a25c26fe24668a35caf3b7a9ea5fcb847b8d
languageName: node
linkType: hard
"type-is@npm:^1.6.4":
version: 1.6.18
resolution: "type-is@npm:1.6.18"
@ -8236,6 +8259,9 @@ __metadata:
babel-plugin-import: ^1.13.5
colorette: ^2.0.19
cookie: ^0.5.0
dot-prop: ^7.2.0
dotenv: ^16.0.1
dotenv-expand: ^8.0.3
esbuild: ^0.14.44
eslint: ^7.32.0
eslint-config-next: 12.1.6