feat: clear storage (#244) (#239)

* - fix: use oauth's user id instead of username
 - feat: add login only config for oauth

* Addresses tomato's concerns

* fix: catch same account on different user

* Add storage cleaning

Co-authored-by: dicedtomato <diced@users.noreply.github.com>

* Update src/components/pages/Manage/index.tsx

Co-authored-by: dicedtomato <35403473+diced@users.noreply.github.com>

* Update src/components/pages/Manage/index.tsx

Co-authored-by: dicedtomato <35403473+diced@users.noreply.github.com>

Co-authored-by: dicedtomato <diced@users.noreply.github.com>
Co-authored-by: dicedtomato <35403473+diced@users.noreply.github.com>
This commit is contained in:
TacticalCoderJay 2022-12-10 15:00:39 -08:00 committed by GitHub
parent 58e8c103b7
commit 6349503b00
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 141 additions and 1 deletions

2
.gitignore vendored
View file

@ -41,5 +41,5 @@ yarn-error.log*
# zipline
config.toml
uploads/
uploads*/
dist/

View file

@ -309,6 +309,58 @@ export default function Manage({ oauth_registration, oauth_providers: raw_oauth_
}
};
const openClearData = () => {
modals.openConfirmModal({
title: 'Are you sure you want to clear all uploads in the database?',
closeOnConfirm: false,
labels: { confirm: 'Yes', cancel: 'No' },
onConfirm: () => {
modals.openConfirmModal({
title: 'Do you want to clear storage too?',
labels: { confirm: 'Yes', cancel: 'No' },
onConfirm: () => {
handleClearData(true);
modals.closeAll();
},
onCancel: () => {
handleClearData(false);
modals.closeAll();
},
});
},
});
};
const handleClearData = async (datasource?: boolean) => {
showNotification({
id: 'clear-uploads',
title: 'Clearing...',
message: '',
loading: true,
autoClose: false,
});
const res = await useFetch('/api/admin/clear', 'POST', { datasource });
if (res.error) {
updateNotification({
id: 'clear-uploads',
title: 'Error while clearing uploads',
message: res.error,
color: 'red',
icon: <CrossIcon />,
});
} else {
updateNotification({
id: 'clear-uploads',
title: 'Successfully cleared uploads',
message: '',
color: 'green',
icon: <CheckIcon />,
});
}
};
const handleOauthUnlink = async (provider) => {
const res = await useFetch('/api/auth/oauth', 'DELETE', {
provider,
@ -525,6 +577,9 @@ export default function Manage({ oauth_registration, oauth_providers: raw_oauth_
<Button size='md' onClick={forceUpdateStats} color='red' rightIcon={<RefreshIcon />}>
Force Update Stats
</Button>
<Button size='md' onClick={openClearData} color='red' rightIcon={<TrashIcon />}>
Delete all uploads
</Button>
</Group>
</Box>
)}

View file

@ -5,6 +5,7 @@ export abstract class Datasource {
public abstract save(file: string, data: Buffer): Promise<void>;
public abstract delete(file: string): Promise<void>;
public abstract clear(): Promise<void>;
public abstract size(file: string): Promise<number>;
public abstract get(file: string): Readable | Promise<Readable>;
public abstract fullSize(): Promise<number>;

View file

@ -18,6 +18,13 @@ export class Local extends Datasource {
await rm(join(process.cwd(), this.path, file));
}
public async clear(): Promise<void> {
const files = await readdir(join(process.cwd(), this.path));
for (let i = 0; i !== files.length; ++i) {
await rm(join(process.cwd(), this.path, files[i]));
}
}
public get(file: string): ReadStream {
const full = join(process.cwd(), this.path, file);
if (!existsSync(full)) return null;

View file

@ -28,6 +28,18 @@ export class S3 extends Datasource {
await this.s3.removeObject(this.config.bucket, file);
}
public async clear(): Promise<void> {
const objects = this.s3.listObjectsV2(this.config.bucket, '', true);
const files = [];
objects.on('data', (item) => files.push(item.name));
objects.on('end', async () => {
this.s3.removeObjects(this.config.bucket, files, (err) => {
if (err) throw err;
});
});
}
public get(file: string): Promise<Readable> {
return new Promise((res, rej) => {
this.s3.getObject(this.config.bucket, file, (err, stream) => {

View file

@ -37,6 +37,41 @@ export class Supabase extends Datasource {
});
}
public async clear(): Promise<void> {
try {
const resp = await fetch(`${this.config.url}/storage/v1/object/list/${this.config.bucket}`, {
method: 'POST',
headers: {
Authorization: `Bearer ${this.config.key}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
prefix: '',
}),
});
const objs = await resp.json();
if (objs.error) throw new Error(`${objs.error}: ${objs.message}`);
const res = await fetch(`${this.config.url}/storage/v1/object/${this.config.bucket}`, {
method: 'DELETE',
headers: {
Authorization: `Bearer ${this.config.key}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
prefixes: objs.map((x: { name: string }) => x.name),
}),
});
const j = await res.json();
if (j.error) throw new Error(`${j.error}: ${j.message}`);
return;
} catch (e) {
this.logger.error(e);
}
}
public async get(file: string): Promise<Readable> {
// get a readable stream from the request
const r = await fetch(`${this.config.url}/storage/v1/object/${this.config.bucket}/${file}`, {

View file

@ -0,0 +1,30 @@
import datasource from 'lib/datasource';
import Logger from 'lib/logger';
import prisma from 'lib/prisma';
import { NextApiReq, NextApiRes, UserExtended, withZipline } from 'middleware/withZipline';
const logger = Logger.get('admin');
async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
try {
const { count } = await prisma.image.deleteMany({});
logger.info(`User ${user.username} (${user.id}) cleared the database of ${count} images`);
if (req.body.datasource) {
await datasource.clear();
logger.info(`User ${user.username} (${user.id}) cleared storage`);
}
} catch (e) {
logger.error(`User ${user.username} (${user.id}) failed to clear the database or storage`);
logger.error(e);
return res.badRequest(`failed to clear the database or storage: ${e}`);
}
return res.json({ message: 'cleared storage' });
}
export default withZipline(handler, {
methods: ['POST'],
user: true,
administrator: true,
});