From 6349503b00e9257154e03aba77d5ec0e43b33a54 Mon Sep 17 00:00:00 2001 From: TacticalCoderJay Date: Sat, 10 Dec 2022 15:00:39 -0800 Subject: [PATCH] feat: clear storage (#244) (#239) * - fix: use oauth's user id instead of username - feat: add login only config for oauth * Addresses tomato's concerns * fix: catch same account on different user * Add storage cleaning Co-authored-by: dicedtomato * Update src/components/pages/Manage/index.tsx Co-authored-by: dicedtomato <35403473+diced@users.noreply.github.com> * Update src/components/pages/Manage/index.tsx Co-authored-by: dicedtomato <35403473+diced@users.noreply.github.com> Co-authored-by: dicedtomato Co-authored-by: dicedtomato <35403473+diced@users.noreply.github.com> --- .gitignore | 2 +- src/components/pages/Manage/index.tsx | 55 +++++++++++++++++++++++++++ src/lib/datasources/Datasource.ts | 1 + src/lib/datasources/Local.ts | 7 ++++ src/lib/datasources/S3.ts | 12 ++++++ src/lib/datasources/Supabase.ts | 35 +++++++++++++++++ src/pages/api/admin/clear.ts | 30 +++++++++++++++ 7 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 src/pages/api/admin/clear.ts diff --git a/.gitignore b/.gitignore index 21e258e..2b54b9f 100644 --- a/.gitignore +++ b/.gitignore @@ -41,5 +41,5 @@ yarn-error.log* # zipline config.toml -uploads/ +uploads*/ dist/ \ No newline at end of file diff --git a/src/components/pages/Manage/index.tsx b/src/components/pages/Manage/index.tsx index 157572d..4eab265 100644 --- a/src/components/pages/Manage/index.tsx +++ b/src/components/pages/Manage/index.tsx @@ -309,6 +309,58 @@ export default function Manage({ oauth_registration, oauth_providers: raw_oauth_ } }; + const openClearData = () => { + modals.openConfirmModal({ + title: 'Are you sure you want to clear all uploads in the database?', + closeOnConfirm: false, + labels: { confirm: 'Yes', cancel: 'No' }, + onConfirm: () => { + modals.openConfirmModal({ + title: 'Do you want to clear storage too?', + labels: { confirm: 'Yes', cancel: 'No' }, + onConfirm: () => { + handleClearData(true); + modals.closeAll(); + }, + onCancel: () => { + handleClearData(false); + modals.closeAll(); + }, + }); + }, + }); + }; + + const handleClearData = async (datasource?: boolean) => { + showNotification({ + id: 'clear-uploads', + title: 'Clearing...', + message: '', + loading: true, + autoClose: false, + }); + + const res = await useFetch('/api/admin/clear', 'POST', { datasource }); + + if (res.error) { + updateNotification({ + id: 'clear-uploads', + title: 'Error while clearing uploads', + message: res.error, + color: 'red', + icon: , + }); + } else { + updateNotification({ + id: 'clear-uploads', + title: 'Successfully cleared uploads', + message: '', + color: 'green', + icon: , + }); + } + }; + const handleOauthUnlink = async (provider) => { const res = await useFetch('/api/auth/oauth', 'DELETE', { provider, @@ -525,6 +577,9 @@ export default function Manage({ oauth_registration, oauth_providers: raw_oauth_ + )} diff --git a/src/lib/datasources/Datasource.ts b/src/lib/datasources/Datasource.ts index 28b22bd..34a6f94 100644 --- a/src/lib/datasources/Datasource.ts +++ b/src/lib/datasources/Datasource.ts @@ -5,6 +5,7 @@ export abstract class Datasource { public abstract save(file: string, data: Buffer): Promise; public abstract delete(file: string): Promise; + public abstract clear(): Promise; public abstract size(file: string): Promise; public abstract get(file: string): Readable | Promise; public abstract fullSize(): Promise; diff --git a/src/lib/datasources/Local.ts b/src/lib/datasources/Local.ts index ed2678a..e69578b 100644 --- a/src/lib/datasources/Local.ts +++ b/src/lib/datasources/Local.ts @@ -18,6 +18,13 @@ export class Local extends Datasource { await rm(join(process.cwd(), this.path, file)); } + public async clear(): Promise { + const files = await readdir(join(process.cwd(), this.path)); + + for (let i = 0; i !== files.length; ++i) { + await rm(join(process.cwd(), this.path, files[i])); + } + } public get(file: string): ReadStream { const full = join(process.cwd(), this.path, file); if (!existsSync(full)) return null; diff --git a/src/lib/datasources/S3.ts b/src/lib/datasources/S3.ts index 4fdb7fb..9e0797d 100644 --- a/src/lib/datasources/S3.ts +++ b/src/lib/datasources/S3.ts @@ -28,6 +28,18 @@ export class S3 extends Datasource { await this.s3.removeObject(this.config.bucket, file); } + public async clear(): Promise { + const objects = this.s3.listObjectsV2(this.config.bucket, '', true); + const files = []; + + objects.on('data', (item) => files.push(item.name)); + objects.on('end', async () => { + this.s3.removeObjects(this.config.bucket, files, (err) => { + if (err) throw err; + }); + }); + } + public get(file: string): Promise { return new Promise((res, rej) => { this.s3.getObject(this.config.bucket, file, (err, stream) => { diff --git a/src/lib/datasources/Supabase.ts b/src/lib/datasources/Supabase.ts index 9851e60..5c5e517 100644 --- a/src/lib/datasources/Supabase.ts +++ b/src/lib/datasources/Supabase.ts @@ -37,6 +37,41 @@ export class Supabase extends Datasource { }); } + public async clear(): Promise { + try { + const resp = await fetch(`${this.config.url}/storage/v1/object/list/${this.config.bucket}`, { + method: 'POST', + headers: { + Authorization: `Bearer ${this.config.key}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + prefix: '', + }), + }); + const objs = await resp.json(); + if (objs.error) throw new Error(`${objs.error}: ${objs.message}`); + + const res = await fetch(`${this.config.url}/storage/v1/object/${this.config.bucket}`, { + method: 'DELETE', + headers: { + Authorization: `Bearer ${this.config.key}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + prefixes: objs.map((x: { name: string }) => x.name), + }), + }); + + const j = await res.json(); + if (j.error) throw new Error(`${j.error}: ${j.message}`); + + return; + } catch (e) { + this.logger.error(e); + } + } + public async get(file: string): Promise { // get a readable stream from the request const r = await fetch(`${this.config.url}/storage/v1/object/${this.config.bucket}/${file}`, { diff --git a/src/pages/api/admin/clear.ts b/src/pages/api/admin/clear.ts new file mode 100644 index 0000000..ce46c45 --- /dev/null +++ b/src/pages/api/admin/clear.ts @@ -0,0 +1,30 @@ +import datasource from 'lib/datasource'; +import Logger from 'lib/logger'; +import prisma from 'lib/prisma'; +import { NextApiReq, NextApiRes, UserExtended, withZipline } from 'middleware/withZipline'; + +const logger = Logger.get('admin'); + +async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) { + try { + const { count } = await prisma.image.deleteMany({}); + logger.info(`User ${user.username} (${user.id}) cleared the database of ${count} images`); + + if (req.body.datasource) { + await datasource.clear(); + logger.info(`User ${user.username} (${user.id}) cleared storage`); + } + } catch (e) { + logger.error(`User ${user.username} (${user.id}) failed to clear the database or storage`); + logger.error(e); + return res.badRequest(`failed to clear the database or storage: ${e}`); + } + + return res.json({ message: 'cleared storage' }); +} + +export default withZipline(handler, { + methods: ['POST'], + user: true, + administrator: true, +});