feat: clear storage (#244) (#239)

* - fix: use oauth's user id instead of username
 - feat: add login only config for oauth

* Addresses tomato's concerns

* fix: catch same account on different user

* Add storage cleaning

Co-authored-by: dicedtomato <diced@users.noreply.github.com>

* Update src/components/pages/Manage/index.tsx

Co-authored-by: dicedtomato <35403473+diced@users.noreply.github.com>

* Update src/components/pages/Manage/index.tsx

Co-authored-by: dicedtomato <35403473+diced@users.noreply.github.com>

Co-authored-by: dicedtomato <diced@users.noreply.github.com>
Co-authored-by: dicedtomato <35403473+diced@users.noreply.github.com>

.gitignore

@@ -41,5 +41,5 @@ yarn-error.log*
 
 # zipline
 config.toml
-uploads/
+uploads*/
 dist/

src/components/pages/Manage/index.tsx

@@ -309,6 +309,58 @@ export default function Manage({ oauth_registration, oauth_providers: raw_oauth_
     }
   };
 
+  const openClearData = () => {
+    modals.openConfirmModal({
+      title: 'Are you sure you want to clear all uploads in the database?',
+      closeOnConfirm: false,
+      labels: { confirm: 'Yes', cancel: 'No' },
+      onConfirm: () => {
+        modals.openConfirmModal({
+          title: 'Do you want to clear storage too?',
+          labels: { confirm: 'Yes', cancel: 'No' },
+          onConfirm: () => {
+            handleClearData(true);
+            modals.closeAll();
+          },
+          onCancel: () => {
+            handleClearData(false);
+            modals.closeAll();
+          },
+        });
+      },
+    });
+  };
+
+  const handleClearData = async (datasource?: boolean) => {
+    showNotification({
+      id: 'clear-uploads',
+      title: 'Clearing...',
+      message: '',
+      loading: true,
+      autoClose: false,
+    });
+
+    const res = await useFetch('/api/admin/clear', 'POST', { datasource });
+
+    if (res.error) {
+      updateNotification({
+        id: 'clear-uploads',
+        title: 'Error while clearing uploads',
+        message: res.error,
+        color: 'red',
+        icon: <CrossIcon />,
+      });
+    } else {
+      updateNotification({
+        id: 'clear-uploads',
+        title: 'Successfully cleared uploads',
+        message: '',
+        color: 'green',
+        icon: <CheckIcon />,
+      });
+    }
+  };
+
   const handleOauthUnlink = async (provider) => {
     const res = await useFetch('/api/auth/oauth', 'DELETE', {
       provider,
@@ -525,6 +577,9 @@ export default function Manage({ oauth_registration, oauth_providers: raw_oauth_
             <Button size='md' onClick={forceUpdateStats} color='red' rightIcon={<RefreshIcon />}>
               Force Update Stats
             </Button>
+            <Button size='md' onClick={openClearData} color='red' rightIcon={<TrashIcon />}>
+              Delete all uploads
+            </Button>
           </Group>
         </Box>
       )}

src/lib/datasources/Datasource.ts

@@ -5,6 +5,7 @@ export abstract class Datasource {
 
   public abstract save(file: string, data: Buffer): Promise<void>;
   public abstract delete(file: string): Promise<void>;
+  public abstract clear(): Promise<void>;
   public abstract size(file: string): Promise<number>;
   public abstract get(file: string): Readable | Promise<Readable>;
   public abstract fullSize(): Promise<number>;

src/lib/datasources/Local.ts

@@ -18,6 +18,13 @@ export class Local extends Datasource {
     await rm(join(process.cwd(), this.path, file));
   }
 
+  public async clear(): Promise<void> {
+    const files = await readdir(join(process.cwd(), this.path));
+
+    for (let i = 0; i !== files.length; ++i) {
+      await rm(join(process.cwd(), this.path, files[i]));
+    }
+  }
   public get(file: string): ReadStream {
     const full = join(process.cwd(), this.path, file);
     if (!existsSync(full)) return null;

src/lib/datasources/S3.ts

@@ -28,6 +28,18 @@ export class S3 extends Datasource {
     await this.s3.removeObject(this.config.bucket, file);
   }
 
+  public async clear(): Promise<void> {
+    const objects = this.s3.listObjectsV2(this.config.bucket, '', true);
+    const files = [];
+
+    objects.on('data', (item) => files.push(item.name));
+    objects.on('end', async () => {
+      this.s3.removeObjects(this.config.bucket, files, (err) => {
+        if (err) throw err;
+      });
+    });
+  }
+
   public get(file: string): Promise<Readable> {
     return new Promise((res, rej) => {
       this.s3.getObject(this.config.bucket, file, (err, stream) => {

src/lib/datasources/Supabase.ts

@@ -37,6 +37,41 @@ export class Supabase extends Datasource {
     });
   }
 
+  public async clear(): Promise<void> {
+    try {
+      const resp = await fetch(`${this.config.url}/storage/v1/object/list/${this.config.bucket}`, {
+        method: 'POST',
+        headers: {
+          Authorization: `Bearer ${this.config.key}`,
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          prefix: '',
+        }),
+      });
+      const objs = await resp.json();
+      if (objs.error) throw new Error(`${objs.error}: ${objs.message}`);
+
+      const res = await fetch(`${this.config.url}/storage/v1/object/${this.config.bucket}`, {
+        method: 'DELETE',
+        headers: {
+          Authorization: `Bearer ${this.config.key}`,
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          prefixes: objs.map((x: { name: string }) => x.name),
+        }),
+      });
+
+      const j = await res.json();
+      if (j.error) throw new Error(`${j.error}: ${j.message}`);
+
+      return;
+    } catch (e) {
+      this.logger.error(e);
+    }
+  }
+
   public async get(file: string): Promise<Readable> {
     // get a readable stream from the request
     const r = await fetch(`${this.config.url}/storage/v1/object/${this.config.bucket}/${file}`, {

src/pages/api/admin/clear.ts

@@ -0,0 +1,30 @@
+import datasource from 'lib/datasource';
+import Logger from 'lib/logger';
+import prisma from 'lib/prisma';
+import { NextApiReq, NextApiRes, UserExtended, withZipline } from 'middleware/withZipline';
+
+const logger = Logger.get('admin');
+
+async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
+  try {
+    const { count } = await prisma.image.deleteMany({});
+    logger.info(`User ${user.username} (${user.id}) cleared the database of ${count} images`);
+
+    if (req.body.datasource) {
+      await datasource.clear();
+      logger.info(`User ${user.username} (${user.id}) cleared storage`);
+    }
+  } catch (e) {
+    logger.error(`User ${user.username} (${user.id}) failed to clear the database or storage`);
+    logger.error(e);
+    return res.badRequest(`failed to clear the database or storage: ${e}`);
+  }
+
+  return res.json({ message: 'cleared storage' });
+}
+
+export default withZipline(handler, {
+  methods: ['POST'],
+  user: true,
+  administrator: true,
+});