feat(v3.4.5): exporting images and more stuff

next.config.js

@@ -8,4 +8,7 @@ module.exports = {
       },
     ];
   },
+  api: {
+    responseLimit: false,
+  },
 };

src/components/Layout.tsx

@@ -123,6 +123,7 @@ export default function Layout({ children, user }) {
   const openResetToken = () => modals.openConfirmModal({
     title: 'Reset Token',
     centered: true,
+    overlayBlur: 3,
     children: (
       <Text size='sm'>
         Once you reset your token, you will have to update any uploaders to use this new token.
@@ -155,6 +156,7 @@ export default function Layout({ children, user }) {
   const openCopyToken = () => modals.openConfirmModal({
     title: 'Copy Token',
     centered: true,
+    overlayBlur: 3,
     children: (
       <Text size='sm'>
         Make sure you don&apos;t share this token with anyone as they will be able to upload files on your behalf.

src/components/pages/Manage.tsx

@@ -1,13 +1,14 @@
-import React, { useState } from 'react';
+import React, { useEffect, useState } from 'react';
 
 import useFetch from 'hooks/useFetch';
 import Link from 'components/Link';
 import { useStoreDispatch, useStoreSelector } from 'lib/redux/store';
 import { updateUser } from 'lib/redux/reducers/user';
-import { randomId, useForm } from '@mantine/hooks';
-import { Tooltip, TextInput, Button, Text, Title, Group, ColorInput, MultiSelect, Space } from '@mantine/core';
-import { DownloadIcon, Cross1Icon } from '@modulz/radix-icons';
+import { randomId, useForm, useInterval } from '@mantine/hooks';
+import { Card, Tooltip, TextInput, Button, Text, Title, Group, ColorInput, MultiSelect, Space, Box, Table } from '@mantine/core';
+import { DownloadIcon, Cross1Icon, TrashIcon } from '@modulz/radix-icons';
 import { useNotifications } from '@mantine/notifications';
+import { useModals } from '@mantine/modals';
 
 function VarsTooltip({ children }) {
   return (
@@ -25,11 +26,44 @@ function VarsTooltip({ children }) {
   );
 }
 
+function ExportDataTooltip({ children }) {
+  return <Tooltip position='top' placement='center' color='' label='After clicking, if you have a lot of files the export can take a while to complete. A list of previous exports will be below to download.'>{children}</Tooltip>;
+}
+
+function ExportTable({ rows, columns }) {
+  return (
+    <Box sx={{ pt: 1 }} >
+      <Table highlightOnHover>
+        <thead>
+          <tr>
+            {columns.map(col => (
+              <th key={randomId()}>{col.name}</th>
+            ))}
+          </tr>
+        </thead>
+        <tbody>
+          {rows.map(row => (
+            <tr key={randomId()}>
+              {columns.map(col => (
+                <td key={randomId()}>
+                  {col.format ? col.format(row[col.id]) : row[col.id]}
+                </td>
+              ))}
+            </tr>
+          ))}
+        </tbody>
+      </Table>
+    </Box>
+  );
+}
+
 export default function Manage() {
   const user = useStoreSelector(state => state.user);
   const dispatch = useStoreDispatch();
   const notif = useNotifications();
+  const modals = useModals();
 
+  const [exports, setExports] = useState([]);
   const [domains, setDomains] = useState(user.domains ?? []);
 
   const genShareX = (withEmbed: boolean = false, withZws: boolean = false) => {
@@ -41,8 +75,8 @@ export default function Manage() {
       RequestURL: `${window.location.protocol + '//' + window.location.hostname + (window.location.port ? ':' + window.location.port : '')}/api/upload`,
       Headers: {
         Authorization: user?.token,
-        ...(withEmbed && {Embed: 'true'}),
-        ...(withZws && {ZWS: 'true'}),
+        ...(withEmbed && { Embed: 'true' }),
+        ...(withZws && { ZWS: 'true' }),
       },
       URL: '$json:files[0]$',
       Body: 'MultipartFormData',
@@ -127,6 +161,78 @@ export default function Manage() {
     }
   };
 
+  const exportData = async () => {
+    const res = await useFetch('/api/user/export', 'POST');
+    if (res.url) {
+      notif.showNotification({
+        title: 'Export started...',
+        loading: true,
+        message: 'If you have a lot of files, the export may take a while. The list of exports will be updated every 30s.',
+      });
+    }
+  };
+
+  const getExports = async () => {
+    const res = await useFetch('/api/user/export');
+
+    setExports(res.exports.map(s => ({
+      date: new Date(Number(s.split('_')[3].slice(0, -4))),
+      full: s,
+    })).sort((a, b) => a.date.getTime() - b.date.getTime()));
+  };
+
+  const handleDelete = async () => {
+    const res = await useFetch('/api/user/files', 'DELETE', {
+      all: true,
+    });
+
+    if (!res.count) {
+      notif.showNotification({
+        title: 'Couldn\'t delete files',
+        message: res.error,
+        color: 'red',
+        icon: <Cross1Icon />,
+      });
+    } else {
+      notif.showNotification({
+        title: 'Deleted files',
+        message: `${res.count} files deleted`,
+        color: 'green',
+        icon: <TrashIcon />,
+      });
+    }
+  };
+
+  const openDeleteModal = () => modals.openConfirmModal({
+    title: 'Are you sure you want to delete all of your images?',
+    closeOnConfirm: false,
+    centered: true,
+    overlayBlur: 3,
+    labels: { confirm: 'Yes', cancel: 'No' },
+    onConfirm: () => {
+      modals.openConfirmModal({
+        title: 'Are you really sure?',
+        centered: true,
+        overlayBlur: 3,
+        labels: { confirm: 'Yes', cancel: 'No' },
+        onConfirm: () => {
+          handleDelete();
+          modals.closeAll();
+        },
+        onCancel: () => {
+          modals.closeAll();
+        },
+      });
+    },
+  });
+
+
+  const interval = useInterval(() => getExports(), 30000);
+  useEffect(() => {
+    getExports();
+    interval.start();
+  }, []);
+
   return (
     <>
       <Title>Manage User</Title>
@@ -159,6 +265,24 @@ export default function Manage() {
         </Group>
       </form>
 
+      <Title sx={{ paddingTop: 12 }}>Manage Data</Title>
+      <Text color='gray' sx={{ paddingBottom: 12 }}>Delete, or export your data into a zip file.</Text>
+      <Group>
+        <Button onClick={openDeleteModal} rightIcon={<TrashIcon />}>Delete All Data</Button>
+        <ExportDataTooltip><Button onClick={exportData} rightIcon={<DownloadIcon />}>Export Data</Button></ExportDataTooltip>
+      </Group>
+      <Card mt={22}>
+        <ExportTable
+          columns={[
+            { id: 'name', name: 'Name' },
+            { id: 'date', name: 'Date' },
+          ]}
+          rows={exports ? exports.map((x, i) => ({
+            name: <Link href={'/api/user/export?name=' + x.full}>Export {i + 1}</Link>,
+            date: x.date.toLocaleString(),
+          })) : []} />
+      </Card>
+
       <Title sx={{ paddingTop: 12, paddingBottom: 12 }}>ShareX Config</Title>
       <Group>
         <Button onClick={() => genShareX(false)} rightIcon={<DownloadIcon />}>ShareX Config</Button>
@@ -167,4 +291,4 @@ export default function Manage() {
       </Group>
     </>
   );
-}
+}

src/components/pages/Users.tsx

@@ -109,11 +109,14 @@ export default function Users() {
     title: `Delete ${user.username}?`,
     closeOnConfirm: false,
     centered: true,
+    overlayBlur: 3,
     labels: { confirm: 'Yes', cancel: 'No' },
     onConfirm: () => {
       modals.openConfirmModal({
         title: `Delete ${user.username}'s images?`,
         labels: { confirm: 'Yes', cancel: 'No' },
+        centered: true,
+        overlayBlur: 3,
         onConfirm: () => {
           handleDelete(user, true);
           modals.closeAll();

src/lib/datasource/Local.ts

@@ -1,4 +1,4 @@
-import { createReadStream, ReadStream } from 'fs';
+import { createReadStream, existsSync, ReadStream } from 'fs';
 import { readdir, rm, stat, writeFile } from 'fs/promises';
 import { join } from 'path';
 import { Datasource } from './datasource';
@@ -19,8 +19,11 @@ export class Local extends Datasource {
   }
 
   public get(file: string): ReadStream {
+    const full = join(process.cwd(), this.path, file);
+    if (!existsSync(full)) return null;
+
     try {
-      return createReadStream(join(process.cwd(), this.path, file));
+      return createReadStream(full);
     } catch (e) {
       return null;
     }

src/pages/api/user/export.ts

@@ -0,0 +1,134 @@
+import { NextApiReq, NextApiRes, withZipline } from 'middleware/withZipline';
+import prisma from 'lib/prisma';
+import Logger from 'lib/logger';
+import { Zip, ZipPassThrough } from 'fflate';
+import datasource from 'lib/ds';
+import { readdir } from 'fs/promises';
+import { createReadStream, createWriteStream } from 'fs';
+
+async function handler(req: NextApiReq, res: NextApiRes) {
+  const user = await req.user();
+  if (!user) return res.forbid('not logged in');
+
+  if (req.method === 'POST') {
+    const files = await prisma.image.findMany({
+      where: {
+        userId: user.id,
+      },
+    });
+
+    const zip = new Zip();
+    const export_name = `zipline_export_${user.id}_${Date.now()}.zip`;
+    const write_stream = createWriteStream(`/tmp/${export_name}`);
+
+    const onBackpressure = (stream, outputStream, cb) => {
+      const runCb = () => {
+        // Pause if either output or internal backpressure should be applied
+        cb(applyOutputBackpressure || backpressureBytes > backpressureThreshold);
+      };
+
+      // Internal backpressure (for when AsyncZipDeflate is slow)
+      const backpressureThreshold = 65536;
+      let backpressure = [];
+      let backpressureBytes = 0;
+      const push = stream.push;
+      stream.push = (dat, final) => {
+        backpressure.push(dat.length);
+        backpressureBytes += dat.length;
+        runCb();
+        push.call(stream, dat, final);
+      };
+      let ondata = stream.ondata;
+      const ondataPatched = (err, dat, final) => {
+        ondata.call(stream, err, dat, final);
+        backpressureBytes -= backpressure.shift();
+        runCb();
+      };
+      if (ondata) {
+        stream.ondata = ondataPatched;
+      } else {
+        // You can remove this condition if you make sure to
+        // call zip.add(file) before calling onBackpressure
+        Object.defineProperty(stream, 'ondata', {
+          get: () => ondataPatched,
+          set: cb => ondata = cb,
+        });
+      }
+
+      // Output backpressure (for when outputStream is slow)
+      let applyOutputBackpressure = false;
+      const write = outputStream.write;
+      outputStream.write = (data) => {
+        const outputNotFull = write.call(outputStream, data);
+        applyOutputBackpressure = !outputNotFull;
+        runCb();
+      };
+      outputStream.on('drain', () => {
+        applyOutputBackpressure = false;
+        runCb();
+      });
+    };
+
+
+
+    zip.ondata = async (err, data, final) => {
+      if (!err) {
+        write_stream.write(data);
+        if (final) {
+          write_stream.close();
+          Logger.get('user').info(`Export for ${user.username} (${user.id}) has completed and is available at ${export_name}`);
+        }
+      } else {
+        write_stream.close();
+
+        Logger.get('user').error(`Export for ${user.username} (${user.id}) has failed\n${err}`);
+      }
+    };
+
+    // for (const file of files) {
+    Logger.get('user').info(`Export for ${user.username} (${user.id}) has started`);
+    for (let i = 0; i !== files.length; ++i) {
+      const file = files[i];
+      const stream = datasource.get(file.file);
+      if (stream) {
+        const def = new ZipPassThrough(file.file);
+        zip.add(def);
+        onBackpressure(def, stream, shouldApplyBackpressure => {
+          if (shouldApplyBackpressure) {
+            stream.pause();
+          } else if (stream.isPaused()) {
+            stream.resume();
+          }
+        });
+        stream.on('data', c => def.push(c));
+        stream.on('end', () => def.push(new Uint8Array(0), true));
+      }
+    }
+
+    zip.end();
+
+    res.json({
+      url: '/api/user/export?name=' + export_name,
+    });
+  } else {
+    const export_name = req.query.name as string;
+    if (export_name) {
+      const parts = export_name.split('_');
+      if (Number(parts[2]) !== user.id) return res.forbid('cannot access export');
+
+      const stream = createReadStream(`/tmp/${export_name}`);
+
+      res.setHeader('Content-Type', 'application/zip');
+      res.setHeader('Content-Disposition', `attachment; filename="${export_name}"`);
+      stream.pipe(res);
+    } else {
+      const files = await readdir('/tmp');
+      const exports = files.filter(f => f.startsWith('zipline_export_'));
+      res.json({
+        exports,
+      });
+    }
+  }
+}
+
+export default withZipline(handler);

src/pages/api/user/files.ts

@@ -9,20 +9,41 @@ async function handler(req: NextApiReq, res: NextApiRes) {
   if (!user) return res.forbid('not logged in');
 
   if (req.method === 'DELETE') {
-    if (!req.body.id) return res.error('no file id');
+    if (req.body.all) {
+      const files = await prisma.image.findMany({
+        where: {
+          userId: user.id,
+        },
+      });
 
-    const image = await prisma.image.delete({
-      where: {
-        id: req.body.id,
-      },
-    });
+      for (let i = 0; i !== files.length; ++i) {
+        await datasource.delete(files[i].file);
+      }
 
-    await datasource.delete(image.file);
+      const { count } = await prisma.image.deleteMany({
+        where: {
+          userId: user.id,
+        },
+      });
+      Logger.get('image').info(`User ${user.username} (${user.id}) deleted ${count} images.`);
 
-    Logger.get('image').info(`User ${user.username} (${user.id}) deleted an image ${image.file} (${image.id})`);
+      return res.json({ count });
+    } else {
+      if (!req.body.id) return res.error('no file id');
 
-    delete image.password;
-    return res.json(image);
+      const image = await prisma.image.delete({
+        where: {
+          id: req.body.id,
+        },
+      });
+
+      await datasource.delete(image.file);
+
+      Logger.get('image').info(`User ${user.username} (${user.id}) deleted an image ${image.file} (${image.id})`);
+
+      delete image.password;
+      return res.json(image);
+    }
   } else if (req.method === 'PATCH') {
     if (!req.body.id) return res.error('no file id');