Merge branch 'trunk' into feature/oauth-authentik

.env.local.example

@@ -1,7 +1,7 @@
 # every field in here is optional except, CORE_SECRET and CORE_DATABASE_URL.
 # if CORE_SECRET is still "changethis" then zipline will exit and tell you to change it.
 
-# if using s3/supabase make sure to comment out the other datasources
+# if using s3/supabase make sure to uncomment or comment out the correct lines needed.
 
 CORE_RETURN_HTTPS=true
 CORE_SECRET="changethis"
@@ -10,34 +10,36 @@ CORE_PORT=3000
 CORE_DATABASE_URL="postgres://postgres:postgres@localhost/zip10"
 CORE_LOGGER=false
 CORE_STATS_INTERVAL=1800
+CORE_INVITES_INTERVAL=1800
+CORE_THUMBNAILS_INTERVAL=600
 
 # default
 DATASOURCE_TYPE=local
 DATASOURCE_LOCAL_DIRECTORY=./uploads
 
 # or you can choose to use s3
-DATASOURCE_TYPE=s3
+# DATASOURCE_TYPE=s3
-DATASOURCE_S3_ACCESS_KEY_ID=key
+# DATASOURCE_S3_ACCESS_KEY_ID=key
-DATASOURCE_S3_SECRET_ACCESS_KEY=secret
+# DATASOURCE_S3_SECRET_ACCESS_KEY=secret
-DATASOURCE_S3_BUCKET=bucket
+# DATASOURCE_S3_BUCKET=bucket
-DATASOURCE_S3_ENDPOINT=s3.amazonaws.com
+# DATASOURCE_S3_ENDPOINT=s3.amazonaws.com
-DATASOURCE_S3_REGION=us-west-2
+# DATASOURCE_S3_REGION=us-west-2
-DATASOURCE_S3_FORCE_S3_PATH=false
+# DATASOURCE_S3_FORCE_S3_PATH=false
-DATASOURCE_S3_USE_SSL=false
+# DATASOURCE_S3_USE_SSL=false
 
 # or supabase
-DATASOURCE_TYPE=supabase
+# DATASOURCE_TYPE=supabase
-DATASOURCE_SUPABASE_KEY=xxx
+# DATASOURCE_SUPABASE_KEY=xxx
 # remember: no leading slash
-DATASOURCE_SUPABASE_URL=https://something.supabase.co
+# DATASOURCE_SUPABASE_URL=https://something.supabase.co
-DATASOURCE_SUPABASE_BUCKET=zipline
+# DATASOURCE_SUPABASE_BUCKET=zipline
 
 UPLOADER_DEFAULT_FORMAT=RANDOM
 UPLOADER_ROUTE=/u
 UPLOADER_LENGTH=6
 UPLOADER_ADMIN_LIMIT=104900000
 UPLOADER_USER_LIMIT=104900000
-UPLOADER_DISABLED_EXTENSIONS=someext
+UPLOADER_DISABLED_EXTENSIONS=someext,anotherext
 
 URLS_ROUTE=/go
 URLS_LENGTH=6

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2022 dicedtomato
+Copyright (c) 2023 dicedtomato
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

package.json

@@ -54,6 +54,7 @@
     "fastify": "^4.15.0",
     "fastify-plugin": "^4.5.0",
     "fflate": "^0.7.4",
+    "ffmpeg-static": "^5.1.0",
     "find-my-way": "^7.6.0",
     "katex": "^0.16.4",
     "mantine-datatable": "^2.2.6",

prisma/migrations/20230523025656_thumbnails/migration.sql

@@ -0,0 +1,16 @@
+
+-- CreateTable
+CREATE TABLE "Thumbnail" (
+    "id" SERIAL NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "name" TEXT NOT NULL,
+    "fileId" INTEGER NOT NULL,
+
+    CONSTRAINT "Thumbnail_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "Thumbnail_fileId_key" ON "Thumbnail"("fileId");
+
+-- AddForeignKey
+ALTER TABLE "Thumbnail" ADD CONSTRAINT "Thumbnail_fileId_fkey" FOREIGN KEY ("fileId") REFERENCES "File"("id") ON DELETE CASCADE ON UPDATE CASCADE;

prisma/schema.prisma

@@ -62,6 +62,17 @@ model File {
 
   folder   Folder? @relation(fields: [folderId], references: [id], onDelete: SetNull)
   folderId Int?
+
+  thumbnail   Thumbnail?
+}
+
+model Thumbnail {
+  id        Int      @id @default(autoincrement())
+  createdAt DateTime @default(now())
+  name      String
+
+  fileId Int  @unique
+  file   File @relation(fields: [fileId], references: [id], onDelete: Cascade)
 }
 
 model InvisibleFile {

src/components/File/index.tsx

@@ -63,7 +63,19 @@ export default function File({
         otherUser={otherUser}
       />
 
-      <Card sx={{ maxWidth: '100%', height: '100%' }} shadow='md' onClick={() => setOpen(true)}>
+      <Card
+        sx={{
+          maxWidth: '100%',
+          height: '100%',
+          '&:hover': {
+            filter: 'brightness(0.75)',
+          },
+          transition: 'filter 0.2s ease-in-out',
+          cursor: 'pointer',
+        }}
+        shadow='md'
+        onClick={() => setOpen(true)}
+      >
         <Card.Section>
           <LoadingOverlay visible={loading} />
           <Type

src/components/Layout.tsx

@@ -351,7 +351,11 @@ export default function Layout({ children, props }) {
                 <Menu.Target>
                   <Button
                     leftIcon={
-                      avatar ? <Image src={avatar} height={32} radius='md' /> : <IconUserCog size='1rem' />
+                      avatar ? (
+                        <Image src={avatar} height={32} width={32} fit='cover' radius='md' />
+                      ) : (
+                        <IconUserCog size='1rem' />
+                      )
                     }
                     variant='subtle'
                     color='gray'

src/components/Type.tsx

@@ -53,6 +53,35 @@ function Placeholder({ text, Icon, ...props }) {
   );
 }
 
+function VideoThumbnailPlaceholder({ file, mediaPreview, ...props }) {
+  if (!file.thumbnail || !mediaPreview)
+    return <Placeholder Icon={IconPlayerPlay} text={`Click to view video (${file.name})`} {...props} />;
+
+  return (
+    <Box sx={{ position: 'relative' }}>
+      <Image
+        src={file.thumbnail}
+        sx={{
+          width: '100%',
+          height: 'auto',
+        }}
+      />
+
+      <Center
+        sx={{
+          position: 'absolute',
+          height: '100%',
+          top: '50%',
+          left: '50%',
+          transform: 'translate(-50%, -50%)',
+        }}
+      >
+        <IconPlayerPlay size={48} />
+      </Center>
+    </Box>
+  );
+}
+
 export default function Type({ file, popup = false, disableMediaPreview, ...props }) {
   const type =
     (file.type ?? file.mimetype) === ''
@@ -159,7 +188,8 @@ export default function Type({ file, popup = false, disableMediaPreview, ...prop
     )
   ) : media ? (
     {
-      video: <Placeholder Icon={IconPlayerPlay} text={`Click to view video (${file.name})`} {...props} />,
+      // video: <Placeholder Icon={IconPlayerPlay} text={`Click to view video (${file.name})`} {...props} />,
+      video: <VideoThumbnailPlaceholder file={file} mediaPreview={!disableMediaPreview} />,
       image: (
         <Image
           placeholder={<PlaceholderContent Icon={IconPhotoCancel} text={'Image failed to load...'} />}

src/components/pages/Files/index.tsx

@@ -7,12 +7,16 @@ import Link from 'next/link';
 import { useEffect, useState } from 'react';
 import FilePagation from './FilePagation';
 import PendingFilesModal from './PendingFilesModal';
+import { showNonMediaSelector } from 'lib/recoil/settings';
+import { useRecoilState } from 'recoil';
 
 export default function Files({ disableMediaPreview, exifEnabled, queryPage, compress }) {
+  const [checked] = useRecoilState(showNonMediaSelector);
+
   const [favoritePage, setFavoritePage] = useState(1);
   const [favoriteNumPages, setFavoriteNumPages] = useState(0);
   const favoritePages = usePaginatedFiles(favoritePage, {
-    filter: 'media',
+    filter: checked ? 'none' : 'media',
     favorite: true,
   });
 

src/components/pages/Invites.tsx

@@ -55,7 +55,7 @@ function CreateInviteModal({ open, setOpen, updateInvites }) {
     setOpen(false);
 
     const res = await useFetch('/api/auth/invite', 'POST', {
-      expiresAt: expiresAt === null ? null : `date=${expiresAt.toISOString()}`,
+      expiresAt: `date=${expiresAt.toISOString()}`,
       count: values.count,
     });
 
@@ -95,7 +95,6 @@ function CreateInviteModal({ open, setOpen, updateInvites }) {
             { value: '3d', label: '3 days' },
             { value: '5d', label: '5 days' },
             { value: '7d', label: '7 days' },
-            { value: 'never', label: 'Never' },
           ]}
         />
 
@@ -299,8 +298,16 @@ export default function Invites() {
         />
       ) : (
         <SimpleGrid cols={3} spacing='lg' breakpoints={[{ maxWidth: 'sm', cols: 1, spacing: 'sm' }]}>
-          {invites.length
+          {!ok && !invites.length && (
-            ? invites.map((invite) => (
+            <>
+              {[1, 2, 3].map((x) => (
+                <Skeleton key={x} width='100%' height={100} radius='sm' />
+              ))}
+            </>
+          )}
+
+          {invites.length && ok ? (
+            invites.map((invite) => (
               <Card key={invite.id} sx={{ maxWidth: '100%' }}>
                 <Group position='apart'>
                   <Group position='left'>
@@ -314,9 +321,7 @@ export default function Invites() {
                       </Title>
                       <Tooltip label={new Date(invite.createdAt).toLocaleString()}>
                         <div>
-                            <MutedText size='sm'>
+                          <MutedText size='sm'>Created {relativeTime(new Date(invite.createdAt))}</MutedText>
-                              Created {relativeTime(new Date(invite.createdAt))}
-                            </MutedText>
                         </div>
                       </Tooltip>
                       <Tooltip label={new Date(invite.expiresAt).toLocaleString()}>
@@ -337,7 +342,21 @@ export default function Invites() {
                 </Group>
               </Card>
             ))
-            : [1, 2, 3].map((x) => <Skeleton key={x} width='100%' height={100} radius='sm' />)}
+          ) : (
+            <>
+              <div></div>
+              <Group>
+                <div>
+                  <IconTag size={48} />
+                </div>
+                <div>
+                  <Title>Nothing here</Title>
+                  <MutedText size='md'>Create some invites and they will show up here</MutedText>
+                </div>
+              </Group>
+              <div></div>
+            </>
+          )}
         </SimpleGrid>
       )}
     </>

src/components/pages/Upload/File.tsx

@@ -122,6 +122,8 @@ export default function File({ chunks: chunks_config }) {
               });
 
               if (j === chunks.length - 1) {
+                window.removeEventListener('beforeunload', beforeUnload);
+                router.events.off('routeChangeStart', beforeRouteChange);
                 updateNotification({
                   id: 'upload-chunked',
                   title: 'Finalizing partial upload',

src/lib/config/Config.ts

@@ -10,6 +10,7 @@ export interface ConfigCore {
 
   stats_interval: number;
   invites_interval: number;
+  thumbnails_interval: number;
 }
 
 export interface ConfigCompression {

src/lib/config/readConfig.ts

@@ -63,8 +63,11 @@ export default function readConfig() {
     map('CORE_PORT', 'number', 'core.port'),
     map('CORE_DATABASE_URL', 'string', 'core.database_url'),
     map('CORE_LOGGER', 'boolean', 'core.logger'),
+
     map('CORE_STATS_INTERVAL', 'number', 'core.stats_interval'),
     map('CORE_INVITES_INTERVAL', 'number', 'core.invites_interval'),
+    map('CORE_THUMBNAILS_INTERVAL', 'number', 'core.thumbnails_interval'),
+
     map('CORE_COMPRESSION_ENABLED', 'boolean', 'core.compression.enabled'),
     map('CORE_COMPRESSION_THRESHOLD', 'human-to-byte', 'core.compression.threshold'),
     map('CORE_COMPRESSION_ON_DASHBOARD', 'boolean', 'core.compression.on_dashboard'),

src/lib/config/validateConfig.ts

@@ -35,8 +35,9 @@ const validator = s.object({
     port: s.number.default(3000),
     database_url: s.string,
     logger: s.boolean.default(false),
-    stats_interval: s.number.default(1800),
+    stats_interval: s.number.default(1800), // 30m
-    invites_interval: s.number.default(1800),
+    invites_interval: s.number.default(1800), // 30m
+    thumbnails_interval: s.number.default(600), // 10m
     compression: s
       .object({
         enabled: s.boolean.default(false),

src/lib/utils/exif.ts

@@ -1,10 +1,10 @@
 import { File } from '@prisma/client';
-import { createWriteStream } from 'fs';
 import { ExifTool, Tags } from 'exiftool-vendored';
+import { createWriteStream } from 'fs';
+import { readFile, rm } from 'fs/promises';
 import datasource from 'lib/datasource';
 import Logger from 'lib/logger';
 import { join } from 'path';
-import { readFile, unlink } from 'fs/promises';
 
 const logger = Logger.get('exif');
 
@@ -43,6 +43,7 @@ export async function removeGPSData(image: File): Promise<void> {
   await new Promise((resolve) => writeStream.on('finish', resolve));
 
   logger.debug(`removing GPS data from ${file}`);
+  try {
     await exiftool.write(file, {
       GPSVersionID: null,
       GPSAltitude: null,
@@ -77,13 +78,19 @@ export async function removeGPSData(image: File): Promise<void> {
       GPSTrack: null,
       GPSTrackRef: null,
     });
+  } catch (e) {
+    logger.debug(`removing temp file: ${file}`);
+    await rm(file);
+
+    return;
+  }
 
   logger.debug(`reading file to upload to datasource: ${file} -> ${image.name}`);
   const buffer = await readFile(file);
   await datasource.save(image.name, buffer);
 
   logger.debug(`removing temp file: ${file}`);
-  await unlink(file);
+  await rm(file);
 
   await exiftool.end(true);
 

src/pages/api/user/[id].ts

@@ -3,6 +3,8 @@ import Logger from 'lib/logger';
 import prisma from 'lib/prisma';
 import { hashPassword } from 'lib/util';
 import { jsonUserReplacer } from 'lib/utils/client';
+import { formatRootUrl } from 'lib/utils/urls';
+import zconfig from 'lib/config';
 import { NextApiReq, NextApiRes, UserExtended, withZipline } from 'middleware/withZipline';
 
 const logger = Logger.get('user');
@@ -15,7 +17,11 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
       id: Number(id),
     },
     include: {
-      files: true,
+      files: {
+        include: {
+          thumbnail: true,
+        },
+      },
       Folder: true,
     },
   });
@@ -179,9 +185,21 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
   } else {
     delete target.password;
 
-    if (user.superAdmin && target.superAdmin) delete target.files;
+    if (user.superAdmin && target.superAdmin) {
-    if (user.administrator && !user.superAdmin && (target.administrator || target.superAdmin))
       delete target.files;
+      return res.json(target);
+    }
+    if (user.administrator && !user.superAdmin && (target.administrator || target.superAdmin)) {
+      delete target.files;
+      return res.json(target);
+    }
+
+    for (const file of target.files) {
+      (file as unknown as { url: string }).url = formatRootUrl(zconfig.uploader.route, file.name);
+      if (file.thumbnail) {
+        (file.thumbnail as unknown as string) = formatRootUrl('/r', file.thumbnail.name);
+      }
+    }
 
     return res.json(target);
   }

src/pages/api/user/files.ts

@@ -14,10 +14,14 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
         where: {
           userId: user.id,
         },
+        include: {
+          thumbnail: true,
+        },
       });
 
       for (let i = 0; i !== files.length; ++i) {
         await datasource.delete(files[i].name);
+        if (files[i].thumbnail?.name) await datasource.delete(files[i].thumbnail.name);
       }
 
       const { count } = await prisma.file.deleteMany({
@@ -45,6 +49,7 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
               id: true,
             },
           },
+          thumbnail: true,
         },
       });
 
@@ -63,10 +68,12 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
               id: true,
             },
           },
+          thumbnail: true,
         },
       });
 
       await datasource.delete(file.name);
+      if (file.thumbnail?.name) await datasource.delete(file.thumbnail.name);
 
       logger.info(
         `User ${user.username} (${user.id}) deleted an image ${file.name} (${file.id}) owned by ${file.user.username} (${file.user.id})`
@@ -134,6 +141,7 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
       views: number;
       size: number;
       originalName: string;
+      thumbnail?: { name: string };
     }[] = await prisma.file.findMany({
       where: {
         userId: user.id,
@@ -154,11 +162,16 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
         maxViews: true,
         size: true,
         originalName: true,
+        thumbnail: true,
       },
     });
 
     for (let i = 0; i !== files.length; ++i) {
       (files[i] as unknown as { url: string }).url = formatRootUrl(config.uploader.route, files[i].name);
+
+      if (files[i].thumbnail) {
+        (files[i].thumbnail as unknown as string) = formatRootUrl('/r', files[i].thumbnail.name);
+      }
     }
 
     if (req.query.filter && req.query.filter === 'media')

src/pages/api/user/paged.ts

@@ -85,6 +85,7 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
     folderId: number;
     size: number;
     password: string | boolean;
+    thumbnail?: { name: string };
   }[] = await prisma.file.findMany({
     where,
     orderBy: {
@@ -102,6 +103,7 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
       folderId: true,
       size: true,
       password: true,
+      thumbnail: true,
     },
     skip: page ? (Number(page) - 1) * pageCount : undefined,
     take: page ? pageCount : undefined,
@@ -112,6 +114,9 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
     if (file.password) file.password = true;
 
     (file as unknown as { url: string }).url = formatRootUrl(config.uploader.route, file.name);
+    if (files[i].thumbnail) {
+      (files[i].thumbnail as unknown as string) = formatRootUrl('/r', files[i].thumbnail.name);
+    }
   }
 
   return res.json(files);

src/pages/api/user/recent.ts

@@ -27,11 +27,15 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
       folderId: true,
       size: true,
       favorite: true,
+      thumbnail: true,
     },
   });
 
   for (let i = 0; i !== files.length; ++i) {
     (files[i] as unknown as { url: string }).url = formatRootUrl(config.uploader.route, files[i].name);
+    if (files[i].thumbnail) {
+      (files[i].thumbnail as unknown as string) = formatRootUrl('/r', files[i].thumbnail.name);
+    }
   }
 
   if (req.query.filter && req.query.filter === 'media')

src/scripts/clear-temp.ts

@@ -11,7 +11,9 @@ async function main() {
     process.exit(0);
   }
 
-  const files = (await readdir(temp)).filter((x) => x.startsWith('zipline_partial_'));
+  const files = (await readdir(temp)).filter(
+    (x) => x.startsWith('zipline_partial_') || x.startsWith('zipline_thumb_')
+  );
   if (files.length === 0) {
     console.log('No partial files found, exiting..');
     process.exit(0);

src/scripts/clear-zero-byte.ts

@@ -47,6 +47,9 @@ async function main() {
       },
     },
   });
+
+  await prisma.$disconnect();
+
   console.log(`Deleted ${count} files from the database.`);
 
   for (let i = 0; i !== toDelete.length; ++i) {

src/scripts/import-dir.ts

@@ -52,6 +52,8 @@ async function main() {
     await datasource.save(file, await readFile(join(directory, file)));
   }
   console.log(`Finished copying files to ${config.datasource.type} storage.`);
+
+  process.exit(0);
 }
 
 main();

src/scripts/list-users.ts

@@ -1,6 +1,7 @@
 import { PrismaClient } from '@prisma/client';
 import config from 'lib/config';
 import { migrations } from 'server/util';
+import { inspect } from 'util';
 
 async function main() {
   const extras = (process.argv[2] ?? '').split(',');
@@ -13,6 +14,7 @@ async function main() {
   const select = {
     username: true,
     administrator: true,
+    superAdmin: true,
     id: true,
   };
   for (let i = 0; i !== extras.length; ++i) {
@@ -30,7 +32,11 @@ async function main() {
     select,
   });
 
-  console.log(JSON.stringify(users, null, 2));
+  await prisma.$disconnect();
+
+  console.log(inspect(users, false, 4, true));
+
+  process.exit(0);
 }
 
 main();

src/scripts/query-size.ts

@@ -60,11 +60,14 @@ async function main() {
     }
   }
 
+  await prisma.$disconnect();
+
   notFound
     ? console.log(
         'At least one file has been found to not exist in the datasource but was on the database. To remove these files, run the script with the --force-delete flag.'
       )
     : console.log('Done.');
+
   process.exit(0);
 }
 

src/scripts/set-user.ts

@@ -66,11 +66,15 @@ async function main() {
     data,
   });
 
+  await prisma.$disconnect();
+
   if (args[1] === 'password') {
     parsed = '***';
   }
 
   console.log(`Updated user ${user.id} with ${args[1]} = ${parsed}`);
+
+  process.exit(0);
 }
 
 main();

src/server/index.ts

@@ -1,11 +1,12 @@
 import config from 'lib/config';
 import datasource from 'lib/datasource';
 import Logger from 'lib/logger';
-import { version } from '../../package.json';
 import { getStats } from 'server/util';
+import { version } from '../../package.json';
 
 import fastify, { FastifyInstance, FastifyServerOptions } from 'fastify';
 import { createReadStream, existsSync, readFileSync } from 'fs';
+import { Worker } from 'worker_threads';
 import dbFileDecorator from './decorators/dbFile';
 import notFound from './decorators/notFound';
 import postFileDecorator from './decorators/postFile';
@@ -183,9 +184,11 @@ Disallow: ${config.urls.route}
 
   await clearInvites.bind(server)();
   await stats.bind(server)();
+  await thumbs.bind(server)();
 
   setInterval(() => clearInvites.bind(server)(), config.core.invites_interval * 1000);
   setInterval(() => stats.bind(server)(), config.core.stats_interval * 1000);
+  setInterval(() => thumbs.bind(server)(), config.core.thumbnails_interval * 1000);
 }
 
 async function stats(this: FastifyInstance) {
@@ -217,6 +220,51 @@ async function clearInvites(this: FastifyInstance) {
   logger.child('invites').debug(`deleted ${count} used invites`);
 }
 
+async function thumbs(this: FastifyInstance) {
+  const videoFiles = await this.prisma.file.findMany({
+    where: {
+      mimetype: {
+        startsWith: 'video/',
+      },
+      thumbnail: null,
+    },
+    include: {
+      thumbnail: true,
+    },
+  });
+
+  // avoids reaching prisma connection limit
+  const MAX_THUMB_THREADS = 4;
+
+  // make all the files fit into 4 arrays
+  const chunks = [];
+
+  for (let i = 0; i !== MAX_THUMB_THREADS; ++i) {
+    chunks.push([]);
+
+    for (let j = i; j < videoFiles.length; j += MAX_THUMB_THREADS) {
+      chunks[i].push(videoFiles[j]);
+    }
+  }
+
+  logger.child('thumbnail').debug(`starting ${chunks.length} thumbnail threads`);
+
+  for (let i = 0; i !== chunks.length; ++i) {
+    const chunk = chunks[i];
+    if (chunk.length === 0) continue;
+
+    logger.child('thumbnail').debug(`starting thumbnail generation for ${chunk.length} videos`);
+
+    new Worker('./dist/worker/thumbnail.js', {
+      workerData: {
+        videos: chunk,
+        config,
+        datasource,
+      },
+    });
+  }
+}
+
 function genFastifyOpts(): FastifyServerOptions {
   const opts = {};
 

src/worker/thumbnail.ts

@@ -0,0 +1,128 @@
+import { type File, PrismaClient, type Thumbnail } from '@prisma/client';
+import { spawn } from 'child_process';
+import ffmpeg from 'ffmpeg-static';
+import { createWriteStream } from 'fs';
+import { rm } from 'fs/promises';
+import type { Config } from 'lib/config/Config';
+import Logger from 'lib/logger';
+import { randomChars } from 'lib/util';
+import { join } from 'path';
+import { isMainThread, workerData } from 'worker_threads';
+import datasource from 'lib/datasource';
+
+const { videos, config } = workerData as {
+  videos: (File & {
+    thumbnail: Thumbnail;
+  })[];
+  config: Config;
+};
+
+const logger = Logger.get('worker::thumbnail').child(randomChars(4));
+
+logger.debug(`thumbnail generation for ${videos.length} videos`);
+
+if (isMainThread) {
+  logger.error('worker is not a thread');
+  process.exit(1);
+}
+
+async function loadThumbnail(path) {
+  const args = ['-i', path, '-frames:v', '1', '-f', 'mjpeg', 'pipe:1'];
+
+  const child = spawn(ffmpeg, args, { stdio: ['ignore', 'pipe', 'ignore'] });
+
+  const data: Buffer = await new Promise((resolve, reject) => {
+    const buffers = [];
+
+    child.stdout.on('data', (chunk) => {
+      buffers.push(chunk);
+    });
+
+    child.once('error', reject);
+    child.once('close', (code) => {
+      if (code !== 0) {
+        reject(new Error(`child exited with code ${code}`));
+      } else {
+        const buffer = Buffer.allocUnsafe(buffers.reduce((acc, val) => acc + val.length, 0));
+
+        let offset = 0;
+        for (let i = 0; i !== buffers.length; ++i) {
+          const chunk = buffers[i];
+          chunk.copy(buffer, offset);
+          offset += chunk.length;
+        }
+
+        resolve(buffer);
+      }
+    });
+  });
+
+  return data;
+}
+
+async function loadFileTmp(file: File) {
+  const stream = await datasource.get(file.name);
+
+  // pipe to tmp file
+  const tmpFile = join(config.core.temp_directory, `zipline_thumb_${file.id}_${file.id}.tmp`);
+  const fileWriteStream = createWriteStream(tmpFile);
+
+  await new Promise((resolve, reject) => {
+    stream.pipe(fileWriteStream);
+    stream.once('error', reject);
+    stream.once('end', resolve);
+  });
+
+  return tmpFile;
+}
+
+async function start() {
+  const prisma = new PrismaClient();
+
+  for (let i = 0; i !== videos.length; ++i) {
+    const file = videos[i];
+    if (!file.mimetype.startsWith('video/')) {
+      logger.info('file is not a video');
+      process.exit(0);
+    }
+
+    if (file.thumbnail) {
+      logger.info('thumbnail already exists');
+      process.exit(0);
+    }
+
+    const tmpFile = await loadFileTmp(file);
+    logger.debug(`loaded file to tmp: ${tmpFile}`);
+    const thumbnail = await loadThumbnail(tmpFile);
+    logger.debug(`loaded thumbnail: ${thumbnail.length} bytes mjpeg`);
+
+    const { thumbnail: thumb } = await prisma.file.update({
+      where: {
+        id: file.id,
+      },
+      data: {
+        thumbnail: {
+          create: {
+            name: `.thumb-${file.id}.jpg`,
+          },
+        },
+      },
+      select: {
+        thumbnail: true,
+      },
+    });
+
+    await datasource.save(thumb.name, thumbnail);
+
+    logger.info(`thumbnail saved - ${thumb.name}`);
+    logger.debug(`thumbnail ${JSON.stringify(thumb)}`);
+
+    logger.debug(`removing tmp file: ${tmpFile}`);
+    await rm(tmpFile);
+  }
+
+  await prisma.$disconnect();
+  process.exit(0);
+}
+
+start();

tsup.config.ts

@@ -19,6 +19,11 @@ export default defineConfig([
     outDir: 'dist/worker',
     ...opts,
   },
+  {
+    entryPoints: ['src/worker/thumbnail.ts'],
+    outDir: 'dist/worker',
+    ...opts,
+  },
   // scripts
   {
     entryPoints: ['src/scripts/import-dir.ts'],

yarn.lock

@@ -1142,6 +1142,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@derhuerst/http-basic@npm:^8.2.0":
+  version: 8.2.4
+  resolution: "@derhuerst/http-basic@npm:8.2.4"
+  dependencies:
+    caseless: ^0.12.0
+    concat-stream: ^2.0.0
+    http-response-object: ^3.0.1
+    parse-cache-control: ^1.0.1
+  checksum: dfb2f30c23fb907988d1c34318fa74c54dcd3c3ba6b4b0e64cdb584d03303ad212dd3b3874328a9367d7282a232976acbd33a20bb9c7a6ea20752e879459253b
+  languageName: node
+  linkType: hard
+
 "@emotion/babel-plugin@npm:^11.10.6":
   version: 11.10.6
   resolution: "@emotion/babel-plugin@npm:11.10.6"
@@ -2719,6 +2731,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/node@npm:^10.0.3":
+  version: 10.17.60
+  resolution: "@types/node@npm:10.17.60"
+  checksum: 2cdb3a77d071ba8513e5e8306fa64bf50e3c3302390feeaeff1fd325dd25c8441369715dfc8e3701011a72fed5958c7dfa94eb9239a81b3c286caa4d97db6eef
+  languageName: node
+  linkType: hard
+
 "@types/node@npm:^17.0.45":
   version: 17.0.45
   resolution: "@types/node@npm:17.0.45"
@@ -3822,9 +3841,16 @@ __metadata:
   linkType: hard
 
 "caniuse-lite@npm:^1.0.30001406":
-  version: 1.0.30001439
+  version: 1.0.30001494
-  resolution: "caniuse-lite@npm:1.0.30001439"
+  resolution: "caniuse-lite@npm:1.0.30001494"
-  checksum: 3912dd536c9735713ca85e47721988bbcefb881ddb4886b0b9923fa984247fd22cba032cf268e57d158af0e8a2ae2eae042ae01942a1d6d7849fa9fa5d62fb82
+  checksum: 770b742ebba6076da72e94f979ef609bbc855369d1b937c52227935d966b11c3b02baa6511fba04a804802b6eb22af0a2a4a82405963bbb769772530e6be7a8e
+  languageName: node
+  linkType: hard
+
+"caseless@npm:^0.12.0":
+  version: 0.12.0
+  resolution: "caseless@npm:0.12.0"
+  checksum: b43bd4c440aa1e8ee6baefee8063b4850fd0d7b378f6aabc796c9ec8cb26d27fb30b46885350777d9bd079c5256c0e1329ad0dc7c2817e0bb466810ebb353751
   languageName: node
   linkType: hard
 
@@ -4136,6 +4162,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"concat-stream@npm:^2.0.0":
+  version: 2.0.0
+  resolution: "concat-stream@npm:2.0.0"
+  dependencies:
+    buffer-from: ^1.0.0
+    inherits: ^2.0.3
+    readable-stream: ^3.0.2
+    typedarray: ^0.0.6
+  checksum: d7f75d48f0ecd356c1545d87e22f57b488172811b1181d96021c7c4b14ab8855f5313280263dca44bb06e5222f274d047da3e290a38841ef87b59719bde967c7
+  languageName: node
+  linkType: hard
+
 "console-control-strings@npm:^1.0.0, console-control-strings@npm:^1.1.0":
   version: 1.1.0
   resolution: "console-control-strings@npm:1.1.0"
@@ -5625,6 +5663,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"ffmpeg-static@npm:^5.1.0":
+  version: 5.1.0
+  resolution: "ffmpeg-static@npm:5.1.0"
+  dependencies:
+    "@derhuerst/http-basic": ^8.2.0
+    env-paths: ^2.2.0
+    https-proxy-agent: ^5.0.0
+    progress: ^2.0.3
+  checksum: 0e27d671a0be1f585ef03e48c2af7c2be14f4e61470ffa02e3b8919551243ee854028a898dfcd16cdf1e3c01916f3c5e9938f42cbc7e877d7dd80d566867db8b
+  languageName: node
+  linkType: hard
+
 "file-entry-cache@npm:^6.0.1":
   version: 6.0.1
   resolution: "file-entry-cache@npm:6.0.1"
@@ -6335,6 +6385,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"http-response-object@npm:^3.0.1":
+  version: 3.0.2
+  resolution: "http-response-object@npm:3.0.2"
+  dependencies:
+    "@types/node": ^10.0.3
+  checksum: 6cbdcb4ce7b27c9158a131b772c903ed54add2ba831e29cc165e91c3969fa6f8105ddf924aac5b954b534ad15a1ae697b693331b2be5281ee24d79aae20c3264
+  languageName: node
+  linkType: hard
+
 "https-proxy-agent@npm:5.0.1, https-proxy-agent@npm:^5.0.0":
   version: 5.0.1
   resolution: "https-proxy-agent@npm:5.0.1"
@@ -8874,6 +8933,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"parse-cache-control@npm:^1.0.1":
+  version: 1.0.1
+  resolution: "parse-cache-control@npm:1.0.1"
+  checksum: 5a70868792124eb07c2dd07a78fcb824102e972e908254e9e59ce59a4796c51705ff28196d2b20d3b7353d14e9f98e65ed0e4eda9be072cc99b5297dc0466fee
+  languageName: node
+  linkType: hard
+
 "parse-json@npm:^4.0.0":
   version: 4.0.0
   resolution: "parse-json@npm:4.0.0"
@@ -9301,7 +9367,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"progress@npm:2.0.3":
+"progress@npm:2.0.3, progress@npm:^2.0.3":
   version: 2.0.3
   resolution: "progress@npm:2.0.3"
   checksum: f67403fe7b34912148d9252cb7481266a354bd99ce82c835f79070643bb3c6583d10dbcfda4d41e04bbc1d8437e9af0fb1e1f2135727878f5308682a579429b7
@@ -9736,6 +9802,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"readable-stream@npm:^3.0.2":
+  version: 3.6.2
+  resolution: "readable-stream@npm:3.6.2"
+  dependencies:
+    inherits: ^2.0.3
+    string_decoder: ^1.1.1
+    util-deprecate: ^1.0.1
+  checksum: bdcbe6c22e846b6af075e32cf8f4751c2576238c5043169a1c221c92ee2878458a816a4ea33f4c67623c0b6827c8a400409bfb3cf0bf3381392d0b1dfb52ac8d
+  languageName: node
+  linkType: hard
+
 "readable-stream@npm:^4.0.0":
   version: 4.2.0
   resolution: "readable-stream@npm:4.2.0"
@@ -11902,6 +11979,7 @@ __metadata:
     fastify: ^4.15.0
     fastify-plugin: ^4.5.0
     fflate: ^0.7.4
+    ffmpeg-static: ^5.1.0
     find-my-way: ^7.6.0
     katex: ^0.16.4
     mantine-datatable: ^2.2.6