feat: video thumbnails (#413) (#376)

* feat: thumbnails workers

* feat: thumbnails final

* fix: no thumbnailId

* fix: unecessary stuff

.env.local.example

@@ -1,7 +1,7 @@
 # every field in here is optional except, CORE_SECRET and CORE_DATABASE_URL.
 # if CORE_SECRET is still "changethis" then zipline will exit and tell you to change it.
 
-# if using s3/supabase make sure to comment out the other datasources
+# if using s3/supabase make sure to uncomment or comment out the correct lines needed.
 
 CORE_RETURN_HTTPS=true
 CORE_SECRET="changethis"
@@ -10,34 +10,36 @@ CORE_PORT=3000
 CORE_DATABASE_URL="postgres://postgres:postgres@localhost/zip10"
 CORE_LOGGER=false
 CORE_STATS_INTERVAL=1800
+CORE_INVITES_INTERVAL=1800
+CORE_THUMBNAILS_INTERVAL=600
 
 # default
 DATASOURCE_TYPE=local
 DATASOURCE_LOCAL_DIRECTORY=./uploads
 
 # or you can choose to use s3
-DATASOURCE_TYPE=s3
+# DATASOURCE_TYPE=s3
-DATASOURCE_S3_ACCESS_KEY_ID=key
+# DATASOURCE_S3_ACCESS_KEY_ID=key
-DATASOURCE_S3_SECRET_ACCESS_KEY=secret
+# DATASOURCE_S3_SECRET_ACCESS_KEY=secret
-DATASOURCE_S3_BUCKET=bucket
+# DATASOURCE_S3_BUCKET=bucket
-DATASOURCE_S3_ENDPOINT=s3.amazonaws.com
+# DATASOURCE_S3_ENDPOINT=s3.amazonaws.com
-DATASOURCE_S3_REGION=us-west-2
+# DATASOURCE_S3_REGION=us-west-2
-DATASOURCE_S3_FORCE_S3_PATH=false
+# DATASOURCE_S3_FORCE_S3_PATH=false
-DATASOURCE_S3_USE_SSL=false
+# DATASOURCE_S3_USE_SSL=false
 
 # or supabase
-DATASOURCE_TYPE=supabase
+# DATASOURCE_TYPE=supabase
-DATASOURCE_SUPABASE_KEY=xxx
+# DATASOURCE_SUPABASE_KEY=xxx
 # remember: no leading slash
-DATASOURCE_SUPABASE_URL=https://something.supabase.co
+# DATASOURCE_SUPABASE_URL=https://something.supabase.co
-DATASOURCE_SUPABASE_BUCKET=zipline
+# DATASOURCE_SUPABASE_BUCKET=zipline
 
 UPLOADER_DEFAULT_FORMAT=RANDOM
 UPLOADER_ROUTE=/u
 UPLOADER_LENGTH=6
 UPLOADER_ADMIN_LIMIT=104900000
 UPLOADER_USER_LIMIT=104900000
-UPLOADER_DISABLED_EXTENSIONS=someext
+UPLOADER_DISABLED_EXTENSIONS=someext,anotherext
 
 URLS_ROUTE=/go
 URLS_LENGTH=6

package.json

@@ -54,6 +54,7 @@
     "fastify": "^4.15.0",
     "fastify-plugin": "^4.5.0",
     "fflate": "^0.7.4",
+    "ffmpeg-static": "^5.1.0",
     "find-my-way": "^7.6.0",
     "katex": "^0.16.4",
     "mantine-datatable": "^2.2.6",

prisma/migrations/20230523025656_thumbnails/migration.sql

@@ -0,0 +1,16 @@
+
+-- CreateTable
+CREATE TABLE "Thumbnail" (
+    "id" SERIAL NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "name" TEXT NOT NULL,
+    "fileId" INTEGER NOT NULL,
+
+    CONSTRAINT "Thumbnail_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "Thumbnail_fileId_key" ON "Thumbnail"("fileId");
+
+-- AddForeignKey
+ALTER TABLE "Thumbnail" ADD CONSTRAINT "Thumbnail_fileId_fkey" FOREIGN KEY ("fileId") REFERENCES "File"("id") ON DELETE CASCADE ON UPDATE CASCADE;

prisma/schema.prisma

@@ -8,24 +8,24 @@ generator client {
 }
 
 model User {
-  id            Int       @id @default(autoincrement())
+  id             Int              @id @default(autoincrement())
-  uuid          String    @unique @default(dbgenerated("gen_random_uuid()")) @db.Uuid
+  uuid           String           @unique @default(dbgenerated("gen_random_uuid()")) @db.Uuid
-  username      String
+  username       String
-  password      String?
+  password       String?
-  avatar        String?
+  avatar         String?
-  token         String
+  token          String
-  administrator Boolean   @default(false)
+  administrator  Boolean          @default(false)
-  superAdmin    Boolean   @default(false)
+  superAdmin     Boolean          @default(false)
-  systemTheme   String    @default("system")
+  systemTheme    String           @default("system")
-  embed         Json      @default("{}")
+  embed          Json             @default("{}")
-  ratelimit     DateTime?
+  ratelimit      DateTime?
-  totpSecret    String?
+  totpSecret     String?
-  domains       String[]
+  domains        String[]
-  oauth         OAuth[]
+  oauth          OAuth[]
-  files         File[]
+  files          File[]
-  urls          Url[]
+  urls           Url[]
-  Invite        Invite[]
+  Invite         Invite[]
-  Folder        Folder[]
+  Folder         Folder[]
   IncompleteFile IncompleteFile[]
 }
 
@@ -62,6 +62,17 @@ model File {
 
   folder   Folder? @relation(fields: [folderId], references: [id], onDelete: SetNull)
   folderId Int?
+
+  thumbnail   Thumbnail?
+}
+
+model Thumbnail {
+  id        Int      @id @default(autoincrement())
+  createdAt DateTime @default(now())
+  name      String
+
+  fileId Int  @unique
+  file   File @relation(fields: [fileId], references: [id], onDelete: Cascade)
 }
 
 model InvisibleFile {

src/components/File/index.tsx

@@ -63,7 +63,19 @@ export default function File({
         otherUser={otherUser}
       />
 
-      <Card sx={{ maxWidth: '100%', height: '100%' }} shadow='md' onClick={() => setOpen(true)}>
+      <Card
+        sx={{
+          maxWidth: '100%',
+          height: '100%',
+          '&:hover': {
+            filter: 'brightness(0.75)',
+          },
+          transition: 'filter 0.2s ease-in-out',
+          cursor: 'pointer',
+        }}
+        shadow='md'
+        onClick={() => setOpen(true)}
+      >
         <Card.Section>
           <LoadingOverlay visible={loading} />
           <Type

src/components/Type.tsx

@@ -53,6 +53,31 @@ function Placeholder({ text, Icon, ...props }) {
   );
 }
 
+function VideoThumbnailPlaceholder({ file, mediaPreview, ...props }) {
+  if (!file.thumbnail || !mediaPreview)
+    return <Placeholder Icon={IconPlayerPlay} text={`Click to view video (${file.name})`} {...props} />;
+
+  return (
+    <Box>
+      <Image
+        src={file.thumbnail}
+        sx={{
+          position: 'absolute',
+          width: '100%',
+          height: '100%',
+          objectFit: 'cover',
+        }}
+      />
+
+      <Center sx={{ position: 'absolute', width: '100%', height: '100%' }}>
+        <IconPlayerPlay size={48} />
+      </Center>
+    </Box>
+
+    // </Placeholder>
+  );
+}
+
 export default function Type({ file, popup = false, disableMediaPreview, ...props }) {
   const type =
     (file.type ?? file.mimetype) === ''
@@ -159,7 +184,8 @@ export default function Type({ file, popup = false, disableMediaPreview, ...prop
     )
   ) : media ? (
     {
-      video: <Placeholder Icon={IconPlayerPlay} text={`Click to view video (${file.name})`} {...props} />,
+      // video: <Placeholder Icon={IconPlayerPlay} text={`Click to view video (${file.name})`} {...props} />,
+      video: <VideoThumbnailPlaceholder file={file} mediaPreview={!disableMediaPreview} />,
       image: (
         <Image
           placeholder={<PlaceholderContent Icon={IconPhotoCancel} text={'Image failed to load...'} />}

src/lib/config/Config.ts

@@ -10,6 +10,7 @@ export interface ConfigCore {
 
   stats_interval: number;
   invites_interval: number;
+  thumbnails_interval: number;
 }
 
 export interface ConfigCompression {

src/lib/config/readConfig.ts

@@ -63,8 +63,11 @@ export default function readConfig() {
     map('CORE_PORT', 'number', 'core.port'),
     map('CORE_DATABASE_URL', 'string', 'core.database_url'),
     map('CORE_LOGGER', 'boolean', 'core.logger'),
+
     map('CORE_STATS_INTERVAL', 'number', 'core.stats_interval'),
     map('CORE_INVITES_INTERVAL', 'number', 'core.invites_interval'),
+    map('CORE_THUMBNAILS_INTERVAL', 'number', 'core.thumbnails_interval'),
+
     map('CORE_COMPRESSION_ENABLED', 'boolean', 'core.compression.enabled'),
     map('CORE_COMPRESSION_THRESHOLD', 'human-to-byte', 'core.compression.threshold'),
     map('CORE_COMPRESSION_ON_DASHBOARD', 'boolean', 'core.compression.on_dashboard'),

src/lib/config/validateConfig.ts

@@ -35,8 +35,9 @@ const validator = s.object({
     port: s.number.default(3000),
     database_url: s.string,
     logger: s.boolean.default(false),
-    stats_interval: s.number.default(1800),
+    stats_interval: s.number.default(1800), // 30m
-    invites_interval: s.number.default(1800),
+    invites_interval: s.number.default(1800), // 30m
+    thumbnails_interval: s.number.default(600), // 10m
     compression: s
       .object({
         enabled: s.boolean.default(false),

src/pages/api/user/files.ts

@@ -134,6 +134,7 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
       views: number;
       size: number;
       originalName: string;
+      thumbnail?: { name: string };
     }[] = await prisma.file.findMany({
       where: {
         userId: user.id,
@@ -154,11 +155,16 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
         maxViews: true,
         size: true,
         originalName: true,
+        thumbnail: true,
       },
     });
 
     for (let i = 0; i !== files.length; ++i) {
       (files[i] as unknown as { url: string }).url = formatRootUrl(config.uploader.route, files[i].name);
+
+      if (files[i].thumbnail) {
+        (files[i].thumbnail as unknown as string) = formatRootUrl('/r', files[i].thumbnail.name);
+      }
     }
 
     if (req.query.filter && req.query.filter === 'media')

src/pages/api/user/paged.ts

@@ -85,6 +85,7 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
     folderId: number;
     size: number;
     password: string | boolean;
+    thumbnail?: { name: string };
   }[] = await prisma.file.findMany({
     where,
     orderBy: {
@@ -102,6 +103,7 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
       folderId: true,
       size: true,
       password: true,
+      thumbnail: true,
     },
     skip: page ? (Number(page) - 1) * pageCount : undefined,
     take: page ? pageCount : undefined,
@@ -112,6 +114,9 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
     if (file.password) file.password = true;
 
     (file as unknown as { url: string }).url = formatRootUrl(config.uploader.route, file.name);
+    if (files[i].thumbnail) {
+      (files[i].thumbnail as unknown as string) = formatRootUrl('/r', files[i].thumbnail.name);
+    }
   }
 
   return res.json(files);

src/pages/api/user/recent.ts

@@ -27,11 +27,15 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
       folderId: true,
       size: true,
       favorite: true,
+      thumbnail: true,
     },
   });
 
   for (let i = 0; i !== files.length; ++i) {
     (files[i] as unknown as { url: string }).url = formatRootUrl(config.uploader.route, files[i].name);
+    if (files[i].thumbnail) {
+      (files[i].thumbnail as unknown as string) = formatRootUrl('/r', files[i].thumbnail.name);
+    }
   }
 
   if (req.query.filter && req.query.filter === 'media')

src/server/index.ts

@@ -21,6 +21,7 @@ import prismaPlugin from './plugins/prisma';
 import rawRoute from './routes/raw';
 import uploadsRoute, { uploadsRouteOnResponse } from './routes/uploads';
 import urlsRoute, { urlsRouteOnResponse } from './routes/urls';
+import { Worker } from 'worker_threads';
 
 const dev = process.env.NODE_ENV === 'development';
 const logger = Logger.get('server');
@@ -183,9 +184,11 @@ Disallow: ${config.urls.route}
 
   await clearInvites.bind(server)();
   await stats.bind(server)();
+  await thumbs.bind(server)();
 
   setInterval(() => clearInvites.bind(server)(), config.core.invites_interval * 1000);
   setInterval(() => stats.bind(server)(), config.core.stats_interval * 1000);
+  setInterval(() => thumbs.bind(server)(), config.core.thumbnails_interval * 1000);
 }
 
 async function stats(this: FastifyInstance) {
@@ -217,6 +220,27 @@ async function clearInvites(this: FastifyInstance) {
   logger.child('invites').debug(`deleted ${count} used invites`);
 }
 
+async function thumbs(this: FastifyInstance) {
+  const videoFiles = await this.prisma.file.findMany({
+    where: {
+      mimetype: {
+        startsWith: 'video/',
+      },
+      thumbnail: null,
+    },
+  });
+
+  logger.child('thumb').debug(`found ${videoFiles.length} videos without thumbnails`);
+
+  for (const file of videoFiles) {
+    new Worker('./dist/worker/thumbnail.js', {
+      workerData: {
+        id: file.id,
+      },
+    });
+  }
+}
+
 function genFastifyOpts(): FastifyServerOptions {
   const opts = {};
 

src/worker/thumbnail.ts

@@ -0,0 +1,108 @@
+import { File } from '@prisma/client';
+import { spawn } from 'child_process';
+import ffmpeg from 'ffmpeg-static';
+import { createWriteStream } from 'fs';
+import { rm } from 'fs/promises';
+import config from 'lib/config';
+import datasource from 'lib/datasource';
+import Logger from 'lib/logger';
+import prisma from 'lib/prisma';
+import { join } from 'path';
+import { isMainThread, workerData } from 'worker_threads';
+
+const { id } = workerData as { id: number };
+
+const logger = Logger.get('worker::thumbnail').child(id.toString() ?? 'unknown-ident');
+
+if (isMainThread) {
+  logger.error('worker is not a thread');
+  process.exit(1);
+}
+
+async function loadThumbnail(path) {
+  const args = ['-i', path, '-frames:v', '1', '-f', 'mjpeg', 'pipe:1'];
+
+  const child = spawn(ffmpeg, args, { stdio: ['ignore', 'pipe', 'ignore'] });
+
+  const data: Promise<Buffer> = new Promise((resolve, reject) => {
+    child.stdout.once('data', resolve);
+    child.once('error', reject);
+  });
+
+  return data;
+}
+
+async function loadFileTmp(file: File) {
+  const stream = await datasource.get(file.name);
+
+  // pipe to tmp file
+  const tmpFile = join(config.core.temp_directory, `zipline_thumb_${file.id}_${file.id}.tmp`);
+  const fileWriteStream = createWriteStream(tmpFile);
+
+  await new Promise((resolve, reject) => {
+    stream.pipe(fileWriteStream);
+    stream.once('error', reject);
+    stream.once('end', resolve);
+  });
+
+  return tmpFile;
+}
+
+async function start() {
+  const file = await prisma.file.findUnique({
+    where: {
+      id,
+    },
+    include: {
+      thumbnail: true,
+    },
+  });
+
+  if (!file) {
+    logger.error('file not found');
+    process.exit(1);
+  }
+
+  if (!file.mimetype.startsWith('video/')) {
+    logger.info('file is not a video');
+    process.exit(0);
+  }
+
+  if (file.thumbnail) {
+    logger.info('thumbnail already exists');
+    process.exit(0);
+  }
+
+  const tmpFile = await loadFileTmp(file);
+  logger.debug(`loaded file to tmp: ${tmpFile}`);
+  const thumbnail = await loadThumbnail(tmpFile);
+  logger.debug(`loaded thumbnail: ${thumbnail.length} bytes mjpeg`);
+
+  const { thumbnail: thumb } = await prisma.file.update({
+    where: {
+      id: file.id,
+    },
+    data: {
+      thumbnail: {
+        create: {
+          name: `.thumb-${file.id}.jpg`,
+        },
+      },
+    },
+    select: {
+      thumbnail: true,
+    },
+  });
+
+  await datasource.save(thumb.name, thumbnail);
+
+  logger.info(`thumbnail saved - ${thumb.name}`);
+  logger.debug(`thumbnail ${JSON.stringify(thumb)}`);
+
+  logger.debug(`removing tmp file: ${tmpFile}`);
+  await rm(tmpFile);
+
+  process.exit(0);
+}
+
+start();

tsup.config.ts

@@ -19,6 +19,11 @@ export default defineConfig([
     outDir: 'dist/worker',
     ...opts,
   },
+  {
+    entryPoints: ['src/worker/thumbnail.ts'],
+    outDir: 'dist/worker',
+    ...opts,
+  },
   // scripts
   {
     entryPoints: ['src/scripts/import-dir.ts'],

yarn.lock

@@ -1142,6 +1142,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@derhuerst/http-basic@npm:^8.2.0":
+  version: 8.2.4
+  resolution: "@derhuerst/http-basic@npm:8.2.4"
+  dependencies:
+    caseless: ^0.12.0
+    concat-stream: ^2.0.0
+    http-response-object: ^3.0.1
+    parse-cache-control: ^1.0.1
+  checksum: dfb2f30c23fb907988d1c34318fa74c54dcd3c3ba6b4b0e64cdb584d03303ad212dd3b3874328a9367d7282a232976acbd33a20bb9c7a6ea20752e879459253b
+  languageName: node
+  linkType: hard
+
 "@emotion/babel-plugin@npm:^11.10.6":
   version: 11.10.6
   resolution: "@emotion/babel-plugin@npm:11.10.6"
@@ -2719,6 +2731,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/node@npm:^10.0.3":
+  version: 10.17.60
+  resolution: "@types/node@npm:10.17.60"
+  checksum: 2cdb3a77d071ba8513e5e8306fa64bf50e3c3302390feeaeff1fd325dd25c8441369715dfc8e3701011a72fed5958c7dfa94eb9239a81b3c286caa4d97db6eef
+  languageName: node
+  linkType: hard
+
 "@types/node@npm:^17.0.45":
   version: 17.0.45
   resolution: "@types/node@npm:17.0.45"
@@ -3828,6 +3847,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"caseless@npm:^0.12.0":
+  version: 0.12.0
+  resolution: "caseless@npm:0.12.0"
+  checksum: b43bd4c440aa1e8ee6baefee8063b4850fd0d7b378f6aabc796c9ec8cb26d27fb30b46885350777d9bd079c5256c0e1329ad0dc7c2817e0bb466810ebb353751
+  languageName: node
+  linkType: hard
+
 "ccount@npm:^2.0.0":
   version: 2.0.1
   resolution: "ccount@npm:2.0.1"
@@ -4136,6 +4162,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"concat-stream@npm:^2.0.0":
+  version: 2.0.0
+  resolution: "concat-stream@npm:2.0.0"
+  dependencies:
+    buffer-from: ^1.0.0
+    inherits: ^2.0.3
+    readable-stream: ^3.0.2
+    typedarray: ^0.0.6
+  checksum: d7f75d48f0ecd356c1545d87e22f57b488172811b1181d96021c7c4b14ab8855f5313280263dca44bb06e5222f274d047da3e290a38841ef87b59719bde967c7
+  languageName: node
+  linkType: hard
+
 "console-control-strings@npm:^1.0.0, console-control-strings@npm:^1.1.0":
   version: 1.1.0
   resolution: "console-control-strings@npm:1.1.0"
@@ -5625,6 +5663,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"ffmpeg-static@npm:^5.1.0":
+  version: 5.1.0
+  resolution: "ffmpeg-static@npm:5.1.0"
+  dependencies:
+    "@derhuerst/http-basic": ^8.2.0
+    env-paths: ^2.2.0
+    https-proxy-agent: ^5.0.0
+    progress: ^2.0.3
+  checksum: 0e27d671a0be1f585ef03e48c2af7c2be14f4e61470ffa02e3b8919551243ee854028a898dfcd16cdf1e3c01916f3c5e9938f42cbc7e877d7dd80d566867db8b
+  languageName: node
+  linkType: hard
+
 "file-entry-cache@npm:^6.0.1":
   version: 6.0.1
   resolution: "file-entry-cache@npm:6.0.1"
@@ -6335,6 +6385,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"http-response-object@npm:^3.0.1":
+  version: 3.0.2
+  resolution: "http-response-object@npm:3.0.2"
+  dependencies:
+    "@types/node": ^10.0.3
+  checksum: 6cbdcb4ce7b27c9158a131b772c903ed54add2ba831e29cc165e91c3969fa6f8105ddf924aac5b954b534ad15a1ae697b693331b2be5281ee24d79aae20c3264
+  languageName: node
+  linkType: hard
+
 "https-proxy-agent@npm:5.0.1, https-proxy-agent@npm:^5.0.0":
   version: 5.0.1
   resolution: "https-proxy-agent@npm:5.0.1"
@@ -8874,6 +8933,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"parse-cache-control@npm:^1.0.1":
+  version: 1.0.1
+  resolution: "parse-cache-control@npm:1.0.1"
+  checksum: 5a70868792124eb07c2dd07a78fcb824102e972e908254e9e59ce59a4796c51705ff28196d2b20d3b7353d14e9f98e65ed0e4eda9be072cc99b5297dc0466fee
+  languageName: node
+  linkType: hard
+
 "parse-json@npm:^4.0.0":
   version: 4.0.0
   resolution: "parse-json@npm:4.0.0"
@@ -9301,7 +9367,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"progress@npm:2.0.3":
+"progress@npm:2.0.3, progress@npm:^2.0.3":
   version: 2.0.3
   resolution: "progress@npm:2.0.3"
   checksum: f67403fe7b34912148d9252cb7481266a354bd99ce82c835f79070643bb3c6583d10dbcfda4d41e04bbc1d8437e9af0fb1e1f2135727878f5308682a579429b7
@@ -9736,6 +9802,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"readable-stream@npm:^3.0.2":
+  version: 3.6.2
+  resolution: "readable-stream@npm:3.6.2"
+  dependencies:
+    inherits: ^2.0.3
+    string_decoder: ^1.1.1
+    util-deprecate: ^1.0.1
+  checksum: bdcbe6c22e846b6af075e32cf8f4751c2576238c5043169a1c221c92ee2878458a816a4ea33f4c67623c0b6827c8a400409bfb3cf0bf3381392d0b1dfb52ac8d
+  languageName: node
+  linkType: hard
+
 "readable-stream@npm:^4.0.0":
   version: 4.2.0
   resolution: "readable-stream@npm:4.2.0"
@@ -11902,6 +11979,7 @@ __metadata:
     fastify: ^4.15.0
     fastify-plugin: ^4.5.0
     fflate: ^0.7.4
+    ffmpeg-static: ^5.1.0
     find-my-way: ^7.6.0
     katex: ^0.16.4
     mantine-datatable: ^2.2.6