Infrastructure/zipline
1
Fork 0
mirror of https://github.com/diced/zipline.git synced 2025-04-04 23:21:17 -05:00
Code Issues Activity

feat: thumbnails workers

Browse source
This commit is contained in:
diced 2023-05-23 22:46:46 -07:00
parent a2c085719a
commit 4893f4a09e
No known key found for this signature in database
GPG key ID: 370BD1BA142842D1
7 changed files with 254 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
package.json
View file

@ -54,6 +54,7 @@
"fastify": "^4.15.0",
"fastify-plugin": "^4.5.0",
"fflate": "^0.7.4",
"ffmpeg-static": "^5.1.0",
"find-my-way": "^7.6.0",
"katex": "^0.16.4",
"mantine-datatable": "^2.2.6",

18
prisma/migrations/20230523025656_thumbnails/migration.sql Normal file
View file

@ -0,0 +1,18 @@
-- AlterTable
ALTER TABLE "File" ADD COLUMN "thumbnailId" INTEGER;
-- CreateTable
CREATE TABLE "Thumbnail" (
"id" SERIAL NOT NULL,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"name" TEXT NOT NULL,
"fileId" INTEGER NOT NULL,
CONSTRAINT "Thumbnail_pkey" PRIMARY KEY ("id")
);
-- CreateIndex
CREATE UNIQUE INDEX "Thumbnail_fileId_key" ON "Thumbnail"("fileId");
-- AddForeignKey
ALTER TABLE "Thumbnail" ADD CONSTRAINT "Thumbnail_fileId_fkey" FOREIGN KEY ("fileId") REFERENCES "File"("id") ON DELETE CASCADE ON UPDATE CASCADE;

48
prisma/schema.prisma
View file

@ -8,24 +8,24 @@ generator client {
}
model User {
id Int @id @default(autoincrement())
uuid String @unique @default(dbgenerated("gen_random_uuid()")) @db.Uuid
username String
password String?
avatar String?
token String
administrator Boolean @default(false)
superAdmin Boolean @default(false)
systemTheme String @default("system")
embed Json @default("{}")
ratelimit DateTime?
totpSecret String?
domains String[]
oauth OAuth[]
files File[]
urls Url[]
Invite Invite[]
Folder Folder[]
id Int @id @default(autoincrement())
uuid String @unique @default(dbgenerated("gen_random_uuid()")) @db.Uuid
username String
password String?
avatar String?
token String
administrator Boolean @default(false)
superAdmin Boolean @default(false)
systemTheme String @default("system")
embed Json @default("{}")
ratelimit DateTime?
totpSecret String?
domains String[]
oauth OAuth[]
files File[]
urls Url[]
Invite Invite[]
Folder Folder[]
IncompleteFile IncompleteFile[]
}
@ -62,6 +62,18 @@ model File {
folder Folder? @relation(fields: [folderId], references: [id], onDelete: SetNull)
folderId Int?
thumbnail Thumbnail?
thumbnailId Int?
}
model Thumbnail {
id Int @id @default(autoincrement())
createdAt DateTime @default(now())
name String
fileId Int @unique
file File @relation(fields: [fileId], references: [id], onDelete: Cascade)
}
model InvisibleFile {

14
src/server/index.ts
View file

@ -183,6 +183,7 @@ Disallow: ${config.urls.route}
await clearInvites.bind(server)();
await stats.bind(server)();
await thumbs.bind(server)(); // TODO: TEMPORARY - this will be done somewhere else i think
setInterval(() => clearInvites.bind(server)(), config.core.invites_interval * 1000);
setInterval(() => stats.bind(server)(), config.core.stats_interval * 1000);
@ -217,6 +218,19 @@ async function clearInvites(this: FastifyInstance) {
logger.child('invites').debug(`deleted ${count} used invites`);
}
async function thumbs(this: FastifyInstance) {
const videoFiles = await this.prisma.file.findMany({
where: {
mimetype: {
startsWith: 'video/',
},
thumbnail: null,
},
});
logger.child('thumb').debug(`found ${videoFiles.length} videos without thumbnails`);
}
function genFastifyOpts(): FastifyServerOptions {
const opts = {};

107
src/worker/thumbnail.ts Normal file
View file

@ -0,0 +1,107 @@
import Logger from 'lib/logger';
import { isMainThread, workerData } from 'worker_threads';
// import ffmpeg from 'ffmpeg-static';
import prisma from 'lib/prisma';
import datasource from 'lib/datasource';
import config from 'lib/config';
import { spawn } from 'child_process';
import { join } from 'path';
import { createWriteStream } from 'fs';
import { File } from '@prisma/client';
import ffmpeg from 'ffmpeg-static';
import { rm } from 'fs/promises';
const { id } = workerData as { id: number };
const logger = Logger.get('worker::thumbnail').child(id.toString() ?? 'unknown-ident');
if (isMainThread) {
logger.error('worker is not a thread');
process.exit(1);
}
async function loadThumbnail(path) {
const args = ['-i', path, '-frames:v', '1', '-f', 'mjpeg', 'pipe:1'];
const child = spawn(ffmpeg, args, { stdio: ['ignore', 'pipe', 'ignore'] });
const data = await new Promise((resolve, reject) => {
child.stdout.once('data', resolve);
child.once('error', reject);
});
return data as unknown as Buffer;
}
async function loadFileTmp(file: File) {
const stream = await datasource.get(file.name);
// pipe to tmp file
const tmpFile = join(config.core.temp_directory, `zipline_thumb_${file.id}_${file.id}.tmp`);
const fileWriteStream = createWriteStream(tmpFile);
await new Promise((resolve, reject) => {
stream.pipe(fileWriteStream);
stream.once('error', reject);
stream.once('end', resolve);
});
return tmpFile;
}
async function start() {
const file = await prisma.file.findUnique({
where: {
id,
},
});
if (!file) {
logger.error('file not found');
process.exit(1);
}
if (!file.mimetype.startsWith('video/')) {
logger.info('file is not a video');
process.exit(0);
}
if (file.thumbnailId) {
logger.info('thumbnail already exists');
process.exit(0);
}
const tmpFile = await loadFileTmp(file);
logger.debug(`loaded file to tmp: ${tmpFile}`);
const thumbnail = await loadThumbnail(tmpFile);
logger.debug(`loaded thumbnail: ${thumbnail.length} bytes mjpeg`);
const { thumbnail: thumb } = await prisma.file.update({
where: {
id: file.id,
},
data: {
thumbnail: {
create: {
name: `.thumb-${file.id}.jpg`,
},
},
},
select: {
thumbnail: true,
},
});
await datasource.save(thumb.name, thumbnail);
logger.info(`thumbnail saved - ${thumb.name}`);
logger.debug(`thumbnail ${JSON.stringify(thumb)}`);
logger.debug(`removing tmp file: ${tmpFile}`);
await rm(tmpFile);
process.exit(0);
}
start();

5
tsup.config.ts
View file

@ -19,6 +19,11 @@ export default defineConfig([
outDir: 'dist/worker',
...opts,
},
{
entryPoints: ['src/worker/thumbnail.ts'],
outDir: 'dist/worker',
...opts,
},
// scripts
{
entryPoints: ['src/scripts/import-dir.ts'],

80
yarn.lock
View file

@ -1142,6 +1142,18 @@ __metadata:
languageName: node
linkType: hard
"@derhuerst/http-basic@npm:^8.2.0":
version: 8.2.4
resolution: "@derhuerst/http-basic@npm:8.2.4"
dependencies:
caseless: ^0.12.0
concat-stream: ^2.0.0
http-response-object: ^3.0.1
parse-cache-control: ^1.0.1
checksum: dfb2f30c23fb907988d1c34318fa74c54dcd3c3ba6b4b0e64cdb584d03303ad212dd3b3874328a9367d7282a232976acbd33a20bb9c7a6ea20752e879459253b
languageName: node
linkType: hard
"@emotion/babel-plugin@npm:^11.10.6":
version: 11.10.6
resolution: "@emotion/babel-plugin@npm:11.10.6"
@ -2719,6 +2731,13 @@ __metadata:
languageName: node
linkType: hard
"@types/node@npm:^10.0.3":
version: 10.17.60
resolution: "@types/node@npm:10.17.60"
checksum: 2cdb3a77d071ba8513e5e8306fa64bf50e3c3302390feeaeff1fd325dd25c8441369715dfc8e3701011a72fed5958c7dfa94eb9239a81b3c286caa4d97db6eef
languageName: node
linkType: hard
"@types/node@npm:^17.0.45":
version: 17.0.45
resolution: "@types/node@npm:17.0.45"
@ -3828,6 +3847,13 @@ __metadata:
languageName: node
linkType: hard
"caseless@npm:^0.12.0":
version: 0.12.0
resolution: "caseless@npm:0.12.0"
checksum: b43bd4c440aa1e8ee6baefee8063b4850fd0d7b378f6aabc796c9ec8cb26d27fb30b46885350777d9bd079c5256c0e1329ad0dc7c2817e0bb466810ebb353751
languageName: node
linkType: hard
"ccount@npm:^2.0.0":
version: 2.0.1
resolution: "ccount@npm:2.0.1"
@ -4136,6 +4162,18 @@ __metadata:
languageName: node
linkType: hard
"concat-stream@npm:^2.0.0":
version: 2.0.0
resolution: "concat-stream@npm:2.0.0"
dependencies:
buffer-from: ^1.0.0
inherits: ^2.0.3
readable-stream: ^3.0.2
typedarray: ^0.0.6
checksum: d7f75d48f0ecd356c1545d87e22f57b488172811b1181d96021c7c4b14ab8855f5313280263dca44bb06e5222f274d047da3e290a38841ef87b59719bde967c7
languageName: node
linkType: hard
"console-control-strings@npm:^1.0.0, console-control-strings@npm:^1.1.0":
version: 1.1.0
resolution: "console-control-strings@npm:1.1.0"
@ -5625,6 +5663,18 @@ __metadata:
languageName: node
linkType: hard
"ffmpeg-static@npm:^5.1.0":
version: 5.1.0
resolution: "ffmpeg-static@npm:5.1.0"
dependencies:
"@derhuerst/http-basic": ^8.2.0
env-paths: ^2.2.0
https-proxy-agent: ^5.0.0
progress: ^2.0.3
checksum: 0e27d671a0be1f585ef03e48c2af7c2be14f4e61470ffa02e3b8919551243ee854028a898dfcd16cdf1e3c01916f3c5e9938f42cbc7e877d7dd80d566867db8b
languageName: node
linkType: hard
"file-entry-cache@npm:^6.0.1":
version: 6.0.1
resolution: "file-entry-cache@npm:6.0.1"
@ -6335,6 +6385,15 @@ __metadata:
languageName: node
linkType: hard
"http-response-object@npm:^3.0.1":
version: 3.0.2
resolution: "http-response-object@npm:3.0.2"
dependencies:
"@types/node": ^10.0.3
checksum: 6cbdcb4ce7b27c9158a131b772c903ed54add2ba831e29cc165e91c3969fa6f8105ddf924aac5b954b534ad15a1ae697b693331b2be5281ee24d79aae20c3264
languageName: node
linkType: hard
"https-proxy-agent@npm:5.0.1, https-proxy-agent@npm:^5.0.0":
version: 5.0.1
resolution: "https-proxy-agent@npm:5.0.1"
@ -8874,6 +8933,13 @@ __metadata:
languageName: node
linkType: hard
"parse-cache-control@npm:^1.0.1":
version: 1.0.1
resolution: "parse-cache-control@npm:1.0.1"
checksum: 5a70868792124eb07c2dd07a78fcb824102e972e908254e9e59ce59a4796c51705ff28196d2b20d3b7353d14e9f98e65ed0e4eda9be072cc99b5297dc0466fee
languageName: node
linkType: hard
"parse-json@npm:^4.0.0":
version: 4.0.0
resolution: "parse-json@npm:4.0.0"
@ -9301,7 +9367,7 @@ __metadata:
languageName: node
linkType: hard
"progress@npm:2.0.3":
"progress@npm:2.0.3, progress@npm:^2.0.3":
version: 2.0.3
resolution: "progress@npm:2.0.3"
checksum: f67403fe7b34912148d9252cb7481266a354bd99ce82c835f79070643bb3c6583d10dbcfda4d41e04bbc1d8437e9af0fb1e1f2135727878f5308682a579429b7
@ -9736,6 +9802,17 @@ __metadata:
languageName: node
linkType: hard
"readable-stream@npm:^3.0.2":
version: 3.6.2
resolution: "readable-stream@npm:3.6.2"
dependencies:
inherits: ^2.0.3
string_decoder: ^1.1.1
util-deprecate: ^1.0.1
checksum: bdcbe6c22e846b6af075e32cf8f4751c2576238c5043169a1c221c92ee2878458a816a4ea33f4c67623c0b6827c8a400409bfb3cf0bf3381392d0b1dfb52ac8d
languageName: node
linkType: hard
"readable-stream@npm:^4.0.0":
version: 4.2.0
resolution: "readable-stream@npm:4.2.0"
@ -11902,6 +11979,7 @@ __metadata:
fastify: ^4.15.0
fastify-plugin: ^4.5.0
fflate: ^0.7.4
ffmpeg-static: ^5.1.0
find-my-way: ^7.6.0
katex: ^0.16.4
mantine-datatable: ^2.2.6