Infrastructure/zipline
1
Fork 0
Code Issues Pull requests Activity

Merge branch 'trunk' into feature/oauth-authentik

Browse source
This commit is contained in:
Jayvin Hernandez 2023-06-20 11:32:15 -07:00 committed by GitHub
commit 106c0418b7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
29 changed files with 557 additions and 125 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
.env.local.example
View file

@ -1,7 +1,7 @@
# every field in here is optional except, CORE_SECRET and CORE_DATABASE_URL.
# if CORE_SECRET is still "changethis" then zipline will exit and tell you to change it.
# if using s3/supabase make sure to comment out the other datasources
# if using s3/supabase make sure to uncomment or comment out the correct lines needed.
CORE_RETURN_HTTPS=true
CORE_SECRET="changethis"
@ -10,34 +10,36 @@ CORE_PORT=3000
CORE_DATABASE_URL="postgres://postgres:postgres@localhost/zip10"
CORE_LOGGER=false
CORE_STATS_INTERVAL=1800
CORE_INVITES_INTERVAL=1800
CORE_THUMBNAILS_INTERVAL=600
# default
DATASOURCE_TYPE=local
DATASOURCE_LOCAL_DIRECTORY=./uploads
# or you can choose to use s3
DATASOURCE_TYPE=s3
DATASOURCE_S3_ACCESS_KEY_ID=key
DATASOURCE_S3_SECRET_ACCESS_KEY=secret
DATASOURCE_S3_BUCKET=bucket
DATASOURCE_S3_ENDPOINT=s3.amazonaws.com
DATASOURCE_S3_REGION=us-west-2
DATASOURCE_S3_FORCE_S3_PATH=false
DATASOURCE_S3_USE_SSL=false
# DATASOURCE_TYPE=s3
# DATASOURCE_S3_ACCESS_KEY_ID=key
# DATASOURCE_S3_SECRET_ACCESS_KEY=secret
# DATASOURCE_S3_BUCKET=bucket
# DATASOURCE_S3_ENDPOINT=s3.amazonaws.com
# DATASOURCE_S3_REGION=us-west-2
# DATASOURCE_S3_FORCE_S3_PATH=false
# DATASOURCE_S3_USE_SSL=false
# or supabase
DATASOURCE_TYPE=supabase
DATASOURCE_SUPABASE_KEY=xxx
# DATASOURCE_TYPE=supabase
# DATASOURCE_SUPABASE_KEY=xxx
# remember: no leading slash
DATASOURCE_SUPABASE_URL=https://something.supabase.co
DATASOURCE_SUPABASE_BUCKET=zipline
# DATASOURCE_SUPABASE_URL=https://something.supabase.co
# DATASOURCE_SUPABASE_BUCKET=zipline
UPLOADER_DEFAULT_FORMAT=RANDOM
UPLOADER_ROUTE=/u
UPLOADER_LENGTH=6
UPLOADER_ADMIN_LIMIT=104900000
UPLOADER_USER_LIMIT=104900000
UPLOADER_DISABLED_EXTENSIONS=someext
UPLOADER_DISABLED_EXTENSIONS=someext,anotherext
URLS_ROUTE=/go
URLS_LENGTH=6

2
LICENSE
View file

@ -1,6 +1,6 @@
MIT License
Copyright (c) 2022 dicedtomato
Copyright (c) 2023 dicedtomato
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal

1
package.json
View file

@ -54,6 +54,7 @@
"fastify": "^4.15.0",
"fastify-plugin": "^4.5.0",
"fflate": "^0.7.4",
"ffmpeg-static": "^5.1.0",
"find-my-way": "^7.6.0",
"katex": "^0.16.4",
"mantine-datatable": "^2.2.6",

16
prisma/migrations/20230523025656_thumbnails/migration.sql Normal file
View file

@ -0,0 +1,16 @@
-- CreateTable
CREATE TABLE "Thumbnail" (
"id" SERIAL NOT NULL,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"name" TEXT NOT NULL,
"fileId" INTEGER NOT NULL,
CONSTRAINT "Thumbnail_pkey" PRIMARY KEY ("id")
);
-- CreateIndex
CREATE UNIQUE INDEX "Thumbnail_fileId_key" ON "Thumbnail"("fileId");
-- AddForeignKey
ALTER TABLE "Thumbnail" ADD CONSTRAINT "Thumbnail_fileId_fkey" FOREIGN KEY ("fileId") REFERENCES "File"("id") ON DELETE CASCADE ON UPDATE CASCADE;

47
prisma/schema.prisma
View file

@ -8,24 +8,24 @@ generator client {
}
model User {
id Int @id @default(autoincrement())
uuid String @unique @default(dbgenerated("gen_random_uuid()")) @db.Uuid
username String
password String?
avatar String?
token String
administrator Boolean @default(false)
superAdmin Boolean @default(false)
systemTheme String @default("system")
embed Json @default("{}")
ratelimit DateTime?
totpSecret String?
domains String[]
oauth OAuth[]
files File[]
urls Url[]
Invite Invite[]
Folder Folder[]
id Int @id @default(autoincrement())
uuid String @unique @default(dbgenerated("gen_random_uuid()")) @db.Uuid
username String
password String?
avatar String?
token String
administrator Boolean @default(false)
superAdmin Boolean @default(false)
systemTheme String @default("system")
embed Json @default("{}")
ratelimit DateTime?
totpSecret String?
domains String[]
oauth OAuth[]
files File[]
urls Url[]
Invite Invite[]
Folder Folder[]
IncompleteFile IncompleteFile[]
}
@ -62,6 +62,17 @@ model File {
folder Folder? @relation(fields: [folderId], references: [id], onDelete: SetNull)
folderId Int?
thumbnail Thumbnail?
}
model Thumbnail {
id Int @id @default(autoincrement())
createdAt DateTime @default(now())
name String
fileId Int @unique
file File @relation(fields: [fileId], references: [id], onDelete: Cascade)
}
model InvisibleFile {

14
src/components/File/index.tsx
View file

@ -63,7 +63,19 @@ export default function File({
otherUser={otherUser}
/>
<Card sx={{ maxWidth: '100%', height: '100%' }} shadow='md' onClick={() => setOpen(true)}>
<Card
sx={{
maxWidth: '100%',
height: '100%',
'&:hover': {
filter: 'brightness(0.75)',
},
transition: 'filter 0.2s ease-in-out',
cursor: 'pointer',
}}
shadow='md'
onClick={() => setOpen(true)}
>
<Card.Section>
<LoadingOverlay visible={loading} />
<Type

6
src/components/Layout.tsx
View file

@ -351,7 +351,11 @@ export default function Layout({ children, props }) {
<Menu.Target>
<Button
leftIcon={
avatar ? <Image src={avatar} height={32} radius='md' /> : <IconUserCog size='1rem' />
avatar ? (
<Image src={avatar} height={32} width={32} fit='cover' radius='md' />
) : (
<IconUserCog size='1rem' />
)
}
variant='subtle'
color='gray'

32
src/components/Type.tsx
View file

@ -53,6 +53,35 @@ function Placeholder({ text, Icon, ...props }) {
);
}
function VideoThumbnailPlaceholder({ file, mediaPreview, ...props }) {
if (!file.thumbnail || !mediaPreview)
return <Placeholder Icon={IconPlayerPlay} text={`Click to view video (${file.name})`} {...props} />;
return (
<Box sx={{ position: 'relative' }}>
<Image
src={file.thumbnail}
sx={{
width: '100%',
height: 'auto',
}}
/>
<Center
sx={{
position: 'absolute',
height: '100%',
top: '50%',
left: '50%',
transform: 'translate(-50%, -50%)',
}}
>
<IconPlayerPlay size={48} />
</Center>
</Box>
);
}
export default function Type({ file, popup = false, disableMediaPreview, ...props }) {
const type =
(file.type ?? file.mimetype) === ''
@ -159,7 +188,8 @@ export default function Type({ file, popup = false, disableMediaPreview, ...prop
)
) : media ? (
{
video: <Placeholder Icon={IconPlayerPlay} text={`Click to view video (${file.name})`} {...props} />,
// video: <Placeholder Icon={IconPlayerPlay} text={`Click to view video (${file.name})`} {...props} />,
video: <VideoThumbnailPlaceholder file={file} mediaPreview={!disableMediaPreview} />,
image: (
<Image
placeholder={<PlaceholderContent Icon={IconPhotoCancel} text={'Image failed to load...'} />}

6
src/components/pages/Files/index.tsx
View file

@ -7,12 +7,16 @@ import Link from 'next/link';
import { useEffect, useState } from 'react';
import FilePagation from './FilePagation';
import PendingFilesModal from './PendingFilesModal';
import { showNonMediaSelector } from 'lib/recoil/settings';
import { useRecoilState } from 'recoil';
export default function Files({ disableMediaPreview, exifEnabled, queryPage, compress }) {
const [checked] = useRecoilState(showNonMediaSelector);
const [favoritePage, setFavoritePage] = useState(1);
const [favoriteNumPages, setFavoriteNumPages] = useState(0);
const favoritePages = usePaginatedFiles(favoritePage, {
filter: 'media',
filter: checked ? 'none' : 'media',
favorite: true,
});

97
src/components/pages/Invites.tsx
View file

@ -55,7 +55,7 @@ function CreateInviteModal({ open, setOpen, updateInvites }) {
setOpen(false);
const res = await useFetch('/api/auth/invite', 'POST', {
expiresAt: expiresAt === null ? null : `date=${expiresAt.toISOString()}`,
expiresAt: `date=${expiresAt.toISOString()}`,
count: values.count,
});
@ -95,7 +95,6 @@ function CreateInviteModal({ open, setOpen, updateInvites }) {
{ value: '3d', label: '3 days' },
{ value: '5d', label: '5 days' },
{ value: '7d', label: '7 days' },
{ value: 'never', label: 'Never' },
]}
/>
@ -299,45 +298,65 @@ export default function Invites() {
/>
) : (
<SimpleGrid cols={3} spacing='lg' breakpoints={[{ maxWidth: 'sm', cols: 1, spacing: 'sm' }]}>
{invites.length
? invites.map((invite) => (
<Card key={invite.id} sx={{ maxWidth: '100%' }}>
<Group position='apart'>
<Group position='left'>
<Avatar size='lg' color={invite.used ? 'dark' : 'primary'}>
{invite.id}
</Avatar>
<Stack spacing={0}>
<Title>
{invite.code}
{invite.used && <> (Used)</>}
</Title>
<Tooltip label={new Date(invite.createdAt).toLocaleString()}>
<div>
<MutedText size='sm'>
Created {relativeTime(new Date(invite.createdAt))}
</MutedText>
</div>
</Tooltip>
<Tooltip label={new Date(invite.expiresAt).toLocaleString()}>
<div>
<MutedText size='sm'>{expireText(invite.expiresAt.toString())}</MutedText>
</div>
</Tooltip>
</Stack>
</Group>
<Stack>
<ActionIcon aria-label='copy' onClick={() => handleCopy(invite)}>
<IconClipboardCopy size='1rem' />
</ActionIcon>
<ActionIcon aria-label='delete' onClick={() => openDeleteModal(invite)}>
<IconTrash size='1rem' />
</ActionIcon>
{!ok && !invites.length && (
<>
{[1, 2, 3].map((x) => (
<Skeleton key={x} width='100%' height={100} radius='sm' />
))}
</>
)}
{invites.length && ok ? (
invites.map((invite) => (
<Card key={invite.id} sx={{ maxWidth: '100%' }}>
<Group position='apart'>
<Group position='left'>
<Avatar size='lg' color={invite.used ? 'dark' : 'primary'}>
{invite.id}
</Avatar>
<Stack spacing={0}>
<Title>
{invite.code}
{invite.used && <> (Used)</>}
</Title>
<Tooltip label={new Date(invite.createdAt).toLocaleString()}>
<div>
<MutedText size='sm'>Created {relativeTime(new Date(invite.createdAt))}</MutedText>
</div>
</Tooltip>
<Tooltip label={new Date(invite.expiresAt).toLocaleString()}>
<div>
<MutedText size='sm'>{expireText(invite.expiresAt.toString())}</MutedText>
</div>
</Tooltip>
</Stack>
</Group>
</Card>
))
: [1, 2, 3].map((x) => <Skeleton key={x} width='100%' height={100} radius='sm' />)}
<Stack>
<ActionIcon aria-label='copy' onClick={() => handleCopy(invite)}>
<IconClipboardCopy size='1rem' />
</ActionIcon>
<ActionIcon aria-label='delete' onClick={() => openDeleteModal(invite)}>
<IconTrash size='1rem' />
</ActionIcon>
</Stack>
</Group>
</Card>
))
) : (
<>
<div></div>
<Group>
<div>
<IconTag size={48} />
</div>
<div>
<Title>Nothing here</Title>
<MutedText size='md'>Create some invites and they will show up here</MutedText>
</div>
</Group>
<div></div>
</>
)}
</SimpleGrid>
)}
</>

2
src/components/pages/Upload/File.tsx
View file

@ -122,6 +122,8 @@ export default function File({ chunks: chunks_config }) {
});
if (j === chunks.length - 1) {
window.removeEventListener('beforeunload', beforeUnload);
router.events.off('routeChangeStart', beforeRouteChange);
updateNotification({
id: 'upload-chunked',
title: 'Finalizing partial upload',

1
src/lib/config/Config.ts
View file

@ -10,6 +10,7 @@ export interface ConfigCore {
stats_interval: number;
invites_interval: number;
thumbnails_interval: number;
}
export interface ConfigCompression {

3
src/lib/config/readConfig.ts
View file

@ -63,8 +63,11 @@ export default function readConfig() {
map('CORE_PORT', 'number', 'core.port'),
map('CORE_DATABASE_URL', 'string', 'core.database_url'),
map('CORE_LOGGER', 'boolean', 'core.logger'),
map('CORE_STATS_INTERVAL', 'number', 'core.stats_interval'),
map('CORE_INVITES_INTERVAL', 'number', 'core.invites_interval'),
map('CORE_THUMBNAILS_INTERVAL', 'number', 'core.thumbnails_interval'),
map('CORE_COMPRESSION_ENABLED', 'boolean', 'core.compression.enabled'),
map('CORE_COMPRESSION_THRESHOLD', 'human-to-byte', 'core.compression.threshold'),
map('CORE_COMPRESSION_ON_DASHBOARD', 'boolean', 'core.compression.on_dashboard'),

5
src/lib/config/validateConfig.ts
View file

@ -35,8 +35,9 @@ const validator = s.object({
port: s.number.default(3000),
database_url: s.string,
logger: s.boolean.default(false),
stats_interval: s.number.default(1800),
invites_interval: s.number.default(1800),
stats_interval: s.number.default(1800), // 30m
invites_interval: s.number.default(1800), // 30m
thumbnails_interval: s.number.default(600), // 10m
compression: s
.object({
enabled: s.boolean.default(false),

81
src/lib/utils/exif.ts
View file

@ -1,10 +1,10 @@
import { File } from '@prisma/client';
import { createWriteStream } from 'fs';
import { ExifTool, Tags } from 'exiftool-vendored';
import { createWriteStream } from 'fs';
import { readFile, rm } from 'fs/promises';
import datasource from 'lib/datasource';
import Logger from 'lib/logger';
import { join } from 'path';
import { readFile, unlink } from 'fs/promises';
const logger = Logger.get('exif');
@ -43,47 +43,54 @@ export async function removeGPSData(image: File): Promise<void> {
await new Promise((resolve) => writeStream.on('finish', resolve));
logger.debug(`removing GPS data from ${file}`);
await exiftool.write(file, {
GPSVersionID: null,
GPSAltitude: null,
GPSAltitudeRef: null,
GPSAreaInformation: null,
GPSDateStamp: null,
GPSDateTime: null,
GPSDestBearing: null,
GPSDestBearingRef: null,
GPSDestDistance: null,
GPSDestLatitude: null,
GPSDestLatitudeRef: null,
GPSDestLongitude: null,
GPSDestLongitudeRef: null,
GPSDifferential: null,
GPSDOP: null,
GPSHPositioningError: null,
GPSImgDirection: null,
GPSImgDirectionRef: null,
GPSLatitude: null,
GPSLatitudeRef: null,
GPSLongitude: null,
GPSLongitudeRef: null,
GPSMapDatum: null,
GPSPosition: null,
GPSProcessingMethod: null,
GPSSatellites: null,
GPSSpeed: null,
GPSSpeedRef: null,
GPSStatus: null,
GPSTimeStamp: null,
GPSTrack: null,
GPSTrackRef: null,
});
try {
await exiftool.write(file, {
GPSVersionID: null,
GPSAltitude: null,
GPSAltitudeRef: null,
GPSAreaInformation: null,
GPSDateStamp: null,
GPSDateTime: null,
GPSDestBearing: null,
GPSDestBearingRef: null,
GPSDestDistance: null,
GPSDestLatitude: null,
GPSDestLatitudeRef: null,
GPSDestLongitude: null,
GPSDestLongitudeRef: null,
GPSDifferential: null,
GPSDOP: null,
GPSHPositioningError: null,
GPSImgDirection: null,
GPSImgDirectionRef: null,
GPSLatitude: null,
GPSLatitudeRef: null,
GPSLongitude: null,
GPSLongitudeRef: null,
GPSMapDatum: null,
GPSPosition: null,
GPSProcessingMethod: null,
GPSSatellites: null,
GPSSpeed: null,
GPSSpeedRef: null,
GPSStatus: null,
GPSTimeStamp: null,
GPSTrack: null,
GPSTrackRef: null,
});
} catch (e) {
logger.debug(`removing temp file: ${file}`);
await rm(file);
return;
}
logger.debug(`reading file to upload to datasource: ${file} -> ${image.name}`);
const buffer = await readFile(file);
await datasource.save(image.name, buffer);
logger.debug(`removing temp file: ${file}`);
await unlink(file);
await rm(file);
await exiftool.end(true);

24
src/pages/api/user/[id].ts
View file

@ -3,6 +3,8 @@ import Logger from 'lib/logger';
import prisma from 'lib/prisma';
import { hashPassword } from 'lib/util';
import { jsonUserReplacer } from 'lib/utils/client';
import { formatRootUrl } from 'lib/utils/urls';
import zconfig from 'lib/config';
import { NextApiReq, NextApiRes, UserExtended, withZipline } from 'middleware/withZipline';
const logger = Logger.get('user');
@ -15,7 +17,11 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
id: Number(id),
},
include: {
files: true,
files: {
include: {
thumbnail: true,
},
},
Folder: true,
},
});
@ -179,9 +185,21 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
} else {
delete target.password;
if (user.superAdmin && target.superAdmin) delete target.files;
if (user.administrator && !user.superAdmin && (target.administrator || target.superAdmin))
if (user.superAdmin && target.superAdmin) {
delete target.files;
return res.json(target);
}
if (user.administrator && !user.superAdmin && (target.administrator || target.superAdmin)) {
delete target.files;
return res.json(target);
}
for (const file of target.files) {
(file as unknown as { url: string }).url = formatRootUrl(zconfig.uploader.route, file.name);
if (file.thumbnail) {
(file.thumbnail as unknown as string) = formatRootUrl('/r', file.thumbnail.name);
}
}
return res.json(target);
}

13
src/pages/api/user/files.ts
View file

@ -14,10 +14,14 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
where: {
userId: user.id,
},
include: {
thumbnail: true,
},
});
for (let i = 0; i !== files.length; ++i) {
await datasource.delete(files[i].name);
if (files[i].thumbnail?.name) await datasource.delete(files[i].thumbnail.name);
}
const { count } = await prisma.file.deleteMany({
@ -45,6 +49,7 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
id: true,
},
},
thumbnail: true,
},
});
@ -63,10 +68,12 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
id: true,
},
},
thumbnail: true,
},
});
await datasource.delete(file.name);
if (file.thumbnail?.name) await datasource.delete(file.thumbnail.name);
logger.info(
`User ${user.username} (${user.id}) deleted an image ${file.name} (${file.id}) owned by ${file.user.username} (${file.user.id})`
@ -134,6 +141,7 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
views: number;
size: number;
originalName: string;
thumbnail?: { name: string };
}[] = await prisma.file.findMany({
where: {
userId: user.id,
@ -154,11 +162,16 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
maxViews: true,
size: true,
originalName: true,
thumbnail: true,
},
});
for (let i = 0; i !== files.length; ++i) {
(files[i] as unknown as { url: string }).url = formatRootUrl(config.uploader.route, files[i].name);
if (files[i].thumbnail) {
(files[i].thumbnail as unknown as string) = formatRootUrl('/r', files[i].thumbnail.name);
}
}
if (req.query.filter && req.query.filter === 'media')

5
src/pages/api/user/paged.ts
View file

@ -85,6 +85,7 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
folderId: number;
size: number;
password: string | boolean;
thumbnail?: { name: string };
}[] = await prisma.file.findMany({
where,
orderBy: {
@ -102,6 +103,7 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
folderId: true,
size: true,
password: true,
thumbnail: true,
},
skip: page ? (Number(page) - 1) * pageCount : undefined,
take: page ? pageCount : undefined,
@ -112,6 +114,9 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
if (file.password) file.password = true;
(file as unknown as { url: string }).url = formatRootUrl(config.uploader.route, file.name);
if (files[i].thumbnail) {
(files[i].thumbnail as unknown as string) = formatRootUrl('/r', files[i].thumbnail.name);
}
}
return res.json(files);

4
src/pages/api/user/recent.ts
View file

@ -27,11 +27,15 @@ async function handler(req: NextApiReq, res: NextApiRes, user: UserExtended) {
folderId: true,
size: true,
favorite: true,
thumbnail: true,
},
});
for (let i = 0; i !== files.length; ++i) {
(files[i] as unknown as { url: string }).url = formatRootUrl(config.uploader.route, files[i].name);
if (files[i].thumbnail) {
(files[i].thumbnail as unknown as string) = formatRootUrl('/r', files[i].thumbnail.name);
}
}
if (req.query.filter && req.query.filter === 'media')

4
src/scripts/clear-temp.ts
View file

@ -11,7 +11,9 @@ async function main() {
process.exit(0);
}
const files = (await readdir(temp)).filter((x) => x.startsWith('zipline_partial_'));
const files = (await readdir(temp)).filter(
(x) => x.startsWith('zipline_partial_') || x.startsWith('zipline_thumb_')
);
if (files.length === 0) {
console.log('No partial files found, exiting..');
process.exit(0);

3
src/scripts/clear-zero-byte.ts
View file

@ -47,6 +47,9 @@ async function main() {
},
},
});
await prisma.$disconnect();
console.log(`Deleted ${count} files from the database.`);
for (let i = 0; i !== toDelete.length; ++i) {

2
src/scripts/import-dir.ts
View file

@ -52,6 +52,8 @@ async function main() {
await datasource.save(file, await readFile(join(directory, file)));
}
console.log(`Finished copying files to ${config.datasource.type} storage.`);
process.exit(0);
}
main();

8
src/scripts/list-users.ts
View file

@ -1,6 +1,7 @@
import { PrismaClient } from '@prisma/client';
import config from 'lib/config';
import { migrations } from 'server/util';
import { inspect } from 'util';
async function main() {
const extras = (process.argv[2] ?? '').split(',');
@ -13,6 +14,7 @@ async function main() {
const select = {
username: true,
administrator: true,
superAdmin: true,
id: true,
};
for (let i = 0; i !== extras.length; ++i) {
@ -30,7 +32,11 @@ async function main() {
select,
});
console.log(JSON.stringify(users, null, 2));
await prisma.$disconnect();
console.log(inspect(users, false, 4, true));
process.exit(0);
}
main();

3
src/scripts/query-size.ts
View file

@ -60,11 +60,14 @@ async function main() {
}
}
await prisma.$disconnect();
notFound
? console.log(
'At least one file has been found to not exist in the datasource but was on the database. To remove these files, run the script with the --force-delete flag.'
)
: console.log('Done.');
process.exit(0);
}

4
src/scripts/set-user.ts
View file

@ -66,11 +66,15 @@ async function main() {
data,
});
await prisma.$disconnect();
if (args[1] === 'password') {
parsed = '***';
}
console.log(`Updated user ${user.id} with ${args[1]} = ${parsed}`);
process.exit(0);
}
main();

50
src/server/index.ts
View file

@ -1,11 +1,12 @@
import config from 'lib/config';
import datasource from 'lib/datasource';
import Logger from 'lib/logger';
import { version } from '../../package.json';
import { getStats } from 'server/util';
import { version } from '../../package.json';
import fastify, { FastifyInstance, FastifyServerOptions } from 'fastify';
import { createReadStream, existsSync, readFileSync } from 'fs';
import { Worker } from 'worker_threads';
import dbFileDecorator from './decorators/dbFile';
import notFound from './decorators/notFound';
import postFileDecorator from './decorators/postFile';
@ -183,9 +184,11 @@ Disallow: ${config.urls.route}
await clearInvites.bind(server)();
await stats.bind(server)();
await thumbs.bind(server)();
setInterval(() => clearInvites.bind(server)(), config.core.invites_interval * 1000);
setInterval(() => stats.bind(server)(), config.core.stats_interval * 1000);
setInterval(() => thumbs.bind(server)(), config.core.thumbnails_interval * 1000);
}
async function stats(this: FastifyInstance) {
@ -217,6 +220,51 @@ async function clearInvites(this: FastifyInstance) {
logger.child('invites').debug(`deleted ${count} used invites`);
}
async function thumbs(this: FastifyInstance) {
const videoFiles = await this.prisma.file.findMany({
where: {
mimetype: {
startsWith: 'video/',
},
thumbnail: null,
},
include: {
thumbnail: true,
},
});
// avoids reaching prisma connection limit
const MAX_THUMB_THREADS = 4;
// make all the files fit into 4 arrays
const chunks = [];
for (let i = 0; i !== MAX_THUMB_THREADS; ++i) {
chunks.push([]);
for (let j = i; j < videoFiles.length; j += MAX_THUMB_THREADS) {
chunks[i].push(videoFiles[j]);
}
}
logger.child('thumbnail').debug(`starting ${chunks.length} thumbnail threads`);
for (let i = 0; i !== chunks.length; ++i) {
const chunk = chunks[i];
if (chunk.length === 0) continue;
logger.child('thumbnail').debug(`starting thumbnail generation for ${chunk.length} videos`);
new Worker('./dist/worker/thumbnail.js', {
workerData: {
videos: chunk,
config,
datasource,
},
});
}
}
function genFastifyOpts(): FastifyServerOptions {
const opts = {};

128
src/worker/thumbnail.ts Normal file
View file

@ -0,0 +1,128 @@
import { type File, PrismaClient, type Thumbnail } from '@prisma/client';
import { spawn } from 'child_process';
import ffmpeg from 'ffmpeg-static';
import { createWriteStream } from 'fs';
import { rm } from 'fs/promises';
import type { Config } from 'lib/config/Config';
import Logger from 'lib/logger';
import { randomChars } from 'lib/util';
import { join } from 'path';
import { isMainThread, workerData } from 'worker_threads';
import datasource from 'lib/datasource';
const { videos, config } = workerData as {
videos: (File & {
thumbnail: Thumbnail;
})[];
config: Config;
};
const logger = Logger.get('worker::thumbnail').child(randomChars(4));
logger.debug(`thumbnail generation for ${videos.length} videos`);
if (isMainThread) {
logger.error('worker is not a thread');
process.exit(1);
}
async function loadThumbnail(path) {
const args = ['-i', path, '-frames:v', '1', '-f', 'mjpeg', 'pipe:1'];
const child = spawn(ffmpeg, args, { stdio: ['ignore', 'pipe', 'ignore'] });
const data: Buffer = await new Promise((resolve, reject) => {
const buffers = [];
child.stdout.on('data', (chunk) => {
buffers.push(chunk);
});
child.once('error', reject);
child.once('close', (code) => {
if (code !== 0) {
reject(new Error(`child exited with code ${code}`));
} else {
const buffer = Buffer.allocUnsafe(buffers.reduce((acc, val) => acc + val.length, 0));
let offset = 0;
for (let i = 0; i !== buffers.length; ++i) {
const chunk = buffers[i];
chunk.copy(buffer, offset);
offset += chunk.length;
}
resolve(buffer);
}
});
});
return data;
}
async function loadFileTmp(file: File) {
const stream = await datasource.get(file.name);
// pipe to tmp file
const tmpFile = join(config.core.temp_directory, `zipline_thumb_${file.id}_${file.id}.tmp`);
const fileWriteStream = createWriteStream(tmpFile);
await new Promise((resolve, reject) => {
stream.pipe(fileWriteStream);
stream.once('error', reject);
stream.once('end', resolve);
});
return tmpFile;
}
async function start() {
const prisma = new PrismaClient();
for (let i = 0; i !== videos.length; ++i) {
const file = videos[i];
if (!file.mimetype.startsWith('video/')) {
logger.info('file is not a video');
process.exit(0);
}
if (file.thumbnail) {
logger.info('thumbnail already exists');
process.exit(0);
}
const tmpFile = await loadFileTmp(file);
logger.debug(`loaded file to tmp: ${tmpFile}`);
const thumbnail = await loadThumbnail(tmpFile);
logger.debug(`loaded thumbnail: ${thumbnail.length} bytes mjpeg`);
const { thumbnail: thumb } = await prisma.file.update({
where: {
id: file.id,
},
data: {
thumbnail: {
create: {
name: `.thumb-${file.id}.jpg`,
},
},
},
select: {
thumbnail: true,
},
});
await datasource.save(thumb.name, thumbnail);
logger.info(`thumbnail saved - ${thumb.name}`);
logger.debug(`thumbnail ${JSON.stringify(thumb)}`);
logger.debug(`removing tmp file: ${tmpFile}`);
await rm(tmpFile);
}
await prisma.$disconnect();
process.exit(0);
}
start();

5
tsup.config.ts
View file

@ -19,6 +19,11 @@ export default defineConfig([
outDir: 'dist/worker',
...opts,
},
{
entryPoints: ['src/worker/thumbnail.ts'],
outDir: 'dist/worker',
...opts,
},
// scripts
{
entryPoints: ['src/scripts/import-dir.ts'],

86
yarn.lock
View file

@ -1142,6 +1142,18 @@ __metadata:
languageName: node
linkType: hard
"@derhuerst/http-basic@npm:^8.2.0":
version: 8.2.4
resolution: "@derhuerst/http-basic@npm:8.2.4"
dependencies:
caseless: ^0.12.0
concat-stream: ^2.0.0
http-response-object: ^3.0.1
parse-cache-control: ^1.0.1
checksum: dfb2f30c23fb907988d1c34318fa74c54dcd3c3ba6b4b0e64cdb584d03303ad212dd3b3874328a9367d7282a232976acbd33a20bb9c7a6ea20752e879459253b
languageName: node
linkType: hard
"@emotion/babel-plugin@npm:^11.10.6":
version: 11.10.6
resolution: "@emotion/babel-plugin@npm:11.10.6"
@ -2719,6 +2731,13 @@ __metadata:
languageName: node
linkType: hard
"@types/node@npm:^10.0.3":
version: 10.17.60
resolution: "@types/node@npm:10.17.60"
checksum: 2cdb3a77d071ba8513e5e8306fa64bf50e3c3302390feeaeff1fd325dd25c8441369715dfc8e3701011a72fed5958c7dfa94eb9239a81b3c286caa4d97db6eef
languageName: node
linkType: hard
"@types/node@npm:^17.0.45":
version: 17.0.45
resolution: "@types/node@npm:17.0.45"
@ -3822,9 +3841,16 @@ __metadata:
linkType: hard
"caniuse-lite@npm:^1.0.30001406":
version: 1.0.30001439
resolution: "caniuse-lite@npm:1.0.30001439"
checksum: 3912dd536c9735713ca85e47721988bbcefb881ddb4886b0b9923fa984247fd22cba032cf268e57d158af0e8a2ae2eae042ae01942a1d6d7849fa9fa5d62fb82
version: 1.0.30001494
resolution: "caniuse-lite@npm:1.0.30001494"
checksum: 770b742ebba6076da72e94f979ef609bbc855369d1b937c52227935d966b11c3b02baa6511fba04a804802b6eb22af0a2a4a82405963bbb769772530e6be7a8e
languageName: node
linkType: hard
"caseless@npm:^0.12.0":
version: 0.12.0
resolution: "caseless@npm:0.12.0"
checksum: b43bd4c440aa1e8ee6baefee8063b4850fd0d7b378f6aabc796c9ec8cb26d27fb30b46885350777d9bd079c5256c0e1329ad0dc7c2817e0bb466810ebb353751
languageName: node
linkType: hard
@ -4136,6 +4162,18 @@ __metadata:
languageName: node
linkType: hard
"concat-stream@npm:^2.0.0":
version: 2.0.0
resolution: "concat-stream@npm:2.0.0"
dependencies:
buffer-from: ^1.0.0
inherits: ^2.0.3
readable-stream: ^3.0.2
typedarray: ^0.0.6
checksum: d7f75d48f0ecd356c1545d87e22f57b488172811b1181d96021c7c4b14ab8855f5313280263dca44bb06e5222f274d047da3e290a38841ef87b59719bde967c7
languageName: node
linkType: hard
"console-control-strings@npm:^1.0.0, console-control-strings@npm:^1.1.0":
version: 1.1.0
resolution: "console-control-strings@npm:1.1.0"
@ -5625,6 +5663,18 @@ __metadata:
languageName: node
linkType: hard
"ffmpeg-static@npm:^5.1.0":
version: 5.1.0
resolution: "ffmpeg-static@npm:5.1.0"
dependencies:
"@derhuerst/http-basic": ^8.2.0
env-paths: ^2.2.0
https-proxy-agent: ^5.0.0
progress: ^2.0.3
checksum: 0e27d671a0be1f585ef03e48c2af7c2be14f4e61470ffa02e3b8919551243ee854028a898dfcd16cdf1e3c01916f3c5e9938f42cbc7e877d7dd80d566867db8b
languageName: node
linkType: hard
"file-entry-cache@npm:^6.0.1":
version: 6.0.1
resolution: "file-entry-cache@npm:6.0.1"
@ -6335,6 +6385,15 @@ __metadata:
languageName: node
linkType: hard
"http-response-object@npm:^3.0.1":
version: 3.0.2
resolution: "http-response-object@npm:3.0.2"
dependencies:
"@types/node": ^10.0.3
checksum: 6cbdcb4ce7b27c9158a131b772c903ed54add2ba831e29cc165e91c3969fa6f8105ddf924aac5b954b534ad15a1ae697b693331b2be5281ee24d79aae20c3264
languageName: node
linkType: hard
"https-proxy-agent@npm:5.0.1, https-proxy-agent@npm:^5.0.0":
version: 5.0.1
resolution: "https-proxy-agent@npm:5.0.1"
@ -8874,6 +8933,13 @@ __metadata:
languageName: node
linkType: hard
"parse-cache-control@npm:^1.0.1":
version: 1.0.1
resolution: "parse-cache-control@npm:1.0.1"
checksum: 5a70868792124eb07c2dd07a78fcb824102e972e908254e9e59ce59a4796c51705ff28196d2b20d3b7353d14e9f98e65ed0e4eda9be072cc99b5297dc0466fee
languageName: node
linkType: hard
"parse-json@npm:^4.0.0":
version: 4.0.0
resolution: "parse-json@npm:4.0.0"
@ -9301,7 +9367,7 @@ __metadata:
languageName: node
linkType: hard
"progress@npm:2.0.3":
"progress@npm:2.0.3, progress@npm:^2.0.3":
version: 2.0.3
resolution: "progress@npm:2.0.3"
checksum: f67403fe7b34912148d9252cb7481266a354bd99ce82c835f79070643bb3c6583d10dbcfda4d41e04bbc1d8437e9af0fb1e1f2135727878f5308682a579429b7
@ -9736,6 +9802,17 @@ __metadata:
languageName: node
linkType: hard
"readable-stream@npm:^3.0.2":
version: 3.6.2
resolution: "readable-stream@npm:3.6.2"
dependencies:
inherits: ^2.0.3
string_decoder: ^1.1.1
util-deprecate: ^1.0.1
checksum: bdcbe6c22e846b6af075e32cf8f4751c2576238c5043169a1c221c92ee2878458a816a4ea33f4c67623c0b6827c8a400409bfb3cf0bf3381392d0b1dfb52ac8d
languageName: node
linkType: hard
"readable-stream@npm:^4.0.0":
version: 4.2.0
resolution: "readable-stream@npm:4.2.0"
@ -11902,6 +11979,7 @@ __metadata:
fastify: ^4.15.0
fastify-plugin: ^4.5.0
fflate: ^0.7.4
ffmpeg-static: ^5.1.0
find-my-way: ^7.6.0
katex: ^0.16.4
mantine-datatable: ^2.2.6