0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-01-21 00:52:43 -05:00

feat(server): in upload folder, split the files into folders based on the first four of the files uuid (#6175)

This commit is contained in:
Zack Pollard 2024-01-04 20:45:16 +00:00 committed by GitHub
parent 2aaf941dda
commit 4cf1e553d2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 24 additions and 13 deletions

View file

@ -63,6 +63,7 @@ const uploadFile = {
auth: null, auth: null,
fieldName: UploadFieldName.ASSET_DATA, fieldName: UploadFieldName.ASSET_DATA,
file: { file: {
uuid: 'random-uuid',
checksum: Buffer.from('checksum', 'utf8'), checksum: Buffer.from('checksum', 'utf8'),
originalPath: 'upload/admin/image.jpeg', originalPath: 'upload/admin/image.jpeg',
originalName: 'image.jpeg', originalName: 'image.jpeg',
@ -73,6 +74,7 @@ const uploadFile = {
auth: authStub.admin, auth: authStub.admin,
fieldName, fieldName,
file: { file: {
uuid: 'random-uuid',
mimeType: 'image/jpeg', mimeType: 'image/jpeg',
checksum: Buffer.from('checksum', 'utf8'), checksum: Buffer.from('checksum', 'utf8'),
originalPath: `upload/admin/${filename}`, originalPath: `upload/admin/${filename}`,
@ -280,9 +282,9 @@ describe(AssetService.name, () => {
it('should return upload for everything else', () => { it('should return upload for everything else', () => {
expect(sut.getUploadFolder(uploadFile.filename(UploadFieldName.ASSET_DATA, 'image.jpg'))).toEqual( expect(sut.getUploadFolder(uploadFile.filename(UploadFieldName.ASSET_DATA, 'image.jpg'))).toEqual(
'upload/upload/admin_id', 'upload/upload/admin_id/ra/nd',
); );
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/upload/admin_id'); expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/upload/admin_id/ra/nd');
}); });
}); });

View file

@ -71,6 +71,7 @@ export interface UploadRequest {
} }
export interface UploadFile { export interface UploadFile {
uuid: string;
checksum: Buffer; checksum: Buffer;
originalPath: string; originalPath: string;
originalName: string; originalName: string;
@ -170,13 +171,13 @@ export class AssetService {
[UploadFieldName.PROFILE_DATA]: originalExt, [UploadFieldName.PROFILE_DATA]: originalExt,
}; };
return sanitize(`${this.cryptoRepository.randomUUID()}${lookup[fieldName]}`); return sanitize(`${file.uuid}${lookup[fieldName]}`);
} }
getUploadFolder({ auth, fieldName }: UploadRequest): string { getUploadFolder({ auth, fieldName, file }: UploadRequest): string {
auth = this.access.requireUploadAccess(auth); auth = this.access.requireUploadAccess(auth);
let folder = StorageCore.getFolderLocation(StorageFolder.UPLOAD, auth.user.id); let folder = StorageCore.getNestedFolder(StorageFolder.UPLOAD, auth.user.id, file.uuid);
if (fieldName === UploadFieldName.PROFILE_DATA) { if (fieldName === UploadFieldName.PROFILE_DATA) {
folder = StorageCore.getFolderLocation(StorageFolder.PROFILE, auth.user.id); folder = StorageCore.getFolderLocation(StorageFolder.PROFILE, auth.user.id);
} }

View file

@ -281,12 +281,11 @@ export class StorageCore {
} }
} }
private static getNestedPath(folder: StorageFolder, ownerId: string, filename: string): string { static getNestedFolder(folder: StorageFolder, ownerId: string, filename: string): string {
return join( return join(StorageCore.getFolderLocation(folder, ownerId), filename.substring(0, 2), filename.substring(2, 4));
StorageCore.getFolderLocation(folder, ownerId), }
filename.substring(0, 2),
filename.substring(2, 4), static getNestedPath(folder: StorageFolder, ownerId: string, filename: string): string {
filename, return join(this.getNestedFolder(folder, ownerId, filename), filename);
);
} }
} }

View file

@ -122,6 +122,7 @@ describe('AssetService', () => {
it('should handle a file upload', async () => { it('should handle a file upload', async () => {
const assetEntity = _getAsset_1(); const assetEntity = _getAsset_1();
const file = { const file = {
uuid: 'random-uuid',
originalPath: 'fake_path/asset_1.jpeg', originalPath: 'fake_path/asset_1.jpeg',
mimeType: 'image/jpeg', mimeType: 'image/jpeg',
checksum: Buffer.from('file hash', 'utf8'), checksum: Buffer.from('file hash', 'utf8'),
@ -139,6 +140,7 @@ describe('AssetService', () => {
it('should handle a duplicate', async () => { it('should handle a duplicate', async () => {
const file = { const file = {
uuid: 'random-uuid',
originalPath: 'fake_path/asset_1.jpeg', originalPath: 'fake_path/asset_1.jpeg',
mimeType: 'image/jpeg', mimeType: 'image/jpeg',
checksum: Buffer.from('file hash', 'utf8'), checksum: Buffer.from('file hash', 'utf8'),

View file

@ -4,7 +4,7 @@ import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from '@nes
import { PATH_METADATA } from '@nestjs/common/constants'; import { PATH_METADATA } from '@nestjs/common/constants';
import { Reflector } from '@nestjs/core'; import { Reflector } from '@nestjs/core';
import { transformException } from '@nestjs/platform-express/multer/multer/multer.utils'; import { transformException } from '@nestjs/platform-express/multer/multer/multer.utils';
import { createHash } from 'crypto'; import { createHash, randomUUID } from 'crypto';
import { NextFunction, RequestHandler } from 'express'; import { NextFunction, RequestHandler } from 'express';
import multer, { StorageEngine, diskStorage } from 'multer'; import multer, { StorageEngine, diskStorage } from 'multer';
import { Observable } from 'rxjs'; import { Observable } from 'rxjs';
@ -17,11 +17,13 @@ export enum Route {
export interface ImmichFile extends Express.Multer.File { export interface ImmichFile extends Express.Multer.File {
/** sha1 hash of file */ /** sha1 hash of file */
uuid: string;
checksum: Buffer; checksum: Buffer;
} }
export function mapToUploadFile(file: ImmichFile): UploadFile { export function mapToUploadFile(file: ImmichFile): UploadFile {
return { return {
uuid: file.uuid,
checksum: file.checksum, checksum: file.checksum,
originalPath: file.path, originalPath: file.path,
originalName: Buffer.from(file.originalname, 'latin1').toString('utf8'), originalName: Buffer.from(file.originalname, 'latin1').toString('utf8'),
@ -30,6 +32,8 @@ export function mapToUploadFile(file: ImmichFile): UploadFile {
type DiskStorageCallback = (error: Error | null, result: string) => void; type DiskStorageCallback = (error: Error | null, result: string) => void;
type ImmichMulterFile = Express.Multer.File & { uuid: string };
interface Callback<T> { interface Callback<T> {
(error: Error): void; (error: Error): void;
(error: null, result: T): void; (error: null, result: T): void;
@ -118,6 +122,7 @@ export class FileUploadInterceptor implements NestInterceptor {
} }
private handleFile(req: AuthRequest, file: Express.Multer.File, callback: Callback<Partial<ImmichFile>>) { private handleFile(req: AuthRequest, file: Express.Multer.File, callback: Callback<Partial<ImmichFile>>) {
(file as ImmichMulterFile).uuid = randomUUID();
if (!this.isAssetUploadFile(file)) { if (!this.isAssetUploadFile(file)) {
this.defaultStorage._handleFile(req, file, callback); this.defaultStorage._handleFile(req, file, callback);
return; return;

View file

@ -1,10 +1,12 @@
export const fileStub = { export const fileStub = {
livePhotoStill: Object.freeze({ livePhotoStill: Object.freeze({
uuid: 'random-uuid',
originalPath: 'fake_path/asset_1.jpeg', originalPath: 'fake_path/asset_1.jpeg',
checksum: Buffer.from('file hash', 'utf8'), checksum: Buffer.from('file hash', 'utf8'),
originalName: 'asset_1.jpeg', originalName: 'asset_1.jpeg',
}), }),
livePhotoMotion: Object.freeze({ livePhotoMotion: Object.freeze({
uuid: 'random-uuid',
originalPath: 'fake_path/asset_1.mp4', originalPath: 'fake_path/asset_1.mp4',
checksum: Buffer.from('live photo file hash', 'utf8'), checksum: Buffer.from('live photo file hash', 'utf8'),
originalName: 'asset_1.mp4', originalName: 'asset_1.mp4',