mirror of
https://github.com/immich-app/immich.git
synced 2025-01-21 00:52:43 -05:00
feat(server): in upload folder, split the files into folders based on the first four of the files uuid (#6175)
This commit is contained in:
parent
2aaf941dda
commit
4cf1e553d2
6 changed files with 24 additions and 13 deletions
|
@ -63,6 +63,7 @@ const uploadFile = {
|
||||||
auth: null,
|
auth: null,
|
||||||
fieldName: UploadFieldName.ASSET_DATA,
|
fieldName: UploadFieldName.ASSET_DATA,
|
||||||
file: {
|
file: {
|
||||||
|
uuid: 'random-uuid',
|
||||||
checksum: Buffer.from('checksum', 'utf8'),
|
checksum: Buffer.from('checksum', 'utf8'),
|
||||||
originalPath: 'upload/admin/image.jpeg',
|
originalPath: 'upload/admin/image.jpeg',
|
||||||
originalName: 'image.jpeg',
|
originalName: 'image.jpeg',
|
||||||
|
@ -73,6 +74,7 @@ const uploadFile = {
|
||||||
auth: authStub.admin,
|
auth: authStub.admin,
|
||||||
fieldName,
|
fieldName,
|
||||||
file: {
|
file: {
|
||||||
|
uuid: 'random-uuid',
|
||||||
mimeType: 'image/jpeg',
|
mimeType: 'image/jpeg',
|
||||||
checksum: Buffer.from('checksum', 'utf8'),
|
checksum: Buffer.from('checksum', 'utf8'),
|
||||||
originalPath: `upload/admin/${filename}`,
|
originalPath: `upload/admin/${filename}`,
|
||||||
|
@ -280,9 +282,9 @@ describe(AssetService.name, () => {
|
||||||
|
|
||||||
it('should return upload for everything else', () => {
|
it('should return upload for everything else', () => {
|
||||||
expect(sut.getUploadFolder(uploadFile.filename(UploadFieldName.ASSET_DATA, 'image.jpg'))).toEqual(
|
expect(sut.getUploadFolder(uploadFile.filename(UploadFieldName.ASSET_DATA, 'image.jpg'))).toEqual(
|
||||||
'upload/upload/admin_id',
|
'upload/upload/admin_id/ra/nd',
|
||||||
);
|
);
|
||||||
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/upload/admin_id');
|
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/upload/admin_id/ra/nd');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
@ -71,6 +71,7 @@ export interface UploadRequest {
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface UploadFile {
|
export interface UploadFile {
|
||||||
|
uuid: string;
|
||||||
checksum: Buffer;
|
checksum: Buffer;
|
||||||
originalPath: string;
|
originalPath: string;
|
||||||
originalName: string;
|
originalName: string;
|
||||||
|
@ -170,13 +171,13 @@ export class AssetService {
|
||||||
[UploadFieldName.PROFILE_DATA]: originalExt,
|
[UploadFieldName.PROFILE_DATA]: originalExt,
|
||||||
};
|
};
|
||||||
|
|
||||||
return sanitize(`${this.cryptoRepository.randomUUID()}${lookup[fieldName]}`);
|
return sanitize(`${file.uuid}${lookup[fieldName]}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
getUploadFolder({ auth, fieldName }: UploadRequest): string {
|
getUploadFolder({ auth, fieldName, file }: UploadRequest): string {
|
||||||
auth = this.access.requireUploadAccess(auth);
|
auth = this.access.requireUploadAccess(auth);
|
||||||
|
|
||||||
let folder = StorageCore.getFolderLocation(StorageFolder.UPLOAD, auth.user.id);
|
let folder = StorageCore.getNestedFolder(StorageFolder.UPLOAD, auth.user.id, file.uuid);
|
||||||
if (fieldName === UploadFieldName.PROFILE_DATA) {
|
if (fieldName === UploadFieldName.PROFILE_DATA) {
|
||||||
folder = StorageCore.getFolderLocation(StorageFolder.PROFILE, auth.user.id);
|
folder = StorageCore.getFolderLocation(StorageFolder.PROFILE, auth.user.id);
|
||||||
}
|
}
|
||||||
|
|
|
@ -281,12 +281,11 @@ export class StorageCore {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static getNestedPath(folder: StorageFolder, ownerId: string, filename: string): string {
|
static getNestedFolder(folder: StorageFolder, ownerId: string, filename: string): string {
|
||||||
return join(
|
return join(StorageCore.getFolderLocation(folder, ownerId), filename.substring(0, 2), filename.substring(2, 4));
|
||||||
StorageCore.getFolderLocation(folder, ownerId),
|
}
|
||||||
filename.substring(0, 2),
|
|
||||||
filename.substring(2, 4),
|
static getNestedPath(folder: StorageFolder, ownerId: string, filename: string): string {
|
||||||
filename,
|
return join(this.getNestedFolder(folder, ownerId, filename), filename);
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -122,6 +122,7 @@ describe('AssetService', () => {
|
||||||
it('should handle a file upload', async () => {
|
it('should handle a file upload', async () => {
|
||||||
const assetEntity = _getAsset_1();
|
const assetEntity = _getAsset_1();
|
||||||
const file = {
|
const file = {
|
||||||
|
uuid: 'random-uuid',
|
||||||
originalPath: 'fake_path/asset_1.jpeg',
|
originalPath: 'fake_path/asset_1.jpeg',
|
||||||
mimeType: 'image/jpeg',
|
mimeType: 'image/jpeg',
|
||||||
checksum: Buffer.from('file hash', 'utf8'),
|
checksum: Buffer.from('file hash', 'utf8'),
|
||||||
|
@ -139,6 +140,7 @@ describe('AssetService', () => {
|
||||||
|
|
||||||
it('should handle a duplicate', async () => {
|
it('should handle a duplicate', async () => {
|
||||||
const file = {
|
const file = {
|
||||||
|
uuid: 'random-uuid',
|
||||||
originalPath: 'fake_path/asset_1.jpeg',
|
originalPath: 'fake_path/asset_1.jpeg',
|
||||||
mimeType: 'image/jpeg',
|
mimeType: 'image/jpeg',
|
||||||
checksum: Buffer.from('file hash', 'utf8'),
|
checksum: Buffer.from('file hash', 'utf8'),
|
||||||
|
|
|
@ -4,7 +4,7 @@ import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from '@nes
|
||||||
import { PATH_METADATA } from '@nestjs/common/constants';
|
import { PATH_METADATA } from '@nestjs/common/constants';
|
||||||
import { Reflector } from '@nestjs/core';
|
import { Reflector } from '@nestjs/core';
|
||||||
import { transformException } from '@nestjs/platform-express/multer/multer/multer.utils';
|
import { transformException } from '@nestjs/platform-express/multer/multer/multer.utils';
|
||||||
import { createHash } from 'crypto';
|
import { createHash, randomUUID } from 'crypto';
|
||||||
import { NextFunction, RequestHandler } from 'express';
|
import { NextFunction, RequestHandler } from 'express';
|
||||||
import multer, { StorageEngine, diskStorage } from 'multer';
|
import multer, { StorageEngine, diskStorage } from 'multer';
|
||||||
import { Observable } from 'rxjs';
|
import { Observable } from 'rxjs';
|
||||||
|
@ -17,11 +17,13 @@ export enum Route {
|
||||||
|
|
||||||
export interface ImmichFile extends Express.Multer.File {
|
export interface ImmichFile extends Express.Multer.File {
|
||||||
/** sha1 hash of file */
|
/** sha1 hash of file */
|
||||||
|
uuid: string;
|
||||||
checksum: Buffer;
|
checksum: Buffer;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function mapToUploadFile(file: ImmichFile): UploadFile {
|
export function mapToUploadFile(file: ImmichFile): UploadFile {
|
||||||
return {
|
return {
|
||||||
|
uuid: file.uuid,
|
||||||
checksum: file.checksum,
|
checksum: file.checksum,
|
||||||
originalPath: file.path,
|
originalPath: file.path,
|
||||||
originalName: Buffer.from(file.originalname, 'latin1').toString('utf8'),
|
originalName: Buffer.from(file.originalname, 'latin1').toString('utf8'),
|
||||||
|
@ -30,6 +32,8 @@ export function mapToUploadFile(file: ImmichFile): UploadFile {
|
||||||
|
|
||||||
type DiskStorageCallback = (error: Error | null, result: string) => void;
|
type DiskStorageCallback = (error: Error | null, result: string) => void;
|
||||||
|
|
||||||
|
type ImmichMulterFile = Express.Multer.File & { uuid: string };
|
||||||
|
|
||||||
interface Callback<T> {
|
interface Callback<T> {
|
||||||
(error: Error): void;
|
(error: Error): void;
|
||||||
(error: null, result: T): void;
|
(error: null, result: T): void;
|
||||||
|
@ -118,6 +122,7 @@ export class FileUploadInterceptor implements NestInterceptor {
|
||||||
}
|
}
|
||||||
|
|
||||||
private handleFile(req: AuthRequest, file: Express.Multer.File, callback: Callback<Partial<ImmichFile>>) {
|
private handleFile(req: AuthRequest, file: Express.Multer.File, callback: Callback<Partial<ImmichFile>>) {
|
||||||
|
(file as ImmichMulterFile).uuid = randomUUID();
|
||||||
if (!this.isAssetUploadFile(file)) {
|
if (!this.isAssetUploadFile(file)) {
|
||||||
this.defaultStorage._handleFile(req, file, callback);
|
this.defaultStorage._handleFile(req, file, callback);
|
||||||
return;
|
return;
|
||||||
|
|
2
server/test/fixtures/file.stub.ts
vendored
2
server/test/fixtures/file.stub.ts
vendored
|
@ -1,10 +1,12 @@
|
||||||
export const fileStub = {
|
export const fileStub = {
|
||||||
livePhotoStill: Object.freeze({
|
livePhotoStill: Object.freeze({
|
||||||
|
uuid: 'random-uuid',
|
||||||
originalPath: 'fake_path/asset_1.jpeg',
|
originalPath: 'fake_path/asset_1.jpeg',
|
||||||
checksum: Buffer.from('file hash', 'utf8'),
|
checksum: Buffer.from('file hash', 'utf8'),
|
||||||
originalName: 'asset_1.jpeg',
|
originalName: 'asset_1.jpeg',
|
||||||
}),
|
}),
|
||||||
livePhotoMotion: Object.freeze({
|
livePhotoMotion: Object.freeze({
|
||||||
|
uuid: 'random-uuid',
|
||||||
originalPath: 'fake_path/asset_1.mp4',
|
originalPath: 'fake_path/asset_1.mp4',
|
||||||
checksum: Buffer.from('live photo file hash', 'utf8'),
|
checksum: Buffer.from('live photo file hash', 'utf8'),
|
||||||
originalName: 'asset_1.mp4',
|
originalName: 'asset_1.mp4',
|
||||||
|
|
Loading…
Add table
Reference in a new issue