From a467936e73fd2ca942d310607e469a7baf56a6bf Mon Sep 17 00:00:00 2001 From: Thanh Pham Date: Tue, 6 Sep 2022 02:45:38 +0700 Subject: [PATCH] feat(server): de-duplication (#557) * feat(server): remove un-used deviceAssetId cols. * feat(server): return 409 if asset is duplicated * feat(server): replace old unique constaint * feat(server): strip deviceId in file path * feat(server): skip duplicate asset * chore(server): revert changes * fix(server): asset test spec * fix(server): checksum generation for uploaded assets * fix(server): make sure generation queue run after migraion * feat(server): remove temp file * chore(server): remove dead code --- .../src/api-v1/asset/asset-repository.ts | 17 ++++++++++ .../src/api-v1/asset/asset.service.spec.ts | 1 + .../immich/src/api-v1/asset/asset.service.ts | 32 +++++++++++++------ .../immich/src/config/asset-upload.config.ts | 4 +-- .../src/microservices.service.ts | 4 ++- .../processors/generate-checksum.processor.ts | 4 --- .../database/src/entities/asset.entity.ts | 2 +- .../1661881837496-AddAssetChecksum.ts | 2 +- ...UpdateAssetTableWithNewUniqueConstraint.ts | 16 ++++++++++ 9 files changed, 64 insertions(+), 18 deletions(-) create mode 100644 server/libs/database/src/migrations/1661971370662-UpdateAssetTableWithNewUniqueConstraint.ts diff --git a/server/apps/immich/src/api-v1/asset/asset-repository.ts b/server/apps/immich/src/api-v1/asset/asset-repository.ts index 02d9116da7..fcbcb44748 100644 --- a/server/apps/immich/src/api-v1/asset/asset-repository.ts +++ b/server/apps/immich/src/api-v1/asset/asset-repository.ts @@ -26,6 +26,7 @@ export interface IAssetRepository { getSearchPropertiesByUserId(userId: string): Promise; getAssetCountByTimeBucket(userId: string, timeBucket: TimeGroupEnum): Promise; getAssetByTimeBucket(userId: string, getAssetByTimeBucketDto: GetAssetByTimeBucketDto): Promise; + getAssetByChecksum(userId: string, checksum: Buffer): Promise; } export const ASSET_REPOSITORY = 'ASSET_REPOSITORY'; @@ -208,4 +209,20 @@ export class AssetRepository implements IAssetRepository { return res; } + + /** + * Get asset by checksum on the database + * @param userId + * @param checksum + * + */ + getAssetByChecksum(userId: string, checksum: Buffer): Promise { + return this.assetRepository.findOneOrFail({ + where: { + userId, + checksum + }, + relations: ['exifInfo'], + }); + } } diff --git a/server/apps/immich/src/api-v1/asset/asset.service.spec.ts b/server/apps/immich/src/api-v1/asset/asset.service.spec.ts index acc1c63e77..0b164ded29 100644 --- a/server/apps/immich/src/api-v1/asset/asset.service.spec.ts +++ b/server/apps/immich/src/api-v1/asset/asset.service.spec.ts @@ -60,6 +60,7 @@ describe('AssetService', () => { getLocationsByUserId: jest.fn(), getSearchPropertiesByUserId: jest.fn(), getAssetByTimeBucket: jest.fn(), + getAssetByChecksum: jest.fn(), }; sui = new AssetService(assetRepositoryMock, a); diff --git a/server/apps/immich/src/api-v1/asset/asset.service.ts b/server/apps/immich/src/api-v1/asset/asset.service.ts index 2fbe75a56f..7d06a9ba47 100644 --- a/server/apps/immich/src/api-v1/asset/asset.service.ts +++ b/server/apps/immich/src/api-v1/asset/asset.service.ts @@ -10,7 +10,7 @@ import { } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { createHash } from 'node:crypto'; -import { Repository } from 'typeorm'; +import { QueryFailedError, Repository } from 'typeorm'; import { AuthUserDto } from '../../decorators/auth-user.decorator'; import { AssetEntity, AssetType } from '@app/database/entities/asset.entity'; import { constants, createReadStream, ReadStream, stat } from 'fs'; @@ -55,15 +55,29 @@ export class AssetService { mimeType: string, ): Promise { const checksum = await this.calculateChecksum(originalPath); - const assetEntity = await this._assetRepository.create( - createAssetDto, - authUser.id, - originalPath, - mimeType, - checksum, - ); - return assetEntity; + try { + const assetEntity = await this._assetRepository.create( + createAssetDto, + authUser.id, + originalPath, + mimeType, + checksum, + ); + + return assetEntity; + } catch (err) { + if (err instanceof QueryFailedError && (err as any).constraint === 'UQ_userid_checksum') { + const [assetEntity, _] = await Promise.all([ + this._assetRepository.getAssetByChecksum(authUser.id, checksum), + fs.unlink(originalPath) + ]); + + return assetEntity; + } + + throw err; + } } public async getUserAssetsByDeviceId(authUser: AuthUserDto, deviceId: string) { diff --git a/server/apps/immich/src/config/asset-upload.config.ts b/server/apps/immich/src/config/asset-upload.config.ts index befc8b2678..6cd77f270f 100644 --- a/server/apps/immich/src/config/asset-upload.config.ts +++ b/server/apps/immich/src/config/asset-upload.config.ts @@ -3,7 +3,7 @@ import { HttpException, HttpStatus } from '@nestjs/common'; import { MulterOptions } from '@nestjs/platform-express/multer/interfaces/multer-options.interface'; import { existsSync, mkdirSync } from 'fs'; import { diskStorage } from 'multer'; -import { extname } from 'path'; +import { extname, join } from 'path'; import { Request } from 'express'; import { randomUUID } from 'crypto'; @@ -29,7 +29,7 @@ export const assetUploadOption: MulterOptions = { return; } - const originalUploadFolder = `${basePath}/${req.user.id}/original/${req.body['deviceId']}`; + const originalUploadFolder = join(basePath, req.user.id, 'original', req.body['deviceId']); if (!existsSync(originalUploadFolder)) { mkdirSync(originalUploadFolder, { recursive: true }); diff --git a/server/apps/microservices/src/microservices.service.ts b/server/apps/microservices/src/microservices.service.ts index 39dd887e9d..e10fe46b83 100644 --- a/server/apps/microservices/src/microservices.service.ts +++ b/server/apps/microservices/src/microservices.service.ts @@ -12,6 +12,8 @@ export class MicroservicesService implements OnModuleInit { ) {} async onModuleInit() { - await this.generateChecksumQueue.add({}, { jobId: randomUUID() },); + await this.generateChecksumQueue.add({}, { + jobId: randomUUID(), delay: 10000 // wait for migration + }); } } diff --git a/server/apps/microservices/src/processors/generate-checksum.processor.ts b/server/apps/microservices/src/processors/generate-checksum.processor.ts index e7514de3c9..cebadc98cf 100644 --- a/server/apps/microservices/src/processors/generate-checksum.processor.ts +++ b/server/apps/microservices/src/processors/generate-checksum.processor.ts @@ -19,14 +19,12 @@ export class GenerateChecksumProcessor { async generateChecksum() { let hasNext = true; let pageSize = 200; - let offset = 0; while (hasNext) { const assets = await this.assetRepository.find({ where: { checksum: IsNull() }, - skip: offset, take: pageSize, }); @@ -43,8 +41,6 @@ export class GenerateChecksumProcessor { if (assets.length < pageSize) { hasNext = false; - } else { - offset += pageSize; } } } diff --git a/server/libs/database/src/entities/asset.entity.ts b/server/libs/database/src/entities/asset.entity.ts index d963d56a69..1cddedc79d 100644 --- a/server/libs/database/src/entities/asset.entity.ts +++ b/server/libs/database/src/entities/asset.entity.ts @@ -3,7 +3,7 @@ import { ExifEntity } from './exif.entity'; import { SmartInfoEntity } from './smart-info.entity'; @Entity('assets') -@Unique(['deviceAssetId', 'userId', 'deviceId']) +@Unique('UQ_userid_checksum', ['userId', 'checksum']) export class AssetEntity { @PrimaryGeneratedColumn('uuid') id!: string; diff --git a/server/libs/database/src/migrations/1661881837496-AddAssetChecksum.ts b/server/libs/database/src/migrations/1661881837496-AddAssetChecksum.ts index a5f1140d9a..5126961d1d 100644 --- a/server/libs/database/src/migrations/1661881837496-AddAssetChecksum.ts +++ b/server/libs/database/src/migrations/1661881837496-AddAssetChecksum.ts @@ -1,4 +1,4 @@ -import { MigrationInterface, QueryRunner } from "typeorm"; +import { MigrationInterface, QueryRunner } from 'typeorm'; export class AddAssetChecksum1661881837496 implements MigrationInterface { name = 'AddAssetChecksum1661881837496' diff --git a/server/libs/database/src/migrations/1661971370662-UpdateAssetTableWithNewUniqueConstraint.ts b/server/libs/database/src/migrations/1661971370662-UpdateAssetTableWithNewUniqueConstraint.ts new file mode 100644 index 0000000000..75cc3f6021 --- /dev/null +++ b/server/libs/database/src/migrations/1661971370662-UpdateAssetTableWithNewUniqueConstraint.ts @@ -0,0 +1,16 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class UpdateAssetTableWithNewUniqueConstraint1661971370662 implements MigrationInterface { + name = 'UpdateAssetTableWithNewUniqueConstraint1661971370662' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "assets" DROP CONSTRAINT "UQ_b599ab0bd9574958acb0b30a90e"`); + await queryRunner.query(`ALTER TABLE "assets" ADD CONSTRAINT "UQ_userid_checksum" UNIQUE ("userId", "checksum")`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "assets" DROP CONSTRAINT "UQ_userid_checksum"`); + await queryRunner.query(`ALTER TABLE "assets" ADD CONSTRAINT "UQ_b599ab0bd9574958acb0b30a90e" UNIQUE ("deviceAssetId", "userId", "deviceId")`); + } + +}