diff --git a/machine-learning/src/main.py b/machine-learning/src/main.py index 6243eef74b..8d7a5f6976 100644 --- a/machine-learning/src/main.py +++ b/machine-learning/src/main.py @@ -22,7 +22,6 @@ class ClipRequestBody(BaseModel): classification_model = os.getenv( "MACHINE_LEARNING_CLASSIFICATION_MODEL", "microsoft/resnet-50" ) -object_model = os.getenv("MACHINE_LEARNING_OBJECT_MODEL", "hustvl/yolos-tiny") clip_image_model = os.getenv("MACHINE_LEARNING_CLIP_IMAGE_MODEL", "clip-ViT-B-32") clip_text_model = os.getenv("MACHINE_LEARNING_CLIP_TEXT_MODEL", "clip-ViT-B-32") facial_recognition_model = os.getenv( @@ -39,7 +38,6 @@ app = FastAPI() @app.on_event("startup") async def startup_event(): # Get all models - _get_model(object_model, "object-detection") _get_model(classification_model, "image-classification") _get_model(clip_image_model) _get_model(clip_text_model) @@ -55,14 +53,6 @@ async def root(): def ping(): return "pong" - -@app.post("/object-detection/detect-object", status_code=200) -def object_detection(payload: MlRequestBody): - model = _get_model(object_model, "object-detection") - assetPath = payload.thumbnailPath - return run_engine(model, assetPath) - - @app.post("/image-classifier/tag-image", status_code=200) def image_classification(payload: MlRequestBody): model = _get_model(classification_model, "image-classification") diff --git a/server/apps/microservices/src/processor.service.ts b/server/apps/microservices/src/processor.service.ts index 27da2bed0d..082b9adcdf 100644 --- a/server/apps/microservices/src/processor.service.ts +++ b/server/apps/microservices/src/processor.service.ts @@ -52,7 +52,6 @@ export class ProcessorService { [JobName.USER_DELETE_CHECK]: () => this.userService.handleUserDeleteCheck(), [JobName.USER_DELETION]: (data) => this.userService.handleUserDelete(data), [JobName.QUEUE_OBJECT_TAGGING]: (data) => this.smartInfoService.handleQueueObjectTagging(data), - [JobName.DETECT_OBJECTS]: (data) => this.smartInfoService.handleDetectObjects(data), [JobName.CLASSIFY_IMAGE]: (data) => this.smartInfoService.handleClassifyImage(data), [JobName.QUEUE_ENCODE_CLIP]: (data) => this.smartInfoService.handleQueueEncodeClip(data), [JobName.ENCODE_CLIP]: (data) => this.smartInfoService.handleEncodeClip(data), diff --git a/server/libs/domain/src/job/job.constants.ts b/server/libs/domain/src/job/job.constants.ts index 169d3cbf6f..88f7a853b6 100644 --- a/server/libs/domain/src/job/job.constants.ts +++ b/server/libs/domain/src/job/job.constants.ts @@ -43,7 +43,6 @@ export enum JobName { // object tagging QUEUE_OBJECT_TAGGING = 'queue-object-tagging', - DETECT_OBJECTS = 'detect-objects', CLASSIFY_IMAGE = 'classify-image', // facial recognition @@ -105,7 +104,6 @@ export const JOBS_TO_QUEUE: Record = { // object tagging [JobName.QUEUE_OBJECT_TAGGING]: QueueName.OBJECT_TAGGING, - [JobName.DETECT_OBJECTS]: QueueName.OBJECT_TAGGING, [JobName.CLASSIFY_IMAGE]: QueueName.OBJECT_TAGGING, // facial recognition diff --git a/server/libs/domain/src/job/job.repository.ts b/server/libs/domain/src/job/job.repository.ts index 1433162f3e..d88123ed50 100644 --- a/server/libs/domain/src/job/job.repository.ts +++ b/server/libs/domain/src/job/job.repository.ts @@ -52,7 +52,6 @@ export type JobItem = // Object Tagging | { name: JobName.QUEUE_OBJECT_TAGGING; data: IBaseJob } - | { name: JobName.DETECT_OBJECTS; data: IEntityJob } | { name: JobName.CLASSIFY_IMAGE; data: IEntityJob } // Recognize Faces diff --git a/server/libs/domain/src/job/job.service.ts b/server/libs/domain/src/job/job.service.ts index 26c044e7ad..3d09f27b3e 100644 --- a/server/libs/domain/src/job/job.service.ts +++ b/server/libs/domain/src/job/job.service.ts @@ -119,7 +119,6 @@ export class JobService { case JobName.GENERATE_JPEG_THUMBNAIL: { await this.jobRepository.queue({ name: JobName.GENERATE_WEBP_THUMBNAIL, data: item.data }); await this.jobRepository.queue({ name: JobName.CLASSIFY_IMAGE, data: item.data }); - await this.jobRepository.queue({ name: JobName.DETECT_OBJECTS, data: item.data }); await this.jobRepository.queue({ name: JobName.ENCODE_CLIP, data: item.data }); await this.jobRepository.queue({ name: JobName.RECOGNIZE_FACES, data: item.data }); @@ -134,7 +133,6 @@ export class JobService { // In addition to the above jobs, all of these should queue `SEARCH_INDEX_ASSET` switch (item.name) { case JobName.CLASSIFY_IMAGE: - case JobName.DETECT_OBJECTS: case JobName.ENCODE_CLIP: case JobName.RECOGNIZE_FACES: case JobName.METADATA_EXTRACTION: diff --git a/server/libs/domain/src/smart-info/machine-learning.interface.ts b/server/libs/domain/src/smart-info/machine-learning.interface.ts index 7fb06d29ed..bbcf62adf1 100644 --- a/server/libs/domain/src/smart-info/machine-learning.interface.ts +++ b/server/libs/domain/src/smart-info/machine-learning.interface.ts @@ -21,7 +21,6 @@ export interface DetectFaceResult { export interface IMachineLearningRepository { classifyImage(input: MachineLearningInput): Promise; - detectObjects(input: MachineLearningInput): Promise; encodeImage(input: MachineLearningInput): Promise; encodeText(input: string): Promise; detectFaces(input: MachineLearningInput): Promise; diff --git a/server/libs/domain/src/smart-info/smart-info.service.spec.ts b/server/libs/domain/src/smart-info/smart-info.service.spec.ts index 0ba89769b7..2af80cc9f3 100644 --- a/server/libs/domain/src/smart-info/smart-info.service.spec.ts +++ b/server/libs/domain/src/smart-info/smart-info.service.spec.ts @@ -49,7 +49,6 @@ describe(SmartInfoService.name, () => { expect(jobMock.queue.mock.calls).toEqual([ [{ name: JobName.CLASSIFY_IMAGE, data: { id: assetEntityStub.image.id } }], - [{ name: JobName.DETECT_OBJECTS, data: { id: assetEntityStub.image.id } }], ]); expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.OBJECT_TAGS); }); @@ -64,7 +63,6 @@ describe(SmartInfoService.name, () => { expect(jobMock.queue.mock.calls).toEqual([ [{ name: JobName.CLASSIFY_IMAGE, data: { id: assetEntityStub.image.id } }], - [{ name: JobName.DETECT_OBJECTS, data: { id: assetEntityStub.image.id } }], ]); expect(assetMock.getAll).toHaveBeenCalled(); }); @@ -103,39 +101,6 @@ describe(SmartInfoService.name, () => { }); }); - describe('handleDetectObjects', () => { - it('should skip assets without a resize path', async () => { - const asset = { resizePath: '' } as AssetEntity; - assetMock.getByIds.mockResolvedValue([asset]); - - await sut.handleDetectObjects({ id: asset.id }); - - expect(smartMock.upsert).not.toHaveBeenCalled(); - expect(machineMock.detectObjects).not.toHaveBeenCalled(); - }); - - it('should save the returned objects', async () => { - machineMock.detectObjects.mockResolvedValue(['obj1', 'obj2', 'obj3']); - - await sut.handleDetectObjects({ id: asset.id }); - - expect(machineMock.detectObjects).toHaveBeenCalledWith({ thumbnailPath: 'path/to/resize.ext' }); - expect(smartMock.upsert).toHaveBeenCalledWith({ - assetId: 'asset-1', - objects: ['obj1', 'obj2', 'obj3'], - }); - }); - - it('should no update the smart info if no objects were returned', async () => { - machineMock.detectObjects.mockResolvedValue([]); - - await sut.handleDetectObjects({ id: asset.id }); - - expect(machineMock.detectObjects).toHaveBeenCalled(); - expect(smartMock.upsert).not.toHaveBeenCalled(); - }); - }); - describe('handleQueueEncodeClip', () => { it('should queue the assets without clip embeddings', async () => { assetMock.getWithout.mockResolvedValue({ diff --git a/server/libs/domain/src/smart-info/smart-info.service.ts b/server/libs/domain/src/smart-info/smart-info.service.ts index cd84b97f97..a07ad0a6cb 100644 --- a/server/libs/domain/src/smart-info/smart-info.service.ts +++ b/server/libs/domain/src/smart-info/smart-info.service.ts @@ -27,30 +27,12 @@ export class SmartInfoService { for await (const assets of assetPagination) { for (const asset of assets) { await this.jobRepository.queue({ name: JobName.CLASSIFY_IMAGE, data: { id: asset.id } }); - await this.jobRepository.queue({ name: JobName.DETECT_OBJECTS, data: { id: asset.id } }); } } return true; } - async handleDetectObjects({ id }: IEntityJob) { - const [asset] = await this.assetRepository.getByIds([id]); - - if (!MACHINE_LEARNING_ENABLED || !asset.resizePath) { - return false; - } - - const objects = await this.machineLearning.detectObjects({ thumbnailPath: asset.resizePath }); - if (objects.length === 0) { - return false; - } - - await this.repository.upsert({ assetId: asset.id, objects }); - - return true; - } - async handleClassifyImage({ id }: IEntityJob) { const [asset] = await this.assetRepository.getByIds([id]); diff --git a/server/libs/domain/test/machine-learning.repository.mock.ts b/server/libs/domain/test/machine-learning.repository.mock.ts index 0988ce8dca..13e335e697 100644 --- a/server/libs/domain/test/machine-learning.repository.mock.ts +++ b/server/libs/domain/test/machine-learning.repository.mock.ts @@ -3,7 +3,6 @@ import { IMachineLearningRepository } from '../src'; export const newMachineLearningRepositoryMock = (): jest.Mocked => { return { classifyImage: jest.fn(), - detectObjects: jest.fn(), encodeImage: jest.fn(), encodeText: jest.fn(), detectFaces: jest.fn(), diff --git a/server/libs/infra/src/repositories/machine-learning.repository.ts b/server/libs/infra/src/repositories/machine-learning.repository.ts index 4931b82c43..0f8d1707bd 100644 --- a/server/libs/infra/src/repositories/machine-learning.repository.ts +++ b/server/libs/infra/src/repositories/machine-learning.repository.ts @@ -14,10 +14,6 @@ export class MachineLearningRepository implements IMachineLearningRepository { return client.post('/facial-recognition/detect-faces', input).then((res) => res.data); } - detectObjects(input: MachineLearningInput): Promise { - return client.post('/object-detection/detect-object', input).then((res) => res.data); - } - encodeImage(input: MachineLearningInput): Promise { return client.post('/sentence-transformer/encode-image', input).then((res) => res.data); }