do e2e instead

e2e/docker-compose.yml

@@ -23,7 +23,6 @@ services:
       - IMMICH_ENV=testing
       - IMMICH_PORT=2285
       - IMMICH_IGNORE_MOUNT_CHECK_ERRORS=true
-      - IMMICH_LOG_LEVEL=verbose
     volumes:
       - ./test-assets:/test-assets
     extra_hosts:

e2e/src/api/specs/jobs.e2e-spec.ts

@@ -1,5 +1,5 @@
 import { JobCommand, JobName, LoginResponseDto, updateConfig } from '@immich/sdk';
-import { cpSync } from 'node:fs';
+import { cpSync, rmSync } from 'node:fs';
 import { readFile, writeFile } from 'node:fs/promises';
 import { basename } from 'node:path';
 import { errorDto } from 'src/responses';
@@ -32,6 +32,16 @@ describe('/jobs', () => {
         force: false,
       });
 
+      await utils.jobCommand(admin.accessToken, JobName.SmartSearch, {
+        command: JobCommand.Resume,
+        force: false,
+      });
+
+      await utils.jobCommand(admin.accessToken, JobName.DuplicateDetection, {
+        command: JobCommand.Resume,
+        force: false,
+      });
+
       const config = await utils.getSystemConfig(admin.accessToken);
       config.machineLearning.duplicateDetection.enabled = false;
       config.machineLearning.enabled = false;
@@ -47,14 +57,7 @@ describe('/jobs', () => {
     });
 
     it('should queue metadata extraction for missing assets', async () => {
-      const path1 = `${testAssetDir}/formats/raw/Nikon/D700/philadelphia.nef`;
-      const path2 = `${testAssetDir}/formats/raw/Nikon/D80/glarus.nef`;
-
-      await utils.createAsset(admin.accessToken, {
-        assetData: { bytes: await readFile(path1), filename: basename(path1) },
-      });
-
-      await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');
+      const path = `${testAssetDir}/formats/raw/Nikon/D700/philadelphia.nef`;
 
       await utils.jobCommand(admin.accessToken, JobName.MetadataExtraction, {
         command: JobCommand.Pause,
@@ -62,7 +65,7 @@ describe('/jobs', () => {
       });
 
       const { id } = await utils.createAsset(admin.accessToken, {
-        assetData: { bytes: await readFile(path2), filename: basename(path2) },
+        assetData: { bytes: await readFile(path), filename: basename(path) },
       });
 
       await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');
@@ -101,16 +104,45 @@ describe('/jobs', () => {
       }
     });
 
-    it('should queue thumbnail extraction for missing assets', async () => {
-      const path1 = `${testAssetDir}/formats/raw/Nikon/D700/philadelphia.nef`;
-      const path2 = `${testAssetDir}/formats/raw/Nikon/D80/glarus.nef`;
+    it('should not re-extract metadata for existing assets', async () => {
+      const path = `${testAssetDir}/temp/metadata/asset.jpg`;
 
-      await utils.createAsset(admin.accessToken, {
-        assetData: { bytes: await readFile(path1), filename: basename(path1) },
+      cpSync(`${testAssetDir}/formats/raw/Nikon/D700/philadelphia.nef`, path);
+
+      const { id } = await utils.createAsset(admin.accessToken, {
+        assetData: { bytes: await readFile(path), filename: basename(path) },
       });
 
-      await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);
-      await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);
+      await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');
+
+      {
+        const asset = await utils.getAssetInfo(admin.accessToken, id);
+
+        expect(asset.exifInfo).toBeDefined();
+        expect(asset.exifInfo?.model).toBe('NIKON D700');
+      }
+
+      cpSync(`${testAssetDir}/formats/raw/Nikon/D80/glarus.nef`, path);
+
+      await utils.jobCommand(admin.accessToken, JobName.MetadataExtraction, {
+        command: JobCommand.Start,
+        force: false,
+      });
+
+      await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');
+
+      {
+        const asset = await utils.getAssetInfo(admin.accessToken, id);
+
+        expect(asset.exifInfo).toBeDefined();
+        expect(asset.exifInfo?.model).toBe('NIKON D700');
+      }
+
+      rmSync(path);
+    });
+
+    it('should queue thumbnail extraction for assets missing thumbs', async () => {
+      const path = `${testAssetDir}/albums/nature/tanners_ridge.jpg`;
 
       await utils.jobCommand(admin.accessToken, JobName.ThumbnailGeneration, {
         command: JobCommand.Pause,
@@ -118,19 +150,16 @@ describe('/jobs', () => {
       });
 
       const { id } = await utils.createAsset(admin.accessToken, {
-        assetData: { bytes: await readFile(path2), filename: basename(path2) },
+        assetData: { bytes: await readFile(path), filename: basename(path) },
       });
 
       await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);
       await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);
 
-      {
-        const asset = await utils.getAssetInfo(admin.accessToken, id);
+      const assetBefore = await utils.getAssetInfo(admin.accessToken, id);
+      expect(assetBefore.thumbhash).toBeNull();
 
-        expect(asset.thumbhash).toBeNull();
-      }
-
-      await utils.jobCommand(admin.accessToken, JobName.MetadataExtraction, {
+      await utils.jobCommand(admin.accessToken, JobName.ThumbnailGeneration, {
         command: JobCommand.Empty,
         force: false,
       });
@@ -151,11 +180,46 @@ describe('/jobs', () => {
       await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);
       await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);
 
-      {
-        const asset = await utils.getAssetInfo(admin.accessToken, id);
+      const assetAfter = await utils.getAssetInfo(admin.accessToken, id);
+      expect(assetAfter.thumbhash).not.toBeNull();
+    });
 
-        expect(asset.thumbhash).not.toBeNull();
-      }
+    it('should not reload existing thumbnail when running thumb job for missing assets', async () => {
+      const path = `${testAssetDir}/temp/thumbs/asset1.jpg`;
+
+      cpSync(`${testAssetDir}/albums/nature/tanners_ridge.jpg`, path);
+
+      const { id } = await utils.createAsset(admin.accessToken, {
+        assetData: { bytes: await readFile(path), filename: basename(path) },
+      });
+
+      await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);
+      await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);
+
+      const assetBefore = await utils.getAssetInfo(admin.accessToken, id);
+
+      cpSync(`${testAssetDir}/albums/nature/notocactus_minimus.jpg`, path);
+
+      await utils.jobCommand(admin.accessToken, JobName.ThumbnailGeneration, {
+        command: JobCommand.Resume,
+        force: false,
+      });
+
+      // This runs the missing thumbnail job
+      await utils.jobCommand(admin.accessToken, JobName.ThumbnailGeneration, {
+        command: JobCommand.Start,
+        force: false,
+      });
+
+      await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);
+      await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);
+
+      const assetAfter = await utils.getAssetInfo(admin.accessToken, id);
+
+      // Asset 1 thumbnail should be untouched since its thumb should not have been reloaded, even though the file was changed
+      expect(assetAfter.thumbhash).toEqual(assetBefore.thumbhash);
+
+      rmSync(path);
     });
 
     it('should queue duplicate detection for missing duplicates', async () => {
@@ -235,12 +299,14 @@ describe('/jobs', () => {
 
         expect(asset1.duplicateId).toEqual(asset2.duplicateId);
       }
+
+      rmSync(`${testAssetDir}/temp/dupes/asset1.jpg`);
+      rmSync(`${testAssetDir}/temp/dupes/asset2.jpg`);
     }, 120_000);
 
     it('should queue smart search for missing assets', async () => {
       {
         const config = await utils.getSystemConfig(admin.accessToken);
-        config.machineLearning.duplicateDetection.enabled = false;
         config.machineLearning.enabled = false;
         config.machineLearning.clip.enabled = false;
         await updateConfig({ systemConfigDto: config }, { headers: asBearerAuth(admin.accessToken) });
@@ -248,7 +314,7 @@ describe('/jobs', () => {
 
       const path = `${testAssetDir}/albums/nature/prairie_falcon.jpg`;
 
-      const { id: id1 } = await utils.createAsset(admin.accessToken, {
+      await utils.createAsset(admin.accessToken, {
         assetData: { bytes: await readFile(path), filename: basename(path) },
       });
 
@@ -281,7 +347,49 @@ describe('/jobs', () => {
         const results = await utils.searchSmart(admin.accessToken, { query: 'bird' });
         expect(results.assets.count).toBeGreaterThanOrEqual(1);
       }
-    }, 120_000);
+    }, 60_000);
+
+    it('should not re-do smart search for already-indexed assets', async () => {
+      {
+        const config = await utils.getSystemConfig(admin.accessToken);
+        config.machineLearning.enabled = true;
+        config.machineLearning.clip.enabled = true;
+        await updateConfig({ systemConfigDto: config }, { headers: asBearerAuth(admin.accessToken) });
+      }
+
+      const path = `${testAssetDir}/temp/smart/asset.jpg`;
+
+      cpSync(`${testAssetDir}/albums/nature/tanners_ridge.jpg`, path);
+
+      await utils.createAsset(admin.accessToken, {
+        assetData: { bytes: await readFile(path), filename: basename(path) },
+      });
+
+      await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);
+      await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);
+      await utils.waitForQueueFinish(admin.accessToken, JobName.SmartSearch);
+
+      {
+        const results = await utils.searchSmart(admin.accessToken, { query: 'bird' });
+        expect(results.assets.count).toBeGreaterThanOrEqual(1);
+      }
+
+      cpSync(`${testAssetDir}/albums/nature/el_torcal_rocks.jpg`, path);
+
+      await utils.jobCommand(admin.accessToken, JobName.SmartSearch, {
+        command: JobCommand.Start,
+        force: false,
+      });
+
+      await utils.waitForQueueFinish(admin.accessToken, JobName.SmartSearch, 60_000);
+
+      {
+        const results = await utils.searchSmart(admin.accessToken, { query: 'bird' });
+        expect(results.assets.count).toBeGreaterThanOrEqual(1);
+      }
+
+      rmSync(path);
+    }, 60_000);
 
     it('should queue face detection for missing faces', async () => {
       const path = `${testAssetDir}/metadata/faces/solvay.jpg`;
@@ -350,6 +458,51 @@ describe('/jobs', () => {
         expect(asset.people).toEqual([]);
         expect(asset.unassignedFaces?.length).toBeGreaterThan(10);
       }
-    }, 120_000);
+    }, 60_000);
+
+    it('should not rerun face detection for existing faces', async () => {
+      const config = await utils.getSystemConfig(admin.accessToken);
+      config.metadata.faces.import = true;
+      config.machineLearning.enabled = true;
+      await updateConfig({ systemConfigDto: config }, { headers: asBearerAuth(admin.accessToken) });
+
+      const path = `${testAssetDir}/temp/faces/asset.jpg`;
+
+      cpSync(`${testAssetDir}/metadata/faces/solvay.jpg`, path);
+
+      const { id } = await utils.createAsset(admin.accessToken, {
+        assetData: { bytes: await readFile(path), filename: basename(path) },
+      });
+
+      await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);
+      await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);
+      await utils.waitForQueueFinish(admin.accessToken, JobName.FaceDetection);
+
+      {
+        const asset = await utils.getAssetInfo(admin.accessToken, id);
+        expect(asset.people).toEqual([]);
+        expect(asset.unassignedFaces?.length).toBeGreaterThan(10);
+      }
+
+      cpSync(`${testAssetDir}/albums/nature/el_torcal_rocks.jpg`, path);
+
+      await utils.jobCommand(admin.accessToken, JobName.FaceDetection, {
+        command: JobCommand.Start,
+        force: false,
+      });
+
+      await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);
+      await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);
+      await utils.waitForQueueFinish(admin.accessToken, JobName.FaceDetection, 60_000);
+
+      {
+        const asset = await utils.getAssetInfo(admin.accessToken, id);
+
+        expect(asset.people).toEqual([]);
+        expect(asset.unassignedFaces?.length).toBeGreaterThan(10);
+      }
+
+      rmSync(path);
+    }, 60_000);
   });
 });

e2e/src/utils.ts

@@ -13,7 +13,6 @@ import {
   PersonCreateDto,
   SharedLinkCreateDto,
   SmartSearchDto,
-  SystemConfigDto,
   UpdateLibraryDto,
   UserAdminCreateDto,
   UserPreferencesUpdateDto,
@@ -61,10 +60,9 @@ import { io, type Socket } from 'socket.io-client';
 import { loginDto, signupDto } from 'src/fixtures';
 import { makeRandomImage } from 'src/generators';
 import request from 'supertest';
-import { a } from 'vitest/dist/chunks/suite.B2jumIFP.js';
 
 type CommandResponse = { stdout: string; stderr: string; exitCode: number | null };
-type EventType = 'assetUpload' | 'assetUpdate' | 'assetDelete' | 'userDelete' | 'assetHidden' | 'configUpdate';
+type EventType = 'assetUpload' | 'assetUpdate' | 'assetDelete' | 'userDelete' | 'assetHidden';
 type WaitOptions = { event: EventType; id?: string; total?: number; timeout?: number };
 type AdminSetupOptions = { onboarding?: boolean };
 type FileData = { bytes?: Buffer; filename: string };
@@ -115,7 +113,6 @@ const events: Record<EventType, Set<string>> = {
   assetUpdate: new Set<string>(),
   assetDelete: new Set<string>(),
   userDelete: new Set<string>(),
-  configUpdate: new Set<string>(),
 };
 
 const idCallbacks: Record<string, () => void> = {};
@@ -123,18 +120,16 @@ const countCallbacks: Record<string, { count: number; callback: () => void }> =
 
 const execPromise = promisify(exec);
 
-const onEvent = ({ event, id }: { event: EventType; id?: string }) => {
+const onEvent = ({ event, id }: { event: EventType; id: string }) => {
   // console.log(`Received event: ${event} [id=${id}]`);
   const set = events[event];
 
-  if (id) {
-    set.add(id);
+  set.add(id);
 
-    const idCallback = idCallbacks[id];
-    if (idCallback) {
-      idCallback();
-      delete idCallbacks[id];
-    }
+  const idCallback = idCallbacks[id];
+  if (idCallback) {
+    idCallback();
+    delete idCallbacks[id];
   }
 
   const item = countCallbacks[event];
@@ -225,8 +220,6 @@ export const utils = {
         .on('on_asset_hidden', (assetId: string) => onEvent({ event: 'assetHidden', id: assetId }))
         .on('on_asset_delete', (assetId: string) => onEvent({ event: 'assetDelete', id: assetId }))
         .on('on_user_delete', (userId: string) => onEvent({ event: 'userDelete', id: userId }))
-        //.on('on_config_update', () => onEvent({ event: 'configUpdate' }))
-        .onAny((event, ...args) => console.log(`Received event: ${event}`, args))
         .connect();
     });
   },
@@ -249,10 +242,6 @@ export const utils = {
 
   waitForWebsocketEvent: ({ event, id, total: count, timeout: ms }: WaitOptions): Promise<void> => {
     return new Promise<void>((resolve, reject) => {
-      if (event === 'configUpdate') {
-        count = 1;
-      }
-
       if (!id && !count) {
         reject(new Error('id or count must be provided for waitForWebsocketEvent'));
       }

e2e/test-assets

@@ -1 +1 @@
-Subproject commit f75059bb2b953831babefe287518050b28fb2719
+Subproject commit 9e3b964b080dca6f035b29b86e66454ae8aeda78

server/src/repositories/asset.repository.ts

@@ -454,8 +454,8 @@ export class AssetRepository implements IAssetRepository {
       .selectAll('assets')
       .$if(property === WithoutProperty.DUPLICATE, (qb) =>
         qb
-          .leftJoin('asset_job_status as job_status', 'assets.id', 'job_status.assetId')
-          .where((eb) => eb.or([eb('job_status.duplicatesDetectedAt', 'is', null), eb('assetId', 'is', null)]))
+          .innerJoin('asset_job_status as job_status', 'assets.id', 'job_status.assetId')
+          .where('job_status.duplicatesDetectedAt', 'is', null)
           .where('job_status.previewAt', 'is not', null)
           .where((eb) => eb.exists(eb.selectFrom('smart_search').where('assetId', '=', eb.ref('assets.id'))))
           .where('assets.isVisible', '=', true),
@@ -473,9 +473,9 @@ export class AssetRepository implements IAssetRepository {
       )
       .$if(property === WithoutProperty.FACES, (qb) =>
         qb
-          .leftJoin('asset_job_status as job_status', 'assetId', 'assets.id')
-          .where((eb) => eb.or([eb('job_status.facesRecognizedAt', 'is', null), eb('assetId', 'is', null)]))
+          .innerJoin('asset_job_status as job_status', 'assetId', 'assets.id')
           .where('job_status.previewAt', 'is not', null)
+          .where('job_status.facesRecognizedAt', 'is', null)
           .where('assets.isVisible', '=', true),
       )
       .$if(property === WithoutProperty.SIDECAR, (qb) =>
@@ -485,8 +485,8 @@ export class AssetRepository implements IAssetRepository {
       )
       .$if(property === WithoutProperty.SMART_SEARCH, (qb) =>
         qb
-          .leftJoin('asset_job_status as job_status', 'assetId', 'assets.id')
-          .where((eb) => eb.or([eb('job_status.previewAt', 'is not', null), eb('assetId', 'is', null)]))
+          .innerJoin('asset_job_status as job_status', 'assetId', 'assets.id')
+          .where('job_status.previewAt', 'is not', null)
           .where('assets.isVisible', '=', true)
           .where((eb) =>
             eb.not((eb) => eb.exists(eb.selectFrom('smart_search').whereRef('assetId', '=', 'assets.id'))),
@@ -494,11 +494,10 @@ export class AssetRepository implements IAssetRepository {
       )
       .$if(property === WithoutProperty.THUMBNAIL, (qb) =>
         qb
-          .leftJoin('asset_job_status as job_status', 'assetId', 'assets.id')
+          .innerJoin('asset_job_status as job_status', 'assetId', 'assets.id')
           .where('assets.isVisible', '=', true)
           .where((eb) =>
             eb.or([
-              eb('assetId', 'is', null),
               eb('job_status.previewAt', 'is', null),
               eb('job_status.thumbnailAt', 'is', null),
               eb('assets.thumbhash', 'is', null),