wip

server/src/interfaces/asset.interface.ts

@@ -1,7 +1,6 @@
 import { Insertable, Updateable, UpdateResult } from 'kysely';
 import { AssetJobStatus, Assets, Exif } from 'src/db';
 import { AssetEntity } from 'src/entities/asset.entity';
-import { LibraryEntity } from 'src/entities/library.entity';
 import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum';
 import { AssetSearchOptions, SearchExploreItem } from 'src/interfaces/search.interface';
 import { Paginated, PaginationOptions } from 'src/utils/pagination';
@@ -171,7 +170,11 @@ export interface IAssetRepository {
   getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise<AssetEntity[]>;
   upsertFile(file: UpsertFileOptions): Promise<void>;
   upsertFiles(files: UpsertFileOptions[]): Promise<void>;
-  detectOfflineExternalAssets(library: LibraryEntity): Promise<UpdateResult>;
+  detectOfflineExternalAssets(
+    libraryId: string,
+    importPaths: string[],
+    exclusionPatterns: string[],
+  ): Promise<UpdateResult>;
   filterNewExternalAssetPaths(libraryId: string, paths: string[]): Promise<string[]>;
   getLibraryAssetCount(options: AssetSearchOptions): Promise<number | undefined>;
 }

server/src/main.ts

@@ -13,7 +13,7 @@ if (immichApp) {
 let apiProcess: ChildProcess | undefined;
 
 const onError = (name: string, error: Error) => {
-  console.error(`${name} worker error: ${error}`);
+  console.error(`${name} worker error: ${error}, stack: ${error.stack}`);
 };
 
 const onExit = (name: string, exitCode: number | null) => {

server/src/queries/asset.repository.sql

@@ -262,8 +262,9 @@ with
     from
       "assets"
     where
-      "assets"."deletedAt" is null
-      and "assets"."isVisible" = $2
+      "assets"."fileCreatedAt" <= $2
+      and "assets"."deletedAt" is null
+      and "assets"."isVisible" = $3
   )
 select
   "timeBucket",
@@ -283,9 +284,10 @@ from
   "assets"
   left join "exif" on "assets"."id" = "exif"."assetId"
 where
-  "assets"."deletedAt" is null
-  and "assets"."isVisible" = $1
-  and date_trunc($2, "localDateTime" at time zone 'UTC') at time zone 'UTC' = $3
+  "assets"."fileCreatedAt" <= $1
+  and "assets"."deletedAt" is null
+  and "assets"."isVisible" = $2
+  and date_trunc($3, "localDateTime" at time zone 'UTC') at time zone 'UTC' = $4
 order by
   "assets"."localDateTime" desc
 

server/src/repositories/asset.repository.ts

@@ -22,7 +22,6 @@ import {
   withStack,
   withTags,
 } from 'src/entities/asset.entity';
-import { LibraryEntity } from 'src/entities/library.entity';
 import { AssetFileType, AssetStatus, AssetType } from 'src/enum';
 import {
   AssetDeltaSyncOptions,
@@ -50,6 +49,8 @@ import { anyUuid, asUuid, mapUpsertColumns } from 'src/utils/database';
 import { globToSqlPattern } from 'src/utils/misc';
 import { Paginated, PaginationOptions, paginationHelper } from 'src/utils/pagination';
 
+const ASSET_CUTOFF_DATE = new Date('9000-01-01');
+
 @Injectable()
 export class AssetRepository implements IAssetRepository {
   constructor(@InjectKysely() private db: Kysely<DB>) {}
@@ -527,6 +528,7 @@ export class AssetRepository implements IAssetRepository {
     return this.db
       .selectFrom('assets')
       .selectAll('assets')
+      .where('assets.fileCreatedAt', '<=', ASSET_CUTOFF_DATE)
       .$call(withExif)
       .where('ownerId', '=', anyUuid(userIds))
       .where('isVisible', '=', true)
@@ -543,6 +545,7 @@ export class AssetRepository implements IAssetRepository {
         .with('assets', (qb) =>
           qb
             .selectFrom('assets')
+            .where('assets.fileCreatedAt', '<=', ASSET_CUTOFF_DATE)
             .select(truncatedDate<Date>(options.size).as('timeBucket'))
             .$if(!!options.isTrashed, (qb) => qb.where('assets.status', '!=', AssetStatus.DELETED))
             .where('assets.deletedAt', options.isTrashed ? 'is not' : 'is', null)
@@ -592,6 +595,7 @@ export class AssetRepository implements IAssetRepository {
   async getTimeBucket(timeBucket: string, options: TimeBucketOptions): Promise<AssetEntity[]> {
     return hasPeople(this.db, options.personId ? [options.personId] : undefined)
       .selectAll('assets')
+      .where('assets.fileCreatedAt', '<=', ASSET_CUTOFF_DATE)
       .$call(withExif)
       .$if(!!options.albumId, (qb) => withAlbums(qb, { albumId: options.albumId }))
       .$if(!!options.userIds, (qb) => qb.where('assets.ownerId', '=', anyUuid(options.userIds!)))
@@ -748,9 +752,16 @@ export class AssetRepository implements IAssetRepository {
       .execute();
   }
 
-  async detectOfflineExternalAssets(library: LibraryEntity): Promise<UpdateResult> {
-    const paths = library.importPaths.map((importPath) => `${importPath}%`);
-    const exclusions = library.exclusionPatterns.map((pattern) => globToSqlPattern(pattern));
+  @GenerateSql({
+    params: [{ libraryId: DummyValue.UUID, importPaths: [DummyValue.STRING], exclusionPatterns: [DummyValue.STRING] }],
+  })
+  async detectOfflineExternalAssets(
+    libraryId: string,
+    importPaths: string[],
+    exclusionPatterns: string[],
+  ): Promise<UpdateResult> {
+    const paths = importPaths.map((importPath) => `${importPath}%`);
+    const exclusions = exclusionPatterns.map((pattern) => globToSqlPattern(pattern));
 
     return this.db
       .updateTable('assets')
@@ -760,13 +771,16 @@ export class AssetRepository implements IAssetRepository {
       })
       .where('isOffline', '=', false)
       .where('isExternal', '=', true)
-      .where('libraryId', '=', asUuid(library.id))
+      .where('libraryId', '=', asUuid(libraryId))
       .where((eb) =>
         eb.or([eb('originalPath', 'not like', paths.join('|')), eb('originalPath', 'like', exclusions.join('|'))]),
       )
       .executeTakeFirstOrThrow();
   }
 
+  @GenerateSql({
+    params: [{ libraryId: DummyValue.UUID, paths: [DummyValue.STRING] }],
+  })
   async filterNewExternalAssetPaths(libraryId: string, paths: string[]): Promise<string[]> {
     const result = await this.db
       .selectFrom(

server/src/services/library.service.spec.ts

@@ -231,7 +231,11 @@ describe(LibraryService.name, () => {
       const response = await sut.handleQueueSyncAssets({ id: libraryStub.externalLibrary1.id });
 
       expect(response).toBe(JobStatus.SUCCESS);
-      expect(assetMock.detectOfflineExternalAssets).toHaveBeenCalledWith(libraryStub.externalLibrary1);
+      expect(assetMock.detectOfflineExternalAssets).toHaveBeenCalledWith(
+        libraryStub.externalLibrary1.id,
+        libraryStub.externalLibrary1.importPaths,
+        libraryStub.externalLibrary1.exclusionPatterns,
+      );
     });
 
     it('should skip an empty library', async () => {
@@ -270,7 +274,11 @@ describe(LibraryService.name, () => {
       });
 
       expect(response).toBe(JobStatus.SUCCESS);
-      expect(assetMock.detectOfflineExternalAssets).toHaveBeenCalledWith(libraryStub.externalLibraryWithImportPaths1);
+      expect(assetMock.detectOfflineExternalAssets).toHaveBeenCalledWith(
+        libraryStub.externalLibraryWithImportPaths1.id,
+        libraryStub.externalLibraryWithImportPaths1.importPaths,
+        libraryStub.externalLibraryWithImportPaths1.exclusionPatterns,
+      );
     });
 
     it("should fail if library can't be found", async () => {

server/src/services/library.service.ts

@@ -24,6 +24,8 @@ import { mimeTypes } from 'src/utils/mime-types';
 import { handlePromiseError } from 'src/utils/misc';
 import { usePagination } from 'src/utils/pagination';
 
+const ASSET_IMPORT_DATE = new Date('9999-12-31');
+
 @Injectable()
 export class LibraryService extends BaseService {
   private watchLibraries = false;
@@ -181,7 +183,7 @@ export class LibraryService extends BaseService {
 
   async getStatistics(id: string): Promise<number> {
     const count = await this.assetRepository.getLibraryAssetCount({ libraryId: id });
-    if (count == undefined) {
+    if (count === undefined) {
       throw new InternalServerErrorException(`Failed to get asset count for library ${id}`);
     }
     return count;
@@ -378,9 +380,9 @@ export class LibraryService extends BaseService {
       checksum: this.cryptoRepository.hashSha1(`path:${assetPath}`),
       originalPath: assetPath,
 
-      fileCreatedAt: new Date(),
-      fileModifiedAt: new Date(),
-      localDateTime: new Date(),
+      fileCreatedAt: ASSET_IMPORT_DATE,
+      fileModifiedAt: ASSET_IMPORT_DATE,
+      localDateTime: ASSET_IMPORT_DATE,
       // TODO: device asset id is deprecated, remove it
       deviceAssetId: `${basename(assetPath)}`.replaceAll(/\s+/g, ''),
       deviceId: 'Library Import',
@@ -470,22 +472,25 @@ export class LibraryService extends BaseService {
         case AssetSyncResult.CHECK_OFFLINE: {
           const isInImportPath = job.importPaths.find((path) => asset.originalPath.startsWith(path));
 
-          if (isInImportPath) {
-            const isExcluded = job.exclusionPatterns.some((pattern) => picomatch.isMatch(asset.originalPath, pattern));
-
-            if (isExcluded) {
-              this.logger.verbose(
-                `Offline asset ${asset.originalPath} is in an import path but still covered by exclusion pattern, keeping offline in library ${job.libraryId}`,
-              );
-            } else {
-              this.logger.debug(`Offline asset ${asset.originalPath} is now online in library ${job.libraryId}`);
-              assetIdsToOnline.push(asset.id);
-            }
-          } else {
+          if (!isInImportPath) {
             this.logger.verbose(
               `Offline asset ${asset.originalPath} is still not in any import path, keeping offline in library ${job.libraryId}`,
             );
+            break;
           }
+
+          const isExcluded = job.exclusionPatterns.some((pattern) => picomatch.isMatch(asset.originalPath, pattern));
+
+          if (!isExcluded) {
+            this.logger.debug(`Offline asset ${asset.originalPath} is now online in library ${job.libraryId}`);
+            assetIdsToOnline.push(asset.id);
+            break;
+          }
+
+          this.logger.verbose(
+            `Offline asset ${asset.originalPath} is in an import path but still covered by exclusion pattern, keeping offline in library ${job.libraryId}`,
+          );
+
           break;
         }
       }
@@ -696,7 +701,11 @@ export class LibraryService extends BaseService {
       `Checking ${assetCount} asset(s) against import paths and exclusion patterns in library ${library.id}...`,
     );
 
-    const offlineResult = await this.assetRepository.detectOfflineExternalAssets(library);
+    const offlineResult = await this.assetRepository.detectOfflineExternalAssets(
+      library.id,
+      library.importPaths,
+      library.exclusionPatterns,
+    );
 
     const affectedAssetCount = Number(offlineResult.numUpdatedRows);
 

server/src/services/metadata.service.ts

@@ -175,10 +175,10 @@ export class MetadataService extends BaseService {
     let fileCreatedAtDate = dateTimeOriginal;
     let fileModifiedAtDate = modifyDate;
 
-    /* if (asset.isExternal) {
+    if (asset.isExternal) {
       fileCreatedAtDate = fileCreatedAt;
       fileModifiedAtDate = fileModifiedAt;
-    } */
+    }
 
     const exifData: Insertable<Exif> = {
       assetId: asset.id,

server/test/repositories/asset.repository.mock.ts

@@ -43,5 +43,6 @@ export const newAssetRepositoryMock = (): Mocked<IAssetRepository> => {
     upsertFiles: vitest.fn(),
     detectOfflineExternalAssets: vitest.fn(),
     filterNewExternalAssetPaths: vitest.fn(),
+    updateByLibraryId: vitest.fn(),
   };
 };