diff --git a/server/src/interfaces/asset.interface.ts b/server/src/interfaces/asset.interface.ts index 2502cb6f4b..9ec8c96825 100644 --- a/server/src/interfaces/asset.interface.ts +++ b/server/src/interfaces/asset.interface.ts @@ -1,7 +1,6 @@ import { Insertable, Updateable, UpdateResult } from 'kysely'; import { AssetJobStatus, Assets, Exif } from 'src/db'; import { AssetEntity } from 'src/entities/asset.entity'; -import { LibraryEntity } from 'src/entities/library.entity'; import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum'; import { AssetSearchOptions, SearchExploreItem } from 'src/interfaces/search.interface'; import { Paginated, PaginationOptions } from 'src/utils/pagination'; @@ -171,7 +170,11 @@ export interface IAssetRepository { getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise; upsertFile(file: UpsertFileOptions): Promise; upsertFiles(files: UpsertFileOptions[]): Promise; - detectOfflineExternalAssets(library: LibraryEntity): Promise; + detectOfflineExternalAssets( + libraryId: string, + importPaths: string[], + exclusionPatterns: string[], + ): Promise; filterNewExternalAssetPaths(libraryId: string, paths: string[]): Promise; getLibraryAssetCount(options: AssetSearchOptions): Promise; } diff --git a/server/src/main.ts b/server/src/main.ts index 3097eee69b..95b35c6915 100644 --- a/server/src/main.ts +++ b/server/src/main.ts @@ -13,7 +13,7 @@ if (immichApp) { let apiProcess: ChildProcess | undefined; const onError = (name: string, error: Error) => { - console.error(`${name} worker error: ${error}`); + console.error(`${name} worker error: ${error}, stack: ${error.stack}`); }; const onExit = (name: string, exitCode: number | null) => { diff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sql index d50069f0a9..d5a203ead7 100644 --- a/server/src/queries/asset.repository.sql +++ b/server/src/queries/asset.repository.sql @@ -262,8 +262,9 @@ with from "assets" where - "assets"."deletedAt" is null - and "assets"."isVisible" = $2 + "assets"."fileCreatedAt" <= $2 + and "assets"."deletedAt" is null + and "assets"."isVisible" = $3 ) select "timeBucket", @@ -283,9 +284,10 @@ from "assets" left join "exif" on "assets"."id" = "exif"."assetId" where - "assets"."deletedAt" is null - and "assets"."isVisible" = $1 - and date_trunc($2, "localDateTime" at time zone 'UTC') at time zone 'UTC' = $3 + "assets"."fileCreatedAt" <= $1 + and "assets"."deletedAt" is null + and "assets"."isVisible" = $2 + and date_trunc($3, "localDateTime" at time zone 'UTC') at time zone 'UTC' = $4 order by "assets"."localDateTime" desc diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index b7c0d13aa7..f26992c5f7 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -22,7 +22,6 @@ import { withStack, withTags, } from 'src/entities/asset.entity'; -import { LibraryEntity } from 'src/entities/library.entity'; import { AssetFileType, AssetStatus, AssetType } from 'src/enum'; import { AssetDeltaSyncOptions, @@ -50,6 +49,8 @@ import { anyUuid, asUuid, mapUpsertColumns } from 'src/utils/database'; import { globToSqlPattern } from 'src/utils/misc'; import { Paginated, PaginationOptions, paginationHelper } from 'src/utils/pagination'; +const ASSET_CUTOFF_DATE = new Date('9000-01-01'); + @Injectable() export class AssetRepository implements IAssetRepository { constructor(@InjectKysely() private db: Kysely) {} @@ -527,6 +528,7 @@ export class AssetRepository implements IAssetRepository { return this.db .selectFrom('assets') .selectAll('assets') + .where('assets.fileCreatedAt', '<=', ASSET_CUTOFF_DATE) .$call(withExif) .where('ownerId', '=', anyUuid(userIds)) .where('isVisible', '=', true) @@ -543,6 +545,7 @@ export class AssetRepository implements IAssetRepository { .with('assets', (qb) => qb .selectFrom('assets') + .where('assets.fileCreatedAt', '<=', ASSET_CUTOFF_DATE) .select(truncatedDate(options.size).as('timeBucket')) .$if(!!options.isTrashed, (qb) => qb.where('assets.status', '!=', AssetStatus.DELETED)) .where('assets.deletedAt', options.isTrashed ? 'is not' : 'is', null) @@ -592,6 +595,7 @@ export class AssetRepository implements IAssetRepository { async getTimeBucket(timeBucket: string, options: TimeBucketOptions): Promise { return hasPeople(this.db, options.personId ? [options.personId] : undefined) .selectAll('assets') + .where('assets.fileCreatedAt', '<=', ASSET_CUTOFF_DATE) .$call(withExif) .$if(!!options.albumId, (qb) => withAlbums(qb, { albumId: options.albumId })) .$if(!!options.userIds, (qb) => qb.where('assets.ownerId', '=', anyUuid(options.userIds!))) @@ -748,9 +752,16 @@ export class AssetRepository implements IAssetRepository { .execute(); } - async detectOfflineExternalAssets(library: LibraryEntity): Promise { - const paths = library.importPaths.map((importPath) => `${importPath}%`); - const exclusions = library.exclusionPatterns.map((pattern) => globToSqlPattern(pattern)); + @GenerateSql({ + params: [{ libraryId: DummyValue.UUID, importPaths: [DummyValue.STRING], exclusionPatterns: [DummyValue.STRING] }], + }) + async detectOfflineExternalAssets( + libraryId: string, + importPaths: string[], + exclusionPatterns: string[], + ): Promise { + const paths = importPaths.map((importPath) => `${importPath}%`); + const exclusions = exclusionPatterns.map((pattern) => globToSqlPattern(pattern)); return this.db .updateTable('assets') @@ -760,13 +771,16 @@ export class AssetRepository implements IAssetRepository { }) .where('isOffline', '=', false) .where('isExternal', '=', true) - .where('libraryId', '=', asUuid(library.id)) + .where('libraryId', '=', asUuid(libraryId)) .where((eb) => eb.or([eb('originalPath', 'not like', paths.join('|')), eb('originalPath', 'like', exclusions.join('|'))]), ) .executeTakeFirstOrThrow(); } + @GenerateSql({ + params: [{ libraryId: DummyValue.UUID, paths: [DummyValue.STRING] }], + }) async filterNewExternalAssetPaths(libraryId: string, paths: string[]): Promise { const result = await this.db .selectFrom( diff --git a/server/src/services/library.service.spec.ts b/server/src/services/library.service.spec.ts index c7a383b552..f8257345b0 100644 --- a/server/src/services/library.service.spec.ts +++ b/server/src/services/library.service.spec.ts @@ -231,7 +231,11 @@ describe(LibraryService.name, () => { const response = await sut.handleQueueSyncAssets({ id: libraryStub.externalLibrary1.id }); expect(response).toBe(JobStatus.SUCCESS); - expect(assetMock.detectOfflineExternalAssets).toHaveBeenCalledWith(libraryStub.externalLibrary1); + expect(assetMock.detectOfflineExternalAssets).toHaveBeenCalledWith( + libraryStub.externalLibrary1.id, + libraryStub.externalLibrary1.importPaths, + libraryStub.externalLibrary1.exclusionPatterns, + ); }); it('should skip an empty library', async () => { @@ -270,7 +274,11 @@ describe(LibraryService.name, () => { }); expect(response).toBe(JobStatus.SUCCESS); - expect(assetMock.detectOfflineExternalAssets).toHaveBeenCalledWith(libraryStub.externalLibraryWithImportPaths1); + expect(assetMock.detectOfflineExternalAssets).toHaveBeenCalledWith( + libraryStub.externalLibraryWithImportPaths1.id, + libraryStub.externalLibraryWithImportPaths1.importPaths, + libraryStub.externalLibraryWithImportPaths1.exclusionPatterns, + ); }); it("should fail if library can't be found", async () => { diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.ts index 1eac809134..1281207e25 100644 --- a/server/src/services/library.service.ts +++ b/server/src/services/library.service.ts @@ -24,6 +24,8 @@ import { mimeTypes } from 'src/utils/mime-types'; import { handlePromiseError } from 'src/utils/misc'; import { usePagination } from 'src/utils/pagination'; +const ASSET_IMPORT_DATE = new Date('9999-12-31'); + @Injectable() export class LibraryService extends BaseService { private watchLibraries = false; @@ -181,7 +183,7 @@ export class LibraryService extends BaseService { async getStatistics(id: string): Promise { const count = await this.assetRepository.getLibraryAssetCount({ libraryId: id }); - if (count == undefined) { + if (count === undefined) { throw new InternalServerErrorException(`Failed to get asset count for library ${id}`); } return count; @@ -378,9 +380,9 @@ export class LibraryService extends BaseService { checksum: this.cryptoRepository.hashSha1(`path:${assetPath}`), originalPath: assetPath, - fileCreatedAt: new Date(), - fileModifiedAt: new Date(), - localDateTime: new Date(), + fileCreatedAt: ASSET_IMPORT_DATE, + fileModifiedAt: ASSET_IMPORT_DATE, + localDateTime: ASSET_IMPORT_DATE, // TODO: device asset id is deprecated, remove it deviceAssetId: `${basename(assetPath)}`.replaceAll(/\s+/g, ''), deviceId: 'Library Import', @@ -470,22 +472,25 @@ export class LibraryService extends BaseService { case AssetSyncResult.CHECK_OFFLINE: { const isInImportPath = job.importPaths.find((path) => asset.originalPath.startsWith(path)); - if (isInImportPath) { - const isExcluded = job.exclusionPatterns.some((pattern) => picomatch.isMatch(asset.originalPath, pattern)); - - if (isExcluded) { - this.logger.verbose( - `Offline asset ${asset.originalPath} is in an import path but still covered by exclusion pattern, keeping offline in library ${job.libraryId}`, - ); - } else { - this.logger.debug(`Offline asset ${asset.originalPath} is now online in library ${job.libraryId}`); - assetIdsToOnline.push(asset.id); - } - } else { + if (!isInImportPath) { this.logger.verbose( `Offline asset ${asset.originalPath} is still not in any import path, keeping offline in library ${job.libraryId}`, ); + break; } + + const isExcluded = job.exclusionPatterns.some((pattern) => picomatch.isMatch(asset.originalPath, pattern)); + + if (!isExcluded) { + this.logger.debug(`Offline asset ${asset.originalPath} is now online in library ${job.libraryId}`); + assetIdsToOnline.push(asset.id); + break; + } + + this.logger.verbose( + `Offline asset ${asset.originalPath} is in an import path but still covered by exclusion pattern, keeping offline in library ${job.libraryId}`, + ); + break; } } @@ -696,7 +701,11 @@ export class LibraryService extends BaseService { `Checking ${assetCount} asset(s) against import paths and exclusion patterns in library ${library.id}...`, ); - const offlineResult = await this.assetRepository.detectOfflineExternalAssets(library); + const offlineResult = await this.assetRepository.detectOfflineExternalAssets( + library.id, + library.importPaths, + library.exclusionPatterns, + ); const affectedAssetCount = Number(offlineResult.numUpdatedRows); diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.ts index f1d6389e02..4b092d2761 100644 --- a/server/src/services/metadata.service.ts +++ b/server/src/services/metadata.service.ts @@ -175,10 +175,10 @@ export class MetadataService extends BaseService { let fileCreatedAtDate = dateTimeOriginal; let fileModifiedAtDate = modifyDate; - /* if (asset.isExternal) { + if (asset.isExternal) { fileCreatedAtDate = fileCreatedAt; fileModifiedAtDate = fileModifiedAt; - } */ + } const exifData: Insertable = { assetId: asset.id, diff --git a/server/test/repositories/asset.repository.mock.ts b/server/test/repositories/asset.repository.mock.ts index 01d73e418d..0ab49478ba 100644 --- a/server/test/repositories/asset.repository.mock.ts +++ b/server/test/repositories/asset.repository.mock.ts @@ -43,5 +43,6 @@ export const newAssetRepositoryMock = (): Mocked => { upsertFiles: vitest.fn(), detectOfflineExternalAssets: vitest.fn(), filterNewExternalAssetPaths: vitest.fn(), + updateByLibraryId: vitest.fn(), }; };