diff --git a/server/src/microservices/processors/metadata-extraction.processor.ts b/server/src/microservices/processors/metadata-extraction.processor.ts index 12c651c45e..51f1599983 100644 --- a/server/src/microservices/processors/metadata-extraction.processor.ts +++ b/server/src/microservices/processors/metadata-extraction.processor.ts @@ -1,12 +1,16 @@ import { IAssetRepository, IBaseJob, + ICryptoRepository, IEntityJob, IGeocodingRepository, IJobRepository, + IStorageRepository, JobName, JOBS_ASSET_PAGINATION_SIZE, QueueName, + StorageCore, + StorageFolder, usePagination, WithoutProperty, } from '@app/domain'; @@ -19,6 +23,7 @@ import { exiftool, Tags } from 'exiftool-vendored'; import ffmpeg, { FfprobeData } from 'fluent-ffmpeg'; import { Duration } from 'luxon'; import fs from 'node:fs'; +import path from 'node:path'; import sharp from 'sharp'; import { Repository } from 'typeorm/repository/Repository'; import { promisify } from 'util'; @@ -29,18 +34,35 @@ import { toNumberOrNull } from '../utils/numbers'; const ffprobe = promisify(ffmpeg.ffprobe); +interface DirectoryItem { + Length?: number; + Mime: string; + Padding?: number; + Semantic?: string; +} + +interface DirectoryEntry { + Item: DirectoryItem; +} + interface ImmichTags extends Tags { ContentIdentifier?: string; + MotionPhoto?: number; + MotionPhotoVersion?: number; + MotionPhotoPresentationTimestampUs?: number; } export class MetadataExtractionProcessor { private logger = new Logger(MetadataExtractionProcessor.name); private reverseGeocodingEnabled: boolean; + private storageCore = new StorageCore(); constructor( @Inject(IAssetRepository) private assetRepository: IAssetRepository, @Inject(IJobRepository) private jobRepository: IJobRepository, @Inject(IGeocodingRepository) private geocodingRepository: IGeocodingRepository, + @Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository, + @Inject(IStorageRepository) private storageRepository: IStorageRepository, @InjectRepository(ExifEntity) private exifRepository: Repository, configService: ConfigService, @@ -100,6 +122,131 @@ export class MetadataExtractionProcessor { } } + async addExtractedLivePhoto(sourceAsset: AssetEntity, video: string, created: Date | null): Promise { + if (sourceAsset.livePhotoVideoId) { + const [liveAsset] = await this.assetRepository.getByIds([sourceAsset.livePhotoVideoId]); + // already exists so no need to generate ID. + if (liveAsset.originalPath == video) { + return liveAsset; + } + liveAsset.originalPath = video; + return this.assetRepository.save(liveAsset); + } + const liveAsset = await this.assetRepository.save({ + ownerId: sourceAsset.ownerId, + owner: sourceAsset.owner, + + checksum: await this.cryptoRepository.hashFile(video), + originalPath: video, + + fileCreatedAt: created ?? sourceAsset.fileCreatedAt, + fileModifiedAt: sourceAsset.fileModifiedAt, + + deviceAssetId: 'NONE', + deviceId: 'NONE', + + type: AssetType.VIDEO, + isFavorite: false, + isArchived: sourceAsset.isArchived, + duration: null, + isVisible: false, + livePhotoVideo: null, + resizePath: null, + webpPath: null, + thumbhash: null, + encodedVideoPath: null, + tags: [], + sharedLinks: [], + originalFileName: path.parse(video).name, + faces: [], + sidecarPath: null, + isReadOnly: sourceAsset.isReadOnly, + }); + + sourceAsset.livePhotoVideoId = liveAsset.id; + await this.assetRepository.save(sourceAsset); + return liveAsset; + } + + private async extractNewPixelLivePhoto( + asset: AssetEntity, + directory: DirectoryEntry[], + fileCreatedAt: Date | null, + ): Promise { + if (asset.livePhotoVideoId) { + // Already extracted, don't try again. + const [ret] = await this.assetRepository.getByIds([asset.livePhotoVideoId]); + this.logger.log(`Already extracted asset ${ret.originalPath}.`); + return ret; + } + let foundMotionPhoto = false; + let motionPhotoOffsetFromEnd = 0; + let motionPhotoLength = 0; + + // Look for the directory entry with semantic label "MotionPhoto", which is the embedded video. + // Then, determine the length from the end of the file to the start of the embedded video. + for (const entry of directory) { + if (entry.Item.Semantic == 'MotionPhoto') { + if (foundMotionPhoto) { + this.logger.error(`Asset ${asset.originalPath} has more than one motion photo.`); + continue; + } + foundMotionPhoto = true; + motionPhotoLength = entry.Item.Length ?? 0; + } + if (foundMotionPhoto) { + motionPhotoOffsetFromEnd += entry.Item.Length ?? 0; + motionPhotoOffsetFromEnd += entry.Item.Padding ?? 0; + } + } + + if (!foundMotionPhoto || motionPhotoLength == 0) { + return null; + } + return this.extractEmbeddedVideo(asset, motionPhotoOffsetFromEnd, motionPhotoLength, fileCreatedAt); + } + + private async extractEmbeddedVideo( + asset: AssetEntity, + offsetFromEnd: number, + length: number | null, + fileCreatedAt: Date | null, + ) { + let file = null; + try { + file = await fs.promises.open(asset.originalPath); + let extracted = null; + // Read in embedded video. + const stat = await file.stat(); + if (length == null) { + length = offsetFromEnd; + } + const offset = stat.size - offsetFromEnd; + extracted = await file.read({ + buffer: Buffer.alloc(length), + position: offset, + length: length, + }); + + // Write out extracted video, and add it to the asset repository. + const encodedVideoFolder = this.storageCore.getFolderLocation(StorageFolder.ENCODED_VIDEO, asset.ownerId); + this.storageRepository.mkdirSync(encodedVideoFolder); + const livePhotoPath = path.join(encodedVideoFolder, path.parse(asset.originalPath).name + '.mp4'); + await fs.promises.writeFile(livePhotoPath, extracted.buffer); + + const result = await this.addExtractedLivePhoto(asset, livePhotoPath, fileCreatedAt); + await this.handleMetadataExtraction({ id: result.id }); + return result; + } catch (e) { + this.logger.error(`Failed to extract live photo ${asset.originalPath}: ${e}`); + return null; + } finally { + if (file) { + await file.close(); + } + } + } + private async handlePhotoMetadataExtraction(asset: AssetEntity) { const mediaExifData = await exiftool.read(asset.originalPath).catch((error: any) => { this.logger.warn( @@ -163,7 +310,32 @@ export class MetadataExtractionProcessor { const longitude = getExifProperty('GPSLongitude'); newExif.latitude = latitude !== null ? parseLatitude(latitude) : null; newExif.longitude = longitude !== null ? parseLongitude(longitude) : null; - + if (getExifProperty('MotionPhoto')) { + // Seen on more recent Pixel phones: starting as early as Pixel 4a, possibly earlier. + const rawDirectory = getExifProperty('Directory'); + if (Array.isArray(rawDirectory)) { + // exiftool-vendor thinks directory is a string, but actually it's an array of DirectoryEntry. + const directory = rawDirectory as DirectoryEntry[]; + await this.extractNewPixelLivePhoto(asset, directory, fileCreatedAt); + } else { + this.logger.warn(`Failed to get Pixel motionPhoto information: directory: ${JSON.stringify(rawDirectory)}`); + } + } else if (getExifProperty('MicroVideo')) { + // Seen on earlier Pixel phones - Pixel 2 and earlier, possibly Pixel 3. + let offset = getExifProperty('MicroVideoOffset'); // offset from end of file. + if (typeof offset == 'string') { + offset = parseInt(offset); + } + if (Number.isNaN(offset) || offset == null) { + this.logger.warn( + `Failed to get MicroVideo information for ${asset.originalPath}, offset=${getExifProperty( + 'MicroVideoOffset', + )}`, + ); + } else { + await this.extractEmbeddedVideo(asset, offset, null, fileCreatedAt); + } + } newExif.livePhotoCID = getExifProperty('MediaGroupUUID'); if (newExif.livePhotoCID && !asset.livePhotoVideoId) { const motionAsset = await this.assetRepository.findLivePhotoMatch({