diff --git a/server/src/entities/system-metadata.entity.ts b/server/src/entities/system-metadata.entity.ts index ae01c47b84..0a238e1da5 100644 --- a/server/src/entities/system-metadata.entity.ts +++ b/server/src/entities/system-metadata.entity.ts @@ -12,12 +12,14 @@ export class SystemMetadataEntity> { - [SystemMetadataKey.REVERSE_GEOCODING_STATE]: { lastUpdate?: string; lastImportFileName?: string }; - [SystemMetadataKey.FACIAL_RECOGNITION_STATE]: { lastRun?: string }; [SystemMetadataKey.ADMIN_ONBOARDING]: { isOnboarded: boolean }; - [SystemMetadataKey.SYSTEM_CONFIG]: DeepPartial; - [SystemMetadataKey.VERSION_CHECK_STATE]: VersionCheckMetadata; + [SystemMetadataKey.FACIAL_RECOGNITION_STATE]: { lastRun?: string }; [SystemMetadataKey.LICENSE]: { licenseKey: string; activationKey: string; activatedAt: Date }; + [SystemMetadataKey.REVERSE_GEOCODING_STATE]: { lastUpdate?: string; lastImportFileName?: string }; + [SystemMetadataKey.SYSTEM_CONFIG]: DeepPartial; + [SystemMetadataKey.SYSTEM_FLAGS]: SystemFlags; + [SystemMetadataKey.VERSION_CHECK_STATE]: VersionCheckMetadata; } diff --git a/server/src/enum.ts b/server/src/enum.ts index 28973e0205..32254854e4 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -153,6 +153,7 @@ export enum SystemMetadataKey { FACIAL_RECOGNITION_STATE = 'facial-recognition-state', ADMIN_ONBOARDING = 'admin-onboarding', SYSTEM_CONFIG = 'system-config', + SYSTEM_FLAGS = 'system-flags', VERSION_CHECK_STATE = 'version-check-state', LICENSE = 'license', } diff --git a/server/src/interfaces/database.interface.ts b/server/src/interfaces/database.interface.ts index 373f109142..51b39b95a8 100644 --- a/server/src/interfaces/database.interface.ts +++ b/server/src/interfaces/database.interface.ts @@ -15,6 +15,7 @@ export enum VectorIndex { export enum DatabaseLock { GeodataImport = 100, Migrations = 200, + SystemFileMounts = 300, StorageTemplateMigration = 420, CLIPDimSize = 512, LibraryWatch = 1337, diff --git a/server/src/main.ts b/server/src/main.ts index 7839bafd2f..ee4de1a259 100644 --- a/server/src/main.ts +++ b/server/src/main.ts @@ -17,7 +17,13 @@ async function bootstrapImmichAdmin() { function bootstrapWorker(name: string) { console.log(`Starting ${name} worker`); + const worker = name === 'api' ? fork(`./dist/workers/${name}.js`) : new Worker(`./dist/workers/${name}.js`); + + worker.on('error', (error) => { + console.error(`${name} worker error: ${error}`); + }); + worker.on('exit', (exitCode) => { if (exitCode !== 0) { console.error(`${name} worker exited with code ${exitCode}`); diff --git a/server/src/services/storage.service.spec.ts b/server/src/services/storage.service.spec.ts index d9b4c8eefb..b0f38554cb 100644 --- a/server/src/services/storage.service.spec.ts +++ b/server/src/services/storage.service.spec.ts @@ -1,19 +1,29 @@ +import { SystemMetadataKey } from 'src/enum'; +import { IDatabaseRepository } from 'src/interfaces/database.interface'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { IStorageRepository } from 'src/interfaces/storage.interface'; +import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface'; import { StorageService } from 'src/services/storage.service'; +import { newDatabaseRepositoryMock } from 'test/repositories/database.repository.mock'; import { newLoggerRepositoryMock } from 'test/repositories/logger.repository.mock'; import { newStorageRepositoryMock } from 'test/repositories/storage.repository.mock'; +import { newSystemMetadataRepositoryMock } from 'test/repositories/system-metadata.repository.mock'; import { Mocked } from 'vitest'; describe(StorageService.name, () => { let sut: StorageService; + let databaseMock: Mocked; let storageMock: Mocked; let loggerMock: Mocked; + let systemMock: Mocked; beforeEach(() => { + databaseMock = newDatabaseRepositoryMock(); storageMock = newStorageRepositoryMock(); loggerMock = newLoggerRepositoryMock(); - sut = new StorageService(storageMock, loggerMock); + systemMock = newSystemMetadataRepositoryMock(); + + sut = new StorageService(databaseMock, storageMock, loggerMock, systemMock); }); it('should work', () => { @@ -21,9 +31,35 @@ describe(StorageService.name, () => { }); describe('onBootstrap', () => { - it('should create the library folder on initialization', () => { - sut.onBootstrap(); + it('should enable mount folder checking', async () => { + systemMock.get.mockResolvedValue(null); + + await expect(sut.onBootstrap()).resolves.toBeUndefined(); + + expect(systemMock.set).toHaveBeenCalledWith(SystemMetadataKey.SYSTEM_FLAGS, { mountFiles: true }); + expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/encoded-video'); expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/library'); + expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/profile'); + expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs'); + }); + + it('should throw an error if .immich is missing', async () => { + systemMock.get.mockResolvedValue({ mountFiles: true }); + storageMock.readFile.mockRejectedValue(new Error("ENOENT: no such file or directory, open '/app/.immich'")); + + await expect(sut.onBootstrap()).rejects.toThrow('Failed to validate folder mount'); + + expect(storageMock.writeFile).not.toHaveBeenCalled(); + expect(systemMock.set).not.toHaveBeenCalled(); + }); + + it('should throw an error if .immich is present but read-only', async () => { + systemMock.get.mockResolvedValue({ mountFiles: true }); + storageMock.writeFile.mockRejectedValue(new Error("ENOENT: no such file or directory, open '/app/.immich'")); + + await expect(sut.onBootstrap()).rejects.toThrow('Failed to validate folder mount'); + + expect(systemMock.set).not.toHaveBeenCalled(); }); }); diff --git a/server/src/services/storage.service.ts b/server/src/services/storage.service.ts index c3f2c06438..a8f6a76e74 100644 --- a/server/src/services/storage.service.ts +++ b/server/src/services/storage.service.ts @@ -1,23 +1,52 @@ import { Inject, Injectable } from '@nestjs/common'; +import { join } from 'node:path'; import { StorageCore, StorageFolder } from 'src/cores/storage.core'; import { OnEmit } from 'src/decorators'; +import { SystemMetadataKey } from 'src/enum'; +import { DatabaseLock, IDatabaseRepository } from 'src/interfaces/database.interface'; import { IDeleteFilesJob, JobStatus } from 'src/interfaces/job.interface'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { IStorageRepository } from 'src/interfaces/storage.interface'; +import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface'; +import { ImmichStartupError } from 'src/utils/events'; @Injectable() export class StorageService { constructor( + @Inject(IDatabaseRepository) private databaseRepository: IDatabaseRepository, @Inject(IStorageRepository) private storageRepository: IStorageRepository, @Inject(ILoggerRepository) private logger: ILoggerRepository, + @Inject(ISystemMetadataRepository) private systemMetadata: ISystemMetadataRepository, ) { this.logger.setContext(StorageService.name); } @OnEmit({ event: 'app.bootstrap' }) - onBootstrap() { - const libraryBase = StorageCore.getBaseFolder(StorageFolder.LIBRARY); - this.storageRepository.mkdirSync(libraryBase); + async onBootstrap() { + await this.databaseRepository.withLock(DatabaseLock.SystemFileMounts, async () => { + const flags = (await this.systemMetadata.get(SystemMetadataKey.SYSTEM_FLAGS)) || { mountFiles: false }; + + this.logger.log('Verifying system mount folder checks'); + + // check each folder exists and is writable + for (const folder of Object.values(StorageFolder)) { + if (!flags.mountFiles) { + this.logger.log(`Writing initial mount file for the ${folder} folder`); + await this.verifyWriteAccess(folder); + } + + await this.verifyReadAccess(folder); + await this.verifyWriteAccess(folder); + } + + if (!flags.mountFiles) { + flags.mountFiles = true; + await this.systemMetadata.set(SystemMetadataKey.SYSTEM_FLAGS, flags); + this.logger.log('Successfully enabled system mount folders checks'); + } + + this.logger.log('Successfully verified system mount folder checks'); + }); } async handleDeleteFiles(job: IDeleteFilesJob) { @@ -38,4 +67,38 @@ export class StorageService { return JobStatus.SUCCESS; } + + private async verifyReadAccess(folder: StorageFolder) { + const { filePath } = this.getMountFilePaths(folder); + try { + await this.storageRepository.readFile(filePath); + } catch (error) { + this.logger.error(`Failed to read ${filePath}: ${error}`); + this.logger.error( + `The "${folder}" folder appears to be offline/missing, please make sure the volume is mounted with the correct permissions`, + ); + throw new ImmichStartupError(`Failed to validate folder mount (read from "/${folder}")`); + } + } + + private async verifyWriteAccess(folder: StorageFolder) { + const { folderPath, filePath } = this.getMountFilePaths(folder); + try { + this.storageRepository.mkdirSync(folderPath); + await this.storageRepository.writeFile(filePath, Buffer.from(`${Date.now()}`)); + } catch (error) { + this.logger.error(`Failed to write ${filePath}: ${error}`); + this.logger.error( + `The "${folder}" folder cannot be written to, please make sure the volume is mounted with the correct permissions`, + ); + throw new ImmichStartupError(`Failed to validate folder mount (write to "/${folder}")`); + } + } + + private getMountFilePaths(folder: StorageFolder) { + const folderPath = StorageCore.getBaseFolder(folder); + const filePath = join(folderPath, '.immich'); + + return { folderPath, filePath }; + } } diff --git a/server/src/utils/events.ts b/server/src/utils/events.ts index 2dd7e7fd5d..064c9f7507 100644 --- a/server/src/utils/events.ts +++ b/server/src/utils/events.ts @@ -12,6 +12,9 @@ type Item = { label: string; }; +export class ImmichStartupError extends Error {} +export const isStartUpError = (error: unknown): error is ImmichStartupError => error instanceof ImmichStartupError; + export const setupEventHandlers = (moduleRef: ModuleRef) => { const reflector = moduleRef.get(Reflector, { strict: false }); const repository = moduleRef.get(IEventRepository); diff --git a/server/src/workers/api.ts b/server/src/workers/api.ts index 5857f587a0..629c50c653 100644 --- a/server/src/workers/api.ts +++ b/server/src/workers/api.ts @@ -9,6 +9,7 @@ import { envName, excludePaths, isDev, resourcePaths, serverVersion } from 'src/ import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { WebSocketAdapter } from 'src/middleware/websocket.adapter'; import { ApiService } from 'src/services/api.service'; +import { isStartUpError } from 'src/utils/events'; import { otelStart } from 'src/utils/instrumentation'; import { useSwagger } from 'src/utils/misc'; @@ -73,6 +74,9 @@ async function bootstrap() { } bootstrap().catch((error) => { - console.error(error); - throw error; + if (!isStartUpError(error)) { + console.error(error); + } + // eslint-disable-next-line unicorn/no-process-exit + process.exit(1); }); diff --git a/server/src/workers/microservices.ts b/server/src/workers/microservices.ts index f920e8c947..789b6f5287 100644 --- a/server/src/workers/microservices.ts +++ b/server/src/workers/microservices.ts @@ -4,6 +4,7 @@ import { MicroservicesModule } from 'src/app.module'; import { envName, serverVersion } from 'src/constants'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { WebSocketAdapter } from 'src/middleware/websocket.adapter'; +import { isStartUpError } from 'src/utils/events'; import { otelStart } from 'src/utils/instrumentation'; export async function bootstrap() { @@ -25,7 +26,9 @@ export async function bootstrap() { if (!isMainThread) { bootstrap().catch((error) => { - console.error(error); - process.exit(1); + if (!isStartUpError(error)) { + console.error(error); + } + throw error; }); }