feat: editor endpoints

open-api/immich-openapi-specs.json

@@ -2469,6 +2469,48 @@
         ]
       }
     },
+    "/editor": {
+      "post": {
+        "operationId": "createAssetFromEdits",
+        "parameters": [],
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/EditorCreateAssetDto"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {
+          "201": {
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/AssetResponseDto"
+                }
+              }
+            },
+            "description": ""
+          }
+        },
+        "security": [
+          {
+            "bearer": []
+          },
+          {
+            "cookie": []
+          },
+          {
+            "api_key": []
+          }
+        ],
+        "tags": [
+          "Editor"
+        ]
+      }
+    },
     "/faces": {
       "get": {
         "operationId": "getFaces",
@@ -8562,6 +8604,144 @@
         ],
         "type": "object"
       },
+      "EditorActionAdjust": {
+        "properties": {
+          "action": {
+            "$ref": "#/components/schemas/EditorActionType"
+          },
+          "brightness": {
+            "type": "integer"
+          },
+          "hue": {
+            "type": "integer"
+          },
+          "lightness": {
+            "type": "integer"
+          },
+          "saturation": {
+            "type": "integer"
+          }
+        },
+        "required": [
+          "action",
+          "brightness",
+          "hue",
+          "lightness",
+          "saturation"
+        ],
+        "type": "object"
+      },
+      "EditorActionBlur": {
+        "properties": {
+          "action": {
+            "$ref": "#/components/schemas/EditorActionType"
+          }
+        },
+        "required": [
+          "action"
+        ],
+        "type": "object"
+      },
+      "EditorActionCrop": {
+        "properties": {
+          "action": {
+            "$ref": "#/components/schemas/EditorActionType"
+          },
+          "region": {
+            "$ref": "#/components/schemas/EditorCropRegion"
+          }
+        },
+        "required": [
+          "action",
+          "region"
+        ],
+        "type": "object"
+      },
+      "EditorActionRotate": {
+        "properties": {
+          "action": {
+            "$ref": "#/components/schemas/EditorActionType"
+          },
+          "angle": {
+            "type": "integer"
+          }
+        },
+        "required": [
+          "action",
+          "angle"
+        ],
+        "type": "object"
+      },
+      "EditorActionType": {
+        "enum": [
+          "crop",
+          "rotate",
+          "blur",
+          "adjust"
+        ],
+        "type": "string"
+      },
+      "EditorCreateAssetDto": {
+        "properties": {
+          "edits": {
+            "description": "list of edits",
+            "items": {
+              "anyOf": [
+                {
+                  "$ref": "#/components/schemas/EditorActionCrop"
+                },
+                {
+                  "$ref": "#/components/schemas/EditorActionRotate"
+                },
+                {
+                  "$ref": "#/components/schemas/EditorActionBlur"
+                },
+                {
+                  "$ref": "#/components/schemas/EditorActionAdjust"
+                }
+              ]
+            },
+            "type": "array"
+          },
+          "id": {
+            "description": "Source asset id",
+            "format": "uuid",
+            "type": "string"
+          },
+          "stack": {
+            "description": "Stack the edit and the original",
+            "type": "boolean"
+          }
+        },
+        "required": [
+          "edits",
+          "id"
+        ],
+        "type": "object"
+      },
+      "EditorCropRegion": {
+        "properties": {
+          "height": {
+            "type": "integer"
+          },
+          "left": {
+            "type": "integer"
+          },
+          "top": {
+            "type": "integer"
+          },
+          "width": {
+            "type": "integer"
+          }
+        },
+        "required": [
+          "height",
+          "left",
+          "top",
+          "width"
+        ],
+        "type": "object"
+      },
       "EmailNotificationsResponse": {
         "properties": {
           "albumInvite": {

open-api/typescript-sdk/src/fetch-client.ts

@@ -444,6 +444,38 @@ export type DuplicateResponseDto = {
     assets: AssetResponseDto[];
     duplicateId: string;
 };
+export type EditorCropRegion = {
+    height: number;
+    left: number;
+    top: number;
+    width: number;
+};
+export type EditorActionCrop = {
+    action: EditorActionType;
+    region: EditorCropRegion;
+};
+export type EditorActionRotate = {
+    action: EditorActionType;
+    angle: number;
+};
+export type EditorActionBlur = {
+    action: EditorActionType;
+};
+export type EditorActionAdjust = {
+    action: EditorActionType;
+    brightness: number;
+    hue: number;
+    lightness: number;
+    saturation: number;
+};
+export type EditorCreateAssetDto = {
+    /** list of edits */
+    edits: (EditorActionCrop | EditorActionRotate | EditorActionBlur | EditorActionAdjust)[];
+    /** Source asset id */
+    id: string;
+    /** Stack the edit and the original */
+    stack?: boolean;
+};
 export type PersonResponseDto = {
     birthDate: string | null;
     id: string;
@@ -1836,6 +1868,18 @@ export function getAssetDuplicates(opts?: Oazapfts.RequestOpts) {
         ...opts
     }));
 }
+export function createAssetFromEdits({ editorCreateAssetDto }: {
+    editorCreateAssetDto: EditorCreateAssetDto;
+}, opts?: Oazapfts.RequestOpts) {
+    return oazapfts.ok(oazapfts.fetchJson<{
+        status: 201;
+        data: AssetResponseDto;
+    }>("/editor", oazapfts.json({
+        ...opts,
+        method: "POST",
+        body: editorCreateAssetDto
+    })));
+}
 export function getFaces({ id }: {
     id: string;
 }, opts?: Oazapfts.RequestOpts) {
@@ -3138,6 +3182,12 @@ export enum EntityType {
     Asset = "ASSET",
     Album = "ALBUM"
 }
+export enum EditorActionType {
+    Crop = "crop",
+    Rotate = "rotate",
+    Blur = "blur",
+    Adjust = "adjust"
+}
 export enum JobName {
     ThumbnailGeneration = "thumbnailGeneration",
     MetadataExtraction = "metadataExtraction",

server/src/controllers/editor.controller.ts

@@ -0,0 +1,19 @@
+import { Body, Controller, Post } from '@nestjs/common';
+import { ApiTags } from '@nestjs/swagger';
+import { AssetResponseDto } from 'src/dtos/asset-response.dto';
+import { AuthDto } from 'src/dtos/auth.dto';
+import { EditorCreateAssetDto } from 'src/dtos/editor.dto';
+import { Auth, Authenticated } from 'src/middleware/auth.guard';
+import { EditorService } from 'src/services/editor.service';
+
+@ApiTags('Editor')
+@Controller('editor')
+export class EditorController {
+  constructor(private service: EditorService) {}
+
+  @Post()
+  @Authenticated()
+  createAssetFromEdits(@Auth() auth: AuthDto, @Body() dto: EditorCreateAssetDto): Promise<AssetResponseDto> {
+    return this.service.createAssetFromEdits(auth, dto);
+  }
+}

server/src/controllers/index.ts

@@ -8,6 +8,7 @@ import { AuditController } from 'src/controllers/audit.controller';
 import { AuthController } from 'src/controllers/auth.controller';
 import { DownloadController } from 'src/controllers/download.controller';
 import { DuplicateController } from 'src/controllers/duplicate.controller';
+import { EditorController } from 'src/controllers/editor.controller';
 import { FaceController } from 'src/controllers/face.controller';
 import { ReportController } from 'src/controllers/file-report.controller';
 import { JobController } from 'src/controllers/job.controller';
@@ -43,6 +44,7 @@ export const controllers = [
   AuthController,
   DownloadController,
   DuplicateController,
+  EditorController,
   FaceController,
   JobController,
   LibraryController,

server/src/dtos/editor.dto.ts

@@ -0,0 +1,98 @@
+import { ApiExtraModels, ApiProperty, getSchemaPath } from '@nestjs/swagger';
+import { ClassConstructor, plainToInstance, Transform, Type } from 'class-transformer';
+import { IsEnum, IsInt, ValidateNested } from 'class-validator';
+import { ValidateBoolean, ValidateUUID } from 'src/validation';
+
+export enum EditorActionType {
+  Crop = 'crop',
+  Rotate = 'rotate',
+  Blur = 'blur',
+  Adjust = 'adjust',
+}
+
+export class EditorActionItem {
+  @IsEnum(EditorActionType)
+  @ApiProperty({ enum: EditorActionType, enumName: 'EditorActionType' })
+  action!: EditorActionType;
+}
+
+export class EditorActionAdjust extends EditorActionItem {
+  @IsInt()
+  @ApiProperty({ type: 'integer' })
+  brightness!: number;
+
+  @IsInt()
+  @ApiProperty({ type: 'integer' })
+  saturation!: number;
+
+  @IsInt()
+  @ApiProperty({ type: 'integer' })
+  hue!: number;
+
+  @IsInt()
+  @ApiProperty({ type: 'integer' })
+  lightness!: number;
+}
+
+export class EditorActionBlur extends EditorActionItem {}
+
+class EditorCropRegion {
+  @IsInt()
+  @ApiProperty({ type: 'integer' })
+  left!: number;
+
+  @IsInt()
+  @ApiProperty({ type: 'integer' })
+  top!: number;
+
+  @IsInt()
+  @ApiProperty({ type: 'integer' })
+  width!: number;
+
+  @IsInt()
+  @ApiProperty({ type: 'integer' })
+  height!: number;
+}
+
+export class EditorActionCrop extends EditorActionItem {
+  @Type(() => EditorCropRegion)
+  @ValidateNested()
+  region!: EditorCropRegion;
+}
+
+export class EditorActionRotate extends EditorActionItem {
+  @IsInt()
+  @ApiProperty({ type: 'integer' })
+  angle!: number;
+}
+
+export type EditorAction = EditorActionRotate | EditorActionBlur | EditorActionCrop | EditorActionAdjust;
+
+const actionToClass: Record<EditorActionType, ClassConstructor<EditorAction>> = {
+  [EditorActionType.Crop]: EditorActionCrop,
+  [EditorActionType.Rotate]: EditorActionRotate,
+  [EditorActionType.Blur]: EditorActionBlur,
+  [EditorActionType.Adjust]: EditorActionAdjust,
+};
+
+const getActionClass = (item: EditorActionItem): ClassConstructor<EditorAction> =>
+  actionToClass[item.action] || EditorActionItem;
+
+@ApiExtraModels(EditorActionRotate, EditorActionBlur, EditorActionCrop, EditorActionAdjust)
+export class EditorCreateAssetDto {
+  /** Source asset id */
+  @ValidateUUID()
+  id!: string;
+
+  /** Stack the edit and the original */
+  @ValidateBoolean({ optional: true })
+  stack?: boolean;
+
+  /** list of edits */
+  @ValidateNested({ each: true })
+  @Transform(({ value: edits }) =>
+    Array.isArray(edits) ? edits.map((item) => plainToInstance(getActionClass(item), item)) : edits,
+  )
+  @ApiProperty({ anyOf: Object.values(actionToClass).map((target) => ({ $ref: getSchemaPath(target) })) })
+  edits!: EditorAction[];
+}

server/src/interfaces/job.interface.ts

@@ -117,7 +117,7 @@ export interface IBaseJob {
 
 export interface IEntityJob extends IBaseJob {
   id: string;
-  source?: 'upload' | 'sidecar-write' | 'copy';
+  source?: 'upload' | 'sidecar-write' | 'copy' | 'editor';
 }
 
 export interface IAssetDeleteJob extends IEntityJob {

server/src/interfaces/media.interface.ts

@@ -1,4 +1,5 @@
 import { Writable } from 'node:stream';
+import { Region } from 'sharp';
 import { ImageFormat, TranscodeTarget, VideoCodec } from 'src/config';
 
 export const IMediaRepository = 'IMediaRepository';
@@ -71,6 +72,20 @@ export interface BitrateDistribution {
   unit: string;
 }
 
+export type MediaEditItem =
+  | { action: 'crop'; region: Region }
+  | { action: 'rotate'; angle: number }
+  | { action: 'blur' }
+  | {
+      action: 'modulate';
+      brightness?: number;
+      saturation?: number;
+      hue?: number;
+      lightness?: number;
+    };
+
+export type MediaEdits = MediaEditItem[];
+
 export interface VideoCodecSWConfig {
   getCommand(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream: AudioStreamInfo): TranscodeCommand;
 }
@@ -89,4 +104,7 @@ export interface IMediaRepository {
   // video
   probe(input: string): Promise<VideoInfo>;
   transcode(input: string, output: string | Writable, command: TranscodeCommand): Promise<void>;
+
+  // editor
+  applyEdits(input: string, output: string, edits: MediaEditItem[]): Promise<void>;
 }

server/src/repositories/media.repository.ts

@@ -8,8 +8,9 @@ import sharp from 'sharp';
 import { Colorspace } from 'src/config';
 import { ILoggerRepository } from 'src/interfaces/logger.interface';
 import {
-  IMediaRepository,
   ImageDimensions,
+  IMediaRepository,
+  MediaEdits,
   ThumbnailOptions,
   TranscodeCommand,
   VideoInfo,
@@ -44,6 +45,41 @@ export class MediaRepository implements IMediaRepository {
     return true;
   }
 
+  async applyEdits(input: string, output: string, edits: MediaEdits) {
+    const pipeline = sharp(input, { failOn: 'error', limitInputPixels: false }).keepMetadata();
+
+    for (const edit of edits) {
+      switch (edit.action) {
+        case 'crop': {
+          pipeline.extract(edit.region);
+          break;
+        }
+
+        case 'rotate': {
+          pipeline.rotate(edit.angle);
+          break;
+        }
+
+        case 'blur': {
+          pipeline.blur(true);
+          break;
+        }
+
+        case 'modulate': {
+          pipeline.modulate({
+            brightness: edit.brightness,
+            saturation: edit.saturation,
+            hue: edit.hue,
+            lightness: edit.lightness,
+          });
+          break;
+        }
+      }
+    }
+
+    await pipeline.toFile(output);
+  }
+
   async generateThumbnail(input: string | Buffer, output: string, options: ThumbnailOptions): Promise<void> {
     // some invalid images can still be processed by sharp, but we want to fail on them by default to avoid crashes
     const pipeline = sharp(input, { failOn: options.processInvalidImages ? 'none' : 'error', limitInputPixels: false })

server/src/services/editor.service.ts

@@ -0,0 +1,128 @@
+import { BadRequestException, Inject, Injectable, InternalServerErrorException } from '@nestjs/common';
+import { dirname } from 'node:path';
+import { AccessCore, Permission } from 'src/cores/access.core';
+import { StorageCore, StorageFolder } from 'src/cores/storage.core';
+import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto';
+import { AuthDto } from 'src/dtos/auth.dto';
+import {
+  EditorAction,
+  EditorActionAdjust,
+  EditorActionBlur,
+  EditorActionCrop,
+  EditorActionRotate,
+  EditorActionType,
+  EditorCreateAssetDto,
+} from 'src/dtos/editor.dto';
+import { AssetType } from 'src/entities/asset.entity';
+import { IAccessRepository } from 'src/interfaces/access.interface';
+import { IAssetRepository } from 'src/interfaces/asset.interface';
+import { ICryptoRepository } from 'src/interfaces/crypto.interface';
+import { IJobRepository, JobName } from 'src/interfaces/job.interface';
+import { ILoggerRepository } from 'src/interfaces/logger.interface';
+import { IMediaRepository, MediaEditItem } from 'src/interfaces/media.interface';
+import { IStorageRepository } from 'src/interfaces/storage.interface';
+
+@Injectable()
+export class EditorService {
+  private access: AccessCore;
+
+  constructor(
+    @Inject(IAccessRepository) accessRepository: IAccessRepository,
+    @Inject(IAssetRepository) private assetRepository: IAssetRepository,
+    @Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository,
+    @Inject(IJobRepository) private jobRepository: IJobRepository,
+    @Inject(ILoggerRepository) private logger: ILoggerRepository,
+    @Inject(IMediaRepository) private mediaRepository: IMediaRepository,
+    @Inject(IStorageRepository) private storageRepository: IStorageRepository,
+  ) {
+    this.access = AccessCore.create(accessRepository);
+  }
+
+  async createAssetFromEdits(auth: AuthDto, dto: EditorCreateAssetDto): Promise<AssetResponseDto> {
+    await this.access.requirePermission(auth, Permission.ASSET_VIEW, dto.id);
+
+    const asset = await this.assetRepository.getById(dto.id);
+    if (!asset) {
+      throw new BadRequestException('Asset not found');
+    }
+
+    if (asset.type !== AssetType.IMAGE) {
+      throw new BadRequestException('Only images can be edited');
+    }
+
+    const uuid = this.cryptoRepository.randomUUID();
+    const outputFile = StorageCore.getNestedPath(StorageFolder.UPLOAD, auth.user.id, uuid);
+    this.storageRepository.mkdirSync(dirname(outputFile));
+
+    await this.mediaRepository.applyEdits(asset.originalPath, outputFile, this.asMediaEdits(dto.edits));
+
+    try {
+      const checksum = await this.cryptoRepository.hashFile(outputFile);
+      const { size } = await this.storageRepository.stat(outputFile);
+
+      const newAsset = await this.assetRepository.create({
+        id: uuid,
+        ownerId: auth.user.id,
+        deviceId: 'immich-editor',
+        deviceAssetId: asset.deviceAssetId + `-edit-${Date.now()}`,
+        libraryId: null,
+        type: asset.type,
+        originalPath: outputFile,
+        localDateTime: asset.localDateTime,
+        fileCreatedAt: asset.fileCreatedAt,
+        fileModifiedAt: asset.fileModifiedAt,
+        isFavorite: false,
+        isArchived: false,
+        isExternal: false,
+        isOffline: false,
+        checksum,
+        isVisible: true,
+        originalFileName: asset.originalFileName,
+        sidecarPath: null,
+        tags: asset.tags,
+        duplicateId: null,
+      });
+
+      await this.assetRepository.upsertExif({ assetId: newAsset.id, fileSizeInByte: size });
+      await this.jobRepository.queue({
+        name: JobName.METADATA_EXTRACTION,
+        data: { id: newAsset.id, source: 'editor' },
+      });
+
+      return mapAsset(newAsset, { auth });
+    } catch (error: Error | any) {
+      this.logger.error(`Failed to create asset from edits: ${error}`, error?.stack);
+      await this.jobRepository.queue({ name: JobName.DELETE_FILES, data: { files: [outputFile] } });
+      throw new InternalServerErrorException('Failed to create asset from edits');
+    }
+  }
+
+  private asMediaEdits(edits: EditorAction[]) {
+    const mediaEdits: MediaEditItem[] = [];
+    for (const { action, ...options } of edits) {
+      switch (action) {
+        case EditorActionType.Crop: {
+          mediaEdits.push({ ...(options as EditorActionCrop), action: 'crop' });
+          break;
+        }
+
+        case EditorActionType.Rotate: {
+          mediaEdits.push({ ...(options as EditorActionRotate), action: 'rotate' });
+          break;
+        }
+
+        case EditorActionType.Blur: {
+          mediaEdits.push({ ...(options as EditorActionBlur), action: 'blur' });
+          break;
+        }
+
+        case EditorActionType.Adjust: {
+          mediaEdits.push({ ...(options as EditorActionAdjust), action: 'modulate' });
+          break;
+        }
+      }
+    }
+
+    return mediaEdits;
+  }
+}

server/src/services/index.ts

@@ -10,6 +10,7 @@ import { CliService } from 'src/services/cli.service';
 import { DatabaseService } from 'src/services/database.service';
 import { DownloadService } from 'src/services/download.service';
 import { DuplicateService } from 'src/services/duplicate.service';
+import { EditorService } from 'src/services/editor.service';
 import { JobService } from 'src/services/job.service';
 import { LibraryService } from 'src/services/library.service';
 import { MapService } from 'src/services/map.service';
@@ -50,6 +51,7 @@ export const services = [
   DatabaseService,
   DownloadService,
   DuplicateService,
+  EditorService,
   JobService,
   LibraryService,
   MapService,

server/src/services/job.service.ts

@@ -250,7 +250,7 @@ export class JobService {
       }
 
       case JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE: {
-        if (item.data.source === 'upload' || item.data.source === 'copy') {
+        if (item.data.source === 'upload' || item.data.source === 'copy' || item.data.source === 'editor') {
           await this.jobRepository.queue({ name: JobName.GENERATE_PREVIEW, data: item.data });
         }
         break;
@@ -271,7 +271,7 @@ export class JobService {
           { name: JobName.GENERATE_THUMBHASH, data: item.data },
         ];
 
-        if (item.data.source === 'upload') {
+        if (item.data.source === 'upload' || item.data.source === 'editor') {
           jobs.push({ name: JobName.SMART_SEARCH, data: item.data }, { name: JobName.FACE_DETECTION, data: item.data });
 
           const [asset] = await this.assetRepository.getByIds([item.data.id]);
@@ -289,7 +289,7 @@ export class JobService {
       }
 
       case JobName.GENERATE_THUMBNAIL: {
-        if (item.data.source !== 'upload') {
+        if (item.data.source !== 'upload' && item.data.source !== 'editor') {
           break;
         }
 

server/test/repositories/media.repository.mock.ts

@@ -9,5 +9,6 @@ export const newMediaRepositoryMock = (): Mocked<IMediaRepository> => {
     probe: vitest.fn(),
     transcode: vitest.fn(),
     getImageDimensions: vitest.fn(),
+    applyEdits: vitest.fn(),
   };
 };