From 93863b0629c5fde6a53fd0bf7ccdea5aa6b3295f Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Wed, 17 May 2023 13:07:17 -0400 Subject: [PATCH] feat: facial recognition (#2180) --- machine-learning/.dockerignore | 4 +- machine-learning/.gitignore | 168 ++++++- machine-learning/Dockerfile | 3 +- machine-learning/src/main.py | 71 ++- mobile/openapi/.openapi-generator/FILES | 9 + mobile/openapi/README.md | Bin 16326 -> 16901 bytes mobile/openapi/doc/AllJobStatusResponseDto.md | Bin 942 -> 1010 bytes mobile/openapi/doc/AssetResponseDto.md | Bin 1297 -> 1400 bytes mobile/openapi/doc/PersonApi.md | Bin 0 -> 10296 bytes mobile/openapi/doc/PersonResponseDto.md | Bin 0 -> 474 bytes mobile/openapi/doc/PersonUpdateDto.md | Bin 0 -> 409 bytes mobile/openapi/lib/api.dart | Bin 5116 -> 5220 bytes mobile/openapi/lib/api/person_api.dart | Bin 0 -> 8362 bytes mobile/openapi/lib/api_client.dart | Bin 16830 -> 16998 bytes .../model/all_job_status_response_dto.dart | Bin 6443 -> 6834 bytes .../openapi/lib/model/asset_response_dto.dart | Bin 10173 -> 10409 bytes mobile/openapi/lib/model/job_name.dart | Bin 3709 -> 3883 bytes .../lib/model/person_response_dto.dart | Bin 0 -> 3771 bytes .../openapi/lib/model/person_update_dto.dart | Bin 0 -> 3247 bytes .../all_job_status_response_dto_test.dart | Bin 1542 -> 1671 bytes .../openapi/test/asset_response_dto_test.dart | Bin 2732 -> 2872 bytes mobile/openapi/test/person_api_test.dart | Bin 0 -> 1128 bytes .../test/person_response_dto_test.dart | Bin 0 -> 767 bytes .../openapi/test/person_update_dto_test.dart | Bin 0 -> 561 bytes .../src/api-v1/asset/asset-repository.ts | 31 +- .../immich/src/api-v1/asset/asset.core.ts | 1 + .../immich/src/api-v1/asset/asset.service.ts | 8 + server/apps/immich/src/app.cron-jobs.ts | 10 +- server/apps/immich/src/app.module.ts | 2 + server/apps/immich/src/controllers/index.ts | 1 + .../src/controllers/person.controller.ts | 57 +++ .../microservices/src/microservices.module.ts | 2 + server/apps/microservices/src/processors.ts | 50 ++ server/immich-openapi-specs.json | 258 +++++++++- .../libs/domain/src/asset/asset.repository.ts | 1 + .../asset/response-dto/asset-response.dto.ts | 6 +- server/libs/domain/src/domain.module.ts | 6 +- .../src/facial-recognition/face.repository.ts | 14 + .../facial-recognition.service.spec.ts | 320 +++++++++++++ .../facial-recognition.services.ts | 144 ++++++ .../domain/src/facial-recognition/index.ts | 2 + server/libs/domain/src/index.ts | 2 + server/libs/domain/src/job/job.constants.ts | 10 + server/libs/domain/src/job/job.interface.ts | 14 + server/libs/domain/src/job/job.repository.ts | 15 +- .../libs/domain/src/job/job.service.spec.ts | 12 + server/libs/domain/src/job/job.service.ts | 8 + .../all-job-status-response.dto.ts | 3 + server/libs/domain/src/media/index.ts | 1 + .../libs/domain/src/media/media.constant.ts | 3 + .../libs/domain/src/media/media.repository.ts | 10 +- .../domain/src/media/media.service.spec.ts | 1 + server/libs/domain/src/media/media.service.ts | 9 +- server/libs/domain/src/person/dto/index.ts | 1 + .../src/person/dto/person-update.dto.ts | 7 + server/libs/domain/src/person/index.ts | 4 + .../domain/src/person/person.repository.ts | 19 + .../domain/src/person/person.service.spec.ts | 135 ++++++ .../libs/domain/src/person/person.service.ts | 82 ++++ .../domain/src/person/response-dto/index.ts | 1 + .../response-dto/person-response.dto.ts | 19 + .../domain/src/search/search.repository.ts | 20 +- .../domain/src/search/search.service.spec.ts | 129 ++++- .../libs/domain/src/search/search.service.ts | 94 +++- .../smart-info/machine-learning.interface.ts | 16 + .../libs/domain/src/user/user.service.spec.ts | 4 +- server/libs/domain/src/user/user.service.ts | 2 +- .../libs/domain/test/face.repository.mock.ts | 9 + server/libs/domain/test/fixtures.ts | 51 ++ server/libs/domain/test/index.ts | 2 + .../test/machine-learning.repository.mock.ts | 1 + .../libs/domain/test/media.repository.mock.ts | 1 + .../domain/test/person.repository.mock.ts | 15 + .../domain/test/search.repository.mock.ts | 4 + .../infra/src/entities/asset-face.entity.ts | 25 + .../libs/infra/src/entities/asset.entity.ts | 5 + server/libs/infra/src/entities/index.ts | 8 +- .../libs/infra/src/entities/person.entity.ts | 38 ++ server/libs/infra/src/infra.module.ts | 6 + .../1684255168091-AddFacialTables.ts | 22 + .../src/repositories/asset.repository.ts | 21 + .../infra/src/repositories/face.repository.ts | 22 + server/libs/infra/src/repositories/index.ts | 2 + .../infra/src/repositories/job.repository.ts | 21 + .../machine-learning.repository.ts | 6 +- .../src/repositories/media.repository.ts | 15 +- .../src/repositories/person.repository.ts | 78 +++ .../src/repositories/typesense.repository.ts | 86 +++- .../src/typesense-schemas/asset.schema.ts | 3 +- .../src/typesense-schemas/face.schema.ts | 12 + .../libs/infra/src/typesense-schemas/index.ts | 1 + web/src/api/api.ts | 8 + web/src/api/open-api/api.ts | 451 ++++++++++++++++++ .../admin-page/jobs/jobs-panel.svelte | 4 + .../components/album-page/album-viewer.svelte | 6 +- .../asset-viewer/asset-viewer.svelte | 6 + .../asset-viewer/detail-panel.svelte | 52 +- .../assets/thumbnail/image-thumbnail.svelte | 14 +- .../faces-page/edit-name-input.svelte | 42 ++ web/src/lib/constants.ts | 1 + web/src/routes/(user)/explore/+page.server.ts | 3 +- web/src/routes/(user)/explore/+page.svelte | 59 ++- web/src/routes/(user)/people/+page.server.ts | 19 + web/src/routes/(user)/people/+page.svelte | 48 ++ .../(user)/people/[personId]/+page.server.ts | 21 + .../(user)/people/[personId]/+page.svelte | 113 +++++ web/src/routes/(user)/search/+page.svelte | 2 +- 107 files changed, 2965 insertions(+), 127 deletions(-) create mode 100644 mobile/openapi/doc/PersonApi.md create mode 100644 mobile/openapi/doc/PersonResponseDto.md create mode 100644 mobile/openapi/doc/PersonUpdateDto.md create mode 100644 mobile/openapi/lib/api/person_api.dart create mode 100644 mobile/openapi/lib/model/person_response_dto.dart create mode 100644 mobile/openapi/lib/model/person_update_dto.dart create mode 100644 mobile/openapi/test/person_api_test.dart create mode 100644 mobile/openapi/test/person_response_dto_test.dart create mode 100644 mobile/openapi/test/person_update_dto_test.dart create mode 100644 server/apps/immich/src/controllers/person.controller.ts create mode 100644 server/libs/domain/src/facial-recognition/face.repository.ts create mode 100644 server/libs/domain/src/facial-recognition/facial-recognition.service.spec.ts create mode 100644 server/libs/domain/src/facial-recognition/facial-recognition.services.ts create mode 100644 server/libs/domain/src/facial-recognition/index.ts create mode 100644 server/libs/domain/src/media/media.constant.ts create mode 100644 server/libs/domain/src/person/dto/index.ts create mode 100644 server/libs/domain/src/person/dto/person-update.dto.ts create mode 100644 server/libs/domain/src/person/index.ts create mode 100644 server/libs/domain/src/person/person.repository.ts create mode 100644 server/libs/domain/src/person/person.service.spec.ts create mode 100644 server/libs/domain/src/person/person.service.ts create mode 100644 server/libs/domain/src/person/response-dto/index.ts create mode 100644 server/libs/domain/src/person/response-dto/person-response.dto.ts create mode 100644 server/libs/domain/test/face.repository.mock.ts create mode 100644 server/libs/domain/test/person.repository.mock.ts create mode 100644 server/libs/infra/src/entities/asset-face.entity.ts create mode 100644 server/libs/infra/src/entities/person.entity.ts create mode 100644 server/libs/infra/src/migrations/1684255168091-AddFacialTables.ts create mode 100644 server/libs/infra/src/repositories/face.repository.ts create mode 100644 server/libs/infra/src/repositories/person.repository.ts create mode 100644 server/libs/infra/src/typesense-schemas/face.schema.ts create mode 100644 web/src/lib/components/faces-page/edit-name-input.svelte create mode 100644 web/src/routes/(user)/people/+page.server.ts create mode 100644 web/src/routes/(user)/people/+page.svelte create mode 100644 web/src/routes/(user)/people/[personId]/+page.server.ts create mode 100644 web/src/routes/(user)/people/[personId]/+page.svelte diff --git a/machine-learning/.dockerignore b/machine-learning/.dockerignore index eba74f4cd2..d5a4cf7d2b 100644 --- a/machine-learning/.dockerignore +++ b/machine-learning/.dockerignore @@ -1 +1,3 @@ -venv/ \ No newline at end of file +venv/ +*.zip +*.onnx \ No newline at end of file diff --git a/machine-learning/.gitignore b/machine-learning/.gitignore index 14d9798b60..e31c7773ee 100644 --- a/machine-learning/.gitignore +++ b/machine-learning/.gitignore @@ -3,4 +3,170 @@ upload/ venv/ __pycache__/ -model-cache/ \ No newline at end of file +model-cache/ + + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ + + +*.onnx +*.zip \ No newline at end of file diff --git a/machine-learning/Dockerfile b/machine-learning/Dockerfile index 77fc960f33..e9896807ba 100644 --- a/machine-learning/Dockerfile +++ b/machine-learning/Dockerfile @@ -8,7 +8,8 @@ RUN python -m venv /opt/venv RUN /opt/venv/bin/pip install torch --index-url https://download.pytorch.org/whl/cpu RUN /opt/venv/bin/pip install transformers tqdm numpy scikit-learn scipy nltk sentencepiece fastapi Pillow uvicorn[standard] RUN /opt/venv/bin/pip install --no-deps sentence-transformers - +# Facial Recognition Stuff +RUN /opt/venv/bin/pip install insightface onnxruntime FROM python:3.10-slim diff --git a/machine-learning/src/main.py b/machine-learning/src/main.py index cd6debbfbf..b09a43464f 100644 --- a/machine-learning/src/main.py +++ b/machine-learning/src/main.py @@ -1,9 +1,13 @@ +import os +import numpy as np +import cv2 as cv +import uvicorn + +from insightface.app import FaceAnalysis from transformers import pipeline from sentence_transformers import SentenceTransformer, util from PIL import Image from fastapi import FastAPI -import uvicorn -import os from pydantic import BaseModel @@ -15,15 +19,6 @@ class ClipRequestBody(BaseModel): text: str -is_dev = os.getenv('NODE_ENV') == 'development' -server_port = os.getenv('MACHINE_LEARNING_PORT', 3003) -server_host = os.getenv('MACHINE_LEARNING_HOST', '0.0.0.0') - -app = FastAPI() - -""" -Model Initialization -""" classification_model = os.getenv( 'MACHINE_LEARNING_CLASSIFICATION_MODEL', 'microsoft/resnet-50') object_model = os.getenv('MACHINE_LEARNING_OBJECT_MODEL', 'hustvl/yolos-tiny') @@ -31,9 +26,15 @@ clip_image_model = os.getenv( 'MACHINE_LEARNING_CLIP_IMAGE_MODEL', 'clip-ViT-B-32') clip_text_model = os.getenv( 'MACHINE_LEARNING_CLIP_TEXT_MODEL', 'clip-ViT-B-32') +facial_recognition_model = os.getenv( + 'MACHINE_LEARNING_FACIAL_RECOGNITION_MODEL', 'buffalo_l') + +cache_folder = os.getenv('MACHINE_LEARNING_CACHE_FOLDER', '/cache') _model_cache = {} +app = FastAPI() + @app.get("/") async def root(): @@ -73,6 +74,36 @@ def clip_encode_text(payload: ClipRequestBody): return model.encode(text).tolist() +@app.post("/facial-recognition/detect-faces", status_code=200) +def facial_recognition(payload: MlRequestBody): + model = _get_model(facial_recognition_model, 'facial-recognition') + assetPath = payload.thumbnailPath + img = cv.imread(assetPath) + height, width, _ = img.shape + results = [] + faces = model.get(img) + for face in faces: + if face.det_score < 0.7: + continue + x1, y1, x2, y2 = face.bbox + # min face size as percent of original image + # if (x2 - x1) / width < 0.03 or (y2 - y1) / height < 0.05: + # continue + results.append({ + "imageWidth": width, + "imageHeight": height, + "boundingBox": { + "x1": round(x1), + "y1": round(y1), + "x2": round(x2), + "y2": round(y2), + }, + "score": face.det_score.item(), + "embedding": face.normed_embedding.tolist() + }) + return results + + def run_engine(engine, path): result = [] predictions = engine(path) @@ -93,12 +124,22 @@ def _get_model(model, task=None): key = '|'.join([model, str(task)]) if key not in _model_cache: if task: - _model_cache[key] = pipeline(model=model, task=task) + if task == 'facial-recognition': + face_model = FaceAnalysis( + name=model, root=cache_folder, allowed_modules=["detection", "recognition"]) + face_model.prepare(ctx_id=0, det_size=(640, 640)) + _model_cache[key] = face_model + else: + _model_cache[key] = pipeline(model=model, task=task) else: - _model_cache[key] = SentenceTransformer(model) + _model_cache[key] = SentenceTransformer( + model, cache_folder=cache_folder) return _model_cache[key] if __name__ == "__main__": - uvicorn.run("main:app", host=server_host, - port=int(server_port), reload=is_dev, workers=1) + host = os.getenv('MACHINE_LEARNING_HOST', '0.0.0.0') + port = int(os.getenv('MACHINE_LEARNING_PORT', 3003)) + is_dev = os.getenv('NODE_ENV') == 'development' + + uvicorn.run("main:app", host=host, port=port, reload=is_dev, workers=1) diff --git a/mobile/openapi/.openapi-generator/FILES b/mobile/openapi/.openapi-generator/FILES index 42a8c3106d..094bcbe40a 100644 --- a/mobile/openapi/.openapi-generator/FILES +++ b/mobile/openapi/.openapi-generator/FILES @@ -61,6 +61,9 @@ doc/OAuthCallbackDto.md doc/OAuthConfigDto.md doc/OAuthConfigResponseDto.md doc/PartnerApi.md +doc/PersonApi.md +doc/PersonResponseDto.md +doc/PersonUpdateDto.md doc/QueueStatusDto.md doc/RemoveAssetsDto.md doc/SearchAlbumResponseDto.md @@ -113,6 +116,7 @@ lib/api/authentication_api.dart lib/api/job_api.dart lib/api/o_auth_api.dart lib/api/partner_api.dart +lib/api/person_api.dart lib/api/search_api.dart lib/api/server_info_api.dart lib/api/share_api.dart @@ -178,6 +182,8 @@ lib/model/map_marker_response_dto.dart lib/model/o_auth_callback_dto.dart lib/model/o_auth_config_dto.dart lib/model/o_auth_config_response_dto.dart +lib/model/person_response_dto.dart +lib/model/person_update_dto.dart lib/model/queue_status_dto.dart lib/model/remove_assets_dto.dart lib/model/search_album_response_dto.dart @@ -274,6 +280,9 @@ test/o_auth_callback_dto_test.dart test/o_auth_config_dto_test.dart test/o_auth_config_response_dto_test.dart test/partner_api_test.dart +test/person_api_test.dart +test/person_response_dto_test.dart +test/person_update_dto_test.dart test/queue_status_dto_test.dart test/remove_assets_dto_test.dart test/search_album_response_dto_test.dart diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index 25c6c7a97a3701975c8f18b5560a9acfca76d82a..1e87cce85581837641043891150d30829f2dc32c 100644 GIT binary patch delta 562 zcmX?B-`c{sVU0q4Kx$EOex74Nrj|mDLbR4vdTNPdPEJ5-enC#EmR77rN`A7wK3tJr zZi+HcMPg1)0a%45P?MIHyK9J+mV$l(*eIX~mlj+D!c>S5x{>t-nNTw!y0I9jU!9p! zi{Dho;^NejVsx|7j7|hALbD(Y+w1Bo%uIL3{yGfgDBwp-6#_2&3TC#G>R3P<(FQrXV9G2Qf4#wYVTZuQ=7E cBp)S8Co5QrC_ru2VnpJ diff --git a/mobile/openapi/doc/AllJobStatusResponseDto.md b/mobile/openapi/doc/AllJobStatusResponseDto.md index e0e8e379585e8b3f1142632dc5ae2449f029cf28..cac4b0f44b83faabc4c79475241d57afe96e76f6 100644 GIT binary patch delta 32 ncmZ3-{)v49C$oT-R#9qletKSJRjONJa%%BpTSk+~oXkrBv@{C9 delta 11 ScmeywzK(qZC-Y=q=A{4{00Z*? diff --git a/mobile/openapi/doc/AssetResponseDto.md b/mobile/openapi/doc/AssetResponseDto.md index ea9c3784351d04412192edf86cf225ac16ec6302..2bb059ceb29b51a6bfaa4524297961c214ca4a71 100644 GIT binary patch delta 65 zcmbQp^@D3e4vV3dRzYfhK~AccmO_m}w3e1nW^sv4Kx$EOeqK;&aY24wajHv6zMYm< QtOi7M@=j)j$zd$J0pkc3@c;k- delta 11 ScmeytHIZvW4$I`rEK30$UIfhm diff --git a/mobile/openapi/doc/PersonApi.md b/mobile/openapi/doc/PersonApi.md new file mode 100644 index 0000000000000000000000000000000000000000..dd1c0eb8e41bfe56159ddd2de18e166253d5cd69 GIT binary patch literal 10296 zcmeHMZExE)5dN-TaXtd2h(N^4XHa6hBWD)2gLc9A86bo z4`l?O;BBwhC5gYrm}+|6$23g1(p(D1FF({*e)X&G&bjAZqb9>$lqyEgyKswz1*2$# zI$0d6tg}g@J?~AY=Xual+489+S7ApzHr`4;GPrfeqx-6ccABD27wchJ{~~QI+g(-K z8;(bP!Faq_ubzr5R9#5q+Q%wl8W*hFc9xgkJuCUGSULp{`E-5@oLzX&Q~lDaKEYH; zk>b9VyHFN%n+`o26nvdqHloe@l6Y&ipm)4~40}=x`1P3L%>?cZPJoRm#uEMy1)mf< z5K4K&(P_0J9I$b$Dd6{FjzR}!E+k9;@-P#ANqq-6F^#)!!wzgT&0(Qd_#GM_j?Ydm z$l%bM!VKt2U*%q2D-(xAlb^vGe zbE*?AuAwJyP%P)cw^}P}ArCXFn4Cy)(+)#S(@dM22ix9W=k)ZlcYM$}YBaNu%j`d~ z_Fq%{nJ|ZsLmm$4bO}u8Oj8)rgaR|$e1;P_PBOi3tcXmeL)|9aK@bFR|CJZ8*d(pP z8F0a3FuTM>s+kCp4wwauw&Jc4(a8kv9H7SvCz#UtC9(3T>=aMY-wZSIigrc2R-T&f zfzCI%$3pUTXNMe^m=LSxqJc%d69S4vfSxFH3<)j3%o*gDkd&y6(9=x7gi!%^ zq{qV6g&PE|lTLg8uwz)zHa0imJU3AT}-Js$KBSS^dUl1KYvBv9R@6VbStP@;!QTwPeF9jY*5XfhAuX#olo7^DDj6BsxzW~{74ya;mi}zacph=z1KKmJ zXB%B^wyh?;n)FukS&$#&f{|;|TQ}0POvfW)xu?&cZJEb^9uBtr?d^W#_n$mPe|xL{ z+pqWx2U}aw3z$(nWircC<_wNjz?fF~=jhToBI2Y}#Bc@ulu|&TS7HuKzbqG|QFFPX zL&(e{%JodvXTO6y?~_sfIR8Gt?V5QF8LtQPPSfsb?wP%pKA5u9)9$bpt*N-qKUbKc z*Hpaj`RBK#;u$EdjF7E~wWH-MEq(y!hMTL6byA$C``=emTq<4@J0DUFhf^s;zveN{ zGUGz1=H;4~YhHF6bzOKtUHIGb@*FO$jG7B8Ysb!WaBfI>uHnaGWEpDIeyS5>qjfG< z&tLpLTCpyR5F>Ab%C5`LvMXb{$*rr{_0{dFF1PE{`R7iZEx}g+r??$1Ac!t|P4(N> z^orK3y1U%X@n5D z=2buey$-f^WN@rojEVdfQy`zPVG^RM;E}We9uOuG1l_;tBZ{u;jE*d<3lte->FSe^?vtHHCREGMK nL(jULq+Tehsk}6Y&+TRV@Xy^`r9y84S?(I#&KJ35(;^2qTeIV;o(12hdjvUMo0gcAjm9p3DbpznKUvw#nYv*7H~zr6@9 zs%Rz5?kaUs>T(%o@ifB52Eq?~eowWuG!EG(@f-`g>hX&!kG;&EaJ0LDNO?f?J) delta 17 ZcmaE&@kf2bD(20{m<2dDEAcw90{}-422B6} diff --git a/mobile/openapi/lib/api/person_api.dart b/mobile/openapi/lib/api/person_api.dart new file mode 100644 index 0000000000000000000000000000000000000000..37f8bf8a307358c579014b775e6557a0c73d3db6 GIT binary patch literal 8362 zcmeHM-BTMk5P$byvB^VhcU(JV`p_YK)B%$)lYmRiWI7Dv2;UkjJxhwDi{mo*#P&6QNw_9*!N&Bhn5JKg+7NSX7LyO8FqR41=Mf#qha9yhHwzDoxj@s8r8uW; zcEm*gx0n6Fh-p))&~X;z5TruE6mQb!Ri#oH#LO7zk`z+36^Dxo%&uxDw6qg*I(i$3 zApAY8!}{zBHVgl^w;{qIOJbW7(jfptE?5lt+Kj^AZIP)-0y6X102A=G*Xu&{O{WJf zW}3j?gHw5}r2cpuL3d;6}E#1?Onnaok0zV*JSM=&iMC zUkqif4$Mr&z$*-pJ9{0J#vz%{TfQG587Q1=h<&DsVs=D|>uM&T*JBYIuQ|D7z~aE* z@a!`V$ec(*Mqh)$F36 zT(*=MA-gW>m95Rh&sK^;Ho57Hm7T&ih$M~)nWeivvI1S*+2qn`k%HtMH+CZ+Kf}we zm0R0o&|CpZ@RLq4=fFQ<++G8-;0Zp2T1M2Qh_;E@krBeyCOAZX@GH$T79CNNKx6AE zE`czv<-~-E&4$@kX1`py8(iv;(NBcxy=e~zU^FsQ$x z&I3zGI3pG?p@^YPL133X)?WVFfC=p%F*3H0mg3zV2;92x^PJn(2d3p?bR-kaErFBP zxM{#Amtoh*M|7P1L|{-=Naogf-D~A?HBa0G<~wNcE6Lir#rv@{W)@Av$KxDWkzzJx9cHPXt8d zp;QY9%F}BzLh=S@G5;IeE=A=&bo?7oU4q-H%V^+k*1fr$H<|;s5aYGN7AqZ1jqu^g z0zg8B^&ypq&5dcgb-|;{@6XkqUtR>)>2z8N-|!Qhgw%5iJaVBEN@v~eT9o5M7?%=! zpeORiy{7u0q&F4WSEGvA0I9^(s$TmB9hjMNzNoCK!T+xs+!2Oz1tipuuv}XP!gAus zlLkGlEI_h@@T!bF;WCnD+Zzi(*s#2^P;r=pW7Vg>^qIc!oD%@$^VnTeev4WNwB|=< z{g7=6x^{4_=hr&!a|Ity+nw~0MT1lgars+)rD1iK`p_R zM(WQr4W`25 z+_8$7MH(2Ng7PmZ@`anmRf(qzTuSjZ8y2MP+KlCuBOcR8i-sFlq<a}6*~Jw0V|;e)Bpeg diff --git a/mobile/openapi/lib/model/all_job_status_response_dto.dart b/mobile/openapi/lib/model/all_job_status_response_dto.dart index aced194303c0bd34c603353204c773118896a7f9..bcf1990fac0b99ac4a1fd23298f3cba16a855414 100644 GIT binary patch delta 318 zcmZ2&w8?bC4W`LEnFRQXQj_!3^D?Va-4c^ii#MNVa%7y$$;^u`&c@=yWTK#^#-#uP z`6U^tMS7@O14~m&Qx$A&6;OpY`?A$DPF~N>fo|en_H~S^I%ulFhS;j08Ii_Wz+|SN zkX4+Y7hR+dm()!IIa#+5WU+dzf~`VHMrJXZt2UqJc3~4xK(l6Y7QYHVnmL>Q3Tm(` SD=6SFUkB*(T5GOaE-nD_2X9vZ delta 49 zcmV-10M7riHLEhP*aEZv0yqJ)Tm(V_v)2ZC0kd`pp8>N!3Tpzh>kK&tvw0I12L^pR H3VjL+*SHXN diff --git a/mobile/openapi/lib/model/asset_response_dto.dart b/mobile/openapi/lib/model/asset_response_dto.dart index e9f0185ce8e9eac69c06dc5a4b08cb93b59c0333..a238e73497eda5e4e499afd3576a017837805929 100644 GIT binary patch delta 211 zcmdn%zcO&cMP^D8C+)fDTlJtqN4M xS+Jf>UO^!XXj^oVIz&)CR>2m;MF0N3Q?? delta 42 zcmV+_0M-AgQN2&F(F3#X1GxmV$q0%AvvLee1GCi+Bm%Pu5`PA>tr)%tv#cTg28r4c AOaK4? diff --git a/mobile/openapi/lib/model/job_name.dart b/mobile/openapi/lib/model/job_name.dart index 9d9424515e24f930bdff50e642a33cb99ebc699a..b76e44e3ff25f0df2290298da2e8c6654fe2fa1c 100644 GIT binary patch delta 126 zcmew>vs!LL60>1ZYI1&hUS?IQTVir*abRg`X{v&)f>(Z$Ut(^mUc5$;I$VWr8c4}x kMrO&)Nz5-9`QQ>@t(zTKCE5AmVo=3_yb|c@oO!I60A3<2vj6}9 delta 22 ecmZ22_g7{^67%LY%r6-?uVdw7-^|Xd!UOCtE~8o{f-Z)T@|O|PccuWqK7@cPaBX$Y4yxSHL-huP(u*Z&@& z7)icNnYQDz=*8K9Ud63cn#U`p z^RK1Qs4m$WKMSVu+j4EtxHgB?6D5sh(#9f1hhinTcIxJ!vs_4S;#|pF6tfwV@o&Gz zNyfAp4A7kgwE|Ue$qEtS|HWXC8D$1dCo0aROWFr5_Y(@gR6O0?+lhs1rf=JtM zWTgGxQFa?I)i6n-ti7o8ImTbb{PVc9y-0c64%HX<xFIwZ#Wl_XOh!TxdiajHq4*=yc18Li&WSxEcsSp z(U+X&ja2fK>UiJW8JWYp5@H~J8yrtXfu@AS@t1REFhJZps=~mr%e4&}z*1;+yzeMn z3}MQWEC?|8eWggOQYCOlb-t+L1Vb2JSH}#XB0TG}?ikk>&nU*VFotp>?7-Vkv#V%` z1tu5%ujOb?N^pep3UhzF6C5mbbP(>U&da_d8tApliXc{(4as)F$-eE?elM$_u^`fO zLc%{pi`nXCTkr}-lYlA~f@Ay1kw0t5FERQ-9XABpS{-&jsNRtt?(I~m?_Bs*a#XHn zI#gtPFOUwstS6A;`fF@ohjl*BJ9oU@YPDg@98U#UvE1-*SA)ruPIFV{mWE{C`h4LV zL&PPUgc%A1k1{vHB4Ln9({*862qo$Sm2I^l_>*ewle9F+_nlf-T2@hBL0znKsGpn_H>Rsb;M{7)Qomw^yo2qi zWA(Yyq-q^YeLM95)vi2x{%PS|hsM4!KboF2HvOgc9Gd%Qj6BChy3zV80k&x0L|oO| zx{VRN32y3ip43ifDRO;pHRpl0VyWg{(M_PSpCkK$@sr-7YC|K;8((P{xiTq#(c1`{ z;S!OEuExDSoO(G_tIk=xW}rO%NFZ|vrvs=UBpo2rgfwXHAU;BP99_{y8#zDn^?&g5 zNN79;X4No$3W_kXZl8P2h^S3X@mgRC8@BFw8#V|#Kzj7m0>%h#@bW;$2?O;ROs6N^ zBJggpE^%wg0dGhK-4Yj=KgIR{aMz|r`5oyt1fSfwde9})A38&Km{iZ*lfnJKxftNY F`43H0yc_@k literal 0 HcmV?d00001 diff --git a/mobile/openapi/lib/model/person_update_dto.dart b/mobile/openapi/lib/model/person_update_dto.dart new file mode 100644 index 0000000000000000000000000000000000000000..773cf6aedf202fdb34997e60ca413ec8f53e82e1 GIT binary patch literal 3247 zcmbVOZExE)5dQ98aVd&e0Tg-7ry-fW7K=0VEgsTzz+e~xEzuEMnN&%thLQTe?~bHo zMe(*|25ed6y+6+#IT{T{BY5-qc6R#v^m=;p=5~4sS8qQ|W4N5b_3Rct&Mx0x{dI(5 zB>6gL+K$hXS7!ry6?al;p01QmSEA(SP|MozJmn={b7|w|U92mm?LiGzZrIkORc%wv zztuvcx@2qot(eBA<=UWeZ4RqvN*c?gO+}6l#Y%AP)XhO>rI6gDrIL3jW(y|MpMOfT zf@w1tpgRj{1*+nbRU*Of#bA(?%oz9sgHZBwl{3pPt%CajzzFld;kMS&00YT4Fz<;i zK)B#Co}hehGa?%RP0nBoVG>F_#`WC^x(4`UwUBo}tM2`^`p&h+OmT0iR7tUFW|fBV zIGQcL@C-As1=lAqo`A1N#>$9n$IZ@$un*Y(^~=)@hP|w#)(CgRMIH>bi3(=)UEP=BA3yySHn3O8Na08#n4sg zBddP#8ZqQs+z{CWta^v&p$T8|m5{82FSrC_cqDzlyYMw|5_3Hy>%#?#wg`Cv?XY)J zw6G(6os~5=u+nNBOwHXA(rASOL!#Ln6I{d~>k`2yq&LDic6G{L zX}&^wyUx&chV(Eg5huk)u_8;crwWx541S@{%wJsyu4JKvKD4HO7)|5?BL-nS%{st? z$~uuGW5zzJapdtP{$t!zX==@}yv*Vchc&AJgSQO3$1Suy@|AB5wNg;LR+hu7b0-su zM=Tp!$O@i)=u@MtFYut?s`heQ>tR$+!>j}yfUIRJ`J{DRiP;%u~$pZ z3MR?I? z{!3h4yr2lz!WhbkcmwY~&90LnmX{p(Psh!il;8+E8Z&>Y3+z#vZ^XEOx_s=023F%R zG=|kRPA$KyHaIrV2dJWI+d7yvzS0nIMN
sL42C-<)@J xQqhTx=lHtDttAJ%ix_lE9l-plv4`GWP+H~pq}w+2ly3G~)2NkkTNzyZ;ZK%n@0h_~o=l)iK896u6W$Ju z4dTXbr*}$P$E9P`2SZvCnnbr-%>+rbOei@7m|JnlzI|cciW^%LIU`Io?|$+DkCjHr zHI?BxjJKYgY(N}ZSH|TP5Svm$+_4av0yiPNl9i_7x1t_KPI7txe+z&BTax4gDCO@E z&fd|}gOeg)2BW;5PPWKRX7HjDl@{K4gQu5d-D!5bMXN(&uy*SF90qii5I3l_z(C{n zNp?XVQqZjbA113OSjrlqA#dB%O`<*Lcx%?1u2C@WPVbx{IRNz^TW93{zUj)9wdgi( zoF&tl-919a?;qsK3rAsK-6H%3spHwdUHS1*auT~oc%$r}B73g40=(|I!+h|bVfHd> Z_vtiD3FtJbZ}kVCYF%x|cyU;si+^u>YOnwR literal 0 HcmV?d00001 diff --git a/mobile/openapi/test/person_response_dto_test.dart b/mobile/openapi/test/person_response_dto_test.dart new file mode 100644 index 0000000000000000000000000000000000000000..9adcbe1546210fcc2e2a9d3004623818c381b44b GIT binary patch literal 767 zcmbV}K}*9x5QXpg72_!_RHHqKrJ#@~f;OergQs;(rpaKkyY6n3BK_~qrV2_6iid1= z;C=7CNs=T=61ccu=7(3=EL&VGvlK3`Z?X}jIn44U+~(=^<--BXg7UnS)=$SrC*vrL zR4Hvyth5y?YVZs?)j2E$HrS%_qpz7-qpjad$mUJ1oiLpj7GFBDj4RplX{6=1N9)+! zy8U>emGe@0K_v&$iqQ7B+Z)!9LMIxnDwg?LDsg%&idtG1Mca-L8(BQd3eU7bB@HF~ z9EGjp$&SRM^GP_q1R*rc4)~M6oU3cKKxeolrd~&{no4LTslgE5BLJ4KNXZ7M-=aX6 z?oxx{Bw}r4b!P@~u;10hBlxuJP4B!VRRNXELEnf$-Cw}h2*&bu*014_%Ev7$-C7lW vseo{?WuE3ezS-t~Hx57OSL-19kGZeA_DM-<=F-={ru zi@esB*RsVa^s0AQacr?eU5_qQQT5U}wJZ4_APc19xgj9ned@}$N*HVcVWZa=dn4dF) z;f#^8vAQ>dBv_v-k`escMZ;HbrD`DjAaM}W5U67Teve=*ZD#!ec0sf|@xmQc8AAn# Rf^CYdhz7@yfU5W&*$<+CxC;OP literal 0 HcmV?d00001 diff --git a/server/apps/immich/src/api-v1/asset/asset-repository.ts b/server/apps/immich/src/api-v1/asset/asset-repository.ts index f01d35119c..0a0d9feeee 100644 --- a/server/apps/immich/src/api-v1/asset/asset-repository.ts +++ b/server/apps/immich/src/api-v1/asset/asset-repository.ts @@ -210,7 +210,15 @@ export class AssetRepository implements IAssetRepository { where: { id: assetId, }, - relations: ['exifInfo', 'tags', 'sharedLinks', 'smartInfo'], + relations: { + exifInfo: true, + tags: true, + sharedLinks: true, + smartInfo: true, + faces: { + person: true, + }, + }, }); } @@ -239,7 +247,14 @@ export class AssetRepository implements IAssetRepository { } get(id: string): Promise { - return this.assetRepository.findOne({ where: { id } }); + return this.assetRepository.findOne({ + where: { id }, + relations: { + faces: { + person: true, + }, + }, + }); } async create( @@ -264,11 +279,6 @@ export class AssetRepository implements IAssetRepository { asset.isFavorite = dto.isFavorite ?? asset.isFavorite; asset.isArchived = dto.isArchived ?? asset.isArchived; - if (dto.tagIds) { - const tags = await this._tagRepository.getByIds(userId, dto.tagIds); - asset.tags = tags; - } - if (asset.exifInfo != null) { asset.exifInfo.description = dto.description || ''; await this.exifRepository.save(asset.exifInfo); @@ -280,7 +290,12 @@ export class AssetRepository implements IAssetRepository { asset.exifInfo = exifInfo; } - return await this.assetRepository.save(asset); + await this.assetRepository.update(asset.id, { + isFavorite: asset.isFavorite, + isArchived: asset.isArchived, + }); + + return asset; } /** diff --git a/server/apps/immich/src/api-v1/asset/asset.core.ts b/server/apps/immich/src/api-v1/asset/asset.core.ts index 972fafd445..f0a819f2f5 100644 --- a/server/apps/immich/src/api-v1/asset/asset.core.ts +++ b/server/apps/immich/src/api-v1/asset/asset.core.ts @@ -38,6 +38,7 @@ export class AssetCore { tags: [], sharedLinks: [], originalFileName: parse(file.originalName).name, + faces: [], }); await this.jobRepository.queue({ name: JobName.ASSET_UPLOADED, data: { asset, fileName: file.originalName } }); diff --git a/server/apps/immich/src/api-v1/asset/asset.service.ts b/server/apps/immich/src/api-v1/asset/asset.service.ts index dcaae0d956..3d40b7984f 100644 --- a/server/apps/immich/src/api-v1/asset/asset.service.ts +++ b/server/apps/immich/src/api-v1/asset/asset.service.ts @@ -355,6 +355,14 @@ export class AssetService { } try { + if (asset.faces) { + await Promise.all( + asset.faces.map(({ assetId, personId }) => + this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_FACE, data: { assetId, personId } }), + ), + ); + } + await this._assetRepository.remove(asset); await this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_ASSET, data: { ids: [id] } }); diff --git a/server/apps/immich/src/app.cron-jobs.ts b/server/apps/immich/src/app.cron-jobs.ts index c05f3e908f..7473c90b5b 100644 --- a/server/apps/immich/src/app.cron-jobs.ts +++ b/server/apps/immich/src/app.cron-jobs.ts @@ -1,13 +1,13 @@ -import { UserService } from '@app/domain'; +import { JobService } from '@app/domain'; import { Injectable } from '@nestjs/common'; import { Cron, CronExpression } from '@nestjs/schedule'; @Injectable() export class AppCronJobs { - constructor(private userService: UserService) {} + constructor(private jobService: JobService) {} - @Cron(CronExpression.EVERY_DAY_AT_11PM) - async onQueueUserDeleteCheck() { - await this.userService.handleQueueUserDelete(); + @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT) + async onNightlyJob() { + await this.jobService.handleNightlyJobs(); } } diff --git a/server/apps/immich/src/app.module.ts b/server/apps/immich/src/app.module.ts index 11b5f15257..7d78292aee 100644 --- a/server/apps/immich/src/app.module.ts +++ b/server/apps/immich/src/app.module.ts @@ -10,6 +10,7 @@ import { AlbumController, APIKeyController, AuthController, + PersonController, JobController, OAuthController, PartnerController, @@ -44,6 +45,7 @@ import { AppCronJobs } from './app.cron-jobs'; SharedLinkController, SystemConfigController, UserController, + PersonController, ], providers: [ // diff --git a/server/apps/immich/src/controllers/index.ts b/server/apps/immich/src/controllers/index.ts index 9846b43a33..1ffb3b79cb 100644 --- a/server/apps/immich/src/controllers/index.ts +++ b/server/apps/immich/src/controllers/index.ts @@ -4,6 +4,7 @@ export * from './auth.controller'; export * from './job.controller'; export * from './oauth.controller'; export * from './partner.controller'; +export * from './person.controller'; export * from './search.controller'; export * from './server-info.controller'; export * from './shared-link.controller'; diff --git a/server/apps/immich/src/controllers/person.controller.ts b/server/apps/immich/src/controllers/person.controller.ts new file mode 100644 index 0000000000..a2a597ca4c --- /dev/null +++ b/server/apps/immich/src/controllers/person.controller.ts @@ -0,0 +1,57 @@ +import { + AssetResponseDto, + AuthUserDto, + ImmichReadStream, + PersonResponseDto, + PersonService, + PersonUpdateDto, +} from '@app/domain'; +import { Body, Controller, Get, Param, Put, StreamableFile } from '@nestjs/common'; +import { ApiOkResponse, ApiTags } from '@nestjs/swagger'; +import { GetAuthUser } from '../decorators/auth-user.decorator'; + +import { Authenticated } from '../decorators/authenticated.decorator'; +import { UseValidation } from '../decorators/use-validation.decorator'; +import { UUIDParamDto } from './dto/uuid-param.dto'; + +function asStreamableFile({ stream, type, length }: ImmichReadStream) { + return new StreamableFile(stream, { type, length }); +} + +@ApiTags('Person') +@Controller('person') +@Authenticated() +@UseValidation() +export class PersonController { + constructor(private service: PersonService) {} + + @Get() + getAllPeople(@GetAuthUser() authUser: AuthUserDto): Promise { + return this.service.getAll(authUser); + } + + @Get(':id') + getPerson(@GetAuthUser() authUser: AuthUserDto, @Param() { id }: UUIDParamDto): Promise { + return this.service.getById(authUser, id); + } + + @Put(':id') + updatePerson( + @GetAuthUser() authUser: AuthUserDto, + @Param() { id }: UUIDParamDto, + @Body() dto: PersonUpdateDto, + ): Promise { + return this.service.update(authUser, id, dto); + } + + @Get(':id/thumbnail') + @ApiOkResponse({ content: { 'application/octet-stream': { schema: { type: 'string', format: 'binary' } } } }) + getPersonThumbnail(@GetAuthUser() authUser: AuthUserDto, @Param() { id }: UUIDParamDto) { + return this.service.getThumbnail(authUser, id).then(asStreamableFile); + } + + @Get(':id/assets') + getPersonAssets(@GetAuthUser() authUser: AuthUserDto, @Param() { id }: UUIDParamDto): Promise { + return this.service.getAssets(authUser, id); + } +} diff --git a/server/apps/microservices/src/microservices.module.ts b/server/apps/microservices/src/microservices.module.ts index 5db6916c50..76d7957964 100644 --- a/server/apps/microservices/src/microservices.module.ts +++ b/server/apps/microservices/src/microservices.module.ts @@ -6,6 +6,7 @@ import { TypeOrmModule } from '@nestjs/typeorm'; import { BackgroundTaskProcessor, ClipEncodingProcessor, + FacialRecognitionProcessor, ObjectTaggingProcessor, SearchIndexProcessor, StorageTemplateMigrationProcessor, @@ -29,6 +30,7 @@ import { MetadataExtractionProcessor } from './processors/metadata-extraction.pr StorageTemplateMigrationProcessor, BackgroundTaskProcessor, SearchIndexProcessor, + FacialRecognitionProcessor, ], }) export class MicroservicesModule {} diff --git a/server/apps/microservices/src/processors.ts b/server/apps/microservices/src/processors.ts index a8c213361d..22133718ba 100644 --- a/server/apps/microservices/src/processors.ts +++ b/server/apps/microservices/src/processors.ts @@ -1,13 +1,17 @@ import { AssetService, + FacialRecognitionService, + IAssetFaceJob, IAssetJob, IAssetUploadedJob, IBaseJob, IBulkEntityJob, IDeleteFilesJob, + IFaceThumbnailJob, IUserDeletionJob, JobName, MediaService, + PersonService, QueueName, SearchService, SmartInfoService, @@ -23,6 +27,7 @@ import { Job } from 'bull'; export class BackgroundTaskProcessor { constructor( private assetService: AssetService, + private personService: PersonService, private storageService: StorageService, private systemConfigService: SystemConfigService, private userService: UserService, @@ -43,10 +48,20 @@ export class BackgroundTaskProcessor { await this.systemConfigService.refreshConfig(); } + @Process(JobName.USER_DELETE_CHECK) + async onUserDeleteCheck() { + await this.userService.handleUserDeleteCheck(); + } + @Process(JobName.USER_DELETION) async onUserDelete(job: Job) { await this.userService.handleUserDelete(job.data); } + + @Process(JobName.PERSON_CLEANUP) + async onPersonCleanup() { + await this.personService.handlePersonCleanup(); + } } @Processor(QueueName.OBJECT_TAGGING) @@ -69,6 +84,26 @@ export class ObjectTaggingProcessor { } } +@Processor(QueueName.RECOGNIZE_FACES) +export class FacialRecognitionProcessor { + constructor(private facialRecognitionService: FacialRecognitionService) {} + + @Process({ name: JobName.QUEUE_RECOGNIZE_FACES, concurrency: 1 }) + async onQueueRecognizeFaces(job: Job) { + await this.facialRecognitionService.handleQueueRecognizeFaces(job.data); + } + + @Process({ name: JobName.RECOGNIZE_FACES, concurrency: 1 }) + async onRecognizeFaces(job: Job) { + await this.facialRecognitionService.handleRecognizeFaces(job.data); + } + + @Process({ name: JobName.GENERATE_FACE_THUMBNAIL, concurrency: 1 }) + async onGenerateFaceThumbnail(job: Job) { + await this.facialRecognitionService.handleGenerateFaceThumbnail(job.data); + } +} + @Processor(QueueName.CLIP_ENCODING) export class ClipEncodingProcessor { constructor(private smartInfoService: SmartInfoService) {} @@ -98,6 +133,11 @@ export class SearchIndexProcessor { await this.searchService.handleIndexAssets(); } + @Process(JobName.SEARCH_INDEX_FACES) + async onIndexFaces() { + await this.searchService.handleIndexFaces(); + } + @Process(JobName.SEARCH_INDEX_ALBUM) onIndexAlbum(job: Job) { this.searchService.handleIndexAlbum(job.data); @@ -108,6 +148,11 @@ export class SearchIndexProcessor { this.searchService.handleIndexAsset(job.data); } + @Process(JobName.SEARCH_INDEX_FACE) + async onIndexFace(job: Job) { + await this.searchService.handleIndexFace(job.data); + } + @Process(JobName.SEARCH_REMOVE_ALBUM) onRemoveAlbum(job: Job) { this.searchService.handleRemoveAlbum(job.data); @@ -117,6 +162,11 @@ export class SearchIndexProcessor { onRemoveAsset(job: Job) { this.searchService.handleRemoveAsset(job.data); } + + @Process(JobName.SEARCH_REMOVE_FACE) + onRemoveFace(job: Job) { + this.searchService.handleRemoveFace(job.data); + } } @Processor(QueueName.STORAGE_TEMPLATE_MIGRATION) diff --git a/server/immich-openapi-specs.json b/server/immich-openapi-specs.json index 20d2f686a4..adacbf522f 100644 --- a/server/immich-openapi-specs.json +++ b/server/immich-openapi-specs.json @@ -1988,6 +1988,221 @@ ] } }, + "/person": { + "get": { + "operationId": "getAllPeople", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/PersonResponseDto" + } + } + } + } + } + }, + "tags": [ + "Person" + ], + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ] + } + }, + "/person/{id}": { + "get": { + "operationId": "getPerson", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "format": "uuid", + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PersonResponseDto" + } + } + } + } + }, + "tags": [ + "Person" + ], + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ] + }, + "put": { + "operationId": "updatePerson", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "format": "uuid", + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PersonUpdateDto" + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PersonResponseDto" + } + } + } + } + }, + "tags": [ + "Person" + ], + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ] + } + }, + "/person/{id}/thumbnail": { + "get": { + "operationId": "getPersonThumbnail", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "format": "uuid", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/octet-stream": { + "schema": { + "type": "string", + "format": "binary" + } + } + }, + "description": "" + } + }, + "tags": [ + "Person" + ], + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ] + } + }, + "/person/{id}/assets": { + "get": { + "operationId": "getPersonAssets", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "format": "uuid", + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AssetResponseDto" + } + } + } + } + } + }, + "tags": [ + "Person" + ], + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ] + } + }, "/asset/upload": { "post": { "operationId": "uploadFile", @@ -4130,6 +4345,25 @@ "userId" ] }, + "PersonResponseDto": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "thumbnailPath": { + "type": "string" + } + }, + "required": [ + "id", + "name", + "thumbnailPath" + ] + }, "AssetResponseDto": { "type": "object", "properties": { @@ -4203,6 +4437,12 @@ "items": { "$ref": "#/components/schemas/TagResponseDto" } + }, + "people": { + "type": "array", + "items": { + "$ref": "#/components/schemas/PersonResponseDto" + } } }, "required": [ @@ -4618,6 +4858,9 @@ }, "search-queue": { "$ref": "#/components/schemas/JobStatusDto" + }, + "recognize-faces-queue": { + "$ref": "#/components/schemas/JobStatusDto" } }, "required": [ @@ -4628,7 +4871,8 @@ "clip-encoding-queue", "storage-template-migration-queue", "background-task-queue", - "search-queue" + "search-queue", + "recognize-faces-queue" ] }, "JobName": { @@ -4638,6 +4882,7 @@ "metadata-extraction-queue", "video-conversion-queue", "object-tagging-queue", + "recognize-faces-queue", "clip-encoding-queue", "background-task-queue", "storage-template-migration-queue", @@ -5375,6 +5620,17 @@ "profileImagePath" ] }, + "PersonUpdateDto": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ] + }, "CreateAssetDto": { "type": "object", "properties": { diff --git a/server/libs/domain/src/asset/asset.repository.ts b/server/libs/domain/src/asset/asset.repository.ts index 0df1a40ee8..ed3a66a29c 100644 --- a/server/libs/domain/src/asset/asset.repository.ts +++ b/server/libs/domain/src/asset/asset.repository.ts @@ -18,6 +18,7 @@ export enum WithoutProperty { EXIF = 'exif', CLIP_ENCODING = 'clip-embedding', OBJECT_TAGS = 'object-tags', + FACES = 'faces', } export const IAssetRepository = 'IAssetRepository'; diff --git a/server/libs/domain/src/asset/response-dto/asset-response.dto.ts b/server/libs/domain/src/asset/response-dto/asset-response.dto.ts index e78c4ddf9c..1cb9a58e35 100644 --- a/server/libs/domain/src/asset/response-dto/asset-response.dto.ts +++ b/server/libs/domain/src/asset/response-dto/asset-response.dto.ts @@ -1,8 +1,9 @@ import { AssetEntity, AssetType } from '@app/infra/entities'; import { ApiProperty } from '@nestjs/swagger'; +import { mapFace, PersonResponseDto } from '../../person'; import { mapTag, TagResponseDto } from '../../tag'; import { ExifResponseDto, mapExif } from './exif-response.dto'; -import { SmartInfoResponseDto, mapSmartInfo } from './smart-info-response.dto'; +import { mapSmartInfo, SmartInfoResponseDto } from './smart-info-response.dto'; export class AssetResponseDto { id!: string; @@ -28,6 +29,7 @@ export class AssetResponseDto { smartInfo?: SmartInfoResponseDto; livePhotoVideoId?: string | null; tags?: TagResponseDto[]; + people?: PersonResponseDto[]; } export function mapAsset(entity: AssetEntity): AssetResponseDto { @@ -53,6 +55,7 @@ export function mapAsset(entity: AssetEntity): AssetResponseDto { smartInfo: entity.smartInfo ? mapSmartInfo(entity.smartInfo) : undefined, livePhotoVideoId: entity.livePhotoVideoId, tags: entity.tags?.map(mapTag), + people: entity.faces?.map(mapFace), }; } @@ -79,5 +82,6 @@ export function mapAssetWithoutExif(entity: AssetEntity): AssetResponseDto { smartInfo: entity.smartInfo ? mapSmartInfo(entity.smartInfo) : undefined, livePhotoVideoId: entity.livePhotoVideoId, tags: entity.tags?.map(mapTag), + people: entity.faces?.map(mapFace), }; } diff --git a/server/libs/domain/src/domain.module.ts b/server/libs/domain/src/domain.module.ts index b2c866f3ac..d06103b0ee 100644 --- a/server/libs/domain/src/domain.module.ts +++ b/server/libs/domain/src/domain.module.ts @@ -3,27 +3,31 @@ import { AlbumService } from './album'; import { APIKeyService } from './api-key'; import { AssetService } from './asset'; import { AuthService } from './auth'; +import { FacialRecognitionService } from './facial-recognition'; import { JobService } from './job'; import { MediaService } from './media'; import { OAuthService } from './oauth'; import { PartnerService } from './partner'; +import { PersonService } from './person'; import { SearchService } from './search'; import { ServerInfoService } from './server-info'; import { ShareService } from './share'; import { SmartInfoService } from './smart-info'; import { StorageService } from './storage'; import { StorageTemplateService } from './storage-template'; -import { UserService } from './user'; import { INITIAL_SYSTEM_CONFIG, SystemConfigService } from './system-config'; +import { UserService } from './user'; const providers: Provider[] = [ AlbumService, APIKeyService, AssetService, AuthService, + FacialRecognitionService, JobService, MediaService, OAuthService, + PersonService, PartnerService, SearchService, ServerInfoService, diff --git a/server/libs/domain/src/facial-recognition/face.repository.ts b/server/libs/domain/src/facial-recognition/face.repository.ts new file mode 100644 index 0000000000..e45171854e --- /dev/null +++ b/server/libs/domain/src/facial-recognition/face.repository.ts @@ -0,0 +1,14 @@ +import { AssetFaceEntity } from '@app/infra/entities'; + +export const IFaceRepository = 'IFaceRepository'; + +export interface AssetFaceId { + assetId: string; + personId: string; +} + +export interface IFaceRepository { + getAll(): Promise; + getByIds(ids: AssetFaceId[]): Promise; + create(entity: Partial): Promise; +} diff --git a/server/libs/domain/src/facial-recognition/facial-recognition.service.spec.ts b/server/libs/domain/src/facial-recognition/facial-recognition.service.spec.ts new file mode 100644 index 0000000000..0c1f62d811 --- /dev/null +++ b/server/libs/domain/src/facial-recognition/facial-recognition.service.spec.ts @@ -0,0 +1,320 @@ +import { + assetEntityStub, + faceStub, + newAssetRepositoryMock, + newFaceRepositoryMock, + newJobRepositoryMock, + newMachineLearningRepositoryMock, + newMediaRepositoryMock, + newPersonRepositoryMock, + newSearchRepositoryMock, + newStorageRepositoryMock, + personStub, +} from '../../test'; +import { IAssetRepository, WithoutProperty } from '../asset'; +import { IJobRepository, JobName } from '../job'; +import { IMediaRepository } from '../media'; +import { IPersonRepository } from '../person'; +import { ISearchRepository } from '../search'; +import { IMachineLearningRepository } from '../smart-info'; +import { IStorageRepository } from '../storage'; +import { IFaceRepository } from './face.repository'; +import { FacialRecognitionService } from './facial-recognition.services'; + +const croppedFace = Buffer.from('Cropped Face'); + +const face = { + start: { + assetId: 'asset-1', + personId: 'person-1', + boundingBox: { + x1: 5, + y1: 5, + x2: 505, + y2: 505, + }, + imageHeight: 1000, + imageWidth: 1000, + }, + middle: { + assetId: 'asset-1', + personId: 'person-1', + boundingBox: { + x1: 100, + y1: 100, + x2: 200, + y2: 200, + }, + imageHeight: 500, + imageWidth: 400, + embedding: [1, 2, 3, 4], + score: 0.2, + }, + end: { + assetId: 'asset-1', + personId: 'person-1', + boundingBox: { + x1: 300, + y1: 300, + x2: 495, + y2: 495, + }, + imageHeight: 500, + imageWidth: 500, + }, +}; + +const faceSearch = { + noMatch: { + total: 0, + count: 0, + page: 1, + items: [], + distances: [], + facets: [], + }, + oneMatch: { + total: 1, + count: 1, + page: 1, + items: [faceStub.face1], + distances: [0.1], + facets: [], + }, + oneRemoteMatch: { + total: 1, + count: 1, + page: 1, + items: [faceStub.face1], + distances: [0.8], + facets: [], + }, +}; + +describe(FacialRecognitionService.name, () => { + let sut: FacialRecognitionService; + let assetMock: jest.Mocked; + let faceMock: jest.Mocked; + let jobMock: jest.Mocked; + let machineLearningMock: jest.Mocked; + let mediaMock: jest.Mocked; + let personMock: jest.Mocked; + let searchMock: jest.Mocked; + let storageMock: jest.Mocked; + + beforeEach(async () => { + assetMock = newAssetRepositoryMock(); + faceMock = newFaceRepositoryMock(); + jobMock = newJobRepositoryMock(); + machineLearningMock = newMachineLearningRepositoryMock(); + mediaMock = newMediaRepositoryMock(); + personMock = newPersonRepositoryMock(); + searchMock = newSearchRepositoryMock(); + storageMock = newStorageRepositoryMock(); + + mediaMock.crop.mockResolvedValue(croppedFace); + + sut = new FacialRecognitionService( + assetMock, + faceMock, + jobMock, + machineLearningMock, + mediaMock, + personMock, + searchMock, + storageMock, + ); + }); + + it('should be defined', () => { + expect(sut).toBeDefined(); + }); + + describe('handleQueueRecognizeFaces', () => { + it('should queue missing assets', async () => { + assetMock.getWithout.mockResolvedValue([assetEntityStub.image]); + await sut.handleQueueRecognizeFaces({}); + + expect(assetMock.getWithout).toHaveBeenCalledWith(WithoutProperty.FACES); + expect(jobMock.queue).toHaveBeenCalledWith({ + name: JobName.RECOGNIZE_FACES, + data: { asset: assetEntityStub.image }, + }); + }); + + it('should queue all assets', async () => { + assetMock.getAll.mockResolvedValue([assetEntityStub.image]); + personMock.deleteAll.mockResolvedValue(5); + searchMock.deleteAllFaces.mockResolvedValue(100); + + await sut.handleQueueRecognizeFaces({ force: true }); + + expect(assetMock.getAll).toHaveBeenCalled(); + expect(jobMock.queue).toHaveBeenCalledWith({ + name: JobName.RECOGNIZE_FACES, + data: { asset: assetEntityStub.image }, + }); + }); + + it('should log an error', async () => { + assetMock.getWithout.mockRejectedValue(new Error('Database unavailable')); + await sut.handleQueueRecognizeFaces({}); + }); + }); + + describe('handleRecognizeFaces', () => { + it('should skip when no resize path', async () => { + await sut.handleRecognizeFaces({ asset: assetEntityStub.noResizePath }); + expect(machineLearningMock.detectFaces).not.toHaveBeenCalled(); + }); + + it('should handle no results', async () => { + machineLearningMock.detectFaces.mockResolvedValue([]); + await sut.handleRecognizeFaces({ asset: assetEntityStub.image }); + expect(machineLearningMock.detectFaces).toHaveBeenCalledWith({ + thumbnailPath: assetEntityStub.image.resizePath, + }); + expect(faceMock.create).not.toHaveBeenCalled(); + expect(jobMock.queue).not.toHaveBeenCalled(); + }); + + it('should match existing people', async () => { + machineLearningMock.detectFaces.mockResolvedValue([face.middle]); + searchMock.searchFaces.mockResolvedValue(faceSearch.oneMatch); + + await sut.handleRecognizeFaces({ asset: assetEntityStub.image }); + + expect(faceMock.create).toHaveBeenCalledWith({ + personId: 'person-1', + assetId: 'asset-id', + embedding: [1, 2, 3, 4], + }); + expect(jobMock.queue.mock.calls).toEqual([ + [{ name: JobName.SEARCH_INDEX_FACE, data: { personId: 'person-1', assetId: 'asset-id' } }], + [{ name: JobName.SEARCH_INDEX_ASSET, data: { ids: ['asset-id'] } }], + ]); + }); + + it('should create a new person', async () => { + machineLearningMock.detectFaces.mockResolvedValue([face.middle]); + searchMock.searchFaces.mockResolvedValue(faceSearch.oneRemoteMatch); + personMock.create.mockResolvedValue(personStub.noName); + + await sut.handleRecognizeFaces({ asset: assetEntityStub.image }); + + expect(personMock.create).toHaveBeenCalledWith({ ownerId: assetEntityStub.image.ownerId }); + expect(faceMock.create).toHaveBeenCalledWith({ + personId: 'person-1', + assetId: 'asset-id', + embedding: [1, 2, 3, 4], + }); + expect(jobMock.queue.mock.calls).toEqual([ + [ + { + name: JobName.GENERATE_FACE_THUMBNAIL, + data: { + assetId: 'asset-1', + personId: 'person-1', + boundingBox: { + x1: 100, + y1: 100, + x2: 200, + y2: 200, + }, + imageHeight: 500, + imageWidth: 400, + score: 0.2, + }, + }, + ], + [{ name: JobName.SEARCH_INDEX_FACE, data: { personId: 'person-1', assetId: 'asset-id' } }], + [{ name: JobName.SEARCH_INDEX_ASSET, data: { ids: ['asset-id'] } }], + ]); + }); + + it('should log an error', async () => { + machineLearningMock.detectFaces.mockRejectedValue(new Error('machine learning unavailable')); + await sut.handleRecognizeFaces({ asset: assetEntityStub.image }); + }); + }); + + describe('handleGenerateFaceThumbnail', () => { + it('should skip an asset not found', async () => { + assetMock.getByIds.mockResolvedValue([]); + + await sut.handleGenerateFaceThumbnail(face.middle); + + expect(mediaMock.crop).not.toHaveBeenCalled(); + }); + + it('should skip an asset without a thumbnail', async () => { + assetMock.getByIds.mockResolvedValue([assetEntityStub.noResizePath]); + + await sut.handleGenerateFaceThumbnail(face.middle); + + expect(mediaMock.crop).not.toHaveBeenCalled(); + }); + + it('should generate a thumbnail', async () => { + assetMock.getByIds.mockResolvedValue([assetEntityStub.image]); + + await sut.handleGenerateFaceThumbnail(face.middle); + + expect(assetMock.getByIds).toHaveBeenCalledWith(['asset-1']); + expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id'); + expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.ext', { + left: 95, + top: 95, + width: 110, + height: 110, + }); + expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', { + format: 'jpeg', + size: 250, + }); + expect(personMock.update).toHaveBeenCalledWith({ + id: 'person-1', + thumbnailPath: 'upload/thumbs/user-id/person-1.jpeg', + }); + }); + + it('should generate a thumbnail without going negative', async () => { + assetMock.getByIds.mockResolvedValue([assetEntityStub.image]); + + await sut.handleGenerateFaceThumbnail(face.start); + + expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.ext', { + left: 0, + top: 0, + width: 510, + height: 510, + }); + expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', { + format: 'jpeg', + size: 250, + }); + }); + + it('should generate a thumbnail without overflowing', async () => { + assetMock.getByIds.mockResolvedValue([assetEntityStub.image]); + + await sut.handleGenerateFaceThumbnail(face.end); + + expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.ext', { + left: 297, + top: 297, + width: 202, + height: 202, + }); + expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', { + format: 'jpeg', + size: 250, + }); + }); + + it('should log an error', async () => { + assetMock.getByIds.mockRejectedValue(new Error('Database unavailable')); + await sut.handleGenerateFaceThumbnail(face.middle); + }); + }); +}); diff --git a/server/libs/domain/src/facial-recognition/facial-recognition.services.ts b/server/libs/domain/src/facial-recognition/facial-recognition.services.ts new file mode 100644 index 0000000000..39d6b0ffbf --- /dev/null +++ b/server/libs/domain/src/facial-recognition/facial-recognition.services.ts @@ -0,0 +1,144 @@ +import { Inject, Logger } from '@nestjs/common'; +import { join } from 'path'; +import { IAssetRepository, WithoutProperty } from '../asset'; +import { MACHINE_LEARNING_ENABLED } from '../domain.constant'; +import { IAssetJob, IBaseJob, IFaceThumbnailJob, IJobRepository, JobName } from '../job'; +import { CropOptions, FACE_THUMBNAIL_SIZE, IMediaRepository } from '../media'; +import { IPersonRepository } from '../person/person.repository'; +import { ISearchRepository } from '../search/search.repository'; +import { IMachineLearningRepository } from '../smart-info'; +import { IStorageRepository, StorageCore, StorageFolder } from '../storage'; +import { AssetFaceId, IFaceRepository } from './face.repository'; + +export class FacialRecognitionService { + private logger = new Logger(FacialRecognitionService.name); + private storageCore = new StorageCore(); + + constructor( + @Inject(IAssetRepository) private assetRepository: IAssetRepository, + @Inject(IFaceRepository) private faceRepository: IFaceRepository, + @Inject(IJobRepository) private jobRepository: IJobRepository, + @Inject(IMachineLearningRepository) private machineLearning: IMachineLearningRepository, + @Inject(IMediaRepository) private mediaRepository: IMediaRepository, + @Inject(IPersonRepository) private personRepository: IPersonRepository, + @Inject(ISearchRepository) private searchRepository: ISearchRepository, + @Inject(IStorageRepository) private storageRepository: IStorageRepository, + ) {} + + async handleQueueRecognizeFaces({ force }: IBaseJob) { + try { + const assets = force + ? await this.assetRepository.getAll() + : await this.assetRepository.getWithout(WithoutProperty.FACES); + + if (force) { + const people = await this.personRepository.deleteAll(); + const faces = await this.searchRepository.deleteAllFaces(); + this.logger.debug(`Deleted ${people} people and ${faces} faces`); + } + for (const asset of assets) { + await this.jobRepository.queue({ name: JobName.RECOGNIZE_FACES, data: { asset } }); + } + } catch (error: any) { + this.logger.error(`Unable to queue recognize faces`, error?.stack); + } + } + + async handleRecognizeFaces(data: IAssetJob) { + const { asset } = data; + + if (!MACHINE_LEARNING_ENABLED || !asset.resizePath) { + return; + } + + try { + const faces = await this.machineLearning.detectFaces({ thumbnailPath: asset.resizePath }); + + this.logger.debug(`${faces.length} faces detected in ${asset.resizePath}`); + this.logger.verbose(faces.map((face) => ({ ...face, embedding: `float[${face.embedding.length}]` }))); + + for (const { embedding, ...rest } of faces) { + const faceSearchResult = await this.searchRepository.searchFaces(embedding, { ownerId: asset.ownerId }); + + let personId: string | null = null; + + // try to find a matching face and link to the associated person + // The closer to 0, the better the match. Range is from 0 to 2 + if (faceSearchResult.total && faceSearchResult.distances[0] < 0.6) { + this.logger.verbose(`Match face with distance ${faceSearchResult.distances[0]}`); + personId = faceSearchResult.items[0].personId; + } + + if (!personId) { + this.logger.debug('No matches, creating a new person.'); + const person = await this.personRepository.create({ ownerId: asset.ownerId }); + personId = person.id; + await this.jobRepository.queue({ + name: JobName.GENERATE_FACE_THUMBNAIL, + data: { assetId: asset.id, personId, ...rest }, + }); + } + + const faceId: AssetFaceId = { assetId: asset.id, personId }; + + await this.faceRepository.create({ ...faceId, embedding }); + await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_FACE, data: faceId }); + await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids: [asset.id] } }); + } + + // queue all faces for asset + } catch (error: any) { + this.logger.error(`Unable run facial recognition pipeline: ${asset.id}`, error?.stack); + } + } + + async handleGenerateFaceThumbnail(data: IFaceThumbnailJob) { + const { assetId, personId, boundingBox, imageWidth, imageHeight } = data; + + try { + const [asset] = await this.assetRepository.getByIds([assetId]); + if (!asset || !asset.resizePath) { + this.logger.warn(`Asset not found for facial cropping: ${assetId}`); + return null; + } + + this.logger.verbose(`Cropping face for person: ${personId}`); + + const outputFolder = this.storageCore.getFolderLocation(StorageFolder.THUMBNAILS, asset.ownerId); + const output = join(outputFolder, `${personId}.jpeg`); + this.storageRepository.mkdirSync(outputFolder); + + const { x1, y1, x2, y2 } = boundingBox; + + const halfWidth = (x2 - x1) / 2; + const halfHeight = (y2 - y1) / 2; + + const middleX = Math.round(x1 + halfWidth); + const middleY = Math.round(y1 + halfHeight); + + // zoom out 10% + const targetHalfSize = Math.floor(Math.max(halfWidth, halfHeight) * 1.1); + + // get the longest distance from the center of the image without overflowing + const newHalfSize = Math.min( + middleX - Math.max(0, middleX - targetHalfSize), + middleY - Math.max(0, middleY - targetHalfSize), + Math.min(imageWidth - 1, middleX + targetHalfSize) - middleX, + Math.min(imageHeight - 1, middleY + targetHalfSize) - middleY, + ); + + const cropOptions: CropOptions = { + left: middleX - newHalfSize, + top: middleY - newHalfSize, + width: newHalfSize * 2, + height: newHalfSize * 2, + }; + + const croppedOutput = await this.mediaRepository.crop(asset.resizePath, cropOptions); + await this.mediaRepository.resize(croppedOutput, output, { size: FACE_THUMBNAIL_SIZE, format: 'jpeg' }); + await this.personRepository.update({ id: personId, thumbnailPath: output }); + } catch (error: Error | any) { + this.logger.error(`Failed to crop face for asset: ${assetId}, person: ${personId} - ${error}`, error.stack); + } + } +} diff --git a/server/libs/domain/src/facial-recognition/index.ts b/server/libs/domain/src/facial-recognition/index.ts new file mode 100644 index 0000000000..affe817a02 --- /dev/null +++ b/server/libs/domain/src/facial-recognition/index.ts @@ -0,0 +1,2 @@ +export * from './facial-recognition.services'; +export * from './face.repository'; diff --git a/server/libs/domain/src/index.ts b/server/libs/domain/src/index.ts index b5e846fe97..7ce05c6ba3 100644 --- a/server/libs/domain/src/index.ts +++ b/server/libs/domain/src/index.ts @@ -8,10 +8,12 @@ export * from './domain.config'; export * from './domain.constant'; export * from './domain.module'; export * from './domain.util'; +export * from './facial-recognition'; export * from './job'; export * from './media'; export * from './metadata'; export * from './oauth'; +export * from './person'; export * from './search'; export * from './server-info'; export * from './partner'; diff --git a/server/libs/domain/src/job/job.constants.ts b/server/libs/domain/src/job/job.constants.ts index ab272fff24..1370a4bbe1 100644 --- a/server/libs/domain/src/job/job.constants.ts +++ b/server/libs/domain/src/job/job.constants.ts @@ -3,6 +3,7 @@ export enum QueueName { METADATA_EXTRACTION = 'metadata-extraction-queue', VIDEO_CONVERSION = 'video-conversion-queue', OBJECT_TAGGING = 'object-tagging-queue', + RECOGNIZE_FACES = 'recognize-faces-queue', CLIP_ENCODING = 'clip-encoding-queue', BACKGROUND_TASK = 'background-task-queue', STORAGE_TEMPLATE_MIGRATION = 'storage-template-migration-queue', @@ -48,16 +49,25 @@ export enum JobName { DETECT_OBJECTS = 'detect-objects', CLASSIFY_IMAGE = 'classify-image', + // facial recognition + QUEUE_RECOGNIZE_FACES = 'queue-recognize-faces', + RECOGNIZE_FACES = 'recognize-faces', + GENERATE_FACE_THUMBNAIL = 'generate-face-thumbnail', + PERSON_CLEANUP = 'person-cleanup', + // cleanup DELETE_FILES = 'delete-files', // search SEARCH_INDEX_ASSETS = 'search-index-assets', SEARCH_INDEX_ASSET = 'search-index-asset', + SEARCH_INDEX_FACE = 'search-index-face', + SEARCH_INDEX_FACES = 'search-index-faces', SEARCH_INDEX_ALBUMS = 'search-index-albums', SEARCH_INDEX_ALBUM = 'search-index-album', SEARCH_REMOVE_ALBUM = 'search-remove-album', SEARCH_REMOVE_ASSET = 'search-remove-asset', + SEARCH_REMOVE_FACE = 'search-remove-face', // clip QUEUE_ENCODE_CLIP = 'queue-clip-encode', diff --git a/server/libs/domain/src/job/job.interface.ts b/server/libs/domain/src/job/job.interface.ts index 912afb0c3f..369724bec8 100644 --- a/server/libs/domain/src/job/job.interface.ts +++ b/server/libs/domain/src/job/job.interface.ts @@ -1,4 +1,5 @@ import { AlbumEntity, AssetEntity, UserEntity } from '@app/infra/entities'; +import { BoundingBox } from '../smart-info'; export interface IBaseJob { force?: boolean; @@ -12,6 +13,19 @@ export interface IAssetJob extends IBaseJob { asset: AssetEntity; } +export interface IAssetFaceJob extends IBaseJob { + assetId: string; + personId: string; +} + +export interface IFaceThumbnailJob extends IAssetFaceJob { + imageWidth: number; + imageHeight: number; + boundingBox: BoundingBox; + assetId: string; + personId: string; +} + export interface IBulkEntityJob extends IBaseJob { ids: string[]; } diff --git a/server/libs/domain/src/job/job.repository.ts b/server/libs/domain/src/job/job.repository.ts index f4a0509627..c09ff3c129 100644 --- a/server/libs/domain/src/job/job.repository.ts +++ b/server/libs/domain/src/job/job.repository.ts @@ -1,10 +1,12 @@ import { JobName, QueueName } from './job.constants'; import { + IAssetFaceJob, IAssetJob, IAssetUploadedJob, IBaseJob, IBulkEntityJob, IDeleteFilesJob, + IFaceThumbnailJob, IUserDeletionJob, } from './job.interface'; @@ -54,6 +56,11 @@ export type JobItem = | { name: JobName.DETECT_OBJECTS; data: IAssetJob } | { name: JobName.CLASSIFY_IMAGE; data: IAssetJob } + // Recognize Faces + | { name: JobName.QUEUE_RECOGNIZE_FACES; data: IBaseJob } + | { name: JobName.RECOGNIZE_FACES; data: IAssetJob } + | { name: JobName.GENERATE_FACE_THUMBNAIL; data: IFaceThumbnailJob } + // Clip Embedding | { name: JobName.QUEUE_ENCODE_CLIP; data: IBaseJob } | { name: JobName.ENCODE_CLIP; data: IAssetJob } @@ -61,13 +68,19 @@ export type JobItem = // Filesystem | { name: JobName.DELETE_FILES; data: IDeleteFilesJob } + // Asset Deletion + | { name: JobName.PERSON_CLEANUP } + // Search | { name: JobName.SEARCH_INDEX_ASSETS } | { name: JobName.SEARCH_INDEX_ASSET; data: IBulkEntityJob } + | { name: JobName.SEARCH_INDEX_FACES } + | { name: JobName.SEARCH_INDEX_FACE; data: IAssetFaceJob } | { name: JobName.SEARCH_INDEX_ALBUMS } | { name: JobName.SEARCH_INDEX_ALBUM; data: IBulkEntityJob } | { name: JobName.SEARCH_REMOVE_ASSET; data: IBulkEntityJob } - | { name: JobName.SEARCH_REMOVE_ALBUM; data: IBulkEntityJob }; + | { name: JobName.SEARCH_REMOVE_ALBUM; data: IBulkEntityJob } + | { name: JobName.SEARCH_REMOVE_FACE; data: IAssetFaceJob }; export const IJobRepository = 'IJobRepository'; diff --git a/server/libs/domain/src/job/job.service.spec.ts b/server/libs/domain/src/job/job.service.spec.ts index bb3f62dc0a..5b95ecaa08 100644 --- a/server/libs/domain/src/job/job.service.spec.ts +++ b/server/libs/domain/src/job/job.service.spec.ts @@ -15,6 +15,17 @@ describe(JobService.name, () => { expect(sut).toBeDefined(); }); + describe('handleNightlyJobs', () => { + it('should run the scheduled jobs', async () => { + await sut.handleNightlyJobs(); + + expect(jobMock.queue.mock.calls).toEqual([ + [{ name: JobName.USER_DELETE_CHECK }], + [{ name: JobName.PERSON_CLEANUP }], + ]); + }); + }); + describe('getAllJobStatus', () => { it('should get all job statuses', async () => { jobMock.getJobCounts.mockResolvedValue({ @@ -54,6 +65,7 @@ describe(JobService.name, () => { 'storage-template-migration-queue': expectedJobStatus, 'thumbnail-generation-queue': expectedJobStatus, 'video-conversion-queue': expectedJobStatus, + 'recognize-faces-queue': expectedJobStatus, }); }); }); diff --git a/server/libs/domain/src/job/job.service.ts b/server/libs/domain/src/job/job.service.ts index fcf9696822..26824810aa 100644 --- a/server/libs/domain/src/job/job.service.ts +++ b/server/libs/domain/src/job/job.service.ts @@ -11,6 +11,11 @@ export class JobService { constructor(@Inject(IJobRepository) private jobRepository: IJobRepository) {} + async handleNightlyJobs() { + await this.jobRepository.queue({ name: JobName.USER_DELETE_CHECK }); + await this.jobRepository.queue({ name: JobName.PERSON_CLEANUP }); + } + handleCommand(queueName: QueueName, dto: JobCommandDto): Promise { this.logger.debug(`Handling command: queue=${queueName},force=${dto.force}`); @@ -73,6 +78,9 @@ export class JobService { case QueueName.THUMBNAIL_GENERATION: return this.jobRepository.queue({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force } }); + case QueueName.RECOGNIZE_FACES: + return this.jobRepository.queue({ name: JobName.QUEUE_RECOGNIZE_FACES, data: { force } }); + default: throw new BadRequestException(`Invalid job name: ${name}`); } diff --git a/server/libs/domain/src/job/response-dto/all-job-status-response.dto.ts b/server/libs/domain/src/job/response-dto/all-job-status-response.dto.ts index 5004923115..a988bbfcc2 100644 --- a/server/libs/domain/src/job/response-dto/all-job-status-response.dto.ts +++ b/server/libs/domain/src/job/response-dto/all-job-status-response.dto.ts @@ -53,4 +53,7 @@ export class AllJobStatusResponseDto implements Record @ApiProperty({ type: JobStatusDto }) [QueueName.SEARCH]!: JobStatusDto; + + @ApiProperty({ type: JobStatusDto }) + [QueueName.RECOGNIZE_FACES]!: JobStatusDto; } diff --git a/server/libs/domain/src/media/index.ts b/server/libs/domain/src/media/index.ts index 5ab3597d70..ad52a4afbf 100644 --- a/server/libs/domain/src/media/index.ts +++ b/server/libs/domain/src/media/index.ts @@ -1,2 +1,3 @@ +export * from './media.constant'; export * from './media.repository'; export * from './media.service'; diff --git a/server/libs/domain/src/media/media.constant.ts b/server/libs/domain/src/media/media.constant.ts new file mode 100644 index 0000000000..97dd3f1d72 --- /dev/null +++ b/server/libs/domain/src/media/media.constant.ts @@ -0,0 +1,3 @@ +export const JPEG_THUMBNAIL_SIZE = 1440; +export const WEBP_THUMBNAIL_SIZE = 250; +export const FACE_THUMBNAIL_SIZE = 250; diff --git a/server/libs/domain/src/media/media.repository.ts b/server/libs/domain/src/media/media.repository.ts index ff2c2e0456..bcb63ccb40 100644 --- a/server/libs/domain/src/media/media.repository.ts +++ b/server/libs/domain/src/media/media.repository.ts @@ -31,10 +31,18 @@ export interface VideoInfo { audioStreams: AudioStreamInfo[]; } +export interface CropOptions { + top: number; + left: number; + width: number; + height: number; +} + export interface IMediaRepository { // image extractThumbnailFromExif(input: string, output: string): Promise; - resize(input: string, output: string, options: ResizeOptions): Promise; + resize(input: string | Buffer, output: string, options: ResizeOptions): Promise; + crop(input: string, options: CropOptions): Promise; // video extractVideoThumbnail(input: string, output: string, size: number): Promise; diff --git a/server/libs/domain/src/media/media.service.spec.ts b/server/libs/domain/src/media/media.service.spec.ts index 0697581770..8ff0b3ce69 100644 --- a/server/libs/domain/src/media/media.service.spec.ts +++ b/server/libs/domain/src/media/media.service.spec.ts @@ -34,6 +34,7 @@ describe(MediaService.name, () => { jobMock = newJobRepositoryMock(); mediaMock = newMediaRepositoryMock(); storageMock = newStorageRepositoryMock(); + sut = new MediaService(assetMock, communicationMock, jobMock, mediaMock, storageMock, configMock); }); diff --git a/server/libs/domain/src/media/media.service.ts b/server/libs/domain/src/media/media.service.ts index 81cae61fd4..dbf7fc6b0a 100644 --- a/server/libs/domain/src/media/media.service.ts +++ b/server/libs/domain/src/media/media.service.ts @@ -7,6 +7,7 @@ import { IAssetJob, IBaseJob, IJobRepository, JobName } from '../job'; import { IStorageRepository, StorageCore, StorageFolder } from '../storage'; import { ISystemConfigRepository, SystemConfigFFmpegDto } from '../system-config'; import { SystemConfigCore } from '../system-config/system-config.core'; +import { JPEG_THUMBNAIL_SIZE, WEBP_THUMBNAIL_SIZE } from './media.constant'; import { AudioStreamInfo, IMediaRepository, VideoStreamInfo } from './media.repository'; @Injectable() @@ -57,11 +58,10 @@ export class MediaService { this.storageRepository.mkdirSync(resizePath); const jpegThumbnailPath = join(resizePath, `${asset.id}.jpeg`); - const thumbnailDimension = 1440; if (asset.type == AssetType.IMAGE) { try { await this.mediaRepository.resize(asset.originalPath, jpegThumbnailPath, { - size: thumbnailDimension, + size: JPEG_THUMBNAIL_SIZE, format: 'jpeg', }); } catch (error) { @@ -74,7 +74,7 @@ export class MediaService { if (asset.type == AssetType.VIDEO) { this.logger.log('Start Generating Video Thumbnail'); - await this.mediaRepository.extractVideoThumbnail(asset.originalPath, jpegThumbnailPath, thumbnailDimension); + await this.mediaRepository.extractVideoThumbnail(asset.originalPath, jpegThumbnailPath, JPEG_THUMBNAIL_SIZE); this.logger.log(`Generating Video Thumbnail Success ${asset.id}`); } @@ -86,6 +86,7 @@ export class MediaService { await this.jobRepository.queue({ name: JobName.CLASSIFY_IMAGE, data: { asset } }); await this.jobRepository.queue({ name: JobName.DETECT_OBJECTS, data: { asset } }); await this.jobRepository.queue({ name: JobName.ENCODE_CLIP, data: { asset } }); + await this.jobRepository.queue({ name: JobName.RECOGNIZE_FACES, data: { asset } }); this.communicationRepository.send(CommunicationEvent.UPLOAD_SUCCESS, asset.ownerId, mapAsset(asset)); } catch (error: any) { @@ -103,7 +104,7 @@ export class MediaService { const webpPath = asset.resizePath.replace('jpeg', 'webp'); try { - await this.mediaRepository.resize(asset.resizePath, webpPath, { size: 250, format: 'webp' }); + await this.mediaRepository.resize(asset.resizePath, webpPath, { size: WEBP_THUMBNAIL_SIZE, format: 'webp' }); await this.assetRepository.save({ id: asset.id, webpPath: webpPath }); } catch (error: any) { this.logger.error(`Failed to generate webp thumbnail for asset: ${asset.id}`, error.stack); diff --git a/server/libs/domain/src/person/dto/index.ts b/server/libs/domain/src/person/dto/index.ts new file mode 100644 index 0000000000..5668429a29 --- /dev/null +++ b/server/libs/domain/src/person/dto/index.ts @@ -0,0 +1 @@ +export * from './person-update.dto'; diff --git a/server/libs/domain/src/person/dto/person-update.dto.ts b/server/libs/domain/src/person/dto/person-update.dto.ts new file mode 100644 index 0000000000..72312378ca --- /dev/null +++ b/server/libs/domain/src/person/dto/person-update.dto.ts @@ -0,0 +1,7 @@ +import { IsNotEmpty, IsString } from 'class-validator'; + +export class PersonUpdateDto { + @IsNotEmpty() + @IsString() + name!: string; +} diff --git a/server/libs/domain/src/person/index.ts b/server/libs/domain/src/person/index.ts new file mode 100644 index 0000000000..02a38887b7 --- /dev/null +++ b/server/libs/domain/src/person/index.ts @@ -0,0 +1,4 @@ +export * from './dto'; +export * from './person.repository'; +export * from './person.service'; +export * from './response-dto'; diff --git a/server/libs/domain/src/person/person.repository.ts b/server/libs/domain/src/person/person.repository.ts new file mode 100644 index 0000000000..a9ae9dcbea --- /dev/null +++ b/server/libs/domain/src/person/person.repository.ts @@ -0,0 +1,19 @@ +import { AssetEntity, PersonEntity } from '@app/infra/entities'; + +export const IPersonRepository = 'IPersonRepository'; + +export interface PersonSearchOptions { + minimumFaceCount: number; +} + +export interface IPersonRepository { + getAll(userId: string, options: PersonSearchOptions): Promise; + getAllWithoutFaces(): Promise; + getById(userId: string, personId: string): Promise; + getAssets(userId: string, id: string): Promise; + + create(entity: Partial): Promise; + update(entity: Partial): Promise; + delete(entity: PersonEntity): Promise; + deleteAll(): Promise; +} diff --git a/server/libs/domain/src/person/person.service.spec.ts b/server/libs/domain/src/person/person.service.spec.ts new file mode 100644 index 0000000000..d0011283cd --- /dev/null +++ b/server/libs/domain/src/person/person.service.spec.ts @@ -0,0 +1,135 @@ +import { BadRequestException, NotFoundException } from '@nestjs/common'; +import { IJobRepository, JobName } from '..'; +import { + assetEntityStub, + authStub, + newJobRepositoryMock, + newPersonRepositoryMock, + newStorageRepositoryMock, + personStub, +} from '../../test'; +import { IStorageRepository } from '../storage'; +import { IPersonRepository } from './person.repository'; +import { PersonService } from './person.service'; +import { PersonResponseDto } from './response-dto'; + +const responseDto: PersonResponseDto = { + id: 'person-1', + name: 'Person 1', + thumbnailPath: '/path/to/thumbnail', +}; + +describe(PersonService.name, () => { + let sut: PersonService; + let personMock: jest.Mocked; + let storageMock: jest.Mocked; + let jobMock: jest.Mocked; + + beforeEach(async () => { + personMock = newPersonRepositoryMock(); + storageMock = newStorageRepositoryMock(); + jobMock = newJobRepositoryMock(); + sut = new PersonService(personMock, storageMock, jobMock); + }); + + it('should be defined', () => { + expect(sut).toBeDefined(); + }); + + describe('getAll', () => { + it('should get all people with thumbnails', async () => { + personMock.getAll.mockResolvedValue([personStub.withName, personStub.noThumbnail]); + await expect(sut.getAll(authStub.admin)).resolves.toEqual([responseDto]); + expect(personMock.getAll).toHaveBeenCalledWith(authStub.admin.id, { minimumFaceCount: 1 }); + }); + }); + + describe('getById', () => { + it('should throw a bad request when person is not found', async () => { + personMock.getById.mockResolvedValue(null); + await expect(sut.getById(authStub.admin, 'person-1')).rejects.toBeInstanceOf(BadRequestException); + }); + + it('should get a person by id', async () => { + personMock.getById.mockResolvedValue(personStub.withName); + await expect(sut.getById(authStub.admin, 'person-1')).resolves.toEqual(responseDto); + expect(personMock.getById).toHaveBeenCalledWith(authStub.admin.id, 'person-1'); + }); + }); + + describe('getThumbnail', () => { + it('should throw an error when personId is invalid', async () => { + personMock.getById.mockResolvedValue(null); + await expect(sut.getThumbnail(authStub.admin, 'person-1')).rejects.toBeInstanceOf(NotFoundException); + expect(storageMock.createReadStream).not.toHaveBeenCalled(); + }); + + it('should throw an error when person has no thumbnail', async () => { + personMock.getById.mockResolvedValue(personStub.noThumbnail); + await expect(sut.getThumbnail(authStub.admin, 'person-1')).rejects.toBeInstanceOf(NotFoundException); + expect(storageMock.createReadStream).not.toHaveBeenCalled(); + }); + + it('should serve the thumbnail', async () => { + personMock.getById.mockResolvedValue(personStub.noName); + await sut.getThumbnail(authStub.admin, 'person-1'); + expect(storageMock.createReadStream).toHaveBeenCalledWith('/path/to/thumbnail', 'image/jpeg'); + }); + }); + + describe('getAssets', () => { + it("should return a person's assets", async () => { + personMock.getAssets.mockResolvedValue([assetEntityStub.image, assetEntityStub.video]); + await sut.getAssets(authStub.admin, 'person-1'); + expect(personMock.getAssets).toHaveBeenCalledWith('admin_id', 'person-1'); + }); + }); + + describe('update', () => { + it('should throw an error when personId is invalid', async () => { + personMock.getById.mockResolvedValue(null); + await expect(sut.update(authStub.admin, 'person-1', { name: 'Person 1' })).rejects.toBeInstanceOf( + BadRequestException, + ); + expect(personMock.update).not.toHaveBeenCalled(); + }); + + it("should update a person's name", async () => { + personMock.getById.mockResolvedValue(personStub.noName); + personMock.update.mockResolvedValue(personStub.withName); + personMock.getAssets.mockResolvedValue([assetEntityStub.image]); + + await expect(sut.update(authStub.admin, 'person-1', { name: 'Person 1' })).resolves.toEqual(responseDto); + + expect(personMock.getById).toHaveBeenCalledWith('admin_id', 'person-1'); + expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', name: 'Person 1' }); + expect(jobMock.queue).toHaveBeenCalledWith({ + name: JobName.SEARCH_INDEX_ASSET, + data: { ids: [assetEntityStub.image.id] }, + }); + }); + }); + + describe('handlePersonCleanup', () => { + it('should delete people without faces', async () => { + personMock.getAllWithoutFaces.mockResolvedValue([personStub.noName]); + + await sut.handlePersonCleanup(); + + expect(personMock.delete).toHaveBeenCalledWith(personStub.noName); + expect(jobMock.queue).toHaveBeenCalledWith({ + name: JobName.DELETE_FILES, + data: { files: ['/path/to/thumbnail'] }, + }); + }); + + it('should log an error', async () => { + personMock.getAllWithoutFaces.mockResolvedValue([personStub.noName]); + personMock.delete.mockRejectedValue(new Error('database unavailable')); + + await sut.handlePersonCleanup(); + + expect(jobMock.queue).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/server/libs/domain/src/person/person.service.ts b/server/libs/domain/src/person/person.service.ts new file mode 100644 index 0000000000..80618e4524 --- /dev/null +++ b/server/libs/domain/src/person/person.service.ts @@ -0,0 +1,82 @@ +import { BadRequestException, Inject, Injectable, Logger, NotFoundException } from '@nestjs/common'; +import { AssetResponseDto, mapAsset } from '../asset'; +import { AuthUserDto } from '../auth'; +import { IJobRepository, JobName } from '../job'; +import { ImmichReadStream, IStorageRepository } from '../storage'; +import { PersonUpdateDto } from './dto'; +import { IPersonRepository } from './person.repository'; +import { mapPerson, PersonResponseDto } from './response-dto'; + +@Injectable() +export class PersonService { + readonly logger = new Logger(PersonService.name); + + constructor( + @Inject(IPersonRepository) private repository: IPersonRepository, + @Inject(IStorageRepository) private storageRepository: IStorageRepository, + @Inject(IJobRepository) private jobRepository: IJobRepository, + ) {} + + async getAll(authUser: AuthUserDto): Promise { + const people = await this.repository.getAll(authUser.id, { minimumFaceCount: 1 }); + const named = people.filter((person) => !!person.name); + const unnamed = people.filter((person) => !person.name); + return ( + [...named, ...unnamed] + // with thumbnails + .filter((person) => !!person.thumbnailPath) + .map((person) => mapPerson(person)) + ); + } + + async getById(authUser: AuthUserDto, personId: string): Promise { + const person = await this.repository.getById(authUser.id, personId); + if (!person) { + throw new BadRequestException(); + } + + return mapPerson(person); + } + + async getThumbnail(authUser: AuthUserDto, personId: string): Promise { + const person = await this.repository.getById(authUser.id, personId); + if (!person || !person.thumbnailPath) { + throw new NotFoundException(); + } + + return this.storageRepository.createReadStream(person.thumbnailPath, 'image/jpeg'); + } + + async getAssets(authUser: AuthUserDto, personId: string): Promise { + const assets = await this.repository.getAssets(authUser.id, personId); + return assets.map(mapAsset); + } + + async update(authUser: AuthUserDto, personId: string, dto: PersonUpdateDto): Promise { + const exists = await this.repository.getById(authUser.id, personId); + if (!exists) { + throw new BadRequestException(); + } + + const person = await this.repository.update({ id: personId, name: dto.name }); + + const relatedAsset = await this.getAssets(authUser, personId); + const assetIds = relatedAsset.map((asset) => asset.id); + await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids: assetIds } }); + + return mapPerson(person); + } + + async handlePersonCleanup(): Promise { + const people = await this.repository.getAllWithoutFaces(); + for (const person of people) { + this.logger.debug(`Person ${person.name || person.id} no longer has any faces, deleting.`); + try { + await this.repository.delete(person); + await this.jobRepository.queue({ name: JobName.DELETE_FILES, data: { files: [person.thumbnailPath] } }); + } catch (error: Error | any) { + this.logger.error(`Unable to delete person: ${error}`, error?.stack); + } + } + } +} diff --git a/server/libs/domain/src/person/response-dto/index.ts b/server/libs/domain/src/person/response-dto/index.ts new file mode 100644 index 0000000000..0e48d9b0e6 --- /dev/null +++ b/server/libs/domain/src/person/response-dto/index.ts @@ -0,0 +1 @@ +export * from './person-response.dto'; diff --git a/server/libs/domain/src/person/response-dto/person-response.dto.ts b/server/libs/domain/src/person/response-dto/person-response.dto.ts new file mode 100644 index 0000000000..c5e4a221ce --- /dev/null +++ b/server/libs/domain/src/person/response-dto/person-response.dto.ts @@ -0,0 +1,19 @@ +import { AssetFaceEntity, PersonEntity } from '@app/infra/entities'; + +export class PersonResponseDto { + id!: string; + name!: string; + thumbnailPath!: string; +} + +export function mapPerson(person: PersonEntity): PersonResponseDto { + return { + id: person.id, + name: person.name, + thumbnailPath: person.thumbnailPath, + }; +} + +export function mapFace(face: AssetFaceEntity): PersonResponseDto { + return mapPerson(face.person); +} diff --git a/server/libs/domain/src/search/search.repository.ts b/server/libs/domain/src/search/search.repository.ts index 9878c825d0..78885bff00 100644 --- a/server/libs/domain/src/search/search.repository.ts +++ b/server/libs/domain/src/search/search.repository.ts @@ -1,8 +1,9 @@ -import { AlbumEntity, AssetEntity, AssetType } from '@app/infra/entities'; +import { AlbumEntity, AssetEntity, AssetFaceEntity, AssetType } from '@app/infra/entities'; export enum SearchCollection { ASSETS = 'assets', ALBUMS = 'albums', + FACES = 'faces', } export enum SearchStrategy { @@ -10,6 +11,10 @@ export enum SearchStrategy { TEXT = 'TEXT', } +export interface SearchFaceFilter { + ownerId: string; +} + export interface SearchFilter { id?: string; userId: string; @@ -37,6 +42,8 @@ export interface SearchResult { page: number; /** items for page */ items: T[]; + /** score */ + distances: number[]; facets: SearchFacet[]; } @@ -56,6 +63,13 @@ export interface SearchExploreItem { }>; } +export type OwnedFaceEntity = Pick & { + /** computed as assetId|personId */ + id: string; + /** copied from asset.id */ + ownerId: string; +}; + export type SearchCollectionIndexStatus = Record; export const ISearchRepository = 'ISearchRepository'; @@ -66,13 +80,17 @@ export interface ISearchRepository { importAlbums(items: AlbumEntity[], done: boolean): Promise; importAssets(items: AssetEntity[], done: boolean): Promise; + importFaces(items: OwnedFaceEntity[], done: boolean): Promise; deleteAlbums(ids: string[]): Promise; deleteAssets(ids: string[]): Promise; + deleteFaces(ids: string[]): Promise; + deleteAllFaces(): Promise; searchAlbums(query: string, filters: SearchFilter): Promise>; searchAssets(query: string, filters: SearchFilter): Promise>; vectorSearch(query: number[], filters: SearchFilter): Promise>; + searchFaces(query: number[], filters: SearchFaceFilter): Promise>; explore(userId: string): Promise[]>; } diff --git a/server/libs/domain/src/search/search.service.spec.ts b/server/libs/domain/src/search/search.service.spec.ts index 72dc61449f..306f33777d 100644 --- a/server/libs/domain/src/search/search.service.spec.ts +++ b/server/libs/domain/src/search/search.service.spec.ts @@ -6,8 +6,10 @@ import { assetEntityStub, asyncTick, authStub, + faceStub, newAlbumRepositoryMock, newAssetRepositoryMock, + newFaceRepositoryMock, newJobRepositoryMock, newMachineLearningRepositoryMock, newSearchRepositoryMock, @@ -15,6 +17,7 @@ import { } from '../../test'; import { IAlbumRepository } from '../album/album.repository'; import { IAssetRepository } from '../asset/asset.repository'; +import { IFaceRepository } from '../facial-recognition'; import { JobName } from '../job'; import { IJobRepository } from '../job/job.repository'; import { IMachineLearningRepository } from '../smart-info'; @@ -28,20 +31,29 @@ describe(SearchService.name, () => { let sut: SearchService; let albumMock: jest.Mocked; let assetMock: jest.Mocked; + let faceMock: jest.Mocked; let jobMock: jest.Mocked; let machineMock: jest.Mocked; let searchMock: jest.Mocked; let configMock: jest.Mocked; + const makeSut = (value?: string) => { + if (value) { + configMock.get.mockReturnValue(value); + } + return new SearchService(albumMock, assetMock, faceMock, jobMock, machineMock, searchMock, configMock); + }; + beforeEach(() => { albumMock = newAlbumRepositoryMock(); assetMock = newAssetRepositoryMock(); + faceMock = newFaceRepositoryMock(); jobMock = newJobRepositoryMock(); machineMock = newMachineLearningRepositoryMock(); searchMock = newSearchRepositoryMock(); configMock = { get: jest.fn() } as unknown as jest.Mocked; - sut = new SearchService(albumMock, assetMock, jobMock, machineMock, searchMock, configMock); + sut = makeSut(); }); afterEach(() => { @@ -80,8 +92,7 @@ describe(SearchService.name, () => { }); it('should be disabled via an env variable', () => { - configMock.get.mockReturnValue('false'); - const sut = new SearchService(albumMock, assetMock, jobMock, machineMock, searchMock, configMock); + const sut = makeSut('false'); expect(sut.isEnabled()).toBe(false); }); @@ -93,8 +104,7 @@ describe(SearchService.name, () => { }); it('should return the config when search is disabled', () => { - configMock.get.mockReturnValue('false'); - const sut = new SearchService(albumMock, assetMock, jobMock, machineMock, searchMock, configMock); + const sut = makeSut('false'); expect(sut.getConfig()).toEqual({ enabled: false }); }); @@ -102,8 +112,7 @@ describe(SearchService.name, () => { describe(`bootstrap`, () => { it('should skip when search is disabled', async () => { - configMock.get.mockReturnValue('false'); - const sut = new SearchService(albumMock, assetMock, jobMock, machineMock, searchMock, configMock); + const sut = makeSut('false'); await sut.bootstrap(); @@ -115,7 +124,7 @@ describe(SearchService.name, () => { }); it('should skip schema migration if not needed', async () => { - searchMock.checkMigrationStatus.mockResolvedValue({ assets: false, albums: false }); + searchMock.checkMigrationStatus.mockResolvedValue({ assets: false, albums: false, faces: false }); await sut.bootstrap(); expect(searchMock.setup).toHaveBeenCalled(); @@ -123,21 +132,21 @@ describe(SearchService.name, () => { }); it('should do schema migration if needed', async () => { - searchMock.checkMigrationStatus.mockResolvedValue({ assets: true, albums: true }); + searchMock.checkMigrationStatus.mockResolvedValue({ assets: true, albums: true, faces: true }); await sut.bootstrap(); expect(searchMock.setup).toHaveBeenCalled(); expect(jobMock.queue.mock.calls).toEqual([ [{ name: JobName.SEARCH_INDEX_ASSETS }], [{ name: JobName.SEARCH_INDEX_ALBUMS }], + [{ name: JobName.SEARCH_INDEX_FACES }], ]); }); }); describe('search', () => { it('should throw an error is search is disabled', async () => { - configMock.get.mockReturnValue('false'); - const sut = new SearchService(albumMock, assetMock, jobMock, machineMock, searchMock, configMock); + const sut = makeSut('false'); await expect(sut.search(authStub.admin, {})).rejects.toBeInstanceOf(BadRequestException); @@ -157,6 +166,7 @@ describe(SearchService.name, () => { page: 1, items: [], facets: [], + distances: [], }, assets: { total: 0, @@ -164,6 +174,7 @@ describe(SearchService.name, () => { page: 1, items: [], facets: [], + distances: [], }, }); @@ -202,8 +213,7 @@ describe(SearchService.name, () => { }); it('should skip if search is disabled', async () => { - configMock.get.mockReturnValue('false'); - const sut = new SearchService(albumMock, assetMock, jobMock, machineMock, searchMock, configMock); + const sut = makeSut('false'); await sut.handleIndexAssets(); @@ -214,8 +224,7 @@ describe(SearchService.name, () => { describe('handleIndexAsset', () => { it('should skip if search is disabled', () => { - configMock.get.mockReturnValue('false'); - const sut = new SearchService(albumMock, assetMock, jobMock, machineMock, searchMock, configMock); + const sut = makeSut('false'); sut.handleIndexAsset({ ids: [assetEntityStub.image.id] }); }); @@ -226,8 +235,7 @@ describe(SearchService.name, () => { describe('handleIndexAlbums', () => { it('should skip if search is disabled', () => { - configMock.get.mockReturnValue('false'); - const sut = new SearchService(albumMock, assetMock, jobMock, machineMock, searchMock, configMock); + const sut = makeSut('false'); sut.handleIndexAlbums(); }); @@ -251,8 +259,7 @@ describe(SearchService.name, () => { describe('handleIndexAlbum', () => { it('should skip if search is disabled', () => { - configMock.get.mockReturnValue('false'); - const sut = new SearchService(albumMock, assetMock, jobMock, machineMock, searchMock, configMock); + const sut = makeSut('false'); sut.handleIndexAlbum({ ids: [albumStub.empty.id] }); }); @@ -263,8 +270,7 @@ describe(SearchService.name, () => { describe('handleRemoveAlbum', () => { it('should skip if search is disabled', () => { - configMock.get.mockReturnValue('false'); - const sut = new SearchService(albumMock, assetMock, jobMock, machineMock, searchMock, configMock); + const sut = makeSut('false'); sut.handleRemoveAlbum({ ids: ['album1'] }); }); @@ -275,8 +281,7 @@ describe(SearchService.name, () => { describe('handleRemoveAsset', () => { it('should skip if search is disabled', () => { - configMock.get.mockReturnValue('false'); - const sut = new SearchService(albumMock, assetMock, jobMock, machineMock, searchMock, configMock); + const sut = makeSut('false'); sut.handleRemoveAsset({ ids: ['asset1'] }); }); @@ -285,6 +290,84 @@ describe(SearchService.name, () => { }); }); + describe('handleIndexFaces', () => { + it('should call done, even when there are no faces', async () => { + faceMock.getAll.mockResolvedValue([]); + + await sut.handleIndexFaces(); + + expect(searchMock.importFaces).toHaveBeenCalledWith([], true); + }); + + it('should index all the faces', async () => { + faceMock.getAll.mockResolvedValue([faceStub.face1]); + + await sut.handleIndexFaces(); + + expect(searchMock.importFaces.mock.calls).toEqual([ + [ + [ + { + id: 'asset-id|person-1', + ownerId: 'user-id', + assetId: 'asset-id', + personId: 'person-1', + embedding: [1, 2, 3, 4], + }, + ], + false, + ], + [[], true], + ]); + }); + + it('should log an error', async () => { + faceMock.getAll.mockResolvedValue([faceStub.face1]); + searchMock.importFaces.mockRejectedValue(new Error('import failed')); + + await sut.handleIndexFaces(); + + expect(searchMock.importFaces).toHaveBeenCalled(); + }); + + it('should skip if search is disabled', async () => { + const sut = makeSut('false'); + + await sut.handleIndexFaces(); + + expect(searchMock.importFaces).not.toHaveBeenCalled(); + }); + }); + + describe('handleIndexAsset', () => { + it('should skip if search is disabled', () => { + const sut = makeSut('false'); + sut.handleIndexFace({ assetId: 'asset-1', personId: 'person-1' }); + + expect(searchMock.importFaces).not.toHaveBeenCalled(); + expect(faceMock.getByIds).not.toHaveBeenCalled(); + }); + + it('should index the face', () => { + faceMock.getByIds.mockResolvedValue([faceStub.face1]); + + sut.handleIndexFace({ assetId: 'asset-1', personId: 'person-1' }); + + expect(faceMock.getByIds).toHaveBeenCalledWith([{ assetId: 'asset-1', personId: 'person-1' }]); + }); + }); + + describe('handleRemoveFace', () => { + it('should skip if search is disabled', () => { + const sut = makeSut('false'); + sut.handleRemoveFace({ assetId: 'asset-1', personId: 'person-1' }); + }); + + it('should remove the face', () => { + sut.handleRemoveFace({ assetId: 'asset-1', personId: 'person-1' }); + }); + }); + describe('flush', () => { it('should flush queued album updates', async () => { albumMock.getByIds.mockResolvedValue([albumStub.empty]); diff --git a/server/libs/domain/src/search/search.service.ts b/server/libs/domain/src/search/search.service.ts index f026a0695a..be1a9c4674 100644 --- a/server/libs/domain/src/search/search.service.ts +++ b/server/libs/domain/src/search/search.service.ts @@ -1,4 +1,4 @@ -import { AlbumEntity, AssetEntity } from '@app/infra/entities'; +import { AlbumEntity, AssetEntity, AssetFaceEntity } from '@app/infra/entities'; import { BadRequestException, Inject, Injectable, Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { mapAlbum } from '../album'; @@ -7,12 +7,14 @@ import { mapAsset } from '../asset'; import { IAssetRepository } from '../asset/asset.repository'; import { AuthUserDto } from '../auth'; import { MACHINE_LEARNING_ENABLED } from '../domain.constant'; -import { IBulkEntityJob, IJobRepository, JobName } from '../job'; +import { AssetFaceId, IFaceRepository } from '../facial-recognition'; +import { IAssetFaceJob, IBulkEntityJob, IJobRepository, JobName } from '../job'; import { IMachineLearningRepository } from '../smart-info'; import { SearchDto } from './dto'; import { SearchConfigResponseDto, SearchResponseDto } from './response-dto'; import { ISearchRepository, + OwnedFaceEntity, SearchCollection, SearchExploreItem, SearchResult, @@ -40,9 +42,15 @@ export class SearchService { delete: new Set(), }; + private faceQueue: SyncQueue = { + upsert: new Set(), + delete: new Set(), + }; + constructor( @Inject(IAlbumRepository) private albumRepository: IAlbumRepository, @Inject(IAssetRepository) private assetRepository: IAssetRepository, + @Inject(IFaceRepository) private faceRepository: IFaceRepository, @Inject(IJobRepository) private jobRepository: IJobRepository, @Inject(IMachineLearningRepository) private machineLearning: IMachineLearningRepository, @Inject(ISearchRepository) private searchRepository: ISearchRepository, @@ -88,6 +96,10 @@ export class SearchService { this.logger.debug('Queueing job to re-index all albums'); await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ALBUMS }); } + if (migrationStatus[SearchCollection.FACES]) { + this.logger.debug('Queueing job to re-index all faces'); + await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_FACES }); + } } async getExploreData(authUser: AuthUserDto): Promise[]> { @@ -159,6 +171,29 @@ export class SearchService { } } + async handleIndexFaces() { + if (!this.enabled) { + return; + } + + try { + // TODO: do this in batches based on searchIndexVersion + const faces = this.patchFaces(await this.faceRepository.getAll()); + this.logger.log(`Indexing ${faces.length} faces`); + + const chunkSize = 1000; + for (let i = 0; i < faces.length; i += chunkSize) { + await this.searchRepository.importFaces(faces.slice(i, i + chunkSize), false); + } + + await this.searchRepository.importFaces([], true); + + this.logger.debug('Finished re-indexing all faces'); + } catch (error: any) { + this.logger.error(`Unable to index all faces`, error?.stack); + } + } + handleIndexAlbum({ ids }: IBulkEntityJob) { if (!this.enabled) { return; @@ -179,6 +214,15 @@ export class SearchService { } } + async handleIndexFace({ assetId, personId }: IAssetFaceJob) { + if (!this.enabled) { + return; + } + + // immediately push to typesense + await this.searchRepository.importFaces(await this.idsToFaces([{ assetId, personId }]), false); + } + handleRemoveAlbum({ ids }: IBulkEntityJob) { if (!this.enabled) { return; @@ -199,6 +243,14 @@ export class SearchService { } } + handleRemoveFace({ assetId, personId }: IAssetFaceJob) { + if (!this.enabled) { + return; + } + + this.faceQueue.delete.add(this.asKey({ assetId, personId })); + } + private async flush() { if (this.albumQueue.upsert.size > 0) { const ids = [...this.albumQueue.upsert.keys()]; @@ -229,6 +281,21 @@ export class SearchService { await this.searchRepository.deleteAssets(ids); this.assetQueue.delete.clear(); } + + if (this.faceQueue.upsert.size > 0) { + const ids = [...this.faceQueue.upsert.keys()].map((key) => this.asParts(key)); + const items = await this.idsToFaces(ids); + this.logger.debug(`Flushing ${items.length} face upserts`); + await this.searchRepository.importFaces(items, false); + this.faceQueue.upsert.clear(); + } + + if (this.faceQueue.delete.size > 0) { + const ids = [...this.faceQueue.delete.keys()]; + this.logger.debug(`Flushing ${ids.length} face deletes`); + await this.searchRepository.deleteFaces(ids); + this.faceQueue.delete.clear(); + } } private assertEnabled() { @@ -247,6 +314,10 @@ export class SearchService { return this.patchAssets(entities.filter((entity) => entity.isVisible)); } + private async idsToFaces(ids: AssetFaceId[]): Promise { + return this.patchFaces(await this.faceRepository.getByIds(ids)); + } + private patchAssets(assets: AssetEntity[]): AssetEntity[] { return assets; } @@ -254,4 +325,23 @@ export class SearchService { private patchAlbums(albums: AlbumEntity[]): AlbumEntity[] { return albums.map((entity) => ({ ...entity, assets: [] })); } + + private patchFaces(faces: AssetFaceEntity[]): OwnedFaceEntity[] { + return faces.map((face) => ({ + id: this.asKey(face), + ownerId: face.asset.ownerId, + assetId: face.assetId, + personId: face.personId, + embedding: face.embedding, + })); + } + + private asKey(face: AssetFaceId): string { + return `${face.assetId}|${face.personId}`; + } + + private asParts(key: string): AssetFaceId { + const [assetId, personId] = key.split('|'); + return { assetId, personId }; + } } diff --git a/server/libs/domain/src/smart-info/machine-learning.interface.ts b/server/libs/domain/src/smart-info/machine-learning.interface.ts index 9d22e08572..7fb06d29ed 100644 --- a/server/libs/domain/src/smart-info/machine-learning.interface.ts +++ b/server/libs/domain/src/smart-info/machine-learning.interface.ts @@ -4,9 +4,25 @@ export interface MachineLearningInput { thumbnailPath: string; } +export interface BoundingBox { + x1: number; + y1: number; + x2: number; + y2: number; +} + +export interface DetectFaceResult { + imageWidth: number; + imageHeight: number; + boundingBox: BoundingBox; + score: number; + embedding: number[]; +} + export interface IMachineLearningRepository { classifyImage(input: MachineLearningInput): Promise; detectObjects(input: MachineLearningInput): Promise; encodeImage(input: MachineLearningInput): Promise; encodeText(input: string): Promise; + detectFaces(input: MachineLearningInput): Promise; } diff --git a/server/libs/domain/src/user/user.service.spec.ts b/server/libs/domain/src/user/user.service.spec.ts index 647c059345..d999522951 100644 --- a/server/libs/domain/src/user/user.service.spec.ts +++ b/server/libs/domain/src/user/user.service.spec.ts @@ -435,7 +435,7 @@ describe(UserService.name, () => { { deletedAt: makeDeletedAt(5) }, ] as UserEntity[]); - await sut.handleQueueUserDelete(); + await sut.handleUserDeleteCheck(); expect(userRepositoryMock.getDeletedUsers).toHaveBeenCalled(); expect(jobMock.queue).not.toHaveBeenCalled(); @@ -445,7 +445,7 @@ describe(UserService.name, () => { const user = { deletedAt: makeDeletedAt(10) }; userRepositoryMock.getDeletedUsers.mockResolvedValue([user] as UserEntity[]); - await sut.handleQueueUserDelete(); + await sut.handleUserDeleteCheck(); expect(userRepositoryMock.getDeletedUsers).toHaveBeenCalled(); expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.USER_DELETION, data: { user } }); diff --git a/server/libs/domain/src/user/user.service.ts b/server/libs/domain/src/user/user.service.ts index 305d5e283a..2f7fc37762 100644 --- a/server/libs/domain/src/user/user.service.ts +++ b/server/libs/domain/src/user/user.service.ts @@ -143,7 +143,7 @@ export class UserService { return { admin, password, provided: !!providedPassword }; } - async handleQueueUserDelete() { + async handleUserDeleteCheck() { const users = await this.userRepository.getDeletedUsers(); for (const user of users) { if (this.isReadyForDeletion(user)) { diff --git a/server/libs/domain/test/face.repository.mock.ts b/server/libs/domain/test/face.repository.mock.ts new file mode 100644 index 0000000000..7e56de4475 --- /dev/null +++ b/server/libs/domain/test/face.repository.mock.ts @@ -0,0 +1,9 @@ +import { IFaceRepository } from '../src'; + +export const newFaceRepositoryMock = (): jest.Mocked => { + return { + getAll: jest.fn(), + getByIds: jest.fn(), + create: jest.fn(), + }; +}; diff --git a/server/libs/domain/test/fixtures.ts b/server/libs/domain/test/fixtures.ts index 6eebc2e474..eb94855069 100644 --- a/server/libs/domain/test/fixtures.ts +++ b/server/libs/domain/test/fixtures.ts @@ -3,6 +3,7 @@ import { APIKeyEntity, AssetEntity, AssetType, + PersonEntity, PartnerEntity, SharedLinkEntity, SharedLinkType, @@ -10,6 +11,7 @@ import { TranscodePreset, UserEntity, UserTokenEntity, + AssetFaceEntity, } from '@app/infra/entities'; import { AlbumResponseDto, @@ -142,6 +144,7 @@ export const assetEntityStub = { livePhotoVideoId: null, tags: [], sharedLinks: [], + faces: [], }), image: Object.freeze({ id: 'asset-id', @@ -168,6 +171,7 @@ export const assetEntityStub = { tags: [], sharedLinks: [], originalFileName: 'asset-id.ext', + faces: [], }), video: Object.freeze({ id: 'asset-id', @@ -194,6 +198,7 @@ export const assetEntityStub = { livePhotoVideoId: null, tags: [], sharedLinks: [], + faces: [], }), livePhotoMotionAsset: Object.freeze({ id: 'live-photo-motion-asset', @@ -372,6 +377,7 @@ const assetResponse: AssetResponseDto = { exifInfo: assetInfo, livePhotoVideoId: null, tags: [], + people: [], }; const albumResponse: AlbumResponseDto = { @@ -655,6 +661,7 @@ export const sharedLinkStub = { }, tags: [], sharedLinks: [], + faces: [], }, ], }, @@ -729,6 +736,7 @@ export const searchStub = { page: 1, items: [], facets: [], + distances: [], }), }; @@ -826,6 +834,39 @@ export const probeStub = { }), }; +export const personStub = { + noName: Object.freeze({ + id: 'person-1', + createdAt: new Date('2021-01-01'), + updatedAt: new Date('2021-01-01'), + ownerId: userEntityStub.admin.id, + owner: userEntityStub.admin, + name: '', + thumbnailPath: '/path/to/thumbnail', + faces: [], + }), + withName: Object.freeze({ + id: 'person-1', + createdAt: new Date('2021-01-01'), + updatedAt: new Date('2021-01-01'), + ownerId: userEntityStub.admin.id, + owner: userEntityStub.admin, + name: 'Person 1', + thumbnailPath: '/path/to/thumbnail', + faces: [], + }), + noThumbnail: Object.freeze({ + id: 'person-1', + createdAt: new Date('2021-01-01'), + updatedAt: new Date('2021-01-01'), + ownerId: userEntityStub.admin.id, + owner: userEntityStub.admin, + name: '', + thumbnailPath: '', + faces: [], + }), +}; + export const partnerStub = { adminToUser1: Object.freeze({ createdAt: new Date('2023-02-23T05:06:29.716Z'), @@ -844,3 +885,13 @@ export const partnerStub = { sharedWith: userEntityStub.admin, }), }; + +export const faceStub = { + face1: Object.freeze({ + assetId: assetEntityStub.image.id, + asset: assetEntityStub.image, + personId: personStub.withName.id, + person: personStub.withName, + embedding: [1, 2, 3, 4], + }), +}; diff --git a/server/libs/domain/test/index.ts b/server/libs/domain/test/index.ts index 842d291319..546b7ced56 100644 --- a/server/libs/domain/test/index.ts +++ b/server/libs/domain/test/index.ts @@ -3,11 +3,13 @@ export * from './api-key.repository.mock'; export * from './asset.repository.mock'; export * from './communication.repository.mock'; export * from './crypto.repository.mock'; +export * from './face.repository.mock'; export * from './fixtures'; export * from './job.repository.mock'; export * from './machine-learning.repository.mock'; export * from './media.repository.mock'; export * from './partner.repository.mock'; +export * from './person.repository.mock'; export * from './search.repository.mock'; export * from './shared-link.repository.mock'; export * from './smart-info.repository.mock'; diff --git a/server/libs/domain/test/machine-learning.repository.mock.ts b/server/libs/domain/test/machine-learning.repository.mock.ts index b9205d44e8..0988ce8dca 100644 --- a/server/libs/domain/test/machine-learning.repository.mock.ts +++ b/server/libs/domain/test/machine-learning.repository.mock.ts @@ -6,5 +6,6 @@ export const newMachineLearningRepositoryMock = (): jest.Mocked => { extractThumbnailFromExif: jest.fn(), extractVideoThumbnail: jest.fn(), resize: jest.fn(), + crop: jest.fn(), probe: jest.fn(), transcode: jest.fn(), }; diff --git a/server/libs/domain/test/person.repository.mock.ts b/server/libs/domain/test/person.repository.mock.ts new file mode 100644 index 0000000000..6a1b5e8c58 --- /dev/null +++ b/server/libs/domain/test/person.repository.mock.ts @@ -0,0 +1,15 @@ +import { IPersonRepository } from '../src'; + +export const newPersonRepositoryMock = (): jest.Mocked => { + return { + getById: jest.fn(), + getAll: jest.fn(), + getAssets: jest.fn(), + getAllWithoutFaces: jest.fn(), + + create: jest.fn(), + update: jest.fn(), + deleteAll: jest.fn(), + delete: jest.fn(), + }; +}; diff --git a/server/libs/domain/test/search.repository.mock.ts b/server/libs/domain/test/search.repository.mock.ts index 5a4fcdf217..b845e30f90 100644 --- a/server/libs/domain/test/search.repository.mock.ts +++ b/server/libs/domain/test/search.repository.mock.ts @@ -6,11 +6,15 @@ export const newSearchRepositoryMock = (): jest.Mocked => { checkMigrationStatus: jest.fn(), importAssets: jest.fn(), importAlbums: jest.fn(), + importFaces: jest.fn(), deleteAlbums: jest.fn(), deleteAssets: jest.fn(), + deleteFaces: jest.fn(), + deleteAllFaces: jest.fn(), searchAssets: jest.fn(), searchAlbums: jest.fn(), vectorSearch: jest.fn(), explore: jest.fn(), + searchFaces: jest.fn(), }; }; diff --git a/server/libs/infra/src/entities/asset-face.entity.ts b/server/libs/infra/src/entities/asset-face.entity.ts new file mode 100644 index 0000000000..96aa17a2ca --- /dev/null +++ b/server/libs/infra/src/entities/asset-face.entity.ts @@ -0,0 +1,25 @@ +import { Column, Entity, ManyToOne, PrimaryColumn } from 'typeorm'; +import { AssetEntity } from './asset.entity'; +import { PersonEntity } from './person.entity'; + +@Entity('asset_faces') +export class AssetFaceEntity { + @PrimaryColumn() + assetId!: string; + + @PrimaryColumn() + personId!: string; + + @Column({ + type: 'float4', + array: true, + nullable: true, + }) + embedding!: number[] | null; + + @ManyToOne(() => AssetEntity, (asset) => asset.faces, { onDelete: 'CASCADE', onUpdate: 'CASCADE' }) + asset!: AssetEntity; + + @ManyToOne(() => PersonEntity, (person) => person.faces, { onDelete: 'CASCADE', onUpdate: 'CASCADE' }) + person!: PersonEntity; +} diff --git a/server/libs/infra/src/entities/asset.entity.ts b/server/libs/infra/src/entities/asset.entity.ts index d370f06478..cba5518ea9 100644 --- a/server/libs/infra/src/entities/asset.entity.ts +++ b/server/libs/infra/src/entities/asset.entity.ts @@ -7,12 +7,14 @@ import { JoinTable, ManyToMany, ManyToOne, + OneToMany, OneToOne, PrimaryGeneratedColumn, Unique, UpdateDateColumn, } from 'typeorm'; import { AlbumEntity } from './album.entity'; +import { AssetFaceEntity } from './asset-face.entity'; import { ExifEntity } from './exif.entity'; import { SharedLinkEntity } from './shared-link.entity'; import { SmartInfoEntity } from './smart-info.entity'; @@ -109,6 +111,9 @@ export class AssetEntity { @ManyToMany(() => AlbumEntity, (album) => album.assets, { onDelete: 'CASCADE', onUpdate: 'CASCADE' }) albums?: AlbumEntity[]; + + @OneToMany(() => AssetFaceEntity, (assetFace) => assetFace.asset) + faces!: AssetFaceEntity[]; } export enum AssetType { diff --git a/server/libs/infra/src/entities/index.ts b/server/libs/infra/src/entities/index.ts index 8e4a73b147..f166b5927f 100644 --- a/server/libs/infra/src/entities/index.ts +++ b/server/libs/infra/src/entities/index.ts @@ -1,18 +1,22 @@ import { AlbumEntity } from './album.entity'; import { APIKeyEntity } from './api-key.entity'; +import { AssetFaceEntity } from './asset-face.entity'; import { AssetEntity } from './asset.entity'; import { PartnerEntity } from './partner.entity'; +import { PersonEntity } from './person.entity'; import { SharedLinkEntity } from './shared-link.entity'; import { SmartInfoEntity } from './smart-info.entity'; import { SystemConfigEntity } from './system-config.entity'; -import { UserEntity } from './user.entity'; import { UserTokenEntity } from './user-token.entity'; +import { UserEntity } from './user.entity'; export * from './album.entity'; export * from './api-key.entity'; +export * from './asset-face.entity'; export * from './asset.entity'; export * from './exif.entity'; export * from './partner.entity'; +export * from './person.entity'; export * from './shared-link.entity'; export * from './smart-info.entity'; export * from './system-config.entity'; @@ -24,7 +28,9 @@ export const databaseEntities = [ AlbumEntity, APIKeyEntity, AssetEntity, + AssetFaceEntity, PartnerEntity, + PersonEntity, SharedLinkEntity, SmartInfoEntity, SystemConfigEntity, diff --git a/server/libs/infra/src/entities/person.entity.ts b/server/libs/infra/src/entities/person.entity.ts new file mode 100644 index 0000000000..40ad593e93 --- /dev/null +++ b/server/libs/infra/src/entities/person.entity.ts @@ -0,0 +1,38 @@ +import { + Column, + CreateDateColumn, + Entity, + ManyToOne, + OneToMany, + PrimaryGeneratedColumn, + UpdateDateColumn, +} from 'typeorm'; +import { AssetFaceEntity } from './asset-face.entity'; +import { UserEntity } from './user.entity'; + +@Entity('person') +export class PersonEntity { + @PrimaryGeneratedColumn('uuid') + id!: string; + + @CreateDateColumn({ type: 'timestamptz' }) + createdAt!: Date; + + @UpdateDateColumn({ type: 'timestamptz' }) + updatedAt!: Date; + + @Column() + ownerId!: string; + + @ManyToOne(() => UserEntity, { onDelete: 'CASCADE', onUpdate: 'CASCADE', nullable: false }) + owner!: UserEntity; + + @Column({ default: '' }) + name!: string; + + @Column({ default: '' }) + thumbnailPath!: string; + + @OneToMany(() => AssetFaceEntity, (assetFace) => assetFace.person) + faces!: AssetFaceEntity[]; +} diff --git a/server/libs/infra/src/infra.module.ts b/server/libs/infra/src/infra.module.ts index 51cc1bef14..34cd5f7c88 100644 --- a/server/libs/infra/src/infra.module.ts +++ b/server/libs/infra/src/infra.module.ts @@ -3,6 +3,7 @@ import { IAssetRepository, ICommunicationRepository, ICryptoRepository, + IFaceRepository, IGeocodingRepository, IJobRepository, IKeyRepository, @@ -10,6 +11,7 @@ import { IMediaRepository, immichAppConfig, IPartnerRepository, + IPersonRepository, ISearchRepository, ISharedLinkRepository, ISmartInfoRepository, @@ -32,12 +34,14 @@ import { AssetRepository, CommunicationRepository, CryptoRepository, + FaceRepository, FilesystemProvider, GeocodingRepository, JobRepository, MachineLearningRepository, MediaRepository, PartnerRepository, + PersonRepository, SharedLinkRepository, SmartInfoRepository, SystemConfigRepository, @@ -51,12 +55,14 @@ const providers: Provider[] = [ { provide: IAssetRepository, useClass: AssetRepository }, { provide: ICommunicationRepository, useClass: CommunicationRepository }, { provide: ICryptoRepository, useClass: CryptoRepository }, + { provide: IFaceRepository, useClass: FaceRepository }, { provide: IGeocodingRepository, useClass: GeocodingRepository }, { provide: IJobRepository, useClass: JobRepository }, { provide: IKeyRepository, useClass: APIKeyRepository }, { provide: IMachineLearningRepository, useClass: MachineLearningRepository }, { provide: IMediaRepository, useClass: MediaRepository }, { provide: IPartnerRepository, useClass: PartnerRepository }, + { provide: IPersonRepository, useClass: PersonRepository }, { provide: ISearchRepository, useClass: TypesenseRepository }, { provide: ISharedLinkRepository, useClass: SharedLinkRepository }, { provide: ISmartInfoRepository, useClass: SmartInfoRepository }, diff --git a/server/libs/infra/src/migrations/1684255168091-AddFacialTables.ts b/server/libs/infra/src/migrations/1684255168091-AddFacialTables.ts new file mode 100644 index 0000000000..1f2426bb0c --- /dev/null +++ b/server/libs/infra/src/migrations/1684255168091-AddFacialTables.ts @@ -0,0 +1,22 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class AddFacialTables1684255168091 implements MigrationInterface { + name = 'AddFacialTables1684255168091' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE TABLE "person" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "ownerId" uuid NOT NULL, "name" character varying NOT NULL DEFAULT '', "thumbnailPath" character varying NOT NULL DEFAULT '', CONSTRAINT "PK_5fdaf670315c4b7e70cce85daa3" PRIMARY KEY ("id"))`); + await queryRunner.query(`CREATE TABLE "asset_faces" ("assetId" uuid NOT NULL, "personId" uuid NOT NULL, "embedding" real array, CONSTRAINT "PK_bf339a24070dac7e71304ec530a" PRIMARY KEY ("assetId", "personId"))`); + await queryRunner.query(`ALTER TABLE "person" ADD CONSTRAINT "FK_5527cc99f530a547093f9e577b6" FOREIGN KEY ("ownerId") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE`); + await queryRunner.query(`ALTER TABLE "asset_faces" ADD CONSTRAINT "FK_02a43fd0b3c50fb6d7f0cb7282c" FOREIGN KEY ("assetId") REFERENCES "assets"("id") ON DELETE CASCADE ON UPDATE CASCADE`); + await queryRunner.query(`ALTER TABLE "asset_faces" ADD CONSTRAINT "FK_95ad7106dd7b484275443f580f9" FOREIGN KEY ("personId") REFERENCES "person"("id") ON DELETE CASCADE ON UPDATE CASCADE`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "asset_faces" DROP CONSTRAINT "FK_95ad7106dd7b484275443f580f9"`); + await queryRunner.query(`ALTER TABLE "asset_faces" DROP CONSTRAINT "FK_02a43fd0b3c50fb6d7f0cb7282c"`); + await queryRunner.query(`ALTER TABLE "person" DROP CONSTRAINT "FK_5527cc99f530a547093f9e577b6"`); + await queryRunner.query(`DROP TABLE "asset_faces"`); + await queryRunner.query(`DROP TABLE "person"`); + } + +} diff --git a/server/libs/infra/src/repositories/asset.repository.ts b/server/libs/infra/src/repositories/asset.repository.ts index 4861b9c4e2..dda9572f56 100644 --- a/server/libs/infra/src/repositories/asset.repository.ts +++ b/server/libs/infra/src/repositories/asset.repository.ts @@ -15,6 +15,9 @@ export class AssetRepository implements IAssetRepository { exifInfo: true, smartInfo: true, tags: true, + faces: { + person: true, + }, }, }); } @@ -35,6 +38,9 @@ export class AssetRepository implements IAssetRepository { exifInfo: true, smartInfo: true, tags: true, + faces: { + person: true, + }, }, }); } @@ -48,6 +54,7 @@ export class AssetRepository implements IAssetRepository { owner: true, smartInfo: true, tags: true, + faces: true, }, }); } @@ -129,6 +136,20 @@ export class AssetRepository implements IAssetRepository { }; break; + case WithoutProperty.FACES: + relations = { + faces: true, + }; + where = { + resizePath: IsNull(), + isVisible: true, + faces: { + assetId: IsNull(), + personId: IsNull(), + }, + }; + break; + default: throw new Error(`Invalid getWithout property: ${property}`); } diff --git a/server/libs/infra/src/repositories/face.repository.ts b/server/libs/infra/src/repositories/face.repository.ts new file mode 100644 index 0000000000..34b8bf800d --- /dev/null +++ b/server/libs/infra/src/repositories/face.repository.ts @@ -0,0 +1,22 @@ +import { AssetFaceId, IFaceRepository } from '@app/domain'; +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { AssetFaceEntity } from '../entities/asset-face.entity'; + +@Injectable() +export class FaceRepository implements IFaceRepository { + constructor(@InjectRepository(AssetFaceEntity) private repository: Repository) {} + + getAll(): Promise { + return this.repository.find({ relations: { asset: true } }); + } + + getByIds(ids: AssetFaceId[]): Promise { + return this.repository.find({ where: ids, relations: { asset: true } }); + } + + create(entity: Partial): Promise { + return this.repository.save(entity); + } +} diff --git a/server/libs/infra/src/repositories/index.ts b/server/libs/infra/src/repositories/index.ts index 28b208c31e..75347863e1 100644 --- a/server/libs/infra/src/repositories/index.ts +++ b/server/libs/infra/src/repositories/index.ts @@ -3,12 +3,14 @@ export * from './api-key.repository'; export * from './asset.repository'; export * from './communication.repository'; export * from './crypto.repository'; +export * from './face.repository'; export * from './filesystem.provider'; export * from './geocoding.repository'; export * from './job.repository'; export * from './machine-learning.repository'; export * from './media.repository'; export * from './partner.repository'; +export * from './person.repository'; export * from './shared-link.repository'; export * from './smart-info.repository'; export * from './system-config.repository'; diff --git a/server/libs/infra/src/repositories/job.repository.ts b/server/libs/infra/src/repositories/job.repository.ts index f129ade23b..557ab07802 100644 --- a/server/libs/infra/src/repositories/job.repository.ts +++ b/server/libs/infra/src/repositories/job.repository.ts @@ -20,6 +20,7 @@ export class JobRepository implements IJobRepository { [QueueName.THUMBNAIL_GENERATION]: this.generateThumbnail, [QueueName.METADATA_EXTRACTION]: this.metadataExtraction, [QueueName.OBJECT_TAGGING]: this.objectTagging, + [QueueName.RECOGNIZE_FACES]: this.recognizeFaces, [QueueName.CLIP_ENCODING]: this.clipEmbedding, [QueueName.VIDEO_CONVERSION]: this.videoTranscode, [QueueName.BACKGROUND_TASK]: this.backgroundTask, @@ -31,6 +32,7 @@ export class JobRepository implements IJobRepository { @InjectQueue(QueueName.OBJECT_TAGGING) private objectTagging: Queue, @InjectQueue(QueueName.CLIP_ENCODING) private clipEmbedding: Queue, @InjectQueue(QueueName.METADATA_EXTRACTION) private metadataExtraction: Queue, + @InjectQueue(QueueName.RECOGNIZE_FACES) private recognizeFaces: Queue, @InjectQueue(QueueName.STORAGE_TEMPLATE_MIGRATION) private storageTemplateMigration: Queue, @InjectQueue(QueueName.THUMBNAIL_GENERATION) private generateThumbnail: Queue, @InjectQueue(QueueName.VIDEO_CONVERSION) private videoTranscode: Queue, @@ -91,6 +93,19 @@ export class JobRepository implements IJobRepository { await this.metadataExtraction.add(item.name, item.data); break; + case JobName.QUEUE_RECOGNIZE_FACES: + case JobName.RECOGNIZE_FACES: + await this.recognizeFaces.add(item.name, item.data); + break; + + case JobName.GENERATE_FACE_THUMBNAIL: + await this.recognizeFaces.add(item.name, item.data, { priority: 1 }); + break; + + case JobName.PERSON_CLEANUP: + await this.backgroundTask.add(item.name); + break; + case JobName.QUEUE_GENERATE_THUMBNAILS: case JobName.GENERATE_JPEG_THUMBNAIL: case JobName.GENERATE_WEBP_THUMBNAIL: @@ -120,13 +135,19 @@ export class JobRepository implements IJobRepository { case JobName.SEARCH_INDEX_ASSETS: case JobName.SEARCH_INDEX_ALBUMS: + case JobName.SEARCH_INDEX_FACES: await this.searchIndex.add(item.name, {}); break; case JobName.SEARCH_INDEX_ASSET: case JobName.SEARCH_INDEX_ALBUM: + case JobName.SEARCH_INDEX_FACE: + await this.searchIndex.add(item.name, item.data); + break; + case JobName.SEARCH_REMOVE_ALBUM: case JobName.SEARCH_REMOVE_ASSET: + case JobName.SEARCH_REMOVE_FACE: await this.searchIndex.add(item.name, item.data); break; diff --git a/server/libs/infra/src/repositories/machine-learning.repository.ts b/server/libs/infra/src/repositories/machine-learning.repository.ts index 7fe90db99d..4931b82c43 100644 --- a/server/libs/infra/src/repositories/machine-learning.repository.ts +++ b/server/libs/infra/src/repositories/machine-learning.repository.ts @@ -1,4 +1,4 @@ -import { IMachineLearningRepository, MachineLearningInput, MACHINE_LEARNING_URL } from '@app/domain'; +import { DetectFaceResult, IMachineLearningRepository, MachineLearningInput, MACHINE_LEARNING_URL } from '@app/domain'; import { Injectable } from '@nestjs/common'; import axios from 'axios'; @@ -10,6 +10,10 @@ export class MachineLearningRepository implements IMachineLearningRepository { return client.post('/image-classifier/tag-image', input).then((res) => res.data); } + detectFaces(input: MachineLearningInput): Promise { + return client.post('/facial-recognition/detect-faces', input).then((res) => res.data); + } + detectObjects(input: MachineLearningInput): Promise { return client.post('/object-detection/detect-object', input).then((res) => res.data); } diff --git a/server/libs/infra/src/repositories/media.repository.ts b/server/libs/infra/src/repositories/media.repository.ts index 94d5f70d3f..ef7ca0f942 100644 --- a/server/libs/infra/src/repositories/media.repository.ts +++ b/server/libs/infra/src/repositories/media.repository.ts @@ -1,4 +1,4 @@ -import { IMediaRepository, ResizeOptions, VideoInfo } from '@app/domain'; +import { CropOptions, IMediaRepository, ResizeOptions, VideoInfo } from '@app/domain'; import { exiftool } from 'exiftool-vendored'; import ffmpeg, { FfprobeData } from 'fluent-ffmpeg'; import sharp from 'sharp'; @@ -7,11 +7,22 @@ import { promisify } from 'util'; const probe = promisify(ffmpeg.ffprobe); export class MediaRepository implements IMediaRepository { + crop(input: string, options: CropOptions): Promise { + return sharp(input, { failOnError: false }) + .extract({ + left: options.left, + top: options.top, + width: options.width, + height: options.height, + }) + .toBuffer(); + } + extractThumbnailFromExif(input: string, output: string): Promise { return exiftool.extractThumbnail(input, output); } - async resize(input: string, output: string, options: ResizeOptions): Promise { + async resize(input: string | Buffer, output: string, options: ResizeOptions): Promise { switch (options.format) { case 'webp': await sharp(input, { failOnError: false }) diff --git a/server/libs/infra/src/repositories/person.repository.ts b/server/libs/infra/src/repositories/person.repository.ts new file mode 100644 index 0000000000..96100fb524 --- /dev/null +++ b/server/libs/infra/src/repositories/person.repository.ts @@ -0,0 +1,78 @@ +import { IPersonRepository, PersonSearchOptions } from '@app/domain'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { AssetEntity, AssetFaceEntity, PersonEntity } from '../entities'; + +export class PersonRepository implements IPersonRepository { + constructor( + @InjectRepository(AssetEntity) private assetRepository: Repository, + @InjectRepository(PersonEntity) private personRepository: Repository, + @InjectRepository(AssetFaceEntity) private assetFaceRepository: Repository, + ) {} + + delete(entity: PersonEntity): Promise { + return this.personRepository.remove(entity); + } + + async deleteAll(): Promise { + const people = await this.personRepository.find(); + await this.personRepository.remove(people); + return people.length; + } + + getAll(userId: string, options?: PersonSearchOptions): Promise { + return this.personRepository + .createQueryBuilder('person') + .leftJoin('person.faces', 'face') + .where('person.ownerId = :userId', { userId }) + .orderBy('COUNT(face.assetId)', 'DESC') + .having('COUNT(face.assetId) >= :faces', { faces: options?.minimumFaceCount || 1 }) + .groupBy('person.id') + .getMany(); + } + + getAllWithoutFaces(): Promise { + return this.personRepository + .createQueryBuilder('person') + .leftJoin('person.faces', 'face') + .having('COUNT(face.assetId) = 0') + .groupBy('person.id') + .getMany(); + } + + getById(ownerId: string, personId: string): Promise { + return this.personRepository.findOne({ where: { id: personId, ownerId } }); + } + + getAssets(ownerId: string, personId: string): Promise { + return this.assetRepository.find({ + where: { + ownerId, + faces: { + personId, + }, + isVisible: true, + }, + relations: { + faces: { + person: true, + }, + exifInfo: true, + }, + order: { + createdAt: 'ASC', + }, + // TODO: remove after either (1) pagination or (2) time bucket is implemented for this query + take: 3000, + }); + } + + create(entity: Partial): Promise { + return this.personRepository.save(entity); + } + + async update(entity: Partial): Promise { + const { id } = await this.personRepository.save(entity); + return this.personRepository.findOneByOrFail({ id }); + } +} diff --git a/server/libs/infra/src/repositories/typesense.repository.ts b/server/libs/infra/src/repositories/typesense.repository.ts index c56cef053d..b7995ad81f 100644 --- a/server/libs/infra/src/repositories/typesense.repository.ts +++ b/server/libs/infra/src/repositories/typesense.repository.ts @@ -1,8 +1,10 @@ import { ISearchRepository, + OwnedFaceEntity, SearchCollection, SearchCollectionIndexStatus, SearchExploreItem, + SearchFaceFilter, SearchFilter, SearchResult, } from '@app/domain'; @@ -12,9 +14,9 @@ import { catchError, filter, firstValueFrom, from, map, mergeMap, of, toArray } import { Client } from 'typesense'; import { CollectionCreateSchema } from 'typesense/lib/Typesense/Collections'; import { DocumentSchema, SearchResponse } from 'typesense/lib/Typesense/Documents'; -import { AlbumEntity, AssetEntity } from '../entities'; +import { AlbumEntity, AssetEntity, AssetFaceEntity } from '../entities'; import { typesenseConfig } from '../infra.config'; -import { albumSchema, assetSchema } from '../typesense-schemas'; +import { albumSchema, assetSchema, faceSchema } from '../typesense-schemas'; function removeNil>(item: T): T { _.forOwn(item, (value, key) => { @@ -26,14 +28,21 @@ function removeNil>(item: T): T { return item; } +interface MultiSearchError { + code: number; + error: string; +} + interface CustomAssetEntity extends AssetEntity { geo?: [number, number]; motion?: boolean; + people?: string[]; } const schemaMap: Record = { [SearchCollection.ASSETS]: assetSchema, [SearchCollection.ALBUMS]: albumSchema, + [SearchCollection.FACES]: faceSchema, }; const schemas = Object.entries(schemaMap) as [SearchCollection, CollectionCreateSchema][]; @@ -61,7 +70,7 @@ export class TypesenseRepository implements ISearchRepository { async setup(): Promise { const collections = await this.client.collections().retrieve(); for (const collection of collections) { - this.logger.debug(`${collection.name} => ${collection.num_documents}`); + this.logger.debug(`${collection.name} collection has ${collection.num_documents} documents`); // await this.client.collections(collection.name).delete(); } @@ -84,6 +93,7 @@ export class TypesenseRepository implements ISearchRepository { const migrationMap: SearchCollectionIndexStatus = { [SearchCollection.ASSETS]: false, [SearchCollection.ALBUMS]: false, + [SearchCollection.FACES]: false, }; // check if alias is using the current schema @@ -110,9 +120,13 @@ export class TypesenseRepository implements ISearchRepository { await this.import(SearchCollection.ASSETS, items, done); } + async importFaces(items: OwnedFaceEntity[], done: boolean): Promise { + await this.import(SearchCollection.FACES, items, done); + } + private async import( collection: SearchCollection, - items: AlbumEntity[] | AssetEntity[], + items: AlbumEntity[] | AssetEntity[] | OwnedFaceEntity[], done: boolean, ): Promise { try { @@ -198,6 +212,15 @@ export class TypesenseRepository implements ISearchRepository { await this.delete(SearchCollection.ASSETS, ids); } + async deleteFaces(ids: string[]): Promise { + await this.delete(SearchCollection.FACES, ids); + } + + async deleteAllFaces(): Promise { + const records = await this.client.collections(faceSchema.name).documents().delete({ filter_by: 'ownerId:!=null' }); + return records.num_deleted; + } + async delete(collection: SearchCollection, ids: string[]): Promise { await this.client .collections(schemaMap[collection].name) @@ -232,6 +255,7 @@ export class TypesenseRepository implements ISearchRepository { 'exifInfo.description', 'smartInfo.tags', 'smartInfo.objects', + 'people', ].join(','), per_page: 250, facet_by: this.getFacetFieldNames(SearchCollection.ASSETS), @@ -242,6 +266,22 @@ export class TypesenseRepository implements ISearchRepository { return this.asResponse(results, filters.debug); } + async searchFaces(input: number[], filters: SearchFaceFilter): Promise> { + const { results } = await this.client.multiSearch.perform({ + searches: [ + { + collection: faceSchema.name, + q: '*', + vector_query: `embedding:([${input.join(',')}], k:5)`, + per_page: 250, + filter_by: this.buildFilterBy('ownerId', filters.ownerId, true), + } as any, + ], + }); + + return this.asResponse(results[0] as SearchResponse); + } + async vectorSearch(input: number[], filters: SearchFilter): Promise> { const { results } = await this.client.multiSearch.perform({ searches: [ @@ -259,12 +299,23 @@ export class TypesenseRepository implements ISearchRepository { return this.asResponse(results[0] as SearchResponse, filters.debug); } - private asResponse(results: SearchResponse, debug?: boolean): SearchResult { + private asResponse( + resultsOrError: SearchResponse | MultiSearchError, + debug?: boolean, + ): SearchResult { + const { error, code } = resultsOrError as MultiSearchError; + if (error) { + throw new Error(`Typesense multi-search error: ${code} - ${error}`); + } + + const results = resultsOrError as SearchResponse; + return { page: results.page, total: results.found, count: results.out_of, items: (results.hits || []).map((hit) => hit.document), + distances: (results.hits || []).map((hit: any) => hit.vector_distance), facets: (results.facet_counts || []).map((facet) => ({ counts: facet.counts.map((item) => ({ count: item.count, value: item.value })), fieldName: facet.field_name as string, @@ -306,12 +357,17 @@ export class TypesenseRepository implements ISearchRepository { } } - private patch(collection: SearchCollection, items: AssetEntity[] | AlbumEntity[]) { - return items.map((item) => - collection === SearchCollection.ASSETS - ? this.patchAsset(item as AssetEntity) - : this.patchAlbum(item as AlbumEntity), - ); + private patch(collection: SearchCollection, items: AssetEntity[] | AlbumEntity[] | OwnedFaceEntity[]) { + return items.map((item) => { + switch (collection) { + case SearchCollection.ASSETS: + return this.patchAsset(item as AssetEntity); + case SearchCollection.ALBUMS: + return this.patchAlbum(item as AlbumEntity); + case SearchCollection.FACES: + return this.patchFace(item as OwnedFaceEntity); + } + }); } private patchAlbum(album: AlbumEntity): AlbumEntity { @@ -327,9 +383,17 @@ export class TypesenseRepository implements ISearchRepository { custom = { ...custom, geo: [lat, lng] }; } + const people = asset.faces?.map((face) => face.person.name).filter((name) => name) || []; + if (people.length) { + custom = { ...custom, people }; + } return removeNil({ ...custom, motion: !!asset.livePhotoVideoId }); } + private patchFace(face: OwnedFaceEntity): OwnedFaceEntity { + return removeNil(face); + } + private getFacetFieldNames(collection: SearchCollection) { return (schemaMap[collection].fields || []) .filter((field) => field.facet) diff --git a/server/libs/infra/src/typesense-schemas/asset.schema.ts b/server/libs/infra/src/typesense-schemas/asset.schema.ts index 0b778fbeae..5d31e67b2b 100644 --- a/server/libs/infra/src/typesense-schemas/asset.schema.ts +++ b/server/libs/infra/src/typesense-schemas/asset.schema.ts @@ -1,6 +1,6 @@ import { CollectionCreateSchema } from 'typesense/lib/Typesense/Collections'; -export const assetSchemaVersion = 5; +export const assetSchemaVersion = 7; export const assetSchema: CollectionCreateSchema = { name: `assets-v${assetSchemaVersion}`, fields: [ @@ -32,6 +32,7 @@ export const assetSchema: CollectionCreateSchema = { // computed { name: 'geo', type: 'geopoint', facet: false, optional: true }, { name: 'motion', type: 'bool', facet: true }, + { name: 'people', type: 'string[]', facet: true, optional: true }, ], token_separators: ['.'], enable_nested_fields: true, diff --git a/server/libs/infra/src/typesense-schemas/face.schema.ts b/server/libs/infra/src/typesense-schemas/face.schema.ts new file mode 100644 index 0000000000..c978057ec9 --- /dev/null +++ b/server/libs/infra/src/typesense-schemas/face.schema.ts @@ -0,0 +1,12 @@ +import { CollectionCreateSchema } from 'typesense/lib/Typesense/Collections'; + +export const faceSchemaVersion = 1; +export const faceSchema: CollectionCreateSchema = { + name: `faces-v${faceSchemaVersion}`, + fields: [ + { name: 'ownerId', type: 'string', facet: false }, + { name: 'assetId', type: 'string', facet: false }, + { name: 'personId', type: 'string', facet: false }, + { name: 'embedding', type: 'float[]', facet: false, num_dim: 512 }, + ], +}; diff --git a/server/libs/infra/src/typesense-schemas/index.ts b/server/libs/infra/src/typesense-schemas/index.ts index d002910221..a2ab861947 100644 --- a/server/libs/infra/src/typesense-schemas/index.ts +++ b/server/libs/infra/src/typesense-schemas/index.ts @@ -1,2 +1,3 @@ export * from './album.schema'; export * from './asset.schema'; +export * from './face.schema'; diff --git a/web/src/api/api.ts b/web/src/api/api.ts index 26499db26f..79397a9003 100644 --- a/web/src/api/api.ts +++ b/web/src/api/api.ts @@ -8,6 +8,7 @@ import { ConfigurationParameters, JobApi, OAuthApi, + PersonApi, PartnerApi, SearchApi, ServerInfoApi, @@ -31,6 +32,7 @@ export class ImmichApi { public searchApi: SearchApi; public serverInfoApi: ServerInfoApi; public shareApi: ShareApi; + public personApi: PersonApi; public systemConfigApi: SystemConfigApi; public userApi: UserApi; @@ -49,6 +51,7 @@ export class ImmichApi { this.searchApi = new SearchApi(this.config); this.serverInfoApi = new ServerInfoApi(this.config); this.shareApi = new ShareApi(this.config); + this.personApi = new PersonApi(this.config); this.systemConfigApi = new SystemConfigApi(this.config); this.userApi = new UserApi(this.config); } @@ -98,6 +101,11 @@ export class ImmichApi { const path = `/user/profile-image/${userId}`; return this.createUrl(path); } + + public getPeopleThumbnailUrl(personId: string) { + const path = `/person/${personId}/thumbnail`; + return this.createUrl(path); + } } export const api = new ImmichApi({ basePath: '/api' }); diff --git a/web/src/api/open-api/api.ts b/web/src/api/open-api/api.ts index 602bdac389..c2475e8d23 100644 --- a/web/src/api/open-api/api.ts +++ b/web/src/api/open-api/api.ts @@ -339,6 +339,12 @@ export interface AllJobStatusResponseDto { * @memberof AllJobStatusResponseDto */ 'search-queue': JobStatusDto; + /** + * + * @type {JobStatusDto} + * @memberof AllJobStatusResponseDto + */ + 'recognize-faces-queue': JobStatusDto; } /** * @@ -566,6 +572,12 @@ export interface AssetResponseDto { * @memberof AssetResponseDto */ 'tags'?: Array; + /** + * + * @type {Array} + * @memberof AssetResponseDto + */ + 'people'?: Array; } @@ -1329,6 +1341,7 @@ export const JobName = { MetadataExtractionQueue: 'metadata-extraction-queue', VideoConversionQueue: 'video-conversion-queue', ObjectTaggingQueue: 'object-tagging-queue', + RecognizeFacesQueue: 'recognize-faces-queue', ClipEncodingQueue: 'clip-encoding-queue', BackgroundTaskQueue: 'background-task-queue', StorageTemplateMigrationQueue: 'storage-template-migration-queue', @@ -1546,6 +1559,44 @@ export interface OAuthConfigResponseDto { */ 'autoLaunch'?: boolean; } +/** + * + * @export + * @interface PersonResponseDto + */ +export interface PersonResponseDto { + /** + * + * @type {string} + * @memberof PersonResponseDto + */ + 'id': string; + /** + * + * @type {string} + * @memberof PersonResponseDto + */ + 'name': string; + /** + * + * @type {string} + * @memberof PersonResponseDto + */ + 'thumbnailPath': string; +} +/** + * + * @export + * @interface PersonUpdateDto + */ +export interface PersonUpdateDto { + /** + * + * @type {string} + * @memberof PersonUpdateDto + */ + 'name': string; +} /** * * @export @@ -7460,6 +7511,406 @@ export class PartnerApi extends BaseAPI { } +/** + * PersonApi - axios parameter creator + * @export + */ +export const PersonApiAxiosParamCreator = function (configuration?: Configuration) { + return { + /** + * + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + getAllPeople: async (options: AxiosRequestConfig = {}): Promise => { + const localVarPath = `/person`; + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'GET', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + // authentication cookie required + + // authentication api_key required + await setApiKeyToObject(localVarHeaderParameter, "x-api-key", configuration) + + // authentication bearer required + // http bearer authentication required + await setBearerAuthToObject(localVarHeaderParameter, configuration) + + + + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, + /** + * + * @param {string} id + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + getPerson: async (id: string, options: AxiosRequestConfig = {}): Promise => { + // verify required parameter 'id' is not null or undefined + assertParamExists('getPerson', 'id', id) + const localVarPath = `/person/{id}` + .replace(`{${"id"}}`, encodeURIComponent(String(id))); + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'GET', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + // authentication cookie required + + // authentication api_key required + await setApiKeyToObject(localVarHeaderParameter, "x-api-key", configuration) + + // authentication bearer required + // http bearer authentication required + await setBearerAuthToObject(localVarHeaderParameter, configuration) + + + + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, + /** + * + * @param {string} id + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + getPersonAssets: async (id: string, options: AxiosRequestConfig = {}): Promise => { + // verify required parameter 'id' is not null or undefined + assertParamExists('getPersonAssets', 'id', id) + const localVarPath = `/person/{id}/assets` + .replace(`{${"id"}}`, encodeURIComponent(String(id))); + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'GET', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + // authentication cookie required + + // authentication api_key required + await setApiKeyToObject(localVarHeaderParameter, "x-api-key", configuration) + + // authentication bearer required + // http bearer authentication required + await setBearerAuthToObject(localVarHeaderParameter, configuration) + + + + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, + /** + * + * @param {string} id + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + getPersonThumbnail: async (id: string, options: AxiosRequestConfig = {}): Promise => { + // verify required parameter 'id' is not null or undefined + assertParamExists('getPersonThumbnail', 'id', id) + const localVarPath = `/person/{id}/thumbnail` + .replace(`{${"id"}}`, encodeURIComponent(String(id))); + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'GET', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + // authentication cookie required + + // authentication api_key required + await setApiKeyToObject(localVarHeaderParameter, "x-api-key", configuration) + + // authentication bearer required + // http bearer authentication required + await setBearerAuthToObject(localVarHeaderParameter, configuration) + + + + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, + /** + * + * @param {string} id + * @param {PersonUpdateDto} personUpdateDto + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + updatePerson: async (id: string, personUpdateDto: PersonUpdateDto, options: AxiosRequestConfig = {}): Promise => { + // verify required parameter 'id' is not null or undefined + assertParamExists('updatePerson', 'id', id) + // verify required parameter 'personUpdateDto' is not null or undefined + assertParamExists('updatePerson', 'personUpdateDto', personUpdateDto) + const localVarPath = `/person/{id}` + .replace(`{${"id"}}`, encodeURIComponent(String(id))); + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'PUT', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + // authentication cookie required + + // authentication api_key required + await setApiKeyToObject(localVarHeaderParameter, "x-api-key", configuration) + + // authentication bearer required + // http bearer authentication required + await setBearerAuthToObject(localVarHeaderParameter, configuration) + + + + localVarHeaderParameter['Content-Type'] = 'application/json'; + + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + localVarRequestOptions.data = serializeDataIfNeeded(personUpdateDto, localVarRequestOptions, configuration) + + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, + } +}; + +/** + * PersonApi - functional programming interface + * @export + */ +export const PersonApiFp = function(configuration?: Configuration) { + const localVarAxiosParamCreator = PersonApiAxiosParamCreator(configuration) + return { + /** + * + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async getAllPeople(options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise>> { + const localVarAxiosArgs = await localVarAxiosParamCreator.getAllPeople(options); + return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration); + }, + /** + * + * @param {string} id + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async getPerson(id: string, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.getPerson(id, options); + return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration); + }, + /** + * + * @param {string} id + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async getPersonAssets(id: string, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise>> { + const localVarAxiosArgs = await localVarAxiosParamCreator.getPersonAssets(id, options); + return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration); + }, + /** + * + * @param {string} id + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async getPersonThumbnail(id: string, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.getPersonThumbnail(id, options); + return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration); + }, + /** + * + * @param {string} id + * @param {PersonUpdateDto} personUpdateDto + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async updatePerson(id: string, personUpdateDto: PersonUpdateDto, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.updatePerson(id, personUpdateDto, options); + return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration); + }, + } +}; + +/** + * PersonApi - factory interface + * @export + */ +export const PersonApiFactory = function (configuration?: Configuration, basePath?: string, axios?: AxiosInstance) { + const localVarFp = PersonApiFp(configuration) + return { + /** + * + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + getAllPeople(options?: any): AxiosPromise> { + return localVarFp.getAllPeople(options).then((request) => request(axios, basePath)); + }, + /** + * + * @param {string} id + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + getPerson(id: string, options?: any): AxiosPromise { + return localVarFp.getPerson(id, options).then((request) => request(axios, basePath)); + }, + /** + * + * @param {string} id + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + getPersonAssets(id: string, options?: any): AxiosPromise> { + return localVarFp.getPersonAssets(id, options).then((request) => request(axios, basePath)); + }, + /** + * + * @param {string} id + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + getPersonThumbnail(id: string, options?: any): AxiosPromise { + return localVarFp.getPersonThumbnail(id, options).then((request) => request(axios, basePath)); + }, + /** + * + * @param {string} id + * @param {PersonUpdateDto} personUpdateDto + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + updatePerson(id: string, personUpdateDto: PersonUpdateDto, options?: any): AxiosPromise { + return localVarFp.updatePerson(id, personUpdateDto, options).then((request) => request(axios, basePath)); + }, + }; +}; + +/** + * PersonApi - object-oriented interface + * @export + * @class PersonApi + * @extends {BaseAPI} + */ +export class PersonApi extends BaseAPI { + /** + * + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof PersonApi + */ + public getAllPeople(options?: AxiosRequestConfig) { + return PersonApiFp(this.configuration).getAllPeople(options).then((request) => request(this.axios, this.basePath)); + } + + /** + * + * @param {string} id + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof PersonApi + */ + public getPerson(id: string, options?: AxiosRequestConfig) { + return PersonApiFp(this.configuration).getPerson(id, options).then((request) => request(this.axios, this.basePath)); + } + + /** + * + * @param {string} id + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof PersonApi + */ + public getPersonAssets(id: string, options?: AxiosRequestConfig) { + return PersonApiFp(this.configuration).getPersonAssets(id, options).then((request) => request(this.axios, this.basePath)); + } + + /** + * + * @param {string} id + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof PersonApi + */ + public getPersonThumbnail(id: string, options?: AxiosRequestConfig) { + return PersonApiFp(this.configuration).getPersonThumbnail(id, options).then((request) => request(this.axios, this.basePath)); + } + + /** + * + * @param {string} id + * @param {PersonUpdateDto} personUpdateDto + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof PersonApi + */ + public updatePerson(id: string, personUpdateDto: PersonUpdateDto, options?: AxiosRequestConfig) { + return PersonApiFp(this.configuration).updatePerson(id, personUpdateDto, options).then((request) => request(this.axios, this.basePath)); + } +} + + /** * SearchApi - axios parameter creator * @export diff --git a/web/src/lib/components/admin-page/jobs/jobs-panel.svelte b/web/src/lib/components/admin-page/jobs/jobs-panel.svelte index b103d4deb0..70db761442 100644 --- a/web/src/lib/components/admin-page/jobs/jobs-panel.svelte +++ b/web/src/lib/components/admin-page/jobs/jobs-panel.svelte @@ -36,6 +36,10 @@ title: 'Encode Clip', subtitle: 'Run machine learning to generate clip embeddings' }, + [JobName.RecognizeFacesQueue]: { + title: 'Recognize Faces', + subtitle: 'Run machine learning to recognize faces' + }, [JobName.VideoConversionQueue]: { title: 'Transcode Videos', subtitle: 'Transcode videos not in the desired format' diff --git a/web/src/lib/components/album-page/album-viewer.svelte b/web/src/lib/components/album-page/album-viewer.svelte index bd832f11f0..702ed4acf3 100644 --- a/web/src/lib/components/album-page/album-viewer.svelte +++ b/web/src/lib/components/album-page/album-viewer.svelte @@ -120,10 +120,6 @@ } }); - const clearMultiSelectAssetAssetHandler = () => { - multiSelectAsset = new Set(); - }; - // Update Album Name $: { if (!isEditingTitle && currentAlbumName != album.albumName && isOwned) { @@ -340,7 +336,7 @@ {#if isMultiSelectionMode} (multiSelectAsset = new Set())} > {#if isOwned} diff --git a/web/src/lib/components/asset-viewer/asset-viewer.svelte b/web/src/lib/components/asset-viewer/asset-viewer.svelte index 74c12f4c73..4c827fca25 100644 --- a/web/src/lib/components/asset-viewer/asset-viewer.svelte +++ b/web/src/lib/components/asset-viewer/asset-viewer.svelte @@ -91,6 +91,11 @@ } }; + const handleCloseViewer = () => { + isShowDetail = false; + closeViewer(); + }; + const closeViewer = () => { dispatch('close'); }; @@ -398,6 +403,7 @@ {asset} albums={appearsInAlbums} on:close={() => (isShowDetail = false)} + on:close-viewer={handleCloseViewer} on:description-focus-in={disableKeyDownEvent} on:description-focus-out={enableKeyDownEvent} /> diff --git a/web/src/lib/components/asset-viewer/detail-panel.svelte b/web/src/lib/components/asset-viewer/detail-panel.svelte index 96616d774e..80e5207614 100644 --- a/web/src/lib/components/asset-viewer/detail-panel.svelte +++ b/web/src/lib/components/asset-viewer/detail-panel.svelte @@ -1,16 +1,17 @@ @@ -13,7 +17,11 @@ src={url} alt={altText} class="object-cover transition-opacity duration-300" + class:rounded-lg={curve} + class:shadow-lg={shadow} + class:rounded-full={circle} class:opacity-0={loading} draggable="false" - on:load|once={() => (loading = false)} + use:imageLoad + on:image-load|once={() => (loading = false)} /> diff --git a/web/src/lib/components/faces-page/edit-name-input.svelte b/web/src/lib/components/faces-page/edit-name-input.svelte new file mode 100644 index 0000000000..9eca6402b3 --- /dev/null +++ b/web/src/lib/components/faces-page/edit-name-input.svelte @@ -0,0 +1,42 @@ + + +
+ +
+ + + +
+
diff --git a/web/src/lib/constants.ts b/web/src/lib/constants.ts index 41563b72ad..4c77563811 100644 --- a/web/src/lib/constants.ts +++ b/web/src/lib/constants.ts @@ -10,6 +10,7 @@ export enum AppRoute { ALBUMS = '/albums', ARCHIVE = '/archive', FAVORITES = '/favorites', + PEOPLE = '/people', PHOTOS = '/photos', EXPLORE = '/explore', SHARING = '/sharing', diff --git a/web/src/routes/(user)/explore/+page.server.ts b/web/src/routes/(user)/explore/+page.server.ts index 9eaf78ca19..8d4d074742 100644 --- a/web/src/routes/(user)/explore/+page.server.ts +++ b/web/src/routes/(user)/explore/+page.server.ts @@ -8,10 +8,11 @@ export const load = (async ({ locals, parent }) => { } const { data: items } = await locals.api.searchApi.getExploreData(); - + const { data: people } = await locals.api.personApi.getAllPeople(); return { user, items, + people, meta: { title: 'Explore' } diff --git a/web/src/routes/(user)/explore/+page.svelte b/web/src/routes/(user)/explore/+page.svelte index d8fc3adf2c..11b515cc3c 100644 --- a/web/src/routes/(user)/explore/+page.svelte +++ b/web/src/routes/(user)/explore/+page.svelte @@ -1,12 +1,13 @@ -
+
+ {#if people.length > 0} +
+
+

People

+ {#if data.people.length > MAX_ITEMS} + View All + {/if} +
+
+ {#each people as person (person.id)} + + +

{person.name}

+ + {/each} +
+
+ {/if} + {#if places.length > 0}
diff --git a/web/src/routes/(user)/people/+page.server.ts b/web/src/routes/(user)/people/+page.server.ts new file mode 100644 index 0000000000..8fabca2c4e --- /dev/null +++ b/web/src/routes/(user)/people/+page.server.ts @@ -0,0 +1,19 @@ +import { redirect } from '@sveltejs/kit'; +import type { PageServerLoad } from './$types'; + +export const load = (async ({ locals, parent }) => { + const { user } = await parent(); + if (!user) { + throw redirect(302, '/auth/login'); + } + + const { data: people } = await locals.api.personApi.getAllPeople(); + + return { + user, + people, + meta: { + title: 'People' + } + }; +}) satisfies PageServerLoad; diff --git a/web/src/routes/(user)/people/+page.svelte b/web/src/routes/(user)/people/+page.svelte new file mode 100644 index 0000000000..1f83d9d46e --- /dev/null +++ b/web/src/routes/(user)/people/+page.svelte @@ -0,0 +1,48 @@ + + + + {#if data.people.length > 0} +
+
+ {#each data.people as person (person.id)} +
+ +
+ +
+ {#if person.name} + + {person.name} + + {/if} + +
+ {/each} +
+
+ {:else} +
+
+ +

No people

+
+
+ {/if} + diff --git a/web/src/routes/(user)/people/[personId]/+page.server.ts b/web/src/routes/(user)/people/[personId]/+page.server.ts new file mode 100644 index 0000000000..b12f7a166b --- /dev/null +++ b/web/src/routes/(user)/people/[personId]/+page.server.ts @@ -0,0 +1,21 @@ +import { redirect } from '@sveltejs/kit'; +import type { PageServerLoad } from './$types'; + +export const load = (async ({ locals, parent, params }) => { + const { user } = await parent(); + if (!user) { + throw redirect(302, '/auth/login'); + } + + const { data: person } = await locals.api.personApi.getPerson(params.personId); + const { data: assets } = await locals.api.personApi.getPersonAssets(params.personId); + + return { + user, + assets, + person, + meta: { + title: person.name || 'Person' + } + }; +}) satisfies PageServerLoad; diff --git a/web/src/routes/(user)/people/[personId]/+page.svelte b/web/src/routes/(user)/people/[personId]/+page.svelte new file mode 100644 index 0000000000..7c22215396 --- /dev/null +++ b/web/src/routes/(user)/people/[personId]/+page.svelte @@ -0,0 +1,113 @@ + + +{#if isMultiSelectionMode} + (multiSelectAsset = new Set())} + > + + + + + + + + + + +{:else} + goto(AppRoute.EXPLORE)} + /> +{/if} + + +
+ {#if isEditName} + handleNameChange(event.detail)} + on:blur={() => (isEditName = false)} + /> + {:else} + + + {/if} +
+ + +
+
+
+ +
+
+
diff --git a/web/src/routes/(user)/search/+page.svelte b/web/src/routes/(user)/search/+page.svelte index 4c50243d0b..96c9361e75 100644 --- a/web/src/routes/(user)/search/+page.svelte +++ b/web/src/routes/(user)/search/+page.svelte @@ -91,7 +91,7 @@
{:else}