diff --git a/server/src/domain/person/person.service.spec.ts b/server/src/domain/person/person.service.spec.ts
index e1937524af..9d55abc8e6 100644
--- a/server/src/domain/person/person.service.spec.ts
+++ b/server/src/domain/person/person.service.spec.ts
@@ -866,11 +866,29 @@ describe(PersonService.name, () => {
       });
     });
 
-    it('should defer non-core faces to end of queue', async () => {
+    it('should not queue face with no matches', async () => {
       const faces = [{ face: faceStub.noPerson1, distance: 0 }] as FaceSearchResult[];
 
+      smartInfoMock.searchFaces.mockResolvedValue(faces);
+      personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1);
+      personMock.create.mockResolvedValue(personStub.withName);
+
+      await sut.handleRecognizeFaces({ id: faceStub.noPerson1.id });
+
+      expect(jobMock.queue).not.toHaveBeenCalled();
+      expect(smartInfoMock.searchFaces).toHaveBeenCalledTimes(1);
+      expect(personMock.create).not.toHaveBeenCalled();
+      expect(personMock.reassignFaces).not.toHaveBeenCalled();
+    });
+
+    it('should defer non-core faces to end of queue', async () => {
+      const faces = [
+        { face: faceStub.noPerson1, distance: 0 },
+        { face: faceStub.noPerson2, distance: 0.4 },
+      ] as FaceSearchResult[];
+
       configMock.load.mockResolvedValue([
-        { key: SystemConfigKey.MACHINE_LEARNING_FACIAL_RECOGNITION_MIN_FACES, value: 2 },
+        { key: SystemConfigKey.MACHINE_LEARNING_FACIAL_RECOGNITION_MIN_FACES, value: 3 },
       ]);
       smartInfoMock.searchFaces.mockResolvedValue(faces);
       personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1);
@@ -887,11 +905,14 @@ describe(PersonService.name, () => {
       expect(personMock.reassignFaces).not.toHaveBeenCalled();
     });
 
-    it('should not assign person to non-core face with no matching person', async () => {
-      const faces = [{ face: faceStub.noPerson1, distance: 0 }] as FaceSearchResult[];
+    it('should not assign person to deferred non-core face with no matching person', async () => {
+      const faces = [
+        { face: faceStub.noPerson1, distance: 0 },
+        { face: faceStub.noPerson2, distance: 0.4 },
+      ] as FaceSearchResult[];
 
       configMock.load.mockResolvedValue([
-        { key: SystemConfigKey.MACHINE_LEARNING_FACIAL_RECOGNITION_MIN_FACES, value: 2 },
+        { key: SystemConfigKey.MACHINE_LEARNING_FACIAL_RECOGNITION_MIN_FACES, value: 3 },
       ]);
       smartInfoMock.searchFaces.mockResolvedValueOnce(faces).mockResolvedValueOnce([]);
       personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1);
diff --git a/server/src/domain/person/person.service.ts b/server/src/domain/person/person.service.ts
index 576f94c491..63fc350002 100644
--- a/server/src/domain/person/person.service.ts
+++ b/server/src/domain/person/person.service.ts
@@ -417,7 +417,13 @@ export class PersonService {
       numResults: machineLearning.facialRecognition.minFaces,
     });
 
-    this.logger.debug(`Face ${id} has ${matches.length} match${matches.length == 1 ? '' : 'es'}`);
+    // `matches` also includes the face itself
+    if (matches.length <= 1) {
+      this.logger.debug(`Face ${id} has no matches`);
+      return true;
+    }
+
+    this.logger.debug(`Face ${id} has ${matches.length} matches`);
 
     const isCore = matches.length >= machineLearning.facialRecognition.minFaces;
     if (!isCore && !deferred) {
@@ -426,7 +432,7 @@ export class PersonService {
       return true;
     }
 
-    let personId = matches.find((match) => match.face.personId)?.face.personId; // `matches` also includes the face itself
+    let personId = matches.find((match) => match.face.personId)?.face.personId;
     if (!personId) {
       const matchWithPerson = await this.smartInfoRepository.searchFaces({
         userIds: [face.asset.ownerId],