diff --git a/web/src/lib/components/faces-page/person-side-panel.svelte b/web/src/lib/components/faces-page/person-side-panel.svelte
index df7082d785..8ca462cf79 100644
--- a/web/src/lib/components/faces-page/person-side-panel.svelte
+++ b/web/src/lib/components/faces-page/person-side-panel.svelte
@@ -16,6 +16,7 @@
   import { photoViewer } from '$lib/stores/assets.store';
   import UnassignedFacesSidePannel from './unassigned-faces-side-pannel.svelte';
   import type { FaceWithGeneretedThumbnail } from '$lib/utils/people-utils';
+  import { cloneDeep } from 'lodash-es';
 
   export let assetId: string;
 
@@ -28,8 +29,10 @@
   let selectedPersonToReassign: (PersonResponseDto | null)[];
   let selectedPersonToCreate: (string | null)[];
   let selectedPersonToAdd: FaceWithGeneretedThumbnail[] = [];
+  let selectedPersonToUnassign: AssetFaceResponseDto[] = [];
   let selectedPersonToRemove: boolean[] = [];
   let unassignedFaces: (FaceWithGeneretedThumbnail | null)[] = [];
+  let unassignedFacesOriginal: (FaceWithGeneretedThumbnail | null)[] = [];
   let editedPersonIndex: number;
   let shouldRefresh: boolean = false;
 
@@ -91,6 +94,7 @@
           }
         }),
       );
+      unassignedFacesOriginal = cloneDeep(unassignedFaces);
     } catch (error) {
       handleError(error, "Can't get faces");
     } finally {
@@ -147,38 +151,31 @@
 
   const handleUnassignFaces = async () => {
     if (numberOfFacesToUnassign > 0) {
-      try {
-        for (let i = 0; i < peopleWithFaces.length; i++) {
-          if (selectedPersonToRemove[i]) {
-            await api.faceApi.unassignFace({
-              id: peopleWithFaces[i].id,
-            });
-            shouldRefresh = true;
-            peopleWithFaces[i].person = null;
-            const image = await zoomImageToBase64(peopleWithFaces[i], $photoViewer);
-            if (image) {
-              unassignedFaces[i] = { ...peopleWithFaces[i], customThumbnail: image };
-            }
+      for (let i = 0; i < peopleWithFaces.length; i++) {
+        if (selectedPersonToRemove[i]) {
+          const image = await zoomImageToBase64(peopleWithFaces[i], $photoViewer);
+          if (image) {
+            unassignedFaces[i] = { ...peopleWithFaces[i], customThumbnail: image };
+            selectedPersonToUnassign.push(peopleWithFaces[i]);
           }
         }
-
-        notificationController.show({
-          message: `Unassigned ${numberOfFacesToUnassign} face${numberOfFacesToUnassign > 1 ? 's' : ''}`,
-          type: NotificationType.Info,
-        });
-      } catch (error) {
-        handleError(error, "Can't apply changes");
       }
+      const uniqueIds = new Set(selectedPersonToUnassign.map((objA) => objA.id));
+      selectedPersonToAdd = selectedPersonToAdd.filter((objB) => !uniqueIds.has(objB.id));
     }
     isSelectingFaces = false;
   };
 
   const handleEditFaces = async () => {
     loaderLoadingDoneTimeout = setTimeout(() => (isShowLoadingDone = true), 100);
+    const uniqueIds = new Set(unassignedFacesOriginal.map((objA) => objA && objA.id));
+    selectedPersonToUnassign = selectedPersonToUnassign.filter((objB) => !uniqueIds.has(objB.id));
+
     const numberOfChanges =
       selectedPersonToCreate.filter((person) => person !== null).length +
       selectedPersonToReassign.filter((person) => person !== null).length +
-      selectedPersonToAdd.length;
+      selectedPersonToAdd.length +
+      selectedPersonToUnassign.length;
     if (numberOfChanges > 0) {
       try {
         for (let i = 0; i < peopleWithFaces.length; i++) {
@@ -214,6 +211,12 @@
           }
         }
 
+        for (const face of selectedPersonToUnassign) {
+          await api.faceApi.unassignFace({
+            id: face.id,
+          });
+        }
+
         notificationController.show({
           message: `Edited ${numberOfChanges} ${numberOfChanges > 1 ? 'people' : 'person'}`,
           type: NotificationType.Info,
@@ -249,12 +252,16 @@
 
   const handleCreatePersonFromUnassignedFace = (face: FaceWithGeneretedThumbnail) => {
     selectedPersonToAdd.push(face);
+    const uniqueIds = new Set(selectedPersonToAdd.map((objA) => objA.id));
+    selectedPersonToUnassign = selectedPersonToUnassign.filter((objB) => !uniqueIds.has(objB.id));
     selectedPersonToAdd = selectedPersonToAdd;
     showUnassignedFaces = false;
   };
 
   const handleReassignFaceFromUnassignedFace = (face: FaceWithGeneretedThumbnail) => {
     selectedPersonToAdd.push(face);
+    const uniqueIds = new Set(selectedPersonToAdd.map((objA) => objA.id));
+    selectedPersonToUnassign = selectedPersonToUnassign.filter((objB) => !uniqueIds.has(objB.id));
     selectedPersonToAdd = selectedPersonToAdd;
     showUnassignedFaces = false;
   };
@@ -350,7 +357,7 @@
         </div>
       {:else}
         {#each peopleWithFaces as face, index}
-          {#if face.person}
+          {#if face.person && unassignedFaces[index] === null}
             <div class="relative z-[20001] h-[115px] w-[95px]">
               <div
                 role="button"
diff --git a/web/src/lib/components/faces-page/unassigned-faces-side-pannel.svelte b/web/src/lib/components/faces-page/unassigned-faces-side-pannel.svelte
index 833fb11136..ed79337ba0 100644
--- a/web/src/lib/components/faces-page/unassigned-faces-side-pannel.svelte
+++ b/web/src/lib/components/faces-page/unassigned-faces-side-pannel.svelte
@@ -55,7 +55,7 @@
 
 <section
   transition:fly={{ x: 360, duration: 100, easing: linear }}
-  class="absolute top-0 z-[2001] h-full w-[360px] overflow-x-hidden p-2 dark:bg-immich-dark-bg dark:text-immich-dark-fg"
+  class="absolute top-0 z-[2001] h-full w-[360px] overflow-x-hidden p-2 bg-immich-bg dark:bg-immich-dark-bg dark:text-immich-dark-fg"
 >
   <div class="flex place-items-center justify-between gap-2">
     <div class="flex items-center gap-2">
@@ -70,7 +70,7 @@
       <p class="flex text-lg text-immich-fg dark:text-immich-dark-fg">Faces Available</p>
     </div>
   </div>
-  {#if unassignedFaces.length > 0}
+  {#if unassignedFaces.some((face) => face)}
     <div class="px-4 py-4 text-sm">
       <div class="mt-4 flex flex-wrap gap-2">
         {#each unassignedFaces as face, index}
@@ -103,8 +103,10 @@
     </div>
   {:else}
     <div class="flex items-center justify-center">
-      <Icon path={mdiAccountOff} size="3.5em" />
-      <p class="mt-5 font-medium">No faces available</p>
+      <div class="grid place-items-center">
+        <Icon path={mdiAccountOff} size="3.5em" />
+        <p class="mt-5 font-medium">No faces available</p>
+      </div>
     </div>
   {/if}
 </section>