feat(web): Improve duplicate suggestion (#14947)

* feat: Improve duplicate suggestion

* format

* feat(web): Add deduplication info popup

* fix: lint

* fmt

---------

Co-authored-by: Alex <alex.tran1502@gmail.com>

i18n/de.json

@@ -523,6 +523,10 @@
   "date_range": "Datumsbereich",
   "day": "Tag",
   "deduplicate_all": "Alle Duplikate entfernen",
+  "deduplication_info": "Deduplizierungsinformationen",
+  "deduplication_info_description": "Für die automatische Datei-Vorauswahl und das Deduplizieren aller Dateien berücksichtigen wir:",
+  "deduplication_criteria_1": "Bildgröße in Bytes",
+  "deduplication_criteria_2": "Anzahl der EXIF-Daten",
   "default_locale": "Standard-Sprache",
   "default_locale_description": "Datumsangaben und Zahlen basierend auf dem Gebietsschema des Browsers formatieren",
   "delete": "Löschen",

i18n/en.json

@@ -523,6 +523,10 @@
   "date_range": "Date range",
   "day": "Day",
   "deduplicate_all": "Deduplicate All",
+  "deduplication_info": "Deduplication Info",
+  "deduplication_info_description": "To automatically preselect assets and remove duplicates in bulk, we look at:",
+  "deduplication_criteria_1": "Image size in bytes",
+  "deduplication_criteria_2": "Count of EXIF data",
   "default_locale": "Default Locale",
   "default_locale_description": "Format dates and numbers based on your browser locale",
   "delete": "Delete",

web/src/lib/components/shared-components/duplicates-modal.svelte

@@ -0,0 +1,20 @@
+<script lang="ts">
+  import { t } from 'svelte-i18n';
+  import FullScreenModal from './full-screen-modal.svelte';
+
+  interface Props {
+    onClose: () => void;
+  }
+
+  let { onClose }: Props = $props();
+</script>
+
+<FullScreenModal title={$t('deduplication_info')} width="auto" {onClose}>
+  <div class="text-sm dark:text-white">
+    <p>{$t('deduplication_info_description')}</p>
+    <ol class="ml-8 mt-2" style="list-style: decimal">
+      <li>{$t('deduplication_criteria_1')}</li>
+      <li>{$t('deduplication_criteria_2')}</li>
+    </ol>
+  </div>
+</FullScreenModal>

web/src/lib/components/utilities-page/duplicates/duplicate-asset.svelte

@@ -3,8 +3,8 @@
   import { getAssetThumbnailUrl } from '$lib/utils';
   import { getAssetResolution, getFileSize } from '$lib/utils/asset-utils';
   import { getAltText } from '$lib/utils/thumbnail-util';
-  import { getAllAlbums, type AssetResponseDto } from '@immich/sdk';
-  import { mdiHeart, mdiMagnifyPlus, mdiImageMultipleOutline } from '@mdi/js';
+  import { type AssetResponseDto, getAllAlbums } from '@immich/sdk';
+  import { mdiHeart, mdiImageMultipleOutline, mdiMagnifyPlus } from '@mdi/js';
   import { t } from 'svelte-i18n';
 
   interface Props {

web/src/lib/components/utilities-page/duplicates/duplicates-compare-control.svelte

@@ -4,7 +4,8 @@
   import Portal from '$lib/components/shared-components/portal/portal.svelte';
   import DuplicateAsset from '$lib/components/utilities-page/duplicates/duplicate-asset.svelte';
   import { assetViewingStore } from '$lib/stores/asset-viewing.store';
-  import { handlePromiseError, suggestDuplicateByFileSize } from '$lib/utils';
+  import { handlePromiseError } from '$lib/utils';
+  import { suggestDuplicate } from '$lib/utils/duplicate-utils';
   import { navigate } from '$lib/utils/navigation';
   import { shortcuts } from '$lib/actions/shortcut';
   import { type AssetResponseDto } from '@immich/sdk';
@@ -27,7 +28,7 @@
   let trashCount = $derived(assets.length - selectedAssetIds.size);
 
   onMount(() => {
-    const suggestedAsset = suggestDuplicateByFileSize(assets);
+    const suggestedAsset = suggestDuplicate(assets);
 
     if (!suggestedAsset) {
       selectedAssetIds = new SvelteSet(assets[0].id);

web/src/lib/utils.ts

@@ -16,13 +16,11 @@ import {
   linkOAuthAccount,
   startOAuth,
   unlinkOAuthAccount,
-  type AssetResponseDto,
   type PersonResponseDto,
   type SharedLinkResponseDto,
   type UserResponseDto,
 } from '@immich/sdk';
 import { mdiCogRefreshOutline, mdiDatabaseRefreshOutline, mdiHeadSyncOutline, mdiImageRefreshOutline } from '@mdi/js';
-import { sortBy } from 'lodash-es';
 import { init, register, t } from 'svelte-i18n';
 import { derived, get } from 'svelte/store';
 
@@ -332,9 +330,5 @@ export const withError = async <T>(fn: () => Promise<T>): Promise<[undefined, T]
   }
 };
 
-export const suggestDuplicateByFileSize = (assets: AssetResponseDto[]): AssetResponseDto | undefined => {
-  return sortBy(assets, (asset) => asset.exifInfo?.fileSizeInByte).pop();
-};
-
 // eslint-disable-next-line unicorn/prefer-code-point
 export const decodeBase64 = (data: string) => Uint8Array.from(atob(data), (c) => c.charCodeAt(0));

web/src/lib/utils/duplicate-utils.spec.ts

@@ -0,0 +1,37 @@
+import { suggestDuplicate } from '$lib/utils/duplicate-utils';
+import type { AssetResponseDto } from '@immich/sdk';
+
+describe('choosing a duplicate', () => {
+  it('picks the asset with the largest file size', () => {
+    const assets = [
+      { exifInfo: { fileSizeInByte: 300 } },
+      { exifInfo: { fileSizeInByte: 200 } },
+      { exifInfo: { fileSizeInByte: 100 } },
+    ];
+    expect(suggestDuplicate(assets as AssetResponseDto[])).toEqual(assets[0]);
+  });
+
+  it('picks the asset with the most exif data if multiple assets have the same file size', () => {
+    const assets = [
+      { exifInfo: { fileSizeInByte: 200, rating: 5, fNumber: 1 } },
+      { exifInfo: { fileSizeInByte: 200, rating: 5 } },
+      { exifInfo: { fileSizeInByte: 100, rating: 5 } },
+    ];
+    expect(suggestDuplicate(assets as AssetResponseDto[])).toEqual(assets[0]);
+  });
+
+  it('returns undefined for an empty array', () => {
+    const assets: AssetResponseDto[] = [];
+    expect(suggestDuplicate(assets)).toBeUndefined();
+  });
+
+  it('handles assets with no exifInfo', () => {
+    const assets = [{ exifInfo: { fileSizeInByte: 200 } }, {}];
+    expect(suggestDuplicate(assets as AssetResponseDto[])).toEqual(assets[0]);
+  });
+
+  it('handles assets with exifInfo but no fileSizeInByte', () => {
+    const assets = [{ exifInfo: { rating: 5, fNumber: 1 } }, { exifInfo: { rating: 5 } }];
+    expect(suggestDuplicate(assets as AssetResponseDto[])).toEqual(assets[0]);
+  });
+});

web/src/lib/utils/duplicate-utils.ts

@@ -0,0 +1,30 @@
+import { getExifCount } from '$lib/utils/exif-utils';
+import type { AssetResponseDto } from '@immich/sdk';
+import { sortBy } from 'lodash-es';
+
+/**
+ * Suggests the best duplicate asset to keep from a list of duplicates.
+ *
+ * The best asset is determined by the following criteria:
+ *  - Largest image file size in bytes
+ *  - Largest count of exif data
+ *
+ * @param assets List of duplicate assets
+ * @returns The best asset to keep
+ */
+export const suggestDuplicate = (assets: AssetResponseDto[]): AssetResponseDto | undefined => {
+  let duplicateAssets = sortBy(assets, (asset) => asset.exifInfo?.fileSizeInByte ?? 0);
+
+  // Update the list to only include assets with the largest file size
+  duplicateAssets = duplicateAssets.filter(
+    (asset) => asset.exifInfo?.fileSizeInByte === duplicateAssets.at(-1)?.exifInfo?.fileSizeInByte,
+  );
+
+  // If there are multiple assets with the same file size, sort the list by the count of exif data
+  if (duplicateAssets.length >= 2) {
+    duplicateAssets = sortBy(duplicateAssets, getExifCount);
+  }
+
+  // Return the last asset in the list
+  return duplicateAssets.pop();
+};

web/src/lib/utils/exif-utils.spec.ts

@@ -0,0 +1,29 @@
+import { getExifCount } from '$lib/utils/exif-utils';
+import type { AssetResponseDto } from '@immich/sdk';
+
+describe('getting the exif count', () => {
+  it('returns 0 when exifInfo is undefined', () => {
+    const asset = {};
+    expect(getExifCount(asset as AssetResponseDto)).toBe(0);
+  });
+
+  it('returns 0 when exifInfo is empty', () => {
+    const asset = { exifInfo: {} };
+    expect(getExifCount(asset as AssetResponseDto)).toBe(0);
+  });
+
+  it('returns the correct count of non-null exifInfo properties', () => {
+    const asset = { exifInfo: { fileSizeInByte: 200, rating: 5, fNumber: null } };
+    expect(getExifCount(asset as AssetResponseDto)).toBe(2);
+  });
+
+  it('ignores null, undefined and empty properties in exifInfo', () => {
+    const asset = { exifInfo: { fileSizeInByte: 200, rating: null, fNumber: undefined, description: '' } };
+    expect(getExifCount(asset as AssetResponseDto)).toBe(1);
+  });
+
+  it('returns the correct count when all exifInfo properties are non-null', () => {
+    const asset = { exifInfo: { fileSizeInByte: 200, rating: 5, fNumber: 1, description: 'test' } };
+    expect(getExifCount(asset as AssetResponseDto)).toBe(4);
+  });
+});

web/src/lib/utils/exif-utils.ts

@@ -0,0 +1,5 @@
+import type { AssetResponseDto } from '@immich/sdk';
+
+export const getExifCount = (asset: AssetResponseDto) => {
+  return Object.values(asset.exifInfo ?? {}).filter(Boolean).length;
+};

web/src/routes/(user)/utilities/duplicates/[[photos=photos]]/[[assetId=id]]/+page.svelte

@@ -12,11 +12,12 @@
   import { deleteAssets, updateAssets } from '@immich/sdk';
   import { t } from 'svelte-i18n';
   import type { PageData } from './$types';
-  import { suggestDuplicateByFileSize } from '$lib/utils';
+  import { suggestDuplicate } from '$lib/utils/duplicate-utils';
   import LinkButton from '$lib/components/elements/buttons/link-button.svelte';
-  import { mdiCheckOutline, mdiTrashCanOutline } from '@mdi/js';
+  import { mdiCheckOutline, mdiInformationOutline, mdiTrashCanOutline } from '@mdi/js';
   import { stackAssets } from '$lib/utils/asset-utils';
   import ShowShortcuts from '$lib/components/shared-components/show-shortcuts.svelte';
+  import DuplicatesModal from '$lib/components/shared-components/duplicates-modal.svelte';
   import CircleIconButton from '$lib/components/elements/buttons/circle-icon-button.svelte';
   import { mdiKeyboard } from '@mdi/js';
   import Icon from '$lib/components/elements/icon.svelte';
@@ -25,9 +26,14 @@
   interface Props {
     data: PageData;
     isShowKeyboardShortcut?: boolean;
+    isShowDuplicateInfo?: boolean;
   }
 
-  let { data = $bindable(), isShowKeyboardShortcut = $bindable(false) }: Props = $props();
+  let {
+    data = $bindable(),
+    isShowKeyboardShortcut = $bindable(false),
+    isShowDuplicateInfo = $bindable(false),
+  }: Props = $props();
 
   interface Shortcuts {
     general: ExplainedShortcut[];
@@ -103,7 +109,7 @@
   };
 
   const handleDeduplicateAll = async () => {
-    const idsToKeep = duplicates.map((group) => suggestDuplicateByFileSize(group.assets)).map((asset) => asset?.id);
+    const idsToKeep = duplicates.map((group) => suggestDuplicate(group.assets)).map((asset) => asset?.id);
     const idsToDelete = duplicates.flatMap((group, i) =>
       group.assets.map((asset) => asset.id).filter((asset) => asset !== idsToKeep[i]),
     );
@@ -178,11 +184,21 @@
     </div>
   {/snippet}
 
-  <div class="mt-4">
+  <div class="">
     {#if duplicates && duplicates.length > 0}
-      <div class="mb-4 text-sm dark:text-white">
-        <p>{$t('duplicates_description')}</p>
+      <div class="flex items-center mb-2">
+        <div class="text-sm dark:text-white">
+          <p>{$t('duplicates_description')}</p>
+        </div>
+        <CircleIconButton
+          icon={mdiInformationOutline}
+          title={$t('deduplication_info')}
+          size="16"
+          padding="2"
+          onclick={() => (isShowDuplicateInfo = true)}
+        />
       </div>
+
       {#key duplicates[0].duplicateId}
         <DuplicatesCompareControl
           assets={duplicates[0].assets}
@@ -202,3 +218,6 @@
 {#if isShowKeyboardShortcut}
   <ShowShortcuts shortcuts={duplicateShortcuts} onClose={() => (isShowKeyboardShortcut = false)} />
 {/if}
+{#if isShowDuplicateInfo}
+  <DuplicatesModal onClose={() => (isShowDuplicateInfo = false)} />
+{/if}