feat(mobile): check hash before uploading

mobile/lib/models/backup/bulk_upload_check_result.model.dart

@@ -0,0 +1,167 @@
+// ignore_for_file: public_member_api_docs, sort_constructors_first
+import 'dart:convert';
+
+import 'package:collection/collection.dart';
+
+class RejectResult {
+  final String localId;
+  final String remoteId;
+
+  RejectResult({
+    required this.localId,
+    required this.remoteId,
+  });
+
+  RejectResult copyWith({
+    String? localId,
+    String? remoteId,
+  }) {
+    return RejectResult(
+      localId: localId ?? this.localId,
+      remoteId: remoteId ?? this.remoteId,
+    );
+  }
+
+  Map<String, dynamic> toMap() {
+    return <String, dynamic>{
+      'localId': localId,
+      'remoteId': remoteId,
+    };
+  }
+
+  factory RejectResult.fromMap(Map<String, dynamic> map) {
+    return RejectResult(
+      localId: map['localId'] as String,
+      remoteId: map['remoteId'] as String,
+    );
+  }
+
+  String toJson() => json.encode(toMap());
+
+  factory RejectResult.fromJson(String source) =>
+      RejectResult.fromMap(json.decode(source) as Map<String, dynamic>);
+
+  @override
+  String toString() => 'RejectResult(localId: $localId, remoteId: $remoteId)';
+
+  @override
+  bool operator ==(covariant RejectResult other) {
+    if (identical(this, other)) return true;
+
+    return other.localId == localId && other.remoteId == remoteId;
+  }
+
+  @override
+  int get hashCode => localId.hashCode ^ remoteId.hashCode;
+}
+
+class AcceptResult {
+  final String localId;
+
+  AcceptResult({
+    required this.localId,
+  });
+
+  AcceptResult copyWith({
+    String? localId,
+  }) {
+    return AcceptResult(
+      localId: localId ?? this.localId,
+    );
+  }
+
+  Map<String, dynamic> toMap() {
+    return <String, dynamic>{
+      'localId': localId,
+    };
+  }
+
+  factory AcceptResult.fromMap(Map<String, dynamic> map) {
+    return AcceptResult(
+      localId: map['localId'] as String,
+    );
+  }
+
+  String toJson() => json.encode(toMap());
+
+  factory AcceptResult.fromJson(String source) =>
+      AcceptResult.fromMap(json.decode(source) as Map<String, dynamic>);
+
+  @override
+  String toString() => 'AcceptResult(localId: $localId)';
+
+  @override
+  bool operator ==(covariant AcceptResult other) {
+    if (identical(this, other)) return true;
+
+    return other.localId == localId;
+  }
+
+  @override
+  int get hashCode => localId.hashCode;
+}
+
+class BulkUploadCheckResult {
+  List<RejectResult> rejects;
+  List<AcceptResult> accepts;
+
+  BulkUploadCheckResult({
+    required this.rejects,
+    required this.accepts,
+  });
+
+  BulkUploadCheckResult copyWith({
+    List<RejectResult>? rejects,
+    List<AcceptResult>? accepts,
+  }) {
+    return BulkUploadCheckResult(
+      rejects: rejects ?? this.rejects,
+      accepts: accepts ?? this.accepts,
+    );
+  }
+
+  Map<String, dynamic> toMap() {
+    return <String, dynamic>{
+      'rejects': rejects.map((x) => x.toMap()).toList(),
+      'accepts': accepts.map((x) => x.toMap()).toList(),
+    };
+  }
+
+  factory BulkUploadCheckResult.fromMap(Map<String, dynamic> map) {
+    return BulkUploadCheckResult(
+      rejects: List<RejectResult>.from(
+        (map['rejects'] as List<int>).map<RejectResult>(
+          (x) => RejectResult.fromMap(x as Map<String, dynamic>),
+        ),
+      ),
+      accepts: List<AcceptResult>.from(
+        (map['accepts'] as List<int>).map<AcceptResult>(
+          (x) => AcceptResult.fromMap(x as Map<String, dynamic>),
+        ),
+      ),
+    );
+  }
+
+  String toJson() => json.encode(toMap());
+
+  factory BulkUploadCheckResult.fromJson(String source) =>
+      BulkUploadCheckResult.fromMap(
+        json.decode(source) as Map<String, dynamic>,
+      );
+
+  @override
+  String toString() =>
+      'BulkUploadCheckResult(rejects: $rejects, accepts: $accepts)';
+
+  @override
+  bool operator ==(covariant BulkUploadCheckResult other) {
+    if (identical(this, other)) return true;
+    final listEquals = const DeepCollectionEquality().equals;
+
+    return listEquals(other.rejects, rejects) &&
+        listEquals(other.accepts, accepts);
+  }
+
+  @override
+  int get hashCode => rejects.hashCode ^ accepts.hashCode;
+}

mobile/lib/providers/backup/backup.provider.dart

@@ -462,36 +462,39 @@ class BackupNotifier extends StateNotifier<BackUpState> {
         return;
       }
 
-      Set<BackupCandidate> assetsWillBeBackup = Set.from(state.allUniqueAssets);
+      Set<BackupCandidate> candidates = Set.from(state.allUniqueAssets);
       // Remove item that has already been backed up
       for (final assetId in state.allAssetsInDatabase) {
-        assetsWillBeBackup.removeWhere((e) => e.asset.id == assetId);
+        candidates.removeWhere((e) => e.asset.id == assetId);
       }
 
-      if (assetsWillBeBackup.isEmpty) {
+      if (candidates.isEmpty) {
         state = state.copyWith(backupProgress: BackUpProgressEnum.idle);
       }
 
-      // Perform Backup
-      state = state.copyWith(cancelToken: CancellationToken());
+      // Check with server for hash duplication
+      await _backupService.checkBulkUpload(candidates);
 
-      final pmProgressHandler = Platform.isIOS ? PMProgressHandler() : null;
+      // // Perform Backup
+      // state = state.copyWith(cancelToken: CancellationToken());
 
-      pmProgressHandler?.stream.listen((event) {
-        final double progress = event.progress;
-        state = state.copyWith(iCloudDownloadProgress: progress);
-      });
+      // final pmProgressHandler = Platform.isIOS ? PMProgressHandler() : null;
 
-      await _backupService.backupAsset(
-        assetsWillBeBackup,
-        state.cancelToken,
-        pmProgressHandler: pmProgressHandler,
-        onSuccess: _onAssetUploaded,
-        onProgress: _onUploadProgress,
-        onCurrentAsset: _onSetCurrentBackupAsset,
-        onError: _onBackupError,
-      );
-      await notifyBackgroundServiceCanRun();
+      // pmProgressHandler?.stream.listen((event) {
+      //   final double progress = event.progress;
+      //   state = state.copyWith(iCloudDownloadProgress: progress);
+      // });
+
+      // await _backupService.backupAsset(
+      //   candidates,
+      //   state.cancelToken,
+      //   pmProgressHandler: pmProgressHandler,
+      //   onSuccess: _onAssetUploaded,
+      //   onProgress: _onUploadProgress,
+      //   onCurrentAsset: _onSetCurrentBackupAsset,
+      //   onError: _onBackupError,
+      // );
+      // await notifyBackgroundServiceCanRun();
     } else {
       openAppSettings();
     }

mobile/lib/services/background.service.dart

@@ -361,8 +361,13 @@ class BackgroundService {
         UserService(apiService, db, syncSerive, partnerService);
     AlbumService albumService =
         AlbumService(apiService, userService, syncSerive, db);
-    BackupService backupService =
-        BackupService(apiService, db, settingService, albumService);
+    BackupService backupService = BackupService(
+      apiService,
+      db,
+      settingService,
+      albumService,
+      hashService,
+    );
 
     final selectedAlbums = backupService.selectedAlbumsQuery().findAllSync();
     final excludedAlbums = backupService.excludedAlbumsQuery().findAllSync();

mobile/lib/services/backup.service.dart

@@ -7,9 +7,12 @@ import 'package:collection/collection.dart';
 import 'package:flutter/material.dart';
 import 'package:hooks_riverpod/hooks_riverpod.dart';
 import 'package:immich_mobile/entities/backup_album.entity.dart';
+import 'package:immich_mobile/entities/device_asset.entity.dart';
 import 'package:immich_mobile/entities/duplicated_asset.entity.dart';
 import 'package:immich_mobile/entities/store.entity.dart';
+import 'package:immich_mobile/extensions/string_extensions.dart';
 import 'package:immich_mobile/models/backup/backup_candidate.model.dart';
+import 'package:immich_mobile/models/backup/bulk_upload_check_result.model.dart';
 import 'package:immich_mobile/models/backup/current_upload_asset.model.dart';
 import 'package:immich_mobile/models/backup/error_upload_asset.model.dart';
 import 'package:immich_mobile/models/backup/success_upload_asset.model.dart';
@@ -19,6 +22,7 @@ import 'package:immich_mobile/providers/db.provider.dart';
 import 'package:immich_mobile/services/album.service.dart';
 import 'package:immich_mobile/services/api.service.dart';
 import 'package:immich_mobile/services/app_settings.service.dart';
+import 'package:immich_mobile/services/hash.service.dart';
 import 'package:isar/isar.dart';
 import 'package:logging/logging.dart';
 import 'package:openapi/api.dart';
@@ -32,6 +36,7 @@ final backupServiceProvider = Provider(
     ref.watch(dbProvider),
     ref.watch(appSettingsServiceProvider),
     ref.watch(albumServiceProvider),
+    ref.watch(hashServiceProvider),
   ),
 );
 
@@ -42,14 +47,72 @@ class BackupService {
   final Logger _log = Logger("BackupService");
   final AppSettingsService _appSetting;
   final AlbumService _albumService;
+  final HashService _hashService;
 
   BackupService(
     this._apiService,
     this._db,
     this._appSetting,
     this._albumService,
+    this._hashService,
   );
 
+  Future<BulkUploadCheckResult> checkBulkUpload(
+    Set<BackupCandidate> candidates,
+  ) async {
+    List<AssetBulkUploadCheckItem> assets = [];
+
+    final assetEntities = candidates.map((c) => c.asset).toList();
+    final hashedDeviceAssets =
+        await _hashService.getHashedAssetsFromAssetEntity(assetEntities);
+
+    for (final hashedAsset in hashedDeviceAssets) {
+      final AssetBulkUploadCheckItem item = AssetBulkUploadCheckItem(
+        id: hashedAsset.id.toString(),
+        checksum: hashedAsset.checksum,
+      );
+
+      assets.add(item);
+    }
+
+    final response = await _apiService.assetsApi.checkBulkUpload(
+      AssetBulkUploadCheckDto(assets: assets),
+    );
+
+    // AssetBulkUploadCheckResult[action=reject, assetId=6929085c-ad33-489b-a352-af1bdcf19ee6, id=-9223372036854775808, reason=duplicate]
+    if (response == null) {
+      return BulkUploadCheckResult(
+        rejects: [],
+        accepts: [],
+      );
+    }
+
+    final List<RejectResult> rejects = [];
+    final List<AcceptResult> accepts = [];
+
+    for (final result in response.results) {
+      if (result.action == AssetBulkUploadCheckResultActionEnum.reject) {
+        rejects.add(
+          RejectResult(
+            localId: result.id,
+            remoteId: result.assetId ?? "",
+          ),
+        );
+      } else {
+        accepts.add(
+          AcceptResult(
+            localId: result.id,
+          ),
+        );
+      }
+    }
+
+    return BulkUploadCheckResult(
+      rejects: rejects,
+      accepts: accepts,
+    );
+  }
+
   Future<List<String>?> getDeviceBackupAsset() async {
     final String deviceId = Store.get(StoreKey.deviceId);
 

mobile/lib/services/hash.service.dart

@@ -19,8 +19,20 @@ class HashService {
   final BackgroundService _backgroundService;
   final _log = Logger('HashService');
 
+  Future<List<Asset>> getHashedAssetsFromAssetEntity(
+    List<AssetEntity> assets,
+  ) async {
+    final ids = assets
+        .map(Platform.isAndroid ? (a) => a.id.toInt() : (a) => a.id)
+        .toList();
+
+    final List<DeviceAsset?> hashes = await lookupHashes(ids);
+
+    return _mapAllHashedAssets(assets, hashes);
+  }
+
   /// Returns all assets that were successfully hashed
-  Future<List<Asset>> getHashedAssets(
+  Future<List<Asset>> getHashedAssetsFromDeviceAlbum(
     AssetPathEntity album, {
     int start = 0,
     int end = 0x7fffffffffffffff,
@@ -44,7 +56,7 @@ class HashService {
     final ids = assetEntities
         .map(Platform.isAndroid ? (a) => a.id.toInt() : (a) => a.id)
         .toList();
-    final List<DeviceAsset?> hashes = await _lookupHashes(ids);
+    final List<DeviceAsset?> hashes = await lookupHashes(ids);
     final List<DeviceAsset> toAdd = [];
     final List<String> toHash = [];
 
@@ -90,7 +102,7 @@ class HashService {
   }
 
   /// Lookup hashes of assets by their local ID
-  Future<List<DeviceAsset?>> _lookupHashes(List<Object> ids) =>
+  Future<List<DeviceAsset?>> lookupHashes(List<Object> ids) =>
       Platform.isAndroid
           ? _db.androidDeviceAssets.getAll(ids.cast())
           : _db.iOSDeviceAssets.getAllById(ids.cast());

mobile/lib/services/sync.service.dart

@@ -566,8 +566,8 @@ class SyncService {
         .findAll();
     assert(inDb.isSorted(Asset.compareByChecksum), "inDb not sorted!");
     final int assetCountOnDevice = await ape.assetCountAsync;
-    final List<Asset> onDevice =
-        await _hashService.getHashedAssets(ape, excludedAssets: excludedAssets);
+    final List<Asset> onDevice = await _hashService
+        .getHashedAssetsFromDeviceAlbum(ape, excludedAssets: excludedAssets);
     _removeDuplicates(onDevice);
     // _removeDuplicates sorts `onDevice` by checksum
     final (toAdd, toUpdate, toDelete) = _diffAssets(onDevice, inDb);
@@ -649,7 +649,8 @@ class SyncService {
     if (modified == null) {
       return false;
     }
-    final List<Asset> newAssets = await _hashService.getHashedAssets(modified);
+    final List<Asset> newAssets =
+        await _hashService.getHashedAssetsFromDeviceAlbum(modified);
 
     if (totalOnDevice != lastKnownTotal + newAssets.length) {
       return false;
@@ -683,8 +684,8 @@ class SyncService {
   ]) async {
     _log.info("Syncing a new local album to DB: ${ape.name}");
     final Album a = Album.local(ape);
-    final assets =
-        await _hashService.getHashedAssets(ape, excludedAssets: excludedAssets);
+    final assets = await _hashService.getHashedAssetsFromDeviceAlbum(ape,
+        excludedAssets: excludedAssets);
     _removeDuplicates(assets);
     final (existingInDb, updated) = await _linkWithExistingFromDb(assets);
     _log.info(