diff --git a/mobile/android/app/build.gradle b/mobile/android/app/build.gradle index 158a272082..38ef303b6e 100644 --- a/mobile/android/app/build.gradle +++ b/mobile/android/app/build.gradle @@ -84,6 +84,7 @@ flutter { dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version" + implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:$kotlin_coroutines_version" implementation "androidx.work:work-runtime-ktx:$work_version" implementation "androidx.concurrent:concurrent-futures:$concurrent_version" implementation "com.google.guava:guava:$guava_version" diff --git a/mobile/android/app/src/main/kotlin/com/example/mobile/BackgroundServicePlugin.kt b/mobile/android/app/src/main/kotlin/com/example/mobile/BackgroundServicePlugin.kt index 1e9b2502d1..6541ad5755 100644 --- a/mobile/android/app/src/main/kotlin/com/example/mobile/BackgroundServicePlugin.kt +++ b/mobile/android/app/src/main/kotlin/com/example/mobile/BackgroundServicePlugin.kt @@ -1,10 +1,15 @@ package app.alextran.immich import android.content.Context +import android.util.Log import io.flutter.embedding.engine.plugins.FlutterPlugin import io.flutter.plugin.common.BinaryMessenger import io.flutter.plugin.common.MethodCall import io.flutter.plugin.common.MethodChannel +import java.security.MessageDigest +import java.io.File +import java.io.FileInputStream +import kotlinx.coroutines.* /** * Android plugin for Dart `BackgroundService` @@ -16,6 +21,7 @@ class BackgroundServicePlugin : FlutterPlugin, MethodChannel.MethodCallHandler { private var methodChannel: MethodChannel? = null private var context: Context? = null + private val sha1: MessageDigest = MessageDigest.getInstance("SHA-1") override fun onAttachedToEngine(binding: FlutterPlugin.FlutterPluginBinding) { onAttachedToEngine(binding.applicationContext, binding.binaryMessenger) @@ -70,9 +76,40 @@ class BackgroundServicePlugin : FlutterPlugin, MethodChannel.MethodCallHandler { "isIgnoringBatteryOptimizations" -> { result.success(BackupWorker.isIgnoringBatteryOptimizations(ctx)) } + "digestFiles" -> { + val args = call.arguments>()!! + GlobalScope.launch(Dispatchers.IO) { + val buf = ByteArray(BUFSIZE) + val digest: MessageDigest = MessageDigest.getInstance("SHA-1") + val hashes = arrayOfNulls(args.size) + for (i in args.indices) { + val path = args[i] + var len = 0 + try { + val file = FileInputStream(path) + try { + while (true) { + len = file.read(buf) + if (len != BUFSIZE) break + digest.update(buf) + } + } finally { + file.close() + } + digest.update(buf, 0, len) + hashes[i] = digest.digest() + } catch (e: Exception) { + // skip this file + Log.w(TAG, "Failed to hash file ${args[i]}: $e") + } + } + result.success(hashes.asList()) + } + } else -> result.notImplemented() } } } -private const val TAG = "BackgroundServicePlugin" \ No newline at end of file +private const val TAG = "BackgroundServicePlugin" +private const val BUFSIZE = 2*1024*1024; diff --git a/mobile/android/build.gradle b/mobile/android/build.gradle index ee01aa0f15..5d8f2bf6e6 100644 --- a/mobile/android/build.gradle +++ b/mobile/android/build.gradle @@ -1,5 +1,6 @@ buildscript { ext.kotlin_version = '1.8.20' + ext.kotlin_coroutines_version = '1.7.1' ext.work_version = '2.7.1' ext.concurrent_version = '1.1.0' ext.guava_version = '31.0.1-android' diff --git a/mobile/lib/main.dart b/mobile/lib/main.dart index 3c8cedc9de..1dfc630d48 100644 --- a/mobile/lib/main.dart +++ b/mobile/lib/main.dart @@ -19,9 +19,11 @@ import 'package:immich_mobile/modules/settings/providers/notification_permission import 'package:immich_mobile/routing/router.dart'; import 'package:immich_mobile/routing/tab_navigation_observer.dart'; import 'package:immich_mobile/shared/models/album.dart'; +import 'package:immich_mobile/shared/models/android_device_asset.dart'; import 'package:immich_mobile/shared/models/asset.dart'; import 'package:immich_mobile/shared/models/etag.dart'; import 'package:immich_mobile/shared/models/exif_info.dart'; +import 'package:immich_mobile/shared/models/ios_device_asset.dart'; import 'package:immich_mobile/shared/models/logger_message.model.dart'; import 'package:immich_mobile/shared/models/store.dart'; import 'package:immich_mobile/shared/models/user.dart'; @@ -91,6 +93,7 @@ Future loadDb() async { DuplicatedAssetSchema, LoggerMessageSchema, ETagSchema, + Platform.isAndroid ? AndroidDeviceAssetSchema : IOSDeviceAssetSchema, ], directory: dir.path, maxSizeMiB: 256, diff --git a/mobile/lib/modules/asset_viewer/providers/image_viewer_page_state.provider.dart b/mobile/lib/modules/asset_viewer/providers/image_viewer_page_state.provider.dart index ffb2e980eb..b95a31c13d 100644 --- a/mobile/lib/modules/asset_viewer/providers/image_viewer_page_state.provider.dart +++ b/mobile/lib/modules/asset_viewer/providers/image_viewer_page_state.provider.dart @@ -2,6 +2,7 @@ import 'package:easy_localization/easy_localization.dart'; import 'package:flutter/material.dart'; import 'package:fluttertoast/fluttertoast.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/modules/album/services/album.service.dart'; import 'package:immich_mobile/modules/asset_viewer/models/image_viewer_page_state.model.dart'; import 'package:immich_mobile/modules/asset_viewer/services/image_viewer.service.dart'; import 'package:immich_mobile/shared/models/asset.dart'; @@ -12,9 +13,13 @@ import 'package:immich_mobile/shared/ui/share_dialog.dart'; class ImageViewerStateNotifier extends StateNotifier { final ImageViewerService _imageViewerService; final ShareService _shareService; + final AlbumService _albumService; - ImageViewerStateNotifier(this._imageViewerService, this._shareService) - : super( + ImageViewerStateNotifier( + this._imageViewerService, + this._shareService, + this._albumService, + ) : super( ImageViewerPageState( downloadAssetStatus: DownloadAssetStatus.idle, ), @@ -34,6 +39,7 @@ class ImageViewerStateNotifier extends StateNotifier { toastType: ToastType.success, gravity: ToastGravity.BOTTOM, ); + _albumService.refreshDeviceAlbums(); } else { state = state.copyWith(downloadAssetStatus: DownloadAssetStatus.error); ImmichToast.show( @@ -66,5 +72,6 @@ final imageViewerStateProvider = ((ref) => ImageViewerStateNotifier( ref.watch(imageViewerServiceProvider), ref.watch(shareServiceProvider), + ref.watch(albumServiceProvider), )), ); diff --git a/mobile/lib/modules/asset_viewer/ui/top_control_app_bar.dart b/mobile/lib/modules/asset_viewer/ui/top_control_app_bar.dart index b79f240bba..ddd1b40a81 100644 --- a/mobile/lib/modules/asset_viewer/ui/top_control_app_bar.dart +++ b/mobile/lib/modules/asset_viewer/ui/top_control_app_bar.dart @@ -72,15 +72,7 @@ class TopControlAppBar extends HookConsumerWidget { color: Colors.grey[200], ), ), - if (!asset.isLocal) - IconButton( - onPressed: onDownloadPressed, - icon: Icon( - Icons.cloud_download_outlined, - color: Colors.grey[200], - ), - ), - if (asset.storage == AssetState.merged) + if (asset.storage == AssetState.remote) IconButton( onPressed: onDownloadPressed, icon: Icon( diff --git a/mobile/lib/modules/asset_viewer/views/gallery_viewer.dart b/mobile/lib/modules/asset_viewer/views/gallery_viewer.dart index 29e932b354..f67fdf5763 100644 --- a/mobile/lib/modules/asset_viewer/views/gallery_viewer.dart +++ b/mobile/lib/modules/asset_viewer/views/gallery_viewer.dart @@ -287,7 +287,7 @@ class GalleryViewerPage extends HookConsumerWidget { isFavorite: asset().isFavorite, onMoreInfoPressed: showInfo, onFavorite: asset().isRemote ? () => toggleFavorite(asset()) : null, - onDownloadPressed: asset().storage == AssetState.local + onDownloadPressed: asset().isLocal ? null : () => ref.watch(imageViewerStateProvider.notifier).downloadAsset( diff --git a/mobile/lib/modules/backup/background_service/background.service.dart b/mobile/lib/modules/backup/background_service/background.service.dart index 319c2890f7..5817e1d2b4 100644 --- a/mobile/lib/modules/backup/background_service/background.service.dart +++ b/mobile/lib/modules/backup/background_service/background.service.dart @@ -132,6 +132,17 @@ class BackgroundService { } } + Future digestFile(String path) { + return _foregroundChannel.invokeMethod("digestFile", [path]); + } + + Future?> digestFiles(List paths) { + return _foregroundChannel.invokeListMethod( + "digestFiles", + paths, + ); + } + /// Updates the notification shown by the background service Future _updateNotification({ String? title, diff --git a/mobile/lib/modules/home/views/home_page.dart b/mobile/lib/modules/home/views/home_page.dart index 957b15133c..292389c057 100644 --- a/mobile/lib/modules/home/views/home_page.dart +++ b/mobile/lib/modules/home/views/home_page.dart @@ -47,11 +47,11 @@ class HomePage extends HookConsumerWidget { useEffect( () { - ref.watch(websocketProvider.notifier).connect(); - ref.watch(assetProvider.notifier).getAllAsset(); - ref.watch(albumProvider.notifier).getAllAlbums(); - ref.watch(sharedAlbumProvider.notifier).getAllSharedAlbums(); - ref.watch(serverInfoProvider.notifier).getServerVersion(); + ref.read(websocketProvider.notifier).connect(); + Future(() => ref.read(assetProvider.notifier).getAllAsset()); + ref.read(albumProvider.notifier).getAllAlbums(); + ref.read(sharedAlbumProvider.notifier).getAllSharedAlbums(); + ref.read(serverInfoProvider.notifier).getServerVersion(); selectionEnabledHook.addListener(() { multiselectEnabled.state = selectionEnabledHook.value; @@ -144,7 +144,7 @@ class HomePage extends HookConsumerWidget { ); if (remoteAssets.isNotEmpty) { await ref - .watch(assetProvider.notifier) + .read(assetProvider.notifier) .toggleArchive(remoteAssets, true); final assetOrAssets = remoteAssets.length > 1 ? 'assets' : 'asset'; @@ -163,7 +163,7 @@ class HomePage extends HookConsumerWidget { void onDelete() async { processing.value = true; try { - await ref.watch(assetProvider.notifier).deleteAssets(selection.value); + await ref.read(assetProvider.notifier).deleteAssets(selection.value); selectionEnabledHook.value = false; } finally { processing.value = false; diff --git a/mobile/lib/shared/models/album.dart b/mobile/lib/shared/models/album.dart index a3cffa1698..81a834b993 100644 --- a/mobile/lib/shared/models/album.dart +++ b/mobile/lib/shared/models/album.dart @@ -166,23 +166,10 @@ extension AssetsHelper on IsarCollection { } } -extension AssetPathEntityHelper on AssetPathEntity { - Future> getAssets({ - int start = 0, - int end = 0x7fffffffffffffff, - Set? excludedAssets, - }) async { - final assetEntities = await getAssetListRange(start: start, end: end); - if (excludedAssets != null) { - return assetEntities - .where((e) => !excludedAssets.contains(e.id)) - .map(Asset.local) - .toList(); - } - return assetEntities.map(Asset.local).toList(); - } -} - extension AlbumResponseDtoHelper on AlbumResponseDto { List getAssets() => assets.map(Asset.remote).toList(); } + +extension AssetPathEntityHelper on AssetPathEntity { + String get eTagKeyAssetCount => "device-album-$id-asset-count"; +} diff --git a/mobile/lib/shared/models/android_device_asset.dart b/mobile/lib/shared/models/android_device_asset.dart new file mode 100644 index 0000000000..b6b2663fd7 --- /dev/null +++ b/mobile/lib/shared/models/android_device_asset.dart @@ -0,0 +1,10 @@ +import 'package:immich_mobile/shared/models/device_asset.dart'; +import 'package:isar/isar.dart'; + +part 'android_device_asset.g.dart'; + +@Collection() +class AndroidDeviceAsset extends DeviceAsset { + AndroidDeviceAsset({required this.id, required super.hash}); + Id id; +} diff --git a/mobile/lib/shared/models/android_device_asset.g.dart b/mobile/lib/shared/models/android_device_asset.g.dart new file mode 100644 index 0000000000..ca7c822ba0 --- /dev/null +++ b/mobile/lib/shared/models/android_device_asset.g.dart @@ -0,0 +1,493 @@ +// GENERATED CODE - DO NOT MODIFY BY HAND + +part of 'android_device_asset.dart'; + +// ************************************************************************** +// IsarCollectionGenerator +// ************************************************************************** + +// coverage:ignore-file +// ignore_for_file: duplicate_ignore, non_constant_identifier_names, constant_identifier_names, invalid_use_of_protected_member, unnecessary_cast, prefer_const_constructors, lines_longer_than_80_chars, require_trailing_commas, inference_failure_on_function_invocation, unnecessary_parenthesis, unnecessary_raw_strings, unnecessary_null_checks, join_return_with_assignment, prefer_final_locals, avoid_js_rounded_ints, avoid_positional_boolean_parameters, always_specify_types + +extension GetAndroidDeviceAssetCollection on Isar { + IsarCollection get androidDeviceAssets => + this.collection(); +} + +const AndroidDeviceAssetSchema = CollectionSchema( + name: r'AndroidDeviceAsset', + id: -6758387181232899335, + properties: { + r'hash': PropertySchema( + id: 0, + name: r'hash', + type: IsarType.byteList, + ) + }, + estimateSize: _androidDeviceAssetEstimateSize, + serialize: _androidDeviceAssetSerialize, + deserialize: _androidDeviceAssetDeserialize, + deserializeProp: _androidDeviceAssetDeserializeProp, + idName: r'id', + indexes: { + r'hash': IndexSchema( + id: -7973251393006690288, + name: r'hash', + unique: false, + replace: false, + properties: [ + IndexPropertySchema( + name: r'hash', + type: IndexType.hash, + caseSensitive: false, + ) + ], + ) + }, + links: {}, + embeddedSchemas: {}, + getId: _androidDeviceAssetGetId, + getLinks: _androidDeviceAssetGetLinks, + attach: _androidDeviceAssetAttach, + version: '3.1.0+1', +); + +int _androidDeviceAssetEstimateSize( + AndroidDeviceAsset object, + List offsets, + Map> allOffsets, +) { + var bytesCount = offsets.last; + bytesCount += 3 + object.hash.length; + return bytesCount; +} + +void _androidDeviceAssetSerialize( + AndroidDeviceAsset object, + IsarWriter writer, + List offsets, + Map> allOffsets, +) { + writer.writeByteList(offsets[0], object.hash); +} + +AndroidDeviceAsset _androidDeviceAssetDeserialize( + Id id, + IsarReader reader, + List offsets, + Map> allOffsets, +) { + final object = AndroidDeviceAsset( + hash: reader.readByteList(offsets[0]) ?? [], + id: id, + ); + return object; +} + +P _androidDeviceAssetDeserializeProp

( + IsarReader reader, + int propertyId, + int offset, + Map> allOffsets, +) { + switch (propertyId) { + case 0: + return (reader.readByteList(offset) ?? []) as P; + default: + throw IsarError('Unknown property with id $propertyId'); + } +} + +Id _androidDeviceAssetGetId(AndroidDeviceAsset object) { + return object.id; +} + +List> _androidDeviceAssetGetLinks( + AndroidDeviceAsset object) { + return []; +} + +void _androidDeviceAssetAttach( + IsarCollection col, Id id, AndroidDeviceAsset object) { + object.id = id; +} + +extension AndroidDeviceAssetQueryWhereSort + on QueryBuilder { + QueryBuilder anyId() { + return QueryBuilder.apply(this, (query) { + return query.addWhereClause(const IdWhereClause.any()); + }); + } +} + +extension AndroidDeviceAssetQueryWhere + on QueryBuilder { + QueryBuilder + idEqualTo(Id id) { + return QueryBuilder.apply(this, (query) { + return query.addWhereClause(IdWhereClause.between( + lower: id, + upper: id, + )); + }); + } + + QueryBuilder + idNotEqualTo(Id id) { + return QueryBuilder.apply(this, (query) { + if (query.whereSort == Sort.asc) { + return query + .addWhereClause( + IdWhereClause.lessThan(upper: id, includeUpper: false), + ) + .addWhereClause( + IdWhereClause.greaterThan(lower: id, includeLower: false), + ); + } else { + return query + .addWhereClause( + IdWhereClause.greaterThan(lower: id, includeLower: false), + ) + .addWhereClause( + IdWhereClause.lessThan(upper: id, includeUpper: false), + ); + } + }); + } + + QueryBuilder + idGreaterThan(Id id, {bool include = false}) { + return QueryBuilder.apply(this, (query) { + return query.addWhereClause( + IdWhereClause.greaterThan(lower: id, includeLower: include), + ); + }); + } + + QueryBuilder + idLessThan(Id id, {bool include = false}) { + return QueryBuilder.apply(this, (query) { + return query.addWhereClause( + IdWhereClause.lessThan(upper: id, includeUpper: include), + ); + }); + } + + QueryBuilder + idBetween( + Id lowerId, + Id upperId, { + bool includeLower = true, + bool includeUpper = true, + }) { + return QueryBuilder.apply(this, (query) { + return query.addWhereClause(IdWhereClause.between( + lower: lowerId, + includeLower: includeLower, + upper: upperId, + includeUpper: includeUpper, + )); + }); + } + + QueryBuilder + hashEqualTo(List hash) { + return QueryBuilder.apply(this, (query) { + return query.addWhereClause(IndexWhereClause.equalTo( + indexName: r'hash', + value: [hash], + )); + }); + } + + QueryBuilder + hashNotEqualTo(List hash) { + return QueryBuilder.apply(this, (query) { + if (query.whereSort == Sort.asc) { + return query + .addWhereClause(IndexWhereClause.between( + indexName: r'hash', + lower: [], + upper: [hash], + includeUpper: false, + )) + .addWhereClause(IndexWhereClause.between( + indexName: r'hash', + lower: [hash], + includeLower: false, + upper: [], + )); + } else { + return query + .addWhereClause(IndexWhereClause.between( + indexName: r'hash', + lower: [hash], + includeLower: false, + upper: [], + )) + .addWhereClause(IndexWhereClause.between( + indexName: r'hash', + lower: [], + upper: [hash], + includeUpper: false, + )); + } + }); + } +} + +extension AndroidDeviceAssetQueryFilter + on QueryBuilder { + QueryBuilder + hashElementEqualTo(int value) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.equalTo( + property: r'hash', + value: value, + )); + }); + } + + QueryBuilder + hashElementGreaterThan( + int value, { + bool include = false, + }) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.greaterThan( + include: include, + property: r'hash', + value: value, + )); + }); + } + + QueryBuilder + hashElementLessThan( + int value, { + bool include = false, + }) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.lessThan( + include: include, + property: r'hash', + value: value, + )); + }); + } + + QueryBuilder + hashElementBetween( + int lower, + int upper, { + bool includeLower = true, + bool includeUpper = true, + }) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.between( + property: r'hash', + lower: lower, + includeLower: includeLower, + upper: upper, + includeUpper: includeUpper, + )); + }); + } + + QueryBuilder + hashLengthEqualTo(int length) { + return QueryBuilder.apply(this, (query) { + return query.listLength( + r'hash', + length, + true, + length, + true, + ); + }); + } + + QueryBuilder + hashIsEmpty() { + return QueryBuilder.apply(this, (query) { + return query.listLength( + r'hash', + 0, + true, + 0, + true, + ); + }); + } + + QueryBuilder + hashIsNotEmpty() { + return QueryBuilder.apply(this, (query) { + return query.listLength( + r'hash', + 0, + false, + 999999, + true, + ); + }); + } + + QueryBuilder + hashLengthLessThan( + int length, { + bool include = false, + }) { + return QueryBuilder.apply(this, (query) { + return query.listLength( + r'hash', + 0, + true, + length, + include, + ); + }); + } + + QueryBuilder + hashLengthGreaterThan( + int length, { + bool include = false, + }) { + return QueryBuilder.apply(this, (query) { + return query.listLength( + r'hash', + length, + include, + 999999, + true, + ); + }); + } + + QueryBuilder + hashLengthBetween( + int lower, + int upper, { + bool includeLower = true, + bool includeUpper = true, + }) { + return QueryBuilder.apply(this, (query) { + return query.listLength( + r'hash', + lower, + includeLower, + upper, + includeUpper, + ); + }); + } + + QueryBuilder + idEqualTo(Id value) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.equalTo( + property: r'id', + value: value, + )); + }); + } + + QueryBuilder + idGreaterThan( + Id value, { + bool include = false, + }) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.greaterThan( + include: include, + property: r'id', + value: value, + )); + }); + } + + QueryBuilder + idLessThan( + Id value, { + bool include = false, + }) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.lessThan( + include: include, + property: r'id', + value: value, + )); + }); + } + + QueryBuilder + idBetween( + Id lower, + Id upper, { + bool includeLower = true, + bool includeUpper = true, + }) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.between( + property: r'id', + lower: lower, + includeLower: includeLower, + upper: upper, + includeUpper: includeUpper, + )); + }); + } +} + +extension AndroidDeviceAssetQueryObject + on QueryBuilder {} + +extension AndroidDeviceAssetQueryLinks + on QueryBuilder {} + +extension AndroidDeviceAssetQuerySortBy + on QueryBuilder {} + +extension AndroidDeviceAssetQuerySortThenBy + on QueryBuilder { + QueryBuilder + thenById() { + return QueryBuilder.apply(this, (query) { + return query.addSortBy(r'id', Sort.asc); + }); + } + + QueryBuilder + thenByIdDesc() { + return QueryBuilder.apply(this, (query) { + return query.addSortBy(r'id', Sort.desc); + }); + } +} + +extension AndroidDeviceAssetQueryWhereDistinct + on QueryBuilder { + QueryBuilder + distinctByHash() { + return QueryBuilder.apply(this, (query) { + return query.addDistinctBy(r'hash'); + }); + } +} + +extension AndroidDeviceAssetQueryProperty + on QueryBuilder { + QueryBuilder idProperty() { + return QueryBuilder.apply(this, (query) { + return query.addPropertyName(r'id'); + }); + } + + QueryBuilder, QQueryOperations> hashProperty() { + return QueryBuilder.apply(this, (query) { + return query.addPropertyName(r'hash'); + }); + } +} diff --git a/mobile/lib/shared/models/asset.dart b/mobile/lib/shared/models/asset.dart index 840653554e..701b6c2dd7 100644 --- a/mobile/lib/shared/models/asset.dart +++ b/mobile/lib/shared/models/asset.dart @@ -1,3 +1,5 @@ +import 'dart:convert'; + import 'package:immich_mobile/shared/models/exif_info.dart'; import 'package:immich_mobile/shared/models/store.dart'; import 'package:immich_mobile/utils/hash.dart'; @@ -14,7 +16,7 @@ part 'asset.g.dart'; class Asset { Asset.remote(AssetResponseDto remote) : remoteId = remote.id, - isLocal = false, + checksum = remote.checksum, fileCreatedAt = remote.fileCreatedAt, fileModifiedAt = remote.fileModifiedAt, updatedAt = remote.updatedAt, @@ -24,23 +26,20 @@ class Asset { height = remote.exifInfo?.exifImageHeight?.toInt(), width = remote.exifInfo?.exifImageWidth?.toInt(), livePhotoVideoId = remote.livePhotoVideoId, - localId = remote.deviceAssetId, - deviceId = fastHash(remote.deviceId), ownerId = fastHash(remote.ownerId), exifInfo = remote.exifInfo != null ? ExifInfo.fromDto(remote.exifInfo!) : null, isFavorite = remote.isFavorite, isArchived = remote.isArchived; - Asset.local(AssetEntity local) + Asset.local(AssetEntity local, List hash) : localId = local.id, - isLocal = true, + checksum = base64.encode(hash), durationInSeconds = local.duration, type = AssetType.values[local.typeInt], height = local.height, width = local.width, fileName = local.title!, - deviceId = Store.get(StoreKey.deviceIdHash), ownerId = Store.get(StoreKey.currentUser).isarId, fileModifiedAt = local.modifiedDateTime, updatedAt = local.modifiedDateTime, @@ -53,13 +52,15 @@ class Asset { if (local.latitude != null) { exifInfo = ExifInfo(lat: local.latitude, long: local.longitude); } + _local = local; + assert(hash.length == 20, "invalid SHA1 hash"); } Asset({ this.id = Isar.autoIncrement, + required this.checksum, this.remoteId, required this.localId, - required this.deviceId, required this.ownerId, required this.fileCreatedAt, required this.fileModifiedAt, @@ -72,7 +73,6 @@ class Asset { this.livePhotoVideoId, this.exifInfo, required this.isFavorite, - required this.isLocal, required this.isArchived, }); @@ -83,7 +83,7 @@ class Asset { AssetEntity? get local { if (isLocal && _local == null) { _local = AssetEntity( - id: localId, + id: localId!, typeInt: isImage ? 1 : 2, width: width ?? 0, height: height ?? 0, @@ -98,18 +98,21 @@ class Asset { Id id = Isar.autoIncrement; + /// stores the raw SHA1 bytes as a base64 String + /// because Isar cannot sort lists of byte arrays + @Index( + unique: true, + replace: false, + type: IndexType.hash, + composite: [CompositeIndex("ownerId")], + ) + String checksum; + @Index(unique: false, replace: false, type: IndexType.hash) String? remoteId; - @Index( - unique: false, - replace: false, - type: IndexType.hash, - composite: [CompositeIndex('deviceId')], - ) - String localId; - - int deviceId; + @Index(unique: false, replace: false, type: IndexType.hash) + String? localId; int ownerId; @@ -134,14 +137,15 @@ class Asset { bool isFavorite; - /// `true` if this [Asset] is present on the device - bool isLocal; - bool isArchived; @ignore ExifInfo? exifInfo; + /// `true` if this [Asset] is present on the device + @ignore + bool get isLocal => localId != null; + @ignore bool get isInDb => id != Isar.autoIncrement; @@ -175,9 +179,9 @@ class Asset { bool operator ==(other) { if (other is! Asset) return false; return id == other.id && + checksum == other.checksum && remoteId == other.remoteId && localId == other.localId && - deviceId == other.deviceId && ownerId == other.ownerId && fileCreatedAt.isAtSameMomentAs(other.fileCreatedAt) && fileModifiedAt.isAtSameMomentAs(other.fileModifiedAt) && @@ -197,9 +201,9 @@ class Asset { @ignore int get hashCode => id.hashCode ^ + checksum.hashCode ^ remoteId.hashCode ^ localId.hashCode ^ - deviceId.hashCode ^ ownerId.hashCode ^ fileCreatedAt.hashCode ^ fileModifiedAt.hashCode ^ @@ -217,8 +221,7 @@ class Asset { /// Returns `true` if this [Asset] can updated with values from parameter [a] bool canUpdate(Asset a) { assert(isInDb); - assert(localId == a.localId); - assert(deviceId == a.deviceId); + assert(checksum == a.checksum); assert(a.storage != AssetState.merged); return a.updatedAt.isAfter(updatedAt) || a.isRemote && !isRemote || @@ -239,11 +242,18 @@ class Asset { if (a.isRemote) { return a._copyWith( id: id, - isLocal: isLocal, + localId: localId, width: a.width ?? width, height: a.height ?? height, exifInfo: a.exifInfo?.copyWith(id: id) ?? exifInfo, ); + } else if (isRemote) { + return _copyWith( + localId: localId ?? a.localId, + width: width ?? a.width, + height: height ?? a.height, + exifInfo: exifInfo ?? a.exifInfo?.copyWith(id: id), + ); } else { return a._copyWith( id: id, @@ -270,7 +280,7 @@ class Asset { } else { // add only missing values (and set isLocal to true) return _copyWith( - isLocal: true, + localId: localId ?? a.localId, width: width ?? a.width, height: height ?? a.height, exifInfo: exifInfo ?? a.exifInfo?.copyWith(id: id), @@ -281,9 +291,9 @@ class Asset { Asset _copyWith({ Id? id, + String? checksum, String? remoteId, String? localId, - int? deviceId, int? ownerId, DateTime? fileCreatedAt, DateTime? fileModifiedAt, @@ -295,15 +305,14 @@ class Asset { String? fileName, String? livePhotoVideoId, bool? isFavorite, - bool? isLocal, bool? isArchived, ExifInfo? exifInfo, }) => Asset( id: id ?? this.id, + checksum: checksum ?? this.checksum, remoteId: remoteId ?? this.remoteId, localId: localId ?? this.localId, - deviceId: deviceId ?? this.deviceId, ownerId: ownerId ?? this.ownerId, fileCreatedAt: fileCreatedAt ?? this.fileCreatedAt, fileModifiedAt: fileModifiedAt ?? this.fileModifiedAt, @@ -315,7 +324,6 @@ class Asset { fileName: fileName ?? this.fileName, livePhotoVideoId: livePhotoVideoId ?? this.livePhotoVideoId, isFavorite: isFavorite ?? this.isFavorite, - isLocal: isLocal ?? this.isLocal, isArchived: isArchived ?? this.isArchived, exifInfo: exifInfo ?? this.exifInfo, ); @@ -328,39 +336,36 @@ class Asset { } } - /// compares assets by [ownerId], [deviceId], [localId] - static int compareByOwnerDeviceLocalId(Asset a, Asset b) { - final int ownerIdOrder = a.ownerId.compareTo(b.ownerId); - if (ownerIdOrder != 0) { - return ownerIdOrder; - } - final int deviceIdOrder = a.deviceId.compareTo(b.deviceId); - if (deviceIdOrder != 0) { - return deviceIdOrder; - } - final int localIdOrder = a.localId.compareTo(b.localId); - return localIdOrder; - } - - /// compares assets by [ownerId], [deviceId], [localId], [fileModifiedAt] - static int compareByOwnerDeviceLocalIdModified(Asset a, Asset b) { - final int order = compareByOwnerDeviceLocalId(a, b); - return order != 0 ? order : a.fileModifiedAt.compareTo(b.fileModifiedAt); - } - static int compareById(Asset a, Asset b) => a.id.compareTo(b.id); - static int compareByLocalId(Asset a, Asset b) => - a.localId.compareTo(b.localId); + static int compareByChecksum(Asset a, Asset b) => + a.checksum.compareTo(b.checksum); + + static int compareByOwnerChecksum(Asset a, Asset b) { + final int ownerIdOrder = a.ownerId.compareTo(b.ownerId); + if (ownerIdOrder != 0) return ownerIdOrder; + return compareByChecksum(a, b); + } + + static int compareByOwnerChecksumCreatedModified(Asset a, Asset b) { + final int ownerIdOrder = a.ownerId.compareTo(b.ownerId); + if (ownerIdOrder != 0) return ownerIdOrder; + final int checksumOrder = compareByChecksum(a, b); + if (checksumOrder != 0) return checksumOrder; + final int createdOrder = a.fileCreatedAt.compareTo(b.fileCreatedAt); + if (createdOrder != 0) return createdOrder; + return a.fileModifiedAt.compareTo(b.fileModifiedAt); + } @override String toString() { return """ { + "id": ${id == Isar.autoIncrement ? '"N/A"' : id}, "remoteId": "${remoteId ?? "N/A"}", - "localId": "$localId", - "deviceId": "$deviceId", - "ownerId": "$ownerId", + "localId": "${localId ?? "N/A"}", + "checksum": "$checksum", + "ownerId": $ownerId, "livePhotoVideoId": "${livePhotoVideoId ?? "N/A"}", "fileCreatedAt": "$fileCreatedAt", "fileModifiedAt": "$fileModifiedAt", @@ -369,9 +374,8 @@ class Asset { "type": "$type", "fileName": "$fileName", "isFavorite": $isFavorite, - "isLocal": $isLocal, "isRemote: $isRemote, - "storage": $storage, + "storage": "$storage", "width": ${width ?? "N/A"}, "height": ${height ?? "N/A"}, "isArchived": $isArchived @@ -424,10 +428,6 @@ extension AssetsHelper on IsarCollection { QueryBuilder _remote(Iterable ids) => where().anyOf(ids, (q, String e) => q.remoteIdEqualTo(e)); QueryBuilder _local(Iterable ids) { - return where().anyOf( - ids, - (q, String e) => - q.localIdDeviceIdEqualTo(e, Store.get(StoreKey.deviceIdHash)), - ); + return where().anyOf(ids, (q, String e) => q.localIdEqualTo(e)); } } diff --git a/mobile/lib/shared/models/asset.g.dart b/mobile/lib/shared/models/asset.g.dart index e7085bad4c..713c26885b 100644 --- a/mobile/lib/shared/models/asset.g.dart +++ b/mobile/lib/shared/models/asset.g.dart @@ -17,10 +17,10 @@ const AssetSchema = CollectionSchema( name: r'Asset', id: -2933289051367723566, properties: { - r'deviceId': PropertySchema( + r'checksum': PropertySchema( id: 0, - name: r'deviceId', - type: IsarType.long, + name: r'checksum', + type: IsarType.string, ), r'durationInSeconds': PropertySchema( id: 1, @@ -57,44 +57,39 @@ const AssetSchema = CollectionSchema( name: r'isFavorite', type: IsarType.bool, ), - r'isLocal': PropertySchema( - id: 8, - name: r'isLocal', - type: IsarType.bool, - ), r'livePhotoVideoId': PropertySchema( - id: 9, + id: 8, name: r'livePhotoVideoId', type: IsarType.string, ), r'localId': PropertySchema( - id: 10, + id: 9, name: r'localId', type: IsarType.string, ), r'ownerId': PropertySchema( - id: 11, + id: 10, name: r'ownerId', type: IsarType.long, ), r'remoteId': PropertySchema( - id: 12, + id: 11, name: r'remoteId', type: IsarType.string, ), r'type': PropertySchema( - id: 13, + id: 12, name: r'type', type: IsarType.byte, enumMap: _AssettypeEnumValueMap, ), r'updatedAt': PropertySchema( - id: 14, + id: 13, name: r'updatedAt', type: IsarType.dateTime, ), r'width': PropertySchema( - id: 15, + id: 14, name: r'width', type: IsarType.int, ) @@ -105,6 +100,24 @@ const AssetSchema = CollectionSchema( deserializeProp: _assetDeserializeProp, idName: r'id', indexes: { + r'checksum_ownerId': IndexSchema( + id: 5611361749756160119, + name: r'checksum_ownerId', + unique: true, + replace: false, + properties: [ + IndexPropertySchema( + name: r'checksum', + type: IndexType.hash, + caseSensitive: true, + ), + IndexPropertySchema( + name: r'ownerId', + type: IndexType.value, + caseSensitive: false, + ) + ], + ), r'remoteId': IndexSchema( id: 6301175856541681032, name: r'remoteId', @@ -118,9 +131,9 @@ const AssetSchema = CollectionSchema( ) ], ), - r'localId_deviceId': IndexSchema( - id: 7649417350086526165, - name: r'localId_deviceId', + r'localId': IndexSchema( + id: 1199848425898359622, + name: r'localId', unique: false, replace: false, properties: [ @@ -128,11 +141,6 @@ const AssetSchema = CollectionSchema( name: r'localId', type: IndexType.hash, caseSensitive: true, - ), - IndexPropertySchema( - name: r'deviceId', - type: IndexType.value, - caseSensitive: false, ) ], ) @@ -151,6 +159,7 @@ int _assetEstimateSize( Map> allOffsets, ) { var bytesCount = offsets.last; + bytesCount += 3 + object.checksum.length * 3; bytesCount += 3 + object.fileName.length * 3; { final value = object.livePhotoVideoId; @@ -158,7 +167,12 @@ int _assetEstimateSize( bytesCount += 3 + value.length * 3; } } - bytesCount += 3 + object.localId.length * 3; + { + final value = object.localId; + if (value != null) { + bytesCount += 3 + value.length * 3; + } + } { final value = object.remoteId; if (value != null) { @@ -174,7 +188,7 @@ void _assetSerialize( List offsets, Map> allOffsets, ) { - writer.writeLong(offsets[0], object.deviceId); + writer.writeString(offsets[0], object.checksum); writer.writeLong(offsets[1], object.durationInSeconds); writer.writeDateTime(offsets[2], object.fileCreatedAt); writer.writeDateTime(offsets[3], object.fileModifiedAt); @@ -182,14 +196,13 @@ void _assetSerialize( writer.writeInt(offsets[5], object.height); writer.writeBool(offsets[6], object.isArchived); writer.writeBool(offsets[7], object.isFavorite); - writer.writeBool(offsets[8], object.isLocal); - writer.writeString(offsets[9], object.livePhotoVideoId); - writer.writeString(offsets[10], object.localId); - writer.writeLong(offsets[11], object.ownerId); - writer.writeString(offsets[12], object.remoteId); - writer.writeByte(offsets[13], object.type.index); - writer.writeDateTime(offsets[14], object.updatedAt); - writer.writeInt(offsets[15], object.width); + writer.writeString(offsets[8], object.livePhotoVideoId); + writer.writeString(offsets[9], object.localId); + writer.writeLong(offsets[10], object.ownerId); + writer.writeString(offsets[11], object.remoteId); + writer.writeByte(offsets[12], object.type.index); + writer.writeDateTime(offsets[13], object.updatedAt); + writer.writeInt(offsets[14], object.width); } Asset _assetDeserialize( @@ -199,7 +212,7 @@ Asset _assetDeserialize( Map> allOffsets, ) { final object = Asset( - deviceId: reader.readLong(offsets[0]), + checksum: reader.readString(offsets[0]), durationInSeconds: reader.readLong(offsets[1]), fileCreatedAt: reader.readDateTime(offsets[2]), fileModifiedAt: reader.readDateTime(offsets[3]), @@ -208,15 +221,14 @@ Asset _assetDeserialize( id: id, isArchived: reader.readBool(offsets[6]), isFavorite: reader.readBool(offsets[7]), - isLocal: reader.readBool(offsets[8]), - livePhotoVideoId: reader.readStringOrNull(offsets[9]), - localId: reader.readString(offsets[10]), - ownerId: reader.readLong(offsets[11]), - remoteId: reader.readStringOrNull(offsets[12]), - type: _AssettypeValueEnumMap[reader.readByteOrNull(offsets[13])] ?? + livePhotoVideoId: reader.readStringOrNull(offsets[8]), + localId: reader.readStringOrNull(offsets[9]), + ownerId: reader.readLong(offsets[10]), + remoteId: reader.readStringOrNull(offsets[11]), + type: _AssettypeValueEnumMap[reader.readByteOrNull(offsets[12])] ?? AssetType.other, - updatedAt: reader.readDateTime(offsets[14]), - width: reader.readIntOrNull(offsets[15]), + updatedAt: reader.readDateTime(offsets[13]), + width: reader.readIntOrNull(offsets[14]), ); return object; } @@ -229,7 +241,7 @@ P _assetDeserializeProp

( ) { switch (propertyId) { case 0: - return (reader.readLong(offset)) as P; + return (reader.readString(offset)) as P; case 1: return (reader.readLong(offset)) as P; case 2: @@ -245,21 +257,19 @@ P _assetDeserializeProp

( case 7: return (reader.readBool(offset)) as P; case 8: - return (reader.readBool(offset)) as P; + return (reader.readStringOrNull(offset)) as P; case 9: return (reader.readStringOrNull(offset)) as P; case 10: - return (reader.readString(offset)) as P; - case 11: return (reader.readLong(offset)) as P; - case 12: + case 11: return (reader.readStringOrNull(offset)) as P; - case 13: + case 12: return (_AssettypeValueEnumMap[reader.readByteOrNull(offset)] ?? AssetType.other) as P; - case 14: + case 13: return (reader.readDateTime(offset)) as P; - case 15: + case 14: return (reader.readIntOrNull(offset)) as P; default: throw IsarError('Unknown property with id $propertyId'); @@ -291,6 +301,94 @@ void _assetAttach(IsarCollection col, Id id, Asset object) { object.id = id; } +extension AssetByIndex on IsarCollection { + Future getByChecksumOwnerId(String checksum, int ownerId) { + return getByIndex(r'checksum_ownerId', [checksum, ownerId]); + } + + Asset? getByChecksumOwnerIdSync(String checksum, int ownerId) { + return getByIndexSync(r'checksum_ownerId', [checksum, ownerId]); + } + + Future deleteByChecksumOwnerId(String checksum, int ownerId) { + return deleteByIndex(r'checksum_ownerId', [checksum, ownerId]); + } + + bool deleteByChecksumOwnerIdSync(String checksum, int ownerId) { + return deleteByIndexSync(r'checksum_ownerId', [checksum, ownerId]); + } + + Future> getAllByChecksumOwnerId( + List checksumValues, List ownerIdValues) { + final len = checksumValues.length; + assert(ownerIdValues.length == len, + 'All index values must have the same length'); + final values = >[]; + for (var i = 0; i < len; i++) { + values.add([checksumValues[i], ownerIdValues[i]]); + } + + return getAllByIndex(r'checksum_ownerId', values); + } + + List getAllByChecksumOwnerIdSync( + List checksumValues, List ownerIdValues) { + final len = checksumValues.length; + assert(ownerIdValues.length == len, + 'All index values must have the same length'); + final values = >[]; + for (var i = 0; i < len; i++) { + values.add([checksumValues[i], ownerIdValues[i]]); + } + + return getAllByIndexSync(r'checksum_ownerId', values); + } + + Future deleteAllByChecksumOwnerId( + List checksumValues, List ownerIdValues) { + final len = checksumValues.length; + assert(ownerIdValues.length == len, + 'All index values must have the same length'); + final values = >[]; + for (var i = 0; i < len; i++) { + values.add([checksumValues[i], ownerIdValues[i]]); + } + + return deleteAllByIndex(r'checksum_ownerId', values); + } + + int deleteAllByChecksumOwnerIdSync( + List checksumValues, List ownerIdValues) { + final len = checksumValues.length; + assert(ownerIdValues.length == len, + 'All index values must have the same length'); + final values = >[]; + for (var i = 0; i < len; i++) { + values.add([checksumValues[i], ownerIdValues[i]]); + } + + return deleteAllByIndexSync(r'checksum_ownerId', values); + } + + Future putByChecksumOwnerId(Asset object) { + return putByIndex(r'checksum_ownerId', object); + } + + Id putByChecksumOwnerIdSync(Asset object, {bool saveLinks = true}) { + return putByIndexSync(r'checksum_ownerId', object, saveLinks: saveLinks); + } + + Future> putAllByChecksumOwnerId(List objects) { + return putAllByIndex(r'checksum_ownerId', objects); + } + + List putAllByChecksumOwnerIdSync(List objects, + {bool saveLinks = true}) { + return putAllByIndexSync(r'checksum_ownerId', objects, + saveLinks: saveLinks); + } +} + extension AssetQueryWhereSort on QueryBuilder { QueryBuilder anyId() { return QueryBuilder.apply(this, (query) { @@ -365,6 +463,145 @@ extension AssetQueryWhere on QueryBuilder { }); } + QueryBuilder checksumEqualToAnyOwnerId( + String checksum) { + return QueryBuilder.apply(this, (query) { + return query.addWhereClause(IndexWhereClause.equalTo( + indexName: r'checksum_ownerId', + value: [checksum], + )); + }); + } + + QueryBuilder checksumNotEqualToAnyOwnerId( + String checksum) { + return QueryBuilder.apply(this, (query) { + if (query.whereSort == Sort.asc) { + return query + .addWhereClause(IndexWhereClause.between( + indexName: r'checksum_ownerId', + lower: [], + upper: [checksum], + includeUpper: false, + )) + .addWhereClause(IndexWhereClause.between( + indexName: r'checksum_ownerId', + lower: [checksum], + includeLower: false, + upper: [], + )); + } else { + return query + .addWhereClause(IndexWhereClause.between( + indexName: r'checksum_ownerId', + lower: [checksum], + includeLower: false, + upper: [], + )) + .addWhereClause(IndexWhereClause.between( + indexName: r'checksum_ownerId', + lower: [], + upper: [checksum], + includeUpper: false, + )); + } + }); + } + + QueryBuilder checksumOwnerIdEqualTo( + String checksum, int ownerId) { + return QueryBuilder.apply(this, (query) { + return query.addWhereClause(IndexWhereClause.equalTo( + indexName: r'checksum_ownerId', + value: [checksum, ownerId], + )); + }); + } + + QueryBuilder + checksumEqualToOwnerIdNotEqualTo(String checksum, int ownerId) { + return QueryBuilder.apply(this, (query) { + if (query.whereSort == Sort.asc) { + return query + .addWhereClause(IndexWhereClause.between( + indexName: r'checksum_ownerId', + lower: [checksum], + upper: [checksum, ownerId], + includeUpper: false, + )) + .addWhereClause(IndexWhereClause.between( + indexName: r'checksum_ownerId', + lower: [checksum, ownerId], + includeLower: false, + upper: [checksum], + )); + } else { + return query + .addWhereClause(IndexWhereClause.between( + indexName: r'checksum_ownerId', + lower: [checksum, ownerId], + includeLower: false, + upper: [checksum], + )) + .addWhereClause(IndexWhereClause.between( + indexName: r'checksum_ownerId', + lower: [checksum], + upper: [checksum, ownerId], + includeUpper: false, + )); + } + }); + } + + QueryBuilder + checksumEqualToOwnerIdGreaterThan( + String checksum, + int ownerId, { + bool include = false, + }) { + return QueryBuilder.apply(this, (query) { + return query.addWhereClause(IndexWhereClause.between( + indexName: r'checksum_ownerId', + lower: [checksum, ownerId], + includeLower: include, + upper: [checksum], + )); + }); + } + + QueryBuilder checksumEqualToOwnerIdLessThan( + String checksum, + int ownerId, { + bool include = false, + }) { + return QueryBuilder.apply(this, (query) { + return query.addWhereClause(IndexWhereClause.between( + indexName: r'checksum_ownerId', + lower: [checksum], + upper: [checksum, ownerId], + includeUpper: include, + )); + }); + } + + QueryBuilder checksumEqualToOwnerIdBetween( + String checksum, + int lowerOwnerId, + int upperOwnerId, { + bool includeLower = true, + bool includeUpper = true, + }) { + return QueryBuilder.apply(this, (query) { + return query.addWhereClause(IndexWhereClause.between( + indexName: r'checksum_ownerId', + lower: [checksum, lowerOwnerId], + includeLower: includeLower, + upper: [checksum, upperOwnerId], + includeUpper: includeUpper, + )); + }); + } + QueryBuilder remoteIdIsNull() { return QueryBuilder.apply(this, (query) { return query.addWhereClause(IndexWhereClause.equalTo( @@ -430,29 +667,49 @@ extension AssetQueryWhere on QueryBuilder { }); } - QueryBuilder localIdEqualToAnyDeviceId( - String localId) { + QueryBuilder localIdIsNull() { return QueryBuilder.apply(this, (query) { return query.addWhereClause(IndexWhereClause.equalTo( - indexName: r'localId_deviceId', + indexName: r'localId', + value: [null], + )); + }); + } + + QueryBuilder localIdIsNotNull() { + return QueryBuilder.apply(this, (query) { + return query.addWhereClause(IndexWhereClause.between( + indexName: r'localId', + lower: [null], + includeLower: false, + upper: [], + )); + }); + } + + QueryBuilder localIdEqualTo( + String? localId) { + return QueryBuilder.apply(this, (query) { + return query.addWhereClause(IndexWhereClause.equalTo( + indexName: r'localId', value: [localId], )); }); } - QueryBuilder localIdNotEqualToAnyDeviceId( - String localId) { + QueryBuilder localIdNotEqualTo( + String? localId) { return QueryBuilder.apply(this, (query) { if (query.whereSort == Sort.asc) { return query .addWhereClause(IndexWhereClause.between( - indexName: r'localId_deviceId', + indexName: r'localId', lower: [], upper: [localId], includeUpper: false, )) .addWhereClause(IndexWhereClause.between( - indexName: r'localId_deviceId', + indexName: r'localId', lower: [localId], includeLower: false, upper: [], @@ -460,13 +717,13 @@ extension AssetQueryWhere on QueryBuilder { } else { return query .addWhereClause(IndexWhereClause.between( - indexName: r'localId_deviceId', + indexName: r'localId', lower: [localId], includeLower: false, upper: [], )) .addWhereClause(IndexWhereClause.between( - indexName: r'localId_deviceId', + indexName: r'localId', lower: [], upper: [localId], includeUpper: false, @@ -474,151 +731,135 @@ extension AssetQueryWhere on QueryBuilder { } }); } - - QueryBuilder localIdDeviceIdEqualTo( - String localId, int deviceId) { - return QueryBuilder.apply(this, (query) { - return query.addWhereClause(IndexWhereClause.equalTo( - indexName: r'localId_deviceId', - value: [localId, deviceId], - )); - }); - } - - QueryBuilder - localIdEqualToDeviceIdNotEqualTo(String localId, int deviceId) { - return QueryBuilder.apply(this, (query) { - if (query.whereSort == Sort.asc) { - return query - .addWhereClause(IndexWhereClause.between( - indexName: r'localId_deviceId', - lower: [localId], - upper: [localId, deviceId], - includeUpper: false, - )) - .addWhereClause(IndexWhereClause.between( - indexName: r'localId_deviceId', - lower: [localId, deviceId], - includeLower: false, - upper: [localId], - )); - } else { - return query - .addWhereClause(IndexWhereClause.between( - indexName: r'localId_deviceId', - lower: [localId, deviceId], - includeLower: false, - upper: [localId], - )) - .addWhereClause(IndexWhereClause.between( - indexName: r'localId_deviceId', - lower: [localId], - upper: [localId, deviceId], - includeUpper: false, - )); - } - }); - } - - QueryBuilder - localIdEqualToDeviceIdGreaterThan( - String localId, - int deviceId, { - bool include = false, - }) { - return QueryBuilder.apply(this, (query) { - return query.addWhereClause(IndexWhereClause.between( - indexName: r'localId_deviceId', - lower: [localId, deviceId], - includeLower: include, - upper: [localId], - )); - }); - } - - QueryBuilder localIdEqualToDeviceIdLessThan( - String localId, - int deviceId, { - bool include = false, - }) { - return QueryBuilder.apply(this, (query) { - return query.addWhereClause(IndexWhereClause.between( - indexName: r'localId_deviceId', - lower: [localId], - upper: [localId, deviceId], - includeUpper: include, - )); - }); - } - - QueryBuilder localIdEqualToDeviceIdBetween( - String localId, - int lowerDeviceId, - int upperDeviceId, { - bool includeLower = true, - bool includeUpper = true, - }) { - return QueryBuilder.apply(this, (query) { - return query.addWhereClause(IndexWhereClause.between( - indexName: r'localId_deviceId', - lower: [localId, lowerDeviceId], - includeLower: includeLower, - upper: [localId, upperDeviceId], - includeUpper: includeUpper, - )); - }); - } } extension AssetQueryFilter on QueryBuilder { - QueryBuilder deviceIdEqualTo(int value) { + QueryBuilder checksumEqualTo( + String value, { + bool caseSensitive = true, + }) { return QueryBuilder.apply(this, (query) { return query.addFilterCondition(FilterCondition.equalTo( - property: r'deviceId', + property: r'checksum', value: value, + caseSensitive: caseSensitive, )); }); } - QueryBuilder deviceIdGreaterThan( - int value, { + QueryBuilder checksumGreaterThan( + String value, { bool include = false, + bool caseSensitive = true, }) { return QueryBuilder.apply(this, (query) { return query.addFilterCondition(FilterCondition.greaterThan( include: include, - property: r'deviceId', + property: r'checksum', value: value, + caseSensitive: caseSensitive, )); }); } - QueryBuilder deviceIdLessThan( - int value, { + QueryBuilder checksumLessThan( + String value, { bool include = false, + bool caseSensitive = true, }) { return QueryBuilder.apply(this, (query) { return query.addFilterCondition(FilterCondition.lessThan( include: include, - property: r'deviceId', + property: r'checksum', value: value, + caseSensitive: caseSensitive, )); }); } - QueryBuilder deviceIdBetween( - int lower, - int upper, { + QueryBuilder checksumBetween( + String lower, + String upper, { bool includeLower = true, bool includeUpper = true, + bool caseSensitive = true, }) { return QueryBuilder.apply(this, (query) { return query.addFilterCondition(FilterCondition.between( - property: r'deviceId', + property: r'checksum', lower: lower, includeLower: includeLower, upper: upper, includeUpper: includeUpper, + caseSensitive: caseSensitive, + )); + }); + } + + QueryBuilder checksumStartsWith( + String value, { + bool caseSensitive = true, + }) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.startsWith( + property: r'checksum', + value: value, + caseSensitive: caseSensitive, + )); + }); + } + + QueryBuilder checksumEndsWith( + String value, { + bool caseSensitive = true, + }) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.endsWith( + property: r'checksum', + value: value, + caseSensitive: caseSensitive, + )); + }); + } + + QueryBuilder checksumContains( + String value, + {bool caseSensitive = true}) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.contains( + property: r'checksum', + value: value, + caseSensitive: caseSensitive, + )); + }); + } + + QueryBuilder checksumMatches( + String pattern, + {bool caseSensitive = true}) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.matches( + property: r'checksum', + wildcard: pattern, + caseSensitive: caseSensitive, + )); + }); + } + + QueryBuilder checksumIsEmpty() { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.equalTo( + property: r'checksum', + value: '', + )); + }); + } + + QueryBuilder checksumIsNotEmpty() { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.greaterThan( + property: r'checksum', + value: '', )); }); } @@ -1053,15 +1294,6 @@ extension AssetQueryFilter on QueryBuilder { }); } - QueryBuilder isLocalEqualTo(bool value) { - return QueryBuilder.apply(this, (query) { - return query.addFilterCondition(FilterCondition.equalTo( - property: r'isLocal', - value: value, - )); - }); - } - QueryBuilder livePhotoVideoIdIsNull() { return QueryBuilder.apply(this, (query) { return query.addFilterCondition(const FilterCondition.isNull( @@ -1210,8 +1442,24 @@ extension AssetQueryFilter on QueryBuilder { }); } + QueryBuilder localIdIsNull() { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(const FilterCondition.isNull( + property: r'localId', + )); + }); + } + + QueryBuilder localIdIsNotNull() { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(const FilterCondition.isNotNull( + property: r'localId', + )); + }); + } + QueryBuilder localIdEqualTo( - String value, { + String? value, { bool caseSensitive = true, }) { return QueryBuilder.apply(this, (query) { @@ -1224,7 +1472,7 @@ extension AssetQueryFilter on QueryBuilder { } QueryBuilder localIdGreaterThan( - String value, { + String? value, { bool include = false, bool caseSensitive = true, }) { @@ -1239,7 +1487,7 @@ extension AssetQueryFilter on QueryBuilder { } QueryBuilder localIdLessThan( - String value, { + String? value, { bool include = false, bool caseSensitive = true, }) { @@ -1254,8 +1502,8 @@ extension AssetQueryFilter on QueryBuilder { } QueryBuilder localIdBetween( - String lower, - String upper, { + String? lower, + String? upper, { bool includeLower = true, bool includeUpper = true, bool caseSensitive = true, @@ -1718,15 +1966,15 @@ extension AssetQueryObject on QueryBuilder {} extension AssetQueryLinks on QueryBuilder {} extension AssetQuerySortBy on QueryBuilder { - QueryBuilder sortByDeviceId() { + QueryBuilder sortByChecksum() { return QueryBuilder.apply(this, (query) { - return query.addSortBy(r'deviceId', Sort.asc); + return query.addSortBy(r'checksum', Sort.asc); }); } - QueryBuilder sortByDeviceIdDesc() { + QueryBuilder sortByChecksumDesc() { return QueryBuilder.apply(this, (query) { - return query.addSortBy(r'deviceId', Sort.desc); + return query.addSortBy(r'checksum', Sort.desc); }); } @@ -1814,18 +2062,6 @@ extension AssetQuerySortBy on QueryBuilder { }); } - QueryBuilder sortByIsLocal() { - return QueryBuilder.apply(this, (query) { - return query.addSortBy(r'isLocal', Sort.asc); - }); - } - - QueryBuilder sortByIsLocalDesc() { - return QueryBuilder.apply(this, (query) { - return query.addSortBy(r'isLocal', Sort.desc); - }); - } - QueryBuilder sortByLivePhotoVideoId() { return QueryBuilder.apply(this, (query) { return query.addSortBy(r'livePhotoVideoId', Sort.asc); @@ -1912,15 +2148,15 @@ extension AssetQuerySortBy on QueryBuilder { } extension AssetQuerySortThenBy on QueryBuilder { - QueryBuilder thenByDeviceId() { + QueryBuilder thenByChecksum() { return QueryBuilder.apply(this, (query) { - return query.addSortBy(r'deviceId', Sort.asc); + return query.addSortBy(r'checksum', Sort.asc); }); } - QueryBuilder thenByDeviceIdDesc() { + QueryBuilder thenByChecksumDesc() { return QueryBuilder.apply(this, (query) { - return query.addSortBy(r'deviceId', Sort.desc); + return query.addSortBy(r'checksum', Sort.desc); }); } @@ -2020,18 +2256,6 @@ extension AssetQuerySortThenBy on QueryBuilder { }); } - QueryBuilder thenByIsLocal() { - return QueryBuilder.apply(this, (query) { - return query.addSortBy(r'isLocal', Sort.asc); - }); - } - - QueryBuilder thenByIsLocalDesc() { - return QueryBuilder.apply(this, (query) { - return query.addSortBy(r'isLocal', Sort.desc); - }); - } - QueryBuilder thenByLivePhotoVideoId() { return QueryBuilder.apply(this, (query) { return query.addSortBy(r'livePhotoVideoId', Sort.asc); @@ -2118,9 +2342,10 @@ extension AssetQuerySortThenBy on QueryBuilder { } extension AssetQueryWhereDistinct on QueryBuilder { - QueryBuilder distinctByDeviceId() { + QueryBuilder distinctByChecksum( + {bool caseSensitive = true}) { return QueryBuilder.apply(this, (query) { - return query.addDistinctBy(r'deviceId'); + return query.addDistinctBy(r'checksum', caseSensitive: caseSensitive); }); } @@ -2167,12 +2392,6 @@ extension AssetQueryWhereDistinct on QueryBuilder { }); } - QueryBuilder distinctByIsLocal() { - return QueryBuilder.apply(this, (query) { - return query.addDistinctBy(r'isLocal'); - }); - } - QueryBuilder distinctByLivePhotoVideoId( {bool caseSensitive = true}) { return QueryBuilder.apply(this, (query) { @@ -2227,9 +2446,9 @@ extension AssetQueryProperty on QueryBuilder { }); } - QueryBuilder deviceIdProperty() { + QueryBuilder checksumProperty() { return QueryBuilder.apply(this, (query) { - return query.addPropertyName(r'deviceId'); + return query.addPropertyName(r'checksum'); }); } @@ -2275,19 +2494,13 @@ extension AssetQueryProperty on QueryBuilder { }); } - QueryBuilder isLocalProperty() { - return QueryBuilder.apply(this, (query) { - return query.addPropertyName(r'isLocal'); - }); - } - QueryBuilder livePhotoVideoIdProperty() { return QueryBuilder.apply(this, (query) { return query.addPropertyName(r'livePhotoVideoId'); }); } - QueryBuilder localIdProperty() { + QueryBuilder localIdProperty() { return QueryBuilder.apply(this, (query) { return query.addPropertyName(r'localId'); }); diff --git a/mobile/lib/shared/models/device_asset.dart b/mobile/lib/shared/models/device_asset.dart new file mode 100644 index 0000000000..0973dd4ff8 --- /dev/null +++ b/mobile/lib/shared/models/device_asset.dart @@ -0,0 +1,8 @@ +import 'package:isar/isar.dart'; + +class DeviceAsset { + DeviceAsset({required this.hash}); + + @Index(unique: false, type: IndexType.hash) + List hash; +} diff --git a/mobile/lib/shared/models/ios_device_asset.dart b/mobile/lib/shared/models/ios_device_asset.dart new file mode 100644 index 0000000000..0c55c74eb9 --- /dev/null +++ b/mobile/lib/shared/models/ios_device_asset.dart @@ -0,0 +1,14 @@ +import 'package:immich_mobile/shared/models/device_asset.dart'; +import 'package:immich_mobile/utils/hash.dart'; +import 'package:isar/isar.dart'; + +part 'ios_device_asset.g.dart'; + +@Collection() +class IOSDeviceAsset extends DeviceAsset { + IOSDeviceAsset({required this.id, required super.hash}); + + @Index(replace: true, unique: true, type: IndexType.hash) + String id; + Id get isarId => fastHash(id); +} diff --git a/mobile/lib/shared/models/ios_device_asset.g.dart b/mobile/lib/shared/models/ios_device_asset.g.dart new file mode 100644 index 0000000000..f10c3decda --- /dev/null +++ b/mobile/lib/shared/models/ios_device_asset.g.dart @@ -0,0 +1,780 @@ +// GENERATED CODE - DO NOT MODIFY BY HAND + +part of 'ios_device_asset.dart'; + +// ************************************************************************** +// IsarCollectionGenerator +// ************************************************************************** + +// coverage:ignore-file +// ignore_for_file: duplicate_ignore, non_constant_identifier_names, constant_identifier_names, invalid_use_of_protected_member, unnecessary_cast, prefer_const_constructors, lines_longer_than_80_chars, require_trailing_commas, inference_failure_on_function_invocation, unnecessary_parenthesis, unnecessary_raw_strings, unnecessary_null_checks, join_return_with_assignment, prefer_final_locals, avoid_js_rounded_ints, avoid_positional_boolean_parameters, always_specify_types + +extension GetIOSDeviceAssetCollection on Isar { + IsarCollection get iOSDeviceAssets => this.collection(); +} + +const IOSDeviceAssetSchema = CollectionSchema( + name: r'IOSDeviceAsset', + id: -1671546753821948030, + properties: { + r'hash': PropertySchema( + id: 0, + name: r'hash', + type: IsarType.byteList, + ), + r'id': PropertySchema( + id: 1, + name: r'id', + type: IsarType.string, + ) + }, + estimateSize: _iOSDeviceAssetEstimateSize, + serialize: _iOSDeviceAssetSerialize, + deserialize: _iOSDeviceAssetDeserialize, + deserializeProp: _iOSDeviceAssetDeserializeProp, + idName: r'isarId', + indexes: { + r'id': IndexSchema( + id: -3268401673993471357, + name: r'id', + unique: true, + replace: true, + properties: [ + IndexPropertySchema( + name: r'id', + type: IndexType.hash, + caseSensitive: true, + ) + ], + ), + r'hash': IndexSchema( + id: -7973251393006690288, + name: r'hash', + unique: false, + replace: false, + properties: [ + IndexPropertySchema( + name: r'hash', + type: IndexType.hash, + caseSensitive: false, + ) + ], + ) + }, + links: {}, + embeddedSchemas: {}, + getId: _iOSDeviceAssetGetId, + getLinks: _iOSDeviceAssetGetLinks, + attach: _iOSDeviceAssetAttach, + version: '3.1.0+1', +); + +int _iOSDeviceAssetEstimateSize( + IOSDeviceAsset object, + List offsets, + Map> allOffsets, +) { + var bytesCount = offsets.last; + bytesCount += 3 + object.hash.length; + bytesCount += 3 + object.id.length * 3; + return bytesCount; +} + +void _iOSDeviceAssetSerialize( + IOSDeviceAsset object, + IsarWriter writer, + List offsets, + Map> allOffsets, +) { + writer.writeByteList(offsets[0], object.hash); + writer.writeString(offsets[1], object.id); +} + +IOSDeviceAsset _iOSDeviceAssetDeserialize( + Id id, + IsarReader reader, + List offsets, + Map> allOffsets, +) { + final object = IOSDeviceAsset( + hash: reader.readByteList(offsets[0]) ?? [], + id: reader.readString(offsets[1]), + ); + return object; +} + +P _iOSDeviceAssetDeserializeProp

( + IsarReader reader, + int propertyId, + int offset, + Map> allOffsets, +) { + switch (propertyId) { + case 0: + return (reader.readByteList(offset) ?? []) as P; + case 1: + return (reader.readString(offset)) as P; + default: + throw IsarError('Unknown property with id $propertyId'); + } +} + +Id _iOSDeviceAssetGetId(IOSDeviceAsset object) { + return object.isarId; +} + +List> _iOSDeviceAssetGetLinks(IOSDeviceAsset object) { + return []; +} + +void _iOSDeviceAssetAttach( + IsarCollection col, Id id, IOSDeviceAsset object) {} + +extension IOSDeviceAssetByIndex on IsarCollection { + Future getById(String id) { + return getByIndex(r'id', [id]); + } + + IOSDeviceAsset? getByIdSync(String id) { + return getByIndexSync(r'id', [id]); + } + + Future deleteById(String id) { + return deleteByIndex(r'id', [id]); + } + + bool deleteByIdSync(String id) { + return deleteByIndexSync(r'id', [id]); + } + + Future> getAllById(List idValues) { + final values = idValues.map((e) => [e]).toList(); + return getAllByIndex(r'id', values); + } + + List getAllByIdSync(List idValues) { + final values = idValues.map((e) => [e]).toList(); + return getAllByIndexSync(r'id', values); + } + + Future deleteAllById(List idValues) { + final values = idValues.map((e) => [e]).toList(); + return deleteAllByIndex(r'id', values); + } + + int deleteAllByIdSync(List idValues) { + final values = idValues.map((e) => [e]).toList(); + return deleteAllByIndexSync(r'id', values); + } + + Future putById(IOSDeviceAsset object) { + return putByIndex(r'id', object); + } + + Id putByIdSync(IOSDeviceAsset object, {bool saveLinks = true}) { + return putByIndexSync(r'id', object, saveLinks: saveLinks); + } + + Future> putAllById(List objects) { + return putAllByIndex(r'id', objects); + } + + List putAllByIdSync(List objects, + {bool saveLinks = true}) { + return putAllByIndexSync(r'id', objects, saveLinks: saveLinks); + } +} + +extension IOSDeviceAssetQueryWhereSort + on QueryBuilder { + QueryBuilder anyIsarId() { + return QueryBuilder.apply(this, (query) { + return query.addWhereClause(const IdWhereClause.any()); + }); + } +} + +extension IOSDeviceAssetQueryWhere + on QueryBuilder { + QueryBuilder isarIdEqualTo( + Id isarId) { + return QueryBuilder.apply(this, (query) { + return query.addWhereClause(IdWhereClause.between( + lower: isarId, + upper: isarId, + )); + }); + } + + QueryBuilder + isarIdNotEqualTo(Id isarId) { + return QueryBuilder.apply(this, (query) { + if (query.whereSort == Sort.asc) { + return query + .addWhereClause( + IdWhereClause.lessThan(upper: isarId, includeUpper: false), + ) + .addWhereClause( + IdWhereClause.greaterThan(lower: isarId, includeLower: false), + ); + } else { + return query + .addWhereClause( + IdWhereClause.greaterThan(lower: isarId, includeLower: false), + ) + .addWhereClause( + IdWhereClause.lessThan(upper: isarId, includeUpper: false), + ); + } + }); + } + + QueryBuilder + isarIdGreaterThan(Id isarId, {bool include = false}) { + return QueryBuilder.apply(this, (query) { + return query.addWhereClause( + IdWhereClause.greaterThan(lower: isarId, includeLower: include), + ); + }); + } + + QueryBuilder + isarIdLessThan(Id isarId, {bool include = false}) { + return QueryBuilder.apply(this, (query) { + return query.addWhereClause( + IdWhereClause.lessThan(upper: isarId, includeUpper: include), + ); + }); + } + + QueryBuilder isarIdBetween( + Id lowerIsarId, + Id upperIsarId, { + bool includeLower = true, + bool includeUpper = true, + }) { + return QueryBuilder.apply(this, (query) { + return query.addWhereClause(IdWhereClause.between( + lower: lowerIsarId, + includeLower: includeLower, + upper: upperIsarId, + includeUpper: includeUpper, + )); + }); + } + + QueryBuilder idEqualTo( + String id) { + return QueryBuilder.apply(this, (query) { + return query.addWhereClause(IndexWhereClause.equalTo( + indexName: r'id', + value: [id], + )); + }); + } + + QueryBuilder idNotEqualTo( + String id) { + return QueryBuilder.apply(this, (query) { + if (query.whereSort == Sort.asc) { + return query + .addWhereClause(IndexWhereClause.between( + indexName: r'id', + lower: [], + upper: [id], + includeUpper: false, + )) + .addWhereClause(IndexWhereClause.between( + indexName: r'id', + lower: [id], + includeLower: false, + upper: [], + )); + } else { + return query + .addWhereClause(IndexWhereClause.between( + indexName: r'id', + lower: [id], + includeLower: false, + upper: [], + )) + .addWhereClause(IndexWhereClause.between( + indexName: r'id', + lower: [], + upper: [id], + includeUpper: false, + )); + } + }); + } + + QueryBuilder hashEqualTo( + List hash) { + return QueryBuilder.apply(this, (query) { + return query.addWhereClause(IndexWhereClause.equalTo( + indexName: r'hash', + value: [hash], + )); + }); + } + + QueryBuilder + hashNotEqualTo(List hash) { + return QueryBuilder.apply(this, (query) { + if (query.whereSort == Sort.asc) { + return query + .addWhereClause(IndexWhereClause.between( + indexName: r'hash', + lower: [], + upper: [hash], + includeUpper: false, + )) + .addWhereClause(IndexWhereClause.between( + indexName: r'hash', + lower: [hash], + includeLower: false, + upper: [], + )); + } else { + return query + .addWhereClause(IndexWhereClause.between( + indexName: r'hash', + lower: [hash], + includeLower: false, + upper: [], + )) + .addWhereClause(IndexWhereClause.between( + indexName: r'hash', + lower: [], + upper: [hash], + includeUpper: false, + )); + } + }); + } +} + +extension IOSDeviceAssetQueryFilter + on QueryBuilder { + QueryBuilder + hashElementEqualTo(int value) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.equalTo( + property: r'hash', + value: value, + )); + }); + } + + QueryBuilder + hashElementGreaterThan( + int value, { + bool include = false, + }) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.greaterThan( + include: include, + property: r'hash', + value: value, + )); + }); + } + + QueryBuilder + hashElementLessThan( + int value, { + bool include = false, + }) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.lessThan( + include: include, + property: r'hash', + value: value, + )); + }); + } + + QueryBuilder + hashElementBetween( + int lower, + int upper, { + bool includeLower = true, + bool includeUpper = true, + }) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.between( + property: r'hash', + lower: lower, + includeLower: includeLower, + upper: upper, + includeUpper: includeUpper, + )); + }); + } + + QueryBuilder + hashLengthEqualTo(int length) { + return QueryBuilder.apply(this, (query) { + return query.listLength( + r'hash', + length, + true, + length, + true, + ); + }); + } + + QueryBuilder + hashIsEmpty() { + return QueryBuilder.apply(this, (query) { + return query.listLength( + r'hash', + 0, + true, + 0, + true, + ); + }); + } + + QueryBuilder + hashIsNotEmpty() { + return QueryBuilder.apply(this, (query) { + return query.listLength( + r'hash', + 0, + false, + 999999, + true, + ); + }); + } + + QueryBuilder + hashLengthLessThan( + int length, { + bool include = false, + }) { + return QueryBuilder.apply(this, (query) { + return query.listLength( + r'hash', + 0, + true, + length, + include, + ); + }); + } + + QueryBuilder + hashLengthGreaterThan( + int length, { + bool include = false, + }) { + return QueryBuilder.apply(this, (query) { + return query.listLength( + r'hash', + length, + include, + 999999, + true, + ); + }); + } + + QueryBuilder + hashLengthBetween( + int lower, + int upper, { + bool includeLower = true, + bool includeUpper = true, + }) { + return QueryBuilder.apply(this, (query) { + return query.listLength( + r'hash', + lower, + includeLower, + upper, + includeUpper, + ); + }); + } + + QueryBuilder idEqualTo( + String value, { + bool caseSensitive = true, + }) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.equalTo( + property: r'id', + value: value, + caseSensitive: caseSensitive, + )); + }); + } + + QueryBuilder + idGreaterThan( + String value, { + bool include = false, + bool caseSensitive = true, + }) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.greaterThan( + include: include, + property: r'id', + value: value, + caseSensitive: caseSensitive, + )); + }); + } + + QueryBuilder + idLessThan( + String value, { + bool include = false, + bool caseSensitive = true, + }) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.lessThan( + include: include, + property: r'id', + value: value, + caseSensitive: caseSensitive, + )); + }); + } + + QueryBuilder idBetween( + String lower, + String upper, { + bool includeLower = true, + bool includeUpper = true, + bool caseSensitive = true, + }) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.between( + property: r'id', + lower: lower, + includeLower: includeLower, + upper: upper, + includeUpper: includeUpper, + caseSensitive: caseSensitive, + )); + }); + } + + QueryBuilder + idStartsWith( + String value, { + bool caseSensitive = true, + }) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.startsWith( + property: r'id', + value: value, + caseSensitive: caseSensitive, + )); + }); + } + + QueryBuilder + idEndsWith( + String value, { + bool caseSensitive = true, + }) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.endsWith( + property: r'id', + value: value, + caseSensitive: caseSensitive, + )); + }); + } + + QueryBuilder + idContains(String value, {bool caseSensitive = true}) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.contains( + property: r'id', + value: value, + caseSensitive: caseSensitive, + )); + }); + } + + QueryBuilder idMatches( + String pattern, + {bool caseSensitive = true}) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.matches( + property: r'id', + wildcard: pattern, + caseSensitive: caseSensitive, + )); + }); + } + + QueryBuilder + idIsEmpty() { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.equalTo( + property: r'id', + value: '', + )); + }); + } + + QueryBuilder + idIsNotEmpty() { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.greaterThan( + property: r'id', + value: '', + )); + }); + } + + QueryBuilder + isarIdEqualTo(Id value) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.equalTo( + property: r'isarId', + value: value, + )); + }); + } + + QueryBuilder + isarIdGreaterThan( + Id value, { + bool include = false, + }) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.greaterThan( + include: include, + property: r'isarId', + value: value, + )); + }); + } + + QueryBuilder + isarIdLessThan( + Id value, { + bool include = false, + }) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.lessThan( + include: include, + property: r'isarId', + value: value, + )); + }); + } + + QueryBuilder + isarIdBetween( + Id lower, + Id upper, { + bool includeLower = true, + bool includeUpper = true, + }) { + return QueryBuilder.apply(this, (query) { + return query.addFilterCondition(FilterCondition.between( + property: r'isarId', + lower: lower, + includeLower: includeLower, + upper: upper, + includeUpper: includeUpper, + )); + }); + } +} + +extension IOSDeviceAssetQueryObject + on QueryBuilder {} + +extension IOSDeviceAssetQueryLinks + on QueryBuilder {} + +extension IOSDeviceAssetQuerySortBy + on QueryBuilder { + QueryBuilder sortById() { + return QueryBuilder.apply(this, (query) { + return query.addSortBy(r'id', Sort.asc); + }); + } + + QueryBuilder sortByIdDesc() { + return QueryBuilder.apply(this, (query) { + return query.addSortBy(r'id', Sort.desc); + }); + } +} + +extension IOSDeviceAssetQuerySortThenBy + on QueryBuilder { + QueryBuilder thenById() { + return QueryBuilder.apply(this, (query) { + return query.addSortBy(r'id', Sort.asc); + }); + } + + QueryBuilder thenByIdDesc() { + return QueryBuilder.apply(this, (query) { + return query.addSortBy(r'id', Sort.desc); + }); + } + + QueryBuilder thenByIsarId() { + return QueryBuilder.apply(this, (query) { + return query.addSortBy(r'isarId', Sort.asc); + }); + } + + QueryBuilder + thenByIsarIdDesc() { + return QueryBuilder.apply(this, (query) { + return query.addSortBy(r'isarId', Sort.desc); + }); + } +} + +extension IOSDeviceAssetQueryWhereDistinct + on QueryBuilder { + QueryBuilder distinctByHash() { + return QueryBuilder.apply(this, (query) { + return query.addDistinctBy(r'hash'); + }); + } + + QueryBuilder distinctById( + {bool caseSensitive = true}) { + return QueryBuilder.apply(this, (query) { + return query.addDistinctBy(r'id', caseSensitive: caseSensitive); + }); + } +} + +extension IOSDeviceAssetQueryProperty + on QueryBuilder { + QueryBuilder isarIdProperty() { + return QueryBuilder.apply(this, (query) { + return query.addPropertyName(r'isarId'); + }); + } + + QueryBuilder, QQueryOperations> hashProperty() { + return QueryBuilder.apply(this, (query) { + return query.addPropertyName(r'hash'); + }); + } + + QueryBuilder idProperty() { + return QueryBuilder.apply(this, (query) { + return query.addPropertyName(r'id'); + }); + } +} diff --git a/mobile/lib/shared/providers/asset.provider.dart b/mobile/lib/shared/providers/asset.provider.dart index c1384f4029..54e9694268 100644 --- a/mobile/lib/shared/providers/asset.provider.dart +++ b/mobile/lib/shared/providers/asset.provider.dart @@ -18,11 +18,7 @@ import 'package:logging/logging.dart'; import 'package:openapi/api.dart'; import 'package:photo_manager/photo_manager.dart'; -/// State does not contain archived assets. -/// Use database provider if you want to access the isArchived assets -class AssetsState {} - -class AssetNotifier extends StateNotifier { +class AssetNotifier extends StateNotifier { final AssetService _assetService; final AlbumService _albumService; final UserService _userService; @@ -38,7 +34,7 @@ class AssetNotifier extends StateNotifier { this._userService, this._syncService, this._db, - ) : super(AssetsState()); + ) : super(false); Future getAllAsset({bool clear = false}) async { if (_getAllAssetInProgress || _deleteInProgress) { @@ -48,14 +44,15 @@ class AssetNotifier extends StateNotifier { final stopwatch = Stopwatch()..start(); try { _getAllAssetInProgress = true; + state = true; if (clear) { await clearAssetsAndAlbums(_db); log.info("Manual refresh requested, cleared assets and albums from db"); } + await _userService.refreshUsers(); final bool newRemote = await _assetService.refreshRemoteAssets(); final bool newLocal = await _albumService.refreshDeviceAlbums(); debugPrint("newRemote: $newRemote, newLocal: $newLocal"); - await _userService.refreshUsers(); final List partners = await _db.users.filter().isPartnerSharedWithEqualTo(true).findAll(); for (User u in partners) { @@ -64,6 +61,7 @@ class AssetNotifier extends StateNotifier { log.info("Load assets: ${stopwatch.elapsedMilliseconds}ms"); } finally { _getAllAssetInProgress = false; + state = false; } } @@ -79,6 +77,7 @@ class AssetNotifier extends StateNotifier { Future deleteAssets(Set deleteAssets) async { _deleteInProgress = true; + state = true; try { final localDeleted = await _deleteLocalAssets(deleteAssets); final remoteDeleted = await _deleteRemoteAssets(deleteAssets); @@ -91,24 +90,14 @@ class AssetNotifier extends StateNotifier { } } finally { _deleteInProgress = false; + state = false; } } Future> _deleteLocalAssets(Set assetsToDelete) async { - final int deviceId = Store.get(StoreKey.deviceIdHash); - final List local = []; + final List local = + assetsToDelete.where((a) => a.isLocal).map((a) => a.localId!).toList(); // Delete asset from device - for (final Asset asset in assetsToDelete) { - if (asset.isLocal) { - local.add(asset.localId); - } else if (asset.deviceId == deviceId) { - // Delete asset on device if it is still present - var localAsset = await AssetEntity.fromId(asset.localId); - if (localAsset != null) { - local.add(localAsset.id); - } - } - } if (local.isNotEmpty) { try { return await PhotoManager.editor.deleteWithIds(local); @@ -153,7 +142,7 @@ class AssetNotifier extends StateNotifier { } } -final assetProvider = StateNotifierProvider((ref) { +final assetProvider = StateNotifierProvider((ref) { return AssetNotifier( ref.watch(assetServiceProvider), ref.watch(albumServiceProvider), diff --git a/mobile/lib/shared/services/hash.service.dart b/mobile/lib/shared/services/hash.service.dart new file mode 100644 index 0000000000..ee272cf5ff --- /dev/null +++ b/mobile/lib/shared/services/hash.service.dart @@ -0,0 +1,175 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:crypto/crypto.dart'; +import 'package:flutter/foundation.dart'; +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/modules/backup/background_service/background.service.dart'; +import 'package:immich_mobile/shared/models/android_device_asset.dart'; +import 'package:immich_mobile/shared/models/asset.dart'; +import 'package:immich_mobile/shared/models/device_asset.dart'; +import 'package:immich_mobile/shared/models/ios_device_asset.dart'; +import 'package:immich_mobile/shared/providers/db.provider.dart'; +import 'package:immich_mobile/utils/builtin_extensions.dart'; +import 'package:isar/isar.dart'; +import 'package:logging/logging.dart'; +import 'package:photo_manager/photo_manager.dart'; + +class HashService { + HashService(this._db, this._backgroundService); + final Isar _db; + final BackgroundService _backgroundService; + final _log = Logger('HashService'); + + /// Returns all assets that were successfully hashed + Future> getHashedAssets( + AssetPathEntity album, { + int start = 0, + int end = 0x7fffffffffffffff, + Set? excludedAssets, + }) async { + final entities = await album.getAssetListRange(start: start, end: end); + final filtered = excludedAssets == null + ? entities + : entities.where((e) => !excludedAssets.contains(e.id)).toList(); + return _hashAssets(filtered); + } + + /// Converts a list of [AssetEntity]s to [Asset]s including only those + /// that were successfully hashed. Hashes are looked up in a DB table + /// [AndroidDeviceAsset] / [IOSDeviceAsset] by local id. Only missing + /// entries are newly hashed and added to the DB table. + Future> _hashAssets(List assetEntities) async { + const int batchFileCount = 128; + const int batchDataSize = 1024 * 1024 * 1024; // 1GB + + final ids = assetEntities + .map(Platform.isAndroid ? (a) => a.id.toInt() : (a) => a.id) + .toList(); + final List hashes = await _lookupHashes(ids); + final List toAdd = []; + final List toHash = []; + + int bytes = 0; + + for (int i = 0; i < assetEntities.length; i++) { + if (hashes[i] != null) { + continue; + } + final file = await assetEntities[i].originFile; + if (file == null) { + _log.warning( + "Failed to get file for asset ${assetEntities[i].id}, skipping", + ); + continue; + } + bytes += await file.length(); + toHash.add(file.path); + final deviceAsset = Platform.isAndroid + ? AndroidDeviceAsset(id: ids[i] as int, hash: const []) + : IOSDeviceAsset(id: ids[i] as String, hash: const []); + toAdd.add(deviceAsset); + hashes[i] = deviceAsset; + if (toHash.length == batchFileCount || bytes >= batchDataSize) { + await _processBatch(toHash, toAdd); + toAdd.clear(); + toHash.clear(); + bytes = 0; + } + } + if (toHash.isNotEmpty) { + await _processBatch(toHash, toAdd); + } + return _mapAllHashedAssets(assetEntities, hashes); + } + + /// Lookup hashes of assets by their local ID + Future> _lookupHashes(List ids) => + Platform.isAndroid + ? _db.androidDeviceAssets.getAll(ids.cast()) + : _db.iOSDeviceAssets.getAllById(ids.cast()); + + /// Processes a batch of files and saves any successfully hashed + /// values to the DB table. + Future _processBatch( + final List toHash, + final List toAdd, + ) async { + final hashes = await _hashFiles(toHash); + bool anyNull = false; + for (int j = 0; j < hashes.length; j++) { + if (hashes[j]?.length == 20) { + toAdd[j].hash = hashes[j]!; + } else { + _log.warning("Failed to hash file ${toHash[j]}, skipping"); + anyNull = true; + } + } + final validHashes = anyNull + ? toAdd.where((e) => e.hash.length == 20).toList(growable: false) + : toAdd; + await _db.writeTxn( + () => Platform.isAndroid + ? _db.androidDeviceAssets.putAll(validHashes.cast()) + : _db.iOSDeviceAssets.putAll(validHashes.cast()), + ); + _log.fine("Hashed ${validHashes.length}/${toHash.length} assets"); + } + + /// Hashes the given files and returns a list of the same length + /// files that could not be hashed have a `null` value + Future> _hashFiles(List paths) async { + if (Platform.isAndroid) { + final List? hashes = + await _backgroundService.digestFiles(paths); + if (hashes == null) { + throw Exception("Hashing ${paths.length} files failed"); + } + return hashes; + } else if (Platform.isIOS) { + final List result = List.filled(paths.length, null); + for (int i = 0; i < paths.length; i++) { + result[i] = await _hashAssetDart(File(paths[i])); + } + return result; + } else { + throw Exception("_hashFiles implementation missing"); + } + } + + /// Hashes a single file using Dart's crypto package + Future _hashAssetDart(File f) async { + late Digest output; + final sink = sha1.startChunkedConversion( + ChunkedConversionSink.withCallback((accumulated) { + output = accumulated.first; + }), + ); + await for (final chunk in f.openRead()) { + sink.add(chunk); + } + sink.close(); + return Uint8List.fromList(output.bytes); + } + + /// Converts [AssetEntity]s that were successfully hashed to [Asset]s + List _mapAllHashedAssets( + List assets, + List hashes, + ) { + final List result = []; + for (int i = 0; i < assets.length; i++) { + if (hashes[i] != null && hashes[i]!.hash.isNotEmpty) { + result.add(Asset.local(assets[i], hashes[i]!.hash)); + } + } + return result; + } +} + +final hashServiceProvider = Provider( + (ref) => HashService( + ref.watch(dbProvider), + ref.watch(backgroundServiceProvider), + ), +); diff --git a/mobile/lib/shared/services/sync.service.dart b/mobile/lib/shared/services/sync.service.dart index f5471d1546..0bb2378361 100644 --- a/mobile/lib/shared/services/sync.service.dart +++ b/mobile/lib/shared/services/sync.service.dart @@ -4,10 +4,12 @@ import 'package:collection/collection.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/shared/models/album.dart'; import 'package:immich_mobile/shared/models/asset.dart'; +import 'package:immich_mobile/shared/models/etag.dart'; import 'package:immich_mobile/shared/models/exif_info.dart'; import 'package:immich_mobile/shared/models/store.dart'; import 'package:immich_mobile/shared/models/user.dart'; import 'package:immich_mobile/shared/providers/db.provider.dart'; +import 'package:immich_mobile/shared/services/hash.service.dart'; import 'package:immich_mobile/utils/async_mutex.dart'; import 'package:immich_mobile/utils/builtin_extensions.dart'; import 'package:immich_mobile/utils/diff.dart'; @@ -16,15 +18,17 @@ import 'package:logging/logging.dart'; import 'package:openapi/api.dart'; import 'package:photo_manager/photo_manager.dart'; -final syncServiceProvider = - Provider((ref) => SyncService(ref.watch(dbProvider))); +final syncServiceProvider = Provider( + (ref) => SyncService(ref.watch(dbProvider), ref.watch(hashServiceProvider)), +); class SyncService { final Isar _db; + final HashService _hashService; final AsyncMutex _lock = AsyncMutex(); final Logger _log = Logger('SyncService'); - SyncService(this._db); + SyncService(this._db, this._hashService); // public methods: @@ -33,6 +37,7 @@ class SyncService { Future syncUsersFromServer(List users) async { users.sortBy((u) => u.id); final dbUsers = await _db.users.where().sortById().findAll(); + assert(dbUsers.isSortedBy((u) => u.id), "dbUsers not sorted!"); final List toDelete = []; final List toUpsert = []; final changes = diffSortedListsSync( @@ -108,40 +113,16 @@ class SyncService { // private methods: /// Syncs a new asset to the db. Returns `true` if successful - Future _syncNewAssetToDb(Asset newAsset) async { - final List inDb = await _db.assets - .where() - .localIdDeviceIdEqualTo(newAsset.localId, newAsset.deviceId) - .findAll(); - Asset? match; - if (inDb.length == 1) { - // exactly one match: trivial case - match = inDb.first; - } else if (inDb.length > 1) { - // TODO instead of this heuristics: match by checksum once available - for (Asset a in inDb) { - if (a.ownerId == newAsset.ownerId && - a.fileModifiedAt.isAtSameMomentAs(newAsset.fileModifiedAt)) { - assert(match == null); - match = a; - } - } - if (match == null) { - for (Asset a in inDb) { - if (a.ownerId == newAsset.ownerId) { - assert(match == null); - match = a; - } - } - } - } - if (match != null) { + Future _syncNewAssetToDb(Asset a) async { + final Asset? inDb = + await _db.assets.getByChecksumOwnerId(a.checksum, a.ownerId); + if (inDb != null) { // unify local/remote assets by replacing the // local-only asset in the DB with a local&remote asset - newAsset = match.updatedCopy(newAsset); + a = inDb.updatedCopy(a); } try { - await _db.writeTxn(() => newAsset.put(_db)); + await _db.writeTxn(() => a.put(_db)); } on IsarError catch (e) { _log.severe("Failed to put new asset into db: $e"); return false; @@ -162,11 +143,11 @@ class SyncService { final List inDb = await _db.assets .filter() .ownerIdEqualTo(user.isarId) - .sortByDeviceId() - .thenByLocalId() - .thenByFileModifiedAt() + .sortByChecksum() .findAll(); - remote.sort(Asset.compareByOwnerDeviceLocalIdModified); + assert(inDb.isSorted(Asset.compareByChecksum), "inDb not sorted!"); + + remote.sort(Asset.compareByChecksum); final (toAdd, toUpdate, toRemove) = _diffAssets(remote, inDb, remote: true); if (toAdd.isEmpty && toUpdate.isEmpty && toRemove.isEmpty) { return false; @@ -199,6 +180,7 @@ class SyncService { query = baseQuery.owner((q) => q.isarIdEqualTo(me.isarId)); } final List dbAlbums = await query.sortByRemoteId().findAll(); + assert(dbAlbums.isSortedBy((e) => e.remoteId!), "dbAlbums not sorted!"); final List toDelete = []; final List existing = []; @@ -245,16 +227,16 @@ class SyncService { if (dto.assetCount != dto.assets.length) { return false; } - final assetsInDb = await album.assets - .filter() - .sortByOwnerId() - .thenByDeviceId() - .thenByLocalId() - .thenByFileModifiedAt() - .findAll(); + final assetsInDb = + await album.assets.filter().sortByOwnerId().thenByChecksum().findAll(); + assert(assetsInDb.isSorted(Asset.compareByOwnerChecksum), "inDb unsorted!"); final List assetsOnRemote = dto.getAssets(); - assetsOnRemote.sort(Asset.compareByOwnerDeviceLocalIdModified); - final (toAdd, toUpdate, toUnlink) = _diffAssets(assetsOnRemote, assetsInDb); + assetsOnRemote.sort(Asset.compareByOwnerChecksum); + final (toAdd, toUpdate, toUnlink) = _diffAssets( + assetsOnRemote, + assetsInDb, + compare: Asset.compareByOwnerChecksum, + ); // update shared users final List sharedUsers = album.sharedUsers.toList(growable: false); @@ -297,6 +279,7 @@ class SyncService { await album.assets.update(link: assetsToLink, unlink: toUnlink.cast()); await _db.albums.put(album); }); + _log.info("Synced changes of remote album ${album.name} to DB"); } on IsarError catch (e) { _log.severe("Failed to sync remote album to database $e"); } @@ -382,10 +365,11 @@ class SyncService { Set? excludedAssets, ]) async { onDevice.sort((a, b) => a.id.compareTo(b.id)); - final List inDb = + final inDb = await _db.albums.where().localIdIsNotNull().sortByLocalId().findAll(); final List deleteCandidates = []; final List existing = []; + assert(inDb.isSorted((a, b) => a.localId!.compareTo(b.localId!)), "sort!"); final bool anyChanges = await diffSortedLists( onDevice, inDb, @@ -447,14 +431,15 @@ class SyncService { final inDb = await album.assets .filter() .ownerIdEqualTo(Store.get(StoreKey.currentUser).isarId) - .deviceIdEqualTo(Store.get(StoreKey.deviceIdHash)) - .sortByLocalId() + .sortByChecksum() .findAll(); + assert(inDb.isSorted(Asset.compareByChecksum), "inDb not sorted!"); + final int assetCountOnDevice = await ape.assetCountAsync; final List onDevice = - await ape.getAssets(excludedAssets: excludedAssets); - onDevice.sort(Asset.compareByLocalId); - final (toAdd, toUpdate, toDelete) = - _diffAssets(onDevice, inDb, compare: Asset.compareByLocalId); + await _hashService.getHashedAssets(ape, excludedAssets: excludedAssets); + _removeDuplicates(onDevice); + // _removeDuplicates sorts `onDevice` by checksum + final (toAdd, toUpdate, toDelete) = _diffAssets(onDevice, inDb); if (toAdd.isEmpty && toUpdate.isEmpty && toDelete.isEmpty && @@ -491,6 +476,9 @@ class SyncService { await _db.albums.put(album); album.thumbnail.value ??= await album.assets.filter().findFirst(); await album.thumbnail.save(); + await _db.eTags.put( + ETag(id: ape.eTagKeyAssetCount, value: assetCountOnDevice.toString()), + ); }); _log.info("Synced changes of local album ${ape.name} to DB"); } on IsarError catch (e) { @@ -503,8 +491,13 @@ class SyncService { /// fast path for common case: only new assets were added to device album /// returns `true` if successfull, else `false` Future _syncDeviceAlbumFast(AssetPathEntity ape, Album album) async { + if (!(ape.lastModified ?? DateTime.now()).isAfter(album.modifiedAt)) { + return false; + } final int totalOnDevice = await ape.assetCountAsync; - final AssetPathEntity? modified = totalOnDevice > album.assetCount + final int lastKnownTotal = + (await _db.eTags.getById(ape.eTagKeyAssetCount))?.value?.toInt() ?? 0; + final AssetPathEntity? modified = totalOnDevice > lastKnownTotal ? await ape.fetchPathProperties( filterOptionGroup: FilterOptionGroup( updateTimeCond: DateTimeCond( @@ -517,17 +510,22 @@ class SyncService { if (modified == null) { return false; } - final List newAssets = await modified.getAssets(); - if (totalOnDevice != album.assets.length + newAssets.length) { + final List newAssets = await _hashService.getHashedAssets(modified); + + if (totalOnDevice != lastKnownTotal + newAssets.length) { return false; } album.modifiedAt = ape.lastModified ?? DateTime.now(); + _removeDuplicates(newAssets); final (existingInDb, updated) = await _linkWithExistingFromDb(newAssets); try { await _db.writeTxn(() async { await _db.assets.putAll(updated); await album.assets.update(link: existingInDb + updated); await _db.albums.put(album); + await _db.eTags.put( + ETag(id: ape.eTagKeyAssetCount, value: totalOnDevice.toString()), + ); }); _log.info("Fast synced local album ${ape.name} to DB"); } on IsarError catch (e) { @@ -547,7 +545,9 @@ class SyncService { ]) async { _log.info("Syncing a new local album to DB: ${ape.name}"); final Album a = Album.local(ape); - final assets = await ape.getAssets(excludedAssets: excludedAssets); + final assets = + await _hashService.getHashedAssets(ape, excludedAssets: excludedAssets); + _removeDuplicates(assets); final (existingInDb, updated) = await _linkWithExistingFromDb(assets); _log.info( "${existingInDb.length} assets already existed in DB, to upsert ${updated.length}", @@ -570,44 +570,29 @@ class SyncService { Future<(List existing, List updated)> _linkWithExistingFromDb( List assets, ) async { - if (assets.isEmpty) { - return ([].cast(), [].cast()); - } - final List inDb = await _db.assets - .where() - .anyOf( - assets, - (q, Asset e) => q.localIdDeviceIdEqualTo(e.localId, e.deviceId), - ) - .sortByOwnerId() - .thenByDeviceId() - .thenByLocalId() - .thenByFileModifiedAt() - .findAll(); - assets.sort(Asset.compareByOwnerDeviceLocalIdModified); - final List existing = [], toUpsert = []; - diffSortedListsSync( - inDb, - assets, - // do not compare by modified date because for some assets dates differ on - // client and server, thus never reaching "both" case below - compare: Asset.compareByOwnerDeviceLocalId, - both: (Asset a, Asset b) { - if (a.canUpdate(b)) { - toUpsert.add(a.updatedCopy(b)); - return true; - } else { - existing.add(a); - return false; - } - }, - onlyFirst: (Asset a) => _log.finer( - "_linkWithExistingFromDb encountered asset only in DB: $a", - null, - StackTrace.current, - ), - onlySecond: (Asset b) => toUpsert.add(b), + if (assets.isEmpty) return ([].cast(), [].cast()); + + final List inDb = await _db.assets.getAllByChecksumOwnerId( + assets.map((a) => a.checksum).toList(growable: false), + assets.map((a) => a.ownerId).toInt64List(), ); + assert(inDb.length == assets.length); + final List existing = [], toUpsert = []; + for (int i = 0; i < assets.length; i++) { + final Asset? b = inDb[i]; + if (b == null) { + toUpsert.add(assets[i]); + continue; + } + if (b.canUpdate(assets[i])) { + final updated = b.updatedCopy(assets[i]); + assert(updated.id != Isar.autoIncrement); + toUpsert.add(updated); + } else { + existing.add(b); + } + } + assert(existing.length + toUpsert.length == assets.length); return (existing, toUpsert); } @@ -627,11 +612,63 @@ class SyncService { }); _log.info("Upserted ${assets.length} assets into the DB"); } on IsarError catch (e) { - _log.warning( + _log.severe( "Failed to upsert ${assets.length} assets into the DB: ${e.toString()}", ); + // give details on the errors + assets.sort(Asset.compareByOwnerChecksum); + final inDb = await _db.assets.getAllByChecksumOwnerId( + assets.map((e) => e.checksum).toList(growable: false), + assets.map((e) => e.ownerId).toInt64List(), + ); + for (int i = 0; i < assets.length; i++) { + final Asset a = assets[i]; + final Asset? b = inDb[i]; + if (b == null) { + if (a.id != Isar.autoIncrement) { + _log.warning( + "Trying to update an asset that does not exist in DB:\n$a", + ); + } + } else if (a.id != b.id) { + _log.warning( + "Trying to insert another asset with the same checksum+owner. In DB:\n$b\nTo insert:\n$a", + ); + } + } + for (int i = 1; i < assets.length; i++) { + if (Asset.compareByOwnerChecksum(assets[i - 1], assets[i]) == 0) { + _log.warning( + "Trying to insert duplicate assets:\n${assets[i - 1]}\n${assets[i]}", + ); + } + } } } + + List _removeDuplicates(List assets) { + final int before = assets.length; + assets.sort(Asset.compareByOwnerChecksumCreatedModified); + assets.uniqueConsecutive( + compare: Asset.compareByOwnerChecksum, + onDuplicate: (a, b) => + _log.info("Ignoring duplicate assets on device:\n$a\n$b"), + ); + final int duplicates = before - assets.length; + if (duplicates > 0) { + _log.warning("Ignored $duplicates duplicate assets on device"); + } + return assets; + } + + /// returns `true` if the albums differ on the surface + Future _hasAssetPathEntityChanged(AssetPathEntity a, Album b) async { + return a.name != b.name || + a.lastModified == null || + !a.lastModified!.isAtSameMomentAs(b.modifiedAt) || + await a.assetCountAsync != + (await _db.eTags.getById(a.eTagKeyAssetCount))?.value?.toInt(); + } } /// Returns a triple(toAdd, toUpdate, toRemove) @@ -639,7 +676,7 @@ class SyncService { List assets, List inDb, { bool? remote, - int Function(Asset, Asset) compare = Asset.compareByOwnerDeviceLocalId, + int Function(Asset, Asset) compare = Asset.compareByChecksum, }) { final List toAdd = []; final List toUpdate = []; @@ -663,7 +700,7 @@ class SyncService { } } else if (remote == false && a.isRemote) { if (a.isLocal) { - a.isLocal = false; + a.localId = null; toUpdate.add(a); } } else { @@ -685,9 +722,9 @@ class SyncService { return const ([], []); } deleteCandidates.sort(Asset.compareById); - deleteCandidates.uniqueConsecutive((a) => a.id); + deleteCandidates.uniqueConsecutive(compare: Asset.compareById); existing.sort(Asset.compareById); - existing.uniqueConsecutive((a) => a.id); + existing.uniqueConsecutive(compare: Asset.compareById); final (tooAdd, toUpdate, toRemove) = _diffAssets( existing, deleteCandidates, @@ -698,14 +735,6 @@ class SyncService { return (toRemove.map((e) => e.id).toList(), toUpdate); } -/// returns `true` if the albums differ on the surface -Future _hasAssetPathEntityChanged(AssetPathEntity a, Album b) async { - return a.name != b.name || - a.lastModified == null || - !a.lastModified!.isAtSameMomentAs(b.modifiedAt) || - await a.assetCountAsync != b.assetCount; -} - /// returns `true` if the albums differ on the surface bool _hasAlbumResponseDtoChanged(AlbumResponseDto dto, Album a) { return dto.assetCount != a.assetCount || diff --git a/mobile/lib/shared/views/tab_controller_page.dart b/mobile/lib/shared/views/tab_controller_page.dart index 4d245d034a..e14f535144 100644 --- a/mobile/lib/shared/views/tab_controller_page.dart +++ b/mobile/lib/shared/views/tab_controller_page.dart @@ -6,12 +6,39 @@ import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/modules/asset_viewer/providers/scroll_notifier.provider.dart'; import 'package:immich_mobile/modules/home/providers/multiselect.provider.dart'; import 'package:immich_mobile/routing/router.dart'; +import 'package:immich_mobile/shared/providers/asset.provider.dart'; -class TabControllerPage extends ConsumerWidget { +class TabControllerPage extends HookConsumerWidget { const TabControllerPage({Key? key}) : super(key: key); @override Widget build(BuildContext context, WidgetRef ref) { + final refreshing = ref.watch(assetProvider); + + Widget buildIcon(Widget icon) { + if (!refreshing) return icon; + return Stack( + alignment: Alignment.center, + clipBehavior: Clip.none, + children: [ + icon, + Positioned( + right: -14, + child: SizedBox( + height: 12, + width: 12, + child: CircularProgressIndicator( + strokeWidth: 2, + valueColor: AlwaysStoppedAnimation( + Theme.of(context).primaryColor, + ), + ), + ), + ), + ], + ); + } + navigationRail(TabsRouter tabsRouter) { return NavigationRail( labelType: NavigationRailLabelType.all, @@ -83,9 +110,12 @@ class TabControllerPage extends ConsumerWidget { icon: const Icon( Icons.photo_library_outlined, ), - selectedIcon: Icon( - Icons.photo_library, - color: Theme.of(context).primaryColor, + selectedIcon: buildIcon( + Icon( + size: 24, + Icons.photo_library, + color: Theme.of(context).primaryColor, + ), ), ), NavigationDestination( @@ -113,9 +143,11 @@ class TabControllerPage extends ConsumerWidget { icon: const Icon( Icons.photo_album_outlined, ), - selectedIcon: Icon( - Icons.photo_album_rounded, - color: Theme.of(context).primaryColor, + selectedIcon: buildIcon( + Icon( + Icons.photo_album_rounded, + color: Theme.of(context).primaryColor, + ), ), ) ], diff --git a/mobile/lib/utils/builtin_extensions.dart b/mobile/lib/utils/builtin_extensions.dart index 3a3a723dc1..5b769f26fd 100644 --- a/mobile/lib/utils/builtin_extensions.dart +++ b/mobile/lib/utils/builtin_extensions.dart @@ -1,3 +1,5 @@ +import 'dart:typed_data'; + import 'package:collection/collection.dart'; extension DurationExtension on String { @@ -22,15 +24,20 @@ extension DurationExtension on String { } extension ListExtension on List { - List uniqueConsecutive([T Function(E element)? key]) { - key ??= (E e) => e as T; + List uniqueConsecutive({ + int Function(E a, E b)? compare, + void Function(E a, E b)? onDuplicate, + }) { + compare ??= (E a, E b) => a == b ? 0 : 1; int i = 1, j = 1; for (; i < length; i++) { - if (key(this[i]) != key(this[i - 1])) { + if (compare(this[i - 1], this[i]) != 0) { if (i != j) { this[j] = this[i]; } j++; + } else if (onDuplicate != null) { + onDuplicate(this[i - 1], this[i]); } } length = length == 0 ? 0 : j; @@ -45,3 +52,11 @@ extension ListExtension on List { return ListSlice(this, start, end); } } + +extension IntListExtension on Iterable { + Int64List toInt64List() { + final list = Int64List(length); + list.setAll(0, this); + return list; + } +} diff --git a/mobile/lib/utils/migration.dart b/mobile/lib/utils/migration.dart index 56e9ba8df7..724f3a8722 100644 --- a/mobile/lib/utils/migration.dart +++ b/mobile/lib/utils/migration.dart @@ -8,11 +8,13 @@ Future migrateDatabaseIfNeeded(Isar db) async { final int version = Store.get(StoreKey.version, 1); switch (version) { case 1: - await _migrateV1ToV2(db); + await _migrateTo(db, 2); + case 2: + await _migrateTo(db, 3); } } -Future _migrateV1ToV2(Isar db) async { +Future _migrateTo(Isar db, int version) async { await clearAssetsAndAlbums(db); - await Store.put(StoreKey.version, 2); + await Store.put(StoreKey.version, version); } diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock index 736b7a4f56..71585f92e7 100644 --- a/mobile/pubspec.lock +++ b/mobile/pubspec.lock @@ -242,13 +242,13 @@ packages: source: hosted version: "0.3.3+4" crypto: - dependency: transitive + dependency: "direct main" description: name: crypto - sha256: aa274aa7774f8964e4f4f38cc994db7b6158dd36e9187aaceaddc994b35c6c67 + sha256: ff625774173754681d66daaf4a448684fb04b78f902da9cb3d308c19cc5e8bab url: "https://pub.dev" source: hosted - version: "3.0.2" + version: "3.0.3" csslib: dependency: transitive description: @@ -333,10 +333,10 @@ packages: dependency: transitive description: name: ffi - sha256: a38574032c5f1dd06c4aee541789906c12ccaab8ba01446e800d9c5b79c4a978 + sha256: ed5337a5660c506388a9f012be0288fb38b49020ce2b45fe1f8b8323fe429f99 url: "https://pub.dev" source: hosted - version: "2.0.1" + version: "2.0.2" file: dependency: transitive description: diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index 8dfc06389d..0a5aa1350d 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -45,6 +45,7 @@ dependencies: isar_flutter_libs: *isar_version # contains Isar Core permission_handler: ^10.2.0 device_info_plus: ^8.1.0 + crypto: ^3.0.3 # TODO remove once native crypto is used on iOS openapi: path: openapi diff --git a/mobile/test/asset_grid_data_structure_test.dart b/mobile/test/asset_grid_data_structure_test.dart index 78c553f5a8..6522bec3d0 100644 --- a/mobile/test/asset_grid_data_structure_test.dart +++ b/mobile/test/asset_grid_data_structure_test.dart @@ -13,8 +13,8 @@ void main() { testAssets.add( Asset( + checksum: "", localId: '$i', - deviceId: 1, ownerId: 1, fileCreatedAt: date, fileModifiedAt: date, @@ -23,7 +23,6 @@ void main() { type: AssetType.image, fileName: '', isFavorite: false, - isLocal: false, isArchived: false, ), ); diff --git a/mobile/test/builtin_extensions_text.dart b/mobile/test/builtin_extensions_test.dart similarity index 88% rename from mobile/test/builtin_extensions_text.dart rename to mobile/test/builtin_extensions_test.dart index 9e4924e44e..875a20fb06 100644 --- a/mobile/test/builtin_extensions_text.dart +++ b/mobile/test/builtin_extensions_test.dart @@ -43,7 +43,12 @@ void main() { test('withKey', () { final a = ["a", "bb", "cc", "ddd"]; - expect(a.uniqueConsecutive((s) => s.length), ["a", "bb", "ddd"]); + expect( + a.uniqueConsecutive( + compare: (s1, s2) => s1.length.compareTo(s2.length), + ), + ["a", "bb", "ddd"], + ); }); }); } diff --git a/mobile/test/sync_service_test.dart b/mobile/test/sync_service_test.dart index 0d1045c729..63b5614bd5 100644 --- a/mobile/test/sync_service_test.dart +++ b/mobile/test/sync_service_test.dart @@ -6,32 +6,33 @@ import 'package:immich_mobile/shared/models/exif_info.dart'; import 'package:immich_mobile/shared/models/logger_message.model.dart'; import 'package:immich_mobile/shared/models/store.dart'; import 'package:immich_mobile/shared/models/user.dart'; +import 'package:immich_mobile/shared/services/hash.service.dart'; import 'package:immich_mobile/shared/services/immich_logger.service.dart'; import 'package:immich_mobile/shared/services/sync.service.dart'; import 'package:isar/isar.dart'; +import 'package:mockito/mockito.dart'; void main() { Asset makeAsset({ - required String localId, + required String checksum, + String? localId, String? remoteId, int deviceId = 1, int ownerId = 590700560494856554, // hash of "1" - bool isLocal = false, }) { final DateTime date = DateTime(2000); return Asset( + checksum: checksum, localId: localId, remoteId: remoteId, - deviceId: deviceId, ownerId: ownerId, fileCreatedAt: date, fileModifiedAt: date, updatedAt: date, durationInSeconds: 0, type: AssetType.image, - fileName: localId, + fileName: localId ?? remoteId ?? "", isFavorite: false, - isLocal: isLocal, isArchived: false, ); } @@ -53,6 +54,7 @@ void main() { group('Test SyncService grouped', () { late final Isar db; + final MockHashService hs = MockHashService(); final owner = User( id: "1", updatedAt: DateTime.now(), @@ -71,11 +73,11 @@ void main() { await Store.put(StoreKey.currentUser, owner); }); final List initialAssets = [ - makeAsset(localId: "1", remoteId: "0-1", deviceId: 0), - makeAsset(localId: "1", remoteId: "2-1", deviceId: 2), - makeAsset(localId: "1", remoteId: "1-1", isLocal: true), - makeAsset(localId: "2", isLocal: true), - makeAsset(localId: "3", isLocal: true), + makeAsset(checksum: "a", remoteId: "0-1", deviceId: 0), + makeAsset(checksum: "b", remoteId: "2-1", deviceId: 2), + makeAsset(checksum: "c", localId: "1", remoteId: "1-1"), + makeAsset(checksum: "d", localId: "2"), + makeAsset(checksum: "e", localId: "3"), ]; setUp(() { db.writeTxnSync(() { @@ -84,11 +86,11 @@ void main() { }); }); test('test inserting existing assets', () async { - SyncService s = SyncService(db); + SyncService s = SyncService(db, hs); final List remoteAssets = [ - makeAsset(localId: "1", remoteId: "0-1", deviceId: 0), - makeAsset(localId: "1", remoteId: "2-1", deviceId: 2), - makeAsset(localId: "1", remoteId: "1-1"), + makeAsset(checksum: "a", remoteId: "0-1", deviceId: 0), + makeAsset(checksum: "b", remoteId: "2-1", deviceId: 2), + makeAsset(checksum: "c", remoteId: "1-1"), ]; expect(db.assets.countSync(), 5); final bool c1 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets); @@ -97,14 +99,14 @@ void main() { }); test('test inserting new assets', () async { - SyncService s = SyncService(db); + SyncService s = SyncService(db, hs); final List remoteAssets = [ - makeAsset(localId: "1", remoteId: "0-1", deviceId: 0), - makeAsset(localId: "1", remoteId: "2-1", deviceId: 2), - makeAsset(localId: "1", remoteId: "1-1"), - makeAsset(localId: "2", remoteId: "1-2"), - makeAsset(localId: "4", remoteId: "1-4"), - makeAsset(localId: "1", remoteId: "3-1", deviceId: 3), + makeAsset(checksum: "a", remoteId: "0-1", deviceId: 0), + makeAsset(checksum: "b", remoteId: "2-1", deviceId: 2), + makeAsset(checksum: "c", remoteId: "1-1"), + makeAsset(checksum: "d", remoteId: "1-2"), + makeAsset(checksum: "f", remoteId: "1-4"), + makeAsset(checksum: "g", remoteId: "3-1", deviceId: 3), ]; expect(db.assets.countSync(), 5); final bool c1 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets); @@ -113,14 +115,14 @@ void main() { }); test('test syncing duplicate assets', () async { - SyncService s = SyncService(db); + SyncService s = SyncService(db, hs); final List remoteAssets = [ - makeAsset(localId: "1", remoteId: "0-1", deviceId: 0), - makeAsset(localId: "1", remoteId: "1-1"), - makeAsset(localId: "1", remoteId: "2-1", deviceId: 2), - makeAsset(localId: "1", remoteId: "2-1b", deviceId: 2), - makeAsset(localId: "1", remoteId: "2-1c", deviceId: 2), - makeAsset(localId: "1", remoteId: "2-1d", deviceId: 2), + makeAsset(checksum: "a", remoteId: "0-1", deviceId: 0), + makeAsset(checksum: "b", remoteId: "1-1"), + makeAsset(checksum: "c", remoteId: "2-1", deviceId: 2), + makeAsset(checksum: "h", remoteId: "2-1b", deviceId: 2), + makeAsset(checksum: "i", remoteId: "2-1c", deviceId: 2), + makeAsset(checksum: "j", remoteId: "2-1d", deviceId: 2), ]; expect(db.assets.countSync(), 5); final bool c1 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets); @@ -133,11 +135,13 @@ void main() { final bool c3 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets); expect(c3, true); expect(db.assets.countSync(), 7); - remoteAssets.add(makeAsset(localId: "1", remoteId: "2-1e", deviceId: 2)); - remoteAssets.add(makeAsset(localId: "2", remoteId: "2-2", deviceId: 2)); + remoteAssets.add(makeAsset(checksum: "k", remoteId: "2-1e", deviceId: 2)); + remoteAssets.add(makeAsset(checksum: "l", remoteId: "2-2", deviceId: 2)); final bool c4 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets); expect(c4, true); expect(db.assets.countSync(), 9); }); }); } + +class MockHashService extends Mock implements HashService {}