From 1811d42d79059ddb7fac34fc11bc5d0083a79892 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Mar 2026 18:52:32 +0000 Subject: [PATCH] feat: skip local hashing for iCloud assets, use server-computed checksum Instead of downloading iCloud assets twice (once to hash, once to upload), skip local hashing for iCloud-only assets and let the server return its computed SHA1 checksum in the upload response. The mobile app stores this checksum locally to prevent re-uploads. Changes: - Server returns checksum in upload response (created + duplicate) - iOS native layer tags iCloud-only assets with ICLOUD_ONLY error - Hash service skips iCloud assets (allowNetworkAccess: false) - Upload result carries server checksum back to mobile - Foreground/background upload services store server checksum - Backup candidates include unhashed assets (onlyHashed: false) https://claude.ai/code/session_01LEs74WpkJ1gWcJrFBsp1i2 --- mobile/ios/Runner/Sync/MessagesImpl.swift | 6 ++++++ mobile/lib/domain/services/hash.service.dart | 8 +++++++- mobile/lib/repositories/upload.repository.dart | 11 ++++++++--- .../lib/services/background_upload.service.dart | 17 ++++++++++++++++- .../lib/services/foreground_upload.service.dart | 11 ++++++++++- server/src/dtos/asset-media-response.dto.ts | 2 ++ server/src/services/asset-media.service.ts | 4 ++-- 7 files changed, 51 insertions(+), 8 deletions(-) diff --git a/mobile/ios/Runner/Sync/MessagesImpl.swift b/mobile/ios/Runner/Sync/MessagesImpl.swift index 8022fb06d2..48196dc681 100644 --- a/mobile/ios/Runner/Sync/MessagesImpl.swift +++ b/mobile/ios/Runner/Sync/MessagesImpl.swift @@ -357,6 +357,12 @@ class NativeSyncApiImpl: ImmichPlugin, NativeSyncApi, FlutterPlugin { completionHandler: { error in let result: HashResult? = switch (error) { case let e as PHPhotosError where e.code == .userCancelled: nil + case let e as PHPhotosError where e.code == .networkAccessRequired: + HashResult( + assetId: asset.localIdentifier, + error: "ICLOUD_ONLY", + hash: nil + ) case let .some(e): HashResult( assetId: asset.localIdentifier, error: "Failed to hash asset: \(e.localizedDescription)", diff --git a/mobile/lib/domain/services/hash.service.dart b/mobile/lib/domain/services/hash.service.dart index 6781507566..2b6f7aaee1 100644 --- a/mobile/lib/domain/services/hash.service.dart +++ b/mobile/lib/domain/services/hash.service.dart @@ -109,9 +109,11 @@ class HashService { _log.fine("Hashing ${toHash.length} files"); final hashed = {}; + // Never download from iCloud just to hash. iCloud-only assets will be + // uploaded directly and the server will compute + return their checksum. final hashResults = await _nativeSyncApi.hashAssets( toHash.keys.toList(), - allowNetworkAccess: album.backupSelection == BackupSelection.selected, + allowNetworkAccess: false, ); assert( hashResults.length == toHash.length, @@ -127,6 +129,10 @@ class HashService { final hashResult = hashResults[i]; if (hashResult.hash != null) { hashed[hashResult.assetId] = hashResult.hash!; + } else if (hashResult.error == 'ICLOUD_ONLY') { + // Asset is in iCloud and not available locally. It will be uploaded + // directly and the server will compute its checksum. + _log.fine("Skipping iCloud-only asset ${hashResult.assetId} from album: ${album.name}"); } else { final asset = toHash[hashResult.assetId]; _log.warning( diff --git a/mobile/lib/repositories/upload.repository.dart b/mobile/lib/repositories/upload.repository.dart index 98c6202e19..c5876b8d74 100644 --- a/mobile/lib/repositories/upload.repository.dart +++ b/mobile/lib/repositories/upload.repository.dart @@ -136,7 +136,10 @@ class UploadRepository { try { final responseBody = jsonDecode(responseBodyString); - return UploadResult.success(remoteAssetId: responseBody['id'] as String); + return UploadResult.success( + remoteAssetId: responseBody['id'] as String, + checksum: responseBody['checksum'] as String?, + ); } catch (e) { return UploadResult.error(errorMessage: 'Failed to parse server response'); } @@ -182,6 +185,7 @@ class UploadResult { final bool isSuccess; final bool isCancelled; final String? remoteAssetId; + final String? checksum; final String? errorMessage; final int? statusCode; @@ -189,12 +193,13 @@ class UploadResult { required this.isSuccess, required this.isCancelled, this.remoteAssetId, + this.checksum, this.errorMessage, this.statusCode, }); - factory UploadResult.success({required String remoteAssetId}) { - return UploadResult(isSuccess: true, isCancelled: false, remoteAssetId: remoteAssetId); + factory UploadResult.success({required String remoteAssetId, String? checksum}) { + return UploadResult(isSuccess: true, isCancelled: false, remoteAssetId: remoteAssetId, checksum: checksum); } factory UploadResult.error({String? errorMessage, int? statusCode}) { diff --git a/mobile/lib/services/background_upload.service.dart b/mobile/lib/services/background_upload.service.dart index d54a677c24..edf452dc54 100644 --- a/mobile/lib/services/background_upload.service.dart +++ b/mobile/lib/services/background_upload.service.dart @@ -162,7 +162,7 @@ class BackgroundUploadService { await _storageRepository.clearCache(); shouldAbortQueuingTasks = false; - final candidates = await _backupRepository.getCandidates(userId); + final candidates = await _backupRepository.getCandidates(userId, onlyHashed: false); if (candidates.isEmpty) { _logger.info("No new backup candidates found, finishing background upload"); return; @@ -210,6 +210,7 @@ class BackgroundUploadService { switch (update.status) { case TaskStatus.complete: unawaited(_handleLivePhoto(update)); + unawaited(_storeServerChecksum(update)); if (CurrentPlatform.isIOS) { try { @@ -227,6 +228,20 @@ class BackgroundUploadService { } } + Future _storeServerChecksum(TaskStatusUpdate update) async { + try { + if (update.responseBody == null || update.responseBody!.isEmpty) return; + final response = jsonDecode(update.responseBody!); + final checksum = response['checksum'] as String?; + if (checksum == null) return; + final deviceAssetId = update.task.taskId; + if (deviceAssetId.isEmpty) return; + await _localAssetRepository.updateHashes({deviceAssetId: checksum}); + } catch (e) { + _logger.warning('Failed to store server checksum: $e'); + } + } + Future _handleLivePhoto(TaskStatusUpdate update) async { try { if (update.task.metaData.isEmpty || update.task.metaData == '') { diff --git a/mobile/lib/services/foreground_upload.service.dart b/mobile/lib/services/foreground_upload.service.dart index ce02c9c56b..c930cc418c 100644 --- a/mobile/lib/services/foreground_upload.service.dart +++ b/mobile/lib/services/foreground_upload.service.dart @@ -11,9 +11,11 @@ import 'package:immich_mobile/extensions/platform_extensions.dart'; import 'package:immich_mobile/extensions/network_capability_extensions.dart'; import 'package:immich_mobile/extensions/translate_extensions.dart'; import 'package:immich_mobile/infrastructure/repositories/backup.repository.dart'; +import 'package:immich_mobile/infrastructure/repositories/local_asset.repository.dart'; import 'package:immich_mobile/infrastructure/repositories/storage.repository.dart'; import 'package:immich_mobile/platform/connectivity_api.g.dart'; import 'package:immich_mobile/providers/app_settings.provider.dart'; +import 'package:immich_mobile/providers/infrastructure/asset.provider.dart'; import 'package:immich_mobile/providers/infrastructure/platform.provider.dart'; import 'package:immich_mobile/providers/infrastructure/storage.provider.dart'; import 'package:immich_mobile/repositories/asset_media.repository.dart'; @@ -37,6 +39,7 @@ final foregroundUploadServiceProvider = Provider((ref) { return ForegroundUploadService( ref.watch(uploadRepositoryProvider), ref.watch(storageRepositoryProvider), + ref.watch(localAssetRepository), ref.watch(backupRepositoryProvider), ref.watch(connectivityApiProvider), ref.watch(appSettingsServiceProvider), @@ -53,6 +56,7 @@ class ForegroundUploadService { ForegroundUploadService( this._uploadRepository, this._storageRepository, + this._localAssetRepository, this._backupRepository, this._connectivityApi, this._appSettingsService, @@ -61,6 +65,7 @@ class ForegroundUploadService { final UploadRepository _uploadRepository; final StorageRepository _storageRepository; + final DriftLocalAssetRepository _localAssetRepository; final DriftBackupRepository _backupRepository; final ConnectivityApi _connectivityApi; final AppSettingsService _appSettingsService; @@ -84,7 +89,7 @@ class ForegroundUploadService { UploadCallbacks callbacks = const UploadCallbacks(), bool useSequentialUpload = false, }) async { - final candidates = await _backupRepository.getCandidates(userId); + final candidates = await _backupRepository.getCandidates(userId, onlyHashed: false); if (candidates.isEmpty) { return; } @@ -387,6 +392,10 @@ class ForegroundUploadService { ); if (result.isSuccess && result.remoteAssetId != null) { + // Store server-computed checksum so iCloud-only assets don't need re-upload + if (result.checksum != null) { + await _localAssetRepository.updateHashes({asset.id: result.checksum!}); + } callbacks.onSuccess?.call(asset.localId!, result.remoteAssetId!); } else if (result.isCancelled) { _logger.warning(() => "Backup was cancelled by the user"); diff --git a/server/src/dtos/asset-media-response.dto.ts b/server/src/dtos/asset-media-response.dto.ts index 345c1bf418..f15faece92 100644 --- a/server/src/dtos/asset-media-response.dto.ts +++ b/server/src/dtos/asset-media-response.dto.ts @@ -11,6 +11,8 @@ export class AssetMediaResponseDto { status!: AssetMediaStatus; @ApiProperty({ description: 'Asset media ID' }) id!: string; + @ApiPropertyOptional({ description: 'Asset checksum (SHA1 base64)' }) + checksum?: string; } export enum AssetUploadAction { diff --git a/server/src/services/asset-media.service.ts b/server/src/services/asset-media.service.ts index 3c981ea61e..5b00c4b057 100644 --- a/server/src/services/asset-media.service.ts +++ b/server/src/services/asset-media.service.ts @@ -157,7 +157,7 @@ export class AssetMediaService extends BaseService { await this.userRepository.updateUsage(auth.user.id, file.size); - return { id: asset.id, status: AssetMediaStatus.CREATED }; + return { id: asset.id, status: AssetMediaStatus.CREATED, checksum: file.checksum.toString('base64') }; } catch (error: any) { return this.handleUploadError(error, auth, file, sidecarFile); } @@ -350,7 +350,7 @@ export class AssetMediaService extends BaseService { await this.sharedLinkRepository.addAssets(auth.sharedLink.id, [duplicateId]); } - return { status: AssetMediaStatus.DUPLICATE, id: duplicateId }; + return { status: AssetMediaStatus.DUPLICATE, id: duplicateId, checksum: file.checksum.toString('base64') }; } this.logger.error(`Error uploading file ${error}`, error?.stack);