mirror of
https://github.com/immich-app/immich.git
synced 2026-03-22 14:29:26 +03:00
feat: skip local hashing for iCloud assets, use server-computed checksum
Instead of downloading iCloud assets twice (once to hash, once to upload), skip local hashing for iCloud-only assets and let the server return its computed SHA1 checksum in the upload response. The mobile app stores this checksum locally to prevent re-uploads. Changes: - Server returns checksum in upload response (created + duplicate) - iOS native layer tags iCloud-only assets with ICLOUD_ONLY error - Hash service skips iCloud assets (allowNetworkAccess: false) - Upload result carries server checksum back to mobile - Foreground/background upload services store server checksum - Backup candidates include unhashed assets (onlyHashed: false) https://claude.ai/code/session_01LEs74WpkJ1gWcJrFBsp1i2
This commit is contained in:
@@ -357,6 +357,12 @@ class NativeSyncApiImpl: ImmichPlugin, NativeSyncApi, FlutterPlugin {
|
||||
completionHandler: { error in
|
||||
let result: HashResult? = switch (error) {
|
||||
case let e as PHPhotosError where e.code == .userCancelled: nil
|
||||
case let e as PHPhotosError where e.code == .networkAccessRequired:
|
||||
HashResult(
|
||||
assetId: asset.localIdentifier,
|
||||
error: "ICLOUD_ONLY",
|
||||
hash: nil
|
||||
)
|
||||
case let .some(e): HashResult(
|
||||
assetId: asset.localIdentifier,
|
||||
error: "Failed to hash asset: \(e.localizedDescription)",
|
||||
|
||||
@@ -109,9 +109,11 @@ class HashService {
|
||||
_log.fine("Hashing ${toHash.length} files");
|
||||
|
||||
final hashed = <String, String>{};
|
||||
// Never download from iCloud just to hash. iCloud-only assets will be
|
||||
// uploaded directly and the server will compute + return their checksum.
|
||||
final hashResults = await _nativeSyncApi.hashAssets(
|
||||
toHash.keys.toList(),
|
||||
allowNetworkAccess: album.backupSelection == BackupSelection.selected,
|
||||
allowNetworkAccess: false,
|
||||
);
|
||||
assert(
|
||||
hashResults.length == toHash.length,
|
||||
@@ -127,6 +129,10 @@ class HashService {
|
||||
final hashResult = hashResults[i];
|
||||
if (hashResult.hash != null) {
|
||||
hashed[hashResult.assetId] = hashResult.hash!;
|
||||
} else if (hashResult.error == 'ICLOUD_ONLY') {
|
||||
// Asset is in iCloud and not available locally. It will be uploaded
|
||||
// directly and the server will compute its checksum.
|
||||
_log.fine("Skipping iCloud-only asset ${hashResult.assetId} from album: ${album.name}");
|
||||
} else {
|
||||
final asset = toHash[hashResult.assetId];
|
||||
_log.warning(
|
||||
|
||||
@@ -136,7 +136,10 @@ class UploadRepository {
|
||||
|
||||
try {
|
||||
final responseBody = jsonDecode(responseBodyString);
|
||||
return UploadResult.success(remoteAssetId: responseBody['id'] as String);
|
||||
return UploadResult.success(
|
||||
remoteAssetId: responseBody['id'] as String,
|
||||
checksum: responseBody['checksum'] as String?,
|
||||
);
|
||||
} catch (e) {
|
||||
return UploadResult.error(errorMessage: 'Failed to parse server response');
|
||||
}
|
||||
@@ -182,6 +185,7 @@ class UploadResult {
|
||||
final bool isSuccess;
|
||||
final bool isCancelled;
|
||||
final String? remoteAssetId;
|
||||
final String? checksum;
|
||||
final String? errorMessage;
|
||||
final int? statusCode;
|
||||
|
||||
@@ -189,12 +193,13 @@ class UploadResult {
|
||||
required this.isSuccess,
|
||||
required this.isCancelled,
|
||||
this.remoteAssetId,
|
||||
this.checksum,
|
||||
this.errorMessage,
|
||||
this.statusCode,
|
||||
});
|
||||
|
||||
factory UploadResult.success({required String remoteAssetId}) {
|
||||
return UploadResult(isSuccess: true, isCancelled: false, remoteAssetId: remoteAssetId);
|
||||
factory UploadResult.success({required String remoteAssetId, String? checksum}) {
|
||||
return UploadResult(isSuccess: true, isCancelled: false, remoteAssetId: remoteAssetId, checksum: checksum);
|
||||
}
|
||||
|
||||
factory UploadResult.error({String? errorMessage, int? statusCode}) {
|
||||
|
||||
@@ -162,7 +162,7 @@ class BackgroundUploadService {
|
||||
await _storageRepository.clearCache();
|
||||
shouldAbortQueuingTasks = false;
|
||||
|
||||
final candidates = await _backupRepository.getCandidates(userId);
|
||||
final candidates = await _backupRepository.getCandidates(userId, onlyHashed: false);
|
||||
if (candidates.isEmpty) {
|
||||
_logger.info("No new backup candidates found, finishing background upload");
|
||||
return;
|
||||
@@ -210,6 +210,7 @@ class BackgroundUploadService {
|
||||
switch (update.status) {
|
||||
case TaskStatus.complete:
|
||||
unawaited(_handleLivePhoto(update));
|
||||
unawaited(_storeServerChecksum(update));
|
||||
|
||||
if (CurrentPlatform.isIOS) {
|
||||
try {
|
||||
@@ -227,6 +228,20 @@ class BackgroundUploadService {
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> _storeServerChecksum(TaskStatusUpdate update) async {
|
||||
try {
|
||||
if (update.responseBody == null || update.responseBody!.isEmpty) return;
|
||||
final response = jsonDecode(update.responseBody!);
|
||||
final checksum = response['checksum'] as String?;
|
||||
if (checksum == null) return;
|
||||
final deviceAssetId = update.task.taskId;
|
||||
if (deviceAssetId.isEmpty) return;
|
||||
await _localAssetRepository.updateHashes({deviceAssetId: checksum});
|
||||
} catch (e) {
|
||||
_logger.warning('Failed to store server checksum: $e');
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> _handleLivePhoto(TaskStatusUpdate update) async {
|
||||
try {
|
||||
if (update.task.metaData.isEmpty || update.task.metaData == '') {
|
||||
|
||||
@@ -11,9 +11,11 @@ import 'package:immich_mobile/extensions/platform_extensions.dart';
|
||||
import 'package:immich_mobile/extensions/network_capability_extensions.dart';
|
||||
import 'package:immich_mobile/extensions/translate_extensions.dart';
|
||||
import 'package:immich_mobile/infrastructure/repositories/backup.repository.dart';
|
||||
import 'package:immich_mobile/infrastructure/repositories/local_asset.repository.dart';
|
||||
import 'package:immich_mobile/infrastructure/repositories/storage.repository.dart';
|
||||
import 'package:immich_mobile/platform/connectivity_api.g.dart';
|
||||
import 'package:immich_mobile/providers/app_settings.provider.dart';
|
||||
import 'package:immich_mobile/providers/infrastructure/asset.provider.dart';
|
||||
import 'package:immich_mobile/providers/infrastructure/platform.provider.dart';
|
||||
import 'package:immich_mobile/providers/infrastructure/storage.provider.dart';
|
||||
import 'package:immich_mobile/repositories/asset_media.repository.dart';
|
||||
@@ -37,6 +39,7 @@ final foregroundUploadServiceProvider = Provider((ref) {
|
||||
return ForegroundUploadService(
|
||||
ref.watch(uploadRepositoryProvider),
|
||||
ref.watch(storageRepositoryProvider),
|
||||
ref.watch(localAssetRepository),
|
||||
ref.watch(backupRepositoryProvider),
|
||||
ref.watch(connectivityApiProvider),
|
||||
ref.watch(appSettingsServiceProvider),
|
||||
@@ -53,6 +56,7 @@ class ForegroundUploadService {
|
||||
ForegroundUploadService(
|
||||
this._uploadRepository,
|
||||
this._storageRepository,
|
||||
this._localAssetRepository,
|
||||
this._backupRepository,
|
||||
this._connectivityApi,
|
||||
this._appSettingsService,
|
||||
@@ -61,6 +65,7 @@ class ForegroundUploadService {
|
||||
|
||||
final UploadRepository _uploadRepository;
|
||||
final StorageRepository _storageRepository;
|
||||
final DriftLocalAssetRepository _localAssetRepository;
|
||||
final DriftBackupRepository _backupRepository;
|
||||
final ConnectivityApi _connectivityApi;
|
||||
final AppSettingsService _appSettingsService;
|
||||
@@ -84,7 +89,7 @@ class ForegroundUploadService {
|
||||
UploadCallbacks callbacks = const UploadCallbacks(),
|
||||
bool useSequentialUpload = false,
|
||||
}) async {
|
||||
final candidates = await _backupRepository.getCandidates(userId);
|
||||
final candidates = await _backupRepository.getCandidates(userId, onlyHashed: false);
|
||||
if (candidates.isEmpty) {
|
||||
return;
|
||||
}
|
||||
@@ -387,6 +392,10 @@ class ForegroundUploadService {
|
||||
);
|
||||
|
||||
if (result.isSuccess && result.remoteAssetId != null) {
|
||||
// Store server-computed checksum so iCloud-only assets don't need re-upload
|
||||
if (result.checksum != null) {
|
||||
await _localAssetRepository.updateHashes({asset.id: result.checksum!});
|
||||
}
|
||||
callbacks.onSuccess?.call(asset.localId!, result.remoteAssetId!);
|
||||
} else if (result.isCancelled) {
|
||||
_logger.warning(() => "Backup was cancelled by the user");
|
||||
|
||||
@@ -11,6 +11,8 @@ export class AssetMediaResponseDto {
|
||||
status!: AssetMediaStatus;
|
||||
@ApiProperty({ description: 'Asset media ID' })
|
||||
id!: string;
|
||||
@ApiPropertyOptional({ description: 'Asset checksum (SHA1 base64)' })
|
||||
checksum?: string;
|
||||
}
|
||||
|
||||
export enum AssetUploadAction {
|
||||
|
||||
@@ -157,7 +157,7 @@ export class AssetMediaService extends BaseService {
|
||||
|
||||
await this.userRepository.updateUsage(auth.user.id, file.size);
|
||||
|
||||
return { id: asset.id, status: AssetMediaStatus.CREATED };
|
||||
return { id: asset.id, status: AssetMediaStatus.CREATED, checksum: file.checksum.toString('base64') };
|
||||
} catch (error: any) {
|
||||
return this.handleUploadError(error, auth, file, sidecarFile);
|
||||
}
|
||||
@@ -350,7 +350,7 @@ export class AssetMediaService extends BaseService {
|
||||
await this.sharedLinkRepository.addAssets(auth.sharedLink.id, [duplicateId]);
|
||||
}
|
||||
|
||||
return { status: AssetMediaStatus.DUPLICATE, id: duplicateId };
|
||||
return { status: AssetMediaStatus.DUPLICATE, id: duplicateId, checksum: file.checksum.toString('base64') };
|
||||
}
|
||||
|
||||
this.logger.error(`Error uploading file ${error}`, error?.stack);
|
||||
|
||||
Reference in New Issue
Block a user