mirror of
https://github.com/immich-app/immich.git
synced 2026-03-22 11:09:21 +03:00
feat: skip local hashing for iCloud assets, use server-computed checksum
Instead of downloading iCloud assets twice (once to hash, once to upload), skip local hashing for iCloud-only assets and let the server return its computed SHA1 checksum in the upload response. The mobile app stores this checksum locally to prevent re-uploads. Changes: - Server returns checksum in upload response (created + duplicate) - iOS native layer tags iCloud-only assets with ICLOUD_ONLY error - Hash service skips iCloud assets (allowNetworkAccess: false) - Upload result carries server checksum back to mobile - Foreground/background upload services store server checksum - Backup candidates include unhashed assets (onlyHashed: false) https://claude.ai/code/session_01LEs74WpkJ1gWcJrFBsp1i2
This commit is contained in:
@@ -357,6 +357,12 @@ class NativeSyncApiImpl: ImmichPlugin, NativeSyncApi, FlutterPlugin {
|
|||||||
completionHandler: { error in
|
completionHandler: { error in
|
||||||
let result: HashResult? = switch (error) {
|
let result: HashResult? = switch (error) {
|
||||||
case let e as PHPhotosError where e.code == .userCancelled: nil
|
case let e as PHPhotosError where e.code == .userCancelled: nil
|
||||||
|
case let e as PHPhotosError where e.code == .networkAccessRequired:
|
||||||
|
HashResult(
|
||||||
|
assetId: asset.localIdentifier,
|
||||||
|
error: "ICLOUD_ONLY",
|
||||||
|
hash: nil
|
||||||
|
)
|
||||||
case let .some(e): HashResult(
|
case let .some(e): HashResult(
|
||||||
assetId: asset.localIdentifier,
|
assetId: asset.localIdentifier,
|
||||||
error: "Failed to hash asset: \(e.localizedDescription)",
|
error: "Failed to hash asset: \(e.localizedDescription)",
|
||||||
|
|||||||
@@ -109,9 +109,11 @@ class HashService {
|
|||||||
_log.fine("Hashing ${toHash.length} files");
|
_log.fine("Hashing ${toHash.length} files");
|
||||||
|
|
||||||
final hashed = <String, String>{};
|
final hashed = <String, String>{};
|
||||||
|
// Never download from iCloud just to hash. iCloud-only assets will be
|
||||||
|
// uploaded directly and the server will compute + return their checksum.
|
||||||
final hashResults = await _nativeSyncApi.hashAssets(
|
final hashResults = await _nativeSyncApi.hashAssets(
|
||||||
toHash.keys.toList(),
|
toHash.keys.toList(),
|
||||||
allowNetworkAccess: album.backupSelection == BackupSelection.selected,
|
allowNetworkAccess: false,
|
||||||
);
|
);
|
||||||
assert(
|
assert(
|
||||||
hashResults.length == toHash.length,
|
hashResults.length == toHash.length,
|
||||||
@@ -127,6 +129,10 @@ class HashService {
|
|||||||
final hashResult = hashResults[i];
|
final hashResult = hashResults[i];
|
||||||
if (hashResult.hash != null) {
|
if (hashResult.hash != null) {
|
||||||
hashed[hashResult.assetId] = hashResult.hash!;
|
hashed[hashResult.assetId] = hashResult.hash!;
|
||||||
|
} else if (hashResult.error == 'ICLOUD_ONLY') {
|
||||||
|
// Asset is in iCloud and not available locally. It will be uploaded
|
||||||
|
// directly and the server will compute its checksum.
|
||||||
|
_log.fine("Skipping iCloud-only asset ${hashResult.assetId} from album: ${album.name}");
|
||||||
} else {
|
} else {
|
||||||
final asset = toHash[hashResult.assetId];
|
final asset = toHash[hashResult.assetId];
|
||||||
_log.warning(
|
_log.warning(
|
||||||
|
|||||||
@@ -136,7 +136,10 @@ class UploadRepository {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
final responseBody = jsonDecode(responseBodyString);
|
final responseBody = jsonDecode(responseBodyString);
|
||||||
return UploadResult.success(remoteAssetId: responseBody['id'] as String);
|
return UploadResult.success(
|
||||||
|
remoteAssetId: responseBody['id'] as String,
|
||||||
|
checksum: responseBody['checksum'] as String?,
|
||||||
|
);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
return UploadResult.error(errorMessage: 'Failed to parse server response');
|
return UploadResult.error(errorMessage: 'Failed to parse server response');
|
||||||
}
|
}
|
||||||
@@ -182,6 +185,7 @@ class UploadResult {
|
|||||||
final bool isSuccess;
|
final bool isSuccess;
|
||||||
final bool isCancelled;
|
final bool isCancelled;
|
||||||
final String? remoteAssetId;
|
final String? remoteAssetId;
|
||||||
|
final String? checksum;
|
||||||
final String? errorMessage;
|
final String? errorMessage;
|
||||||
final int? statusCode;
|
final int? statusCode;
|
||||||
|
|
||||||
@@ -189,12 +193,13 @@ class UploadResult {
|
|||||||
required this.isSuccess,
|
required this.isSuccess,
|
||||||
required this.isCancelled,
|
required this.isCancelled,
|
||||||
this.remoteAssetId,
|
this.remoteAssetId,
|
||||||
|
this.checksum,
|
||||||
this.errorMessage,
|
this.errorMessage,
|
||||||
this.statusCode,
|
this.statusCode,
|
||||||
});
|
});
|
||||||
|
|
||||||
factory UploadResult.success({required String remoteAssetId}) {
|
factory UploadResult.success({required String remoteAssetId, String? checksum}) {
|
||||||
return UploadResult(isSuccess: true, isCancelled: false, remoteAssetId: remoteAssetId);
|
return UploadResult(isSuccess: true, isCancelled: false, remoteAssetId: remoteAssetId, checksum: checksum);
|
||||||
}
|
}
|
||||||
|
|
||||||
factory UploadResult.error({String? errorMessage, int? statusCode}) {
|
factory UploadResult.error({String? errorMessage, int? statusCode}) {
|
||||||
|
|||||||
@@ -162,7 +162,7 @@ class BackgroundUploadService {
|
|||||||
await _storageRepository.clearCache();
|
await _storageRepository.clearCache();
|
||||||
shouldAbortQueuingTasks = false;
|
shouldAbortQueuingTasks = false;
|
||||||
|
|
||||||
final candidates = await _backupRepository.getCandidates(userId);
|
final candidates = await _backupRepository.getCandidates(userId, onlyHashed: false);
|
||||||
if (candidates.isEmpty) {
|
if (candidates.isEmpty) {
|
||||||
_logger.info("No new backup candidates found, finishing background upload");
|
_logger.info("No new backup candidates found, finishing background upload");
|
||||||
return;
|
return;
|
||||||
@@ -210,6 +210,7 @@ class BackgroundUploadService {
|
|||||||
switch (update.status) {
|
switch (update.status) {
|
||||||
case TaskStatus.complete:
|
case TaskStatus.complete:
|
||||||
unawaited(_handleLivePhoto(update));
|
unawaited(_handleLivePhoto(update));
|
||||||
|
unawaited(_storeServerChecksum(update));
|
||||||
|
|
||||||
if (CurrentPlatform.isIOS) {
|
if (CurrentPlatform.isIOS) {
|
||||||
try {
|
try {
|
||||||
@@ -227,6 +228,20 @@ class BackgroundUploadService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Future<void> _storeServerChecksum(TaskStatusUpdate update) async {
|
||||||
|
try {
|
||||||
|
if (update.responseBody == null || update.responseBody!.isEmpty) return;
|
||||||
|
final response = jsonDecode(update.responseBody!);
|
||||||
|
final checksum = response['checksum'] as String?;
|
||||||
|
if (checksum == null) return;
|
||||||
|
final deviceAssetId = update.task.taskId;
|
||||||
|
if (deviceAssetId.isEmpty) return;
|
||||||
|
await _localAssetRepository.updateHashes({deviceAssetId: checksum});
|
||||||
|
} catch (e) {
|
||||||
|
_logger.warning('Failed to store server checksum: $e');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Future<void> _handleLivePhoto(TaskStatusUpdate update) async {
|
Future<void> _handleLivePhoto(TaskStatusUpdate update) async {
|
||||||
try {
|
try {
|
||||||
if (update.task.metaData.isEmpty || update.task.metaData == '') {
|
if (update.task.metaData.isEmpty || update.task.metaData == '') {
|
||||||
|
|||||||
@@ -11,9 +11,11 @@ import 'package:immich_mobile/extensions/platform_extensions.dart';
|
|||||||
import 'package:immich_mobile/extensions/network_capability_extensions.dart';
|
import 'package:immich_mobile/extensions/network_capability_extensions.dart';
|
||||||
import 'package:immich_mobile/extensions/translate_extensions.dart';
|
import 'package:immich_mobile/extensions/translate_extensions.dart';
|
||||||
import 'package:immich_mobile/infrastructure/repositories/backup.repository.dart';
|
import 'package:immich_mobile/infrastructure/repositories/backup.repository.dart';
|
||||||
|
import 'package:immich_mobile/infrastructure/repositories/local_asset.repository.dart';
|
||||||
import 'package:immich_mobile/infrastructure/repositories/storage.repository.dart';
|
import 'package:immich_mobile/infrastructure/repositories/storage.repository.dart';
|
||||||
import 'package:immich_mobile/platform/connectivity_api.g.dart';
|
import 'package:immich_mobile/platform/connectivity_api.g.dart';
|
||||||
import 'package:immich_mobile/providers/app_settings.provider.dart';
|
import 'package:immich_mobile/providers/app_settings.provider.dart';
|
||||||
|
import 'package:immich_mobile/providers/infrastructure/asset.provider.dart';
|
||||||
import 'package:immich_mobile/providers/infrastructure/platform.provider.dart';
|
import 'package:immich_mobile/providers/infrastructure/platform.provider.dart';
|
||||||
import 'package:immich_mobile/providers/infrastructure/storage.provider.dart';
|
import 'package:immich_mobile/providers/infrastructure/storage.provider.dart';
|
||||||
import 'package:immich_mobile/repositories/asset_media.repository.dart';
|
import 'package:immich_mobile/repositories/asset_media.repository.dart';
|
||||||
@@ -37,6 +39,7 @@ final foregroundUploadServiceProvider = Provider((ref) {
|
|||||||
return ForegroundUploadService(
|
return ForegroundUploadService(
|
||||||
ref.watch(uploadRepositoryProvider),
|
ref.watch(uploadRepositoryProvider),
|
||||||
ref.watch(storageRepositoryProvider),
|
ref.watch(storageRepositoryProvider),
|
||||||
|
ref.watch(localAssetRepository),
|
||||||
ref.watch(backupRepositoryProvider),
|
ref.watch(backupRepositoryProvider),
|
||||||
ref.watch(connectivityApiProvider),
|
ref.watch(connectivityApiProvider),
|
||||||
ref.watch(appSettingsServiceProvider),
|
ref.watch(appSettingsServiceProvider),
|
||||||
@@ -53,6 +56,7 @@ class ForegroundUploadService {
|
|||||||
ForegroundUploadService(
|
ForegroundUploadService(
|
||||||
this._uploadRepository,
|
this._uploadRepository,
|
||||||
this._storageRepository,
|
this._storageRepository,
|
||||||
|
this._localAssetRepository,
|
||||||
this._backupRepository,
|
this._backupRepository,
|
||||||
this._connectivityApi,
|
this._connectivityApi,
|
||||||
this._appSettingsService,
|
this._appSettingsService,
|
||||||
@@ -61,6 +65,7 @@ class ForegroundUploadService {
|
|||||||
|
|
||||||
final UploadRepository _uploadRepository;
|
final UploadRepository _uploadRepository;
|
||||||
final StorageRepository _storageRepository;
|
final StorageRepository _storageRepository;
|
||||||
|
final DriftLocalAssetRepository _localAssetRepository;
|
||||||
final DriftBackupRepository _backupRepository;
|
final DriftBackupRepository _backupRepository;
|
||||||
final ConnectivityApi _connectivityApi;
|
final ConnectivityApi _connectivityApi;
|
||||||
final AppSettingsService _appSettingsService;
|
final AppSettingsService _appSettingsService;
|
||||||
@@ -84,7 +89,7 @@ class ForegroundUploadService {
|
|||||||
UploadCallbacks callbacks = const UploadCallbacks(),
|
UploadCallbacks callbacks = const UploadCallbacks(),
|
||||||
bool useSequentialUpload = false,
|
bool useSequentialUpload = false,
|
||||||
}) async {
|
}) async {
|
||||||
final candidates = await _backupRepository.getCandidates(userId);
|
final candidates = await _backupRepository.getCandidates(userId, onlyHashed: false);
|
||||||
if (candidates.isEmpty) {
|
if (candidates.isEmpty) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -387,6 +392,10 @@ class ForegroundUploadService {
|
|||||||
);
|
);
|
||||||
|
|
||||||
if (result.isSuccess && result.remoteAssetId != null) {
|
if (result.isSuccess && result.remoteAssetId != null) {
|
||||||
|
// Store server-computed checksum so iCloud-only assets don't need re-upload
|
||||||
|
if (result.checksum != null) {
|
||||||
|
await _localAssetRepository.updateHashes({asset.id: result.checksum!});
|
||||||
|
}
|
||||||
callbacks.onSuccess?.call(asset.localId!, result.remoteAssetId!);
|
callbacks.onSuccess?.call(asset.localId!, result.remoteAssetId!);
|
||||||
} else if (result.isCancelled) {
|
} else if (result.isCancelled) {
|
||||||
_logger.warning(() => "Backup was cancelled by the user");
|
_logger.warning(() => "Backup was cancelled by the user");
|
||||||
|
|||||||
@@ -11,6 +11,8 @@ export class AssetMediaResponseDto {
|
|||||||
status!: AssetMediaStatus;
|
status!: AssetMediaStatus;
|
||||||
@ApiProperty({ description: 'Asset media ID' })
|
@ApiProperty({ description: 'Asset media ID' })
|
||||||
id!: string;
|
id!: string;
|
||||||
|
@ApiPropertyOptional({ description: 'Asset checksum (SHA1 base64)' })
|
||||||
|
checksum?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export enum AssetUploadAction {
|
export enum AssetUploadAction {
|
||||||
|
|||||||
@@ -157,7 +157,7 @@ export class AssetMediaService extends BaseService {
|
|||||||
|
|
||||||
await this.userRepository.updateUsage(auth.user.id, file.size);
|
await this.userRepository.updateUsage(auth.user.id, file.size);
|
||||||
|
|
||||||
return { id: asset.id, status: AssetMediaStatus.CREATED };
|
return { id: asset.id, status: AssetMediaStatus.CREATED, checksum: file.checksum.toString('base64') };
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
return this.handleUploadError(error, auth, file, sidecarFile);
|
return this.handleUploadError(error, auth, file, sidecarFile);
|
||||||
}
|
}
|
||||||
@@ -350,7 +350,7 @@ export class AssetMediaService extends BaseService {
|
|||||||
await this.sharedLinkRepository.addAssets(auth.sharedLink.id, [duplicateId]);
|
await this.sharedLinkRepository.addAssets(auth.sharedLink.id, [duplicateId]);
|
||||||
}
|
}
|
||||||
|
|
||||||
return { status: AssetMediaStatus.DUPLICATE, id: duplicateId };
|
return { status: AssetMediaStatus.DUPLICATE, id: duplicateId, checksum: file.checksum.toString('base64') };
|
||||||
}
|
}
|
||||||
|
|
||||||
this.logger.error(`Error uploading file ${error}`, error?.stack);
|
this.logger.error(`Error uploading file ${error}`, error?.stack);
|
||||||
|
|||||||
Reference in New Issue
Block a user