feat(mobile): add support for encoded image requests in local/remote image APIs (#26584)

* feat(mobile): add support for encoded image requests in local and remote image APIs

* fix(mobile): handle memory cleanup for cancelled image requests

* refactor(mobile): simplify memory management and response handling for encoded image requests

* fix(mobile): correct formatting in cancellation check for image requests

* Apply suggestion from @mertalev

Co-authored-by: Mert <101130780+mertalev@users.noreply.github.com>

* refactor(mobile): rename 'encoded' parameter to 'preferEncoded' for clarity in image request APIs

* fix(mobile): ensure proper resource cleanup for cancelled image requests

* refactor(mobile): streamline codec handling by removing unnecessary descriptor disposal in loadCodec request

---------

Co-authored-by: Mert <101130780+mertalev@users.noreply.github.com>

mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImages.g.kt

@@ -59,7 +59,7 @@ private open class LocalImagesPigeonCodec : StandardMessageCodec() {
 
 /** Generated interface from Pigeon that represents a handler of messages from Flutter. */
 interface LocalImageApi {
-  fun requestImage(assetId: String, requestId: Long, width: Long, height: Long, isVideo: Boolean, callback: (Result<Map<String, Long>?>) -> Unit)
+  fun requestImage(assetId: String, requestId: Long, width: Long, height: Long, isVideo: Boolean, preferEncoded: Boolean, callback: (Result<Map<String, Long>?>) -> Unit)
   fun cancelRequest(requestId: Long)
   fun getThumbhash(thumbhash: String, callback: (Result<Map<String, Long>>) -> Unit)
 
@@ -82,7 +82,8 @@ interface LocalImageApi {
             val widthArg = args[2] as Long
             val heightArg = args[3] as Long
             val isVideoArg = args[4] as Boolean
-            api.requestImage(assetIdArg, requestIdArg, widthArg, heightArg, isVideoArg) { result: Result<Map<String, Long>?> ->
+            val preferEncodedArg = args[5] as Boolean
+            api.requestImage(assetIdArg, requestIdArg, widthArg, heightArg, isVideoArg, preferEncodedArg) { result: Result<Map<String, Long>?> ->
               val error = result.exceptionOrNull()
               if (error != null) {
                 reply.reply(LocalImagesPigeonUtils.wrapError(error))

mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt

@@ -14,6 +14,7 @@ import android.util.Size
 import androidx.annotation.RequiresApi
 import app.alextran.immich.NativeBuffer
 import kotlin.math.*
+import java.io.IOException
 import java.util.concurrent.Executors
 import com.bumptech.glide.Glide
 import com.bumptech.glide.Priority
@@ -99,12 +100,17 @@ class LocalImagesImpl(context: Context) : LocalImageApi {
     width: Long,
     height: Long,
     isVideo: Boolean,
+    preferEncoded: Boolean,
     callback: (Result<Map<String, Long>?>) -> Unit
   ) {
     val signal = CancellationSignal()
     val task = threadPool.submit {
       try {
-        getThumbnailBufferInternal(assetId, width, height, isVideo, callback, signal)
+        if (preferEncoded) {
+          getEncodedImageInternal(assetId, callback, signal)
+        } else {
+          getThumbnailBufferInternal(assetId, width, height, isVideo, callback, signal)
+        }
       } catch (e: Exception) {
         when (e) {
           is OperationCanceledException -> callback(CANCELLED)
@@ -133,6 +139,35 @@ class LocalImagesImpl(context: Context) : LocalImageApi {
     }
   }
 
+  private fun getEncodedImageInternal(
+    assetId: String,
+    callback: (Result<Map<String, Long>?>) -> Unit,
+    signal: CancellationSignal
+  ) {
+    signal.throwIfCanceled()
+    val id = assetId.toLong()
+    val uri = ContentUris.withAppendedId(Images.Media.EXTERNAL_CONTENT_URI, id)
+
+    signal.throwIfCanceled()
+    val bytes = resolver.openInputStream(uri)?.use { it.readBytes() }
+      ?: throw IOException("Could not read image data for $assetId")
+
+    signal.throwIfCanceled()
+    val pointer = NativeBuffer.allocate(bytes.size)
+    try {
+      val buffer = NativeBuffer.wrap(pointer, bytes.size)
+      buffer.put(bytes)
+      signal.throwIfCanceled()
+      callback(Result.success(mapOf(
+        "pointer" to pointer,
+        "length" to bytes.size.toLong()
+      )))
+    } catch (e: Exception) {
+      NativeBuffer.free(pointer)
+      throw e
+    }
+  }
+
   private fun getThumbnailBufferInternal(
     assetId: String,
     width: Long,

mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImages.g.kt

@@ -47,7 +47,7 @@ private open class RemoteImagesPigeonCodec : StandardMessageCodec() {
 
 /** Generated interface from Pigeon that represents a handler of messages from Flutter. */
 interface RemoteImageApi {
-  fun requestImage(url: String, headers: Map<String, String>, requestId: Long, callback: (Result<Map<String, Long>?>) -> Unit)
+  fun requestImage(url: String, headers: Map<String, String>, requestId: Long, preferEncoded: Boolean, callback: (Result<Map<String, Long>?>) -> Unit)
   fun cancelRequest(requestId: Long)
   fun clearCache(callback: (Result<Long>) -> Unit)
 
@@ -68,7 +68,8 @@ interface RemoteImageApi {
             val urlArg = args[0] as String
             val headersArg = args[1] as Map<String, String>
             val requestIdArg = args[2] as Long
-            api.requestImage(urlArg, headersArg, requestIdArg) { result: Result<Map<String, Long>?> ->
+            val preferEncodedArg = args[3] as Boolean
+            api.requestImage(urlArg, headersArg, requestIdArg, preferEncodedArg) { result: Result<Map<String, Long>?> ->
               val error = result.exceptionOrNull()
               if (error != null) {
                 reply.reply(RemoteImagesPigeonUtils.wrapError(error))

mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImagesImpl.kt

@@ -51,6 +51,7 @@ class RemoteImagesImpl(context: Context) : RemoteImageApi {
     url: String,
     headers: Map<String, String>,
     requestId: Long,
+    @Suppress("UNUSED_PARAMETER") preferEncoded: Boolean, // always returns encoded; setting has no effect on Android
     callback: (Result<Map<String, Long>?>) -> Unit
   ) {
     val signal = CancellationSignal()

mobile/ios/Runner/Images/LocalImages.g.swift

@@ -70,7 +70,7 @@ class LocalImagesPigeonCodec: FlutterStandardMessageCodec, @unchecked Sendable {
 
 /// Generated protocol from Pigeon that represents a handler of messages from Flutter.
 protocol LocalImageApi {
-  func requestImage(assetId: String, requestId: Int64, width: Int64, height: Int64, isVideo: Bool, completion: @escaping (Result<[String: Int64]?, Error>) -> Void)
+  func requestImage(assetId: String, requestId: Int64, width: Int64, height: Int64, isVideo: Bool, preferEncoded: Bool, completion: @escaping (Result<[String: Int64]?, Error>) -> Void)
   func cancelRequest(requestId: Int64) throws
   func getThumbhash(thumbhash: String, completion: @escaping (Result<[String: Int64], Error>) -> Void)
 }
@@ -90,7 +90,8 @@ class LocalImageApiSetup {
         let widthArg = args[2] as! Int64
         let heightArg = args[3] as! Int64
         let isVideoArg = args[4] as! Bool
-        api.requestImage(assetId: assetIdArg, requestId: requestIdArg, width: widthArg, height: heightArg, isVideo: isVideoArg) { result in
+        let preferEncodedArg = args[5] as! Bool
+        api.requestImage(assetId: assetIdArg, requestId: requestIdArg, width: widthArg, height: heightArg, isVideo: isVideoArg, preferEncoded: preferEncodedArg) { result in
           switch result {
           case .success(let res):
             reply(wrapResult(res))

mobile/ios/Runner/Images/LocalImagesImpl.swift

@@ -64,7 +64,7 @@ class LocalImageApiImpl: LocalImageApi {
     }
   }
 
-  func requestImage(assetId: String, requestId: Int64, width: Int64, height: Int64, isVideo: Bool, completion: @escaping (Result<[String: Int64]?, any Error>) -> Void) {
+  func requestImage(assetId: String, requestId: Int64, width: Int64, height: Int64, isVideo: Bool, preferEncoded: Bool, completion: @escaping (Result<[String: Int64]?, any Error>) -> Void) {
     let request = LocalImageRequest(callback: completion)
     let item = DispatchWorkItem {
       if request.isCancelled {
@@ -91,6 +91,49 @@ class LocalImageApiImpl: LocalImageApi {
         return completion(ImageProcessing.cancelledResult)
       }
 
+      if preferEncoded {
+        let dataOptions = PHImageRequestOptions()
+        dataOptions.isNetworkAccessAllowed = true
+        dataOptions.isSynchronous = true
+        dataOptions.version = .current
+
+        var imageData: Data?
+        Self.imageManager.requestImageDataAndOrientation(
+          for: asset,
+          options: dataOptions,
+          resultHandler: { (data, _, _, _) in
+            imageData = data
+          }
+        )
+
+        if request.isCancelled {
+          Self.remove(requestId: requestId)
+          return completion(ImageProcessing.cancelledResult)
+        }
+
+        guard let data = imageData else {
+          Self.remove(requestId: requestId)
+          return completion(.failure(PigeonError(code: "", message: "Could not get image data for \(assetId)", details: nil)))
+        }
+
+        let length = data.count
+        let pointer = malloc(length)!
+        data.copyBytes(to: pointer.assumingMemoryBound(to: UInt8.self), count: length)
+
+        if request.isCancelled {
+          free(pointer)
+          Self.remove(requestId: requestId)
+          return completion(ImageProcessing.cancelledResult)
+        }
+
+        request.callback(.success([
+          "pointer": Int64(Int(bitPattern: pointer)),
+          "length": Int64(length),
+        ]))
+        Self.remove(requestId: requestId)
+        return
+      }
+
       var image: UIImage?
       Self.imageManager.requestImage(
         for: asset,

mobile/ios/Runner/Images/RemoteImages.g.swift

@@ -70,7 +70,7 @@ class RemoteImagesPigeonCodec: FlutterStandardMessageCodec, @unchecked Sendable
 
 /// Generated protocol from Pigeon that represents a handler of messages from Flutter.
 protocol RemoteImageApi {
-  func requestImage(url: String, headers: [String: String], requestId: Int64, completion: @escaping (Result<[String: Int64]?, Error>) -> Void)
+  func requestImage(url: String, headers: [String: String], requestId: Int64, preferEncoded: Bool, completion: @escaping (Result<[String: Int64]?, Error>) -> Void)
   func cancelRequest(requestId: Int64) throws
   func clearCache(completion: @escaping (Result<Int64, Error>) -> Void)
 }
@@ -88,7 +88,8 @@ class RemoteImageApiSetup {
         let urlArg = args[0] as! String
         let headersArg = args[1] as! [String: String]
         let requestIdArg = args[2] as! Int64
-        api.requestImage(url: urlArg, headers: headersArg, requestId: requestIdArg) { result in
+        let preferEncodedArg = args[3] as! Bool
+        api.requestImage(url: urlArg, headers: headersArg, requestId: requestIdArg, preferEncoded: preferEncodedArg) { result in
           switch result {
           case .success(let res):
             reply(wrapResult(res))

mobile/ios/Runner/Images/RemoteImagesImpl.swift

@@ -33,7 +33,7 @@ class RemoteImageApiImpl: NSObject, RemoteImageApi {
     kCGImageSourceCreateThumbnailFromImageAlways: true
   ] as CFDictionary
 
-  func requestImage(url: String, headers: [String : String], requestId: Int64, completion: @escaping (Result<[String : Int64]?, any Error>) -> Void) {
+  func requestImage(url: String, headers: [String : String], requestId: Int64, preferEncoded: Bool, completion: @escaping (Result<[String : Int64]?, any Error>) -> Void) {
     var urlRequest = URLRequest(url: URL(string: url)!)
     urlRequest.cachePolicy = .returnCacheDataElseLoad
     for (key, value) in headers {
@@ -41,7 +41,7 @@ class RemoteImageApiImpl: NSObject, RemoteImageApi {
     }
 
     let task = URLSessionManager.shared.session.dataTask(with: urlRequest) { data, response, error in
-      Self.handleCompletion(requestId: requestId, data: data, response: response, error: error)
+      Self.handleCompletion(requestId: requestId, encoded: preferEncoded, data: data, response: response, error: error)
     }
 
     let request = RemoteImageRequest(id: requestId, task: task, completion: completion)
@@ -53,7 +53,7 @@ class RemoteImageApiImpl: NSObject, RemoteImageApi {
     task.resume()
   }
 
-  private static func handleCompletion(requestId: Int64, data: Data?, response: URLResponse?, error: Error?) {
+  private static func handleCompletion(requestId: Int64, encoded: Bool, data: Data?, response: URLResponse?, error: Error?) {
     os_unfair_lock_lock(&Self.lock)
     guard let request = requests[requestId] else {
       return os_unfair_lock_unlock(&Self.lock)
@@ -84,6 +84,24 @@ class RemoteImageApiImpl: NSObject, RemoteImageApi {
         return request.completion(ImageProcessing.cancelledResult)
       }
 
+      // Return raw encoded bytes when requested (for animated images)
+      if encoded {
+        let length = data.count
+        let pointer = malloc(length)!
+        data.copyBytes(to: pointer.assumingMemoryBound(to: UInt8.self), count: length)
+
+        if request.isCancelled {
+          free(pointer)
+          return request.completion(ImageProcessing.cancelledResult)
+        }
+
+        return request.completion(
+          .success([
+            "pointer": Int64(Int(bitPattern: pointer)),
+            "length": Int64(length),
+          ]))
+      }
+
       guard let imageSource = CGImageSourceCreateWithData(data as CFData, nil),
             let cgImage = CGImageSourceCreateThumbnailAtIndex(imageSource, 0, decodeOptions) else {
         return request.completion(.failure(PigeonError(code: "", message: "Failed to decode image for request", details: nil)))

mobile/lib/infrastructure/loaders/image_request.dart

@@ -24,6 +24,8 @@ abstract class ImageRequest {
 
   Future<ImageInfo?> load(ImageDecoderCallback decode, {double scale = 1.0});
 
+  Future<ui.Codec?> loadCodec();
+
   void cancel() {
     if (_isCancelled) {
       return;
@@ -34,7 +36,7 @@ abstract class ImageRequest {
 
   void _onCancelled();
 
-  Future<ui.FrameInfo?> _fromEncodedPlatformImage(int address, int length) async {
+  Future<(ui.Codec, ui.ImageDescriptor)?> _codecFromEncodedPlatformImage(int address, int length) async {
     final pointer = Pointer<Uint8>.fromAddress(address);
     if (_isCancelled) {
       malloc.free(pointer);
@@ -67,6 +69,20 @@ abstract class ImageRequest {
       return null;
     }
 
+    return (codec, descriptor);
+  }
+
+  Future<ui.FrameInfo?> _fromEncodedPlatformImage(int address, int length) async {
+    final result = await _codecFromEncodedPlatformImage(address, length);
+    if (result == null) return null;
+
+    final (codec, descriptor) = result;
+    if (_isCancelled) {
+      descriptor.dispose();
+      codec.dispose();
+      return null;
+    }
+
     final frame = await codec.getNextFrame();
     descriptor.dispose();
     codec.dispose();

mobile/lib/infrastructure/loaders/local_image_request.dart

@@ -22,6 +22,7 @@ class LocalImageRequest extends ImageRequest {
       width: width,
       height: height,
       isVideo: assetType == AssetType.video,
+      preferEncoded: false,
     );
     if (info == null) {
       return null;
@@ -31,6 +32,26 @@ class LocalImageRequest extends ImageRequest {
     return frame == null ? null : ImageInfo(image: frame.image, scale: scale);
   }
 
+  @override
+  Future<ui.Codec?> loadCodec() async {
+    if (_isCancelled) {
+      return null;
+    }
+
+    final info = await localImageApi.requestImage(
+      localId,
+      requestId: requestId,
+      width: width,
+      height: height,
+      isVideo: assetType == AssetType.video,
+      preferEncoded: true,
+    );
+    if (info == null) return null;
+
+    final (codec, _) = await _codecFromEncodedPlatformImage(info['pointer']!, info['length']!) ?? (null, null);
+    return codec;
+  }
+
   @override
   Future<void> _onCancelled() {
     return localImageApi.cancelRequest(requestId);

mobile/lib/infrastructure/loaders/remote_image_request.dart

@@ -12,7 +12,8 @@ class RemoteImageRequest extends ImageRequest {
       return null;
     }
 
-    final info = await remoteImageApi.requestImage(uri, headers: headers, requestId: requestId);
+    final info = await remoteImageApi.requestImage(uri, headers: headers, requestId: requestId, preferEncoded: false);
+    // Android always returns encoded data, so we need to check for both shapes of the response.
     final frame = switch (info) {
       {'pointer': int pointer, 'length': int length} => await _fromEncodedPlatformImage(pointer, length),
       {'pointer': int pointer, 'width': int width, 'height': int height, 'rowBytes': int rowBytes} =>
@@ -22,6 +23,19 @@ class RemoteImageRequest extends ImageRequest {
     return frame == null ? null : ImageInfo(image: frame.image, scale: scale);
   }
 
+  @override
+  Future<ui.Codec?> loadCodec() async {
+    if (_isCancelled) {
+      return null;
+    }
+
+    final info = await remoteImageApi.requestImage(uri, headers: headers, requestId: requestId, preferEncoded: true);
+    if (info == null) return null;
+
+    final (codec, _) = await _codecFromEncodedPlatformImage(info['pointer']!, info['length']!) ?? (null, null);
+    return codec;
+  }
+
   @override
   Future<void> _onCancelled() {
     return remoteImageApi.cancelRequest(requestId);

mobile/lib/infrastructure/loaders/thumbhash_image_request.dart

@@ -16,6 +16,9 @@ class ThumbhashImageRequest extends ImageRequest {
     return frame == null ? null : ImageInfo(image: frame.image, scale: scale);
   }
 
+  @override
+  Future<ui.Codec?> loadCodec() => throw UnsupportedError('Thumbhash does not support codec loading');
+
   @override
   void _onCancelled() {}
 }

mobile/lib/platform/local_image_api.g.dart

@@ -55,6 +55,7 @@ class LocalImageApi {
     required int width,
     required int height,
     required bool isVideo,
+    required bool preferEncoded,
   }) async {
     final String pigeonVar_channelName =
         'dev.flutter.pigeon.immich_mobile.LocalImageApi.requestImage$pigeonVar_messageChannelSuffix';
@@ -69,6 +70,7 @@ class LocalImageApi {
       width,
       height,
       isVideo,
+      preferEncoded,
     ]);
     final List<Object?>? pigeonVar_replyList = await pigeonVar_sendFuture as List<Object?>?;
     if (pigeonVar_replyList == null) {

mobile/lib/platform/remote_image_api.g.dart

@@ -53,6 +53,7 @@ class RemoteImageApi {
     String url, {
     required Map<String, String> headers,
     required int requestId,
+    required bool preferEncoded,
   }) async {
     final String pigeonVar_channelName =
         'dev.flutter.pigeon.immich_mobile.RemoteImageApi.requestImage$pigeonVar_messageChannelSuffix';
@@ -61,7 +62,12 @@ class RemoteImageApi {
       pigeonChannelCodec,
       binaryMessenger: pigeonVar_binaryMessenger,
     );
-    final Future<Object?> pigeonVar_sendFuture = pigeonVar_channel.send(<Object?>[url, headers, requestId]);
+    final Future<Object?> pigeonVar_sendFuture = pigeonVar_channel.send(<Object?>[
+      url,
+      headers,
+      requestId,
+      preferEncoded,
+    ]);
     final List<Object?>? pigeonVar_replyList = await pigeonVar_sendFuture as List<Object?>?;
     if (pigeonVar_replyList == null) {
       throw _createConnectionError(pigeonVar_channelName);

mobile/lib/presentation/widgets/images/image_provider.dart

@@ -1,3 +1,5 @@
+import 'dart:ui' as ui;
+
 import 'package:async/async.dart';
 import 'package:flutter/widgets.dart';
 import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
@@ -75,6 +77,29 @@ mixin CancellableImageProviderMixin<T extends Object> on CancellableImageProvide
     }
   }
 
+  Future<ui.Codec?> loadCodecRequest(ImageRequest request) async {
+    if (isCancelled) {
+      this.request = null;
+      PaintingBinding.instance.imageCache.evict(this);
+      return null;
+    }
+
+    try {
+      final codec = await request.loadCodec();
+      if (codec == null || isCancelled) {
+        codec?.dispose();
+        PaintingBinding.instance.imageCache.evict(this);
+        return null;
+      }
+      return codec;
+    } catch (e) {
+      PaintingBinding.instance.imageCache.evict(this);
+      rethrow;
+    } finally {
+      this.request = null;
+    }
+  }
+
   Stream<ImageInfo> initialImageStream() async* {
     final cachedOperation = this.cachedOperation;
     if (cachedOperation == null) {

mobile/pigeon/local_image_api.dart

@@ -21,6 +21,7 @@ abstract class LocalImageApi {
     required int width,
     required int height,
     required bool isVideo,
+    required bool preferEncoded,
   });
 
   void cancelRequest(int requestId);

mobile/pigeon/remote_image_api.dart

@@ -19,6 +19,7 @@ abstract class RemoteImageApi {
     String url, {
     required Map<String, String> headers,
     required int requestId,
+    required bool preferEncoded,
   });
 
   void cancelRequest(int requestId);