memory optimization

mobile/android/app/CMakeLists.txt

@@ -8,3 +8,5 @@ project(native_buffer LANGUAGES C)
 add_library(native_buffer SHARED
     src/main/cpp/native_buffer.c
 )
+
+target_link_libraries(native_buffer jnigraphics)

mobile/android/app/src/main/cpp/native_buffer.c

@@ -1,40 +1,39 @@
 #include <jni.h>
 #include <stdlib.h>
+#include <android/bitmap.h>
 
 JNIEXPORT jlong JNICALL
-Java_app_alextran_immich_images_ThumbnailsImpl_00024Companion_allocateNative(
+Java_app_alextran_immich_images_LocalImagesImpl_allocateNative(
-        JNIEnv *env, jclass clazz, jint size) {
-    void *ptr = malloc(size);
-    return (jlong) ptr;
-}
-
-JNIEXPORT jlong JNICALL
-Java_app_alextran_immich_images_ThumbnailsImpl_allocateNative(
         JNIEnv *env, jclass clazz, jint size) {
     void *ptr = malloc(size);
     return (jlong) ptr;
 }
 
 JNIEXPORT void JNICALL
-Java_app_alextran_immich_images_ThumbnailsImpl_00024Companion_freeNative(
+Java_app_alextran_immich_images_LocalImagesImpl_freeNative(
         JNIEnv *env, jclass clazz, jlong address) {
     free((void *) address);
 }
 
+JNIEXPORT jobject JNICALL
+Java_app_alextran_immich_images_LocalImagesImpl_wrapAsBuffer(
+        JNIEnv *env, jclass clazz, jlong address, jint capacity) {
+    return (*env)->NewDirectByteBuffer(env, (void *) address, capacity);
+}
+
+JNIEXPORT jlong JNICALL
+Java_app_alextran_immich_images_RemoteImagesImpl_lockBitmapPixels(
+        JNIEnv *env, jclass clazz, jobject bitmap) {
+    void *pixels = NULL;
+    int result = AndroidBitmap_lockPixels(env, bitmap, &pixels);
+    if (result != ANDROID_BITMAP_RESULT_SUCCESS) {
+        return 0;
+    }
+    return (jlong) pixels;
+}
+
 JNIEXPORT void JNICALL
-Java_app_alextran_immich_images_ThumbnailsImpl_freeNative(
+Java_app_alextran_immich_images_RemoteImagesImpl_unlockBitmapPixels(
-        JNIEnv *env, jclass clazz, jlong address) {
+        JNIEnv *env, jclass clazz, jobject bitmap) {
-    free((void *) address);
+    AndroidBitmap_unlockPixels(env, bitmap);
-}
-
-JNIEXPORT jobject JNICALL
-Java_app_alextran_immich_images_ThumbnailsImpl_00024Companion_wrapAsBuffer(
-        JNIEnv *env, jclass clazz, jlong address, jint capacity) {
-    return (*env)->NewDirectByteBuffer(env, (void *) address, capacity);
-}
-
-JNIEXPORT jobject JNICALL
-Java_app_alextran_immich_images_ThumbnailsImpl_wrapAsBuffer(
-        JNIEnv *env, jclass clazz, jlong address, jint capacity) {
-    return (*env)->NewDirectByteBuffer(env, (void *) address, capacity);
 }

mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt

@@ -12,9 +12,6 @@ import android.provider.MediaStore.Images
 import android.provider.MediaStore.Video
 import android.util.Size
 import androidx.annotation.RequiresApi
-import app.alextran.immich.images.LocalImagesImpl.Companion.allocateNative
-import app.alextran.immich.images.LocalImagesImpl.Companion.freeNative
-import app.alextran.immich.images.LocalImagesImpl.Companion.wrapAsBuffer
 import java.nio.ByteBuffer
 import kotlin.math.*
 import java.util.concurrent.Executors
@@ -47,9 +44,9 @@ inline fun ImageDecoder.Source.decodeBitmap(target: Size = Size(0, 0)): Bitmap {
 
 fun Bitmap.toNativeBuffer(): Map<String, Long>  {
   val size = width * height * 4
-  val pointer = allocateNative(size)
+  val pointer = LocalImagesImpl.allocateNative(size)
   try {
-    val buffer = wrapAsBuffer(pointer, size)
+    val buffer = LocalImagesImpl.wrapAsBuffer(pointer, size)
     copyPixelsToBuffer(buffer)
     recycle()
     return mapOf(
@@ -58,7 +55,7 @@ fun Bitmap.toNativeBuffer(): Map<String, Long>  {
       "height" to height.toLong()
     )
   } catch (e: Exception) {
-    freeNative(pointer)
+    LocalImagesImpl.freeNative(pointer)
     recycle()
     throw e
   }

mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImages.g.kt

@@ -49,6 +49,7 @@ private open class RemoteImagesPigeonCodec : StandardMessageCodec() {
 interface RemoteImageApi {
   fun requestImage(url: String, headers: Map<String, String>, requestId: Long, callback: (Result<Map<String, Long>>) -> Unit)
   fun cancelRequest(requestId: Long)
+  fun releaseImage(requestId: Long)
 
   companion object {
     /** The codec used by RemoteImageApi. */
@@ -99,6 +100,24 @@ interface RemoteImageApi {
           channel.setMessageHandler(null)
         }
       }
+      run {
+        val channel = BasicMessageChannel<Any?>(binaryMessenger, "dev.flutter.pigeon.immich_mobile.RemoteImageApi.releaseImage$separatedMessageChannelSuffix", codec)
+        if (api != null) {
+          channel.setMessageHandler { message, reply ->
+            val args = message as List<Any?>
+            val requestIdArg = args[0] as Long
+            val wrapped: List<Any?> = try {
+              api.releaseImage(requestIdArg)
+              listOf(null)
+            } catch (exception: Throwable) {
+              RemoteImagesPigeonUtils.wrapError(exception)
+            }
+            reply.reply(wrapped)
+          }
+        } else {
+          channel.setMessageHandler(null)
+        }
+      }
     }
   }
 }

mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImagesImpl.kt

@@ -7,7 +7,6 @@ import android.graphics.ColorSpace
 import android.graphics.ImageDecoder
 import android.os.Build
 import android.os.CancellationSignal
-import android.util.Size
 import app.alextran.immich.core.SSLConfig
 import okhttp3.Call
 import okhttp3.Callback
@@ -31,9 +30,9 @@ data class RemoteRequest(
 
 class RemoteImagesImpl(context: Context) : RemoteImageApi {
   private val requestMap = ConcurrentHashMap<Long, RemoteRequest>()
+  private val lockedBitmaps = ConcurrentHashMap<Long, Bitmap>()
 
   init {
-    System.loadLibrary("native_buffer")
     cacheDir = context.cacheDir
     client = buildClient()
   }
@@ -45,17 +44,23 @@ class RemoteImagesImpl(context: Context) : RemoteImageApi {
     private const val CACHE_SIZE_BYTES = 1024L * 1024 * 1024
 
     val CANCELLED = Result.success<Map<String, Long>>(emptyMap())
-    private val decodePool = Executors.newFixedThreadPool(
+    private val decodePool =
-      (Runtime.getRuntime().availableProcessors() / 2).coerceAtLeast(2)
+      Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() / 2 + 1)
-    )
 
     private var cacheDir: File? = null
     private var client: OkHttpClient? = null
 
     init {
+      System.loadLibrary("native_buffer")
       SSLConfig.addListener(::invalidateClient)
     }
 
+    @JvmStatic
+    external fun lockBitmapPixels(bitmap: Bitmap): Long
+
+    @JvmStatic
+    external fun unlockBitmapPixels(bitmap: Bitmap)
+
     private fun invalidateClient() {
       client?.let {
         it.dispatcher.cancelAll()
@@ -123,8 +128,20 @@ class RemoteImagesImpl(context: Context) : RemoteImageApi {
             signal.throwIfCanceled()
             val bitmap = decodeImage(bytes)
             signal.throwIfCanceled()
-            val res = bitmap.toNativeBuffer()
+
-            callback(Result.success(res))
+            val pointer = lockBitmapPixels(bitmap)
+            if (pointer == 0L) {
+              bitmap.recycle()
+              return@execute callback(Result.failure(RuntimeException("Failed to lock bitmap pixels")))
+            }
+
+            lockedBitmaps[requestId] = bitmap
+            callback(Result.success(mapOf(
+              "pointer" to pointer,
+              "width" to bitmap.width.toLong(),
+              "height" to bitmap.height.toLong(),
+              "rowBytes" to bitmap.rowBytes.toLong()
+            )))
           } catch (e: Exception) {
             val result = if (signal.isCanceled) CANCELLED else Result.failure(e)
             callback(result)
@@ -138,8 +155,14 @@ class RemoteImagesImpl(context: Context) : RemoteImageApi {
   }
 
   override fun cancelRequest(requestId: Long) {
-    val request = requestMap.remove(requestId) ?: return
+    requestMap.remove(requestId)?.cancellationSignal?.cancel()
-    request.cancellationSignal.cancel()
+    releaseImage(requestId)
+  }
+
+  override fun releaseImage(requestId: Long) {
+    val bitmap = lockedBitmaps.remove(requestId) ?: return
+    unlockBitmapPixels(bitmap)
+    bitmap.recycle()
   }
 
   private fun decodeImage(bytes: ByteArray): Bitmap {

mobile/ios/Runner/Images/RemoteImages.g.swift

@@ -72,6 +72,7 @@ class RemoteImagesPigeonCodec: FlutterStandardMessageCodec, @unchecked Sendable
 protocol RemoteImageApi {
   func requestImage(url: String, headers: [String: String], requestId: Int64, completion: @escaping (Result<[String: Int64], Error>) -> Void)
   func cancelRequest(requestId: Int64) throws
+  func releaseImage(requestId: Int64) throws
 }
 
 /// Generated setup class from Pigeon to handle messages through the `binaryMessenger`.
@@ -114,5 +115,20 @@ class RemoteImageApiSetup {
     } else {
       cancelRequestChannel.setMessageHandler(nil)
     }
+    let releaseImageChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.RemoteImageApi.releaseImage\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
+    if let api = api {
+      releaseImageChannel.setMessageHandler { message, reply in
+        let args = message as! [Any?]
+        let requestIdArg = args[0] as! Int64
+        do {
+          try api.releaseImage(requestId: requestIdArg)
+          reply(wrapResult(nil))
+        } catch {
+          reply(wrapError(error))
+        }
+      }
+    } else {
+      releaseImageChannel.setMessageHandler(nil)
+    }
   }
 }

mobile/ios/Runner/Images/RemoteImagesImpl.swift

@@ -1,4 +1,5 @@
-import Accelerate
+import CoreImage
+import CoreVideo
 import Flutter
 import MobileCoreServices
 import Photos
@@ -20,7 +21,7 @@ class RemoteImageApiImpl: NSObject, RemoteImageApi {
   private static let delegate = RemoteImageApiDelegate()
   static let session = {
     let config = URLSessionConfiguration.default
-    let thumbnailPath = FileManager.default.temporaryDirectory.appendingPathComponent("thumbnails", isDirectory: true)
+    let thumbnailPath = FileManager.default.temporaryDirectory.appendingPathComponent("thumbnails2", isDirectory: true)
     try! FileManager.default.createDirectory(at: thumbnailPath, withIntermediateDirectories: true)
     config.urlCache = URLCache(
       memoryCapacity: 0,
@@ -47,25 +48,20 @@ class RemoteImageApiImpl: NSObject, RemoteImageApi {
   
   func cancelRequest(requestId: Int64) {
     Self.delegate.cancel(requestId: requestId)
+    Self.delegate.releasePixelBuffer(requestId: requestId)
+  }
+  
+  func releaseImage(requestId: Int64) {
+    Self.delegate.releasePixelBuffer(requestId: requestId)
   }
 }
 
 class RemoteImageApiDelegate: NSObject, URLSessionDataDelegate {
   private static let requestQueue = DispatchQueue(label: "thumbnail.requests", qos: .userInitiated)
-  private static var rgbaFormat = vImage_CGImageFormat(
-    bitsPerComponent: 8,
-    bitsPerPixel: 32,
-    colorSpace: CGColorSpaceCreateDeviceRGB(),
-    bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue),
-    renderingIntent: .perceptual
-  )!
   private static var requests = [Int64: RemoteImageRequest]()
+  private static var lockedPixelBuffers = [Int64: CVPixelBuffer]()
   private static let cancelledResult = Result<[String: Int64], any Error>.success([:])
-  private static let decodeOptions = [
+  private static let ciContext = CIContext(options: [.useSoftwareRenderer: false])
-    kCGImageSourceShouldCache: false,
-    kCGImageSourceShouldCacheImmediately: true,
-    kCGImageSourceCreateThumbnailWithTransform: true,
-  ] as CFDictionary
   
   func urlSession(
     _ session: URLSession, dataTask: URLSessionDataTask,
@@ -108,11 +104,13 @@ class RemoteImageApiDelegate: NSObject, URLSessionDataDelegate {
     defer { remove(requestId: requestId) }
     
     if let error = error {
+      if request.isCancelled || (error as NSError).code == NSURLErrorCancelled {
+        return request.completion(Self.cancelledResult)
+      }
       return request.completion(.failure(error))
     }
     
-    guard let imageSource = CGImageSourceCreateWithData(data, nil),
+    guard let ciImage = CIImage(data: data as Data, options: [.applyOrientationProperty: true]) else {
-          let cgImage = CGImageSourceCreateThumbnailAtIndex(imageSource, 0, Self.decodeOptions) else {
       return request.completion(.failure(PigeonError(code: "", message: "Failed to decode image for request \(requestId)", details: nil)))
     }
     
@@ -120,24 +118,52 @@ class RemoteImageApiDelegate: NSObject, URLSessionDataDelegate {
       return request.completion(Self.cancelledResult)
     }
     
-    do {
+    let extent = ciImage.extent
-      let buffer = try vImage_Buffer(cgImage: cgImage, format: Self.rgbaFormat)
+    let width = Int(extent.width)
+    let height = Int(extent.height)
     
-      if request.isCancelled {
+    guard width > 0 && height > 0 else {
-        buffer.free()
+      return request.completion(.failure(PigeonError(code: "", message: "Invalid image dimensions \(width)x\(height) for request \(requestId)", details: nil)))
-        return request.completion(Self.cancelledResult)
-      }
-      
-      request.completion(
-        .success([
-          "pointer": Int64(Int(bitPattern: buffer.data)),
-          "width": Int64(buffer.width),
-          "height": Int64(buffer.height),
-          "rowBytes": Int64(buffer.rowBytes),
-        ]))
-    } catch {
-      return request.completion(.failure(PigeonError(code: "", message: "Failed to convert image for request \(requestId): \(error)", details: nil)))
     }
+    
+    var pixelBuffer: CVPixelBuffer?
+    let attrs: [String: Any] = [
+      kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
+      kCVPixelBufferWidthKey as String: width,
+      kCVPixelBufferHeightKey as String: height,
+      kCVPixelBufferIOSurfacePropertiesKey as String: [:],
+    ]
+    let status = CVPixelBufferCreate(kCFAllocatorDefault, width, height, kCVPixelFormatType_32BGRA, attrs as CFDictionary, &pixelBuffer)
+    
+    guard status == kCVReturnSuccess, let pixelBuffer = pixelBuffer else {
+      return request.completion(.failure(PigeonError(code: "", message: "Failed to create pixel buffer for request \(requestId), status: \(status)", details: nil)))
+    }
+    
+    if request.isCancelled {
+      return request.completion(Self.cancelledResult)
+    }
+    
+    Self.ciContext.render(ciImage, to: pixelBuffer)
+    
+    CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly)
+    guard let pointer = CVPixelBufferGetBaseAddress(pixelBuffer) else {
+      CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly)
+      return request.completion(.failure(PigeonError(code: "", message: "Failed to lock pixel buffer for request \(requestId)", details: nil)))
+    }
+    
+    let rowBytes = CVPixelBufferGetBytesPerRow(pixelBuffer)
+    
+    Self.requestQueue.sync {
+      Self.lockedPixelBuffers[requestId] = pixelBuffer
+    }
+    
+    request.completion(
+      .success([
+        "pointer": Int64(Int(bitPattern: pointer)),
+        "width": Int64(width),
+        "height": Int64(height),
+        "rowBytes": Int64(rowBytes),
+      ]))
   }
   
   func get(requestId: Int64) -> RemoteImageRequest? {
@@ -157,4 +183,9 @@ class RemoteImageApiDelegate: NSObject, URLSessionDataDelegate {
     request.isCancelled = true
     request.task?.cancel()
   }
+  
+  func releasePixelBuffer(requestId: Int64) -> Void {
+    guard let pixelBuffer = (Self.requestQueue.sync { Self.lockedPixelBuffers.removeValue(forKey: requestId) }) else { return }
+    CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly)
+  }
 }

mobile/lib/infrastructure/loaders/image_request.dart

@@ -34,7 +34,7 @@ abstract class ImageRequest {
 
   void _onCancelled();
 
-  Future<ui.FrameInfo?> _fromPlatformImage(Map<String, int> info) async {
+  Future<ui.FrameInfo?> _fromPlatformImage(Map<String, int> info, ui.PixelFormat pixelFormat, bool shouldFree) async {
     final address = info['pointer'];
     if (address == null) {
       return null;
@@ -42,7 +42,9 @@ abstract class ImageRequest {
 
     final pointer = Pointer<Uint8>.fromAddress(address);
     if (_isCancelled) {
-      malloc.free(pointer);
+      if (shouldFree) {
+        malloc.free(pointer);
+      }
       return null;
     }
 
@@ -58,7 +60,9 @@ abstract class ImageRequest {
       actualSize = rowBytes * actualHeight;
       buffer = await ImmutableBuffer.fromUint8List(pointer.asTypedList(actualSize));
     } finally {
-      malloc.free(pointer);
+      if (shouldFree) {
+        malloc.free(pointer);
+      }
     }
 
     if (_isCancelled) {
@@ -71,7 +75,7 @@ abstract class ImageRequest {
       width: actualWidth,
       height: actualHeight,
       rowBytes: rowBytes,
-      pixelFormat: ui.PixelFormat.rgba8888,
+      pixelFormat: pixelFormat,
     );
     final codec = await descriptor.instantiateCodec();
     if (_isCancelled) {

mobile/lib/infrastructure/loaders/local_image_request.dart

@@ -24,7 +24,7 @@ class LocalImageRequest extends ImageRequest {
       isVideo: assetType == AssetType.video,
     );
 
-    final frame = await _fromPlatformImage(info);
+    final frame = await _fromPlatformImage(info, ui.PixelFormat.rgba8888, true);
     return frame == null ? null : ImageInfo(image: frame.image, scale: scale);
   }
 

mobile/lib/infrastructure/loaders/remote_image_request.dart

@@ -14,12 +14,16 @@ class RemoteImageRequest extends ImageRequest {
 
     final Map<String, int> info = await remoteImageApi.requestImage(uri, headers: headers, requestId: requestId);
 
-    final frame = await _fromPlatformImage(info);
+    try {
-    return frame == null ? null : ImageInfo(image: frame.image, scale: scale);
+      final frame = await _fromPlatformImage(info, ui.PixelFormat.bgra8888, false);
+      return frame == null ? null : ImageInfo(image: frame.image, scale: scale);
+    } finally {
+      unawaited(remoteImageApi.releaseImage(requestId));
+    }
   }
 
   @override
   Future<void> _onCancelled() {
-    return localImageApi.cancelRequest(requestId);
+    return remoteImageApi.cancelRequest(requestId);
   }
 }

mobile/lib/infrastructure/loaders/thumbhash_image_request.dart

@@ -12,7 +12,7 @@ class ThumbhashImageRequest extends ImageRequest {
     }
 
     final Map<String, int> info = await localImageApi.getThumbhash(thumbhash);
-    final frame = await _fromPlatformImage(info);
+    final frame = await _fromPlatformImage(info, ui.PixelFormat.rgba8888, true);
     return frame == null ? null : ImageInfo(image: frame.image, scale: scale);
   }
 

mobile/lib/platform/remote_image_api.g.dart

@@ -103,4 +103,27 @@ class RemoteImageApi {
       return;
     }
   }
+
+  Future<void> releaseImage(int requestId) async {
+    final String pigeonVar_channelName =
+        'dev.flutter.pigeon.immich_mobile.RemoteImageApi.releaseImage$pigeonVar_messageChannelSuffix';
+    final BasicMessageChannel<Object?> pigeonVar_channel = BasicMessageChannel<Object?>(
+      pigeonVar_channelName,
+      pigeonChannelCodec,
+      binaryMessenger: pigeonVar_binaryMessenger,
+    );
+    final Future<Object?> pigeonVar_sendFuture = pigeonVar_channel.send(<Object?>[requestId]);
+    final List<Object?>? pigeonVar_replyList = await pigeonVar_sendFuture as List<Object?>?;
+    if (pigeonVar_replyList == null) {
+      throw _createConnectionError(pigeonVar_channelName);
+    } else if (pigeonVar_replyList.length > 1) {
+      throw PlatformException(
+        code: pigeonVar_replyList[0]! as String,
+        message: pigeonVar_replyList[1] as String?,
+        details: pigeonVar_replyList[2],
+      );
+    } else {
+      return;
+    }
+  }
 }

mobile/pigeon/remote_image_api.dart

@@ -22,4 +22,6 @@ abstract class RemoteImageApi {
   });
 
   void cancelRequest(int requestId);
+
+  void releaseImage(int requestId);
 }