feat(mobile): add playbackStyle to native sync API (#26541)

* feat(mobile): add playbackStyle to native sync API

Adds a `playbackStyle` field to `PlatformAsset` in the pigeon sync API so
native platforms can communicate the asset's playback style (image, video,
animated, livePhoto) to Flutter during sync.

- Add `playbackStyleValue` computed property to `PHAsset` extension (iOS)
- Populate `playbackStyle` in `toPlatformAsset()` and the full-sync path
- Update generated Dart/Kotlin/Swift files

* fix(tests): add playbackStyle to local asset test cases

* fix(tests): update playbackStyle to use integer values in local sync tests

* feat(mobile): extend playbackStyle enum to include videoLooping

* Update PHAssetExtensions.swift

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* fix(playback): simplify playbackStyleValue implementation by removing iOS version check

* feat(android): implement proper playbackStyle detection

* add PlatformAssetPlaybackStyle enum

* linting

---------

Co-authored-by: Mert <101130780+mertalev@users.noreply.github.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Luis Nachtigall
2026-02-28 04:08:51 +01:00
committed by GitHub
parent aa87d1b9a3
commit d6c724b13b
8 changed files with 239 additions and 45 deletions

View File

@@ -78,6 +78,21 @@ class FlutterError (
val details: Any? = null
) : Throwable()
enum class PlatformAssetPlaybackStyle(val raw: Int) {
UNKNOWN(0),
IMAGE(1),
VIDEO(2),
IMAGE_ANIMATED(3),
LIVE_PHOTO(4),
VIDEO_LOOPING(5);
companion object {
fun ofRaw(raw: Int): PlatformAssetPlaybackStyle? {
return values().firstOrNull { it.raw == raw }
}
}
}
/** Generated class from Pigeon that represents data sent in messages. */
data class PlatformAsset (
val id: String,
@@ -92,7 +107,8 @@ data class PlatformAsset (
val isFavorite: Boolean,
val adjustmentTime: Long? = null,
val latitude: Double? = null,
val longitude: Double? = null
val longitude: Double? = null,
val playbackStyle: PlatformAssetPlaybackStyle
)
{
companion object {
@@ -110,7 +126,8 @@ data class PlatformAsset (
val adjustmentTime = pigeonVar_list[10] as Long?
val latitude = pigeonVar_list[11] as Double?
val longitude = pigeonVar_list[12] as Double?
return PlatformAsset(id, name, type, createdAt, updatedAt, width, height, durationInSeconds, orientation, isFavorite, adjustmentTime, latitude, longitude)
val playbackStyle = pigeonVar_list[13] as PlatformAssetPlaybackStyle
return PlatformAsset(id, name, type, createdAt, updatedAt, width, height, durationInSeconds, orientation, isFavorite, adjustmentTime, latitude, longitude, playbackStyle)
}
}
fun toList(): List<Any?> {
@@ -128,6 +145,7 @@ data class PlatformAsset (
adjustmentTime,
latitude,
longitude,
playbackStyle,
)
}
override fun equals(other: Any?): Boolean {
@@ -290,26 +308,31 @@ private open class MessagesPigeonCodec : StandardMessageCodec() {
override fun readValueOfType(type: Byte, buffer: ByteBuffer): Any? {
return when (type) {
129.toByte() -> {
return (readValue(buffer) as? List<Any?>)?.let {
PlatformAsset.fromList(it)
return (readValue(buffer) as Long?)?.let {
PlatformAssetPlaybackStyle.ofRaw(it.toInt())
}
}
130.toByte() -> {
return (readValue(buffer) as? List<Any?>)?.let {
PlatformAlbum.fromList(it)
PlatformAsset.fromList(it)
}
}
131.toByte() -> {
return (readValue(buffer) as? List<Any?>)?.let {
SyncDelta.fromList(it)
PlatformAlbum.fromList(it)
}
}
132.toByte() -> {
return (readValue(buffer) as? List<Any?>)?.let {
HashResult.fromList(it)
SyncDelta.fromList(it)
}
}
133.toByte() -> {
return (readValue(buffer) as? List<Any?>)?.let {
HashResult.fromList(it)
}
}
134.toByte() -> {
return (readValue(buffer) as? List<Any?>)?.let {
CloudIdResult.fromList(it)
}
@@ -319,26 +342,30 @@ private open class MessagesPigeonCodec : StandardMessageCodec() {
}
override fun writeValue(stream: ByteArrayOutputStream, value: Any?) {
when (value) {
is PlatformAsset -> {
is PlatformAssetPlaybackStyle -> {
stream.write(129)
writeValue(stream, value.toList())
writeValue(stream, value.raw)
}
is PlatformAlbum -> {
is PlatformAsset -> {
stream.write(130)
writeValue(stream, value.toList())
}
is SyncDelta -> {
is PlatformAlbum -> {
stream.write(131)
writeValue(stream, value.toList())
}
is HashResult -> {
is SyncDelta -> {
stream.write(132)
writeValue(stream, value.toList())
}
is CloudIdResult -> {
is HashResult -> {
stream.write(133)
writeValue(stream, value.toList())
}
is CloudIdResult -> {
stream.write(134)
writeValue(stream, value.toList())
}
else -> super.writeValue(stream, value)
}
}

View File

@@ -4,11 +4,17 @@ import android.annotation.SuppressLint
import android.content.ContentUris
import android.content.Context
import android.database.Cursor
import androidx.exifinterface.media.ExifInterface
import android.os.Build
import android.os.Bundle
import android.provider.MediaStore
import android.util.Base64
import android.util.Log
import androidx.core.database.getStringOrNull
import app.alextran.immich.core.ImmichPlugin
import com.bumptech.glide.Glide
import com.bumptech.glide.load.ImageHeaderParser
import com.bumptech.glide.load.ImageHeaderParserUtils
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
@@ -28,6 +34,8 @@ sealed class AssetResult {
data class InvalidAsset(val assetId: String) : AssetResult()
}
private const val TAG = "NativeSyncApiImplBase"
@SuppressLint("InlinedApi")
open class NativeSyncApiImplBase(context: Context) : ImmichPlugin() {
private val ctx: Context = context.applicationContext
@@ -39,6 +47,13 @@ open class NativeSyncApiImplBase(context: Context) : ImmichPlugin() {
private val hashSemaphore = Semaphore(MAX_CONCURRENT_HASH_OPERATIONS)
private const val HASHING_CANCELLED_CODE = "HASH_CANCELLED"
// MediaStore.Files.FileColumns.SPECIAL_FORMAT — S Extensions 21+
// https://developer.android.com/reference/android/provider/MediaStore.Files.FileColumns#SPECIAL_FORMAT
private const val SPECIAL_FORMAT_COLUMN = "_special_format"
private const val SPECIAL_FORMAT_GIF = 1
private const val SPECIAL_FORMAT_MOTION_PHOTO = 2
private const val SPECIAL_FORMAT_ANIMATED_WEBP = 3
const val MEDIA_SELECTION =
"(${MediaStore.Files.FileColumns.MEDIA_TYPE} = ? OR ${MediaStore.Files.FileColumns.MEDIA_TYPE} = ?)"
val MEDIA_SELECTION_ARGS = arrayOf(
@@ -60,9 +75,15 @@ open class NativeSyncApiImplBase(context: Context) : ImmichPlugin() {
add(MediaStore.MediaColumns.DURATION)
add(MediaStore.MediaColumns.ORIENTATION)
// IS_FAVORITE is only available on Android 11 and above
if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.R) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
add(MediaStore.MediaColumns.IS_FAVORITE)
}
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
add(SPECIAL_FORMAT_COLUMN)
} else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
// Fallback: read XMP from MediaStore to detect Motion Photos
add(MediaStore.MediaColumns.XMP)
}
}.toTypedArray()
const val HASH_BUFFER_SIZE = 2 * 1024 * 1024
@@ -109,9 +130,12 @@ open class NativeSyncApiImplBase(context: Context) : ImmichPlugin() {
val orientationColumn =
c.getColumnIndexOrThrow(MediaStore.MediaColumns.ORIENTATION)
val favoriteColumn = c.getColumnIndex(MediaStore.MediaColumns.IS_FAVORITE)
val specialFormatColumn = c.getColumnIndex(SPECIAL_FORMAT_COLUMN)
val xmpColumn = c.getColumnIndex(MediaStore.MediaColumns.XMP)
while (c.moveToNext()) {
val id = c.getLong(idColumn).toString()
val numericId = c.getLong(idColumn)
val id = numericId.toString()
val name = c.getStringOrNull(nameColumn)
val bucketId = c.getStringOrNull(bucketIdColumn)
val path = c.getStringOrNull(dataColumn)
@@ -125,10 +149,11 @@ open class NativeSyncApiImplBase(context: Context) : ImmichPlugin() {
continue
}
val mediaType = when (c.getInt(mediaTypeColumn)) {
MediaStore.Files.FileColumns.MEDIA_TYPE_IMAGE -> 1
MediaStore.Files.FileColumns.MEDIA_TYPE_VIDEO -> 2
else -> 0
val rawMediaType = c.getInt(mediaTypeColumn)
val assetType: Long = when (rawMediaType) {
MediaStore.Files.FileColumns.MEDIA_TYPE_IMAGE -> 1L
MediaStore.Files.FileColumns.MEDIA_TYPE_VIDEO -> 2L
else -> 0L
}
// Date taken is milliseconds since epoch, Date added is seconds since epoch
val createdAt = (c.getLong(dateTakenColumn).takeIf { it > 0 }?.div(1000))
@@ -138,15 +163,19 @@ open class NativeSyncApiImplBase(context: Context) : ImmichPlugin() {
val width = c.getInt(widthColumn).toLong()
val height = c.getInt(heightColumn).toLong()
// Duration is milliseconds
val duration = if (mediaType == MediaStore.Files.FileColumns.MEDIA_TYPE_IMAGE) 0
val duration = if (rawMediaType == MediaStore.Files.FileColumns.MEDIA_TYPE_IMAGE) 0L
else c.getLong(durationColumn) / 1000
val orientation = c.getInt(orientationColumn)
val isFavorite = if (favoriteColumn == -1) false else c.getInt(favoriteColumn) != 0
val playbackStyle = detectPlaybackStyle(
numericId, rawMediaType, specialFormatColumn, xmpColumn, c
)
val asset = PlatformAsset(
id,
name,
mediaType.toLong(),
assetType,
createdAt,
modifiedAt,
width,
@@ -154,6 +183,7 @@ open class NativeSyncApiImplBase(context: Context) : ImmichPlugin() {
duration,
orientation.toLong(),
isFavorite,
playbackStyle = playbackStyle,
)
yield(AssetResult.ValidAsset(asset, bucketId))
}
@@ -161,6 +191,81 @@ open class NativeSyncApiImplBase(context: Context) : ImmichPlugin() {
}
}
/**
* Detects the playback style for an asset using _special_format (API 33+)
* or XMP / MIME / RIFF header fallbacks (pre-33).
*/
@SuppressLint("NewApi")
private fun detectPlaybackStyle(
assetId: Long,
rawMediaType: Int,
specialFormatColumn: Int,
xmpColumn: Int,
cursor: Cursor
): PlatformAssetPlaybackStyle {
// video currently has no special formats, so we can short circuit and avoid unnecessary work
if (rawMediaType == MediaStore.Files.FileColumns.MEDIA_TYPE_VIDEO) {
return PlatformAssetPlaybackStyle.VIDEO
}
// API 33+: use _special_format from cursor
if (specialFormatColumn != -1) {
val specialFormat = cursor.getInt(specialFormatColumn)
return when {
specialFormat == SPECIAL_FORMAT_MOTION_PHOTO -> PlatformAssetPlaybackStyle.LIVE_PHOTO
specialFormat == SPECIAL_FORMAT_GIF || specialFormat == SPECIAL_FORMAT_ANIMATED_WEBP -> PlatformAssetPlaybackStyle.IMAGE_ANIMATED
rawMediaType == MediaStore.Files.FileColumns.MEDIA_TYPE_IMAGE -> PlatformAssetPlaybackStyle.IMAGE
else -> PlatformAssetPlaybackStyle.UNKNOWN
}
}
if (rawMediaType != MediaStore.Files.FileColumns.MEDIA_TYPE_IMAGE) {
return PlatformAssetPlaybackStyle.UNKNOWN
}
// Pre-API 33 fallback
val uri = ContentUris.withAppendedId(
MediaStore.Files.getContentUri(MediaStore.VOLUME_EXTERNAL),
assetId
)
// Read XMP from cursor (API 30+) or ExifInterface stream (pre-30)
val xmp: String? = if (xmpColumn != -1) {
cursor.getBlob(xmpColumn)?.toString(Charsets.UTF_8)
} else {
try {
ctx.contentResolver.openInputStream(uri)?.use { stream ->
ExifInterface(stream).getAttribute(ExifInterface.TAG_XMP)
}
} catch (e: Exception) {
Log.w(TAG, "Failed to read XMP for asset $assetId", e)
null
}
}
if (xmp != null && "Camera:MotionPhoto" in xmp) {
return PlatformAssetPlaybackStyle.LIVE_PHOTO
}
try {
ctx.contentResolver.openInputStream(uri)?.use { stream ->
val glide = Glide.get(ctx)
val type = ImageHeaderParserUtils.getType(
glide.registry.imageHeaderParsers,
stream,
glide.arrayPool
)
if (type == ImageHeaderParser.ImageType.GIF || type == ImageHeaderParser.ImageType.ANIMATED_WEBP) {
return PlatformAssetPlaybackStyle.IMAGE_ANIMATED
}
}
} catch (e: Exception) {
Log.w(TAG, "Failed to parse image header for asset $assetId", e)
}
return PlatformAssetPlaybackStyle.IMAGE
}
fun getAlbums(): List<PlatformAlbum> {
val albums = mutableListOf<PlatformAlbum>()
val albumsCount = mutableMapOf<String, Int>()

View File

@@ -128,6 +128,15 @@ func deepHashMessages(value: Any?, hasher: inout Hasher) {
enum PlatformAssetPlaybackStyle: Int {
case unknown = 0
case image = 1
case video = 2
case imageAnimated = 3
case livePhoto = 4
case videoLooping = 5
}
/// Generated class from Pigeon that represents data sent in messages.
struct PlatformAsset: Hashable {
var id: String
@@ -143,6 +152,7 @@ struct PlatformAsset: Hashable {
var adjustmentTime: Int64? = nil
var latitude: Double? = nil
var longitude: Double? = nil
var playbackStyle: PlatformAssetPlaybackStyle
// swift-format-ignore: AlwaysUseLowerCamelCase
@@ -160,6 +170,7 @@ struct PlatformAsset: Hashable {
let adjustmentTime: Int64? = nilOrValue(pigeonVar_list[10])
let latitude: Double? = nilOrValue(pigeonVar_list[11])
let longitude: Double? = nilOrValue(pigeonVar_list[12])
let playbackStyle = pigeonVar_list[13] as! PlatformAssetPlaybackStyle
return PlatformAsset(
id: id,
@@ -174,7 +185,8 @@ struct PlatformAsset: Hashable {
isFavorite: isFavorite,
adjustmentTime: adjustmentTime,
latitude: latitude,
longitude: longitude
longitude: longitude,
playbackStyle: playbackStyle
)
}
func toList() -> [Any?] {
@@ -192,6 +204,7 @@ struct PlatformAsset: Hashable {
adjustmentTime,
latitude,
longitude,
playbackStyle,
]
}
static func == (lhs: PlatformAsset, rhs: PlatformAsset) -> Bool {
@@ -349,14 +362,20 @@ private class MessagesPigeonCodecReader: FlutterStandardReader {
override func readValue(ofType type: UInt8) -> Any? {
switch type {
case 129:
return PlatformAsset.fromList(self.readValue() as! [Any?])
let enumResultAsInt: Int? = nilOrValue(self.readValue() as! Int?)
if let enumResultAsInt = enumResultAsInt {
return PlatformAssetPlaybackStyle(rawValue: enumResultAsInt)
}
return nil
case 130:
return PlatformAlbum.fromList(self.readValue() as! [Any?])
return PlatformAsset.fromList(self.readValue() as! [Any?])
case 131:
return SyncDelta.fromList(self.readValue() as! [Any?])
return PlatformAlbum.fromList(self.readValue() as! [Any?])
case 132:
return HashResult.fromList(self.readValue() as! [Any?])
return SyncDelta.fromList(self.readValue() as! [Any?])
case 133:
return HashResult.fromList(self.readValue() as! [Any?])
case 134:
return CloudIdResult.fromList(self.readValue() as! [Any?])
default:
return super.readValue(ofType: type)
@@ -366,21 +385,24 @@ private class MessagesPigeonCodecReader: FlutterStandardReader {
private class MessagesPigeonCodecWriter: FlutterStandardWriter {
override func writeValue(_ value: Any) {
if let value = value as? PlatformAsset {
if let value = value as? PlatformAssetPlaybackStyle {
super.writeByte(129)
super.writeValue(value.toList())
} else if let value = value as? PlatformAlbum {
super.writeValue(value.rawValue)
} else if let value = value as? PlatformAsset {
super.writeByte(130)
super.writeValue(value.toList())
} else if let value = value as? SyncDelta {
} else if let value = value as? PlatformAlbum {
super.writeByte(131)
super.writeValue(value.toList())
} else if let value = value as? HashResult {
} else if let value = value as? SyncDelta {
super.writeByte(132)
super.writeValue(value.toList())
} else if let value = value as? CloudIdResult {
} else if let value = value as? HashResult {
super.writeByte(133)
super.writeValue(value.toList())
} else if let value = value as? CloudIdResult {
super.writeByte(134)
super.writeValue(value.toList())
} else {
super.writeValue(value)
}

View File

@@ -173,7 +173,8 @@ class NativeSyncApiImpl: ImmichPlugin, NativeSyncApi, FlutterPlugin {
type: 0,
durationInSeconds: 0,
orientation: 0,
isFavorite: false
isFavorite: false,
playbackStyle: .unknown
)
if (updatedAssets.contains(AssetWrapper(with: predicate))) {
continue

View File

@@ -1,6 +1,17 @@
import Photos
extension PHAsset {
var platformPlaybackStyle: PlatformAssetPlaybackStyle {
switch playbackStyle {
case .image: return .image
case .imageAnimated: return .imageAnimated
case .livePhoto: return .livePhoto
case .video: return .video
case .videoLooping: return .videoLooping
@unknown default: return .unknown
}
}
func toPlatformAsset() -> PlatformAsset {
return PlatformAsset(
id: localIdentifier,
@@ -15,7 +26,8 @@ extension PHAsset {
isFavorite: isFavorite,
adjustmentTime: adjustmentTimestamp,
latitude: location?.coordinate.latitude,
longitude: location?.coordinate.longitude
longitude: location?.coordinate.longitude,
playbackStyle: platformPlaybackStyle
)
}
@@ -26,7 +38,7 @@ extension PHAsset {
var filename: String? {
return value(forKey: "filename") as? String
}
var adjustmentTimestamp: Int64? {
if let date = value(forKey: "adjustmentTimestamp") as? Date {
return Int64(date.timeIntervalSince1970)

View File

@@ -29,6 +29,8 @@ bool _deepEquals(Object? a, Object? b) {
return a == b;
}
enum PlatformAssetPlaybackStyle { unknown, image, video, imageAnimated, livePhoto, videoLooping }
class PlatformAsset {
PlatformAsset({
required this.id,
@@ -44,6 +46,7 @@ class PlatformAsset {
this.adjustmentTime,
this.latitude,
this.longitude,
required this.playbackStyle,
});
String id;
@@ -72,6 +75,8 @@ class PlatformAsset {
double? longitude;
PlatformAssetPlaybackStyle playbackStyle;
List<Object?> _toList() {
return <Object?>[
id,
@@ -87,6 +92,7 @@ class PlatformAsset {
adjustmentTime,
latitude,
longitude,
playbackStyle,
];
}
@@ -110,6 +116,7 @@ class PlatformAsset {
adjustmentTime: result[10] as int?,
latitude: result[11] as double?,
longitude: result[12] as double?,
playbackStyle: result[13]! as PlatformAssetPlaybackStyle,
);
}
@@ -316,21 +323,24 @@ class _PigeonCodec extends StandardMessageCodec {
if (value is int) {
buffer.putUint8(4);
buffer.putInt64(value);
} else if (value is PlatformAsset) {
} else if (value is PlatformAssetPlaybackStyle) {
buffer.putUint8(129);
writeValue(buffer, value.encode());
} else if (value is PlatformAlbum) {
writeValue(buffer, value.index);
} else if (value is PlatformAsset) {
buffer.putUint8(130);
writeValue(buffer, value.encode());
} else if (value is SyncDelta) {
} else if (value is PlatformAlbum) {
buffer.putUint8(131);
writeValue(buffer, value.encode());
} else if (value is HashResult) {
} else if (value is SyncDelta) {
buffer.putUint8(132);
writeValue(buffer, value.encode());
} else if (value is CloudIdResult) {
} else if (value is HashResult) {
buffer.putUint8(133);
writeValue(buffer, value.encode());
} else if (value is CloudIdResult) {
buffer.putUint8(134);
writeValue(buffer, value.encode());
} else {
super.writeValue(buffer, value);
}
@@ -340,14 +350,17 @@ class _PigeonCodec extends StandardMessageCodec {
Object? readValueOfType(int type, ReadBuffer buffer) {
switch (type) {
case 129:
return PlatformAsset.decode(readValue(buffer)!);
final int? value = readValue(buffer) as int?;
return value == null ? null : PlatformAssetPlaybackStyle.values[value];
case 130:
return PlatformAlbum.decode(readValue(buffer)!);
return PlatformAsset.decode(readValue(buffer)!);
case 131:
return SyncDelta.decode(readValue(buffer)!);
return PlatformAlbum.decode(readValue(buffer)!);
case 132:
return HashResult.decode(readValue(buffer)!);
return SyncDelta.decode(readValue(buffer)!);
case 133:
return HashResult.decode(readValue(buffer)!);
case 134:
return CloudIdResult.decode(readValue(buffer)!);
default:
return super.readValueOfType(type, buffer);

View File

@@ -11,6 +11,15 @@ import 'package:pigeon/pigeon.dart';
dartPackageName: 'immich_mobile',
),
)
enum PlatformAssetPlaybackStyle {
unknown,
image,
video,
imageAnimated,
livePhoto,
videoLooping,
}
class PlatformAsset {
final String id;
final String name;
@@ -31,6 +40,8 @@ class PlatformAsset {
final double? latitude;
final double? longitude;
final PlatformAssetPlaybackStyle playbackStyle;
const PlatformAsset({
required this.id,
required this.name,
@@ -45,6 +56,7 @@ class PlatformAsset {
this.adjustmentTime,
this.latitude,
this.longitude,
this.playbackStyle = PlatformAssetPlaybackStyle.unknown,
});
}

View File

@@ -131,6 +131,7 @@ void main() {
durationInSeconds: 0,
orientation: 0,
isFavorite: false,
playbackStyle: PlatformAssetPlaybackStyle.image
);
final assetsToRestore = [LocalAssetStub.image1];
@@ -214,6 +215,7 @@ void main() {
isFavorite: false,
createdAt: 1700000000,
updatedAt: 1732000000,
playbackStyle: PlatformAssetPlaybackStyle.image
);
final localAsset = platformAsset.toLocalAsset();