From 6052f84022ed0e5be428bc95714bd77fe09c0f67 Mon Sep 17 00:00:00 2001 From: Yaros Date: Wed, 25 Feb 2026 21:20:28 +0100 Subject: [PATCH] feat(mobile): ocr ui --- .../asset_viewer/asset_page.widget.dart | 10 + .../asset_viewer/asset_viewer.state.dart | 12 +- .../asset_viewer/ocr_overlay.widget.dart | 212 ++++++++++++++++++ .../viewer_top_app_bar.widget.dart | 9 + .../infrastructure/ocr.provider.dart | 6 + 5 files changed, 248 insertions(+), 1 deletion(-) create mode 100644 mobile/lib/presentation/widgets/asset_viewer/ocr_overlay.widget.dart diff --git a/mobile/lib/presentation/widgets/asset_viewer/asset_page.widget.dart b/mobile/lib/presentation/widgets/asset_viewer/asset_page.widget.dart index 686b3fcf10..61692e0538 100644 --- a/mobile/lib/presentation/widgets/asset_viewer/asset_page.widget.dart +++ b/mobile/lib/presentation/widgets/asset_viewer/asset_page.widget.dart @@ -13,6 +13,7 @@ import 'package:immich_mobile/extensions/scroll_extensions.dart'; import 'package:immich_mobile/presentation/widgets/asset_viewer/asset_details.widget.dart'; import 'package:immich_mobile/presentation/widgets/asset_viewer/asset_stack.provider.dart'; import 'package:immich_mobile/presentation/widgets/asset_viewer/asset_viewer.state.dart'; +import 'package:immich_mobile/presentation/widgets/asset_viewer/ocr_overlay.widget.dart'; import 'package:immich_mobile/presentation/widgets/asset_viewer/video_viewer.widget.dart'; import 'package:immich_mobile/presentation/widgets/images/image_provider.dart'; import 'package:immich_mobile/presentation/widgets/images/thumbnail.widget.dart'; @@ -372,6 +373,7 @@ class _AssetPageState extends ConsumerState { _showingDetails = ref.watch(assetViewerProvider.select((s) => s.showingDetails)); final stackIndex = ref.watch(assetViewerProvider.select((s) => s.stackIndex)); final isPlayingMotionVideo = ref.watch(isPlayingMotionVideoProvider); + final showingOcr = ref.watch(assetViewerProvider.select((s) => s.showingOcr)); final asset = ref.read(timelineServiceProvider).getAssetSafe(widget.index); if (asset == null) { @@ -432,6 +434,14 @@ class _AssetPageState extends ConsumerState { backgroundDecoration: BoxDecoration(color: _showingDetails ? Colors.black : Colors.transparent), ), ), + if (showingOcr && !_isZoomed && displayAsset.width != null && displayAsset.height != null) + Positioned.fill( + child: OcrOverlay( + asset: displayAsset, + imageSize: Size(displayAsset.width!.toDouble(), displayAsset.height!.toDouble()), + viewportSize: Size(viewportWidth, viewportHeight), + ), + ), IgnorePointer( ignoring: !_showingDetails, child: Column( diff --git a/mobile/lib/presentation/widgets/asset_viewer/asset_viewer.state.dart b/mobile/lib/presentation/widgets/asset_viewer/asset_viewer.state.dart index dc510d6017..b94f5f0b51 100644 --- a/mobile/lib/presentation/widgets/asset_viewer/asset_viewer.state.dart +++ b/mobile/lib/presentation/widgets/asset_viewer/asset_viewer.state.dart @@ -7,6 +7,7 @@ class AssetViewerState { final bool showingDetails; final bool showingControls; final bool isZoomed; + final bool showingOcr; final BaseAsset? currentAsset; final int stackIndex; @@ -15,6 +16,7 @@ class AssetViewerState { this.showingDetails = false, this.showingControls = true, this.isZoomed = false, + this.showingOcr = false, this.currentAsset, this.stackIndex = 0, }); @@ -24,6 +26,7 @@ class AssetViewerState { bool? showingDetails, bool? showingControls, bool? isZoomed, + bool? showingOcr, BaseAsset? currentAsset, int? stackIndex, }) { @@ -32,6 +35,7 @@ class AssetViewerState { showingDetails: showingDetails ?? this.showingDetails, showingControls: showingControls ?? this.showingControls, isZoomed: isZoomed ?? this.isZoomed, + showingOcr: showingOcr ?? this.showingOcr, currentAsset: currentAsset ?? this.currentAsset, stackIndex: stackIndex ?? this.stackIndex, ); @@ -39,7 +43,7 @@ class AssetViewerState { @override String toString() { - return 'AssetViewerState(opacity: $backgroundOpacity, showingDetails: $showingDetails, controls: $showingControls, isZoomed: $isZoomed)'; + return 'AssetViewerState(opacity: $backgroundOpacity, showingDetails: $showingDetails, controls: $showingControls, isZoomed: $isZoomed, showingOcr: $showingOcr)'; } @override @@ -51,6 +55,7 @@ class AssetViewerState { other.showingDetails == showingDetails && other.showingControls == showingControls && other.isZoomed == isZoomed && + other.showingOcr == showingOcr && other.currentAsset == currentAsset && other.stackIndex == stackIndex; } @@ -61,6 +66,7 @@ class AssetViewerState { showingDetails.hashCode ^ showingControls.hashCode ^ isZoomed.hashCode ^ + showingOcr.hashCode ^ currentAsset.hashCode ^ stackIndex.hashCode; } @@ -123,6 +129,10 @@ class AssetViewerStateNotifier extends Notifier { } state = state.copyWith(stackIndex: index); } + + void toggleOcr() { + state = state.copyWith(showingOcr: !state.showingOcr); + } } final assetViewerProvider = NotifierProvider(AssetViewerStateNotifier.new); diff --git a/mobile/lib/presentation/widgets/asset_viewer/ocr_overlay.widget.dart b/mobile/lib/presentation/widgets/asset_viewer/ocr_overlay.widget.dart new file mode 100644 index 0000000000..0045e86cee --- /dev/null +++ b/mobile/lib/presentation/widgets/asset_viewer/ocr_overlay.widget.dart @@ -0,0 +1,212 @@ +import 'dart:math' as math; +import 'package:flutter/material.dart'; +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; +import 'package:immich_mobile/domain/models/ocr.model.dart'; +import 'package:immich_mobile/extensions/build_context_extensions.dart'; +import 'package:immich_mobile/providers/infrastructure/ocr.provider.dart'; + +class OcrOverlay extends ConsumerStatefulWidget { + final BaseAsset asset; + final Size imageSize; + final Size viewportSize; + + const OcrOverlay({super.key, required this.asset, required this.imageSize, required this.viewportSize}); + + @override + ConsumerState createState() => _OcrOverlayState(); +} + +class _OcrOverlayState extends ConsumerState { + int? _selectedBoxIndex; + + @override + Widget build(BuildContext context) { + if (widget.asset is! RemoteAsset) { + return const SizedBox.shrink(); + } + + final ocrData = ref.watch(driftOcrAssetProvider((widget.asset as RemoteAsset).id)); + + return ocrData.when( + data: (data) { + if (data == null || data.isEmpty) { + return const SizedBox.shrink(); + } + + return _buildOcrBoxes(data); + }, + loading: () => const SizedBox.shrink(), + error: (_, __) => const SizedBox.shrink(), + ); + } + + Widget _buildOcrBoxes(List ocrData) { + // Calculate the scale factor to fit the image in the viewport + final imageWidth = widget.imageSize.width; + final imageHeight = widget.imageSize.height; + final viewportWidth = widget.viewportSize.width; + final viewportHeight = widget.viewportSize.height; + + // Calculate how the image is scaled to fit in the viewport + final scaleX = viewportWidth / imageWidth; + final scaleY = viewportHeight / imageHeight; + final scale = scaleX < scaleY ? scaleX : scaleY; + + // Calculate the actual displayed image size + final displayedWidth = imageWidth * scale; + final displayedHeight = imageHeight * scale; + + // Calculate the offset to center the image + final offsetX = (viewportWidth - displayedWidth) / 2; + final offsetY = (viewportHeight - displayedHeight) / 2; + + return GestureDetector( + behavior: HitTestBehavior.opaque, + onTap: () { + setState(() { + _selectedBoxIndex = null; + }); + }, + child: Stack( + children: [ + // Invisible layer to catch taps outside of boxes + SizedBox(width: viewportWidth, height: viewportHeight), + ...ocrData.asMap().entries.map((entry) { + final index = entry.key; + final ocr = entry.value; + final isSelected = _selectedBoxIndex == index; + + // Normalize coordinates (0-1 range) and scale to displayed image size + final x1 = ocr.x1 * displayedWidth + offsetX; + final y1 = ocr.y1 * displayedHeight + offsetY; + final x2 = ocr.x2 * displayedWidth + offsetX; + final y2 = ocr.y2 * displayedHeight + offsetY; + final x3 = ocr.x3 * displayedWidth + offsetX; + final y3 = ocr.y3 * displayedHeight + offsetY; + final x4 = ocr.x4 * displayedWidth + offsetX; + final y4 = ocr.y4 * displayedHeight + offsetY; + + // Calculate bounding rectangle for hit testing + final minX = [x1, x2, x3, x4].reduce((a, b) => a < b ? a : b); + final maxX = [x1, x2, x3, x4].reduce((a, b) => a > b ? a : b); + final minY = [y1, y2, y3, y4].reduce((a, b) => a < b ? a : b); + final maxY = [y1, y2, y3, y4].reduce((a, b) => a > b ? a : b); + + // Calculate rotation angle from the bottom edge (x1,y1) to (x2,y2) + final angle = math.atan2(y2 - y1, x2 - x1); + final centerX = (minX + maxX) / 2; + final centerY = (minY + maxY) / 2; + + return Positioned( + left: minX, + top: minY, + child: GestureDetector( + onTap: () { + setState(() { + _selectedBoxIndex = isSelected ? null : index; + }); + }, + behavior: HitTestBehavior.translucent, + child: SizedBox( + width: maxX - minX, + height: maxY - minY, + child: Stack( + children: [ + CustomPaint( + painter: _OcrBoxPainter( + points: [ + Offset(x1 - minX, y1 - minY), + Offset(x2 - minX, y2 - minY), + Offset(x3 - minX, y3 - minY), + Offset(x4 - minX, y4 - minY), + ], + isSelected: isSelected, + context: context, + ), + size: Size(maxX - minX, maxY - minY), + ), + if (isSelected) + Positioned( + left: centerX - minX, + top: centerY - minY, + child: FractionalTranslation( + translation: const Offset(-0.5, -0.5), + child: Transform.rotate( + angle: angle, + alignment: Alignment.center, + child: Container( + margin: const EdgeInsets.all(2), + padding: const EdgeInsets.symmetric(horizontal: 4, vertical: 2), + decoration: BoxDecoration( + color: Colors.grey[800]?.withValues(alpha: 0.4), + borderRadius: BorderRadius.circular(4), + ), + child: ConstrainedBox( + constraints: BoxConstraints( + maxWidth: math.max(50, maxX - minX), + maxHeight: math.max(20, maxY - minY), + ), + child: FittedBox( + fit: BoxFit.scaleDown, + child: SelectableText( + ocr.text, + style: TextStyle( + color: Colors.white, + fontSize: math.max(12, (maxY - minY) * 0.6), + fontWeight: FontWeight.bold, + ), + textAlign: TextAlign.center, + ), + ), + ), + ), + ), + ), + ), + ], + ), + ), + ), + ); + }), + ], + ), + ); + } +} + +class _OcrBoxPainter extends CustomPainter { + final List points; + final bool isSelected; + final BuildContext context; + + _OcrBoxPainter({required this.points, required this.isSelected, required this.context}); + + @override + void paint(Canvas canvas, Size size) { + final paint = Paint() + ..color = isSelected ? context.primaryColor : Colors.green + ..style = PaintingStyle.stroke + ..strokeWidth = 2.0; + + final fillPaint = Paint() + ..color = (isSelected ? context.primaryColor : Colors.green).withValues(alpha: 0.1) + ..style = PaintingStyle.fill; + + final path = Path() + ..moveTo(points[0].dx, points[0].dy) + ..lineTo(points[1].dx, points[1].dy) + ..lineTo(points[2].dx, points[2].dy) + ..lineTo(points[3].dx, points[3].dy) + ..close(); + + canvas.drawPath(path, fillPaint); + canvas.drawPath(path, paint); + } + + @override + bool shouldRepaint(_OcrBoxPainter oldDelegate) { + return oldDelegate.isSelected != isSelected || oldDelegate.points != points; + } +} diff --git a/mobile/lib/presentation/widgets/asset_viewer/viewer_top_app_bar.widget.dart b/mobile/lib/presentation/widgets/asset_viewer/viewer_top_app_bar.widget.dart index 4b748abc27..d5f887b244 100644 --- a/mobile/lib/presentation/widgets/asset_viewer/viewer_top_app_bar.widget.dart +++ b/mobile/lib/presentation/widgets/asset_viewer/viewer_top_app_bar.widget.dart @@ -4,6 +4,7 @@ import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/constants/enums.dart'; import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; import 'package:immich_mobile/extensions/build_context_extensions.dart'; +import 'package:immich_mobile/providers/infrastructure/ocr.provider.dart'; import 'package:immich_mobile/routing/router.dart'; import 'package:immich_mobile/presentation/widgets/action_buttons/favorite_action_button.widget.dart'; import 'package:immich_mobile/presentation/widgets/action_buttons/motion_photo_action_button.widget.dart'; @@ -33,6 +34,7 @@ class ViewerTopAppBar extends ConsumerWidget implements PreferredSizeWidget { final isOwner = asset is RemoteAsset && asset.ownerId == user?.id; final isInLockedView = ref.watch(inLockedViewProvider); final isReadonlyModeEnabled = ref.watch(readonlyModeProvider); + final hasOcr = asset is RemoteAsset && ref.watch(driftOcrAssetProvider(asset.id)).valueOrNull?.isNotEmpty == true; final showingDetails = ref.watch(assetViewerProvider.select((state) => state.showingDetails)); double opacity = ref.watch(assetViewerProvider.select((state) => state.backgroundOpacity)); @@ -47,8 +49,15 @@ class ViewerTopAppBar extends ConsumerWidget implements PreferredSizeWidget { } final originalTheme = context.themeData; + final showingOcr = ref.watch(assetViewerProvider.select((state) => state.showingOcr)); final actions = [ + if (hasOcr) + IconButton( + icon: Icon(showingOcr ? Icons.text_fields : Icons.text_fields_outlined), + onPressed: () => ref.read(assetViewerProvider.notifier).toggleOcr(), + color: showingOcr ? context.primaryColor : null, + ), if (asset.isMotionPhoto) const MotionPhotoActionButton(iconOnly: true), if (album != null && album.isActivityEnabled && album.isShared) IconButton( diff --git a/mobile/lib/providers/infrastructure/ocr.provider.dart b/mobile/lib/providers/infrastructure/ocr.provider.dart index 6ed97dbbd9..3ea72e0277 100644 --- a/mobile/lib/providers/infrastructure/ocr.provider.dart +++ b/mobile/lib/providers/infrastructure/ocr.provider.dart @@ -1,3 +1,4 @@ +import 'package:immich_mobile/domain/models/ocr.model.dart'; import 'package:immich_mobile/domain/services/ocr.service.dart'; import 'package:immich_mobile/infrastructure/repositories/ocr.repository.dart'; import 'package:immich_mobile/providers/infrastructure/db.provider.dart'; @@ -8,3 +9,8 @@ final driftOcrRepositoryProvider = Provider((ref) => DriftOc final driftOcrServiceProvider = Provider( (ref) => DriftOcrService(ref.watch(driftOcrRepositoryProvider)), ); + +final driftOcrAssetProvider = FutureProvider.family?, String>((ref, assetId) async { + final service = ref.watch(driftOcrServiceProvider); + return service.get(assetId); +});