feat: 'ken burns' style slideshow effect

Change-Id: I22c2bfbf08b5b0e0f4eb9f25e1efe0e16a6a6964

feat(web): manual compositor-driven crossfade for slideshow transitions

Change-Id: If1119429abd2689595defcb8831442506a6a6964

resolve conflict: event rename

Change-Id: I4d7c8d2a16237b42e49b87734764e18f6a6a6964

Change-Id: I22c2bfbf08b5b0e0f4eb9f25e1efe0e16a6a6964

resolve xxn conflicts

Change-Id: I86c287c6c8b59a5549153f21d1c092586a6a6964

web/package.json

@@ -53,6 +53,7 @@
     "pmtiles": "^4.3.0",
     "qrcode": "^1.5.4",
     "simple-icons": "^15.15.0",
+    "smartcrop": "^2.0.5",
     "socket.io-client": "~4.8.0",
     "svelte-gestures": "^5.2.2",
     "svelte-i18n": "^4.0.1",

web/src/lib/components/asset-viewer/asset-viewer.svelte

@@ -297,8 +297,19 @@
 
     const targetAsset = order === 'previous' ? previousAsset : nextAsset;
     const slideshowAllowsTransition = !slideShowPlaying || $slideshowTransition;
-    const useTransition = canTransition && slideshowAllowsTransition && (slideShowShuffle || !!targetAsset);
-    const hasNext = useTransition ? await startTransition(types, targetTransition, navigate) : await navigate();
+    const useTransition = slideshowAllowsTransition && (slideShowShuffle || !!targetAsset);
+
+    let hasNext: boolean;
+    if (slideShowPlaying && useTransition) {
+      hasNext = false;
+      await crossfadeViewerContent(async () => {
+        hasNext = await navigate();
+      }, 1000);
+    } else if (canTransition && useTransition) {
+      hasNext = await startTransition(types, targetTransition, navigate);
+    } else {
+      hasNext = await navigate();
+    }
 
     if (!slideShowPlaying) {
       return;

web/src/lib/components/asset-viewer/photo-viewer.svelte

@@ -9,6 +9,7 @@
   import { assetViewerManager } from '$lib/managers/asset-viewer-manager.svelte';
   import { castManager } from '$lib/managers/cast-manager.svelte';
   import { eventManager } from '$lib/managers/event-manager.svelte';
+  import { viewTransitionManager } from '$lib/managers/ViewTransitionManager.svelte';
   import { isEditFacesPanelOpen, isFaceEditMode } from '$lib/stores/face-edit.svelte';
   import { ocrManager } from '$lib/stores/ocr.svelte';
   import { boundingBoxesArray, type Faces } from '$lib/stores/people.store';
@@ -17,11 +18,12 @@
   import { canCopyImageToClipboard, copyImageToClipboard } from '$lib/utils/asset-utils';
   import type { Size } from '$lib/utils/container-utils';
   import { handleError } from '$lib/utils/handle-error';
+  import { KenBurnsAnimation } from '$lib/utils/ken-burns';
   import { getOcrBoundingBoxes } from '$lib/utils/ocr-utils';
   import { getBoundingBox } from '$lib/utils/people-utils';
   import { type SharedLinkResponseDto } from '@immich/sdk';
   import { toastManager } from '@immich/ui';
-  import { onDestroy, untrack } from 'svelte';
+  import { onDestroy, onMount, untrack } from 'svelte';
   import { useSwipe, type SwipeCustomEvent } from 'svelte-gestures';
   import { t } from 'svelte-i18n';
   import type { AssetCursor } from './asset-viewer.svelte';
@@ -46,13 +48,17 @@
     onSwipe,
   }: Props = $props();
 
-  const { slideshowState, slideshowLook } = slideshowStore;
-  const objectFit = $derived(
-    $slideshowState !== SlideshowState.None && $slideshowLook === SlideshowLook.Cover ? 'cover' : 'contain',
-  );
+  const { slideshowState, slideshowLook, kenBurnsEffect, slideshowDelay } = slideshowStore;
   const asset = $derived(cursor.current);
 
   let visibleImageReady: boolean = $state(false);
+  const kenBurns = new KenBurnsAnimation();
+  let adaptiveImage = $state<HTMLDivElement | undefined>();
+  onMount(() =>
+    eventManager.on({
+      ViewTransitionOldSnapshotPending: () => kenBurns.freeze(),
+    }),
+  );
 
   let previousAssetId: string | undefined;
   $effect.pre(() => {
@@ -62,13 +68,16 @@
     }
     previousAssetId = id;
     untrack(() => {
+      kenBurns.cancel();
       assetViewerManager.resetZoomState();
       visibleImageReady = false;
       $boundingBoxesArray = [];
+      adaptiveImage?.style.removeProperty('transform');
     });
   });
 
   onDestroy(() => {
+    kenBurns.cancel();
     $boundingBoxesArray = [];
   });
 
@@ -80,6 +89,8 @@
     height: containerHeight,
   });
 
+  const isCoverMode = $derived($slideshowState !== SlideshowState.None && $slideshowLook === SlideshowLook.Cover);
+
   let imageDimensions = $state<Size>({ width: 0, height: 0 });
   let scaledDimensions = $state<Size>({ width: 0, height: 0 });
 
@@ -147,8 +158,6 @@
     $slideshowState !== SlideshowState.None && $slideshowLook === SlideshowLook.BlurredBackground && !!asset.thumbhash,
   );
 
-  let adaptiveImage = $state<HTMLDivElement | undefined>();
-
   const faceToNameMap = $derived.by(() => {
     // eslint-disable-next-line svelte/prefer-svelte-reactivity
     const map = new Map<Faces, string | undefined>();
@@ -167,6 +176,47 @@
   const faces = $derived(Array.from(faceToNameMap.keys()));
   const boundingBoxes = $derived(getBoundingBox(faces, overlaySize));
   const activeBoundingBoxes = $derived(boundingBoxes.filter((box) => $boundingBoxesArray.some((f) => f.id === box.id)));
+
+  const unassignedFaces = $derived((asset.unassignedFaces ?? []) as Faces[]);
+
+  const kenBurnsActive = $derived($slideshowState === SlideshowState.PlaySlideshow && $kenBurnsEffect);
+
+  $effect(() => {
+    if (!kenBurnsActive || !visibleImageReady || !adaptiveImage || !assetViewerManager.imgRef) {
+      return;
+    }
+
+    assetViewerManager.zoomState = { ...untrack(() => assetViewerManager.zoomState), enable: false };
+
+    void kenBurns.startWithSmartCrop(adaptiveImage, {
+      imgRef: assetViewerManager.imgRef,
+      faces,
+      fallbackFaces: unassignedFaces,
+      contentWidth: overlaySize.width,
+      contentHeight: overlaySize.height,
+      containerWidth,
+      containerHeight,
+      slideshowLook: $slideshowLook,
+      isCoverMode,
+      slideshowDelay: $slideshowDelay,
+      assetId: asset.id,
+    });
+
+    return () => {
+      kenBurns.cancel();
+      if (viewTransitionManager.activeViewTransition === null) {
+        assetViewerManager.resetZoomState();
+      }
+    };
+  });
+
+  $effect(() => {
+    if (viewTransitionManager.activeViewTransition === null) {
+      kenBurns.resume();
+    } else {
+      kenBurns.pause();
+    }
+  });
 </script>
 
 <AssetViewerEvents {onCopy} {onZoom} />
@@ -194,7 +244,7 @@
     {asset}
     {sharedLink}
     {container}
-    {objectFit}
+    objectFit={isCoverMode ? 'cover' : 'contain'}
     {onUrlChange}
     onImageReady={() => {
       visibleImageReady = true;

web/src/lib/managers/event-manager.svelte.ts

@@ -98,6 +98,7 @@ export type Events = {
   ViewerCloseTransitionReady: [];
   ViewerOpenTransition: [];
   ViewerOpenTransitionReady: [];
+  ViewTransitionOldSnapshotPending: [];
 };
 
 export const eventManager = new BaseEventManager<Events>();

web/src/lib/modals/SlideshowSettingsModal.svelte

@@ -21,6 +21,7 @@
     slideshowTransition,
     slideshowAutoplay,
     slideshowRepeat,
+    kenBurnsEffect,
     slideshowState,
   } = slideshowStore;
 
@@ -38,6 +39,7 @@
   let tempSlideshowTransition = $state($slideshowTransition);
   let tempSlideshowAutoplay = $state($slideshowAutoplay);
   let tempSlideshowRepeat = $state($slideshowRepeat);
+  let tempKenBurnsEffect = $state($kenBurnsEffect);
 
   const navigationOptions: Record<SlideshowNavigation, RenderedOption> = {
     [SlideshowNavigation.Shuffle]: { icon: mdiShuffle, title: $t('shuffle') },
@@ -70,6 +72,7 @@
     $slideshowTransition = tempSlideshowTransition;
     $slideshowAutoplay = tempSlideshowAutoplay;
     $slideshowRepeat = tempSlideshowRepeat;
+    $kenBurnsEffect = tempKenBurnsEffect;
     $slideshowState = SlideshowState.PlaySlideshow;
     onClose();
   };
@@ -107,6 +110,10 @@
       <Switch bind:checked={tempSlideshowTransition} />
     </Field>
 
+    <Field label={$t('slideshow_ken_burns_effect')}>
+      <Switch bind:checked={tempKenBurnsEffect} />
+    </Field>
+
     <Field label={$t('slideshow_repeat')} description={$t('slideshow_repeat_description')}>
       <Switch bind:checked={tempSlideshowRepeat} />
     </Field>

web/src/lib/stores/slideshow.store.ts

@@ -41,6 +41,7 @@ function createSlideshowStore() {
   const slideshowTransition = persisted<boolean>('slideshow-transition', true);
   const slideshowAutoplay = persisted<boolean>('slideshow-autoplay', true, {});
   const slideshowRepeat = persisted<boolean>('slideshow-repeat', false);
+  const kenBurnsEffect = persisted<boolean>('slideshow-ken-burns-effect', false);
 
   return {
     restartProgress: {
@@ -73,6 +74,7 @@ function createSlideshowStore() {
     slideshowTransition,
     slideshowAutoplay,
     slideshowRepeat,
+    kenBurnsEffect,
   };
 }
 

web/src/lib/utils/ken-burns.ts

@@ -0,0 +1,237 @@
+import type { Faces } from '$lib/stores/people.store';
+import { SlideshowLook } from '$lib/stores/slideshow.store';
+import type { Point } from '$lib/utils/container-utils';
+import { TUNABLES } from '$lib/utils/tunables';
+import { clamp } from 'lodash-es';
+import smartcrop from 'smartcrop';
+
+const KEN_BURNS_MAX_ZOOM_SPEED = 0.08;
+const KEN_BURNS_MAX_PAN_SPEED = 8;
+
+export interface KenBurnsKeyframes {
+  startTransform: string;
+  endTransform: string;
+  duration: number;
+}
+
+interface KenBurnsInput {
+  faces: Faces[];
+  fallbackFaces: Faces[];
+  smartCropCenter: Point | undefined;
+  contentWidth: number;
+  contentHeight: number;
+  containerWidth: number;
+  containerHeight: number;
+  slideshowLook: SlideshowLook;
+  isCoverMode: boolean;
+  slideshowDelay: number;
+  assetId: string;
+}
+
+export async function computeSmartCropCenter(
+  imgRef: HTMLImageElement,
+  faces: Faces[],
+  containerWidth: number,
+  containerHeight: number,
+): Promise<Point | undefined> {
+  if (!TUNABLES.KEN_BURNS.SMARTCROP) {
+    return undefined;
+  }
+  if (!TUNABLES.KEN_BURNS.FACE_BOOST && faces.length > 0) {
+    return undefined;
+  }
+
+  const boosts =
+    TUNABLES.KEN_BURNS.FACE_BOOST && faces.length > 0
+      ? faces.map((face) => ({
+          x: (face.boundingBoxX1 / face.imageWidth) * imgRef.naturalWidth,
+          y: (face.boundingBoxY1 / face.imageHeight) * imgRef.naturalHeight,
+          width: ((face.boundingBoxX2 - face.boundingBoxX1) / face.imageWidth) * imgRef.naturalWidth,
+          height: ((face.boundingBoxY2 - face.boundingBoxY1) / face.imageHeight) * imgRef.naturalHeight,
+          weight: 1,
+        }))
+      : undefined;
+
+  const result = await smartcrop.crop(imgRef, {
+    width: containerWidth,
+    height: containerHeight,
+    ...(boosts && { boost: boosts }),
+  });
+
+  const { x, y, width, height } = result.topCrop;
+  return {
+    x: (x + width / 2) / imgRef.naturalWidth,
+    y: (y + height / 2) / imgRef.naturalHeight,
+  };
+}
+
+const selectKenBurnsFace = (faces: Faces[]): Faces | null => {
+  let best: Faces | null = null;
+  let bestArea = 0;
+  for (const face of faces) {
+    const area = (face.boundingBoxX2 - face.boundingBoxX1) * (face.boundingBoxY2 - face.boundingBoxY1);
+    if (area > bestArea) {
+      best = face;
+      bestArea = area;
+    }
+  }
+  return best;
+};
+
+export function computeKenBurnsKeyframes({
+  faces,
+  fallbackFaces,
+  smartCropCenter,
+  contentWidth,
+  contentHeight,
+  containerWidth,
+  containerHeight,
+  slideshowLook,
+  isCoverMode,
+  slideshowDelay,
+  assetId,
+}: KenBurnsInput): KenBurnsKeyframes {
+  const blurredBackground = slideshowLook === SlideshowLook.BlurredBackground;
+
+  const minZoom =
+    !blurredBackground && contentWidth > 0 && contentHeight > 0
+      ? Math.min(Math.max(containerWidth / contentWidth, containerHeight / contentHeight), 2)
+      : 1;
+
+  const slideDurationMs = slideshowDelay * 1000;
+  const maxZoomChange = KEN_BURNS_MAX_ZOOM_SPEED * (slideDurationMs / 1000);
+
+  const face = selectKenBurnsFace(faces) ?? selectKenBurnsFace(fallbackFaces);
+
+  let targetScale: number;
+  let endX = 0;
+  let endY = 0;
+
+  if (face && contentWidth > 0) {
+    const faceHeightFraction = (face.boundingBoxY2 - face.boundingBoxY1) / face.imageHeight;
+    const faceTargetZoom = (0.4 * containerHeight) / (faceHeightFraction * contentHeight);
+    targetScale = clamp(faceTargetZoom, Math.max(minZoom, 1.2), 2);
+    targetScale = clamp(targetScale, Math.max(minZoom, 1.2), minZoom + maxZoomChange);
+
+    const targetNormalizedX =
+      TUNABLES.KEN_BURNS.FACE_BOOST && smartCropCenter
+        ? smartCropCenter.x
+        : (face.boundingBoxX1 + face.boundingBoxX2) / 2 / face.imageWidth;
+    const targetNormalizedY =
+      TUNABLES.KEN_BURNS.FACE_BOOST && smartCropCenter
+        ? smartCropCenter.y
+        : (face.boundingBoxY1 + face.boundingBoxY2) / 2 / face.imageHeight;
+
+    endX = (((0.5 - targetNormalizedX) * contentWidth) / containerWidth) * 100;
+    endY = (((0.5 - targetNormalizedY) * contentHeight) / containerHeight) * 100;
+  } else {
+    targetScale = clamp(minZoom, 1.2, 2);
+    targetScale = clamp(targetScale, Math.max(minZoom, 1.2), minZoom + maxZoomChange);
+
+    if (smartCropCenter && contentWidth > 0) {
+      endX = (((0.5 - smartCropCenter.x) * contentWidth) / containerWidth) * 100;
+      endY = (((0.5 - smartCropCenter.y) * contentHeight) / containerHeight) * 100;
+    }
+  }
+
+  const clampWidth = blurredBackground || isCoverMode ? containerWidth : contentWidth;
+  const clampHeight = blurredBackground || isCoverMode ? containerHeight : contentHeight;
+  const maxTranslateX = Math.max(0, (clampWidth / (2 * containerWidth) - 1 / (2 * targetScale)) * 100);
+  const maxTranslateY = Math.max(0, (clampHeight / (2 * containerHeight) - 1 / (2 * targetScale)) * 100);
+  endX = clamp(endX, -maxTranslateX, maxTranslateX);
+  endY = clamp(endY, -maxTranslateY, maxTranslateY);
+
+  const panDist = Math.hypot(endX, endY);
+  const maxPan = KEN_BURNS_MAX_PAN_SPEED * (slideDurationMs / 1000);
+  if (panDist > maxPan && panDist > 0) {
+    const ratio = maxPan / panDist;
+    endX *= ratio;
+    endY *= ratio;
+  }
+
+  const zoomIn = Number.parseInt(assetId.at(-1) ?? '0', 16) < 8;
+
+  const startTransform = zoomIn
+    ? `scale(${minZoom}) translate(0%, 0%)`
+    : `scale(${targetScale}) translate(${endX}%, ${endY}%)`;
+  const endTransform = zoomIn
+    ? `scale(${targetScale}) translate(${endX}%, ${endY}%)`
+    : `scale(${minZoom}) translate(0%, 0%)`;
+
+  return { startTransform, endTransform, duration: slideDurationMs };
+}
+
+export class KenBurnsAnimation {
+  #animation: Animation | undefined;
+  #element: HTMLElement | undefined;
+  #cancelToken: { value: boolean } | undefined;
+
+  async startWithSmartCrop(
+    element: HTMLElement,
+    input: Omit<KenBurnsInput, 'smartCropCenter'> & { imgRef: HTMLImageElement },
+  ) {
+    this.cancel();
+    const token = { value: false };
+    this.#cancelToken = token;
+
+    const { imgRef, faces, fallbackFaces, containerWidth, containerHeight, ...rest } = input;
+    const allFaces = faces.length > 0 ? faces : fallbackFaces;
+    const smartCropCenter = await computeSmartCropCenter(imgRef, allFaces, containerWidth, containerHeight);
+
+    if (token.value) {
+      return;
+    }
+
+    const keyframes = computeKenBurnsKeyframes({
+      faces,
+      fallbackFaces,
+      smartCropCenter,
+      containerWidth,
+      containerHeight,
+      ...rest,
+    });
+    this.start(element, keyframes);
+  }
+
+  start(element: HTMLElement, { startTransform, endTransform, duration }: KenBurnsKeyframes) {
+    this.cancel();
+    this.#element = element;
+
+    element.style.transformOrigin = '50% 50%';
+    element.style.transform = startTransform;
+
+    const keyframes: Keyframe[] = [{ transform: startTransform, easing: 'ease-in-out' }, { transform: endTransform }];
+    this.#animation = element.animate(keyframes, { duration, fill: 'forwards' });
+  }
+
+  freeze() {
+    if (!this.#animation || !this.#element) {
+      return;
+    }
+    const frozen = getComputedStyle(this.#element).transform;
+    this.#animation.cancel();
+    this.#animation = undefined;
+    if (frozen && frozen !== 'none') {
+      this.#element.style.transform = frozen;
+    }
+  }
+
+  pause() {
+    this.#animation?.pause();
+  }
+
+  resume() {
+    this.#animation?.play();
+  }
+
+  cancel() {
+    if (this.#cancelToken) {
+      this.#cancelToken.value = true;
+      this.#cancelToken = undefined;
+    }
+    this.#animation?.cancel();
+    this.#animation = undefined;
+    this.#element?.style.removeProperty('transform-origin');
+    this.#element = undefined;
+  }
+}

web/src/lib/utils/transition-utils.ts

@@ -58,12 +58,15 @@ export async function crossfadeViewerContent(updateFn: () => void | Promise<void
 
   removeCrossfadeOverlay();
 
+  eventManager.emit('ViewTransitionOldSnapshotPending');
+
   const clone = viewerContent.cloneNode(true) as HTMLElement;
   Object.assign(clone.style, {
     position: 'absolute',
     inset: '0',
     zIndex: '1',
     pointerEvents: 'none',
+    backgroundColor: 'black',
   });
   delete clone.dataset.viewerContent;
   if (!viewerContent.parentElement) {

web/src/lib/utils/tunables.ts

@@ -31,4 +31,8 @@ export const TUNABLES = {
   IMAGE_THUMBNAIL: {
     THUMBHASH_FADE_DURATION: getNumber(storage.getItem('THUMBHASH_FADE_DURATION'), 100),
   },
+  KEN_BURNS: {
+    SMARTCROP: getBoolean(storage.getItem('KEN_BURNS.SMARTCROP'), true),
+    FACE_BOOST: getBoolean(storage.getItem('KEN_BURNS.FACE_BOOST'), true),
+  },
 };