feat(web): face overlay hover UX and face editor zoom preservation

Change-Id: I7164305d7764bec54fa06b8738cd97fd6a6a6964

refactor(web): use asset metadata for face editor image dimensions instead of DOM

The face editor previously read naturalWidth/naturalHeight from the DOM
element via a $effect + load event listener. This was fragile on slow
hardware (ARM CI) because imgRef changes as AdaptiveImage progresses
through quality levels, and the DOM element's natural dimensions could
be 0 during transitions.

Now the face editor receives imageSize as a prop from the parent, derived
from the asset's metadata dimensions which are always available immediately.

Change-Id: Id4c3a59110feff4c50f429bbd744eac46a6a6964
Change-Id: I7164305d7764bec54fa06b8738cd97fd6a6a6964
This commit is contained in:
midzelis
2026-03-19 13:02:46 +00:00
parent ed04d87273
commit 590a9df7ec
21 changed files with 1147 additions and 208 deletions

View File

@@ -20,7 +20,7 @@ export {
toColumnarFormat,
} from './timeline/rest-response';
export type { Changes } from './timeline/rest-response';
export type { Changes, FaceData } from './timeline/rest-response';
export { randomImage, randomImageFromString, randomPreview, randomThumbnail } from './timeline/images';

View File

@@ -7,8 +7,10 @@ import {
AssetVisibility,
UserAvatarColor,
type AlbumResponseDto,
type AssetFaceWithoutPersonResponseDto,
type AssetResponseDto,
type ExifResponseDto,
type PersonWithFacesResponseDto,
type TimeBucketAssetResponseDto,
type TimeBucketsResponseDto,
type UserResponseDto,
@@ -284,7 +286,16 @@ const createDefaultOwner = (ownerId: string) => {
* Convert a TimelineAssetConfig to a full AssetResponseDto
* This matches the response from GET /api/assets/:id
*/
export function toAssetResponseDto(asset: MockTimelineAsset, owner?: UserResponseDto): AssetResponseDto {
export type FaceData = {
people: PersonWithFacesResponseDto[];
unassignedFaces: AssetFaceWithoutPersonResponseDto[];
};
export function toAssetResponseDto(
asset: MockTimelineAsset,
owner?: UserResponseDto,
faceData?: FaceData,
): AssetResponseDto {
const now = new Date().toISOString();
// Default owner if not provided
@@ -338,8 +349,8 @@ export function toAssetResponseDto(asset: MockTimelineAsset, owner?: UserRespons
exifInfo,
livePhotoVideoId: asset.livePhotoVideoId,
tags: [],
people: [],
unassignedFaces: [],
people: faceData?.people ?? [],
unassignedFaces: faceData?.unassignedFaces ?? [],
stack: asset.stack,
isOffline: false,
hasMetadata: true,

View File

@@ -1,5 +1,6 @@
import type { AssetFaceResponseDto, AssetResponseDto, PersonWithFacesResponseDto, SourceType } from '@immich/sdk';
import { BrowserContext } from '@playwright/test';
import { randomThumbnail } from 'src/ui/generators/timeline';
import { type FaceData, randomThumbnail } from 'src/ui/generators/timeline';
// Minimal valid H.264 MP4 (8x8px, 1 frame) that browsers can decode to get videoWidth/videoHeight
const MINIMAL_MP4_BASE64 =
@@ -125,3 +126,84 @@ export const setupFaceEditorMockApiRoutes = async (
});
});
};
export type MockFaceSpec = {
personId: string;
personName: string;
faceId: string;
boundingBoxX1: number;
boundingBoxY1: number;
boundingBoxX2: number;
boundingBoxY2: number;
};
const toPersonResponseDto = (spec: MockFaceSpec) => ({
id: spec.personId,
name: spec.personName,
birthDate: null,
isHidden: false,
thumbnailPath: `/upload/thumbs/${spec.personId}.jpeg`,
updatedAt: '2025-01-01T00:00:00.000Z',
});
const toBoundingBox = (spec: MockFaceSpec, imageWidth: number, imageHeight: number) => ({
id: spec.faceId,
imageWidth,
imageHeight,
boundingBoxX1: spec.boundingBoxX1,
boundingBoxY1: spec.boundingBoxY1,
boundingBoxX2: spec.boundingBoxX2,
boundingBoxY2: spec.boundingBoxY2,
});
export const createMockFaceData = (specs: MockFaceSpec[], imageWidth: number, imageHeight: number): FaceData => {
const people: PersonWithFacesResponseDto[] = specs.map((spec) => ({
...toPersonResponseDto(spec),
faces: [toBoundingBox(spec, imageWidth, imageHeight)],
}));
return { people, unassignedFaces: [] };
};
export const createMockAssetFaces = (
specs: MockFaceSpec[],
imageWidth: number,
imageHeight: number,
): AssetFaceResponseDto[] => {
return specs.map((spec) => ({
...toBoundingBox(spec, imageWidth, imageHeight),
person: toPersonResponseDto(spec),
sourceType: 'machine-learning' as SourceType,
}));
};
export const setupGetFacesMockApiRoute = async (context: BrowserContext, faces: AssetFaceResponseDto[]) => {
await context.route('**/api/faces?*', async (route, request) => {
if (request.method() !== 'GET') {
return route.fallback();
}
return route.fulfill({
status: 200,
contentType: 'application/json',
json: faces,
});
});
};
export const setupFaceOverlayMockApiRoutes = async (context: BrowserContext, assetDto: AssetResponseDto) => {
await context.route('**/api/assets/*', async (route, request) => {
if (request.method() !== 'GET') {
return route.fallback();
}
const url = new URL(request.url());
const assetId = url.pathname.split('/').at(-1);
if (assetId !== assetDto.id) {
return route.fallback();
}
return route.fulfill({
status: 200,
contentType: 'application/json',
json: assetDto,
});
});
};

View File

@@ -10,16 +10,21 @@ import { assetViewerUtils } from '../timeline/utils';
import { setupAssetViewerFixture } from './utils';
const waitForSelectorTransition = async (page: Page) => {
await page.waitForFunction(
() => {
const selector = document.querySelector('#face-selector') as HTMLElement | null;
if (!selector) {
return false;
}
return selector.getAnimations({ subtree: false }).every((animation) => animation.playState === 'finished');
},
undefined,
{ timeout: 1000, polling: 50 },
await expect(page.locator('#face-editor-data')).toHaveAttribute('data-face-width', /^[1-9]/, { timeout: 10_000 });
await page.locator('#face-selector').evaluate(
(el) =>
new Promise<void>((resolve) => {
requestAnimationFrame(() =>
requestAnimationFrame(() => {
const animations = el.getAnimations();
if (animations.length === 0) {
resolve();
return;
}
void Promise.all(animations.map((a) => a.finished)).then(() => resolve());
}),
);
}),
);
};
@@ -95,7 +100,7 @@ test.describe('face-editor', () => {
await page.mouse.down();
await page.mouse.move(centerX + deltaX, centerY + deltaY, { steps: 5 });
await page.mouse.up();
await page.waitForTimeout(300);
await waitForSelectorTransition(page);
};
test('Face editor opens with person list', async ({ page }) => {
@@ -149,7 +154,7 @@ test.describe('face-editor', () => {
await expect(page.getByRole('dialog')).toBeVisible();
});
test('Confirming tag calls createFace API and closes editor', async ({ page }) => {
test('Confirming tag calls createFace API with valid coordinates and closes editor', async ({ page }) => {
const asset = selectRandom(fixture.assets, rng);
await openFaceEditor(page, asset);
@@ -163,8 +168,15 @@ test.describe('face-editor', () => {
await expect(page.locator('#face-editor')).toBeHidden();
expect(faceCreateCapture.requests).toHaveLength(1);
expect(faceCreateCapture.requests[0].assetId).toBe(asset.id);
expect(faceCreateCapture.requests[0].personId).toBe(personToTag.id);
const request = faceCreateCapture.requests[0];
expect(request.assetId).toBe(asset.id);
expect(request.personId).toBe(personToTag.id);
expect(request.x).toBeGreaterThanOrEqual(0);
expect(request.y).toBeGreaterThanOrEqual(0);
expect(request.width).toBeGreaterThan(0);
expect(request.height).toBeGreaterThan(0);
expect(request.x + request.width).toBeLessThanOrEqual(request.imageWidth);
expect(request.y + request.height).toBeLessThanOrEqual(request.imageHeight);
});
test('Cancel button closes face editor', async ({ page }) => {
@@ -282,4 +294,39 @@ test.describe('face-editor', () => {
expect(afterDrag.left).toBeGreaterThan(beforeDrag.left + 50);
expect(afterDrag.top).toBeGreaterThan(beforeDrag.top + 20);
});
test('Cancel on confirmation dialog keeps face editor open', async ({ page }) => {
const asset = selectRandom(fixture.assets, rng);
await openFaceEditor(page, asset);
const personToTag = mockPeople[0];
await page.locator('#face-selector').getByText(personToTag.name).click();
await expect(page.getByRole('dialog')).toBeVisible();
await page
.getByRole('dialog')
.getByRole('button', { name: /cancel/i })
.click();
await expect(page.getByRole('dialog')).toBeHidden();
await expect(page.locator('#face-selector')).toBeVisible();
await expect(page.locator('#face-editor')).toBeVisible();
expect(faceCreateCapture.requests).toHaveLength(0);
});
test('Clicking on face rect center does not reposition it', async ({ page }) => {
const asset = selectRandom(fixture.assets, rng);
await openFaceEditor(page, asset);
const beforeClick = await getFaceBoxRect(page);
const centerX = beforeClick.left + beforeClick.width / 2;
const centerY = beforeClick.top + beforeClick.height / 2;
await page.mouse.click(centerX, centerY);
await waitForSelectorTransition(page);
const afterClick = await getFaceBoxRect(page);
expect(Math.abs(afterClick.left - beforeClick.left)).toBeLessThan(3);
expect(Math.abs(afterClick.top - beforeClick.top)).toBeLessThan(3);
});
});

View File

@@ -0,0 +1,340 @@
import { expect, test } from '@playwright/test';
import { toAssetResponseDto } from 'src/ui/generators/timeline';
import {
createMockAssetFaces,
createMockFaceData,
createMockPeople,
type MockFaceSpec,
setupFaceEditorMockApiRoutes,
setupFaceOverlayMockApiRoutes,
setupGetFacesMockApiRoute,
} from 'src/ui/mock-network/face-editor-network';
import { assetViewerUtils } from '../timeline/utils';
import { ensureDetailPanelVisible, setupAssetViewerFixture } from './utils';
test.describe.configure({ mode: 'parallel' });
const FACE_SPECS: MockFaceSpec[] = [
{
personId: 'person-alice',
personName: 'Alice Johnson',
faceId: 'face-alice',
boundingBoxX1: 1000,
boundingBoxY1: 500,
boundingBoxX2: 1500,
boundingBoxY2: 1200,
},
{
personId: 'person-bob',
personName: 'Bob Smith',
faceId: 'face-bob',
boundingBoxX1: 2000,
boundingBoxY1: 800,
boundingBoxX2: 2400,
boundingBoxY2: 1600,
},
];
const setupFaceMocks = async (
context: import('@playwright/test').BrowserContext,
fixture: ReturnType<typeof setupAssetViewerFixture>,
) => {
const mockPeople = createMockPeople(4);
const faceData = createMockFaceData(
FACE_SPECS,
fixture.primaryAssetDto.width ?? 3000,
fixture.primaryAssetDto.height ?? 4000,
);
const assetDtoWithFaces = toAssetResponseDto(fixture.primaryAsset, undefined, faceData);
await setupFaceOverlayMockApiRoutes(context, assetDtoWithFaces);
await setupFaceEditorMockApiRoutes(context, mockPeople, { requests: [] });
};
test.describe('face overlay bounding boxes', () => {
const fixture = setupAssetViewerFixture(901);
test.beforeEach(async ({ context }) => {
await setupFaceMocks(context, fixture);
});
test('face overlay divs render with correct aria labels', async ({ page }) => {
await page.goto(`/photos/${fixture.primaryAsset.id}`);
await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);
const aliceOverlay = page.getByLabel('Person: Alice Johnson');
const bobOverlay = page.getByLabel('Person: Bob Smith');
await expect(aliceOverlay).toBeVisible();
await expect(bobOverlay).toBeVisible();
});
test('face overlay shows border on hover', async ({ page }) => {
await page.goto(`/photos/${fixture.primaryAsset.id}`);
await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);
const aliceOverlay = page.getByLabel('Person: Alice Johnson');
await expect(aliceOverlay).toBeVisible();
const activeBorder = page.locator('[data-viewer-content] .border-solid.border-white.border-3');
await expect(activeBorder).toHaveCount(0);
await aliceOverlay.hover();
await expect(activeBorder).toHaveCount(1);
});
test('face name tooltip appears on hover', async ({ page }) => {
await page.goto(`/photos/${fixture.primaryAsset.id}`);
await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);
const aliceOverlay = page.getByLabel('Person: Alice Johnson');
await expect(aliceOverlay).toBeVisible();
await aliceOverlay.hover();
const nameTooltip = page.locator('[data-viewer-content]').getByText('Alice Johnson');
await expect(nameTooltip).toBeVisible();
});
test('face overlays hidden in face edit mode', async ({ page }) => {
await page.goto(`/photos/${fixture.primaryAsset.id}`);
await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);
const aliceOverlay = page.getByLabel('Person: Alice Johnson');
await expect(aliceOverlay).toBeVisible();
await ensureDetailPanelVisible(page);
await page.getByLabel('Tag people').click();
await page.locator('#face-selector').waitFor({ state: 'visible' });
await expect(aliceOverlay).toBeHidden();
});
test('face overlay hover works after exiting face edit mode', async ({ page }) => {
await page.goto(`/photos/${fixture.primaryAsset.id}`);
await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);
const aliceOverlay = page.getByLabel('Person: Alice Johnson');
await expect(aliceOverlay).toBeVisible();
await ensureDetailPanelVisible(page);
await page.getByLabel('Tag people').click();
await page.locator('#face-selector').waitFor({ state: 'visible' });
await expect(aliceOverlay).toBeHidden();
await page.getByRole('button', { name: /cancel/i }).click();
await expect(page.locator('#face-selector')).toBeHidden();
await expect(aliceOverlay).toBeVisible();
const activeBorder = page.locator('[data-viewer-content] .border-solid.border-white.border-3');
await expect(activeBorder).toHaveCount(0);
await aliceOverlay.hover();
await expect(activeBorder).toHaveCount(1);
});
});
test.describe('zoom and face editor interaction', () => {
const fixture = setupAssetViewerFixture(902);
test.beforeEach(async ({ context }) => {
await setupFaceMocks(context, fixture);
});
test('zoom is preserved when entering face edit mode', async ({ page }) => {
await page.goto(`/photos/${fixture.primaryAsset.id}`);
await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);
const { width, height } = page.viewportSize()!;
await page.mouse.move(width / 2, height / 2);
await page.mouse.wheel(0, -1);
const imgLocator = page.locator('[data-viewer-content] img[data-testid="preview"]');
await expect(async () => {
const transform = await imgLocator.evaluate((element) => {
return getComputedStyle(element.closest('[style*="transform"]') ?? element).transform;
});
expect(transform).not.toBe('none');
expect(transform).not.toBe('');
}).toPass({ timeout: 2000 });
await ensureDetailPanelVisible(page);
await page.getByLabel('Tag people').click();
await page.locator('#face-selector').waitFor({ state: 'visible' });
await expect(page.locator('#face-editor')).toBeVisible();
const afterTransform = await imgLocator.evaluate((element) => {
return getComputedStyle(element.closest('[style*="transform"]') ?? element).transform;
});
expect(afterTransform).not.toBe('none');
});
test('modifier+drag pans zoomed image without repositioning face rect', async ({ page }) => {
await page.goto(`/photos/${fixture.primaryAsset.id}`);
await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);
const { width, height } = page.viewportSize()!;
await page.mouse.move(width / 2, height / 2);
for (let i = 0; i < 10; i++) {
await page.mouse.wheel(0, -3);
}
const imgLocator = page.locator('[data-viewer-content] img[data-testid="preview"]');
await expect(async () => {
const transform = await imgLocator.evaluate((element) => {
return getComputedStyle(element.closest('[style*="transform"]') ?? element).transform;
});
expect(transform).not.toBe('none');
}).toPass({ timeout: 2000 });
await ensureDetailPanelVisible(page);
await page.getByLabel('Tag people').click();
await page.locator('#face-selector').waitFor({ state: 'visible' });
const dataEl = page.locator('#face-editor-data');
await expect(dataEl).toHaveAttribute('data-face-width', /^[1-9]/);
const beforeLeft = Number(await dataEl.getAttribute('data-face-left'));
const beforeTop = Number(await dataEl.getAttribute('data-face-top'));
const transformBefore = await imgLocator.evaluate((element) => {
return getComputedStyle(element.closest('[style*="transform"]') ?? element).transform;
});
const panModifier = await page.evaluate(() =>
/Mac|iPhone|iPad|iPod/.test(navigator.userAgent) ? 'Meta' : 'Control',
);
await page.keyboard.down(panModifier);
// Verify face editor becomes transparent to pointer events
await expect(async () => {
const pe = await dataEl.evaluate((el) => getComputedStyle(el).pointerEvents);
expect(pe).toBe('none');
}).toPass({ timeout: 2000 });
await page.mouse.move(width / 2, height / 2);
await page.mouse.down();
await page.mouse.move(width / 2 + 100, height / 2 + 50, { steps: 5 });
await page.mouse.up();
await page.keyboard.up(panModifier);
const transformAfter = await imgLocator.evaluate((element) => {
return getComputedStyle(element.closest('[style*="transform"]') ?? element).transform;
});
expect(transformAfter).not.toBe(transformBefore);
// Extract translate values from matrix(a, b, c, d, tx, ty)
const parseTranslate = (matrix: string) => {
const values =
matrix
.match(/matrix\((.+)\)/)?.[1]
.split(',')
.map(Number) ?? [];
return { tx: values[4], ty: values[5] };
};
const panBefore = parseTranslate(transformBefore);
const panAfter = parseTranslate(transformAfter);
const panDeltaX = panAfter.tx - panBefore.tx;
const panDeltaY = panAfter.ty - panBefore.ty;
// Face rect screen position should have moved by the same amount as the pan
// (it follows the image), NOT been repositioned by a click
const afterLeft = Number(await dataEl.getAttribute('data-face-left'));
const afterTop = Number(await dataEl.getAttribute('data-face-top'));
const faceDeltaX = afterLeft - beforeLeft;
const faceDeltaY = afterTop - beforeTop;
expect(Math.abs(faceDeltaX - panDeltaX)).toBeLessThan(3);
expect(Math.abs(faceDeltaY - panDeltaY)).toBeLessThan(3);
});
});
test.describe('face overlay via detail panel interaction', () => {
const fixture = setupAssetViewerFixture(903);
test.beforeEach(async ({ context }) => {
await setupFaceMocks(context, fixture);
});
test('hovering person in detail panel shows face overlay border', async ({ page }) => {
await page.goto(`/photos/${fixture.primaryAsset.id}`);
await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);
await ensureDetailPanelVisible(page);
const personLink = page.locator('#detail-panel a').filter({ hasText: 'Alice Johnson' });
await expect(personLink).toBeVisible();
const activeBorder = page.locator('[data-viewer-content] .border-solid.border-white.border-3');
await expect(activeBorder).toHaveCount(0);
await personLink.hover();
await expect(activeBorder).toHaveCount(1);
});
test('touch pointer on person in detail panel shows face overlay border', async ({ page }) => {
await page.goto(`/photos/${fixture.primaryAsset.id}`);
await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);
await ensureDetailPanelVisible(page);
const personLink = page.locator('#detail-panel a').filter({ hasText: 'Alice Johnson' });
await expect(personLink).toBeVisible();
const activeBorder = page.locator('[data-viewer-content] .border-solid.border-white.border-3');
await expect(activeBorder).toHaveCount(0);
// Simulate a touch-type pointerover (the fix changed from onmouseover to onpointerover,
// which fires for touch pointers unlike mouseover)
await personLink.dispatchEvent('pointerover', { pointerType: 'touch' });
await expect(activeBorder).toHaveCount(1);
});
test('hovering person in detail panel works after exiting face edit mode', async ({ page }) => {
await page.goto(`/photos/${fixture.primaryAsset.id}`);
await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);
await ensureDetailPanelVisible(page);
await page.getByLabel('Tag people').click();
await page.locator('#face-selector').waitFor({ state: 'visible' });
await page.getByRole('button', { name: /cancel/i }).click();
await expect(page.locator('#face-selector')).toBeHidden();
const personLink = page.locator('#detail-panel a').filter({ hasText: 'Alice Johnson' });
await expect(personLink).toBeVisible();
const activeBorder = page.locator('[data-viewer-content] .border-solid.border-white.border-3');
await personLink.hover();
await expect(activeBorder).toHaveCount(1);
});
});
test.describe('face overlay via edit faces side panel', () => {
const fixture = setupAssetViewerFixture(904);
test.beforeEach(async ({ context }) => {
await setupFaceMocks(context, fixture);
const assetFaces = createMockAssetFaces(
FACE_SPECS,
fixture.primaryAssetDto.width ?? 3000,
fixture.primaryAssetDto.height ?? 4000,
);
await setupGetFacesMockApiRoute(context, assetFaces);
});
test('hovering person in edit faces panel shows face overlay border', async ({ page }) => {
await page.goto(`/photos/${fixture.primaryAsset.id}`);
await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);
await ensureDetailPanelVisible(page);
await page.getByLabel('Edit people').click();
const faceThumbnail = page.getByTestId('face-thumbnail').first();
await expect(faceThumbnail).toBeVisible();
const activeBorder = page.locator('[data-viewer-content] .border-solid.border-white.border-3');
await expect(activeBorder).toHaveCount(0);
await faceThumbnail.hover();
await expect(activeBorder).toHaveCount(1);
});
});