From 5c11d1500843466e143017e9ae0c524289d264bd Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 26 Feb 2026 13:17:50 +0000 Subject: [PATCH] feat: add visual review workflow for automated PR screenshot comparison Adds a label-triggered GitHub Actions workflow that automatically generates before/after screenshots when web UI changes are made in a PR. Uses smart dependency analysis to only screenshot pages affected by the changed files. Key components: - Reverse dependency analyzer: traces changed files through the import graph to find which +page.svelte routes are affected - Screenshot scenarios: Playwright tests using existing mock-network infrastructure (no Docker/backend needed) for fast, deterministic screenshots - Pixel comparison: generates diff images highlighting changed pixels - GitHub Actions workflow: triggered by 'visual-review' label, posts results as a PR comment with change percentages per page https://claude.ai/code/session_01XSTqDJXuR4jaLN7SGm3uES --- .github/workflows/visual-review.yml | 330 +++++++++++++++++++++++++++ .gitignore | 1 + e2e/package.json | 5 +- e2e/playwright.screenshot.config.ts | 27 +++ e2e/src/screenshots/analyze-deps.ts | 249 ++++++++++++++++++++ e2e/src/screenshots/compare.ts | 257 +++++++++++++++++++++ e2e/src/screenshots/page-map.ts | 187 +++++++++++++++ e2e/src/screenshots/run-scenarios.ts | 123 ++++++++++ 8 files changed, 1178 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/visual-review.yml create mode 100644 e2e/playwright.screenshot.config.ts create mode 100644 e2e/src/screenshots/analyze-deps.ts create mode 100644 e2e/src/screenshots/compare.ts create mode 100644 e2e/src/screenshots/page-map.ts create mode 100644 e2e/src/screenshots/run-scenarios.ts diff --git a/.github/workflows/visual-review.yml b/.github/workflows/visual-review.yml new file mode 100644 index 0000000000..7169d79b91 --- /dev/null +++ b/.github/workflows/visual-review.yml @@ -0,0 +1,330 @@ +name: Visual Review + +on: + pull_request: + types: [labeled, synchronize] + +permissions: {} + +jobs: + visual-diff: + name: Visual Diff Screenshots + runs-on: ubuntu-latest + if: >- + (github.event.action == 'labeled' && github.event.label.name == 'visual-review') || + (github.event.action == 'synchronize' && contains(github.event.pull_request.labels.*.name, 'visual-review')) + permissions: + contents: read + pull-requests: write + steps: + - id: token + uses: immich-app/devtools/actions/create-workflow-token@05e16407c0a5492138bb38139c9d9bf067b40886 # create-workflow-token-action-v1.0.1 + with: + app-id: ${{ secrets.PUSH_O_MATIC_APP_ID }} + private-key: ${{ secrets.PUSH_O_MATIC_APP_KEY }} + + - name: Checkout PR branch + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + token: ${{ steps.token.outputs.token }} + fetch-depth: 0 + + - name: Setup pnpm + uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0 + + - name: Setup Node + uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 + with: + node-version-file: './e2e/.nvmrc' + cache: 'pnpm' + cache-dependency-path: '**/pnpm-lock.yaml' + + - name: Install Playwright + run: pnpm exec playwright install chromium --only-shell + working-directory: ./e2e + + - name: Determine changed web files + id: changed-files + run: | + BASE_SHA="${{ github.event.pull_request.base.sha }}" + HEAD_SHA="${{ github.event.pull_request.head.sha }}" + CHANGED=$(git diff --name-only "$BASE_SHA"..."$HEAD_SHA" -- 'web/' 'i18n/' 'open-api/typescript-sdk/' | head -500) + echo "files<> "$GITHUB_OUTPUT" + echo "$CHANGED" >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + # Check if there are any web changes at all + if [ -z "$CHANGED" ]; then + echo "has_changes=false" >> "$GITHUB_OUTPUT" + else + echo "has_changes=true" >> "$GITHUB_OUTPUT" + fi + + - name: Analyze affected routes + if: steps.changed-files.outputs.has_changes == 'true' + id: routes + run: | + # Install dependencies for the analyzer + pnpm install --frozen-lockfile + working_dir=$(pwd) + + # Run the dependency analyzer + CHANGED_FILES="${{ steps.changed-files.outputs.files }}" + ROUTES=$(echo "$CHANGED_FILES" | xargs npx tsx src/screenshots/analyze-deps.ts 2>/dev/null | grep "^ /" | sed 's/^ //' || true) + + echo "routes<> "$GITHUB_OUTPUT" + echo "$ROUTES" >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + if [ -z "$ROUTES" ]; then + echo "has_routes=false" >> "$GITHUB_OUTPUT" + else + echo "has_routes=true" >> "$GITHUB_OUTPUT" + + # Build the scenario filter JSON array + SCENARIO_NAMES=$(npx tsx -e " + import { getScenariosForRoutes } from './src/screenshots/page-map.ts'; + const routes = process.argv.slice(1); + const scenarios = getScenariosForRoutes(routes); + console.log(JSON.stringify(scenarios.map(s => s.name))); + " $ROUTES) + echo "scenarios=$SCENARIO_NAMES" >> "$GITHUB_OUTPUT" + fi + working-directory: ./e2e + + - name: Post initial comment + if: steps.changed-files.outputs.has_changes == 'true' && steps.routes.outputs.has_routes == 'true' + uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2 + with: + github-token: ${{ steps.token.outputs.token }} + message-id: 'visual-review' + message: | + ## Visual Review + + Generating screenshots for affected pages... + + Affected routes: + ``` + ${{ steps.routes.outputs.routes }} + ``` + + # === Screenshot PR version === + - name: Build SDK (PR) + if: steps.routes.outputs.has_routes == 'true' + run: pnpm install --frozen-lockfile && pnpm build + working-directory: ./open-api/typescript-sdk + + - name: Build web (PR) + if: steps.routes.outputs.has_routes == 'true' + run: pnpm install --frozen-lockfile && pnpm build + working-directory: ./web + + - name: Take screenshots (PR) + if: steps.routes.outputs.has_routes == 'true' + env: + PW_EXPERIMENTAL_SERVICE_WORKER_NETWORK_EVENTS: '1' + SCREENSHOT_OUTPUT_DIR: ${{ github.workspace }}/screenshots/pr + SCREENSHOT_SCENARIOS: ${{ steps.routes.outputs.scenarios }} + SCREENSHOT_BASE_URL: http://127.0.0.1:4173 + run: | + # Start the preview server in background + cd ../web && pnpm preview --port 4173 --host 127.0.0.1 & + SERVER_PID=$! + + # Wait for server to be ready + for i in $(seq 1 30); do + if curl -s http://127.0.0.1:4173 > /dev/null 2>&1; then + break + fi + sleep 1 + done + + # Run screenshot tests + pnpm exec playwright test --config playwright.screenshot.config.ts || true + + # Stop the preview server + kill $SERVER_PID 2>/dev/null || true + working-directory: ./e2e + + # === Screenshot base version === + - name: Checkout base web directory + if: steps.routes.outputs.has_routes == 'true' + run: | + BASE_SHA="${{ github.event.pull_request.base.sha }}" + # Restore web directory from base branch + git checkout "$BASE_SHA" -- web/ open-api/typescript-sdk/ i18n/ || true + + - name: Build SDK (base) + if: steps.routes.outputs.has_routes == 'true' + run: pnpm install --frozen-lockfile && pnpm build + working-directory: ./open-api/typescript-sdk + + - name: Build web (base) + if: steps.routes.outputs.has_routes == 'true' + run: pnpm install --frozen-lockfile && pnpm build + working-directory: ./web + + - name: Take screenshots (base) + if: steps.routes.outputs.has_routes == 'true' + env: + PW_EXPERIMENTAL_SERVICE_WORKER_NETWORK_EVENTS: '1' + SCREENSHOT_OUTPUT_DIR: ${{ github.workspace }}/screenshots/base + SCREENSHOT_SCENARIOS: ${{ steps.routes.outputs.scenarios }} + SCREENSHOT_BASE_URL: http://127.0.0.1:4173 + run: | + # Start the preview server in background + cd ../web && pnpm preview --port 4173 --host 127.0.0.1 & + SERVER_PID=$! + + # Wait for server to be ready + for i in $(seq 1 30); do + if curl -s http://127.0.0.1:4173 > /dev/null 2>&1; then + break + fi + sleep 1 + done + + # Run screenshot tests + pnpm exec playwright test --config playwright.screenshot.config.ts || true + + # Stop the preview server + kill $SERVER_PID 2>/dev/null || true + working-directory: ./e2e + + # === Compare and report === + - name: Compare screenshots + if: steps.routes.outputs.has_routes == 'true' + id: compare + run: | + npx tsx src/screenshots/compare.ts \ + "${{ github.workspace }}/screenshots/base" \ + "${{ github.workspace }}/screenshots/pr" \ + "${{ github.workspace }}/screenshots/diff" + working-directory: ./e2e + + - name: Upload screenshot artifacts + if: steps.routes.outputs.has_routes == 'true' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: visual-review-screenshots + path: screenshots/ + retention-days: 14 + + - name: Post comparison results + if: steps.routes.outputs.has_routes == 'true' + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + with: + github-token: ${{ steps.token.outputs.token }} + script: | + const fs = require('fs'); + const path = require('path'); + + const diffDir = path.join('${{ github.workspace }}', 'screenshots', 'diff'); + const baseDir = path.join('${{ github.workspace }}', 'screenshots', 'base'); + const prDir = path.join('${{ github.workspace }}', 'screenshots', 'pr'); + + // Read comparison results + let results; + try { + results = JSON.parse(fs.readFileSync(path.join(diffDir, 'results.json'), 'utf8')); + } catch { + // Post a simple message if comparison failed + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: '## Visual Review\n\nScreenshot comparison failed. Check the workflow artifacts for details.' + }); + return; + } + + const changed = results.filter(r => r.changePercent > 0.1); + const unchanged = results.filter(r => r.changePercent <= 0.1); + + let body = '## Visual Review\n\n'; + + if (changed.length === 0) { + body += 'No visual changes detected in the affected pages.\n'; + } else { + body += `Found **${changed.length}** page(s) with visual changes`; + if (unchanged.length > 0) { + body += ` (${unchanged.length} unchanged)`; + } + body += '.\n\n'; + body += '> **Note:** Download the `visual-review-screenshots` artifact to view full-resolution before/after/diff images.\n\n'; + + for (const result of changed) { + body += `### ${result.name}\n\n`; + + if (!result.baseExists) { + body += '**New page** — no base screenshot to compare against.\n\n'; + continue; + } + + if (!result.prExists) { + body += '**Removed page** — no longer exists in the PR.\n\n'; + continue; + } + + body += `Change: **${result.changePercent.toFixed(1)}%** of pixels (${result.diffPixels.toLocaleString()} pixels differ)\n\n`; + } + } + + if (unchanged.length > 0) { + body += '
\nUnchanged pages (' + unchanged.length + ')\n\n'; + for (const result of unchanged) { + body += `- ${result.name}\n`; + } + body += '\n
\n'; + } + + // Find and update existing comment or create new one + const comments = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + + const botComment = comments.data.find(c => + c.body && c.body.includes('## Visual Review') && c.body.includes('visual-review') + ); + + if (botComment) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } + + - name: No web changes + if: steps.changed-files.outputs.has_changes != 'true' + uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2 + with: + github-token: ${{ steps.token.outputs.token }} + message-id: 'visual-review' + message: | + ## Visual Review + + No web-related file changes detected in this PR. Visual review not needed. + + - name: No affected routes + if: steps.changed-files.outputs.has_changes == 'true' && steps.routes.outputs.has_routes != 'true' + uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2 + with: + github-token: ${{ steps.token.outputs.token }} + message-id: 'visual-review' + message: | + ## Visual Review + + Changed files don't affect any pages with screenshot scenarios configured. + To add coverage, define new scenarios in `e2e/src/screenshots/page-map.ts`. diff --git a/.gitignore b/.gitignore index 3220701cc6..b6db267045 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ open-api/typescript-sdk/build mobile/android/fastlane/report.xml mobile/ios/fastlane/report.xml +screenshots-output vite.config.js.timestamp-* .pnpm-store .devcontainer/library diff --git a/e2e/package.json b/e2e/package.json index ac1ae081b3..32fafe34a5 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -18,7 +18,10 @@ "format:fix": "prettier --write .", "lint": "eslint \"src/**/*.ts\" --max-warnings 0", "lint:fix": "pnpm run lint --fix", - "check": "tsc --noEmit" + "check": "tsc --noEmit", + "screenshots": "pnpm exec playwright test --config playwright.screenshot.config.ts", + "screenshots:compare": "npx tsx src/screenshots/compare.ts", + "screenshots:analyze": "npx tsx src/screenshots/analyze-deps.ts" }, "keywords": [], "author": "", diff --git a/e2e/playwright.screenshot.config.ts b/e2e/playwright.screenshot.config.ts new file mode 100644 index 0000000000..c2b039847e --- /dev/null +++ b/e2e/playwright.screenshot.config.ts @@ -0,0 +1,27 @@ +import { defineConfig, devices } from '@playwright/test'; + +const baseUrl = process.env.SCREENSHOT_BASE_URL ?? 'http://127.0.0.1:4173'; + +export default defineConfig({ + testDir: './src/screenshots', + testMatch: /run-scenarios\.ts/, + fullyParallel: false, + forbidOnly: !!process.env.CI, + retries: 0, + reporter: 'list', + use: { + baseURL: baseUrl, + screenshot: 'off', + trace: 'off', + }, + workers: 1, + projects: [ + { + name: 'screenshots', + use: { + ...devices['Desktop Chrome'], + viewport: { width: 1920, height: 1080 }, + }, + }, + ], +}); diff --git a/e2e/src/screenshots/analyze-deps.ts b/e2e/src/screenshots/analyze-deps.ts new file mode 100644 index 0000000000..f20fa4bc53 --- /dev/null +++ b/e2e/src/screenshots/analyze-deps.ts @@ -0,0 +1,249 @@ +/** + * Reverse dependency analyzer for the Immich web app. + * + * Given a list of changed files, traces upward through the import graph + * to find which +page.svelte routes are affected, then maps those to URL paths. + */ + +import { readFileSync, readdirSync, statSync } from 'node:fs'; +import { dirname, join, relative, resolve } from 'node:path'; + +const WEB_SRC = resolve(import.meta.dirname, '../../../web/src'); +const LIB_ALIAS = resolve(WEB_SRC, 'lib'); + +/** Collect all .svelte, .ts, .js files under web/src/ */ +function collectFiles(dir: string): string[] { + const results: string[] = []; + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + const stat = statSync(full); + if (stat.isDirectory()) { + if (entry === 'node_modules' || entry === '.svelte-kit') { + continue; + } + results.push(...collectFiles(full)); + } else if (/\.(svelte|ts|js)$/.test(entry)) { + results.push(full); + } + } + return results; +} + +/** Extract import specifiers from a file's source text. */ +function extractImports(source: string): string[] { + const specifiers: string[] = []; + + // Match: import ... from '...' / import '...' / export ... from '...' + const importRegex = /(?:import|export)\s+(?:[\s\S]*?\s+from\s+)?['"]([^'"]+)['"]/g; + let match; + while ((match = importRegex.exec(source)) !== null) { + specifiers.push(match[1]); + } + + // Match dynamic imports: import('...') + const dynamicRegex = /import\(\s*['"]([^'"]+)['"]\s*\)/g; + while ((match = dynamicRegex.exec(source)) !== null) { + specifiers.push(match[1]); + } + + return specifiers; +} + +/** Resolve an import specifier to an absolute file path (or null if external). */ +function resolveImport(specifier: string, fromFile: string, allFiles: Set): string | null { + // Handle $lib alias + let resolved: string; + if (specifier.startsWith('$lib/') || specifier === '$lib') { + resolved = specifier.replace('$lib', LIB_ALIAS); + } else if (specifier.startsWith('./') || specifier.startsWith('../')) { + resolved = resolve(dirname(fromFile), specifier); + } else { + // External package import — not relevant + return null; + } + + // Try exact match, then common extensions + const extensions = ['', '.ts', '.js', '.svelte', '/index.ts', '/index.js', '/index.svelte']; + for (const ext of extensions) { + const candidate = resolved + ext; + if (allFiles.has(candidate)) { + return candidate; + } + } + + return null; +} + +/** Build the forward dependency graph: file → set of files it imports. */ +function buildDependencyGraph(files: string[]): Map> { + const fileSet = new Set(files); + const graph = new Map>(); + + for (const file of files) { + const deps = new Set(); + graph.set(file, deps); + + try { + const source = readFileSync(file, 'utf8'); + for (const specifier of extractImports(source)) { + const resolved = resolveImport(specifier, file, fileSet); + if (resolved) { + deps.add(resolved); + } + } + } catch { + // Skip files that can't be read + } + } + + return graph; +} + +/** Invert the dependency graph: file → set of files that import it. */ +function buildReverseDependencyGraph(forwardGraph: Map>): Map> { + const reverse = new Map>(); + + for (const [file, deps] of forwardGraph) { + for (const dep of deps) { + let importers = reverse.get(dep); + if (!importers) { + importers = new Set(); + reverse.set(dep, importers); + } + importers.add(file); + } + } + + return reverse; +} + +/** BFS from changed files upward through reverse deps to find +page.svelte files. */ +function findAffectedPages(changedFiles: string[], reverseGraph: Map>): Set { + const visited = new Set(); + const pages = new Set(); + const queue = [...changedFiles]; + + while (queue.length > 0) { + const file = queue.shift()!; + if (visited.has(file)) { + continue; + } + visited.add(file); + + if (file.endsWith('+page.svelte') || file.endsWith('+layout.svelte')) { + pages.add(file); + // If it's a layout, keep tracing upward because the layout itself + // isn't a page — but the pages under it are affected. + // If it's a +page.svelte, we still want to continue in case + // this page is imported by others. + } + + const importers = reverseGraph.get(file); + if (importers) { + for (const importer of importers) { + if (!visited.has(importer)) { + queue.push(importer); + } + } + } + } + + // For +layout.svelte hits, also find all +page.svelte under the same directory tree + const layoutDirs: string[] = []; + for (const page of pages) { + if (page.endsWith('+layout.svelte')) { + layoutDirs.push(dirname(page)); + pages.delete(page); + } + } + + if (layoutDirs.length > 0) { + for (const file of reverseGraph.keys()) { + if (file.endsWith('+page.svelte')) { + for (const layoutDir of layoutDirs) { + if (file.startsWith(layoutDir)) { + pages.add(file); + } + } + } + } + // Also check the forward graph keys for page files under layout dirs + for (const layoutDir of layoutDirs) { + const allFiles = collectFiles(layoutDir); + for (const f of allFiles) { + if (f.endsWith('+page.svelte')) { + pages.add(f); + } + } + } + } + + return pages; +} + +/** Convert a +page.svelte file path to its URL route. */ +export function pageFileToRoute(pageFile: string): string { + const routesDir = resolve(WEB_SRC, 'routes'); + let rel = relative(routesDir, dirname(pageFile)); + + // Remove SvelteKit group markers: (user), (list), etc. + rel = rel.replaceAll(/\([^)]+\)\/?/g, ''); + + // Remove parameter segments: [albumId=id], [[photos=photos]], [[assetId=id]] + rel = rel.replaceAll(/\[\[?[^\]]+\]\]?\/?/g, ''); + + // Clean up trailing slashes and normalize + rel = rel.replaceAll(/\/+/g, '/').replace(/\/$/, ''); + + return '/' + rel; +} + +export interface AnalysisResult { + affectedPages: string[]; + affectedRoutes: string[]; +} + +/** Main entry: analyze which routes are affected by the given changed files. */ +export function analyzeAffectedRoutes(changedFiles: string[]): AnalysisResult { + // Resolve changed files to absolute paths relative to web/src + const webRoot = resolve(WEB_SRC, '..'); + const resolvedChanged = changedFiles + .filter((f) => f.startsWith('web/')) + .map((f) => resolve(webRoot, '..', f)) + .filter((f) => statSync(f, { throwIfNoEntry: false })?.isFile()); + + if (resolvedChanged.length === 0) { + return { affectedPages: [], affectedRoutes: [] }; + } + + const allFiles = collectFiles(WEB_SRC); + const forwardGraph = buildDependencyGraph(allFiles); + const reverseGraph = buildReverseDependencyGraph(forwardGraph); + + const pages = findAffectedPages(resolvedChanged, reverseGraph); + + const affectedPages = [...pages].sort(); + const affectedRoutes = [...new Set(affectedPages.map(pageFileToRoute))].sort(); + + return { affectedPages, affectedRoutes }; +} + +// CLI usage: node --import tsx analyze-deps.ts file1 file2 ... +if (process.argv[1]?.endsWith('analyze-deps.ts') || process.argv[1]?.endsWith('analyze-deps.js')) { + const files = process.argv.slice(2); + if (files.length === 0) { + console.log('Usage: analyze-deps.ts ...'); + console.log('Files should be relative to the repo root (e.g. web/src/lib/components/Button.svelte)'); + process.exit(1); + } + + const result = analyzeAffectedRoutes(files); + console.log('Affected pages:'); + for (const page of result.affectedPages) { + console.log(` ${page}`); + } + console.log('\nAffected routes:'); + for (const route of result.affectedRoutes) { + console.log(` ${route}`); + } +} diff --git a/e2e/src/screenshots/compare.ts b/e2e/src/screenshots/compare.ts new file mode 100644 index 0000000000..2819b26964 --- /dev/null +++ b/e2e/src/screenshots/compare.ts @@ -0,0 +1,257 @@ +/** + * Pixel-level comparison of base vs PR screenshots. + * + * Uses pixelmatch to generate diff images and calculate change percentages. + * + * Usage: + * npx tsx e2e/src/screenshots/compare.ts + */ + +import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { basename, join, resolve } from 'node:path'; +import { PNG } from 'pngjs'; + +// pixelmatch is a lightweight dependency — use a simple inline implementation +// based on the approach from the pixelmatch library to avoid adding a new dependency. +// The e2e package already has pngjs. + +function pixelMatch( + img1Data: Uint8Array, + img2Data: Uint8Array, + diffData: Uint8Array, + width: number, + height: number, +): number { + let diffCount = 0; + + for (let i = 0; i < img1Data.length; i += 4) { + const r1 = img1Data[i]; + const g1 = img1Data[i + 1]; + const b1 = img1Data[i + 2]; + + const r2 = img2Data[i]; + const g2 = img2Data[i + 1]; + const b2 = img2Data[i + 2]; + + const dr = Math.abs(r1 - r2); + const dg = Math.abs(g1 - g2); + const db = Math.abs(b1 - b2); + + // Threshold: if any channel differs by more than 25, mark as different + const isDiff = dr > 25 || dg > 25 || db > 25; + + if (isDiff) { + // Red highlight for diff pixels + diffData[i] = 255; + diffData[i + 1] = 0; + diffData[i + 2] = 0; + diffData[i + 3] = 255; + diffCount++; + } else { + // Dimmed original for unchanged pixels + const gray = Math.round(0.299 * r1 + 0.587 * g1 + 0.114 * b1); + diffData[i] = gray; + diffData[i + 1] = gray; + diffData[i + 2] = gray; + diffData[i + 3] = 128; + } + } + + return diffCount; +} + +export interface ComparisonResult { + name: string; + baseExists: boolean; + prExists: boolean; + diffPixels: number; + totalPixels: number; + changePercent: number; + diffImagePath: string | null; + baseImagePath: string | null; + prImagePath: string | null; +} + +export function compareScreenshots(baseDir: string, prDir: string, outputDir: string): ComparisonResult[] { + mkdirSync(outputDir, { recursive: true }); + + // Collect all screenshot names from both directories + const baseFiles = existsSync(baseDir) + ? new Set(readdirSync(baseDir).filter((f) => f.endsWith('.png'))) + : new Set(); + const prFiles = existsSync(prDir) + ? new Set(readdirSync(prDir).filter((f) => f.endsWith('.png'))) + : new Set(); + + const allNames = new Set([...baseFiles, ...prFiles]); + const results: ComparisonResult[] = []; + + for (const fileName of [...allNames].sort()) { + const name = basename(fileName, '.png'); + const basePath = join(baseDir, fileName); + const prPath = join(prDir, fileName); + const baseExists = baseFiles.has(fileName); + const prExists = prFiles.has(fileName); + + if (!baseExists || !prExists) { + // New or removed page + results.push({ + name, + baseExists, + prExists, + diffPixels: -1, + totalPixels: -1, + changePercent: 100, + diffImagePath: null, + baseImagePath: baseExists ? basePath : null, + prImagePath: prExists ? prPath : null, + }); + continue; + } + + // Load both PNGs + const basePng = PNG.sync.read(readFileSync(basePath)); + const prPng = PNG.sync.read(readFileSync(prPath)); + + // Handle size mismatches by comparing the overlapping region + const width = Math.max(basePng.width, prPng.width); + const height = Math.max(basePng.height, prPng.height); + + // Resize images to the same dimensions (pad with transparent) + const normalizedBase = normalizeImage(basePng, width, height); + const normalizedPr = normalizeImage(prPng, width, height); + + const diffPng = new PNG({ width, height }); + const totalPixels = width * height; + const diffPixels = pixelMatch( + normalizedBase, + normalizedPr, + diffPng.data as unknown as Uint8Array, + width, + height, + ); + + const diffImagePath = join(outputDir, `${name}-diff.png`); + writeFileSync(diffImagePath, PNG.sync.write(diffPng)); + + results.push({ + name, + baseExists, + prExists, + diffPixels, + totalPixels, + changePercent: totalPixels > 0 ? (diffPixels / totalPixels) * 100 : 0, + diffImagePath, + baseImagePath: basePath, + prImagePath: prPath, + }); + } + + return results; +} + +function normalizeImage(png: PNG, targetWidth: number, targetHeight: number): Uint8Array { + if (png.width === targetWidth && png.height === targetHeight) { + return png.data as unknown as Uint8Array; + } + + const data = new Uint8Array(targetWidth * targetHeight * 4); + for (let y = 0; y < targetHeight; y++) { + for (let x = 0; x < targetWidth; x++) { + const targetIdx = (y * targetWidth + x) * 4; + if (x < png.width && y < png.height) { + const sourceIdx = (y * png.width + x) * 4; + data[targetIdx] = png.data[sourceIdx]; + data[targetIdx + 1] = png.data[sourceIdx + 1]; + data[targetIdx + 2] = png.data[sourceIdx + 2]; + data[targetIdx + 3] = png.data[sourceIdx + 3]; + } else { + // Transparent padding + data[targetIdx + 3] = 0; + } + } + } + return data; +} + +/** Generate a markdown report for PR comment. */ +export function generateMarkdownReport(results: ComparisonResult[], artifactUrl: string): string { + const changed = results.filter((r) => r.changePercent > 0.1); + const unchanged = results.filter((r) => r.changePercent <= 0.1); + + if (changed.length === 0) { + return '## Visual Review\n\nNo visual changes detected in the affected pages.'; + } + + let md = '## Visual Review\n\n'; + md += `Found **${changed.length}** page(s) with visual changes`; + if (unchanged.length > 0) { + md += ` (${unchanged.length} unchanged)`; + } + md += '.\n\n'; + + for (const result of changed) { + md += `### ${result.name}\n\n`; + + if (!result.baseExists) { + md += '**New page** (no base screenshot to compare)\n\n'; + md += `| PR |\n|---|\n| ![${result.name} PR](${artifactUrl}/${result.name}.png) |\n\n`; + continue; + } + + if (!result.prExists) { + md += '**Removed page** (no PR screenshot)\n\n'; + continue; + } + + md += `Change: **${result.changePercent.toFixed(1)}%** (${result.diffPixels.toLocaleString()} pixels)\n\n`; + md += '| Base | PR | Diff |\n'; + md += '|------|-------|------|\n'; + md += `| ![base](${artifactUrl}/base/${result.name}.png) `; + md += `| ![pr](${artifactUrl}/pr/${result.name}.png) `; + md += `| ![diff](${artifactUrl}/diff/${result.name}-diff.png) |\n\n`; + } + + if (unchanged.length > 0) { + md += '
\nUnchanged pages\n\n'; + for (const result of unchanged) { + md += `- ${result.name}\n`; + } + md += '\n
\n'; + } + + return md; +} + +// CLI usage +if (process.argv[1]?.endsWith('compare.ts') || process.argv[1]?.endsWith('compare.js')) { + const [baseDir, prDir, outputDir] = process.argv.slice(2); + + if (!baseDir || !prDir || !outputDir) { + console.log('Usage: compare.ts '); + process.exit(1); + } + + const results = compareScreenshots( + resolve(baseDir), + resolve(prDir), + resolve(outputDir), + ); + + console.log('\nComparison Results:'); + console.log('=================='); + for (const r of results) { + const status = r.changePercent > 0.1 ? 'CHANGED' : 'unchanged'; + console.log(` ${r.name}: ${status} (${r.changePercent.toFixed(1)}%)`); + } + + const report = generateMarkdownReport(results, '.'); + const reportPath = join(resolve(outputDir), 'report.md'); + writeFileSync(reportPath, report); + console.log(`\nReport written to: ${reportPath}`); + + // Also output results as JSON for CI + const jsonPath = join(resolve(outputDir), 'results.json'); + writeFileSync(jsonPath, JSON.stringify(results, null, 2)); + console.log(`Results JSON written to: ${jsonPath}`); +} diff --git a/e2e/src/screenshots/page-map.ts b/e2e/src/screenshots/page-map.ts new file mode 100644 index 0000000000..25fb6a427a --- /dev/null +++ b/e2e/src/screenshots/page-map.ts @@ -0,0 +1,187 @@ +/** + * Maps URL routes to screenshot scenario keys. + * + * Routes discovered by the dependency analyzer are matched against this map + * to determine which screenshot scenarios to run. Routes not in this map + * are skipped (they don't have a scenario defined yet). + */ + +export interface ScenarioDefinition { + /** The URL path to navigate to */ + url: string; + /** Human-readable name for the screenshot file */ + name: string; + /** Which mock networks this scenario needs */ + mocks: ('base' | 'timeline' | 'memory')[]; + /** Optional: selector to wait for before screenshotting */ + waitForSelector?: string; + /** Optional: time to wait after page load (ms) for animations to settle */ + settleTime?: number; +} + +/** + * Map from route paths (as output by analyze-deps) to scenario definitions. + * A single route might map to multiple scenarios (e.g., different states). + */ +export const PAGE_SCENARIOS: Record = { + '/photos': [ + { + url: '/photos', + name: 'photos-timeline', + mocks: ['base', 'timeline'], + waitForSelector: '[data-thumbnail-focus-container]', + settleTime: 500, + }, + ], + '/albums': [ + { + url: '/albums', + name: 'albums-list', + mocks: ['base'], + settleTime: 300, + }, + ], + '/explore': [ + { + url: '/explore', + name: 'explore', + mocks: ['base'], + settleTime: 300, + }, + ], + '/favorites': [ + { + url: '/favorites', + name: 'favorites', + mocks: ['base', 'timeline'], + waitForSelector: '#asset-grid', + settleTime: 300, + }, + ], + '/archive': [ + { + url: '/archive', + name: 'archive', + mocks: ['base', 'timeline'], + waitForSelector: '#asset-grid', + settleTime: 300, + }, + ], + '/trash': [ + { + url: '/trash', + name: 'trash', + mocks: ['base', 'timeline'], + waitForSelector: '#asset-grid', + settleTime: 300, + }, + ], + '/people': [ + { + url: '/people', + name: 'people', + mocks: ['base'], + settleTime: 300, + }, + ], + '/sharing': [ + { + url: '/sharing', + name: 'sharing', + mocks: ['base'], + settleTime: 300, + }, + ], + '/search': [ + { + url: '/search', + name: 'search', + mocks: ['base'], + settleTime: 300, + }, + ], + '/memory': [ + { + url: '/memory', + name: 'memory', + mocks: ['base', 'memory'], + settleTime: 500, + }, + ], + '/user-settings': [ + { + url: '/user-settings', + name: 'user-settings', + mocks: ['base'], + settleTime: 300, + }, + ], + '/map': [ + { + url: '/map', + name: 'map', + mocks: ['base'], + settleTime: 500, + }, + ], + '/admin': [ + { + url: '/admin', + name: 'admin-dashboard', + mocks: ['base'], + settleTime: 300, + }, + ], + '/admin/system-settings': [ + { + url: '/admin/system-settings', + name: 'admin-system-settings', + mocks: ['base'], + settleTime: 300, + }, + ], + '/admin/users': [ + { + url: '/admin/users', + name: 'admin-users', + mocks: ['base'], + settleTime: 300, + }, + ], + '/auth/login': [ + { + url: '/auth/login', + name: 'login', + mocks: [], + settleTime: 300, + }, + ], + '/': [ + { + url: '/', + name: 'landing', + mocks: [], + settleTime: 300, + }, + ], +}; + +/** Given a list of routes from the analyzer, return the matching scenarios. */ +export function getScenariosForRoutes(routes: string[]): ScenarioDefinition[] { + const scenarios: ScenarioDefinition[] = []; + const seen = new Set(); + + for (const route of routes) { + const defs = PAGE_SCENARIOS[route]; + if (defs) { + for (const def of defs) { + if (!seen.has(def.name)) { + seen.add(def.name); + scenarios.push(def); + } + } + } + } + + return scenarios; +} diff --git a/e2e/src/screenshots/run-scenarios.ts b/e2e/src/screenshots/run-scenarios.ts new file mode 100644 index 0000000000..08989d4673 --- /dev/null +++ b/e2e/src/screenshots/run-scenarios.ts @@ -0,0 +1,123 @@ +/** + * Playwright script to capture screenshots for visual diff scenarios. + * + * Usage: + * npx playwright test --config e2e/playwright.screenshot.config.ts + * + * Environment variables: + * SCREENSHOT_SCENARIOS - JSON array of scenario names to run (from page-map.ts) + * If not set, runs all scenarios. + * SCREENSHOT_OUTPUT_DIR - Directory to save screenshots to. Defaults to e2e/screenshots-output. + */ + +import { faker } from '@faker-js/faker'; +import type { MemoryResponseDto } from '@immich/sdk'; +import { test } from '@playwright/test'; +import { mkdirSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { generateMemoriesFromTimeline } from 'src/ui/generators/memory'; +import { + createDefaultTimelineConfig, + generateTimelineData, + type TimelineAssetConfig, + type TimelineData, +} from 'src/ui/generators/timeline'; +import { setupBaseMockApiRoutes } from 'src/ui/mock-network/base-network'; +import { setupMemoryMockApiRoutes } from 'src/ui/mock-network/memory-network'; +import { setupTimelineMockApiRoutes, TimelineTestContext } from 'src/ui/mock-network/timeline-network'; +import { PAGE_SCENARIOS, type ScenarioDefinition } from './page-map'; + +const OUTPUT_DIR = process.env.SCREENSHOT_OUTPUT_DIR || resolve(import.meta.dirname, '../../../screenshots-output'); +const SCENARIO_FILTER: string[] | null = process.env.SCREENSHOT_SCENARIOS + ? JSON.parse(process.env.SCREENSHOT_SCENARIOS) + : null; + +// Collect scenarios to run +const allScenarios: ScenarioDefinition[] = []; +for (const defs of Object.values(PAGE_SCENARIOS)) { + for (const def of defs) { + if (!SCENARIO_FILTER || SCENARIO_FILTER.includes(def.name)) { + allScenarios.push(def); + } + } +} + +// Use a fixed seed so screenshots are deterministic across runs +faker.seed(42); + +let adminUserId: string; +let timelineData: TimelineData; +let timelineAssets: TimelineAssetConfig[]; +let memories: MemoryResponseDto[]; + +test.beforeAll(async () => { + adminUserId = faker.string.uuid(); + timelineData = generateTimelineData({ ...createDefaultTimelineConfig(), ownerId: adminUserId }); + + timelineAssets = []; + for (const timeBucket of timelineData.buckets.values()) { + timelineAssets.push(...timeBucket); + } + + memories = generateMemoriesFromTimeline( + timelineAssets, + adminUserId, + [ + { year: 2024, assetCount: 3 }, + { year: 2023, assetCount: 2 }, + ], + 42, + ); + + mkdirSync(OUTPUT_DIR, { recursive: true }); +}); + +for (const scenario of allScenarios) { + test(`Screenshot: ${scenario.name}`, async ({ context, page }) => { + // Set up mocks based on scenario requirements + if (scenario.mocks.includes('base')) { + await setupBaseMockApiRoutes(context, adminUserId); + } + + if (scenario.mocks.includes('timeline')) { + const testContext = new TimelineTestContext(); + testContext.adminId = adminUserId; + await setupTimelineMockApiRoutes(context, timelineData, { + albumAdditions: [], + assetDeletions: [], + assetArchivals: [], + assetFavorites: [], + }, testContext); + } + + if (scenario.mocks.includes('memory')) { + await setupMemoryMockApiRoutes(context, memories, { + memoryDeletions: [], + assetRemovals: new Map(), + }); + } + + // Navigate to the page + await page.goto(scenario.url, { waitUntil: 'networkidle' }); + + // Wait for specific selector if specified + if (scenario.waitForSelector) { + try { + await page.waitForSelector(scenario.waitForSelector, { timeout: 5000 }); + } catch { + // Continue with screenshot even if selector doesn't appear + } + } + + // Wait for animations/transitions to settle + if (scenario.settleTime) { + await page.waitForTimeout(scenario.settleTime); + } + + // Take the screenshot + await page.screenshot({ + path: resolve(OUTPUT_DIR, `${scenario.name}.png`), + fullPage: false, + }); + }); +}