wip: panorama tiling

server/src/constants.ts

@@ -55,6 +55,7 @@ export const LOGIN_URL = '/auth/login?autoLaunch=0';
 export const excludePaths = ['/.well-known/immich', '/custom.css', '/favicon.ico'];
 
 export const FACE_THUMBNAIL_SIZE = 250;
+export const TILE_TARGET_SIZE = 1024;
 
 type ModelInfo = { dimSize: number };
 export const CLIP_MODEL_INFO: Record<string, ModelInfo> = {

server/src/controllers/asset-media.controller.ts

@@ -7,6 +7,7 @@ import {
   Next,
   Param,
   ParseFilePipe,
+  ParseIntPipe,
   Post,
   Put,
   Query,
@@ -167,6 +168,26 @@ export class AssetMediaController {
     }
   }
 
+  @Get(':id/tiles/:level/:col/:row')
+  @FileResponse()
+  @Authenticated({ permission: Permission.AssetView, sharedLink: true })
+  @Endpoint({
+    summary: 'Get an image tile',
+    description: 'Download a specific tile from an image at the specified level and position',
+    history: new HistoryBuilder().added('v2.4.0').stable('v2.4.0'),
+  })
+  async getAssetTile(
+    @Auth() auth: AuthDto,
+    @Param() { id }: UUIDParamDto,
+    @Param('level', ParseIntPipe) level: number,
+    @Param('col', ParseIntPipe) col: number,
+    @Param('row', ParseIntPipe) row: number,
+    @Res() res: Response,
+    @Next() next: NextFunction,
+  ) {
+    await sendFile(res, next, () => this.service.getAssetTile(auth, id, level, col, row), this.logger);
+  }
+
   @Get(':id/video/playback')
   @FileResponse()
   @Authenticated({ permission: Permission.AssetView, sharedLink: true })

server/src/cores/storage.core.ts

@@ -24,7 +24,11 @@ export interface MoveRequest {
   };
 }
 
-export type GeneratedImageType = AssetPathType.Preview | AssetPathType.Thumbnail | AssetPathType.FullSize;
+export type GeneratedImageType =
+  | AssetPathType.Thumbnail
+  | AssetPathType.Preview
+  | AssetPathType.FullSize
+  | AssetPathType.Tiles;
 export type GeneratedAssetType = GeneratedImageType | AssetPathType.EncodedVideo;
 
 export type ThumbnailPathEntity = { id: string; ownerId: string };
@@ -105,7 +109,7 @@ export class StorageCore {
     return StorageCore.getNestedPath(StorageFolder.Thumbnails, person.ownerId, `${person.id}.jpeg`);
   }
 
-  static getImagePath(asset: ThumbnailPathEntity, type: GeneratedImageType, format: 'jpeg' | 'webp') {
+  static getImagePath(asset: ThumbnailPathEntity, type: GeneratedImageType, format: 'jpeg' | 'webp' | 'dz') {
     return StorageCore.getNestedPath(StorageFolder.Thumbnails, asset.ownerId, `${asset.id}-${type}.${format}`);
   }
 

server/src/enum.ts

@@ -44,6 +44,8 @@ export enum AssetFileType {
   FullSize = 'fullsize',
   Preview = 'preview',
   Thumbnail = 'thumbnail',
+  /** Folder structure containing tiles of the image */
+  Tiles = 'tiles',
 }
 
 export enum AlbumUserRole {
@@ -358,6 +360,8 @@ export enum AssetPathType {
   FullSize = 'fullsize',
   Preview = 'preview',
   Thumbnail = 'thumbnail',
+  /** Folder structure containing tiles of the image */
+  Tiles = 'tiles',
   EncodedVideo = 'encoded_video',
   Sidecar = 'sidecar',
 }

server/src/repositories/media.repository.ts

@@ -152,6 +152,19 @@ export class MediaRepository {
       .toFile(output);
   }
 
+  /**
+   * For output file path 'output.dz', this creates an 'output.dzi' file and 'output_files' directory containing tiles
+   */
+  async generateTiles(input: string | Buffer, options: GenerateThumbnailOptions, output: string): Promise<void> {
+    await this.getImageDecodingPipeline(input, options)
+      .toFormat(options.format)
+      .tile({
+        depth: 'one',
+        size: options.size,
+      })
+      .toFile(output);
+  }
+
   private getImageDecodingPipeline(input: string | Buffer, options: DecodeToBufferOptions) {
     let pipeline = sharp(input, {
       // some invalid images can still be processed by sharp, but we want to fail on them by default to avoid crashes

server/src/services/asset-media.service.ts

@@ -239,6 +239,33 @@ export class AssetMediaService extends BaseService {
     });
   }
 
+  async getAssetTile(auth: AuthDto, id: string, level: number, col: number, row: number): Promise<ImmichFileResponse> {
+    await this.requireAccess({ auth, permission: Permission.AssetView, ids: [id] });
+
+    const asset = await this.findOrFail(id);
+    const { tilesPath } = getAssetFiles(asset.files ?? []);
+    if (!tilesPath) {
+      // TODO: placeholder tiles.
+      return new ImmichFileResponse({
+        fileName: `${level}_${col}_${row}.jpg`,
+        path: `/data/sluis_files/0/${col}_${row}.jpg`,
+        contentType: 'image/jpg',
+        cacheControl: CacheControl.None,
+      });
+      throw new NotFoundException('Asset tiles not found');
+    }
+
+    const tileName = getFileNameWithoutExtension(asset.originalFileName) + `_${level}_${col}_${row}.jpg`;
+    const tilePath = tilesPath.path.replace('.dz', '_files') + `/${level}/${col}_${row}.jpg`;
+
+    return new ImmichFileResponse({
+      fileName: tileName,
+      path: tilePath,
+      contentType: 'image/jpg',
+      cacheControl: CacheControl.PrivateWithCache,
+    });
+  }
+
   async playbackVideo(auth: AuthDto, id: string): Promise<ImmichFileResponse> {
     await this.requireAccess({ auth, permission: Permission.AssetView, ids: [id] });
 

server/src/services/media.service.ts

@@ -1,5 +1,5 @@
 import { Injectable } from '@nestjs/common';
-import { FACE_THUMBNAIL_SIZE, JOBS_ASSET_PAGINATION_SIZE } from 'src/constants';
+import { FACE_THUMBNAIL_SIZE, JOBS_ASSET_PAGINATION_SIZE, TILE_TARGET_SIZE } from 'src/constants';
 import { StorageCore, ThumbnailPathEntity } from 'src/cores/storage.core';
 import { Exif } from 'src/database';
 import { OnEvent, OnJob } from 'src/decorators';
@@ -47,6 +47,15 @@ interface UpsertFileOptions {
   path: string;
 }
 
+interface TileInfo {
+  path: string;
+  info: {
+    width: number;
+    cols: number;
+    rows: number;
+  };
+}
+
 @Injectable()
 export class MediaService extends BaseService {
   videoInterfaces: VideoInterfaces = { dri: [], mali: false };
@@ -149,6 +158,8 @@ export class MediaService extends BaseService {
     await this.storageCore.moveAssetImage(asset, AssetPathType.FullSize, image.fullsize.format);
     await this.storageCore.moveAssetImage(asset, AssetPathType.Preview, image.preview.format);
     await this.storageCore.moveAssetImage(asset, AssetPathType.Thumbnail, image.thumbnail.format);
+    // TODO:
+    // await this.storageCore.moveAssetImage(asset, AssetPathType.Tiles, image.???.format);
     await this.storageCore.moveAssetVideo(asset);
 
     return JobStatus.Success;
@@ -171,6 +182,7 @@ export class MediaService extends BaseService {
       previewPath: string;
       thumbnailPath: string;
       fullsizePath?: string;
+      tileInfo?: TileInfo;
       thumbhash: Buffer;
     };
     if (asset.type === AssetType.Video || asset.originalFileName.toLowerCase().endsWith('.gif')) {
@@ -182,7 +194,7 @@ export class MediaService extends BaseService {
       return JobStatus.Skipped;
     }
 
-    const { previewFile, thumbnailFile, fullsizeFile } = getAssetFiles(asset.files);
+    const { previewFile, thumbnailFile, fullsizeFile, tilesPath } = getAssetFiles(asset.files);
     const toUpsert: UpsertFileOptions[] = [];
     if (previewFile?.path !== generated.previewPath) {
       toUpsert.push({ assetId: asset.id, path: generated.previewPath, type: AssetFileType.Preview });
@@ -196,6 +208,11 @@ export class MediaService extends BaseService {
       toUpsert.push({ assetId: asset.id, path: generated.fullsizePath, type: AssetFileType.FullSize });
     }
 
+    if (generated.tileInfo?.path && tilesPath?.path !== generated.tileInfo.path) {
+      // TODO: save tileInfo.info (width, cols, rows) somewhere in the db.
+      toUpsert.push({ assetId: asset.id, path: generated.tileInfo.path, type: AssetFileType.Tiles });
+    }
+
     if (toUpsert.length > 0) {
       await this.assetRepository.upsertFiles(toUpsert);
     }
@@ -220,6 +237,12 @@ export class MediaService extends BaseService {
       }
     }
 
+    if (tilesPath && tilesPath.path !== generated.tileInfo?.path) {
+      this.logger.debug(`Deleting old tiles for asset ${asset.id}`);
+      pathsToDelete.push(tilesPath.path.replace('.dz', '.dzi'));
+      await this.storageRepository.unlinkDir(tilesPath.path.replace('.dz', '_files'), { recursive: true });
+    }
+
     if (pathsToDelete.length > 0) {
       await Promise.all(pathsToDelete.map((path) => this.storageRepository.unlink(path)));
     }
@@ -314,6 +337,39 @@ export class MediaService extends BaseService {
       );
     }
 
+    // TODO: probably extract to helper method
+    let tileInfo: TileInfo | undefined;
+    if (asset.exifInfo.projectionType === 'EQUIRECTANGULAR') {
+      // TODO: get uncropped width from asset (FullPanoWidthPixels if present).
+      const originalSize = 12988;
+      // Get the number of tiles at the exact target size, rounded up (to at least 1 tile).
+      const numTilesExact = Math.ceil(originalSize / TILE_TARGET_SIZE);
+      // Then round up to the nearest power of 2 (photo-sphere-viewer requirement).
+      const numTiles = Math.pow(2, Math.ceil(Math.log2(numTilesExact)));
+      const tileSize = Math.ceil(originalSize / numTiles);
+
+      const tileOptions = {
+        format: image.preview.format,
+        size: tileSize,
+        quality: image.preview.quality,
+        ...thumbnailOptions,
+      };
+
+      tileInfo = {
+        path: StorageCore.getImagePath(asset, AssetPathType.Tiles, 'dz'),
+        info: {
+          width: originalSize,
+          cols: numTiles,
+          rows: numTiles / 2,
+        }
+      };
+      // TODO: reverse comment state
+      // TODO: handle cropped panoramas here. Tile as normal but save some offset?
+      // promises.push(this.mediaRepository.generateTiles(data, tileOptions, tileInfo.path));
+      console.log(tileOptions, tileInfo);
+      tileInfo = undefined;
+    }
+
     const outputs = await Promise.all(promises);
 
     if (asset.exifInfo.projectionType === 'EQUIRECTANGULAR') {
@@ -326,7 +382,7 @@ export class MediaService extends BaseService {
       await Promise.all(promises);
     }
 
-    return { previewPath, thumbnailPath, fullsizePath, thumbhash: outputs[0] as Buffer };
+    return { previewPath, thumbnailPath, fullsizePath, tileInfo, thumbhash: outputs[0] as Buffer };
   }
 
   @OnJob({ name: JobName.PersonGenerateThumbnail, queue: QueueName.ThumbnailGeneration })

server/src/utils/asset.util.ts

@@ -21,6 +21,7 @@ export const getAssetFiles = (files: AssetFile[]) => ({
   fullsizeFile: getAssetFile(files, AssetFileType.FullSize),
   previewFile: getAssetFile(files, AssetFileType.Preview),
   thumbnailFile: getAssetFile(files, AssetFileType.Thumbnail),
+  tilesPath: getAssetFile(files, AssetFileType.Tiles),
 });
 
 export const addAssets = async (