impl tile generation

server/src/controllers/asset-media.controller.ts

@@ -190,11 +190,11 @@ export class AssetMediaController {
   @FileResponse()
   @Authenticated({ permission: Permission.AssetView, sharedLink: true })
   @Endpoint({
-    summary: 'Get an image tile',
-    description: 'Download a specific tile from an image at the specified level and position',
-    history: new HistoryBuilder().added('v2.4.0').stable('v2.4.0'),
+    summary: 'View an asset tile',
+    description: 'Download a specific tile from an image at the specified level - must currently be 0 - and position',
+    history: new HistoryBuilder().added('v2.7.0').stable('v2.7.0'),
   })
-  async getAssetTile(
+  async viewAssetTile(
     @Auth() auth: AuthDto,
     @Param() { id }: UUIDParamDto,
     @Param('level', ParseIntPipe) level: number,
@@ -203,7 +203,10 @@ export class AssetMediaController {
     @Res() res: Response,
     @Next() next: NextFunction,
   ) {
-    await sendFile(res, next, () => this.service.getAssetTile(auth, id, level, col, row), this.logger);
+    if (level !== 0) {
+      throw new Error(`Invalid level ${level}`);
+    }
+    await sendFile(res, next, () => this.service.viewAssetTile(auth, id, level, col, row), this.logger);
   }
 
   @Get(':id/video/playback')

server/src/cores/storage.core.ts

@@ -9,7 +9,6 @@ import {
   PersonPathType,
   RawExtractedFormat,
   StorageFolder,
-  TilesFormat,
 } from 'src/enum';
 import { AssetRepository } from 'src/repositories/asset.repository';
 import { ConfigRepository } from 'src/repositories/config.repository';
@@ -33,16 +32,9 @@ export interface MoveRequest {
   };
 }
 
-export type GeneratedImageType =
-  | AssetPathType.Thumbnail
-  | AssetPathType.Preview
-  | AssetPathType.FullSize
-  | AssetPathType.Tiles;
-export type GeneratedAssetType = GeneratedImageType | AssetPathType.EncodedVideo;
-
 export type ThumbnailPathEntity = { id: string; ownerId: string };
 
-export type ImagePathOptions = { fileType: AssetFileType; format: ImageFormat | RawExtractedFormat | TilesFormat; isEdited: boolean };
+export type ImagePathOptions = { fileType: AssetFileType; format: ImageFormat | RawExtractedFormat; isEdited: boolean };
 
 let instance: StorageCore | null;
 
@@ -128,6 +120,10 @@ export class StorageCore {
     );
   }
 
+  static getTilesFolder(asset: ThumbnailPathEntity) {
+    return StorageCore.getNestedPath(StorageFolder.Thumbnails, asset.ownerId, `${asset.id}_tiles`);
+  }
+
   static getEncodedVideoPath(asset: ThumbnailPathEntity) {
     return StorageCore.getNestedPath(StorageFolder.EncodedVideo, asset.ownerId, `${asset.id}.mp4`);
   }
@@ -161,6 +157,16 @@ export class StorageCore {
     });
   }
 
+  async moveAssetTiles(asset: StorageAsset) {
+    const oldDir = getAssetFile(asset.files, AssetFileType.Tiles, { isEdited: false });
+    return this.moveFile({
+      entityId: asset.id,
+      pathType: AssetPathType.Tiles,
+      oldPath: oldDir?.path || null,
+      newPath: StorageCore.getTilesFolder(asset),
+    })
+  }
+
   async moveAssetVideo(asset: StorageAsset) {
     return this.moveFile({
       entityId: asset.id,

server/src/enum.ts

@@ -373,9 +373,6 @@ export enum ManualJobName {
 
 export enum AssetPathType {
   Original = 'original',
-  FullSize = 'fullsize',
-  Preview = 'preview',
-  Thumbnail = 'thumbnail',
   /** Folder structure containing tiles of the image */
   Tiles = 'tiles',
   EncodedVideo = 'encoded_video',
@@ -463,10 +460,6 @@ export enum RawExtractedFormat {
   Jxl = 'jxl',
 }
 
-export enum TilesFormat {
-  Dz = 'dz',
-}
-
 export enum LogLevel {
   Verbose = 'verbose',
   Debug = 'debug',

server/src/repositories/asset.repository.ts

@@ -1128,6 +1128,19 @@ export class AssetRepository {
       .executeTakeFirstOrThrow();
   }
 
+  @GenerateSql({ params: [DummyValue.UUID] })
+  async getForTiles(id: string) {
+    // TODO: we don't actually need original path and file name. Plain 'select asset_file.path from asset_file where type = tiles and assetId = id;'?
+    return this.db
+      .selectFrom('asset')
+      .where('asset.id', '=', id)
+      .leftJoin('asset_file', (join) =>
+        join.onRef('asset.id', '=', 'asset_file.assetId').on('asset_file.type', '=', AssetFileType.Tiles),
+      )
+      .select(['asset.originalPath', 'asset.originalFileName', 'asset_file.path as path'])
+      .executeTakeFirstOrThrow();
+  }
+
   @GenerateSql({ params: [DummyValue.UUID] })
   async getForVideo(id: string) {
     return this.db

server/src/repositories/media.repository.ts

@@ -123,6 +123,23 @@ export class MediaRepository {
     }
   }
 
+  async copyTagGroup(tagGroup: string, source: string, target: string): Promise<boolean> {
+    try {
+      await exiftool.write(
+        target,
+        {},
+        {
+          ignoreMinorErrors: true,
+          writeArgs: ['-TagsFromFile', source, `-${tagGroup}:all>${tagGroup}:all`, '-overwrite_original'],
+        },
+      );
+      return true;
+    } catch (error: any) {
+      this.logger.warn(`Could not copy tag data to image: ${error.message}`);
+      return false;
+    }
+  }
+
   async decodeImage(input: string | Buffer, options: DecodeToBufferOptions) {
     const pipeline = await this.getImageDecodingPipeline(input, options);
     return pipeline.raw().toBuffer({ resolveWithObject: true });
@@ -166,17 +183,21 @@ export class MediaRepository {
   }
 
   /**
-   * For output file path 'output.dz', this creates an 'output.dzi' file and 'output_files' directory containing tiles
+   * For output file path 'output.dz', this creates an 'output.dzi' file and 'output_files/0' directory containing tiles
    */
   async generateTiles(input: string | Buffer, options: GenerateThumbnailOptions, output: string): Promise<void> {
-    const pipeline = await this.getImageDecodingPipeline(input, options);
+    // size is intended tile size, don't resize input image.
+    const pipeline = await this.getImageDecodingPipeline(input,  { ...options, size: undefined });
     await pipeline
-      .toFormat(options.format)
+      .toFormat(options.format) // TODO: set quality and chroma ss?
       .tile({
         depth: 'one',
         size: options.size,
       })
       .toFile(output);
+    // TODO: move <uuid>_tiles_files/0 dir to <uuid>_tiles
+    // TODO: delete <uuid>_tiles_files/vips-properties.xml
+    // TODO: delete <uuid>_tiles.dzi
   }
 
   private async getImageDecodingPipeline(input: string | Buffer, options: DecodeToBufferOptions) {

server/src/services/asset-media.service.spec.ts

@@ -720,6 +720,38 @@ describe(AssetMediaService.name, () => {
     });
   });
 
+  describe('getAssetTile', () => {
+    it('should require asset.view permissions', async () => {
+      await expect(sut.viewAssetTile(authStub.admin, 'id', 0, 0, 0)).rejects.toBeInstanceOf(BadRequestException);
+
+      expect(mocks.access.asset.checkOwnerAccess).toHaveBeenCalledWith(userStub.admin.id, new Set(['id']), undefined);
+      expect(mocks.access.asset.checkAlbumAccess).toHaveBeenCalledWith(userStub.admin.id, new Set(['id']));
+      expect(mocks.access.asset.checkPartnerAccess).toHaveBeenCalledWith(userStub.admin.id, new Set(['id']));
+    });
+
+    it('should throw an error if the asset tiles dir could not be found', async () => {
+      const asset = AssetFactory.create();
+      mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([asset.id]));
+      mocks.asset.getForTiles.mockResolvedValue({ ...asset, path: null });
+
+      await expect(sut.viewAssetTile(authStub.admin, asset.id, 0, 0, 0)).rejects.toBeInstanceOf(NotFoundException);
+    });
+
+    it('should get tile file', async () => {
+      const asset = AssetFactory.from().file({ type: AssetFileType.Tiles, path: '/path/to/asset_tiles' }).build();
+      mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([asset.id]));
+      mocks.asset.getForTiles.mockResolvedValue({ ...asset, path: asset.files[0].path });
+      await expect(sut.viewAssetTile(authStub.admin, asset.id, 0, 0, 0)).resolves.toEqual(
+        new ImmichFileResponse({
+          path: `${asset.files[0].path}_files/0/0_0.jpeg`,
+          cacheControl: CacheControl.PrivateWithCache,
+          contentType: 'image/jpeg',
+        }),
+      );
+      expect(mocks.asset.getForTiles).toHaveBeenCalledWith(asset.id);
+    });
+  });
+
   describe('playbackVideo', () => {
     it('should require asset.view permissions', async () => {
       await expect(sut.playbackVideo(authStub.admin, 'id')).rejects.toBeInstanceOf(BadRequestException);

server/src/services/asset-media.service.ts

@@ -262,31 +262,22 @@ export class AssetMediaService extends BaseService {
     });
   }
 
-  async getAssetTile(auth: AuthDto, id: string, level: number, col: number, row: number): Promise<ImmichFileResponse> {
+  async viewAssetTile(auth: AuthDto, id: string, level: number, col: number, row: number): Promise<ImmichFileResponse> {
     await this.requireAccess({ auth, permission: Permission.AssetView, ids: [id] });
 
-    const asset = await this.assetRepository.getForThumbnail(id, AssetFileType.Tiles, false);
-    let tilesPath = undefined; // TODO
-    if (!tilesPath) {
-      // TODO: placeholder tiles.
-      return new ImmichFileResponse({
-        fileName: `${level}_${col}_${row}.jpg`,
-        path: `/data/sluis_files/0/${col}_${row}.jpg`,
-        contentType: 'image/jpg',
-        cacheControl: CacheControl.None,
-      });
+    // TODO: get tile info { width, height } and check against col, row to return NotFound instead of 500 when tile can't be found in sendFile.
+    const { path } = await this.assetRepository.getForTiles(id);
+
+    if (!path) {
       throw new NotFoundException('Asset tiles not found');
     }
 
-    tilesPath = { path: 'tmppath' }; 
-
-    const tileName = getFileNameWithoutExtension(asset.originalFileName) + `_${level}_${col}_${row}.jpg`;
-    const tilePath = tilesPath.path.replace('.dz', '_files') + `/${level}/${col}_${row}.jpg`;
+    // By definition of the tiles format, it's always .jpeg; should ImageFormat.Jpeg be used?
+    const tilePath = `${path}_files/${level}/${col}_${row}.jpeg`;
 
     return new ImmichFileResponse({
-      fileName: tileName,
       path: tilePath,
-      contentType: 'image/jpg',
+      contentType: mimeTypes.lookup(tilePath),
       cacheControl: CacheControl.PrivateWithCache,
     });
   }

server/src/services/media.service.spec.ts

@@ -1210,6 +1210,7 @@ describe(MediaService.name, () => {
       mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: false } } });
       mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg });
       mocks.media.getImageMetadata.mockResolvedValue({ width: 3840, height: 2160, isTransparent: false });
+      mocks.media.copyTagGroup.mockResolvedValue(true);
 
       const asset = AssetFactory.from({ originalFileName: 'panorama.tif' })
         .exif({
@@ -1244,6 +1245,8 @@ describe(MediaService.name, () => {
         },
         expect.any(String),
       );
+
+      expect(mocks.media.copyTagGroup).toHaveBeenCalledExactlyOnceWith('XMP-GPano', asset.originalPath, expect.any(String));
     });
 
     it('should respect encoding options when generating full-size preview', async () => {

server/src/services/media.service.ts

@@ -19,7 +19,6 @@ import {
   QueueName,
   RawExtractedFormat,
   StorageFolder,
-  TilesFormat,
   TranscodeHardwareAcceleration,
   TranscodePolicy,
   TranscodeTarget,
@@ -34,6 +33,7 @@ import {
   DecodeToBufferOptions,
   GenerateThumbnailOptions,
   ImageDimensions,
+  ImageOptions,
   JobItem,
   JobOf,
   VideoFormat,
@@ -56,15 +56,6 @@ interface UpsertFileOptions {
   isTransparent: boolean;
 }
 
-interface TileInfo {
-  path: string;
-  info: {
-    width: number;
-    cols: number;
-    rows: number;
-  };
-}
-
 type ThumbnailAsset = NonNullable<Awaited<ReturnType<AssetJobRepository['getForGenerateThumbnailJob']>>>;
 
 @Injectable()
@@ -173,8 +164,7 @@ export class MediaService extends BaseService {
     await this.storageCore.moveAssetImage(asset, AssetFileType.FullSize, image.fullsize.format);
     await this.storageCore.moveAssetImage(asset, AssetFileType.Preview, image.preview.format);
     await this.storageCore.moveAssetImage(asset, AssetFileType.Thumbnail, image.thumbnail.format);
-    // TODO:
-    // await this.storageCore.moveAssetImage(asset, AssetFileType.Tiles, image.???.format);
+    await this.storageCore.moveAssetTiles(asset);
     await this.storageCore.moveAssetVideo(asset);
 
     return JobStatus.Success;
@@ -288,8 +278,8 @@ export class MediaService extends BaseService {
     const extractEmbedded = image.extractEmbedded && mimeTypes.isRaw(asset.originalFileName);
     const extracted = extractEmbedded ? await this.extractImage(asset.originalPath, image.preview.size) : null;
     const generateFullsize =
-      ((image.fullsize.enabled || asset.exifInfo.projectionType === 'EQUIRECTANGULAR') &&
-        !mimeTypes.isWebSupportedImage(asset.originalPath)) ||
+      (image.fullsize.enabled && !mimeTypes.isWebSupportedImage(asset.originalPath)) ||
+      asset.exifInfo.projectionType === 'EQUIRECTANGULAR' ||
       useEdits;
     const convertFullsize = generateFullsize && (!extracted || !mimeTypes.isWebSupportedImage(` .${extracted.format}`));
 
@@ -396,38 +386,47 @@ export class MediaService extends BaseService {
     }
 
     // TODO: probably extract to helper method
-    let tileInfo: TileInfo | undefined;
+    // TODO: handle cropped panoramas. Tile as normal but save some offset?
+    let tileInfo: UpsertFileOptions | undefined;
     if (asset.exifInfo.projectionType === 'EQUIRECTANGULAR') {
-      // TODO: get uncropped width from asset (FullPanoWidthPixels if present).
-      const originalSize = 12_988;
+      // TODO: get uncropped width from asset (FullPanoWidthPixels if present). -> TODO find out why i wrote this down as a todo
+      const originalSize = asset.exifInfo.exifImageWidth!;
+
       // Get the number of tiles at the exact target size, rounded up (to at least 1 tile).
       const numTilesExact = Math.ceil(originalSize / TILE_TARGET_SIZE);
       // Then round up to the nearest power of 2 (photo-sphere-viewer requirement).
       const numTiles = Math.pow(2, Math.ceil(Math.log2(numTilesExact)));
       const tileSize = Math.ceil(originalSize / numTiles);
 
-      const tileOptions = {
-        ...previewOptions,
+      tileInfo = {
+        assetId: asset.id,
+        type: AssetFileType.Tiles,
+        path: StorageCore.getTilesFolder(asset),
+        isEdited: false,
+        isProgressive: false,
+        isTransparent: false,
+      };
+      const tilesOptions = {
+        ...baseOptions,
+        quality: image.preview.quality,
+        format: ImageFormat.Jpeg,
         size: tileSize,
       };
+      promises.push(this.mediaRepository.generateTiles(data, tilesOptions, tileInfo.path));
 
-      tileInfo = {
-        path: StorageCore.getImagePath(asset, { fileType: AssetFileType.Tiles, format: TilesFormat.Dz, isEdited: useEdits }),
-        info: {
-          width: originalSize,
-          cols: numTiles,
-          rows: numTiles / 2,
-        }
-      };
-      // TODO: reverse comment state
-      // TODO: handle cropped panoramas here. Tile as normal but save some offset?
-      // promises.push(this.mediaRepository.generateTiles(data, tileOptions, tileInfo.path));
-      console.log(tileOptions, tileInfo);
-      tileInfo = undefined;
+      console.log('Tile info for DB:', {
+        width: originalSize,
+        cols: numTiles,
+        rows: numTiles / 2,
+      });
     }
 
     const outputs = await Promise.all(promises);
 
+    if (asset.exifInfo.projectionType === 'EQUIRECTANGULAR') {
+      await this.mediaRepository.copyTagGroup('XMP-GPano', asset.originalPath, previewFile.path);
+    }
+
     const decodedDimensions = { width: info.width, height: info.height };
     const fullsizeDimensions = useEdits ? getOutputDimensions(asset.edits, decodedDimensions) : decodedDimensions;
     const files = [previewFile, thumbnailFile];
@@ -435,8 +434,7 @@ export class MediaService extends BaseService {
       files.push(fullsizeFile);
     }
     if (tileInfo) {
-      console.warn('TODO: should push tile info to files');
-      // files.push(tileInfo);
+      files.push(tileInfo);
     }
 
     return {

server/test/repositories/asset.repository.mock.ts

@@ -53,6 +53,7 @@ export const newAssetRepositoryMock = (): Mocked<RepositoryInterface<AssetReposi
     getForOriginal: vitest.fn(),
     getForOriginals: vitest.fn(),
     getForThumbnail: vitest.fn(),
+    getForTiles: vitest.fn(),
     getForVideo: vitest.fn(),
     getForEdit: vitest.fn(),
     getForOcr: vitest.fn(),