feat: image editing (#24155)

server/src/dtos/asset-media.dto.ts

@@ -19,6 +19,9 @@ export enum AssetMediaSize {
 export class AssetMediaOptionsDto {
   @ValidateEnum({ enum: AssetMediaSize, name: 'AssetMediaSize', optional: true })
   size?: AssetMediaSize;
+
+  @ValidateBoolean({ optional: true, default: false })
+  edited?: boolean;
 }
 
 export enum UploadFieldName {

server/src/dtos/asset-response.dto.ts

@@ -3,6 +3,7 @@ import { Selectable } from 'kysely';
 import { AssetFace, AssetFile, Exif, Stack, Tag, User } from 'src/database';
 import { HistoryBuilder, Property } from 'src/decorators';
 import { AuthDto } from 'src/dtos/auth.dto';
+import { AssetEditActionItem } from 'src/dtos/editing.dto';
 import { ExifResponseDto, mapExif } from 'src/dtos/exif.dto';
 import {
   AssetFaceWithoutPersonResponseDto,
@@ -13,6 +14,8 @@ import {
 import { TagResponseDto, mapTag } from 'src/dtos/tag.dto';
 import { UserResponseDto, mapUser } from 'src/dtos/user.dto';
 import { AssetStatus, AssetType, AssetVisibility } from 'src/enum';
+import { ImageDimensions } from 'src/types';
+import { getDimensions } from 'src/utils/asset.util';
 import { hexOrBufferToBase64 } from 'src/utils/bytes';
 import { mimeTypes } from 'src/utils/mime-types';
 import { ValidateEnum } from 'src/validation';
@@ -34,6 +37,8 @@ export class SanitizedAssetResponseDto {
   duration!: string;
   livePhotoVideoId?: string | null;
   hasMetadata!: boolean;
+  width!: number | null;
+  height!: number | null;
 }
 
 export class AssetResponseDto extends SanitizedAssetResponseDto {
@@ -107,6 +112,7 @@ export type MapAsset = {
   deviceId: string;
   duplicateId: string | null;
   duration: string | null;
+  edits?: AssetEditActionItem[];
   encodedVideoPath: string | null;
   exifInfo?: Selectable<Exif> | null;
   faces?: AssetFace[];
@@ -129,6 +135,8 @@ export type MapAsset = {
   tags?: Tag[];
   thumbhash: Buffer<ArrayBufferLike> | null;
   type: AssetType;
+  width: number | null;
+  height: number | null;
 };
 
 export class AssetStackResponseDto {
@@ -147,7 +155,11 @@ export type AssetMapOptions = {
 };
 
 // TODO: this is inefficient
-const peopleWithFaces = (faces?: AssetFace[]): PersonWithFacesResponseDto[] => {
+const peopleWithFaces = (
+  faces?: AssetFace[],
+  edits?: AssetEditActionItem[],
+  assetDimensions?: ImageDimensions,
+): PersonWithFacesResponseDto[] => {
   const result: PersonWithFacesResponseDto[] = [];
   if (faces) {
     for (const face of faces) {
@@ -156,7 +168,7 @@ const peopleWithFaces = (faces?: AssetFace[]): PersonWithFacesResponseDto[] => {
         if (existingPersonEntry) {
           existingPersonEntry.faces.push(face);
         } else {
-          result.push({ ...mapPerson(face.person!), faces: [mapFacesWithoutPerson(face)] });
+          result.push({ ...mapPerson(face.person!), faces: [mapFacesWithoutPerson(face, edits, assetDimensions)] });
         }
       }
     }
@@ -190,10 +202,14 @@ export function mapAsset(entity: MapAsset, options: AssetMapOptions = {}): Asset
       duration: entity.duration ?? '0:00:00.00000',
       livePhotoVideoId: entity.livePhotoVideoId,
       hasMetadata: false,
+      width: entity.width,
+      height: entity.height,
     };
     return sanitizedAssetResponse as AssetResponseDto;
   }
 
+  const assetDimensions = entity.exifInfo ? getDimensions(entity.exifInfo) : undefined;
+
   return {
     id: entity.id,
     createdAt: entity.createdAt,
@@ -219,7 +235,7 @@ export function mapAsset(entity: MapAsset, options: AssetMapOptions = {}): Asset
     exifInfo: entity.exifInfo ? mapExif(entity.exifInfo) : undefined,
     livePhotoVideoId: entity.livePhotoVideoId,
     tags: entity.tags?.map((tag) => mapTag(tag)),
-    people: peopleWithFaces(entity.faces),
+    people: peopleWithFaces(entity.faces, entity.edits, assetDimensions),
     unassignedFaces: entity.faces?.filter((face) => !face.person).map((a) => mapFacesWithoutPerson(a)),
     checksum: hexOrBufferToBase64(entity.checksum)!,
     stack: withStack ? mapStack(entity) : undefined,
@@ -227,5 +243,7 @@ export function mapAsset(entity: MapAsset, options: AssetMapOptions = {}): Asset
     hasMetadata: true,
     duplicateId: entity.duplicateId,
     resized: true,
+    width: entity.width,
+    height: entity.height,
   };
 }

server/src/dtos/asset.dto.ts

@@ -228,6 +228,11 @@ export class AssetCopyDto {
   favorite?: boolean;
 }
 
+export class AssetDownloadOriginalDto {
+  @ValidateBoolean({ optional: true, default: false })
+  edited?: boolean;
+}
+
 export const mapStats = (stats: AssetStats): AssetStatsResponseDto => {
   return {
     images: stats[AssetType.Image],

server/src/dtos/editing.dto.ts

@@ -0,0 +1,125 @@
+import { ApiExtraModels, ApiProperty, getSchemaPath } from '@nestjs/swagger';
+import { ClassConstructor, plainToInstance, Transform, Type } from 'class-transformer';
+import { ArrayMinSize, IsEnum, IsInt, Min, ValidateNested } from 'class-validator';
+import { IsAxisAlignedRotation, IsUniqueEditActions, ValidateUUID } from 'src/validation';
+
+export enum AssetEditAction {
+  Crop = 'crop',
+  Rotate = 'rotate',
+  Mirror = 'mirror',
+}
+
+export enum MirrorAxis {
+  Horizontal = 'horizontal',
+  Vertical = 'vertical',
+}
+
+export class CropParameters {
+  @IsInt()
+  @Min(0)
+  @ApiProperty({ description: 'Top-Left X coordinate of crop' })
+  x!: number;
+
+  @IsInt()
+  @Min(0)
+  @ApiProperty({ description: 'Top-Left Y coordinate of crop' })
+  y!: number;
+
+  @IsInt()
+  @Min(1)
+  @ApiProperty({ description: 'Width of the crop' })
+  width!: number;
+
+  @IsInt()
+  @Min(1)
+  @ApiProperty({ description: 'Height of the crop' })
+  height!: number;
+}
+
+export class RotateParameters {
+  @IsAxisAlignedRotation()
+  @ApiProperty({ description: 'Rotation angle in degrees' })
+  angle!: number;
+}
+
+export class MirrorParameters {
+  @IsEnum(MirrorAxis)
+  @ApiProperty({ enum: MirrorAxis, enumName: 'MirrorAxis', description: 'Axis to mirror along' })
+  axis!: MirrorAxis;
+}
+
+class AssetEditActionBase {
+  @IsEnum(AssetEditAction)
+  @ApiProperty({ enum: AssetEditAction, enumName: 'AssetEditAction' })
+  action!: AssetEditAction;
+}
+
+export class AssetEditActionCrop extends AssetEditActionBase {
+  @ValidateNested()
+  @Type(() => CropParameters)
+  @ApiProperty({ type: CropParameters })
+  parameters!: CropParameters;
+}
+
+export class AssetEditActionRotate extends AssetEditActionBase {
+  @ValidateNested()
+  @Type(() => RotateParameters)
+  @ApiProperty({ type: RotateParameters })
+  parameters!: RotateParameters;
+}
+
+export class AssetEditActionMirror extends AssetEditActionBase {
+  @ValidateNested()
+  @Type(() => MirrorParameters)
+  @ApiProperty({ type: MirrorParameters })
+  parameters!: MirrorParameters;
+}
+
+export type AssetEditActionItem =
+  | {
+      action: AssetEditAction.Crop;
+      parameters: CropParameters;
+    }
+  | {
+      action: AssetEditAction.Rotate;
+      parameters: RotateParameters;
+    }
+  | {
+      action: AssetEditAction.Mirror;
+      parameters: MirrorParameters;
+    };
+
+export type AssetEditActionParameter = {
+  [AssetEditAction.Crop]: CropParameters;
+  [AssetEditAction.Rotate]: RotateParameters;
+  [AssetEditAction.Mirror]: MirrorParameters;
+};
+
+type AssetEditActions = AssetEditActionCrop | AssetEditActionRotate | AssetEditActionMirror;
+const actionToClass: Record<AssetEditAction, ClassConstructor<AssetEditActions>> = {
+  [AssetEditAction.Crop]: AssetEditActionCrop,
+  [AssetEditAction.Rotate]: AssetEditActionRotate,
+  [AssetEditAction.Mirror]: AssetEditActionMirror,
+} as const;
+
+const getActionClass = (item: { action: AssetEditAction }): ClassConstructor<AssetEditActions> =>
+  actionToClass[item.action];
+
+@ApiExtraModels(AssetEditActionRotate, AssetEditActionMirror, AssetEditActionCrop)
+export class AssetEditActionListDto {
+  /** list of edits */
+  @ArrayMinSize(1)
+  @IsUniqueEditActions()
+  @ValidateNested({ each: true })
+  @Transform(({ value: edits }) =>
+    Array.isArray(edits) ? edits.map((item) => plainToInstance(getActionClass(item), item)) : edits,
+  )
+  @ApiProperty({ anyOf: Object.values(actionToClass).map((target) => ({ $ref: getSchemaPath(target) })) })
+  edits!: AssetEditActionItem[];
+}
+
+export class AssetEditsDto extends AssetEditActionListDto {
+  @ValidateUUID()
+  @ApiProperty()
+  assetId!: string;
+}

server/src/dtos/person.dto.ts

@@ -6,9 +6,12 @@ import { DateTime } from 'luxon';
 import { AssetFace, Person } from 'src/database';
 import { HistoryBuilder, Property } from 'src/decorators';
 import { AuthDto } from 'src/dtos/auth.dto';
+import { AssetEditActionItem } from 'src/dtos/editing.dto';
 import { SourceType } from 'src/enum';
 import { AssetFaceTable } from 'src/schema/tables/asset-face.table';
+import { ImageDimensions } from 'src/types';
 import { asDateString } from 'src/utils/date';
+import { transformFaceBoundingBox } from 'src/utils/transform';
 import {
   IsDateStringFormat,
   MaxDateString,
@@ -233,29 +236,37 @@ export function mapPerson(person: Person): PersonResponseDto {
   };
 }
 
-export function mapFacesWithoutPerson(face: Selectable<AssetFaceTable>): AssetFaceWithoutPersonResponseDto {
+export function mapFacesWithoutPerson(
+  face: Selectable<AssetFaceTable>,
+  edits?: AssetEditActionItem[],
+  assetDimensions?: ImageDimensions,
+): AssetFaceWithoutPersonResponseDto {
   return {
     id: face.id,
-    imageHeight: face.imageHeight,
-    imageWidth: face.imageWidth,
-    boundingBoxX1: face.boundingBoxX1,
-    boundingBoxX2: face.boundingBoxX2,
-    boundingBoxY1: face.boundingBoxY1,
-    boundingBoxY2: face.boundingBoxY2,
+    ...transformFaceBoundingBox(
+      {
+        boundingBoxX1: face.boundingBoxX1,
+        boundingBoxY1: face.boundingBoxY1,
+        boundingBoxX2: face.boundingBoxX2,
+        boundingBoxY2: face.boundingBoxY2,
+        imageWidth: face.imageWidth,
+        imageHeight: face.imageHeight,
+      },
+      edits ?? [],
+      assetDimensions ?? { width: face.imageWidth, height: face.imageHeight },
+    ),
     sourceType: face.sourceType,
   };
 }
 
-export function mapFaces(face: AssetFace, auth: AuthDto): AssetFaceResponseDto {
+export function mapFaces(
+  face: AssetFace,
+  auth: AuthDto,
+  edits?: AssetEditActionItem[],
+  assetDimensions?: ImageDimensions,
+): AssetFaceResponseDto {
   return {
-    id: face.id,
-    imageHeight: face.imageHeight,
-    imageWidth: face.imageWidth,
-    boundingBoxX1: face.boundingBoxX1,
-    boundingBoxX2: face.boundingBoxX2,
-    boundingBoxY1: face.boundingBoxY1,
-    boundingBoxY2: face.boundingBoxY2,
-    sourceType: face.sourceType,
+    ...mapFacesWithoutPerson(face, edits, assetDimensions),
     person: face.person?.ownerId === auth.user.id ? mapPerson(face.person) : null,
   };
 }

server/src/dtos/queue-legacy.dto.ts

@@ -66,6 +66,9 @@ export class QueuesResponseLegacyDto implements Record<QueueName, QueueResponseL
 
   @ApiProperty({ type: QueueResponseLegacyDto })
   [QueueName.Workflow]!: QueueResponseLegacyDto;
+
+  @ApiProperty({ type: QueueResponseLegacyDto })
+  [QueueName.Editor]!: QueueResponseLegacyDto;
 }
 
 export const mapQueueLegacy = (response: QueueResponseDto): QueueResponseLegacyDto => {

server/src/dtos/sync.dto.ts

@@ -117,6 +117,10 @@ export class SyncAssetV1 {
   livePhotoVideoId!: string | null;
   stackId!: string | null;
   libraryId!: string | null;
+  @ApiProperty({ type: 'integer' })
+  width!: number | null;
+  @ApiProperty({ type: 'integer' })
+  height!: number | null;
 }
 
 @ExtraModel()

server/src/dtos/system-config.dto.ts

@@ -230,6 +230,12 @@ class SystemConfigJobDto implements Record<ConcurrentQueueName, JobSettingsDto>
   @IsObject()
   @Type(() => JobSettingsDto)
   [QueueName.Workflow]!: JobSettingsDto;
+
+  @ApiProperty({ type: JobSettingsDto })
+  @ValidateNested()
+  @IsObject()
+  @Type(() => JobSettingsDto)
+  [QueueName.Editor]!: JobSettingsDto;
 }
 
 class SystemConfigLibraryScanDto {