feat: server support for filters

server/src/dtos/editing.dto.ts

@@ -1,12 +1,13 @@
 import { ApiExtraModels, ApiProperty, getSchemaPath } from '@nestjs/swagger';
 import { ClassConstructor, plainToInstance, Transform, Type } from 'class-transformer';
-import { ArrayMinSize, IsEnum, IsInt, Min, ValidateNested } from 'class-validator';
+import { ArrayMinSize, IsEnum, IsInt, IsNumber, Max, Min, ValidateNested } from 'class-validator';
 import { IsAxisAlignedRotation, IsUniqueEditActions, ValidateUUID } from 'src/validation';
 
 export enum AssetEditAction {
   Crop = 'crop',
   Rotate = 'rotate',
   Mirror = 'mirror',
+  Filter = 'filter',
 }
 
 export enum MirrorAxis {
@@ -48,6 +49,68 @@ export class MirrorParameters {
   axis!: MirrorAxis;
 }
 
+// Sharp supports a 3x3 matrix for color manipulation and rgb offsets
+// The matrix representation of a filter is as follows:
+//           | rrBias rgBias rbBias |   | r_offset |
+//  Image x  | grBias ggBias gbBias | + | g_offset |
+//           | brBias bgBias bbBias |   | b_offset |
+
+export class FilterParameters {
+  @IsNumber()
+  @ApiProperty({ description: 'RR Bias' })
+  rrBias!: number;
+
+  @IsNumber()
+  @ApiProperty({ description: 'RG Bias' })
+  rgBias!: number;
+
+  @IsNumber()
+  @ApiProperty({ description: 'RB Bias' })
+  rbBias!: number;
+
+  @IsNumber()
+  @ApiProperty({ description: 'GR Bias' })
+  grBias!: number;
+
+  @IsNumber()
+  @ApiProperty({ description: 'GG Bias' })
+  ggBias!: number;
+
+  @IsNumber()
+  @ApiProperty({ description: 'GB Bias' })
+  gbBias!: number;
+
+  @IsNumber()
+  @ApiProperty({ description: 'BR Bias' })
+  brBias!: number;
+
+  @IsNumber()
+  @ApiProperty({ description: 'BG Bias' })
+  bgBias!: number;
+
+  @IsNumber()
+  @ApiProperty({ description: 'BB Bias' })
+  bbBias!: number;
+
+  @IsInt()
+  @Min(-255)
+  @Max(255)
+  @ApiProperty({ description: 'R Offset (-255 -> 255)' })
+  rOffset!: number;
+
+  @IsInt()
+  @Min(-255)
+  @Max(255)
+  @ApiProperty({ description: 'G Offset (-255 -> 255)' })
+  gOffset!: number;
+
+  @IsInt()
+  @Min(-255)
+  @Max(255)
+  @ApiProperty({ description: 'B Offset (-255 -> 255)' })
+  bOffset!: number;
+}
+
 class AssetEditActionBase {
   @IsEnum(AssetEditAction)
   @ApiProperty({ enum: AssetEditAction, enumName: 'AssetEditAction' })
@@ -74,6 +137,12 @@ export class AssetEditActionMirror extends AssetEditActionBase {
   @ApiProperty({ type: MirrorParameters })
   parameters!: MirrorParameters;
 }
+export class AssetEditActionFilter extends AssetEditActionBase {
+  @ValidateNested()
+  @Type(() => FilterParameters)
+  @ApiProperty({ type: FilterParameters })
+  parameters!: FilterParameters;
+}
 
 export type AssetEditActionItem =
   | {
@@ -87,25 +156,31 @@ export type AssetEditActionItem =
   | {
       action: AssetEditAction.Mirror;
       parameters: MirrorParameters;
+    }
+  | {
+      action: AssetEditAction.Filter;
+      parameters: FilterParameters;
     };
 
 export type AssetEditActionParameter = {
   [AssetEditAction.Crop]: CropParameters;
   [AssetEditAction.Rotate]: RotateParameters;
   [AssetEditAction.Mirror]: MirrorParameters;
+  [AssetEditAction.Filter]: FilterParameters;
 };
 
-type AssetEditActions = AssetEditActionCrop | AssetEditActionRotate | AssetEditActionMirror;
+type AssetEditActions = AssetEditActionCrop | AssetEditActionRotate | AssetEditActionMirror | AssetEditActionFilter;
 const actionToClass: Record<AssetEditAction, ClassConstructor<AssetEditActions>> = {
   [AssetEditAction.Crop]: AssetEditActionCrop,
   [AssetEditAction.Rotate]: AssetEditActionRotate,
   [AssetEditAction.Mirror]: AssetEditActionMirror,
+  [AssetEditAction.Filter]: AssetEditActionFilter,
 } as const;
 
 const getActionClass = (item: { action: AssetEditAction }): ClassConstructor<AssetEditActions> =>
   actionToClass[item.action];
 
-@ApiExtraModels(AssetEditActionRotate, AssetEditActionMirror, AssetEditActionCrop)
+@ApiExtraModels(AssetEditActionRotate, AssetEditActionMirror, AssetEditActionCrop, AssetEditActionFilter)
 export class AssetEditActionListDto {
   /** list of edits */
   @ArrayMinSize(1)

server/src/repositories/media.repository.spec.ts

@@ -165,6 +165,38 @@ describe(MediaRepository.name, () => {
       // bottom-right should now be top-right (blue)
       expect(await getPixelColor(bufferVertical, 990, 990)).toEqual({ r: 0, g: 255, b: 0 });
     });
+
+    it('should apply filter edit correctly', async () => {
+      const resultHorizontal = await sut['applyEdits'](sharp(await buildTestQuadImage()), [
+        {
+          action: AssetEditAction.Filter,
+          parameters: {
+            rrBias: 1,
+            rgBias: 0.5,
+            rbBias: 0.5,
+            grBias: 0.5,
+            ggBias: 1,
+            gbBias: 0.5,
+            brBias: 0.5,
+            bgBias: 0.5,
+            bbBias: 1,
+            rOffset: 5,
+            gOffset: 10,
+            bOffset: 15,
+          },
+        },
+      ]);
+
+      const bufferHorizontal = await resultHorizontal.toBuffer();
+      const metadataHorizontal = await resultHorizontal.metadata();
+      expect(metadataHorizontal.width).toBe(1000);
+      expect(metadataHorizontal.height).toBe(1000);
+
+      expect(await getPixelColor(bufferHorizontal, 10, 10)).toEqual({ r: 255, g: 137, b: 142 });
+      expect(await getPixelColor(bufferHorizontal, 990, 10)).toEqual({ r: 132, g: 255, b: 142 });
+      expect(await getPixelColor(bufferHorizontal, 10, 990)).toEqual({ r: 132, g: 137, b: 255 });
+      expect(await getPixelColor(bufferHorizontal, 990, 990)).toEqual({ r: 255, g: 255, b: 255 });
+    });
   });
 
   describe('applyEdits (multiple sequential edits)', () => {
@@ -307,12 +339,29 @@ describe(MediaRepository.name, () => {
       expect(await getPixelColor(buffer, 490, 490)).toEqual({ r: 255, g: 0, b: 0 });
     });
 
-    it('should apply all operations: crop, rotate, mirror', async () => {
+    it('should apply all operations: crop, rotate, mirror, filter', async () => {
       const imageBuffer = await buildTestQuadImage();
       const result = await sut['applyEdits'](sharp(imageBuffer), [
         { action: AssetEditAction.Crop, parameters: { x: 0, y: 0, width: 500, height: 1000 } },
         { action: AssetEditAction.Rotate, parameters: { angle: 90 } },
         { action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Horizontal } },
+        {
+          action: AssetEditAction.Filter,
+          parameters: {
+            rrBias: 1,
+            rgBias: 0,
+            rbBias: 0,
+            grBias: 0,
+            ggBias: 1,
+            gbBias: 0,
+            brBias: 0,
+            bgBias: 0,
+            bbBias: 1,
+            rOffset: -10,
+            gOffset: 20,
+            bOffset: -30,
+          },
+        },
       ]);
 
       const buffer = await result.png().toBuffer();
@@ -320,8 +369,8 @@ describe(MediaRepository.name, () => {
       expect(metadata.width).toBe(1000);
       expect(metadata.height).toBe(500);
 
-      expect(await getPixelColor(buffer, 10, 10)).toEqual({ r: 255, g: 0, b: 0 });
-      expect(await getPixelColor(buffer, 990, 10)).toEqual({ r: 0, g: 0, b: 255 });
+      expect(await getPixelColor(buffer, 10, 10)).toEqual({ r: 245, g: 20, b: 0 });
+      expect(await getPixelColor(buffer, 990, 10)).toEqual({ r: 0, g: 20, b: 225 });
     });
   });
 

server/src/repositories/media.repository.ts

@@ -19,6 +19,7 @@ import {
   TranscodeCommand,
   VideoInfo,
 } from 'src/types';
+import { convertColorFilterToMatricies } from 'src/utils/color_filter';
 import { handlePromiseError } from 'src/utils/misc';
 import { createAffineMatrix } from 'src/utils/transform';
 
@@ -167,6 +168,12 @@ export class MediaRepository {
       [c, d],
     ]);
 
+    const filter = edits.find((edit) => edit.action === 'filter');
+    if (filter) {
+      const { biasMatrix, offsetMatrix } = convertColorFilterToMatricies(filter.parameters);
+      pipeline = pipeline.recomb(biasMatrix).linear([1, 1, 1], offsetMatrix);
+    }
+
     return pipeline;
   }
 

server/src/utils/color_filter.ts

@@ -0,0 +1,14 @@
+import { Matrix3x3 } from 'sharp';
+import { FilterParameters } from 'src/dtos/editing.dto';
+
+export function convertColorFilterToMatricies(filter: FilterParameters) {
+  const biasMatrix: Matrix3x3 = [
+    [filter.rrBias, filter.rgBias, filter.rbBias],
+    [filter.grBias, filter.ggBias, filter.gbBias],
+    [filter.brBias, filter.bgBias, filter.bbBias],
+  ];
+
+  const offsetMatrix = [filter.rOffset, filter.gOffset, filter.bOffset];
+
+  return { biasMatrix, offsetMatrix };
+}