feat: image editing (#24155)

server/src/repositories/media.repository.ts

@@ -7,6 +7,7 @@ import { Writable } from 'node:stream';
 import sharp from 'sharp';
 import { ORIENTATION_TO_SHARP_ROTATION } from 'src/constants';
 import { Exif } from 'src/database';
+import { AssetEditActionItem } from 'src/dtos/editing.dto';
 import { Colorspace, LogLevel, RawExtractedFormat } from 'src/enum';
 import { LoggingRepository } from 'src/repositories/logging.repository';
 import {
@@ -19,6 +20,7 @@ import {
   VideoInfo,
 } from 'src/types';
 import { handlePromiseError } from 'src/utils/misc';
+import { createAffineMatrix } from 'src/utils/transform';
 
 const probe = (input: string, options: string[]): Promise<FfprobeData> =>
   new Promise((resolve, reject) =>
@@ -138,21 +140,48 @@ export class MediaRepository {
     }
   }
 
-  decodeImage(input: string | Buffer, options: DecodeToBufferOptions) {
-    return this.getImageDecodingPipeline(input, options).raw().toBuffer({ resolveWithObject: true });
+  async decodeImage(input: string | Buffer, options: DecodeToBufferOptions) {
+    const pipeline = await this.getImageDecodingPipeline(input, options);
+    return pipeline.raw().toBuffer({ resolveWithObject: true });
+  }
+
+  private async applyEdits(pipeline: sharp.Sharp, edits: AssetEditActionItem[]): Promise<sharp.Sharp> {
+    const affineEditOperations = edits.filter((edit) => edit.action !== 'crop');
+    const matrix = createAffineMatrix(affineEditOperations);
+
+    const crop = edits.find((edit) => edit.action === 'crop');
+    const dimensions = await pipeline.metadata();
+
+    if (crop) {
+      pipeline = pipeline.extract({
+        left: crop ? Math.round(crop.parameters.x) : 0,
+        top: crop ? Math.round(crop.parameters.y) : 0,
+        width: crop ? Math.round(crop.parameters.width) : dimensions.width || 0,
+        height: crop ? Math.round(crop.parameters.height) : dimensions.height || 0,
+      });
+    }
+
+    const { a, b, c, d } = matrix;
+    pipeline = pipeline.affine([
+      [a, b],
+      [c, d],
+    ]);
+
+    return pipeline;
   }
 
   async generateThumbnail(input: string | Buffer, options: GenerateThumbnailOptions, output: string): Promise<void> {
-    await this.getImageDecodingPipeline(input, options)
-      .toFormat(options.format, {
-        quality: options.quality,
-        // this is default in libvips (except the threshold is 90), but we need to set it manually in sharp
-        chromaSubsampling: options.quality >= 80 ? '4:4:4' : '4:2:0',
-      })
-      .toFile(output);
+    const pipeline = await this.getImageDecodingPipeline(input, options);
+    const decoded = pipeline.toFormat(options.format, {
+      quality: options.quality,
+      // this is default in libvips (except the threshold is 90), but we need to set it manually in sharp
+      chromaSubsampling: options.quality >= 80 ? '4:4:4' : '4:2:0',
+    });
+
+    await decoded.toFile(output);
   }
 
-  private getImageDecodingPipeline(input: string | Buffer, options: DecodeToBufferOptions) {
+  private async getImageDecodingPipeline(input: string | Buffer, options: DecodeToBufferOptions) {
     let pipeline = sharp(input, {
       // some invalid images can still be processed by sharp, but we want to fail on them by default to avoid crashes
       failOn: options.processInvalidImages ? 'none' : 'error',
@@ -175,8 +204,8 @@ export class MediaRepository {
       }
     }
 
-    if (options.crop) {
-      pipeline = pipeline.extract(options.crop);
+    if (options.edits && options.edits.length > 0) {
+      pipeline = await this.applyEdits(pipeline, options.edits);
     }
 
     if (options.size !== undefined) {
@@ -186,14 +215,20 @@ export class MediaRepository {
   }
 
   async generateThumbhash(input: string | Buffer, options: GenerateThumbhashOptions): Promise<Buffer> {
-    const [{ rgbaToThumbHash }, { data, info }] = await Promise.all([
+    const [{ rgbaToThumbHash }, decodingPipeline] = await Promise.all([
       import('thumbhash'),
-      sharp(input, options)
-        .resize(100, 100, { fit: 'inside', withoutEnlargement: true })
-        .raw()
-        .ensureAlpha()
-        .toBuffer({ resolveWithObject: true }),
+      this.getImageDecodingPipeline(input, {
+        colorspace: options.colorspace,
+        processInvalidImages: options.processInvalidImages,
+        raw: options.raw,
+        edits: options.edits,
+      }),
     ]);
+
+    const pipeline = decodingPipeline.resize(100, 100, { fit: 'inside', withoutEnlargement: true }).raw().ensureAlpha();
+
+    const { data, info } = await pipeline.toBuffer({ resolveWithObject: true });
+
     return Buffer.from(rgbaToThumbHash(info.width, info.height, data));
   }