From 80db413d6986a57a2aa9f34fc90dbf754af67e2a Mon Sep 17 00:00:00 2001 From: bwees Date: Tue, 24 Mar 2026 16:34:07 -0500 Subject: [PATCH] chore: more wip --- open-api/immich-openapi-specs.json | 13 +- .../src/controllers/asset-media.controller.ts | 6 +- server/src/cores/storage.core.ts | 4 +- server/src/dtos/asset-media.dto.ts | 8 +- server/src/enum.ts | 3 +- .../src/repositories/asset-job.repository.ts | 45 ++- server/src/repositories/asset.repository.ts | 6 +- server/src/services/asset-media.service.ts | 7 +- server/src/services/asset.service.ts | 32 +- server/src/services/job.service.ts | 3 +- server/src/services/media.service.spec.ts | 18 +- server/src/services/media.service.ts | 355 ++++++++++++------ server/src/types.ts | 3 +- server/src/utils/database.ts | 5 +- server/src/utils/editor.ts | 27 ++ server/src/utils/media.ts | 13 +- .../managers/asset-viewer-manager.svelte.ts | 1 + .../lib/managers/edit/edit-manager.svelte.ts | 2 + web/src/lib/utils.ts | 4 +- 19 files changed, 400 insertions(+), 155 deletions(-) diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 74a0ff8c2c..dee2fb6b2b 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -4402,6 +4402,16 @@ "description": "Streams the video file for the specified asset. This endpoint also supports byte range requests.", "operationId": "playAssetVideo", "parameters": [ + { + "name": "edited", + "required": false, + "in": "query", + "description": "Return edited asset if available", + "schema": { + "default": false, + "type": "boolean" + } + }, { "name": "id", "required": true, @@ -18144,8 +18154,6 @@ "AssetDetectFaces", "AssetDetectDuplicatesQueueAll", "AssetDetectDuplicates", - "AssetEditThumbnailGeneration", - "AssetEditTranscodeGeneration", "AssetEncodeVideoQueueAll", "AssetEncodeVideo", "AssetEmptyTrash", @@ -18154,6 +18162,7 @@ "AssetFileMigration", "AssetGenerateThumbnailsQueueAll", "AssetGenerateThumbnails", + "AssetProcessEdit", "AuditLogCleanup", "AuditTableCleanup", "DatabaseBackup", diff --git a/server/src/controllers/asset-media.controller.ts b/server/src/controllers/asset-media.controller.ts index ec6083cfa8..0d05bc6db0 100644 --- a/server/src/controllers/asset-media.controller.ts +++ b/server/src/controllers/asset-media.controller.ts @@ -30,6 +30,7 @@ import { AssetMediaOptionsDto, AssetMediaReplaceDto, AssetMediaSize, + AssetThumbnailOptionsDto, CheckExistingAssetsDto, UploadFieldName, } from 'src/dtos/asset-media.dto'; @@ -154,7 +155,7 @@ export class AssetMediaController { async viewAsset( @Auth() auth: AuthDto, @Param() { id }: UUIDParamDto, - @Query() dto: AssetMediaOptionsDto, + @Query() dto: AssetThumbnailOptionsDto, @Req() req: Request, @Res() res: Response, @Next() next: NextFunction, @@ -197,9 +198,10 @@ export class AssetMediaController { @Auth() auth: AuthDto, @Param() { id }: UUIDParamDto, @Res() res: Response, + @Query() dto: AssetMediaOptionsDto, @Next() next: NextFunction, ) { - await sendFile(res, next, () => this.service.playbackVideo(auth, id), this.logger); + await sendFile(res, next, () => this.service.playbackVideo(auth, id, dto), this.logger); } @Post('exist') diff --git a/server/src/cores/storage.core.ts b/server/src/cores/storage.core.ts index 3345f6e129..077f60fab9 100644 --- a/server/src/cores/storage.core.ts +++ b/server/src/cores/storage.core.ts @@ -120,8 +120,8 @@ export class StorageCore { ); } - static getEncodedVideoPath(asset: ThumbnailPathEntity) { - return StorageCore.getNestedPath(StorageFolder.EncodedVideo, asset.ownerId, `${asset.id}.mp4`); + static getEncodedVideoPath(asset: ThumbnailPathEntity, isEdited: boolean = false) { + return StorageCore.getNestedPath(StorageFolder.EncodedVideo, asset.ownerId, `${asset.id}${isEdited ? '_edited' : ''}.mp4`); } static getAndroidMotionPath(asset: ThumbnailPathEntity, uuid: string) { diff --git a/server/src/dtos/asset-media.dto.ts b/server/src/dtos/asset-media.dto.ts index 4655850379..0677a40523 100644 --- a/server/src/dtos/asset-media.dto.ts +++ b/server/src/dtos/asset-media.dto.ts @@ -18,13 +18,15 @@ export enum AssetMediaSize { } export class AssetMediaOptionsDto { - @ValidateEnum({ enum: AssetMediaSize, name: 'AssetMediaSize', description: 'Asset media size', optional: true }) - size?: AssetMediaSize; - @ValidateBoolean({ optional: true, description: 'Return edited asset if available', default: false }) edited?: boolean; } +export class AssetThumbnailOptionsDto extends AssetMediaOptionsDto { + @ValidateEnum({ enum: AssetMediaSize, name: 'AssetMediaSize', description: 'Asset media size', optional: true }) + size?: AssetMediaSize; +} + export enum UploadFieldName { ASSET_DATA = 'assetData', SIDECAR_DATA = 'sidecarData', diff --git a/server/src/enum.ts b/server/src/enum.ts index de0e106842..1750300843 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -588,8 +588,6 @@ export enum JobName { AssetDetectFaces = 'AssetDetectFaces', AssetDetectDuplicatesQueueAll = 'AssetDetectDuplicatesQueueAll', AssetDetectDuplicates = 'AssetDetectDuplicates', - AssetEditThumbnailGeneration = 'AssetEditThumbnailGeneration', - AssetEditTranscodeGeneration = 'AssetEditTranscodeGeneration', AssetEncodeVideoQueueAll = 'AssetEncodeVideoQueueAll', AssetEncodeVideo = 'AssetEncodeVideo', AssetEmptyTrash = 'AssetEmptyTrash', @@ -598,6 +596,7 @@ export enum JobName { AssetFileMigration = 'AssetFileMigration', AssetGenerateThumbnailsQueueAll = 'AssetGenerateThumbnailsQueueAll', AssetGenerateThumbnails = 'AssetGenerateThumbnails', + AssetProcessEdit = 'AssetProcessEdit', AuditLogCleanup = 'AuditLogCleanup', AuditTableCleanup = 'AuditTableCleanup', diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.ts index e503ab8dd0..58bef905dc 100644 --- a/server/src/repositories/asset-job.repository.ts +++ b/server/src/repositories/asset-job.repository.ts @@ -138,6 +138,39 @@ export class AssetJobRepository { .executeTakeFirst(); } + @GenerateSql({ params: [DummyValue.UUID] }) + getForAssetEditProcessing(id: string) { + return this.db + .selectFrom('asset') + .select([ + 'asset.id', + 'asset.visibility', + 'asset.originalFileName', + 'asset.originalPath', + 'asset.ownerId', + 'asset.thumbhash', + 'asset.type', + ]) + .select((eb) => + jsonArrayFrom( + eb + .selectFrom('asset_file') + .select(columns.assetFilesForThumbnail) + .whereRef('asset_file.assetId', '=', 'asset.id') + .where('asset_file.type', 'in', [ + AssetFileType.Thumbnail, + AssetFileType.Preview, + AssetFileType.FullSize, + AssetFileType.EncodedVideo, + ]), + ).as('files'), + ) + .select(withEdits) + .$call(withExifInner) + .where('asset.id', '=', id) + .executeTakeFirst(); + } + @GenerateSql({ params: [DummyValue.UUID] }) getForMetadataExtraction(id: string) { return this.db @@ -308,7 +341,7 @@ export class AssetJobRepository { streamForVideoConversion(force?: boolean) { return this.db .selectFrom('asset') - .select(['asset.id']) + .select(['asset.id', 'asset.isEdited']) .where('asset.type', '=', sql.lit(AssetType.Video)) .$if(!force, (qb) => qb @@ -334,7 +367,15 @@ export class AssetJobRepository { return this.db .selectFrom('asset') .select(['asset.id', 'asset.ownerId', 'asset.originalPath']) - .select(withFiles) + .select((eb) => + jsonArrayFrom( + eb + .selectFrom('asset_file') + .select(columns.assetFilesForThumbnail) + .whereRef('asset_file.assetId', '=', 'asset.id') + .where('asset_file.type', '=', sql.lit(AssetFileType.EncodedVideo)), + ).as('files'), + ) .select(withEdits) .where('asset.id', '=', id) .where('asset.type', '=', sql.lit(AssetType.Video)) diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index 2e1d02ef28..8015fb3bb3 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -1149,12 +1149,12 @@ export class AssetRepository { .executeTakeFirstOrThrow(); } - @GenerateSql({ params: [DummyValue.UUID] }) - async getForVideo(id: string) { + @GenerateSql({ params: [DummyValue.UUID, true] }) + async getForVideo(id: string, isEdited: boolean) { return this.db .selectFrom('asset') .select(['asset.originalPath']) - .select((eb) => withFilePath(eb, AssetFileType.EncodedVideo).as('encodedVideoPath')) + .select((eb) => withFilePath(eb, AssetFileType.EncodedVideo, isEdited).as('encodedVideoPath')) .where('asset.id', '=', id) .where('asset.type', '=', AssetType.Video) .executeTakeFirst(); diff --git a/server/src/services/asset-media.service.ts b/server/src/services/asset-media.service.ts index 10132bbb07..a9f429e08d 100644 --- a/server/src/services/asset-media.service.ts +++ b/server/src/services/asset-media.service.ts @@ -17,6 +17,7 @@ import { AssetMediaOptionsDto, AssetMediaReplaceDto, AssetMediaSize, + AssetThumbnailOptionsDto, CheckExistingAssetsDto, UploadFieldName, } from 'src/dtos/asset-media.dto'; @@ -222,7 +223,7 @@ export class AssetMediaService extends BaseService { async viewThumbnail( auth: AuthDto, id: string, - dto: AssetMediaOptionsDto, + dto: AssetThumbnailOptionsDto, ): Promise { await this.requireAccess({ auth, permission: Permission.AssetView, ids: [id] }); @@ -266,10 +267,10 @@ export class AssetMediaService extends BaseService { }); } - async playbackVideo(auth: AuthDto, id: string): Promise { + async playbackVideo(auth: AuthDto, id: string, dto: AssetMediaOptionsDto): Promise { await this.requireAccess({ auth, permission: Permission.AssetView, ids: [id] }); - const asset = await this.assetRepository.getForVideo(id); + const asset = await this.assetRepository.getForVideo(id, dto.edited ?? false); if (!asset) { throw new NotFoundException('Asset not found or asset is not a video'); diff --git a/server/src/services/asset.service.ts b/server/src/services/asset.service.ts index bd8df37afb..0386681edb 100644 --- a/server/src/services/asset.service.ts +++ b/server/src/services/asset.service.ts @@ -47,6 +47,7 @@ import { } from 'src/utils/asset.util'; import { updateLockedColumns } from 'src/utils/database'; import { extractTimeZone } from 'src/utils/date'; +import { scaleEdits } from 'src/utils/editor'; import { transformOcrBoundingBox } from 'src/utils/transform'; @Injectable() @@ -605,10 +606,27 @@ export class AssetService extends BaseService { } const newEdits = await this.assetEditRepository.replaceAll(id, edits); - await this.jobRepository.queue({ name: JobName.AssetEditThumbnailGeneration, data: { id } }); + await this.jobRepository.queue({ name: JobName.AssetProcessEdit, data: { id } }); if (asset.livePhotoVideoId) { - await this.jobRepository.queue({ name: JobName.AssetEditTranscodeGeneration, data: { id } }); + const liveAsset = await this.assetRepository.getForEdit(asset.livePhotoVideoId); + if (!liveAsset) { + throw new BadRequestException('Live photo video not found'); + } + + const { width: liveWidth, height: liveHeight } = getDimensions(liveAsset); + console.log(liveWidth, liveHeight); + const scaledEdits = scaleEdits( + edits, + { width: liveWidth, height: liveHeight }, + { width: assetWidth, height: assetHeight }, + ); + + await this.assetEditRepository.replaceAll(asset.livePhotoVideoId, scaledEdits); + await this.jobRepository.queue({ + name: JobName.AssetProcessEdit, + data: { id: asset.livePhotoVideoId }, + }); } // Return the asset and its applied edits @@ -627,6 +645,14 @@ export class AssetService extends BaseService { } await this.assetEditRepository.replaceAll(id, []); - await this.jobRepository.queue({ name: JobName.AssetEditThumbnailGeneration, data: { id } }); + await this.jobRepository.queue({ name: JobName.AssetProcessEdit, data: { id } }); + + if (asset.livePhotoVideoId) { + await this.assetEditRepository.replaceAll(asset.livePhotoVideoId, []); + await this.jobRepository.queue({ + name: JobName.AssetProcessEdit, + data: { id: asset.livePhotoVideoId }, + }); + } } } diff --git a/server/src/services/job.service.ts b/server/src/services/job.service.ts index 98f369c31a..cf979d707c 100644 --- a/server/src/services/job.service.ts +++ b/server/src/services/job.service.ts @@ -95,8 +95,7 @@ export class JobService extends BaseService { } break; } - - case JobName.AssetEditThumbnailGeneration: { + case JobName.AssetProcessEdit: { const asset = await this.assetRepository.getById(item.data.id); const edits = await this.assetEditRepository.getWithSyncInfo(item.data.id); diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts index 51a10a39c2..c74adf033a 100644 --- a/server/src/services/media.service.spec.ts +++ b/server/src/services/media.service.spec.ts @@ -273,7 +273,7 @@ describe(MediaService.name, () => { data: { id: asset.id }, }, { - name: JobName.AssetEditThumbnailGeneration, + name: JobName.AssetProcessEdit, data: { id: asset.id }, }, ]); @@ -1323,7 +1323,7 @@ describe(MediaService.name, () => { }); }); - describe('handleAssetEditThumbnailGeneration', () => { + describe('handleAssetEditProcessing', () => { let rawInfo: RawImageInfo; beforeEach(() => { @@ -1344,7 +1344,7 @@ describe(MediaService.name, () => { const asset = AssetFactory.from({ type: AssetType.Video }).exif().build(); mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(getForGenerateThumbnail(asset)); - await expect(sut.handleAssetEditThumbnailGeneration({ id: asset.id })).resolves.toBe(JobStatus.Success); + await expect(sut.handleAssetEditProcessing({ id: asset.id })).resolves.toBe(JobStatus.Success); expect(mocks.media.generateThumbnail).not.toHaveBeenCalled(); }); @@ -1365,7 +1365,7 @@ describe(MediaService.name, () => { mocks.person.getFaces.mockResolvedValue([]); mocks.ocr.getByAssetId.mockResolvedValue([]); - await sut.handleAssetEditThumbnailGeneration({ id: asset.id }); + await sut.handleAssetEditProcessing({ id: asset.id }); expect(mocks.asset.upsertFiles).toHaveBeenCalledWith( expect.arrayContaining([ @@ -1385,7 +1385,7 @@ describe(MediaService.name, () => { mocks.person.getFaces.mockResolvedValue([]); mocks.ocr.getByAssetId.mockResolvedValue([]); - await sut.handleAssetEditThumbnailGeneration({ id: asset.id }); + await sut.handleAssetEditProcessing({ id: asset.id }); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( rawBuffer, expect.objectContaining({ @@ -1411,7 +1411,7 @@ describe(MediaService.name, () => { .build(); mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(getForGenerateThumbnail(asset)); - const status = await sut.handleAssetEditThumbnailGeneration({ id: asset.id }); + const status = await sut.handleAssetEditProcessing({ id: asset.id }); expect(mocks.job.queue).toHaveBeenCalledWith({ name: JobName.FileDelete, @@ -1431,7 +1431,7 @@ describe(MediaService.name, () => { mocks.person.getFaces.mockResolvedValue([]); mocks.ocr.getByAssetId.mockResolvedValue([]); - await sut.handleAssetEditThumbnailGeneration({ id: asset.id }); + await sut.handleAssetEditProcessing({ id: asset.id }); expect(mocks.media.generateThumbnail).toHaveBeenCalledTimes(3); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( @@ -1456,7 +1456,7 @@ describe(MediaService.name, () => { mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(getForGenerateThumbnail(asset)); mocks.media.generateThumbhash.mockResolvedValue(factory.buffer()); - await sut.handleAssetEditThumbnailGeneration({ id: asset.id, source: 'upload' }); + await sut.handleAssetEditProcessing({ id: asset.id, source: 'upload' }); expect(mocks.media.generateThumbhash).toHaveBeenCalled(); }); @@ -1469,7 +1469,7 @@ describe(MediaService.name, () => { mocks.person.getFaces.mockResolvedValue([]); mocks.ocr.getByAssetId.mockResolvedValue([]); - await sut.handleAssetEditThumbnailGeneration({ id: asset.id }); + await sut.handleAssetEditProcessing({ id: asset.id }); expect(mocks.asset.update).toHaveBeenCalledWith(expect.objectContaining({ thumbhash: thumbhashBuffer })); }); diff --git a/server/src/services/media.service.ts b/server/src/services/media.service.ts index 455c5de9db..4b15dbbf71 100644 --- a/server/src/services/media.service.ts +++ b/server/src/services/media.service.ts @@ -4,7 +4,7 @@ import { FACE_THUMBNAIL_SIZE, JOBS_ASSET_PAGINATION_SIZE } from 'src/constants'; import { ImagePathOptions, StorageCore, ThumbnailPathEntity } from 'src/cores/storage.core'; import { AssetFile } from 'src/database'; import { OnEvent, OnJob } from 'src/decorators'; -import { AssetEditAction, CropParameters } from 'src/dtos/editing.dto'; +import { AssetEditAction, AssetEditActionItem, CropParameters } from 'src/dtos/editing.dto'; import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto'; import { AssetFileType, @@ -39,7 +39,7 @@ import { VideoInterfaces, VideoStreamInfo, } from 'src/types'; -import { getAssetFile, getDimensions } from 'src/utils/asset.util'; +import { getDimensions } from 'src/utils/asset.util'; import { checkFaceVisibility, checkOcrVisibility } from 'src/utils/editor'; import { BaseConfig, ThumbnailConfig } from 'src/utils/media'; import { mimeTypes } from 'src/utils/mime-types'; @@ -56,6 +56,13 @@ interface UpsertFileOptions { } type ThumbnailAsset = NonNullable>>; +type VideoConversionAsset = NonNullable>>; + +type ThumbnailGenerationResult = { + files: UpsertFileOptions[]; + thumbhash: Buffer; + fullsizeDimensions: ImageDimensions; +}; @Injectable() export class MediaService extends BaseService { @@ -84,7 +91,7 @@ export class MediaService extends BaseService { } if (asset.isEdited) { - jobs.push({ name: JobName.AssetEditThumbnailGeneration, data: { id: asset.id } }); + jobs.push({ name: JobName.AssetProcessEdit, data: { id: asset.id } }); } if (jobs.length >= JOBS_ASSET_PAGINATION_SIZE) { @@ -168,9 +175,9 @@ export class MediaService extends BaseService { return JobStatus.Success; } - @OnJob({ name: JobName.AssetEditThumbnailGeneration, queue: QueueName.Editor }) - async handleAssetEditThumbnailGeneration({ id }: JobOf): Promise { - const asset = await this.assetJobRepository.getForGenerateThumbnailJob(id); + @OnJob({ name: JobName.AssetProcessEdit, queue: QueueName.Editor }) + async handleAssetEditProcessing({ id }: JobOf): Promise { + const asset = await this.assetJobRepository.getForAssetEditProcessing(id); const config = await this.getConfig({ withCache: true }); if (!asset) { @@ -178,7 +185,25 @@ export class MediaService extends BaseService { return JobStatus.Failed; } - const generated = await this.generateEditedThumbnails(asset, config); + switch (asset.type) { + case AssetType.Image: { + await this.handleImageEdit(asset, config); + break; + } + case AssetType.Video: { + await this.handleVideoEdit(asset, config); + break; + } + default: { + this.logger.warn(`Skipping thumbnail generation for asset ${id}: ${asset.type} is not an image or video`); + } + } + + return JobStatus.Success; + } + + private async handleImageEdit(asset: ThumbnailAsset, config: SystemConfig) { + const generated = await this.generateEditedImageThumbnails(asset, config); await this.syncFiles( asset.files.filter((file) => file.isEdited), generated?.files ?? [], @@ -203,54 +228,51 @@ export class MediaService extends BaseService { const fullsizeDimensions = generated?.fullsizeDimensions ?? getDimensions(asset.exifInfo!); await this.assetRepository.update({ id: asset.id, ...fullsizeDimensions }); - - return JobStatus.Success; } - @OnJob({ name: JobName.AssetEditTranscodeGeneration, queue: QueueName.Editor }) - async handleAssetEditTranscodeGeneration({ id }: JobOf): Promise { - const asset = await this.assetJobRepository.getForVideoConversion(id); - if (!asset) { - return JobStatus.Failed; - } + private async handleVideoEdit(asset: ThumbnailAsset, config: SystemConfig) { + // transcode edited video + const generatedVideo = asset.edits.length > 0 ? await this.transcodeVideo(asset, config.ffmpeg, true) : undefined; - const input = asset.originalPath; - const output = StorageCore.getEncodedVideoPath(asset); - this.storageCore.ensureFolders(output); - - const { videoStreams, audioStreams, format } = await this.mediaRepository.probe(input, { - countFrames: this.logger.isLevelEnabled(LogLevel.Debug), // makes frame count more reliable for progress logs - }); - const videoStream = this.getMainStream(videoStreams); - const audioStream = this.getMainStream(audioStreams); - if (!videoStream || !format.formatName) { - return JobStatus.Failed; - } - - if (!videoStream.height || !videoStream.width) { - this.logger.warn(`Skipped transcoding for asset ${asset.id}: no video streams found`); - return JobStatus.Failed; - } - - let { ffmpeg } = await this.getConfig({ withCache: true }); - ffmpeg = { ...ffmpeg, accel: TranscodeHardwareAcceleration.Disabled }; - const command = BaseConfig.create(ffmpeg, this.videoInterfaces).getCommand( - TranscodeTarget.All, - videoStream, - audioStream, - undefined, // TODO: cleaner way to do this? - asset.edits, + await this.syncFiles( + asset.files.filter((file) => file.isEdited && file.type === AssetFileType.EncodedVideo), + generatedVideo ? [generatedVideo.file] : [], ); - await this.mediaRepository.transcode(input, output, command); - await this.assetRepository.upsertFile({ - assetId: asset.id, - type: AssetFileType.EncodedVideo, - path: output, - isEdited: true, - }); + // update asset dimensions + const newDimensions = generatedVideo?.dimensions ?? getDimensions(asset.exifInfo!); + await this.assetRepository.update({ id: asset.id, ...newDimensions }); - return JobStatus.Success; + // if the asset is hidden, we dont need to update the thumbhash or thumbnails + if (asset.visibility === AssetVisibility.Hidden) { + return; + } + + const editedThumbnails = await this.generateEditedVideoThumbnails(asset, config); + await this.syncFiles( + asset.files.filter((file) => file.isEdited && file.type !== AssetFileType.EncodedVideo), + editedThumbnails?.files ?? [], + ); + + let thumbhash: Buffer | undefined = editedThumbnails?.thumbhash; + if (!thumbhash) { + const previewFile = asset.files.find((file) => file.type === AssetFileType.Preview && !file.isEdited); + + if (!previewFile) { + this.logger.warn(`Failed to generate thumbhash for asset ${asset.id}: missing preview file`); + return; + } + + thumbhash = await this.mediaRepository.generateThumbhash(previewFile.path, { + colorspace: config.image.colorspace, + processInvalidImages: process.env.IMMICH_PROCESS_INVALID_IMAGES === 'true', + }); + } + + // update asset table info + if (!asset.thumbhash || Buffer.compare(asset.thumbhash, thumbhash) !== 0) { + await this.assetRepository.update({ id: asset.id, thumbhash }); + } } @OnJob({ name: JobName.AssetGenerateThumbnails, queue: QueueName.ThumbnailGeneration }) @@ -263,31 +285,34 @@ export class MediaService extends BaseService { return JobStatus.Failed; } + let generated: ThumbnailGenerationResult; + let generatedEdited: ThumbnailGenerationResult | undefined; + if (asset.visibility === AssetVisibility.Hidden) { this.logger.verbose(`Thumbnail generation skipped for asset ${id}: not visible`); return JobStatus.Skipped; } - let generated: Awaited>; if (asset.type === AssetType.Video || asset.originalFileName.toLowerCase().endsWith('.gif')) { this.logger.verbose(`Thumbnail generation for video ${id} ${asset.originalPath}`); generated = await this.generateVideoThumbnails(asset, config); + generatedEdited = await this.generateEditedVideoThumbnails(asset, config); } else if (asset.type === AssetType.Image) { this.logger.verbose(`Thumbnail generation for image ${id} ${asset.originalPath}`); generated = await this.generateImageThumbnails(asset, config); + generatedEdited = await this.generateEditedImageThumbnails(asset, config); } else { this.logger.warn(`Skipping thumbnail generation for asset ${id}: ${asset.type} is not an image or video`); return JobStatus.Skipped; } - const editedGenerated = await this.generateEditedThumbnails(asset, config); - if (editedGenerated) { - generated.files.push(...editedGenerated.files); + if (generatedEdited) { + generated.files.push(...generatedEdited.files); } await this.syncFiles(asset.files, generated.files); - const thumbhash = editedGenerated?.thumbhash || generated.thumbhash; + const thumbhash = generatedEdited?.thumbhash || generated.thumbhash; if (!asset.thumbhash || Buffer.compare(asset.thumbhash, thumbhash) !== 0) { await this.assetRepository.update({ id: asset.id, thumbhash }); } @@ -553,20 +578,21 @@ export class MediaService extends BaseService { } private async generateVideoThumbnails( - asset: ThumbnailPathEntity & { originalPath: string }, + asset: ThumbnailPathEntity & { originalPath: string; edits: AssetEditActionItem[] }, { ffmpeg, image }: SystemConfig, + useEdits: boolean = false, ) { const previewFile = this.getImageFile(asset, { fileType: AssetFileType.Preview, format: image.preview.format, - isEdited: false, + isEdited: useEdits, isProgressive: false, isTransparent: false, }); const thumbnailFile = this.getImageFile(asset, { fileType: AssetFileType.Thumbnail, format: image.thumbnail.format, - isEdited: false, + isEdited: useEdits, isProgressive: false, isTransparent: false, }); @@ -579,14 +605,27 @@ export class MediaService extends BaseService { } const mainAudioStream = this.getMainStream(audioStreams); + let edits: AssetEditActionItem[] | undefined; + if (useEdits) { + ffmpeg = { ...ffmpeg, accel: TranscodeHardwareAcceleration.Disabled }; + edits = asset.edits; + } + const previewConfig = ThumbnailConfig.create({ ...ffmpeg, targetResolution: image.preview.size.toString() }); const thumbnailConfig = ThumbnailConfig.create({ ...ffmpeg, targetResolution: image.thumbnail.size.toString() }); - const previewOptions = previewConfig.getCommand(TranscodeTarget.Video, mainVideoStream, mainAudioStream, format); + const previewOptions = previewConfig.getCommand( + TranscodeTarget.Video, + mainVideoStream, + mainAudioStream, + format, + edits, + ); const thumbnailOptions = thumbnailConfig.getCommand( TranscodeTarget.Video, mainVideoStream, mainAudioStream, format, + edits, ); await this.mediaRepository.transcode(asset.originalPath, previewFile.path, previewOptions); @@ -597,73 +636,69 @@ export class MediaService extends BaseService { processInvalidImages: process.env.IMMICH_PROCESS_INVALID_IMAGES === 'true', }); + let fullsizeDimensions = { width: mainVideoStream.width, height: mainVideoStream.height }; + if (useEdits) { + fullsizeDimensions = getOutputDimensions(asset.edits, fullsizeDimensions); + } + return { files: [previewFile, thumbnailFile], thumbhash, - fullsizeDimensions: { width: mainVideoStream.width, height: mainVideoStream.height }, + fullsizeDimensions, }; } - @OnJob({ name: JobName.AssetEncodeVideoQueueAll, queue: QueueName.VideoConversion }) - async handleQueueVideoConversion(job: JobOf): Promise { - const { force } = job; - - let queue: { name: JobName.AssetEncodeVideo; data: { id: string } }[] = []; - for await (const asset of this.assetJobRepository.streamForVideoConversion(force)) { - queue.push({ name: JobName.AssetEncodeVideo, data: { id: asset.id } }); - - if (queue.length >= JOBS_ASSET_PAGINATION_SIZE) { - await this.jobRepository.queueAll(queue); - queue = []; - } - } - - await this.jobRepository.queueAll(queue); - - return JobStatus.Success; - } - - @OnJob({ name: JobName.AssetEncodeVideo, queue: QueueName.VideoConversion }) - async handleVideoConversion({ id }: JobOf): Promise { - const asset = await this.assetJobRepository.getForVideoConversion(id); - if (!asset) { - return JobStatus.Failed; - } - + private async transcodeVideo( + asset: VideoConversionAsset, + ffmpeg: SystemConfigFFmpegDto, + useEdits: boolean = false, + ): Promise<{ file: UpsertFileOptions; dimensions: { width: number; height: number } } | undefined> { const input = asset.originalPath; - const output = StorageCore.getEncodedVideoPath(asset); + const output = StorageCore.getEncodedVideoPath(asset, useEdits); this.storageCore.ensureFolders(output); const { videoStreams, audioStreams, format } = await this.mediaRepository.probe(input, { - countFrames: this.logger.isLevelEnabled(LogLevel.Debug), // makes frame count more reliable for progress logs + countFrames: this.logger.isLevelEnabled(LogLevel.Debug), }); const videoStream = this.getMainStream(videoStreams); const audioStream = this.getMainStream(audioStreams); if (!videoStream || !format.formatName) { - return JobStatus.Failed; + return undefined; } if (!videoStream.height || !videoStream.width) { this.logger.warn(`Skipped transcoding for asset ${asset.id}: no video streams found`); - return JobStatus.Failed; + return undefined; } - let { ffmpeg } = await this.getConfig({ withCache: true }); - const target = this.getTranscodeTarget(ffmpeg, videoStream, audioStream); - if (target === TranscodeTarget.None && !this.isRemuxRequired(ffmpeg, format)) { - const encodedVideo = getAssetFile(asset.files, AssetFileType.EncodedVideo, { isEdited: false }); - if (encodedVideo) { - this.logger.log(`Transcoded video exists for asset ${asset.id}, but is no longer required. Deleting...`); - await this.jobRepository.queue({ name: JobName.FileDelete, data: { files: [encodedVideo.path] } }); - await this.assetRepository.deleteFiles([encodedVideo]); - } else { - this.logger.verbose(`Asset ${asset.id} does not require transcoding based on current policy, skipping`); + let target: TranscodeTarget; + let edits: AssetEditActionItem[] | undefined; + + if (useEdits) { + if (asset.edits.length === 0) { + this.logger.verbose(`Asset ${asset.id} has no edits, skipping edited version transcoding`); + return undefined; } - return JobStatus.Skipped; + ffmpeg = { ...ffmpeg, accel: TranscodeHardwareAcceleration.Disabled }; + target = TranscodeTarget.All; + edits = asset.edits; + } else { + target = this.getTranscodeTarget(ffmpeg, videoStream, audioStream); + if (target === TranscodeTarget.None && !this.isRemuxRequired(ffmpeg, format)) { + this.logger.verbose(`Asset ${asset.id} does not require transcoding based on current policy, skipping`); + return undefined; + } } - const command = BaseConfig.create(ffmpeg, this.videoInterfaces).getCommand(target, videoStream, audioStream); + const command = BaseConfig.create(ffmpeg, this.videoInterfaces).getCommand( + target, + videoStream, + audioStream, + useEdits ? undefined : format, + edits, + ); + if (ffmpeg.accel === TranscodeHardwareAcceleration.Disabled) { this.logger.log(`Transcoding video ${asset.id} without hardware acceleration`); } else { @@ -677,7 +712,7 @@ export class MediaService extends BaseService { } catch (error: any) { this.logger.error(`Error occurred during transcoding: ${error.message}`); if (ffmpeg.accel === TranscodeHardwareAcceleration.Disabled) { - return JobStatus.Failed; + throw error; } let partialFallbackSuccess = false; @@ -685,7 +720,13 @@ export class MediaService extends BaseService { try { this.logger.error(`Retrying with ${ffmpeg.accel.toUpperCase()}-accelerated encoding and software decoding`); ffmpeg = { ...ffmpeg, accelDecode: false }; - const command = BaseConfig.create(ffmpeg, this.videoInterfaces).getCommand(target, videoStream, audioStream); + const command = BaseConfig.create(ffmpeg, this.videoInterfaces).getCommand( + target, + videoStream, + audioStream, + format, + edits, + ); await this.mediaRepository.transcode(input, output, command); partialFallbackSuccess = true; } catch (error: any) { @@ -695,20 +736,92 @@ export class MediaService extends BaseService { if (!partialFallbackSuccess) { this.logger.error(`Retrying with ${ffmpeg.accel.toUpperCase()} acceleration disabled`); - ffmpeg = { ...ffmpeg, accel: TranscodeHardwareAcceleration.Disabled }; // TODO: USE THIS TO DISABLE CPU ENCODING - const command = BaseConfig.create(ffmpeg, this.videoInterfaces).getCommand(target, videoStream, audioStream); + ffmpeg = { ...ffmpeg, accel: TranscodeHardwareAcceleration.Disabled }; + const command = BaseConfig.create(ffmpeg, this.videoInterfaces).getCommand( + target, + videoStream, + audioStream, + format, + edits, + ); await this.mediaRepository.transcode(input, output, command); } } this.logger.log(`Successfully encoded ${asset.id}`); - await this.assetRepository.upsertFile({ - assetId: asset.id, - type: AssetFileType.EncodedVideo, - path: output, - isEdited: false, - }); + let finalDimensions = { width: videoStream.width, height: videoStream.height }; + if (useEdits) { + finalDimensions = getOutputDimensions(asset.edits, finalDimensions); + } + + return { + dimensions: finalDimensions, + file: { + assetId: asset.id, + type: AssetFileType.EncodedVideo, + path: output, + isEdited: useEdits, + isProgressive: false, + isTransparent: false, + }, + }; + } + + @OnJob({ name: JobName.AssetEncodeVideoQueueAll, queue: QueueName.VideoConversion }) + async handleQueueVideoConversion(job: JobOf): Promise { + const { force } = job; + + let jobs: JobItem[] = []; + for await (const asset of this.assetJobRepository.streamForVideoConversion(force)) { + if (force || !asset.isEdited) { + jobs.push({ name: JobName.AssetEncodeVideo, data: { id: asset.id } }); + } + + if (asset.isEdited) { + jobs.push({ name: JobName.AssetProcessEdit, data: { id: asset.id } }); + } + + if (jobs.length >= JOBS_ASSET_PAGINATION_SIZE) { + await this.jobRepository.queueAll(jobs); + jobs = []; + } + } + + await this.jobRepository.queueAll(jobs); + + return JobStatus.Success; + } + + @OnJob({ name: JobName.AssetEncodeVideo, queue: QueueName.VideoConversion }) + async handleVideoConversion({ id }: JobOf): Promise { + const asset = await this.assetJobRepository.getForVideoConversion(id); + if (!asset) { + return JobStatus.Failed; + } + + const { ffmpeg } = await this.getConfig({ withCache: true }); + + const files: UpsertFileOptions[] = []; + + try { + const generated = await this.transcodeVideo(asset, ffmpeg); + if (!generated) { + return JobStatus.Skipped; + } + if (generated?.file) { + files.push(generated.file); + } + + const editedGenerated = await this.transcodeVideo(asset, ffmpeg, true); + if (editedGenerated) { + files.push(editedGenerated.file); + } + } catch { + return JobStatus.Failed; + } + + await this.syncFiles(asset.files, files); return JobStatus.Success; } @@ -920,13 +1033,29 @@ export class MediaService extends BaseService { } } - private async generateEditedThumbnails(asset: ThumbnailAsset, config: SystemConfig) { + private async generateEditedImageThumbnails(asset: ThumbnailAsset, config: SystemConfig) { if (asset.type !== AssetType.Image || (asset.files.length === 0 && asset.edits.length === 0)) { return; } const generated = asset.edits.length > 0 ? await this.generateImageThumbnails(asset, config, true) : undefined; + await this.updateMLVisibilities(asset); + return generated; + } + + private async generateEditedVideoThumbnails(asset: ThumbnailAsset, config: SystemConfig) { + if (asset.type !== AssetType.Video || (asset.files.length === 0 && asset.edits.length === 0)) { + return; + } + + const generated = asset.edits.length > 0 ? await this.generateVideoThumbnails(asset, config, true) : undefined; + await this.updateMLVisibilities(asset); + + return generated; + } + + private async updateMLVisibilities(asset: ThumbnailAsset) { const crop = asset.edits.find((e) => e.action === AssetEditAction.Crop); const cropBox = crop ? { @@ -946,8 +1075,6 @@ export class MediaService extends BaseService { const ocrStatuses = checkOcrVisibility(ocrData, originalDimensions, cropBox); await this.ocrRepository.updateOcrVisibilities(asset.id, ocrStatuses.visible, ocrStatuses.hidden); - - return generated; } private warnOnTransparencyLoss(isTransparent: boolean, format: ImageFormat, assetId: string) { diff --git a/server/src/types.ts b/server/src/types.ts index 5b6d6a0bda..d46583b296 100644 --- a/server/src/types.ts +++ b/server/src/types.ts @@ -391,8 +391,7 @@ export type JobItem = | { name: JobName.WorkflowRun; data: IWorkflowJob } // Editor - | { name: JobName.AssetEditThumbnailGeneration; data: IEntityJob } - | { name: JobName.AssetEditTranscodeGeneration; data: IEntityJob }; + | { name: JobName.AssetProcessEdit; data: IEntityJob }; export type VectorExtension = (typeof VECTOR_EXTENSIONS)[number]; diff --git a/server/src/utils/database.ts b/server/src/utils/database.ts index 03998d9462..ef2a2690dd 100644 --- a/server/src/utils/database.ts +++ b/server/src/utils/database.ts @@ -126,12 +126,13 @@ export function withFiles(eb: ExpressionBuilder, type?: AssetFileTy ).as('files'); } -export function withFilePath(eb: ExpressionBuilder, type: AssetFileType) { +export function withFilePath(eb: ExpressionBuilder, type: AssetFileType, isEdited = false) { return eb .selectFrom('asset_file') .select('asset_file.path') .whereRef('asset_file.assetId', '=', 'asset.id') - .where('asset_file.type', '=', type); + .where('asset_file.type', '=', type) + .where('asset_file.isEdited', '=', isEdited); } export function withFacesAndPeople( diff --git a/server/src/utils/editor.ts b/server/src/utils/editor.ts index 21678f2a82..8b82c39469 100644 --- a/server/src/utils/editor.ts +++ b/server/src/utils/editor.ts @@ -1,4 +1,5 @@ import { AssetFace } from 'src/database'; +import { AssetEditActionItem, CropParameters } from 'src/dtos/editing.dto'; import { AssetOcrResponseDto } from 'src/dtos/ocr.dto'; import { ImageDimensions } from 'src/types'; @@ -31,6 +32,15 @@ const scale = (box: BoundingBox, target: ImageDimensions, source?: ImageDimensio }; }; +const scaleCrop = (crop: CropParameters, target: ImageDimensions, source: ImageDimensions) => { + return { + width: Math.round((crop.width / source.width) * target.width), + height: Math.round((crop.height / source.height) * target.height), + x: Math.round((crop.x / source.width) * target.width), + y: Math.round((crop.y / source.height) * target.height), + }; +}; + export const checkFaceVisibility = ( faces: AssetFace[], originalAssetDimensions: ImageDimensions, @@ -105,3 +115,20 @@ export const checkOcrVisibility = ( hidden: status.filter((s) => !s.isVisible).map((s) => s.ocr), }; }; + +export const scaleEdits = ( + edits: AssetEditActionItem[], + target: ImageDimensions, + source: ImageDimensions, +): AssetEditActionItem[] => { + return edits.map((edit) => { + if (edit.action === 'crop') { + return { + ...edit, + parameters: scaleCrop(edit.parameters as CropParameters, target, source), + } as AssetEditActionItem; + } + + return edit; + }); +}; diff --git a/server/src/utils/media.ts b/server/src/utils/media.ts index 039de27ce2..8b49a26e43 100644 --- a/server/src/utils/media.ts +++ b/server/src/utils/media.ts @@ -8,7 +8,7 @@ import { RotateParameters, } from 'src/dtos/editing.dto'; import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto'; -import { CQMode, LogLevel, ToneMapping, TranscodeHardwareAcceleration, TranscodeTarget, VideoCodec } from 'src/enum'; +import { CQMode, ToneMapping, TranscodeHardwareAcceleration, TranscodeTarget, VideoCodec } from 'src/enum'; import { AudioStreamInfo, BitrateDistribution, @@ -98,8 +98,17 @@ export class BaseConfig implements VideoCodecSWConfig { format?: VideoFormat, edits: AssetEditActionItem[] = [], ) { + const inputOptions = this.getBaseInputOptions(videoStream, format); + + if (edits.length > 0) { + // turns out MOV files can have cropping metadata that ffmpeg automatically applies when decoding + // this means that the video streams dimensions can just be wrong once it hits the filter pipeline + // https://github.com/FFmpeg/FFmpeg/blob/f40fcf802472227851e0b8eeba40b9e6b3b8a3a1/libavutil/frame.h#L1021 + inputOptions.push('-apply_cropping 0'); + } + const options = { - inputOptions: this.getBaseInputOptions(videoStream, format), + inputOptions, outputOptions: [...this.getBaseOutputOptions(target, videoStream, audioStream), '-v verbose'], twoPass: this.eligibleForTwoPass(), progress: { frameCount: videoStream.frameCount, percentInterval: 5 }, diff --git a/web/src/lib/managers/asset-viewer-manager.svelte.ts b/web/src/lib/managers/asset-viewer-manager.svelte.ts index 0bab3aff80..aa921cce82 100644 --- a/web/src/lib/managers/asset-viewer-manager.svelte.ts +++ b/web/src/lib/managers/asset-viewer-manager.svelte.ts @@ -139,6 +139,7 @@ export class AssetViewerManager extends BaseEventManager { openEditor() { this.closeActivityPanel(); + this.isPlayingMotionPhoto = false; this.isShowEditor = true; } diff --git a/web/src/lib/managers/edit/edit-manager.svelte.ts b/web/src/lib/managers/edit/edit-manager.svelte.ts index 9eba741c36..f770425de4 100644 --- a/web/src/lib/managers/edit/edit-manager.svelte.ts +++ b/web/src/lib/managers/edit/edit-manager.svelte.ts @@ -128,6 +128,8 @@ export class EditManager { try { // Setup the websocket listener before sending the edit request const editCompleted = waitForWebsocketEvent('AssetEditReadyV1', (event) => event.asset.id === assetId, 10_000); + if (this.currentAsset.livePhotoVideoId) { + } await (edits.length === 0 ? removeAssetEdits({ id: assetId }) diff --git a/web/src/lib/utils.ts b/web/src/lib/utils.ts index f5d46576cb..f23d6e6eac 100644 --- a/web/src/lib/utils.ts +++ b/web/src/lib/utils.ts @@ -238,8 +238,8 @@ export const getAssetMediaUrl = (options: AssetUrlOptions) => { }; export const getAssetPlaybackUrl = (options: AssetUrlOptions) => { - const { id, cacheKey: c } = options; - return createUrl(getAssetPlaybackPath(id), { ...authManager.params, c }); + const { id, cacheKey: c, edited = true } = options; + return createUrl(getAssetPlaybackPath(id), { ...authManager.params, c, edited }); }; export const getProfileImageUrl = (user: UserResponseDto) =>