From 1556d978edc8f4ce3f130cefea5a97e98638a983 Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Mon, 13 May 2024 19:42:45 -0400 Subject: [PATCH] toggle for hardware decoding, software / hardware decoding for nvenc and rkmpp --- server/src/config.ts | 2 + server/src/dtos/system-config.dto.ts | 3 + server/src/entities/system-config.entity.ts | 1 + server/src/services/media.service.spec.ts | 81 ++++++++++++++++- .../services/system-config.service.spec.ts | 1 + server/src/utils/media.ts | 86 +++++++++++++------ 6 files changed, 148 insertions(+), 26 deletions(-) diff --git a/server/src/config.ts b/server/src/config.ts index a9a9b2398c..69ba84dbff 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -97,6 +97,7 @@ export interface SystemConfig { preferredHwDevice: string; transcode: TranscodePolicy; accel: TranscodeHWAccel; + accelDecode: false; tonemap: ToneMapping; }; job: Record; @@ -224,6 +225,7 @@ export const defaults = Object.freeze({ transcode: TranscodePolicy.REQUIRED, tonemap: ToneMapping.HABLE, accel: TranscodeHWAccel.DISABLED, + accelDecode: false, }, job: { [QueueName.BACKGROUND_TASK]: { concurrency: 5 }, diff --git a/server/src/dtos/system-config.dto.ts b/server/src/dtos/system-config.dto.ts index da68c27478..42fa645c83 100644 --- a/server/src/dtos/system-config.dto.ts +++ b/server/src/dtos/system-config.dto.ts @@ -132,6 +132,9 @@ export class SystemConfigFFmpegDto { @ApiProperty({ enumName: 'TranscodeHWAccel', enum: TranscodeHWAccel }) accel!: TranscodeHWAccel; + @ValidateBoolean() + accelDecode!: boolean; + @IsEnum(ToneMapping) @ApiProperty({ enumName: 'ToneMapping', enum: ToneMapping }) tonemap!: ToneMapping; diff --git a/server/src/entities/system-config.entity.ts b/server/src/entities/system-config.entity.ts index 64342cc195..e956a737dd 100644 --- a/server/src/entities/system-config.entity.ts +++ b/server/src/entities/system-config.entity.ts @@ -43,6 +43,7 @@ export const SystemConfigKey = { FFMPEG_PREFERRED_HW_DEVICE: 'ffmpeg.preferredHwDevice', FFMPEG_TRANSCODE: 'ffmpeg.transcode', FFMPEG_ACCEL: 'ffmpeg.accel', + FFMPEG_ACCEL_DECODE: 'ffmpeg.accelDecode', FFMPEG_TONEMAP: 'ffmpeg.tonemap', JOB_THUMBNAIL_GENERATION_CONCURRENCY: 'job.thumbnailGeneration.concurrency', diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts index 4a7a6836af..3aafcc8513 100644 --- a/server/src/services/media.service.spec.ts +++ b/server/src/services/media.service.spec.ts @@ -1381,6 +1381,52 @@ describe(MediaService.name, () => { ); }); + it('should use hardware decoding for nvenc if enabled', async () => { + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + configMock.load.mockResolvedValue([ + { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.NVENC }, + { key: SystemConfigKey.FFMPEG_ACCEL_DECODE, value: true }, + ]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', + { + inputOptions: expect.arrayContaining(['-hwaccel cuda', '-hwaccel_output_format cuda']), + outputOptions: expect.arrayContaining([ + expect.stringContaining( + 'hwupload=derive_device=vulkan,scale_vulkan=w=1280:h=720,libplacebo=color_primaries=bt709:color_trc=bt709:colorspace=bt709:deband=true:deband_iterations=3:deband_radius=8:deband_threshold=6:downscaler=none:format=yuv420p:tonemapping=clip:upscaler=none,hwupload=derive_device=cuda', + ), + ]), + twoPass: false, + }, + ); + }); + + it('should use hardware tone-mapping for nvenc if hardware decoding is enabled and should tone map', async () => { + mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR); + configMock.load.mockResolvedValue([ + { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.NVENC }, + { key: SystemConfigKey.FFMPEG_ACCEL_DECODE, value: true }, + ]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', + { + inputOptions: expect.arrayContaining(['-hwaccel cuda', '-hwaccel_output_format cuda']), + outputOptions: expect.arrayContaining([ + expect.stringContaining( + 'hwupload=derive_device=vulkan,libplacebo=color_primaries=bt709:color_trc=bt709:colorspace=bt709:deband=true:deband_iterations=3:deband_radius=8:deband_threshold=6:downscaler=none:format=yuv420p:tonemapping=hable:upscaler=none,hwupload=derive_device=cuda', + ), + ]), + twoPass: false, + }, + ); + }); + it('should set options for qsv', async () => { storageMock.readdir.mockResolvedValue(['renderD128']); mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); @@ -1694,7 +1740,10 @@ describe(MediaService.name, () => { it('should set options for rkmpp', async () => { storageMock.readdir.mockResolvedValue(['renderD128']); mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); - configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.RKMPP }]); + configMock.load.mockResolvedValue([ + { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.RKMPP }, + { key: SystemConfigKey.FFMPEG_ACCEL_DECODE, value: true }, + ]); assetMock.getByIds.mockResolvedValue([assetStub.video]); await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( @@ -1726,6 +1775,7 @@ describe(MediaService.name, () => { mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9); configMock.load.mockResolvedValue([ { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.RKMPP }, + { key: SystemConfigKey.FFMPEG_ACCEL_DECODE, value: true }, { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '10000k' }, { key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.HEVC }, ]); @@ -1747,6 +1797,7 @@ describe(MediaService.name, () => { mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); configMock.load.mockResolvedValue([ { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.RKMPP }, + { key: SystemConfigKey.FFMPEG_ACCEL_DECODE, value: true }, { key: SystemConfigKey.FFMPEG_CRF, value: 30 }, { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '0' }, ]); @@ -1769,6 +1820,7 @@ describe(MediaService.name, () => { mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR); configMock.load.mockResolvedValue([ { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.RKMPP }, + { key: SystemConfigKey.FFMPEG_ACCEL_DECODE, value: true }, { key: SystemConfigKey.FFMPEG_CRF, value: 30 }, { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '0' }, ]); @@ -1788,6 +1840,33 @@ describe(MediaService.name, () => { }, ); }); + + it('should use software decoding and tone-mapping if hardware decoding is disabled', async () => { + storageMock.readdir.mockResolvedValue(['renderD128']); + storageMock.stat.mockResolvedValue({ ...new Stats(), isFile: () => true, isCharacterDevice: () => true }); + mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR); + configMock.load.mockResolvedValue([ + { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.RKMPP }, + { key: SystemConfigKey.FFMPEG_ACCEL_DECODE, value: false }, + { key: SystemConfigKey.FFMPEG_CRF, value: 30 }, + { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '0' }, + ]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', + { + inputOptions: [], + outputOptions: expect.arrayContaining([ + expect.stringContaining( + 'zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:range=pc,format=yuv420p', + ), + ]), + twoPass: false, + }, + ); + }); }); it('should tonemap when policy is required and video is hdr', async () => { diff --git a/server/src/services/system-config.service.spec.ts b/server/src/services/system-config.service.spec.ts index d345d55df6..f2f15a8fe0 100644 --- a/server/src/services/system-config.service.spec.ts +++ b/server/src/services/system-config.service.spec.ts @@ -65,6 +65,7 @@ const updatedConfig = Object.freeze({ preferredHwDevice: 'auto', transcode: TranscodePolicy.REQUIRED, accel: TranscodeHWAccel.DISABLED, + accelDecode: false, tonemap: ToneMapping.HABLE, }, logging: { diff --git a/server/src/utils/media.ts b/server/src/utils/media.ts index dcd598ab94..7a7c7464e7 100644 --- a/server/src/utils/media.ts +++ b/server/src/utils/media.ts @@ -26,14 +26,18 @@ class BaseConfig implements VideoCodecSWConfig { } } - options.outputOptions.push(...this.getPresetOptions(), ...this.getThreadOptions(), ...this.getBitrateOptions()); + options.outputOptions.push( + ...this.getPresetOptions(), + ...this.getOutputThreadOptions(), + ...this.getBitrateOptions(), + ); return options; } // eslint-disable-next-line @typescript-eslint/no-unused-vars getBaseInputOptions(videoStream: VideoStreamInfo): string[] { - return []; + return this.getInputThreadOptions(); } getBaseOutputOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) { @@ -80,11 +84,7 @@ class BaseConfig implements VideoCodecSWConfig { options.push(`scale=${this.getScaling(videoStream)}`); } - if (this.shouldToneMap(videoStream)) { - options.push(...this.getToneMapping()); - } - options.push('format=yuv420p'); - + options.push(...this.getToneMapping(videoStream), 'format=yuv420p'); return options; } @@ -112,7 +112,11 @@ class BaseConfig implements VideoCodecSWConfig { } } - getThreadOptions(): Array { + getInputThreadOptions(): Array { + return []; + } + + getOutputThreadOptions(): Array { if (this.config.threads <= 0) { return []; } @@ -218,7 +222,11 @@ class BaseConfig implements VideoCodecSWConfig { } } - getToneMapping() { + getToneMapping(videoStream: VideoStreamInfo) { + if (!this.shouldToneMap(videoStream)) { + return []; + } + const colors = this.getColors(); return [ @@ -348,8 +356,8 @@ export class ThumbnailConfig extends BaseConfig { } export class H264Config extends BaseConfig { - getThreadOptions() { - const options = super.getThreadOptions(); + getOutputThreadOptions() { + const options = super.getOutputThreadOptions(); if (this.config.threads === 1) { options.push('-x264-params frame-threads=1:pools=none'); } @@ -359,8 +367,8 @@ export class H264Config extends BaseConfig { } export class HEVCConfig extends BaseConfig { - getThreadOptions() { - const options = super.getThreadOptions(); + getOutputThreadOptions() { + const options = super.getOutputThreadOptions(); if (this.config.threads === 1) { options.push('-x265-params frame-threads=1:pools=none'); } @@ -391,8 +399,8 @@ export class VP9Config extends BaseConfig { return [`-${this.useCQP() ? 'q:v' : 'crf'} ${this.config.crf}`, `-b:v ${bitrates.max}${bitrates.unit}`]; } - getThreadOptions() { - return ['-row-mt 1', ...super.getThreadOptions()]; + getOutputThreadOptions() { + return ['-row-mt 1', ...super.getOutputThreadOptions()]; } eligibleForTwoPass() { @@ -425,7 +433,7 @@ export class AV1Config extends BaseConfig { return options; } - getThreadOptions() { + getOutputThreadOptions() { return []; // Already set above with svtav1-params } @@ -440,7 +448,11 @@ export class NVENCConfig extends BaseHWConfig { } getBaseInputOptions() { - return ['-hwaccel cuda', '-hwaccel_output_format cuda', ...this.getThreadOptions()]; + if (!this.config.accelDecode) { + return ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda', ...this.getInputThreadOptions()]; + } + + return ['-hwaccel cuda', '-hwaccel_output_format cuda', ...this.getInputThreadOptions()]; } getBaseOutputOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) { @@ -462,12 +474,30 @@ export class NVENCConfig extends BaseHWConfig { } getFilterOptions(videoStream: VideoStreamInfo) { - const options = ['hwupload=derive_device=vulkan']; + const options = []; + if (!this.config.accelDecode) { + options.push('format=nv12', 'hwupload_cuda'); + if (this.shouldScale(videoStream)) { + options.push(`scale_cuda=${this.getScaling(videoStream)}`); + } + return options; + } + + options.push('hwupload=derive_device=vulkan'); if (this.shouldScale(videoStream)) { const { width, height } = this.getSize(videoStream); options.push(`scale_vulkan=w=${width}:h=${height}`); } + options.push(...this.getToneMapping(videoStream), 'hwupload=derive_device=cuda'); + return options; + } + + getToneMapping(videoStream: VideoStreamInfo) { + if (!this.config.accelDecode) { + return super.getToneMapping(videoStream); + } + const colors = this.getColors(); const libplaceboOptions = [ `color_primaries=${colors.primaries}`, @@ -483,9 +513,7 @@ export class NVENCConfig extends BaseHWConfig { 'upscaler=none', ]; - const libplacebo = `libplacebo=${libplaceboOptions.join(':')}`; - options.push(libplacebo, 'hwupload=derive_device=cuda'); - return options; + return [`libplacebo=${libplaceboOptions.join(':')}`]; } getPresetOptions() { @@ -517,10 +545,14 @@ export class NVENCConfig extends BaseHWConfig { } } - getThreadOptions() { + getInputThreadOptions() { return [`-threads ${this.config.threads <= 0 ? 1 : this.config.threads}`]; } + getOutputThreadOptions() { + return []; + } + getRefs() { const bframes = this.getBFrames(); if (bframes > 0 && bframes < 3 && this.config.refs < 3) { @@ -555,7 +587,7 @@ export class QSVConfig extends BaseHWConfig { } getFilterOptions(videoStream: VideoStreamInfo) { - const options = this.shouldToneMap(videoStream) ? this.getToneMapping() : []; + const options = this.getToneMapping(videoStream); options.push('format=nv12', 'hwupload=extra_hw_frames=64'); if (this.shouldScale(videoStream)) { options.push(`scale_qsv=${this.getScaling(videoStream)}`); @@ -621,7 +653,7 @@ export class VAAPIConfig extends BaseHWConfig { } getFilterOptions(videoStream: VideoStreamInfo) { - const options = this.shouldToneMap(videoStream) ? this.getToneMapping() : []; + const options = this.getToneMapping(videoStream); options.push('format=nv12', 'hwupload'); if (this.shouldScale(videoStream)) { options.push(`scale_vaapi=${this.getScaling(videoStream)}`); @@ -693,12 +725,16 @@ export class RKMPPConfig extends BaseHWConfig { if (this.devices.length === 0) { throw new Error('No RKMPP device found'); } - return this.shouldToneMap(videoStream) && !this.hasOpenCL + return !this.config.accelDecode || (this.shouldToneMap(videoStream) && !this.hasOpenCL) ? [] // disable hardware decoding & filters : ['-hwaccel rkmpp', '-hwaccel_output_format drm_prime', '-afbc rga']; } getFilterOptions(videoStream: VideoStreamInfo) { + if (!this.config.accelDecode) { + return super.getFilterOptions(videoStream); + } + if (this.shouldToneMap(videoStream)) { if (!this.hasOpenCL) { return super.getFilterOptions(videoStream);