Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

avfilter/tonemap_videotoolbox: improve quality #416

Merged
merged 11 commits into from
Jul 24, 2024
4 changes: 2 additions & 2 deletions builder/scripts.d/25-xz.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/bin/bash

SCRIPT_REPO="https://git.tukaani.org/xz.git"
SCRIPT_COMMIT="v5.6.1"
SCRIPT_REPO="https://github.com/tukaani-project/xz.git"
SCRIPT_COMMIT="v5.6.2"
SCRIPT_TAGFILTER="v?.*.*"

ffbuild_enabled() {
Expand Down
180 changes: 112 additions & 68 deletions debian/patches/0069-add-tonemap-videotoolbox.patch
Original file line number Diff line number Diff line change
@@ -1,30 +1,22 @@
Subject: [PATCH] avfilter: add vf_tonemap_videotoolbox
---
Index: configure
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
Index: FFmpeg/configure
===================================================================
diff --git a/configure b/configure
--- a/configure (revision 2e3649a10ff797e360eb9fd5d389dfa6611dea59)
+++ b/configure (revision bcb63879b775fef8085a5311f045dc1cc4960a98)
@@ -3748,6 +3748,7 @@
--- FFmpeg.orig/configure
+++ FFmpeg/configure
@@ -3772,6 +3772,7 @@ tinterlace_pad_test_deps="tinterlace_fil
tonemap_filter_deps="const_nan"
tonemap_vaapi_filter_deps="vaapi VAProcFilterParameterBufferHDRToneMapping"
tonemap_opencl_filter_deps="opencl const_nan"
+tonemap_videotoolbox_filter_deps="metal corevideo videotoolbox const_nan"
transpose_opencl_filter_deps="opencl"
transpose_vaapi_filter_deps="vaapi VAProcPipelineCaps_rotation_flags"
transpose_vulkan_filter_deps="vulkan spirv_compiler"
Index: libavfilter/Makefile
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
Index: FFmpeg/libavfilter/Makefile
===================================================================
diff --git a/libavfilter/Makefile b/libavfilter/Makefile
--- a/libavfilter/Makefile (revision 2e3649a10ff797e360eb9fd5d389dfa6611dea59)
+++ b/libavfilter/Makefile (revision bcb63879b775fef8085a5311f045dc1cc4960a98)
@@ -510,6 +510,9 @@
--- FFmpeg.orig/libavfilter/Makefile
+++ FFmpeg/libavfilter/Makefile
@@ -521,6 +521,9 @@ OBJS-$(CONFIG_TONEMAP_OPENCL_FILTER)
OBJS-$(CONFIG_TONEMAP_CUDA_FILTER) += vf_tonemap_cuda.o cuda/tonemap.ptx.o \
cuda/host_util.o
OBJS-$(CONFIG_TONEMAP_VAAPI_FILTER) += vf_tonemap_vaapi.o vaapi_vpp.o
Expand All @@ -34,33 +26,23 @@ diff --git a/libavfilter/Makefile b/libavfilter/Makefile
OBJS-$(CONFIG_TPAD_FILTER) += vf_tpad.o
OBJS-$(CONFIG_TRANSPOSE_FILTER) += vf_transpose.o
OBJS-$(CONFIG_TRANSPOSE_CUDA_FILTER) += vf_transpose_cuda.o vf_transpose_cuda.ptx.o \
cuda/load_helper.o
Index: libavfilter/allfilters.c
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
Index: FFmpeg/libavfilter/allfilters.c
===================================================================
diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
--- a/libavfilter/allfilters.c (revision 2e3649a10ff797e360eb9fd5d389dfa6611dea59)
+++ b/libavfilter/allfilters.c (revision bcb63879b775fef8085a5311f045dc1cc4960a98)
@@ -480,6 +480,7 @@
--- FFmpeg.orig/libavfilter/allfilters.c
+++ FFmpeg/libavfilter/allfilters.c
@@ -487,6 +487,7 @@ extern const AVFilter ff_vf_tonemap;
extern const AVFilter ff_vf_tonemap_cuda;
extern const AVFilter ff_vf_tonemap_opencl;
extern const AVFilter ff_vf_tonemap_vaapi;
+extern const AVFilter ff_vf_tonemap_videotoolbox;
extern const AVFilter ff_vf_tpad;
extern const AVFilter ff_vf_transpose;
extern const AVFilter ff_vf_transpose_cuda;
Index: libavfilter/metal/vf_tonemap_videotoolbox.metal
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
Index: FFmpeg/libavfilter/metal/vf_tonemap_videotoolbox.metal
===================================================================
diff --git a/libavfilter/metal/vf_tonemap_videotoolbox.metal b/libavfilter/metal/vf_tonemap_videotoolbox.metal
new file mode 100644
--- /dev/null (revision bcb63879b775fef8085a5311f045dc1cc4960a98)
+++ b/libavfilter/metal/vf_tonemap_videotoolbox.metal (revision bcb63879b775fef8085a5311f045dc1cc4960a98)
@@ -0,0 +1,688 @@
--- /dev/null
+++ FFmpeg/libavfilter/metal/vf_tonemap_videotoolbox.metal
@@ -0,0 +1,725 @@
+/*
+ * Copyright (c) 2024 Gnattu OC <[email protected]>
+ *
Expand Down Expand Up @@ -142,12 +124,23 @@ new file mode 100644
+constant short linearize_type [[function_constant(36)]];
+constant short delinearize_type [[function_constant(37)]];
+
+enum AVChromaLocation {
+ AVCHROMA_LOC_UNSPECIFIED,
+ AVCHROMA_LOC_LEFT,
+ AVCHROMA_LOC_CENTER,
+ AVCHROMA_LOC_TOPLEFT,
+ AVCHROMA_LOC_TOP,
+ AVCHROMA_LOC_BOTTOMLEFT,
+ AVCHROMA_LOC_BOTTOM,
+ AVCHROMA_LOC_NB
+};
+
+float3 get_chroma_sample(float3 a, float3 b, float3 c,float3 d) {
+ if (chroma_loc == 1) return (((a) + (c)) * 0.5f);
+ if (chroma_loc == 3) return a;
+ if (chroma_loc == 4) return (((a) + (b)) * 0.5f);
+ if (chroma_loc == 5) return c;
+ if (chroma_loc == 6) return (((c) + (d)) * 0.5f);
+ if (chroma_loc == AVCHROMA_LOC_LEFT) return (((a) + (c)) * 0.5f);
+ if (chroma_loc == AVCHROMA_LOC_TOPLEFT) return a;
+ if (chroma_loc == AVCHROMA_LOC_TOP) return (((a) + (b)) * 0.5f);
+ if (chroma_loc == AVCHROMA_LOC_BOTTOMLEFT) return c;
+ if (chroma_loc == AVCHROMA_LOC_BOTTOM) return (((c) + (d)) * 0.5f);
+ return (((a) + (b) + (c) + (d)) * 0.25f);
+}
+
Expand Down Expand Up @@ -287,7 +280,9 @@ new file mode 100644
+
+float rgb2y(float r, float g, float b) {
+ float y = (r*yuv_matrix_1[0]) + (g*yuv_matrix_1[1]) + (b*yuv_matrix_1[2]);
+ y = (219.0f * y + 16.0f) / 255.0f;
+ if (!is_full_range_out) {
+ y = (219.0f * y + 16.0f) / 255.0f;
+ }
+ return y;
+}
+
Expand Down Expand Up @@ -355,6 +350,18 @@ new file mode 100644
+
+//------------
+// Tonemapping methods
+enum TonemapAlgorithm {
+ TONEMAP_NONE,
+ TONEMAP_LINEAR,
+ TONEMAP_GAMMA,
+ TONEMAP_CLIP,
+ TONEMAP_REINHARD,
+ TONEMAP_HABLE,
+ TONEMAP_MOBIUS,
+ TONEMAP_BT2390,
+ TONEMAP_COUNT,
+};
+
+float hable_f(float in) {
+ float a = 0.15f, b = 0.50f, c = 0.10f, d = 0.20f, e = 0.02f, f = 0.30f;
+ return (in * (in * a + b * c) + d * e) / (in * (in * a + b) + d * f) - e / f;
Expand Down Expand Up @@ -419,28 +426,28 @@ new file mode 100644
+}
+
+float tonemap(float s, float peak, float target_peak) {
+ if (tonemap_func_type == 1) {
+ if (tonemap_func_type == TONEMAP_NONE) {
+ return direct(s, peak, target_peak);
+ }
+ if (tonemap_func_type == 2) {
+ if (tonemap_func_type == TONEMAP_LINEAR) {
+ return linear(s, peak, target_peak);
+ }
+ if (tonemap_func_type == 3) {
+ if (tonemap_func_type == TONEMAP_GAMMA) {
+ return gamma(s, peak, target_peak);
+ }
+ if (tonemap_func_type == 4) {
+ if (tonemap_func_type == TONEMAP_CLIP) {
+ return clip(s, peak, target_peak);
+ }
+ if (tonemap_func_type == 5) {
+ if (tonemap_func_type == TONEMAP_REINHARD) {
+ return reinhard(s, peak, target_peak);
+ }
+ if (tonemap_func_type == 6) {
+ if (tonemap_func_type == TONEMAP_HABLE) {
+ return hable(s, peak, target_peak);
+ }
+ if (tonemap_func_type == 7) {
+ if (tonemap_func_type == TONEMAP_MOBIUS) {
+ return mobius(s, peak, target_peak);
+ }
+ if (tonemap_func_type == 8) {
+ if (tonemap_func_type == TONEMAP_BT2390) {
+ return bt2390(s, peak, target_peak);
+ }
+ return direct(s, peak, target_peak);
Expand All @@ -459,9 +466,17 @@ new file mode 100644
+ sig.w = tonemap(sig.w, peak, target_peak); \
+}
+ if (is_tone_mode_rgb) {
+ float4 sig_r = fmax(*r4, FLOAT_EPS), sig_ro = sig_r;
+ float4 sig_g = fmax(*g4, FLOAT_EPS), sig_go = sig_g;
+ float4 sig_b = fmax(*b4, FLOAT_EPS), sig_bo = sig_b;
+ float4 sig_r = fmax(*r4, FLOAT_EPS);
+ float4 sig_g = fmax(*g4, FLOAT_EPS);
+ float4 sig_b = fmax(*b4, FLOAT_EPS);
+ if (is_tone_func_bt2390) {
+ sig_r = fmin(sig_r, peak);
+ sig_g = fmin(sig_g, peak);
+ sig_b = fmin(sig_b, peak);
+ }
+ float4 sig_ro = sig_r;
+ float4 sig_go = sig_g;
+ float4 sig_bo = sig_b;
+ // Desaturate the color using a coefficient dependent on the signal level
+ if (desat_param > 0.0f) {
+ float4 sig = fmax(fmax(*r4, fmax(*g4, *b4)), FLOAT_EPS);
Expand All @@ -475,9 +490,6 @@ new file mode 100644
+ if (is_tone_func_bt2390) {
+ float src_peak_delin_pq = inverse_eotf_st2084(peak);
+ float dst_peak_delin_pq = inverse_eotf_st2084(1.0f);
+ sig_r = fmin(sig_r, dst_peak_delin_pq);
+ sig_g = fmin(sig_g, dst_peak_delin_pq);
+ sig_b = fmin(sig_b, dst_peak_delin_pq);
+ MAP_FOUR_PIXELS(sig_r, src_peak_delin_pq, dst_peak_delin_pq)
+ MAP_FOUR_PIXELS(sig_g, src_peak_delin_pq, dst_peak_delin_pq)
+ MAP_FOUR_PIXELS(sig_b, src_peak_delin_pq, dst_peak_delin_pq)
Expand All @@ -496,7 +508,15 @@ new file mode 100644
+ *g4 *= factor_g;
+ *b4 *= factor_b;
+ } else {
+ float4 sig = fmax(fmax(*r4, fmax(*g4, *b4)), FLOAT_EPS);
+ float4 sig;
+ if (is_tone_mode_max) {
+ sig = fmax(fmax3(*r4, *g4, *b4), FLOAT_EPS);
+ } else {
+ sig = fmax((*r4 * 0.2627f + *g4 * 0.678f + *b4 * 0.0593f), FLOAT_EPS);
+ }
+ if (is_tone_func_bt2390) {
+ sig = fmin(sig, peak);
+ }
+ float4 sig_o = sig;
+ if (desat_param > 0.0f) {
+ float4 luma = get_luma_dst4(*r4, *g4, *b4);
Expand All @@ -509,7 +529,6 @@ new file mode 100644
+ if (is_tone_func_bt2390) {
+ float src_peak_delin_pq = inverse_eotf_st2084(peak);
+ float dst_peak_delin_pq = inverse_eotf_st2084(1.0f);
+ sig = fmin(sig, src_peak_delin_pq);
+ MAP_FOUR_PIXELS(sig, src_peak_delin_pq, dst_peak_delin_pq)
+ } else {
+ MAP_FOUR_PIXELS(sig, peak, 1.0f)
Expand Down Expand Up @@ -749,16 +768,11 @@ new file mode 100644
+ dst2.write(float4(chroma.y, chroma.z, 0.0f, 1.0f), uint2(xi, yi));
+ }
+}
Index: libavfilter/vf_tonemap_videotoolbox.m
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
Index: FFmpeg/libavfilter/vf_tonemap_videotoolbox.m
===================================================================
diff --git a/libavfilter/vf_tonemap_videotoolbox.m b/libavfilter/vf_tonemap_videotoolbox.m
new file mode 100644
--- /dev/null (revision bcb63879b775fef8085a5311f045dc1cc4960a98)
+++ b/libavfilter/vf_tonemap_videotoolbox.m (revision bcb63879b775fef8085a5311f045dc1cc4960a98)
@@ -0,0 +1,1106 @@
--- /dev/null
+++ FFmpeg/libavfilter/vf_tonemap_videotoolbox.m
@@ -0,0 +1,1136 @@
+/*
+ * Copyright (c) 2024 Gnattu OC <[email protected]>
+ *
Expand Down Expand Up @@ -827,6 +841,7 @@ new file mode 100644
+enum TonemapMode {
+ TONEMAP_MODE_MAX,
+ TONEMAP_MODE_RGB,
+ TONEMAP_MODE_LUM,
+ TONEMAP_MODE_COUNT,
+};
+
Expand Down Expand Up @@ -1229,7 +1244,7 @@ new file mode 100644
+ tonemap_func_type = (short)ctx->tonemap;
+ is_tone_func_bt2390 = ctx->tonemap == TONEMAP_BT2390;
+ is_tone_mode_rgb = ctx->tonemap_mode == TONEMAP_MODE_RGB;
+ is_tone_mode_max = !is_tone_mode_rgb;
+ is_tone_mode_max = ctx->tonemap_mode == TONEMAP_MODE_MAX;
+ is_non_semi_planar_in = ctx->in_planes > 2;
+ is_non_semi_planar_out = ctx->out_planes > 2;
+ enable_dither = ctx->in_desc->comp[0].depth > ctx->out_desc->comp[0].depth;
Expand Down Expand Up @@ -1777,6 +1792,34 @@ new file mode 100644
+ if (ctx->trc_out != AVCOL_TRC_SMPTE2084) {
+ av_frame_remove_side_data(output, AV_FRAME_DATA_MASTERING_DISPLAY_METADATA);
+ av_frame_remove_side_data(output, AV_FRAME_DATA_CONTENT_LIGHT_LEVEL);
+ } else {
+ CGColorSpaceRef colorspace = NULL;
+ CFStringRef colormatrix = kCVImageBufferYCbCrMatrix_ITU_R_2020;
+ CFStringRef colorpri = kCVImageBufferColorPrimaries_ITU_R_2020;
+ CFStringRef colortrc = kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ;
+ CFMutableDictionaryRef attachments = CFDictionaryCreateMutable(NULL, 4,
+ &kCFTypeDictionaryKeyCallBacks,
+ &kCFTypeDictionaryValueCallBacks);
+ if (!attachments) {
+ err = AVERROR(ENOMEM);
+ goto fail;
+ }
+ CFDictionarySetValue(attachments, kCVImageBufferYCbCrMatrixKey, colormatrix);
+ CFDictionarySetValue(attachments, kCVImageBufferColorPrimariesKey, colorpri);
+ CFDictionarySetValue(attachments, kCVImageBufferTransferFunctionKey, colortrc);
+ colorspace = CVImageBufferCreateColorSpaceFromAttachments(attachments);
+ if (colorspace) {
+ CFDictionarySetValue(attachments, kCVImageBufferCGColorSpaceKey, colorspace);
+ CFRelease(colorspace);
+ } else {
+ av_log(avctx, AV_LOG_WARNING, "Unable to set proper colorspace for the CVImageBuffer.\n");
+ }
+ CVBufferSetAttachments(
+ (CVPixelBufferRef)output->data[3],
+ attachments,
+ kCVAttachmentMode_ShouldPropagate);
+ CFRelease(attachments);
+ ff_update_hdr_metadata(output, 100.0f);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ff_update_hdr_metadata(output, 100.0f);

I wasn't sure if this was good enough since dovi meta provides per-scene brightness. Add it for cuda/ocl too?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is reserved for future use because for DOVI to PQ direct passthrough without tonemapping will still cause washed out color as a lot of dovi scene is mastered at 4000nit, if we want to ever implement this, we need to tonemap it to 1000nit HDR10 range, so this value should always be 1000nit here.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And in this case, BT2390 would be the only viable method as all other methods lacks a proper way to handle target peak.

+ }
+
+ av_frame_remove_side_data(output, AV_FRAME_DATA_DOVI_RPU_BUFFER);
Expand Down Expand Up @@ -1805,8 +1848,9 @@ new file mode 100644
+ { "mobius", 0, 0, AV_OPT_TYPE_CONST, { .i64 = TONEMAP_MOBIUS }, 0, 0, FLAGS, "tonemap" },
+ { "bt2390", 0, 0, AV_OPT_TYPE_CONST, { .i64 = TONEMAP_BT2390 }, 0, 0, FLAGS, "tonemap" },
+ { "tonemap_mode", "Tonemap mode selection", OFFSET(tonemap_mode), AV_OPT_TYPE_INT, { .i64 = TONEMAP_MODE_MAX }, TONEMAP_MODE_MAX, TONEMAP_MODE_COUNT - 1, FLAGS, "tonemap_mode" },
+ { "max", 0, 0, AV_OPT_TYPE_CONST, { .i64 = TONEMAP_MODE_MAX }, 0, 0, FLAGS, "tonemap_mode" },
+ { "rgb", 0, 0, AV_OPT_TYPE_CONST, { .i64 = TONEMAP_MODE_RGB }, 0, 0, FLAGS, "tonemap_mode" },
+ { "max", "Brightest channel based tonemap", 0, AV_OPT_TYPE_CONST, { .i64 = TONEMAP_MODE_MAX }, 0, 0, FLAGS, "tonemap_mode" },
+ { "rgb", "Per-channel based tonemap", 0, AV_OPT_TYPE_CONST, { .i64 = TONEMAP_MODE_RGB }, 0, 0, FLAGS, "tonemap_mode" },
+ { "lum", "Relative luminance based tonemap", 0, AV_OPT_TYPE_CONST, { .i64 = TONEMAP_MODE_LUM }, 0, 0, FLAGS, "tonemap_mode" },
+ { "transfer", "Set transfer characteristic", OFFSET(trc), AV_OPT_TYPE_INT, { .i64 = AVCOL_TRC_BT709 }, -1, INT_MAX, FLAGS, "transfer" },
+ { "t", "Set transfer characteristic", OFFSET(trc), AV_OPT_TYPE_INT, { .i64 = AVCOL_TRC_BT709 }, -1, INT_MAX, FLAGS, "transfer" },
+ { "bt709", 0, 0, AV_OPT_TYPE_CONST, { .i64 = AVCOL_TRC_BT709 }, 0, 0, FLAGS, "transfer" },
Expand Down
2 changes: 1 addition & 1 deletion docker-build-win64.sh
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ popd
popd

# LZMA
git clone -b v5.6.1 --depth=1 https://git.tukaani.org/xz.git
git clone -b v5.6.2 --depth=1 https://github.com/tukaani-project/xz.git
pushd xz
./autogen.sh --no-po4a --no-doxygen
./configure \
Expand Down
Loading