Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

avfilter/tonemap_videotoolbox: improve quality #416

Merged
merged 11 commits into from
Jul 24, 2024
Merged

avfilter/tonemap_videotoolbox: improve quality #416

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
d37d738
avfilter/tonemapx: add simd optimized tonemapx
gnattu Jun 25, 2024
51f6e85
avfilter/tonemap-videotoolbox: properly set hdr attachements
gnattu Jun 25, 2024
7d42f31
avfilter/tonemap-videotoolbox: add relative luminance mode
gnattu Jul 16, 2024
a5932eb
chore: drop irrelevant path
gnattu Jul 16, 2024
3edaea9
avfilter/tonemap_videotoolbox: fix MAX mode check
gnattu Jul 18, 2024
38e7850
avfilter/tonemap_videotoolbox: use rec2020 color primaries for relaiv…
gnattu Jul 23, 2024
a0053a3
avfilter/tonemap_videotoolbox: add help text for tonemap modes
gnattu Jul 23, 2024
b46e674
ci: bump xz to 5.6.2
gnattu Jul 23, 2024
8ee6565
avfilter/tonemap_videotoolbox: fix spaces
gnattu Jul 23, 2024
93df96e
ci: move xz back to github
gnattu Jul 23, 2024
f2c36b8
ci: set .git
gnattu Jul 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions builder/scripts.d/25-xz.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/bin/bash

SCRIPT_REPO="https://git.tukaani.org/xz.git"
SCRIPT_COMMIT="v5.6.1"
SCRIPT_REPO="https://github.com/tukaani-project/xz"
gnattu marked this conversation as resolved.
Show resolved Hide resolved
SCRIPT_COMMIT="v5.6.2"
SCRIPT_TAGFILTER="v?.*.*"

ffbuild_enabled() {
Expand Down
180 changes: 112 additions & 68 deletions debian/patches/0069-add-tonemap-videotoolbox.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,30 +1,22 @@
Subject: [PATCH] avfilter: add vf_tonemap_videotoolbox
---
Index: configure
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
Index: FFmpeg/configure
===================================================================
diff --git a/configure b/configure
--- a/configure (revision 2e3649a10ff797e360eb9fd5d389dfa6611dea59)
+++ b/configure (revision bcb63879b775fef8085a5311f045dc1cc4960a98)
@@ -3748,6 +3748,7 @@
--- FFmpeg.orig/configure
+++ FFmpeg/configure
@@ -3772,6 +3772,7 @@ tinterlace_pad_test_deps="tinterlace_fil
tonemap_filter_deps="const_nan"
tonemap_vaapi_filter_deps="vaapi VAProcFilterParameterBufferHDRToneMapping"
tonemap_opencl_filter_deps="opencl const_nan"
+tonemap_videotoolbox_filter_deps="metal corevideo videotoolbox const_nan"
transpose_opencl_filter_deps="opencl"
transpose_vaapi_filter_deps="vaapi VAProcPipelineCaps_rotation_flags"
transpose_vulkan_filter_deps="vulkan spirv_compiler"
Index: libavfilter/Makefile
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
Index: FFmpeg/libavfilter/Makefile
===================================================================
diff --git a/libavfilter/Makefile b/libavfilter/Makefile
--- a/libavfilter/Makefile (revision 2e3649a10ff797e360eb9fd5d389dfa6611dea59)
+++ b/libavfilter/Makefile (revision bcb63879b775fef8085a5311f045dc1cc4960a98)
@@ -510,6 +510,9 @@
--- FFmpeg.orig/libavfilter/Makefile
+++ FFmpeg/libavfilter/Makefile
@@ -521,6 +521,9 @@ OBJS-$(CONFIG_TONEMAP_OPENCL_FILTER)
OBJS-$(CONFIG_TONEMAP_CUDA_FILTER) += vf_tonemap_cuda.o cuda/tonemap.ptx.o \
cuda/host_util.o
OBJS-$(CONFIG_TONEMAP_VAAPI_FILTER) += vf_tonemap_vaapi.o vaapi_vpp.o
Expand All @@ -34,33 +26,23 @@ diff --git a/libavfilter/Makefile b/libavfilter/Makefile
OBJS-$(CONFIG_TPAD_FILTER) += vf_tpad.o
OBJS-$(CONFIG_TRANSPOSE_FILTER) += vf_transpose.o
OBJS-$(CONFIG_TRANSPOSE_CUDA_FILTER) += vf_transpose_cuda.o vf_transpose_cuda.ptx.o \
cuda/load_helper.o
Index: libavfilter/allfilters.c
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
Index: FFmpeg/libavfilter/allfilters.c
===================================================================
diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
--- a/libavfilter/allfilters.c (revision 2e3649a10ff797e360eb9fd5d389dfa6611dea59)
+++ b/libavfilter/allfilters.c (revision bcb63879b775fef8085a5311f045dc1cc4960a98)
@@ -480,6 +480,7 @@
--- FFmpeg.orig/libavfilter/allfilters.c
+++ FFmpeg/libavfilter/allfilters.c
@@ -487,6 +487,7 @@ extern const AVFilter ff_vf_tonemap;
extern const AVFilter ff_vf_tonemap_cuda;
extern const AVFilter ff_vf_tonemap_opencl;
extern const AVFilter ff_vf_tonemap_vaapi;
+extern const AVFilter ff_vf_tonemap_videotoolbox;
extern const AVFilter ff_vf_tpad;
extern const AVFilter ff_vf_transpose;
extern const AVFilter ff_vf_transpose_cuda;
Index: libavfilter/metal/vf_tonemap_videotoolbox.metal
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
Index: FFmpeg/libavfilter/metal/vf_tonemap_videotoolbox.metal
===================================================================
diff --git a/libavfilter/metal/vf_tonemap_videotoolbox.metal b/libavfilter/metal/vf_tonemap_videotoolbox.metal
new file mode 100644
--- /dev/null (revision bcb63879b775fef8085a5311f045dc1cc4960a98)
+++ b/libavfilter/metal/vf_tonemap_videotoolbox.metal (revision bcb63879b775fef8085a5311f045dc1cc4960a98)
@@ -0,0 +1,688 @@
--- /dev/null
+++ FFmpeg/libavfilter/metal/vf_tonemap_videotoolbox.metal
@@ -0,0 +1,725 @@
+/*
+ * Copyright (c) 2024 Gnattu OC <[email protected]>
+ *
Expand Down Expand Up @@ -142,12 +124,23 @@ new file mode 100644
+constant short linearize_type [[function_constant(36)]];
+constant short delinearize_type [[function_constant(37)]];
+
+enum AVChromaLocation {
+ AVCHROMA_LOC_UNSPECIFIED,
+ AVCHROMA_LOC_LEFT,
+ AVCHROMA_LOC_CENTER,
+ AVCHROMA_LOC_TOPLEFT,
+ AVCHROMA_LOC_TOP,
+ AVCHROMA_LOC_BOTTOMLEFT,
+ AVCHROMA_LOC_BOTTOM,
+ AVCHROMA_LOC_NB
+};
+
+float3 get_chroma_sample(float3 a, float3 b, float3 c,float3 d) {
+ if (chroma_loc == 1) return (((a) + (c)) * 0.5f);
+ if (chroma_loc == 3) return a;
+ if (chroma_loc == 4) return (((a) + (b)) * 0.5f);
+ if (chroma_loc == 5) return c;
+ if (chroma_loc == 6) return (((c) + (d)) * 0.5f);
+ if (chroma_loc == AVCHROMA_LOC_LEFT) return (((a) + (c)) * 0.5f);
+ if (chroma_loc == AVCHROMA_LOC_TOPLEFT) return a;
+ if (chroma_loc == AVCHROMA_LOC_TOP) return (((a) + (b)) * 0.5f);
+ if (chroma_loc == AVCHROMA_LOC_BOTTOMLEFT) return c;
+ if (chroma_loc == AVCHROMA_LOC_BOTTOM) return (((c) + (d)) * 0.5f);
+ return (((a) + (b) + (c) + (d)) * 0.25f);
+}
+
Expand Down Expand Up @@ -287,7 +280,9 @@ new file mode 100644
+
+float rgb2y(float r, float g, float b) {
+ float y = (r*yuv_matrix_1[0]) + (g*yuv_matrix_1[1]) + (b*yuv_matrix_1[2]);
+ y = (219.0f * y + 16.0f) / 255.0f;
+ if (!is_full_range_out) {
+ y = (219.0f * y + 16.0f) / 255.0f;
+ }
+ return y;
+}
+
Expand Down Expand Up @@ -355,6 +350,18 @@ new file mode 100644
+
+//------------
+// Tonemapping methods
+enum TonemapAlgorithm {
+ TONEMAP_NONE,
+ TONEMAP_LINEAR,
+ TONEMAP_GAMMA,
+ TONEMAP_CLIP,
+ TONEMAP_REINHARD,
+ TONEMAP_HABLE,
+ TONEMAP_MOBIUS,
+ TONEMAP_BT2390,
+ TONEMAP_COUNT,
+};
+
+float hable_f(float in) {
+ float a = 0.15f, b = 0.50f, c = 0.10f, d = 0.20f, e = 0.02f, f = 0.30f;
+ return (in * (in * a + b * c) + d * e) / (in * (in * a + b) + d * f) - e / f;
Expand Down Expand Up @@ -419,28 +426,28 @@ new file mode 100644
+}
+
+float tonemap(float s, float peak, float target_peak) {
+ if (tonemap_func_type == 1) {
+ if (tonemap_func_type == TONEMAP_NONE) {
+ return direct(s, peak, target_peak);
+ }
+ if (tonemap_func_type == 2) {
+ if (tonemap_func_type == TONEMAP_LINEAR) {
+ return linear(s, peak, target_peak);
+ }
+ if (tonemap_func_type == 3) {
+ if (tonemap_func_type == TONEMAP_GAMMA) {
+ return gamma(s, peak, target_peak);
+ }
+ if (tonemap_func_type == 4) {
+ if (tonemap_func_type == TONEMAP_CLIP) {
+ return clip(s, peak, target_peak);
+ }
+ if (tonemap_func_type == 5) {
+ if (tonemap_func_type == TONEMAP_REINHARD) {
+ return reinhard(s, peak, target_peak);
+ }
+ if (tonemap_func_type == 6) {
+ if (tonemap_func_type == TONEMAP_HABLE) {
+ return hable(s, peak, target_peak);
+ }
+ if (tonemap_func_type == 7) {
+ if (tonemap_func_type == TONEMAP_MOBIUS) {
+ return mobius(s, peak, target_peak);
+ }
+ if (tonemap_func_type == 8) {
+ if (tonemap_func_type == TONEMAP_BT2390) {
+ return bt2390(s, peak, target_peak);
+ }
+ return direct(s, peak, target_peak);
Expand All @@ -459,9 +466,17 @@ new file mode 100644
+ sig.w = tonemap(sig.w, peak, target_peak); \
+}
+ if (is_tone_mode_rgb) {
+ float4 sig_r = fmax(*r4, FLOAT_EPS), sig_ro = sig_r;
+ float4 sig_g = fmax(*g4, FLOAT_EPS), sig_go = sig_g;
+ float4 sig_b = fmax(*b4, FLOAT_EPS), sig_bo = sig_b;
+ float4 sig_r = fmax(*r4, FLOAT_EPS);
+ float4 sig_g = fmax(*g4, FLOAT_EPS);
+ float4 sig_b = fmax(*b4, FLOAT_EPS);
+ if (is_tone_func_bt2390) {
+ sig_r = fmin(sig_r, peak);
+ sig_g = fmin(sig_g, peak);
+ sig_b = fmin(sig_b, peak);
+ }
+ float4 sig_ro = sig_r;
+ float4 sig_go = sig_g;
+ float4 sig_bo = sig_b;
+ // Desaturate the color using a coefficient dependent on the signal level
+ if (desat_param > 0.0f) {
+ float4 sig = fmax(fmax(*r4, fmax(*g4, *b4)), FLOAT_EPS);
Expand All @@ -475,9 +490,6 @@ new file mode 100644
+ if (is_tone_func_bt2390) {
+ float src_peak_delin_pq = inverse_eotf_st2084(peak);
+ float dst_peak_delin_pq = inverse_eotf_st2084(1.0f);
+ sig_r = fmin(sig_r, dst_peak_delin_pq);
+ sig_g = fmin(sig_g, dst_peak_delin_pq);
+ sig_b = fmin(sig_b, dst_peak_delin_pq);
+ MAP_FOUR_PIXELS(sig_r, src_peak_delin_pq, dst_peak_delin_pq)
+ MAP_FOUR_PIXELS(sig_g, src_peak_delin_pq, dst_peak_delin_pq)
+ MAP_FOUR_PIXELS(sig_b, src_peak_delin_pq, dst_peak_delin_pq)
Expand All @@ -496,7 +508,15 @@ new file mode 100644
+ *g4 *= factor_g;
+ *b4 *= factor_b;
+ } else {
+ float4 sig = fmax(fmax(*r4, fmax(*g4, *b4)), FLOAT_EPS);
+ float4 sig;
+ if (is_tone_mode_max) {
+ sig = fmax(fmax3(*r4, *g4, *b4), FLOAT_EPS);
+ } else {
+ sig = fmax((*r4 * 0.2627f + *g4 * 0.678f + *b4 * 0.0593f), FLOAT_EPS);
+ }
+ if (is_tone_func_bt2390) {
+ sig = fmin(sig, peak);
+ }
+ float4 sig_o = sig;
+ if (desat_param > 0.0f) {
+ float4 luma = get_luma_dst4(*r4, *g4, *b4);
Expand All @@ -509,7 +529,6 @@ new file mode 100644
+ if (is_tone_func_bt2390) {
+ float src_peak_delin_pq = inverse_eotf_st2084(peak);
+ float dst_peak_delin_pq = inverse_eotf_st2084(1.0f);
+ sig = fmin(sig, src_peak_delin_pq);
+ MAP_FOUR_PIXELS(sig, src_peak_delin_pq, dst_peak_delin_pq)
+ } else {
+ MAP_FOUR_PIXELS(sig, peak, 1.0f)
Expand Down Expand Up @@ -749,16 +768,11 @@ new file mode 100644
+ dst2.write(float4(chroma.y, chroma.z, 0.0f, 1.0f), uint2(xi, yi));
+ }
+}
Index: libavfilter/vf_tonemap_videotoolbox.m
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
Index: FFmpeg/libavfilter/vf_tonemap_videotoolbox.m
===================================================================
diff --git a/libavfilter/vf_tonemap_videotoolbox.m b/libavfilter/vf_tonemap_videotoolbox.m
new file mode 100644
--- /dev/null (revision bcb63879b775fef8085a5311f045dc1cc4960a98)
+++ b/libavfilter/vf_tonemap_videotoolbox.m (revision bcb63879b775fef8085a5311f045dc1cc4960a98)
@@ -0,0 +1,1106 @@
--- /dev/null
+++ FFmpeg/libavfilter/vf_tonemap_videotoolbox.m
@@ -0,0 +1,1136 @@
+/*
+ * Copyright (c) 2024 Gnattu OC <[email protected]>
+ *
Expand Down Expand Up @@ -827,6 +841,7 @@ new file mode 100644
+enum TonemapMode {
+ TONEMAP_MODE_MAX,
+ TONEMAP_MODE_RGB,
+ TONEMAP_MODE_LUM,
+ TONEMAP_MODE_COUNT,
+};
+
Expand Down Expand Up @@ -1229,7 +1244,7 @@ new file mode 100644
+ tonemap_func_type = (short)ctx->tonemap;
+ is_tone_func_bt2390 = ctx->tonemap == TONEMAP_BT2390;
+ is_tone_mode_rgb = ctx->tonemap_mode == TONEMAP_MODE_RGB;
+ is_tone_mode_max = !is_tone_mode_rgb;
+ is_tone_mode_max = ctx->tonemap_mode == TONEMAP_MODE_MAX;
+ is_non_semi_planar_in = ctx->in_planes > 2;
+ is_non_semi_planar_out = ctx->out_planes > 2;
+ enable_dither = ctx->in_desc->comp[0].depth > ctx->out_desc->comp[0].depth;
Expand Down Expand Up @@ -1777,6 +1792,34 @@ new file mode 100644
+ if (ctx->trc_out != AVCOL_TRC_SMPTE2084) {
+ av_frame_remove_side_data(output, AV_FRAME_DATA_MASTERING_DISPLAY_METADATA);
+ av_frame_remove_side_data(output, AV_FRAME_DATA_CONTENT_LIGHT_LEVEL);
+ } else {
+ CGColorSpaceRef colorspace = NULL;
+ CFStringRef colormatrix = kCVImageBufferYCbCrMatrix_ITU_R_2020;
+ CFStringRef colorpri = kCVImageBufferColorPrimaries_ITU_R_2020;
+ CFStringRef colortrc = kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ;
+ CFMutableDictionaryRef attachments = CFDictionaryCreateMutable(NULL, 4,
+ &kCFTypeDictionaryKeyCallBacks,
+ &kCFTypeDictionaryValueCallBacks);
+ if (!attachments) {
+ err = AVERROR(ENOMEM);
+ goto fail;
+ }
+ CFDictionarySetValue(attachments, kCVImageBufferYCbCrMatrixKey, colormatrix);
+ CFDictionarySetValue(attachments, kCVImageBufferColorPrimariesKey, colorpri);
+ CFDictionarySetValue(attachments, kCVImageBufferTransferFunctionKey, colortrc);
+ colorspace = CVImageBufferCreateColorSpaceFromAttachments(attachments);
+ if (colorspace) {
+ CFDictionarySetValue(attachments, kCVImageBufferCGColorSpaceKey, colorspace);
+ CFRelease(colorspace);
+ } else {
+ av_log(avctx, AV_LOG_WARNING, "Unable to set proper colorspace for the CVImageBuffer.\n");
+ }
+ CVBufferSetAttachments(
+ (CVPixelBufferRef)output->data[3],
+ attachments,
+ kCVAttachmentMode_ShouldPropagate);
+ CFRelease(attachments);
+ ff_update_hdr_metadata(output, 100.0f);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ff_update_hdr_metadata(output, 100.0f);

I wasn't sure if this was good enough since dovi meta provides per-scene brightness. Add it for cuda/ocl too?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is reserved for future use because for DOVI to PQ direct passthrough without tonemapping will still cause washed out color as a lot of dovi scene is mastered at 4000nit, if we want to ever implement this, we need to tonemap it to 1000nit HDR10 range, so this value should always be 1000nit here.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And in this case, BT2390 would be the only viable method as all other methods lacks a proper way to handle target peak.

+ }
+
+ av_frame_remove_side_data(output, AV_FRAME_DATA_DOVI_RPU_BUFFER);
Expand Down Expand Up @@ -1805,8 +1848,9 @@ new file mode 100644
+ { "mobius", 0, 0, AV_OPT_TYPE_CONST, { .i64 = TONEMAP_MOBIUS }, 0, 0, FLAGS, "tonemap" },
+ { "bt2390", 0, 0, AV_OPT_TYPE_CONST, { .i64 = TONEMAP_BT2390 }, 0, 0, FLAGS, "tonemap" },
+ { "tonemap_mode", "Tonemap mode selection", OFFSET(tonemap_mode), AV_OPT_TYPE_INT, { .i64 = TONEMAP_MODE_MAX }, TONEMAP_MODE_MAX, TONEMAP_MODE_COUNT - 1, FLAGS, "tonemap_mode" },
+ { "max", 0, 0, AV_OPT_TYPE_CONST, { .i64 = TONEMAP_MODE_MAX }, 0, 0, FLAGS, "tonemap_mode" },
+ { "rgb", 0, 0, AV_OPT_TYPE_CONST, { .i64 = TONEMAP_MODE_RGB }, 0, 0, FLAGS, "tonemap_mode" },
+ { "max", "Brightest channel based tonemap", 0, AV_OPT_TYPE_CONST, { .i64 = TONEMAP_MODE_MAX }, 0, 0, FLAGS, "tonemap_mode" },
+ { "rgb", "Per-channel based tonemap", 0, AV_OPT_TYPE_CONST, { .i64 = TONEMAP_MODE_RGB }, 0, 0, FLAGS, "tonemap_mode" },
+ { "lum", "Relative luminance based tonemap", 0, AV_OPT_TYPE_CONST, { .i64 = TONEMAP_MODE_LUM }, 0, 0, FLAGS, "tonemap_mode" },
+ { "transfer", "Set transfer characteristic", OFFSET(trc), AV_OPT_TYPE_INT, { .i64 = AVCOL_TRC_BT709 }, -1, INT_MAX, FLAGS, "transfer" },
+ { "t", "Set transfer characteristic", OFFSET(trc), AV_OPT_TYPE_INT, { .i64 = AVCOL_TRC_BT709 }, -1, INT_MAX, FLAGS, "transfer" },
+ { "bt709", 0, 0, AV_OPT_TYPE_CONST, { .i64 = AVCOL_TRC_BT709 }, 0, 0, FLAGS, "transfer" },
Expand Down
2 changes: 1 addition & 1 deletion docker-build-win64.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ popd
popd

# LZMA
git clone -b v5.6.1 --depth=1 https://git.tukaani.org/xz.git
git clone -b v5.6.2 --depth=1 https://github.com/tukaani-project/xz
gnattu marked this conversation as resolved.
Show resolved Hide resolved
pushd xz
./autogen.sh --no-po4a --no-doxygen
./configure \
Expand Down
Loading