Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

mini: implement HEVC reading #1351

Merged
merged 1 commit into from
Nov 4, 2024
Merged

mini: implement HEVC reading #1351

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions libheif/box.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -460,6 +460,10 @@ Error Box::read(BitstreamRange& range, std::shared_ptr<Box>* result, const heif_
box = std::make_shared<Box_ftyp>();
break;

case fourcc("free"):
box = std::make_shared<Box_free>();
break;

case fourcc("meta"):
box = std::make_shared<Box_meta>();
break;
Expand Down Expand Up @@ -1152,6 +1156,29 @@ Error Box_ftyp::write(StreamWriter& writer) const
}


Error Box_free::parse(BitstreamRange& range, const heif_security_limits* limits)
{
range.skip_to_end_of_box();
return range.get_error();
}


std::string Box_free::dump(Indent& indent) const
{
std::ostringstream sstr;
sstr << BoxHeader::dump(indent);
return sstr.str();
}


Error Box_free::write(StreamWriter& writer) const
{
size_t box_start = reserve_box_header_space(writer);
prepend_header(writer, box_start);
return Error::Ok;
}


Error Box_meta::parse(BitstreamRange& range, const heif_security_limits* limits)
{
parse_full_box_header(range);
Expand Down
17 changes: 17 additions & 0 deletions libheif/box.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -412,6 +412,23 @@ class Box_ftyp : public Box
};


class Box_free : public Box
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would it be possible to reuse the existing Box_other as the placeholder box instead of introducing a new Box_free class?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That was my first attempt.

It didn't work out because some of the properties are marked essential (e.g. imir and irot, plus the codec might end up getting omitted if we support the JPEG case). We have a check for any Box_other instances that are marked as essential, since its something we should handle and can't. So a Box_free (which is a no-op) seemed better.

The other way to do this would be to use Box_other but to check each association against the properties first, and only make it if we need it.

Or we could do both the free (either as an explicit Box_free box, or as a Box_other with free parameter), and the check, which would exactly align with the CDAM description.

{
public:
Box_free()
{
set_short_type(fourcc("free"));
}

std::string dump(Indent&) const override;

Error write(StreamWriter& writer) const override;

protected:
Error parse(BitstreamRange& range, const heif_security_limits*) override;
};


class Box_meta : public FullBox
{
public:
Expand Down
4 changes: 4 additions & 0 deletions libheif/codecs/hevc_boxes.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@

class Box_hvcC : public Box
{

// allow access to protected parse() method
friend class HeifFile;

public:
Box_hvcC()
{
Expand Down
129 changes: 99 additions & 30 deletions libheif/file.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
#include "image-items/jpeg.h"
#include "image-items/vvc.h"
#include "codecs/avif_boxes.h"
#include "codecs/hevc_boxes.h"
#include "codecs/uncompressed/unc_boxes.h"

#include <cstdint>
Expand Down Expand Up @@ -299,7 +300,8 @@ static uint32_t get_item_type_for_brand(const heif_brand2 brand)
switch(brand) {
case heif_brand2_avif:
return fourcc("av01");
// TODO: more
case heif_brand2_heic:
return fourcc("hvc1");
default:
return 0;
}
Expand Down Expand Up @@ -437,16 +439,35 @@ Error HeifFile::parse_heif_file()

m_ipco_box = std::make_shared<Box_ipco>();

// TODO: we should look this up based on the infe prop, not assume Box_av1C.
std::shared_ptr<Box_av1C> main_item_codec_prop = std::make_shared<Box_av1C>();
std::shared_ptr<StreamReader> istr = std::make_shared<StreamReader_memory>(
m_mini_box->get_main_item_codec_config().data(),
m_mini_box->get_main_item_codec_config().size(),
false
);
BitstreamRange codec_range(istr, m_mini_box->get_main_item_codec_config().size(), nullptr);
main_item_codec_prop->parse(codec_range, heif_get_global_security_limits());
m_ipco_box->append_child_box(main_item_codec_prop); // entry 1
if (m_mini_box->get_main_item_codec_config().size() != 0) {
std::shared_ptr<StreamReader> istr = std::make_shared<StreamReader_memory>(
m_mini_box->get_main_item_codec_config().data(),
m_mini_box->get_main_item_codec_config().size(),
false
);
BitstreamRange codec_range(istr, m_mini_box->get_main_item_codec_config().size(), nullptr);

std::shared_ptr<Box> main_item_codec_prop;
if (infe_type == fourcc("av01")) {
std::shared_ptr<Box_av1C> codec_prop = std::make_shared<Box_av1C>();
codec_prop->parse(codec_range, heif_get_global_security_limits());
main_item_codec_prop = std::move(codec_prop);
} else if (infe_type == fourcc("hvc1")) {
std::shared_ptr<Box_hvcC> codec_prop = std::make_shared<Box_hvcC>();
codec_prop->parse(codec_range, heif_get_global_security_limits());
main_item_codec_prop = std::move(codec_prop);
} else {
// not found
std::stringstream sstr;
sstr << "Minimised file requires infe support for " << fourcc_to_string(infe_type) << " but this is not yet supported.";
return Error(heif_error_Unsupported_filetype,
heif_suberror_Unspecified,
sstr.str());
}
m_ipco_box->append_child_box(main_item_codec_prop); // entry 1
} else {
m_ipco_box->append_child_box(std::make_shared<Box_free>()); // placeholder for entry 1
}

std::shared_ptr<Box_ispe> ispe = std::make_shared<Box_ispe>();
ispe->set_size(m_mini_box->get_width(), m_mini_box->get_height());
Expand All @@ -470,53 +491,101 @@ Error HeifFile::parse_heif_file()
colr->set_color_profile(nclx);
m_ipco_box->append_child_box(colr); // entry 4

std::shared_ptr<Box_colr> colr_icc = std::make_shared<Box_colr>();
std::shared_ptr<color_profile_raw> icc = std::make_shared<color_profile_raw>(fourcc("prof"), m_mini_box->get_icc_data());
colr_icc->set_color_profile(icc);
m_ipco_box->append_child_box(colr_icc); // entry 5
if (m_mini_box->get_icc_flag()) {
std::shared_ptr<Box_colr> colr_icc = std::make_shared<Box_colr>();
std::shared_ptr<color_profile_raw> icc = std::make_shared<color_profile_raw>(fourcc("prof"), m_mini_box->get_icc_data());
colr_icc->set_color_profile(icc);
m_ipco_box->append_child_box(colr_icc); // entry 5
} else {
m_ipco_box->append_child_box(std::make_shared<Box_free>()); // placeholder for entry 5
}

if (m_mini_box->get_alpha_item_codec_config().size() != 0) {
std::shared_ptr<Box_av1C> alpha_item_codec_prop = std::make_shared<Box_av1C>();
std::shared_ptr<StreamReader> istr = std::make_shared<StreamReader_memory>(
m_mini_box->get_alpha_item_codec_config().data(),
m_mini_box->get_alpha_item_codec_config().size(),
false
);
BitstreamRange alpha_codec_range(istr, m_mini_box->get_alpha_item_codec_config().size(), nullptr);
alpha_item_codec_prop->parse(alpha_codec_range, heif_get_global_security_limits());
std::shared_ptr<Box> alpha_item_codec_prop;
if (infe_type == fourcc("av01")) {
std::shared_ptr<Box_av1C> codec_prop = std::make_shared<Box_av1C>();
codec_prop->parse(alpha_codec_range, heif_get_global_security_limits());
alpha_item_codec_prop = std::move(codec_prop);
} else if (infe_type == fourcc("hvc1")) {
std::shared_ptr<Box_hvcC> codec_prop = std::make_shared<Box_hvcC>();
codec_prop->parse(alpha_codec_range, heif_get_global_security_limits());
alpha_item_codec_prop = std::move(codec_prop);
} else {
// not found
std::stringstream sstr;
sstr << "Minimised file requires infe support for " << fourcc_to_string(infe_type) << " but this is not yet supported.";
return Error(heif_error_Unsupported_filetype,
heif_suberror_Unspecified,
sstr.str());
}
m_ipco_box->append_child_box(alpha_item_codec_prop); // entry 6
} else {
m_ipco_box->append_child_box(std::make_shared<Box_free>()); // placeholder for entry 6
}

if (m_mini_box->get_alpha_item_data_size() != 0) {
std::shared_ptr<Box_auxC> aux_type = std::make_shared<Box_auxC>();
aux_type->set_aux_type("urn:mpeg:mpegB:cicp:systems:auxiliary:alpha");
m_ipco_box->append_child_box(aux_type); // entry 7
} else {
m_ipco_box->append_child_box(std::make_shared<Box_free>()); // placeholder for entry 7
}

// 8
// TODO: replace this placeholder with pixi box version 1 once that is supported
m_ipco_box->append_child_box(std::make_shared<Box_free>()); // placeholder for entry 8

if (m_mini_box->get_orientation() == 2) {
std::shared_ptr<Box_irot> irot = std::make_shared<Box_irot>();
irot->set_rotation_ccw(2 * 90);
m_ipco_box->append_child_box(irot); // entry 9
} else if ((m_mini_box->get_orientation() == 4) || (m_mini_box->get_orientation() == 6) || (m_mini_box->get_orientation() == 7)) {
std::shared_ptr<Box_irot> irot = std::make_shared<Box_irot>();
irot->set_rotation_ccw(1 * 90);
m_ipco_box->append_child_box(irot); // entry 9
} else if (m_mini_box->get_orientation() == 5) {
std::shared_ptr<Box_irot> irot = std::make_shared<Box_irot>();
irot->set_rotation_ccw(3 * 90);
m_ipco_box->append_child_box(irot); // entry 9
} else {
m_ipco_box->append_child_box(std::make_shared<Box_free>()); // placeholder for entry 9
}

// 9
if ((m_mini_box->get_orientation() == 1) || (m_mini_box->get_orientation() == 6)) {
std::shared_ptr<Box_imir> imir = std::make_shared<Box_imir>();
imir->set_mirror_direction(heif_transform_mirror_direction_horizontal);
m_ipco_box->append_child_box(imir); // entry 10
} else if ((m_mini_box->get_orientation() == 3) || (m_mini_box->get_orientation() == 4)) {
std::shared_ptr<Box_imir> imir = std::make_shared<Box_imir>();
imir->set_mirror_direction(heif_transform_mirror_direction_vertical);
m_ipco_box->append_child_box(imir); // entry 10
} else {
m_ipco_box->append_child_box(std::make_shared<Box_free>()); // placeholder for entry 10
}

// 10
m_ipma_box = std::make_shared<Box_ipma>();
m_ipma_box->add_property_for_item_ID(1, Box_ipma::PropertyAssociation{true, uint16_t(1)});
m_ipma_box->add_property_for_item_ID(1, Box_ipma::PropertyAssociation{false, uint16_t(2)});
m_ipma_box->add_property_for_item_ID(1, Box_ipma::PropertyAssociation{false, uint16_t(3)});
m_ipma_box->add_property_for_item_ID(1, Box_ipma::PropertyAssociation{true, uint16_t(4)});
if (m_mini_box->get_icc_flag()) {
// m_ipma_box->add_property_for_item_ID(1, Box_ipma::PropertyAssociation{true, uint16_t(5)});
}
if (m_mini_box->get_alpha_item_data_size() > 0) {
m_ipma_box->add_property_for_item_ID(1, Box_ipma::PropertyAssociation{true, uint16_t(5)});
m_ipma_box->add_property_for_item_ID(1, Box_ipma::PropertyAssociation{true, uint16_t(9)});
m_ipma_box->add_property_for_item_ID(1, Box_ipma::PropertyAssociation{true, uint16_t(10)});

if (m_mini_box->get_alpha_item_data_size() != 0) {
m_ipma_box->add_property_for_item_ID(2, Box_ipma::PropertyAssociation{true, uint16_t(6)});
m_ipma_box->add_property_for_item_ID(2, Box_ipma::PropertyAssociation{false, uint16_t(2)});
m_ipma_box->add_property_for_item_ID(2, Box_ipma::PropertyAssociation{true, uint16_t(7)});
// m_ipma_box->add_property_for_item_ID(2, Box_ipma::PropertyAssociation{false, uint16_t(8)});
// m_ipma_box->add_property_for_item_ID(2, Box_ipma::PropertyAssociation{true, uint16_t(9)});
// m_ipma_box->add_property_for_item_ID(2, Box_ipma::PropertyAssociation{true, uint16_t(10)});
m_ipma_box->add_property_for_item_ID(2, Box_ipma::PropertyAssociation{false, uint16_t(8)});
m_ipma_box->add_property_for_item_ID(2, Box_ipma::PropertyAssociation{true, uint16_t(9)});
m_ipma_box->add_property_for_item_ID(2, Box_ipma::PropertyAssociation{true, uint16_t(10)});
}
// TODO: will need more


// TODO: will need more once we support HDR / gainmap representation

m_iloc_box = std::make_shared<Box_iloc>();
Box_iloc::Item main_item;
Expand Down
2 changes: 2 additions & 0 deletions libheif/mini.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ class Box_mini : public Box

uint8_t get_bit_depth() const { return m_bit_depth; }

uint8_t get_orientation() const { return m_orientation; }

std::vector<uint8_t> get_main_item_codec_config() const { return m_main_item_codec_config; }
std::vector<uint8_t> get_alpha_item_codec_config() const { return m_alpha_item_codec_config; }
std::vector<uint8_t> get_icc_data() const { return m_icc_data; }
Expand Down
Binary file added tests/data/lightning_mini.heif
View file
Open in desktop
Binary file not shown.
48 changes: 48 additions & 0 deletions tests/mini_box.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,3 +224,51 @@ TEST_CASE("check mini+exif+xmp version")
"exif_data offset: 770, size: 208\n"
"xmp_data offset: 978, size: 3426\n");
}


TEST_CASE("check heif mini")
{
auto istr = std::unique_ptr<std::istream>(new std::ifstream(tests_data_directory + "/lightning_mini.heif", std::ios::binary));
auto reader = std::make_shared<StreamReader_istream>(std::move(istr));
FileLayout file;
Error err = file.read(reader, heif_get_global_security_limits());
REQUIRE(err.error_code == heif_error_Ok);

std::shared_ptr<Box_mini> mini = file.get_mini_box();
REQUIRE(mini->get_exif_flag() == false);
REQUIRE(mini->get_xmp_flag() == false);
REQUIRE(mini->get_bit_depth() == 8);
REQUIRE(mini->get_colour_primaries() == 1);
REQUIRE(mini->get_transfer_characteristics() == 13);
REQUIRE(mini->get_matrix_coefficients() == 6);
REQUIRE(mini->get_width() == 128);
REQUIRE(mini->get_height() == 128);
REQUIRE(mini->get_main_item_codec_config().size() == 112);
Indent indent;
std::string dumpResult = mini->dump(indent);
REQUIRE(dumpResult == "Box: mini -----\n"
"size: 4710 (header size: 8)\n"
"version: 0\n"
"explicit_codec_types_flag: 0\n"
"float_flag: 0\n"
"full_range_flag: 1\n"
"alpha_flag: 0\n"
"explicit_cicp_flag: 0\n"
"hdr_flag: 0\n"
"icc_flag: 0\n"
"exif_flag: 0\n"
"xmp_flag: 0\n"
"chroma_subsampling: 1\n"
"orientation: 1\n"
"width: 128\n"
"height: 128\n"
"chroma_is_horizontally_centered: 0\n"
"chroma_is_vertically_centered: 0\n"
"bit_depth: 8\n"
"colour_primaries: 1\n"
"transfer_characteristics: 13\n"
"matrix_coefficients: 6\n"
"main_item_code_config size: 112\n"
"main_item_data offset: 144, size: 4582\n");
}

51 changes: 51 additions & 0 deletions tests/mini_decode.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,3 +56,54 @@ TEST_CASE("check image handle size") {
heif_context_free(context);
}

void check_image_size_heif_mini(struct heif_context *&context) {
heif_image_handle *handle = get_primary_image_handle(context);
heif_image *img = get_primary_image_ycbcr(handle, heif_chroma_444);

REQUIRE(heif_image_has_channel(img, heif_channel_Y) == 1);
REQUIRE(heif_image_has_channel(img, heif_channel_Cb) == 1);
REQUIRE(heif_image_has_channel(img, heif_channel_Cr) == 1);
REQUIRE(heif_image_has_channel(img, heif_channel_R) == 0);
REQUIRE(heif_image_has_channel(img, heif_channel_G) == 0);
REQUIRE(heif_image_has_channel(img, heif_channel_B) == 0);
REQUIRE(heif_image_has_channel(img, heif_channel_Alpha) == 0);
REQUIRE(heif_image_has_channel(img, heif_channel_interleaved) == 0);
int width = heif_image_get_primary_width(img);
REQUIRE(width == 128);
int height = heif_image_get_primary_height(img);
REQUIRE(height == 128);
width = heif_image_get_width(img, heif_channel_Y);
REQUIRE(width == 128);
height = heif_image_get_height(img, heif_channel_Y);
REQUIRE(height == 128);
width = heif_image_get_width(img, heif_channel_Cb);
REQUIRE(width == 128);
height = heif_image_get_height(img, heif_channel_Cr);
REQUIRE(height == 128);
width = heif_image_get_width(img, heif_channel_Cr);
REQUIRE(width == 128);
height = heif_image_get_height(img, heif_channel_Cr);
REQUIRE(height == 128);

int pixel_depth = heif_image_get_bits_per_pixel(img, heif_channel_Y);
REQUIRE(pixel_depth == 8);
pixel_depth = heif_image_get_bits_per_pixel(img, heif_channel_Cb);
REQUIRE(pixel_depth == 8);
pixel_depth = heif_image_get_bits_per_pixel(img, heif_channel_Cr);
REQUIRE(pixel_depth == 8);
int pixel_range = heif_image_get_bits_per_pixel_range(img, heif_channel_Y);
REQUIRE(pixel_range == 8);
pixel_range = heif_image_get_bits_per_pixel_range(img, heif_channel_Cb);
REQUIRE(pixel_range == 8);
pixel_range = heif_image_get_bits_per_pixel_range(img, heif_channel_Cr);
REQUIRE(pixel_range == 8);

heif_image_release(img);
heif_image_handle_release(handle);
}

TEST_CASE("check image size HEIF mini") {
auto context = get_context_for_test_file("lightning_mini.heif");
check_image_size_heif_mini(context);
heif_context_free(context);
}
Loading