From bbbd67bd88140e66f116b9c1d7b5c3edcca377bf Mon Sep 17 00:00:00 2001 From: Dirk Farin Date: Sun, 13 Oct 2024 19:23:30 +0200 Subject: [PATCH] change 'grid' encoding API to match other grid image types --- examples/heif_enc.cc | 85 ++++++++++--------------------------- libheif/api/libheif/heif.cc | 23 ++++------ libheif/api/libheif/heif.h | 2 +- libheif/box.cc | 63 +++++++++++++++++++++------ libheif/box.h | 4 ++ libheif/context.cc | 65 ++++++++++++++++++++-------- libheif/context.h | 10 +++-- libheif/file.cc | 7 +++ libheif/file.h | 2 + libheif/image-items/grid.cc | 15 +++++-- libheif/image-items/grid.h | 9 ++++ 11 files changed, 170 insertions(+), 115 deletions(-) diff --git a/examples/heif_enc.cc b/examples/heif_enc.cc index 68b220f124..dec8b9d14a 100644 --- a/examples/heif_enc.cc +++ b/examples/heif_enc.cc @@ -63,7 +63,7 @@ const char* encoderId = nullptr; std::string chroma_downsampling; int tiled_image_width = 0; int tiled_image_height = 0; -std::string tiling_method; +std::string tiling_method = "grid"; heif_metadata_compression unci_compression = heif_metadata_compression_brotli; uint16_t nclx_colour_primaries = 1; @@ -747,80 +747,38 @@ std::optional determine_input_images_tiling(const std::st } -heif_image_handle* encode_tiled_grid(heif_context* ctx, heif_encoder* encoder, heif_encoding_options* options, - int output_bit_depth, - const input_tiles_generator& tile_generator, - const heif_image_tiling& tiling) -{ - std::vector image_ids; - - std::cout << "encoding tiled image, tile size: " << tiling.tile_width << "x" << tiling.tile_height - << " image size: " << tiling.image_width << "x" << tiling.image_height << "\n"; - - for (uint32_t ty = 0; ty < tile_generator.nRows(); ty++) - for (uint32_t tx = 0; tx < tile_generator.nColumns(); tx++) { - std::string input_filename = tile_generator.filename(tx,ty); - - InputImage input_image = load_image(input_filename, output_bit_depth); - - std::cout << "encoding tile " << ty+1 << " " << tx+1 - << " (of " << tile_generator.nRows() << "x" << tile_generator.nColumns() << ") \r"; - std::cout.flush(); - - heif_image_handle* handle; - heif_error error = heif_context_encode_image(ctx, - input_image.image.get(), - encoder, - options, - &handle); - if (error.code != 0) { - std::cerr << "Could not encode HEIF/AVIF file: " << error.message << "\n"; - return nullptr; - } - - image_ids.push_back(heif_image_handle_get_item_id(handle)); - heif_image_handle_release(handle); - } - - std::cout << "\n"; - - heif_image_handle* grid_image; - heif_error error = heif_context_add_grid_image(ctx, tiling.image_width, tiling.image_height, - tiling.num_columns, tiling.num_rows, image_ids.data(), &grid_image); - if (error.code != 0) { - std::cerr << "Could not generate grid image: " << error.message << "\n"; - return nullptr; - } - - return grid_image; -} - - heif_image_handle* encode_tiled(heif_context* ctx, heif_encoder* encoder, heif_encoding_options* options, int output_bit_depth, const input_tiles_generator& tile_generator, const heif_image_tiling& tiling) { - if (tiling_method == "grid") { - return encode_tiled_grid(ctx, encoder, options, output_bit_depth, tile_generator, tiling); - } + heif_image_handle* tiled_image; - heif_image_handle* tiled_image; - heif_error error; - if (tiling_method == "tili") { + // --- create the main grid image + + if (tiling_method == "grid") { + heif_error error = heif_context_add_grid_image(ctx, tiling.image_width, tiling.image_height, + tiling.num_columns, tiling.num_rows, + options, + &tiled_image); + if (error.code != 0) { + std::cerr << "Could not generate grid image: " << error.message << "\n"; + return nullptr; + } + } + else if (tiling_method == "tili") { heif_tiled_image_parameters tiled_params{}; tiled_params.version = 1; tiled_params.image_width = tiling.image_width; tiled_params.image_height = tiling.image_height; tiled_params.tile_width = tiling.tile_width; tiled_params.tile_height = tiling.tile_height; - // tiled_params.compression_format_fourcc = heif_fourcc('a', 'v', '0', '1'); // TODO HACK tiled_params.offset_field_length = 32; tiled_params.size_field_length = 24; tiled_params.tiles_are_sequential = 1; - error = heif_context_add_tiled_image(ctx, &tiled_params, options, encoder, &tiled_image); + heif_error error = heif_context_add_tiled_image(ctx, &tiled_params, options, encoder, &tiled_image); if (error.code != 0) { std::cerr << "Could not generate tili image: " << error.message << "\n"; return nullptr; @@ -835,10 +793,10 @@ heif_image_handle* encode_tiled(heif_context* ctx, heif_encoder* encoder, heif_e params.tile_height = tiling.tile_height; params.compression = unci_compression; - std::string input_filename = tile_generator.filename(0,0); + std::string input_filename = tile_generator.filename(0, 0); InputImage prototype_image = load_image(input_filename, output_bit_depth); - error = heif_context_add_unci_image(ctx, ¶ms, options, prototype_image.image.get(), &tiled_image); + heif_error error = heif_context_add_unci_image(ctx, ¶ms, options, prototype_image.image.get(), &tiled_image); if (error.code != 0) { std::cerr << "Could not generate unci image: " << error.message << "\n"; return nullptr; @@ -849,6 +807,9 @@ heif_image_handle* encode_tiled(heif_context* ctx, heif_encoder* encoder, heif_e exit(10); } + + // --- add all the image tiles + std::cout << "encoding tiled image, tile size: " << tiling.tile_width << "x" << tiling.tile_height << " image size: " << tiling.image_width << "x" << tiling.image_height << "\n"; @@ -863,8 +824,8 @@ heif_image_handle* encode_tiled(heif_context* ctx, heif_encoder* encoder, heif_e std::cout.flush(); heif_error error = heif_context_add_image_tile(ctx, tiled_image, tx, ty, - input_image.image.get(), - encoder); + input_image.image.get(), + encoder); if (error.code != 0) { std::cerr << "Could not encode HEIF/AVIF file: " << error.message << "\n"; return nullptr; diff --git a/libheif/api/libheif/heif.cc b/libheif/api/libheif/heif.cc index 09f55db0f0..7e340fa7cc 100644 --- a/libheif/api/libheif/heif.cc +++ b/libheif/api/libheif/heif.cc @@ -3429,14 +3429,10 @@ struct heif_error heif_context_add_grid_image(struct heif_context* ctx, uint32_t image_height, uint32_t tile_columns, uint32_t tile_rows, - const heif_item_id* image_ids, + const struct heif_encoding_options* encoding_options, struct heif_image_handle** out_grid_image_handle) { - if (!image_ids) { - return Error(heif_error_Usage_error, - heif_suberror_Null_pointer_argument).error_struct(ctx->context.get()); - } - else if (tile_rows == 0 || tile_columns == 0) { + if (tile_rows == 0 || tile_columns == 0) { return Error(heif_error_Usage_error, heif_suberror_Invalid_parameter_value).error_struct(ctx->context.get()); } @@ -3446,16 +3442,11 @@ struct heif_error heif_context_add_grid_image(struct heif_context* ctx, "Number of tile rows/columns may not exceed 65535"}; } - - std::vector tiles(tile_rows * tile_columns); - for (uint64_t i = 0; i < tile_rows * tile_columns; i++) { - tiles[i] = image_ids[i]; - } - - std::shared_ptr gridimage; - Error error = ctx->context->add_grid_item(tiles, image_width, image_height, + std::shared_ptr gridimage; + Error error = ctx->context->add_grid_item(image_width, image_height, static_cast(tile_rows), static_cast(tile_columns), + encoding_options, gridimage); if (error != Error::Ok) { @@ -3565,6 +3556,10 @@ struct heif_error heif_context_add_image_tile(struct heif_context* ctx, return err.error_struct(ctx->context.get()); } #endif + else if (tiled_image->image->get_infe_type() == fourcc("grid")) { + Error err = ctx->context->add_grid_image_tile(tiled_image->image->get_id(), tile_x, tile_y, image->image, encoder); + return err.error_struct(ctx->context.get()); + } else { return { heif_error_Usage_error, diff --git a/libheif/api/libheif/heif.h b/libheif/api/libheif/heif.h index 4e5aa3c5cf..c9f4564483 100644 --- a/libheif/api/libheif/heif.h +++ b/libheif/api/libheif/heif.h @@ -2369,7 +2369,7 @@ struct heif_error heif_context_add_grid_image(struct heif_context* ctx, uint32_t image_height, uint32_t tile_columns, uint32_t tile_rows, - const heif_item_id* image_ids, + const struct heif_encoding_options* encoding_options, struct heif_image_handle** out_grid_image_handle); LIBHEIF_API diff --git a/libheif/box.cc b/libheif/box.cc index c78880efa8..1200f8306c 100644 --- a/libheif/box.cc +++ b/libheif/box.cc @@ -2766,6 +2766,15 @@ void Box_ipma::add_property_for_item_ID(heif_item_id itemID, m_entries.push_back(entry); } + // If the property is already associated with the item, skip. + for (auto const& a : m_entries[idx].associations) { + if (a.property_index == assoc.property_index) { + return; + } + + // TODO: should we check that the essential flag matches and return an internal error if not? + } + // add the property association m_entries[idx].associations.push_back(assoc); } @@ -3209,18 +3218,8 @@ Error Box_iref::parse(BitstreamRange& range, const heif_security_limits* limits) // --- check for duplicate references - for (const auto& ref : m_references) { - std::set to_ids; - for (const auto to_id : ref.to_item_ID) { - if (to_ids.find(to_id) == to_ids.end()) { - to_ids.insert(to_id); - } - else { - return Error(heif_error_Invalid_input, - heif_suberror_Unspecified, - "'iref' has double references"); - } - } + if (auto error = check_for_double_references()) { + return error; } @@ -3279,6 +3278,26 @@ Error Box_iref::parse(BitstreamRange& range, const heif_security_limits* limits) } +Error Box_iref::check_for_double_references() const +{ + for (const auto& ref : m_references) { + std::set to_ids; + for (const auto to_id : ref.to_item_ID) { + if (to_ids.find(to_id) == to_ids.end()) { + to_ids.insert(to_id); + } + else { + return {heif_error_Invalid_input, + heif_suberror_Unspecified, + "'iref' has double references"}; + } + } + } + + return Error::Ok; +} + + void Box_iref::derive_box_version() { uint8_t version = 0; @@ -3303,6 +3322,10 @@ void Box_iref::derive_box_version() Error Box_iref::write(StreamWriter& writer) const { + if (auto error = check_for_double_references()) { + return error; + } + size_t box_start = reserve_box_header_space(writer); int id_size = ((get_version() == 0) ? 2 : 4); @@ -3322,7 +3345,6 @@ Error Box_iref::write(StreamWriter& writer) const } } - prepend_header(writer, box_start); return Error::Ok; @@ -3400,6 +3422,21 @@ void Box_iref::add_references(heif_item_id from_id, uint32_t type, const std::ve } +void Box_iref::overwrite_reference(heif_item_id from_id, uint32_t type, uint32_t reference_idx, heif_item_id to_item) +{ + for (auto& ref : m_references) { + if (ref.from_item_ID == from_id && ref.header.get_short_type() == type) { + assert(reference_idx >= 0 && reference_idx < ref.to_item_ID.size()); + + ref.to_item_ID[reference_idx] = to_item; + return; + } + } + + assert(false); // reference was not found +} + + Error Box_idat::parse(BitstreamRange& range, const heif_security_limits* limits) { //parse_full_box_header(range); diff --git a/libheif/box.h b/libheif/box.h index 4759f62e21..3722b9f74c 100644 --- a/libheif/box.h +++ b/libheif/box.h @@ -932,6 +932,8 @@ class Box_iref : public FullBox void add_references(heif_item_id from_id, uint32_t type, const std::vector& to_ids); + void overwrite_reference(heif_item_id from_id, uint32_t type, uint32_t reference_idx, heif_item_id to_item); + protected: Error parse(BitstreamRange& range, const heif_security_limits*) override; @@ -939,6 +941,8 @@ class Box_iref : public FullBox void derive_box_version() override; + Error check_for_double_references() const; + private: std::vector m_references; }; diff --git a/libheif/context.cc b/libheif/context.cc index 7ccc6c0d97..30bbe9f05c 100644 --- a/libheif/context.cc +++ b/libheif/context.cc @@ -1243,26 +1243,19 @@ Error HeifContext::encode_grid(const std::vector } -Error HeifContext::add_grid_item(const std::vector& tile_ids, - uint32_t output_width, - uint32_t output_height, - uint16_t tile_rows, - uint16_t tile_columns, - std::shared_ptr& out_grid_image) +Error HeifContext::add_grid_item(uint32_t output_width, + uint32_t output_height, + uint16_t tile_rows, + uint16_t tile_columns, + const struct heif_encoding_options* encoding_options, + std::shared_ptr& out_grid_image) { - if (tile_ids.size() > 0xFFFF) { + if (tile_rows > 0xFFFF / tile_columns) { return {heif_error_Usage_error, heif_suberror_Unspecified, "Too many tiles (maximum: 65535)"}; } -#if 1 - for (heif_item_id tile_id : tile_ids) { - m_heif_file->get_infe_box(tile_id)->set_hidden_item(true); // only show the full grid - } -#endif - - // Create ImageGrid ImageGrid grid; @@ -1273,20 +1266,25 @@ Error HeifContext::add_grid_item(const std::vector& tile_ids, // Create Grid Item heif_item_id grid_id = m_heif_file->add_new_image(fourcc("grid")); - out_grid_image = std::make_shared(this, grid_id); + out_grid_image = std::make_shared(this, grid_id); + out_grid_image->set_encoding_options(encoding_options); + out_grid_image->set_grid_spec(grid); + m_all_images.insert(std::make_pair(grid_id, out_grid_image)); const int construction_method = 1; // 0=mdat 1=idat m_heif_file->append_iloc_data(grid_id, grid_data, construction_method); + // generate dummy grid item IDs (0) + std::vector tile_ids; + tile_ids.resize(tile_rows * tile_columns); + // Connect tiles to grid m_heif_file->add_iref_reference(grid_id, fourcc("dimg"), tile_ids); // Add ISPE property m_heif_file->add_ispe_property(grid_id, output_width, output_height, false); - // Add PIXI property (copy from first tile) - auto pixi = m_heif_file->get_property(tile_ids[0]); - m_heif_file->add_property(grid_id, pixi, true); + // PIXI property will be added when the first tile is set // Set Brands //m_heif_file->set_brand(encoder->plugin->compression_format, @@ -1427,6 +1425,37 @@ Error HeifContext::add_tiled_image_tile(heif_item_id tild_id, uint32_t tile_x, u } +Error HeifContext::add_grid_image_tile(heif_item_id grid_id, uint32_t tile_x, uint32_t tile_y, + const std::shared_ptr& image, + struct heif_encoder* encoder) +{ + auto grid_item = std::dynamic_pointer_cast(get_image(grid_id)); + auto encoding_options = grid_item->get_encoding_options(); + + std::shared_ptr encoded_image; + Error error = encode_image(image, + encoder, + *encoding_options, + heif_image_input_class_normal, + encoded_image); + if (error != Error::Ok) { + return error; + } + + m_heif_file->get_infe_box(encoded_image->get_id())->set_hidden_item(true); // grid tiles are hidden items + + // Assign tile to grid + heif_image_tiling tiling = grid_item->get_heif_image_tiling(); + m_heif_file->set_iref_reference(grid_id, fourcc("dimg"), tile_y * tiling.num_columns + tile_x, encoded_image->get_id()); + + // Add PIXI property (copy from first tile) + auto pixi = m_heif_file->get_property(encoded_image->get_id()); + m_heif_file->add_property(grid_id, pixi, true); + + return Error::Ok; +} + + Result> HeifContext::add_unci_item(const heif_unci_image_parameters* parameters, const struct heif_encoding_options* encoding_options, const std::shared_ptr& prototype) diff --git a/libheif/context.h b/libheif/context.h index 0572dcfe4a..8694035b5c 100644 --- a/libheif/context.h +++ b/libheif/context.h @@ -151,12 +151,12 @@ class HeifContext : public ErrorBuffer const struct heif_encoding_options& options, std::shared_ptr& out_image); - Error add_grid_item(const std::vector& tile_ids, - uint32_t output_width, + Error add_grid_item(uint32_t output_width, uint32_t output_height, uint16_t tile_rows, uint16_t tile_columns, - std::shared_ptr& out_grid_image); + const struct heif_encoding_options* encoding_options, + std::shared_ptr& out_grid_image); Result> add_iovl_item(const ImageOverlay& overlayspec); @@ -167,6 +167,10 @@ class HeifContext : public ErrorBuffer const std::shared_ptr& image, struct heif_encoder* encoder); + Error add_grid_image_tile(heif_item_id grid_id, uint32_t tile_x, uint32_t tile_y, + const std::shared_ptr& image, + struct heif_encoder* encoder); + Result> add_unci_item(const heif_unci_image_parameters* parameters, const struct heif_encoding_options* encoding_options, const std::shared_ptr& prototype); diff --git a/libheif/file.cc b/libheif/file.cc index 5196b252c3..83c61ffcb0 100644 --- a/libheif/file.cc +++ b/libheif/file.cc @@ -1045,6 +1045,13 @@ void HeifFile::add_iref_reference(heif_item_id from, uint32_t type, } +void HeifFile::set_iref_reference(heif_item_id from, uint32_t type, int reference_idx, heif_item_id to_item) +{ + assert(m_iref_box); + m_iref_box->overwrite_reference(from, type, reference_idx, to_item); +} + + void HeifFile::add_entity_group_box(const std::shared_ptr& entity_group_box) { if (!m_grpl_box) { diff --git a/libheif/file.h b/libheif/file.h index be17c336d5..2a3eb2024a 100644 --- a/libheif/file.h +++ b/libheif/file.h @@ -187,6 +187,8 @@ class HeifFile void add_iref_reference(heif_item_id from, uint32_t type, const std::vector& to); + void set_iref_reference(heif_item_id from, uint32_t type, int reference_idx, heif_item_id to_item); + void add_entity_group_box(const std::shared_ptr& entity_group_box); void set_auxC_property(heif_item_id id, const std::string& type); diff --git a/libheif/image-items/grid.cc b/libheif/image-items/grid.cc index 7b25f09e78..9e31c4efda 100644 --- a/libheif/image-items/grid.cc +++ b/libheif/image-items/grid.cc @@ -457,10 +457,17 @@ heif_image_tiling ImageItem_Grid::get_heif_image_tiling() const tiling.num_columns = gridspec.get_columns(); tiling.num_rows = gridspec.get_rows(); - heif_item_id tile0_id = get_grid_tiles()[0]; - auto tile0 = get_context()->get_image(tile0_id); - tiling.tile_width = tile0->get_width(); - tiling.tile_height = tile0->get_height(); + auto tile_ids = get_grid_tiles(); + if (!tile_ids.empty() && tile_ids[0] != 0) { + heif_item_id tile0_id = tile_ids[0]; + auto tile0 = get_context()->get_image(tile0_id); + tiling.tile_width = tile0->get_width(); + tiling.tile_height = tile0->get_height(); + } + else { + tiling.tile_width = 0; + tiling.tile_height = 0; + } tiling.image_width = gridspec.get_width(); tiling.image_height = gridspec.get_height(); diff --git a/libheif/image-items/grid.h b/libheif/image-items/grid.h index c040f50b07..055dde8e93 100644 --- a/libheif/image-items/grid.h +++ b/libheif/image-items/grid.h @@ -93,6 +93,12 @@ class ImageItem_Grid : public ImageItem int get_chroma_bits_per_pixel() const override; + void set_encoding_options(const heif_encoding_options* options) { + m_encoding_options = *options; + } + + const heif_encoding_options* get_encoding_options() const { return &m_encoding_options; } + Result encode(const std::shared_ptr& image, struct heif_encoder* encoder, const struct heif_encoding_options& options, @@ -113,6 +119,8 @@ class ImageItem_Grid : public ImageItem const ImageGrid& get_grid_spec() const { return m_grid_spec; } + void set_grid_spec(const ImageGrid& grid) { m_grid_spec = grid; } + const std::vector& get_grid_tiles() const { return m_grid_tile_ids; } heif_image_tiling get_heif_image_tiling() const override; @@ -123,6 +131,7 @@ class ImageItem_Grid : public ImageItem ImageGrid m_grid_spec; std::vector m_grid_tile_ids; + heif_encoding_options m_encoding_options; Error read_grid_spec();