From c6d9d6cdcf2580edcbb5cb62671576bb2e0a5cf7 Mon Sep 17 00:00:00 2001 From: Dirk Farin Date: Mon, 14 Oct 2024 00:41:34 +0200 Subject: [PATCH] heif-enc: option to add pyramid group --- examples/heif_enc.cc | 27 ++++++++ libheif/api/libheif/heif.cc | 14 +++-- libheif/api/libheif/heif.h | 2 +- libheif/api/libheif/heif_experimental.h | 7 +-- libheif/context.cc | 84 +++++++++++++++++++++---- libheif/context.h | 3 +- libheif/image-items/grid.cc | 7 +++ libheif/image-items/grid.h | 4 +- libheif/image-items/image_item.h | 2 +- libheif/image-items/tiled.cc | 1 + libheif/image-items/unc_image.cc | 10 ++- 11 files changed, 131 insertions(+), 30 deletions(-) diff --git a/examples/heif_enc.cc b/examples/heif_enc.cc index 87d6dd7e05..ad8066e4ed 100644 --- a/examples/heif_enc.cc +++ b/examples/heif_enc.cc @@ -65,6 +65,7 @@ int tiled_image_width = 0; int tiled_image_height = 0; std::string tiling_method = "grid"; heif_metadata_compression unci_compression = heif_metadata_compression_brotli; +int add_pyramid_group = 0; uint16_t nclx_colour_primaries = 1; uint16_t nclx_transfer_characteristic = 13; @@ -147,6 +148,7 @@ static struct option long_options[] = { {(char* const) "tiled-image-height", required_argument, nullptr, OPTION_TILED_IMAGE_HEIGHT}, {(char* const) "tiled-input-x-y", no_argument, &tiled_input_x_y, 1}, {(char* const) "tiling-method", required_argument, nullptr, OPTION_TILING_METHOD}, + {(char* const) "add-pyramid-group", no_argument, &add_pyramid_group, 1}, {0, 0, 0, 0}, }; @@ -212,6 +214,7 @@ void show_help(const char* argv0) << " With this option, this can be swapped so that the first number is x, the second number y.\n" #if WITH_EXPERIMENTAL_FEATURES << " --tiling-method METHOD choose one of these methods: grid, tili, unci. The default is 'grid'.\n" + << " --add-pyramid-group when several images are given, put them into a multi-resolution pyramid group.\n" #endif ; } @@ -748,6 +751,7 @@ std::optional determine_input_images_tiling(const std::st return generator; } +#include heif_image_handle* encode_tiled(heif_context* ctx, heif_encoder* encoder, heif_encoding_options* options, int output_bit_depth, @@ -817,12 +821,21 @@ heif_image_handle* encode_tiled(heif_context* ctx, heif_encoder* encoder, heif_e std::cout << "encoding tiled image, tile size: " << tiling.tile_width << "x" << tiling.tile_height << " image size: " << tiling.image_width << "x" << tiling.image_height << "\n"; + uint32_t tile_width = 0, tile_height = 0; + for (uint32_t ty = 0; ty < tile_generator.nRows(); ty++) for (uint32_t tx = 0; tx < tile_generator.nColumns(); tx++) { std::string input_filename = tile_generator.filename(tx,ty); InputImage input_image = load_image(input_filename, output_bit_depth); + if (tile_width == 0) { + tile_width = heif_image_get_primary_width(input_image.image.get()); + tile_height = heif_image_get_primary_height(input_image.image.get()); + } + + input_image.image->image->extend_to_size_with_zero(tile_width, tile_height); + std::cout << "encoding tile " << ty+1 << " " << tx+1 << " (of " << tile_generator.nRows() << "x" << tile_generator.nColumns() << ") \r"; std::cout.flush(); @@ -1168,6 +1181,8 @@ int main(int argc, char** argv) bool is_primary_image = true; + std::vector encoded_image_ids; + for (; optind < argc; optind++) { std::string input_filename = argv[optind]; @@ -1295,6 +1310,8 @@ int main(int argc, char** argv) heif_context_set_primary_image(context.get(), handle); } + encoded_image_ids.push_back(heif_image_handle_get_item_id(handle)); + // write EXIF to HEIC if (!input_image.exif.empty()) { // Note: we do not modify the EXIF Orientation here because we want it to match the HEIF transforms. @@ -1391,6 +1408,16 @@ int main(int argc, char** argv) heif_image_handle_release(primary_image_handle); } +#if WITH_EXPERIMENTAL_FEATURES + if (add_pyramid_group && encoded_image_ids.size() > 1) { + error = heif_context_add_pyramid_entity_group(context.get(), encoded_image_ids.data(), encoded_image_ids.size(), nullptr); + if (error.code) { + std::cerr << "Cannot set multi-resolution pyramid: " << error.message << "\n"; + return 5; + } + } +#endif + error = heif_context_write_to_file(context.get(), output_filename.c_str()); if (error.code) { std::cerr << error.message << "\n"; diff --git a/libheif/api/libheif/heif.cc b/libheif/api/libheif/heif.cc index 4fa31a7eda..661a7d4c96 100644 --- a/libheif/api/libheif/heif.cc +++ b/libheif/api/libheif/heif.cc @@ -1054,13 +1054,17 @@ void heif_entity_groups_release(struct heif_entity_group* grp, int num_groups) struct heif_error heif_context_add_pyramid_entity_group(struct heif_context* ctx, + const heif_item_id* layer_item_ids, + size_t num_layers, + /* uint16_t tile_width, uint16_t tile_height, uint32_t num_layers, const heif_pyramid_layer_info* in_layers, + */ heif_item_id* out_group_id) { - if (!in_layers) { + if (!layer_item_ids) { return error_null_parameter; } @@ -1068,12 +1072,12 @@ struct heif_error heif_context_add_pyramid_entity_group(struct heif_context* ctx return {heif_error_Usage_error, heif_suberror_Invalid_parameter_value, "Number of layers cannot be 0."}; } - std::vector layers(num_layers); - for (uint32_t i=0;i layers(num_layers); + for (size_t i = 0; i < num_layers; i++) { + layers[i] = layer_item_ids[i]; } - Result result = ctx->context->add_pyramid_group(tile_width, tile_height, layers); + Result result = ctx->context->add_pyramid_group(layers); if (result) { if (out_group_id) { diff --git a/libheif/api/libheif/heif.h b/libheif/api/libheif/heif.h index c9f4564483..d39c8bfb7e 100644 --- a/libheif/api/libheif/heif.h +++ b/libheif/api/libheif/heif.h @@ -2319,7 +2319,7 @@ struct heif_encoding_options // version 7 options - // Set this to true to use compressed form of uncC where possible + // Set this to true to use compressed form of uncC where possible. uint8_t prefer_uncC_short_form; }; diff --git a/libheif/api/libheif/heif_experimental.h b/libheif/api/libheif/heif_experimental.h index d8b7c70de5..2999942b7b 100644 --- a/libheif/api/libheif/heif_experimental.h +++ b/libheif/api/libheif/heif_experimental.h @@ -183,12 +183,11 @@ struct heif_pyramid_layer_info { }; #if WITH_EXPERIMENTAL_FEATURES +// The input images are automatically sorted according to resolution. You can provide them in any order. LIBHEIF_API struct heif_error heif_context_add_pyramid_entity_group(struct heif_context* ctx, - uint16_t tile_width, - uint16_t tile_height, - uint32_t num_layers, - const struct heif_pyramid_layer_info* layers, + const heif_item_id* layer_item_ids, + size_t num_layers, heif_item_id* out_group_id); LIBHEIF_API diff --git a/libheif/context.cc b/libheif/context.cc index 8e8a0f8675..d19c456383 100644 --- a/libheif/context.cc +++ b/libheif/context.cc @@ -1326,6 +1326,7 @@ Error HeifContext::add_grid_item(uint32_t output_width, out_grid_image = std::make_shared(this, grid_id); out_grid_image->set_encoding_options(encoding_options); out_grid_image->set_grid_spec(grid); + out_grid_image->set_resolution(output_width, output_height); m_all_images.insert(std::make_pair(grid_id, out_grid_image)); const int construction_method = 1; // 0=mdat 1=idat @@ -1440,6 +1441,11 @@ Error HeifContext::add_tiled_image_tile(heif_item_id tild_id, uint32_t tile_x, u if (image->get_width() != header.get_parameters().tile_width || image->get_height() != header.get_parameters().tile_height) { + + std::cout << "tx:" << tile_x << " ty:" << tile_y << "\n"; + std::cout << image->get_width() << " " << header.get_parameters().tile_width << " | " + << image->get_height() << " " << header.get_parameters().tile_height <<"\n"; + return {heif_error_Usage_error, heif_suberror_Unspecified, "Tile image size does not match the specified tile size."}; @@ -1510,6 +1516,8 @@ Error HeifContext::add_grid_image_tile(heif_item_id grid_id, uint32_t tile_x, ui heif_image_tiling tiling = grid_item->get_heif_image_tiling(); m_heif_file->set_iref_reference(grid_id, fourcc("dimg"), tile_y * tiling.num_columns + tile_x, encoded_image->get_id()); + grid_item->set_grid_tile_id(tile_x, tile_y, encoded_image->get_id()); + // Add PIXI property (copy from first tile) auto pixi = m_heif_file->get_property(encoded_image->get_id()); m_heif_file->add_property(grid_id, pixi, true); @@ -1732,34 +1740,84 @@ heif_property_id HeifContext::add_property(heif_item_id targetItem, std::shared_ } -Result HeifContext::add_pyramid_group(uint16_t tile_size_x, uint16_t tile_size_y, - std::vector in_layers) +Result HeifContext::add_pyramid_group(const std::vector& layer_item_ids) { + struct pymd_entry + { + std::shared_ptr item; + uint32_t width = 0; + }; + + // --- sort all images by size + + std::vector pymd_entries; + for (auto id : layer_item_ids) { + auto image_item = get_image(id, true); + if (auto error = image_item->get_item_error()) { + return error; + } + + pymd_entry entry; + entry.item = image_item; + entry.width = image_item->get_width(); + pymd_entries.emplace_back(entry); + } + + std::sort(pymd_entries.begin(), pymd_entries.end(), [](const pymd_entry& a, const pymd_entry& b) { + return a.width < b.width; + }); + + + // --- generate pymd box + auto pymd = std::make_shared(); std::vector layers; std::vector ids; - for (const auto& l : in_layers) { - if (l.tiles_in_layer_row==0 || l.tiles_in_layer_column==0 || - l.tiles_in_layer_row - 1 > 0xFFFF || l.tiles_in_layer_column - 1 > 0xFFFF) { + auto base_item = pymd_entries.back().item; + + uint32_t tile_w=0, tile_h=0; + base_item->get_tile_size(tile_w, tile_h); + + uint32_t last_width=0, last_height=0; - return {Error(heif_error_Invalid_input, - heif_suberror_Invalid_parameter_value, - "Invalid number of tiles in layer.")}; + for (const auto& entry : pymd_entries) { + auto layer_item = entry.item; + + if (false) { + // according to pymd definition, we should check that all layers have the same tile size + uint32_t item_tile_w = 0, item_tile_h = 0; + base_item->get_tile_size(item_tile_w, item_tile_h); + if (item_tile_w != tile_w || item_tile_h != tile_h) { + // TODO: add warning that tile sizes are not the same + } } + heif_image_tiling tiling = layer_item->get_heif_image_tiling(); + + if (tiling.image_width < last_width || tiling.image_height < last_height) { + return Error{ + heif_error_Invalid_input, + heif_suberror_Invalid_parameter_value, + "Multi-resolution pyramid images have to be provided ordered from smallest to largest." + }; + } + + last_width = tiling.image_width; + last_height = tiling.image_height; + Box_pymd::LayerInfo layer{}; - layer.layer_binning = l.layer_binning; - layer.tiles_in_layer_row_minus1 = static_cast(l.tiles_in_layer_row - 1); - layer.tiles_in_layer_column_minus1 = static_cast(l.tiles_in_layer_column - 1); + layer.layer_binning = (uint16_t)(base_item->get_width() / tiling.image_width); + layer.tiles_in_layer_row_minus1 = static_cast(tiling.num_rows - 1); + layer.tiles_in_layer_column_minus1 = static_cast(tiling.num_columns - 1); layers.push_back(layer); - ids.push_back(l.layer_image_id); + ids.push_back(layer_item->get_id()); } heif_item_id group_id = m_heif_file->get_unused_item_id(); pymd->set_group_id(group_id); - pymd->set_layers(tile_size_x, tile_size_y, layers, ids); + pymd->set_layers((uint16_t)tile_w, (uint16_t)tile_h, layers, ids); m_heif_file->add_entity_group_box(pymd); diff --git a/libheif/context.h b/libheif/context.h index 7edfaddcaa..57c5d7f31a 100644 --- a/libheif/context.h +++ b/libheif/context.h @@ -189,8 +189,7 @@ class HeifContext : public ErrorBuffer heif_property_id add_property(heif_item_id targetItem, std::shared_ptr property, bool essential); - Result add_pyramid_group(uint16_t tile_size_x, uint16_t tile_size_y, - std::vector layers); + Result add_pyramid_group(const std::vector& layers); // --- region items diff --git a/libheif/image-items/grid.cc b/libheif/image-items/grid.cc index 0419b6f9a9..9d99bcf4c0 100644 --- a/libheif/image-items/grid.cc +++ b/libheif/image-items/grid.cc @@ -458,6 +458,13 @@ Result> ImageItem_Grid::decode_grid_tile(const h } +void ImageItem_Grid::set_grid_tile_id(uint32_t tile_x, uint32_t tile_y, heif_item_id id) +{ + uint32_t idx = tile_y * m_grid_spec.get_columns() + tile_x; + m_grid_tile_ids[idx] = id; +} + + heif_image_tiling ImageItem_Grid::get_heif_image_tiling() const { heif_image_tiling tiling{}; diff --git a/libheif/image-items/grid.h b/libheif/image-items/grid.h index 055dde8e93..f6e21e6082 100644 --- a/libheif/image-items/grid.h +++ b/libheif/image-items/grid.h @@ -119,10 +119,12 @@ class ImageItem_Grid : public ImageItem const ImageGrid& get_grid_spec() const { return m_grid_spec; } - void set_grid_spec(const ImageGrid& grid) { m_grid_spec = grid; } + void set_grid_spec(const ImageGrid& grid) { m_grid_spec = grid; m_grid_tile_ids.resize(grid.get_rows() * grid.get_columns()); } const std::vector& get_grid_tiles() const { return m_grid_tile_ids; } + void set_grid_tile_id(uint32_t tile_x, uint32_t tile_y, heif_item_id); + heif_image_tiling get_heif_image_tiling() const override; void get_tile_size(uint32_t& w, uint32_t& h) const override; diff --git a/libheif/image-items/image_item.h b/libheif/image-items/image_item.h index 46043ef86a..39250505db 100644 --- a/libheif/image-items/image_item.h +++ b/libheif/image-items/image_item.h @@ -99,7 +99,7 @@ class ImageItem : public ErrorBuffer Error check_resolution(uint32_t w, uint32_t h) const; - void set_resolution(int w, int h) + void set_resolution(uint32_t w, uint32_t h) { m_width = w; m_height = h; diff --git a/libheif/image-items/tiled.cc b/libheif/image-items/tiled.cc index 3cb140db72..25c41032df 100644 --- a/libheif/image-items/tiled.cc +++ b/libheif/image-items/tiled.cc @@ -517,6 +517,7 @@ ImageItem_Tiled::add_new_tiled_item(HeifContext* ctx, const heif_tiled_image_par heif_item_id tild_id = ctx->get_heif_file()->add_new_image(fourcc("tili")); auto tild_image = std::make_shared(ctx, tild_id); + tild_image->set_resolution(parameters->image_width, parameters->image_height); ctx->insert_new_image(tild_id, tild_image); // Create tilC box diff --git a/libheif/image-items/unc_image.cc b/libheif/image-items/unc_image.cc index 875d95fc33..1bca4fa0b6 100644 --- a/libheif/image-items/unc_image.cc +++ b/libheif/image-items/unc_image.cc @@ -104,8 +104,11 @@ static Result generate_headers(const std::shared_ptrtile_width != parameters->image_width || + parameters->tile_height != parameters->image_height); + std::shared_ptr uncC = std::make_shared(); - if (options && options->prefer_uncC_short_form) { + if (options && options->prefer_uncC_short_form && !uses_tiles) { maybe_make_minimised_uncC(uncC, src_image); } @@ -317,6 +320,7 @@ Result> ImageItem_uncompressed::add_unci heif_item_id unci_id = ctx->get_heif_file()->add_new_image(fourcc("unci")); auto unci_image = std::make_shared(ctx, unci_id); + unci_image->set_resolution(parameters->image_width, parameters->image_height); ctx->insert_new_image(unci_id, unci_image); @@ -400,8 +404,6 @@ Error ImageItem_uncompressed::add_image_tile(uint32_t tile_x, uint32_t tile_y, c uint32_t tile_width = image->get_width(); uint32_t tile_height = image->get_height(); - uint64_t tile_data_size = uncC->compute_tile_data_size_bytes(tile_width, tile_height); - uint32_t tile_idx = tile_y * uncC->get_number_of_tile_columns() + tile_x; Result> codedBitstreamResult = encode_image_tile(image); @@ -418,6 +420,8 @@ Error ImageItem_uncompressed::add_image_tile(uint32_t tile_x, uint32_t tile_y, c // uncompressed + uint64_t tile_data_size = uncC->compute_tile_data_size_bytes(tile_width, tile_height); + get_file()->replace_iloc_data(get_id(), tile_idx * tile_data_size, *codedBitstreamResult, 0); } else {