heif-enc: option to add pyramid group

examples/heif_enc.cc

@@ -65,6 +65,7 @@ int tiled_image_width = 0;
 int tiled_image_height = 0;
 std::string tiling_method = "grid";
 heif_metadata_compression unci_compression = heif_metadata_compression_brotli;
+int add_pyramid_group = 0;
 
 uint16_t nclx_colour_primaries = 1;
 uint16_t nclx_transfer_characteristic = 13;
@@ -147,6 +148,7 @@ static struct option long_options[] = {
     {(char* const) "tiled-image-height",          required_argument, nullptr, OPTION_TILED_IMAGE_HEIGHT},
     {(char* const) "tiled-input-x-y",             no_argument,       &tiled_input_x_y, 1},
     {(char* const) "tiling-method",               required_argument, nullptr, OPTION_TILING_METHOD},
+    {(char* const) "add-pyramid-group",           no_argument,       &add_pyramid_group, 1},
     {0, 0,                                                           0,  0},
 };
 
@@ -212,6 +214,7 @@ void show_help(const char* argv0)
             << "                            With this option, this can be swapped so that the first number is x, the second number y.\n"
 #if WITH_EXPERIMENTAL_FEATURES
             << "  --tiling-method METHOD    choose one of these methods: grid, tili, unci. The default is 'grid'.\n"
+            << "  --add-pyramid-group       when several images are given, put them into a multi-resolution pyramid group.\n"
 #endif
             ;
 }
@@ -748,6 +751,7 @@ std::optional<input_tiles_generator> determine_input_images_tiling(const std::st
   return generator;
 }
 
+#include <libheif/api_structs.h>
 
 heif_image_handle* encode_tiled(heif_context* ctx, heif_encoder* encoder, heif_encoding_options* options,
                                 int output_bit_depth,
@@ -817,12 +821,21 @@ heif_image_handle* encode_tiled(heif_context* ctx, heif_encoder* encoder, heif_e
   std::cout << "encoding tiled image, tile size: " << tiling.tile_width << "x" << tiling.tile_height
             << " image size: " << tiling.image_width << "x" << tiling.image_height << "\n";
 
+  uint32_t tile_width = 0, tile_height = 0;
+
   for (uint32_t ty = 0; ty < tile_generator.nRows(); ty++)
     for (uint32_t tx = 0; tx < tile_generator.nColumns(); tx++) {
       std::string input_filename = tile_generator.filename(tx,ty);
 
       InputImage input_image = load_image(input_filename, output_bit_depth);
 
+      if (tile_width == 0) {
+        tile_width = heif_image_get_primary_width(input_image.image.get());
+        tile_height = heif_image_get_primary_height(input_image.image.get());
+      }
+
+      input_image.image->image->extend_to_size_with_zero(tile_width, tile_height);
+
       std::cout << "encoding tile " << ty+1 << " " << tx+1
                 << " (of " << tile_generator.nRows() << "x" << tile_generator.nColumns() << ")  \r";
       std::cout.flush();
@@ -1168,6 +1181,8 @@ int main(int argc, char** argv)
 
   bool is_primary_image = true;
 
+  std::vector<heif_item_id> encoded_image_ids;
+
   for (; optind < argc; optind++) {
     std::string input_filename = argv[optind];
 
@@ -1295,6 +1310,8 @@ int main(int argc, char** argv)
       heif_context_set_primary_image(context.get(), handle);
     }
 
+    encoded_image_ids.push_back(heif_image_handle_get_item_id(handle));
+
     // write EXIF to HEIC
     if (!input_image.exif.empty()) {
       // Note: we do not modify the EXIF Orientation here because we want it to match the HEIF transforms.
@@ -1391,6 +1408,16 @@ int main(int argc, char** argv)
     heif_image_handle_release(primary_image_handle);
   }
 
+#if WITH_EXPERIMENTAL_FEATURES
+  if (add_pyramid_group && encoded_image_ids.size() > 1) {
+    error = heif_context_add_pyramid_entity_group(context.get(), encoded_image_ids.data(), encoded_image_ids.size(), nullptr);
+    if (error.code) {
+      std::cerr << "Cannot set multi-resolution pyramid: " << error.message << "\n";
+      return 5;
+    }
+  }
+#endif
+
   error = heif_context_write_to_file(context.get(), output_filename.c_str());
   if (error.code) {
     std::cerr << error.message << "\n";

libheif/api/libheif/heif.cc

@@ -1054,26 +1054,30 @@ void heif_entity_groups_release(struct heif_entity_group* grp, int num_groups)
 
 
 struct heif_error heif_context_add_pyramid_entity_group(struct heif_context* ctx,
+                                                        const heif_item_id* layer_item_ids,
+                                                        size_t num_layers,
+                                                        /*
                                                         uint16_t tile_width,
                                                         uint16_t tile_height,
                                                         uint32_t num_layers,
                                                         const heif_pyramid_layer_info* in_layers,
+                                                         */
                                                         heif_item_id* out_group_id)
 {
-  if (!in_layers) {
+  if (!layer_item_ids) {
     return error_null_parameter;
   }
 
   if (num_layers == 0) {
     return {heif_error_Usage_error, heif_suberror_Invalid_parameter_value, "Number of layers cannot be 0."};
   }
 
-  std::vector<heif_pyramid_layer_info> layers(num_layers);
-  for (uint32_t i=0;i<num_layers;i++) {
-    layers[i] = in_layers[i];
+  std::vector<heif_item_id> layers(num_layers);
+  for (size_t i = 0; i < num_layers; i++) {
+    layers[i] = layer_item_ids[i];
   }
 
-  Result<heif_item_id> result = ctx->context->add_pyramid_group(tile_width, tile_height, layers);
+  Result<heif_item_id> result = ctx->context->add_pyramid_group(layers);
 
   if (result) {
     if (out_group_id) {

libheif/api/libheif/heif.h

@@ -2319,7 +2319,7 @@ struct heif_encoding_options
 
   // version 7 options
 
-  // Set this to true to use compressed form of uncC where possible
+  // Set this to true to use compressed form of uncC where possible.
   uint8_t prefer_uncC_short_form;
 };
 

libheif/api/libheif/heif_experimental.h

@@ -183,12 +183,11 @@ struct heif_pyramid_layer_info {
 };
 
 #if WITH_EXPERIMENTAL_FEATURES
+// The input images are automatically sorted according to resolution. You can provide them in any order.
 LIBHEIF_API
 struct heif_error heif_context_add_pyramid_entity_group(struct heif_context* ctx,
-                                                        uint16_t tile_width,
-                                                        uint16_t tile_height,
-                                                        uint32_t num_layers,
-                                                        const struct heif_pyramid_layer_info* layers,
+                                                        const heif_item_id* layer_item_ids,
+                                                        size_t num_layers,
                                                         heif_item_id* out_group_id);
 
 LIBHEIF_API

libheif/context.cc

@@ -1326,6 +1326,7 @@ Error HeifContext::add_grid_item(uint32_t output_width,
   out_grid_image = std::make_shared<ImageItem_Grid>(this, grid_id);
   out_grid_image->set_encoding_options(encoding_options);
   out_grid_image->set_grid_spec(grid);
+  out_grid_image->set_resolution(output_width, output_height);
 
   m_all_images.insert(std::make_pair(grid_id, out_grid_image));
   const int construction_method = 1; // 0=mdat 1=idat
@@ -1440,6 +1441,11 @@ Error HeifContext::add_tiled_image_tile(heif_item_id tild_id, uint32_t tile_x, u
 
   if (image->get_width() != header.get_parameters().tile_width ||
       image->get_height() != header.get_parameters().tile_height) {
+
+    std::cout << "tx:" << tile_x << " ty:" << tile_y << "\n";
+    std::cout << image->get_width() << " " << header.get_parameters().tile_width << " | "
+      << image->get_height() << " " << header.get_parameters().tile_height <<"\n";
+
     return {heif_error_Usage_error,
             heif_suberror_Unspecified,
             "Tile image size does not match the specified tile size."};
@@ -1510,6 +1516,8 @@ Error HeifContext::add_grid_image_tile(heif_item_id grid_id, uint32_t tile_x, ui
   heif_image_tiling tiling = grid_item->get_heif_image_tiling();
   m_heif_file->set_iref_reference(grid_id, fourcc("dimg"), tile_y * tiling.num_columns + tile_x, encoded_image->get_id());
 
+  grid_item->set_grid_tile_id(tile_x, tile_y, encoded_image->get_id());
+
   // Add PIXI property (copy from first tile)
   auto pixi = m_heif_file->get_property<Box_pixi>(encoded_image->get_id());
   m_heif_file->add_property(grid_id, pixi, true);
@@ -1732,34 +1740,84 @@ heif_property_id HeifContext::add_property(heif_item_id targetItem, std::shared_
 }
 
 
-Result<heif_item_id> HeifContext::add_pyramid_group(uint16_t tile_size_x, uint16_t tile_size_y,
-                                                    std::vector<heif_pyramid_layer_info> in_layers)
+Result<heif_item_id> HeifContext::add_pyramid_group(const std::vector<heif_item_id>& layer_item_ids)
 {
+  struct pymd_entry
+  {
+    std::shared_ptr<ImageItem> item;
+    uint32_t width = 0;
+  };
+
+  // --- sort all images by size
+
+  std::vector<pymd_entry> pymd_entries;
+  for (auto id : layer_item_ids) {
+    auto image_item = get_image(id, true);
+    if (auto error = image_item->get_item_error()) {
+      return error;
+    }
+
+    pymd_entry entry;
+    entry.item = image_item;
+    entry.width = image_item->get_width();
+    pymd_entries.emplace_back(entry);
+  }
+
+  std::sort(pymd_entries.begin(), pymd_entries.end(), [](const pymd_entry& a, const pymd_entry& b) {
+    return a.width < b.width;
+  });
+
+
+  // --- generate pymd box
+
   auto pymd = std::make_shared<Box_pymd>();
   std::vector<Box_pymd::LayerInfo> layers;
   std::vector<heif_item_id> ids;
 
-  for (const auto& l : in_layers) {
-    if (l.tiles_in_layer_row==0 || l.tiles_in_layer_column==0 ||
-        l.tiles_in_layer_row - 1 > 0xFFFF || l.tiles_in_layer_column - 1 > 0xFFFF) {
+  auto base_item = pymd_entries.back().item;
+
+  uint32_t tile_w=0, tile_h=0;
+  base_item->get_tile_size(tile_w, tile_h);
+
+  uint32_t last_width=0, last_height=0;
 
-      return {Error(heif_error_Invalid_input,
-                    heif_suberror_Invalid_parameter_value,
-                    "Invalid number of tiles in layer.")};
+  for (const auto& entry : pymd_entries) {
+    auto layer_item = entry.item;
+
+    if (false) {
+      // according to pymd definition, we should check that all layers have the same tile size
+      uint32_t item_tile_w = 0, item_tile_h = 0;
+      base_item->get_tile_size(item_tile_w, item_tile_h);
+      if (item_tile_w != tile_w || item_tile_h != tile_h) {
+        // TODO: add warning that tile sizes are not the same
+      }
     }
 
+    heif_image_tiling tiling = layer_item->get_heif_image_tiling();
+
+    if (tiling.image_width < last_width || tiling.image_height < last_height) {
+      return Error{
+        heif_error_Invalid_input,
+        heif_suberror_Invalid_parameter_value,
+        "Multi-resolution pyramid images have to be provided ordered from smallest to largest."
+      };
+    }
+
+    last_width = tiling.image_width;
+    last_height = tiling.image_height;
+
     Box_pymd::LayerInfo layer{};
-    layer.layer_binning = l.layer_binning;
-    layer.tiles_in_layer_row_minus1 = static_cast<uint16_t>(l.tiles_in_layer_row - 1);
-    layer.tiles_in_layer_column_minus1 = static_cast<uint16_t>(l.tiles_in_layer_column - 1);
+    layer.layer_binning = (uint16_t)(base_item->get_width() / tiling.image_width);
+    layer.tiles_in_layer_row_minus1 = static_cast<uint16_t>(tiling.num_rows - 1);
+    layer.tiles_in_layer_column_minus1 = static_cast<uint16_t>(tiling.num_columns - 1);
     layers.push_back(layer);
-    ids.push_back(l.layer_image_id);
+    ids.push_back(layer_item->get_id());
   }
 
   heif_item_id group_id = m_heif_file->get_unused_item_id();
 
   pymd->set_group_id(group_id);
-  pymd->set_layers(tile_size_x, tile_size_y, layers, ids);
+  pymd->set_layers((uint16_t)tile_w, (uint16_t)tile_h, layers, ids);
 
   m_heif_file->add_entity_group_box(pymd);
 

libheif/context.h

@@ -189,8 +189,7 @@ class HeifContext : public ErrorBuffer
 
   heif_property_id add_property(heif_item_id targetItem, std::shared_ptr<Box> property, bool essential);
 
-  Result<heif_item_id> add_pyramid_group(uint16_t tile_size_x, uint16_t tile_size_y,
-                                         std::vector<heif_pyramid_layer_info> layers);
+  Result<heif_item_id> add_pyramid_group(const std::vector<heif_item_id>& layers);
 
   // --- region items
 

libheif/image-items/grid.cc

@@ -458,6 +458,13 @@ Result<std::shared_ptr<HeifPixelImage>> ImageItem_Grid::decode_grid_tile(const h
 }
 
 
+void ImageItem_Grid::set_grid_tile_id(uint32_t tile_x, uint32_t tile_y, heif_item_id id)
+{
+  uint32_t idx = tile_y * m_grid_spec.get_columns() + tile_x;
+  m_grid_tile_ids[idx] = id;
+}
+
+
 heif_image_tiling ImageItem_Grid::get_heif_image_tiling() const
 {
   heif_image_tiling tiling{};

libheif/image-items/grid.h

@@ -119,10 +119,12 @@ class ImageItem_Grid : public ImageItem
 
   const ImageGrid& get_grid_spec() const { return m_grid_spec; }
 
-  void set_grid_spec(const ImageGrid& grid) { m_grid_spec = grid; }
+  void set_grid_spec(const ImageGrid& grid) { m_grid_spec = grid; m_grid_tile_ids.resize(grid.get_rows() * grid.get_columns()); }
 
   const std::vector<heif_item_id>& get_grid_tiles() const { return m_grid_tile_ids; }
 
+  void set_grid_tile_id(uint32_t tile_x, uint32_t tile_y, heif_item_id);
+
   heif_image_tiling get_heif_image_tiling() const override;
 
   void get_tile_size(uint32_t& w, uint32_t& h) const override;

libheif/image-items/image_item.h

@@ -99,7 +99,7 @@ class ImageItem : public ErrorBuffer
 
   Error check_resolution(uint32_t w, uint32_t h) const;
 
-  void set_resolution(int w, int h)
+  void set_resolution(uint32_t w, uint32_t h)
   {
     m_width = w;
     m_height = h;

libheif/image-items/tiled.cc

@@ -517,6 +517,7 @@ ImageItem_Tiled::add_new_tiled_item(HeifContext* ctx, const heif_tiled_image_par
 
   heif_item_id tild_id = ctx->get_heif_file()->add_new_image(fourcc("tili"));
   auto tild_image = std::make_shared<ImageItem_Tiled>(ctx, tild_id);
+  tild_image->set_resolution(parameters->image_width, parameters->image_height);
   ctx->insert_new_image(tild_id, tild_image);
 
   // Create tilC box

libheif/image-items/unc_image.cc

@@ -104,8 +104,11 @@ static Result<unciHeaders> generate_headers(const std::shared_ptr<const HeifPixe
 {
   unciHeaders headers;
 
+  bool uses_tiles = (parameters->tile_width != parameters->image_width ||
+                     parameters->tile_height != parameters->image_height);
+
   std::shared_ptr<Box_uncC> uncC = std::make_shared<Box_uncC>();
-  if (options && options->prefer_uncC_short_form) {
+  if (options && options->prefer_uncC_short_form && !uses_tiles) {
     maybe_make_minimised_uncC(uncC, src_image);
   }
 
@@ -317,6 +320,7 @@ Result<std::shared_ptr<ImageItem_uncompressed>> ImageItem_uncompressed::add_unci
 
   heif_item_id unci_id = ctx->get_heif_file()->add_new_image(fourcc("unci"));
   auto unci_image = std::make_shared<ImageItem_uncompressed>(ctx, unci_id);
+  unci_image->set_resolution(parameters->image_width, parameters->image_height);
   ctx->insert_new_image(unci_id, unci_image);
 
 
@@ -400,8 +404,6 @@ Error ImageItem_uncompressed::add_image_tile(uint32_t tile_x, uint32_t tile_y, c
   uint32_t tile_width = image->get_width();
   uint32_t tile_height = image->get_height();
 
-  uint64_t tile_data_size = uncC->compute_tile_data_size_bytes(tile_width, tile_height);
-
   uint32_t tile_idx = tile_y * uncC->get_number_of_tile_columns() + tile_x;
 
   Result<std::vector<uint8_t>> codedBitstreamResult = encode_image_tile(image);
@@ -418,6 +420,8 @@ Error ImageItem_uncompressed::add_image_tile(uint32_t tile_x, uint32_t tile_y, c
 
     // uncompressed
 
+    uint64_t tile_data_size = uncC->compute_tile_data_size_bytes(tile_width, tile_height);
+
     get_file()->replace_iloc_data(get_id(), tile_idx * tile_data_size, *codedBitstreamResult, 0);
   }
   else {