From 465645390678c3abb2b62e29072f5fe5742890dc Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Fri, 26 Feb 2021 16:50:46 -0800 Subject: [PATCH] Add encoding details to tippecanoe-decode --stats output --- decode.cpp | 112 +++++++++++++++++- mvt.cpp | 126 ++++++++++++++++++++- mvt.hpp | 36 ++++++ tests/muni/decode/multi.mbtiles.stats.json | 8 +- 4 files changed, 273 insertions(+), 9 deletions(-) diff --git a/decode.cpp b/decode.cpp index 6a0b677ca..c93ad5d75 100644 --- a/decode.cpp +++ b/decode.cpp @@ -79,7 +79,117 @@ void do_stats(mvt_tile &tile, size_t size, bool compressed, int z, unsigned x, u state.json_end_hash(); } - state.json_end_hash(); + state.json_end_hash(); // layers + + state.json_write_string("encoding"); + state.json_write_hash(); + + state.json_write_string("wrapper_overhead_bytes"); + state.json_write_unsigned(tile.reader_overhead); + + state.json_write_string("wrapper_overhead_count"); + state.json_write_unsigned(tile.reader_overhead_count); + + state.json_write_string("layer_overhead_bytes"); + state.json_write_unsigned(tile.layer_read_overhead); + + state.json_write_string("layer_overhead_count"); + state.json_write_unsigned(tile.layer_read_overhead_count); + + state.json_write_string("layer_name_bytes"); + state.json_write_unsigned(tile.layer_names); + + state.json_write_string("layer_name_count"); + state.json_write_unsigned(tile.layer_names_count); + + state.json_write_string("attribute_name_bytes"); + state.json_write_unsigned(tile.layer_keys); + + state.json_write_string("attribute_name_count"); + state.json_write_unsigned(tile.layer_keys_count); + + state.json_write_string("attribute_value_overhead_bytes"); + state.json_write_unsigned(tile.value_overhead); + + state.json_write_string("attribute_value_overhead_count"); + state.json_write_unsigned(tile.value_overhead_count); + + state.json_write_string("attribute_string_value_bytes"); + state.json_write_unsigned(tile.string_values); + + state.json_write_string("attribute_string_value_count"); + state.json_write_unsigned(tile.string_values_count); + + state.json_write_string("attribute_float_value_bytes"); + state.json_write_unsigned(tile.float_values); + + state.json_write_string("attribute_float_value_count"); + state.json_write_unsigned(tile.float_values_count); + + state.json_write_string("attribute_double_value_bytes"); + state.json_write_unsigned(tile.double_values); + + state.json_write_string("attribute_double_value_count"); + state.json_write_unsigned(tile.double_values_count); + + state.json_write_string("attribute_int_value_bytes"); + state.json_write_unsigned(tile.int_values); + + state.json_write_string("attribute_int_value_count"); + state.json_write_unsigned(tile.int_values_count); + + state.json_write_string("attribute_bool_value_bytes"); + state.json_write_unsigned(tile.bool_values); + + state.json_write_string("attribute_bool_value_count"); + state.json_write_unsigned(tile.bool_values_count); + + state.json_write_string("layer_extent_bytes"); + state.json_write_unsigned(tile.layer_extents); + + state.json_write_string("layer_extent_count"); + state.json_write_unsigned(tile.layer_extents_count); + + state.json_write_string("layer_version_bytes"); + state.json_write_unsigned(tile.layer_versions); + + state.json_write_string("layer_version_count"); + state.json_write_unsigned(tile.layer_versions_count); + + state.json_write_string("feature_overhead_bytes"); + state.json_write_unsigned(tile.feature_reader_overhead); + + state.json_write_string("feature_overhead_count"); + state.json_write_unsigned(tile.feature_reader_overhead_count); + + state.json_write_string("feature_id_bytes"); + state.json_write_unsigned(tile.ids); + + state.json_write_string("feature_id_count"); + state.json_write_unsigned(tile.ids_count); + + state.json_write_string("feature_attribute_references_bytes"); + state.json_write_unsigned(tile.attribute_refs); + + state.json_write_string("feature_attribute_references_count"); + state.json_write_unsigned(tile.attribute_refs_count); + + state.json_write_string("feature_type_bytes"); + state.json_write_unsigned(tile.feature_types); + + state.json_write_string("feature_type_count"); + state.json_write_unsigned(tile.feature_types_count); + + state.json_write_string("feature_geometry_bytes"); + state.json_write_unsigned(tile.feature_geometries); + + state.json_write_string("feature_geometry_count"); + state.json_write_unsigned(tile.feature_geometries_count); + + state.json_write_string("feature_geometry_nodes"); + state.json_write_unsigned(tile.feature_geometries_nodes); + + state.json_end_hash(); // encoding state.json_end_hash(); state.json_write_newline(); diff --git a/mvt.cpp b/mvt.cpp index 27486f839..e1eef8b73 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -124,23 +124,79 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) { was_compressed = false; } + reader_overhead = 0; + reader_overhead_count = 0; + layer_read_overhead = 0; + layer_read_overhead_count = 0; + layer_names = 0; + layer_names_count = 0; + layer_keys = 0; + layer_keys_count = 0; + value_overhead = 0; + value_overhead_count = 0; + string_values = 0; + string_values_count = 0; + float_values = 0; + float_values_count = 0; + double_values = 0; + double_values_count = 0; + int_values = 0; + int_values_count = 0; + bool_values = 0; + bool_values_count = 0; + layer_extents = 0; + layer_extents_count = 0; + layer_versions = 0; + layer_versions_count = 0; + feature_reader_overhead = 0; + feature_reader_overhead_count = 0; + ids = 0; + ids_count = 0; + attribute_refs = 0; + attribute_refs_count = 0; + feature_types = 0; + feature_types_count = 0; + feature_geometries = 0; + feature_geometries_count = 0; + feature_geometries_nodes = 0; + protozero::pbf_reader reader(src); - while (reader.next()) { + while (1) { + reader_overhead += reader.length(); + if (!reader.next()) { + break; + } + reader_overhead -= reader.length(); + reader_overhead_count++; + switch (reader.tag()) { case 3: /* layer */ { protozero::pbf_reader layer_reader(reader.get_message()); mvt_layer layer; - while (layer_reader.next()) { + while (1) { + layer_read_overhead += layer_reader.length(); + if (!layer_reader.next()) { + break; + } + layer_read_overhead -= layer_reader.length(); + layer_read_overhead_count++; + switch (layer_reader.tag()) { case 1: /* name */ + layer_names += layer_reader.length(); layer.name = layer_reader.get_string(); + layer_names -= layer_reader.length(); + layer_names_count++; break; case 3: /* key */ + layer_keys += layer_reader.length(); layer.keys.push_back(layer_reader.get_string()); + layer_keys -= layer_reader.length(); + layer_keys_count++; break; case 4: /* value */ @@ -151,45 +207,75 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) { value.type = mvt_null; value.numeric_value.null_value = 0; - while (value_reader.next()) { + while (1) { + value_overhead += value_reader.length(); + if (!value_reader.next()) { + break; + } + value_overhead -= value_reader.length(); + value_overhead_count++; + switch (value_reader.tag()) { case 1: /* string */ value.type = mvt_string; + string_values += value_reader.length(); value.string_value = value_reader.get_string(); + string_values -= value_reader.length(); + string_values_count++; break; case 2: /* float */ value.type = mvt_float; + float_values += value_reader.length(); value.numeric_value.float_value = value_reader.get_float(); + float_values -= value_reader.length(); + float_values_count++; break; case 3: /* double */ value.type = mvt_double; + double_values += value_reader.length(); value.numeric_value.double_value = value_reader.get_double(); + double_values -= value_reader.length(); + double_values_count++; break; case 4: /* int */ value.type = mvt_int; + int_values += value_reader.length(); value.numeric_value.int_value = value_reader.get_int64(); + int_values -= value_reader.length(); + int_values_count++; break; case 5: /* uint */ value.type = mvt_uint; + int_values += value_reader.length(); value.numeric_value.uint_value = value_reader.get_uint64(); + int_values -= value_reader.length(); + int_values_count++; break; case 6: /* sint */ value.type = mvt_sint; + int_values += value_reader.length(); value.numeric_value.sint_value = value_reader.get_sint64(); + int_values -= value_reader.length(); + int_values_count++; break; case 7: /* bool */ value.type = mvt_bool; + bool_values += value_reader.length(); value.numeric_value.bool_value = value_reader.get_bool(); + bool_values -= value_reader.length(); + bool_values_count++; break; default: + value_overhead += value_reader.length(); value_reader.skip(); + value_overhead -= value_reader.length(); break; } } @@ -199,11 +285,17 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) { } case 5: /* extent */ + layer_extents += layer_reader.length(); layer.extent = layer_reader.get_uint32(); + layer_extents -= layer_reader.length(); + layer_extents_count++; break; case 15: /* version */ + layer_versions += layer_reader.length(); layer.version = layer_reader.get_uint32(); + layer_versions -= layer_reader.length(); + layer_versions_count++; break; case 2: /* feature */ @@ -212,16 +304,30 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) { mvt_feature feature; std::vector geoms; - while (feature_reader.next()) { + while (1) { + feature_reader_overhead += feature_reader.length(); + if (!feature_reader.next()) { + break; + } + feature_reader_overhead -= feature_reader.length(); + feature_reader_overhead_count++; + switch (feature_reader.tag()) { case 1: /* id */ + ids += feature_reader.length(); feature.id = feature_reader.get_uint64(); + ids -= feature_reader.length(); + ids_count++; feature.has_id = true; break; case 2: /* tag */ { + attribute_refs += feature_reader.length(); auto pi = feature_reader.get_packed_uint32(); + attribute_refs -= feature_reader.length(); + attribute_refs_count++; + for (auto it = pi.first; it != pi.second; ++it) { feature.tags.push_back(*it); } @@ -229,12 +335,19 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) { } case 3: /* feature type */ + feature_types += feature_reader.length(); feature.type = feature_reader.get_enum(); + feature_types -= feature_reader.length(); + feature_types_count++; break; case 4: /* geometry */ { + feature_geometries += feature_reader.length(); auto pi = feature_reader.get_packed_uint32(); + feature_geometries -= feature_reader.length(); + feature_geometries_count++; + for (auto it = pi.first; it != pi.second; ++it) { geoms.push_back(*it); } @@ -242,7 +355,9 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) { } default: + feature_reader_overhead += feature_reader.length(); feature_reader.skip(); + feature_reader_overhead -= feature_reader.length(); break; } } @@ -266,6 +381,7 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) { } } + feature_geometries_nodes += feature.geometry.size(); layer.features.push_back(feature); break; } @@ -288,7 +404,9 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) { } default: + reader_overhead += reader.length(); reader.skip(); + reader_overhead -= reader.length(); break; } } diff --git a/mvt.hpp b/mvt.hpp index 3bfd16e34..f4d6f6d95 100644 --- a/mvt.hpp +++ b/mvt.hpp @@ -111,6 +111,42 @@ struct mvt_tile { std::string encode(); bool decode(std::string &message, bool &was_compressed); + + size_t reader_overhead = 0; + size_t reader_overhead_count = 0; + size_t layer_read_overhead = 0; + size_t layer_read_overhead_count = 0; + size_t layer_names = 0; + size_t layer_names_count = 0; + size_t layer_keys = 0; + size_t layer_keys_count = 0; + size_t value_overhead = 0; + size_t value_overhead_count = 0; + size_t string_values = 0; + size_t string_values_count = 0; + size_t float_values = 0; + size_t float_values_count = 0; + size_t double_values = 0; + size_t double_values_count = 0; + size_t int_values = 0; + size_t int_values_count = 0; + size_t bool_values = 0; + size_t bool_values_count = 0; + size_t layer_extents = 0; + size_t layer_extents_count = 0; + size_t layer_versions = 0; + size_t layer_versions_count = 0; + size_t feature_reader_overhead = 0; + size_t feature_reader_overhead_count = 0; + size_t ids = 0; + size_t ids_count = 0; + size_t attribute_refs = 0; + size_t attribute_refs_count = 0; + size_t feature_types = 0; + size_t feature_types_count = 0; + size_t feature_geometries = 0; + size_t feature_geometries_count = 0; + size_t feature_geometries_nodes = 0; }; bool is_compressed(std::string const &data); diff --git a/tests/muni/decode/multi.mbtiles.stats.json b/tests/muni/decode/multi.mbtiles.stats.json index 37215ba76..d9fe59b5a 100644 --- a/tests/muni/decode/multi.mbtiles.stats.json +++ b/tests/muni/decode/multi.mbtiles.stats.json @@ -1,9 +1,9 @@ [ -{ "zoom": 11, "x": 326, "y": 791, "bytes": 372, "compressed": true, "layers": { "muni": { "points": 14, "lines": 0, "polygons": 0, "extent": 4096 } } } +{ "zoom": 11, "x": 326, "y": 791, "bytes": 372, "compressed": true, "layers": { "muni": { "points": 14, "lines": 0, "polygons": 0, "extent": 4096 } }, "encoding": { "wrapper_overhead_bytes": 1, "wrapper_overhead_count": 1, "layer_overhead_bytes": 30, "layer_overhead_count": 30, "layer_name_bytes": 5, "layer_name_count": 1, "attribute_name_bytes": 5, "attribute_name_count": 1, "attribute_value_overhead_bytes": 12, "attribute_value_overhead_count": 12, "attribute_string_value_bytes": 283, "attribute_string_value_count": 12, "attribute_float_value_bytes": 0, "attribute_float_value_count": 0, "attribute_double_value_bytes": 0, "attribute_double_value_count": 0, "attribute_int_value_bytes": 0, "attribute_int_value_count": 0, "attribute_bool_value_bytes": 0, "attribute_bool_value_count": 0, "layer_extent_bytes": 2, "layer_extent_count": 1, "layer_version_bytes": 1, "layer_version_count": 1, "feature_overhead_bytes": 42, "feature_overhead_count": 42, "feature_id_bytes": 0, "feature_id_count": 0, "feature_attribute_references_bytes": 42, "feature_attribute_references_count": 14, "feature_type_bytes": 14, "feature_type_count": 14, "feature_geometry_bytes": 84, "feature_geometry_count": 14, "feature_geometry_nodes": 14 } } , -{ "zoom": 11, "x": 327, "y": 792, "bytes": 6481, "compressed": true, "layers": { "muni": { "points": 528, "lines": 0, "polygons": 0, "extent": 4096 } } } +{ "zoom": 11, "x": 327, "y": 792, "bytes": 6481, "compressed": true, "layers": { "muni": { "points": 528, "lines": 0, "polygons": 0, "extent": 4096 } }, "encoding": { "wrapper_overhead_bytes": 1, "wrapper_overhead_count": 1, "layer_overhead_bytes": 888, "layer_overhead_count": 888, "layer_name_bytes": 5, "layer_name_count": 1, "attribute_name_bytes": 5, "attribute_name_count": 1, "attribute_value_overhead_bytes": 356, "attribute_value_overhead_count": 356, "attribute_string_value_bytes": 9075, "attribute_string_value_count": 356, "attribute_float_value_bytes": 0, "attribute_float_value_count": 0, "attribute_double_value_bytes": 0, "attribute_double_value_count": 0, "attribute_int_value_bytes": 0, "attribute_int_value_count": 0, "attribute_bool_value_bytes": 0, "attribute_bool_value_count": 0, "layer_extent_bytes": 2, "layer_extent_count": 1, "layer_version_bytes": 1, "layer_version_count": 1, "feature_overhead_bytes": 1584, "feature_overhead_count": 1584, "feature_id_bytes": 0, "feature_id_count": 0, "feature_attribute_references_bytes": 1918, "feature_attribute_references_count": 528, "feature_type_bytes": 528, "feature_type_count": 528, "feature_geometry_bytes": 2982, "feature_geometry_count": 528, "feature_geometry_nodes": 528 } } , -{ "zoom": 11, "x": 327, "y": 791, "bytes": 44376, "compressed": true, "layers": { "muni": { "points": 4285, "lines": 0, "polygons": 0, "extent": 4096 }, "subway": { "points": 19, "lines": 0, "polygons": 0, "extent": 4096 } } } +{ "zoom": 11, "x": 327, "y": 791, "bytes": 44376, "compressed": true, "layers": { "muni": { "points": 4285, "lines": 0, "polygons": 0, "extent": 4096 }, "subway": { "points": 19, "lines": 0, "polygons": 0, "extent": 4096 } }, "encoding": { "wrapper_overhead_bytes": 2, "wrapper_overhead_count": 2, "layer_overhead_bytes": 7156, "layer_overhead_count": 7156, "layer_name_bytes": 12, "layer_name_count": 2, "attribute_name_bytes": 10, "attribute_name_count": 2, "attribute_value_overhead_bytes": 2844, "attribute_value_overhead_count": 2844, "attribute_string_value_bytes": 67705, "attribute_string_value_count": 2844, "attribute_float_value_bytes": 0, "attribute_float_value_count": 0, "attribute_double_value_bytes": 0, "attribute_double_value_count": 0, "attribute_int_value_bytes": 0, "attribute_int_value_count": 0, "attribute_bool_value_bytes": 0, "attribute_bool_value_count": 0, "layer_extent_bytes": 4, "layer_extent_count": 2, "layer_version_bytes": 2, "layer_version_count": 2, "feature_overhead_bytes": 12912, "feature_overhead_count": 12912, "feature_id_bytes": 0, "feature_id_count": 0, "feature_attribute_references_bytes": 17010, "feature_attribute_references_count": 4304, "feature_type_bytes": 4304, "feature_type_count": 4304, "feature_geometry_bytes": 25824, "feature_geometry_count": 4304, "feature_geometry_nodes": 4304 } } , -{ "zoom": 11, "x": 954, "y": 791, "bytes": 75, "compressed": true, "layers": { "muni": { "points": 12, "lines": 0, "polygons": 0, "extent": 4096 } } } +{ "zoom": 11, "x": 954, "y": 791, "bytes": 75, "compressed": true, "layers": { "muni": { "points": 12, "lines": 0, "polygons": 0, "extent": 4096 } }, "encoding": { "wrapper_overhead_bytes": 1, "wrapper_overhead_count": 1, "layer_overhead_bytes": 17, "layer_overhead_count": 17, "layer_name_bytes": 5, "layer_name_count": 1, "attribute_name_bytes": 5, "attribute_name_count": 1, "attribute_value_overhead_bytes": 1, "attribute_value_overhead_count": 1, "attribute_string_value_bytes": 22, "attribute_string_value_count": 1, "attribute_float_value_bytes": 0, "attribute_float_value_count": 0, "attribute_double_value_bytes": 0, "attribute_double_value_count": 0, "attribute_int_value_bytes": 0, "attribute_int_value_count": 0, "attribute_bool_value_bytes": 0, "attribute_bool_value_count": 0, "layer_extent_bytes": 2, "layer_extent_count": 1, "layer_version_bytes": 1, "layer_version_count": 1, "feature_overhead_bytes": 36, "feature_overhead_count": 36, "feature_id_bytes": 0, "feature_id_count": 0, "feature_attribute_references_bytes": 36, "feature_attribute_references_count": 12, "feature_type_bytes": 12, "feature_type_count": 12, "feature_geometry_bytes": 72, "feature_geometry_count": 12, "feature_geometry_nodes": 12 } } ]