From f957f30f90fe8227cc2d16975dc8462ad221e71e Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Tue, 23 Jan 2024 14:17:19 -0800 Subject: [PATCH] Clean up internal naming related to tilestats (#190) * Get rid of the type_and_string near-synonym for serial_val * Rename file_keys to the more familiar tilestats * "tas" (type_and_string) => "sv" (serial_val) * "fk" (tile_keys) => "ts" (tilestats) * Revert ""tas" (type_and_string) => "sv" (serial_val)" This reverts commit 4854c57e2277c4e6be642cdec9bf307769f0f435. * More carefully this time: "tas" (type_and_string) => "sv" (serial_val) --- main.cpp | 4 +-- mbtiles.cpp | 98 +++++++++++++++++++++++++-------------------------- mbtiles.hpp | 16 +++------ plugin.cpp | 50 +++++++++++++------------- serial.cpp | 8 ++--- serial.hpp | 3 ++ tile-join.cpp | 44 +++++++++++------------ tile.cpp | 10 ++---- 8 files changed, 110 insertions(+), 123 deletions(-) diff --git a/main.cpp b/main.cpp index 038495e0a..d4738acd3 100644 --- a/main.cpp +++ b/main.cpp @@ -479,10 +479,10 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const sst.resize(CPUS); pthread_t pthreads[CPUS]; - std::vector > file_subkeys; + std::vector > file_subkeys; for (size_t i = 0; i < CPUS; i++) { - file_subkeys.push_back(std::set()); + file_subkeys.push_back(std::set()); } for (size_t i = 0; i < CPUS; i++) { diff --git a/mbtiles.cpp b/mbtiles.cpp index 5af9168c6..6788d73ad 100644 --- a/mbtiles.cpp +++ b/mbtiles.cpp @@ -196,21 +196,21 @@ void mbtiles_erase_zoom(sqlite3 *outdb, int z) { } } -bool type_and_string::operator<(const type_and_string &o) const { - if (string < o.string) { +bool serial_val::operator<(const serial_val &o) const { + if (s < o.s) { return true; } - if (string == o.string && type < o.type) { + if (s == o.s && type < o.type) { return true; } return false; } -bool type_and_string::operator!=(const type_and_string &o) const { +bool serial_val::operator!=(const serial_val &o) const { if (type != o.type) { return true; } - if (string != o.string) { + if (s != o.s) { return true; } return false; @@ -261,7 +261,7 @@ void tilestats(std::map const &layermap1, size_t el state.nospace = true; state.json_write_string(geomtype); - size_t attrib_count = layer.second.file_keys.size(); + size_t attrib_count = layer.second.tilestats.size(); if (attrib_count > max_tilestats_attributes) { attrib_count = max_tilestats_attributes; } @@ -277,7 +277,7 @@ void tilestats(std::map const &layermap1, size_t el state.json_write_array(); size_t attrs = 0; - for (auto attribute : layer.second.file_keys) { + for (auto attribute : layer.second.tilestats) { if (attrs == elements) { break; } @@ -338,15 +338,15 @@ void tilestats(std::map const &layermap1, size_t el vals++; state.nospace = true; - state.json_write_stringified(value.string); + state.json_write_stringified(value.s); } else { - std::string trunc = truncate16(value.string, 256); + std::string trunc = truncate16(value.s, 256); - if (trunc.size() == value.string.size()) { + if (trunc.size() == value.s.size()) { vals++; state.nospace = true; - state.json_write_string(value.string); + state.json_write_string(value.s); } } } @@ -714,7 +714,7 @@ metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minla } for (size_t i = 0; i < lnames.size(); i++) { - auto fk = layermap.find(lnames[i]); + auto ts = layermap.find(lnames[i]); state.nospace = true; state.json_write_hash(); @@ -726,17 +726,17 @@ metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minla state.nospace = true; state.json_write_string("description"); state.nospace = true; - state.json_write_string(fk->second.description); + state.json_write_string(ts->second.description); state.nospace = true; state.json_write_string("minzoom"); state.nospace = true; - state.json_write_signed(fk->second.minzoom); + state.json_write_signed(ts->second.minzoom); state.nospace = true; state.json_write_string("maxzoom"); state.nospace = true; - state.json_write_signed(fk->second.maxzoom); + state.json_write_signed(ts->second.maxzoom); state.nospace = true; state.json_write_string("fields"); @@ -745,7 +745,7 @@ metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minla bool first = true; size_t attribute_count = 0; - for (auto j = fk->second.file_keys.begin(); j != fk->second.file_keys.end(); ++j) { + for (auto j = ts->second.tilestats.begin(); j != ts->second.tilestats.end(); ++j) { if (first) { first = false; } @@ -852,35 +852,35 @@ std::map merge_layermaps(std::vectorsecond.file_keys.begin(); fk != map->second.file_keys.end(); ++fk) { - std::string attribname = fk->first; + for (auto ts = map->second.tilestats.begin(); ts != map->second.tilestats.end(); ++ts) { + std::string attribname = ts->first; if (trunc) { attribname = truncate16(attribname, 256); } - auto fk2 = out_entry->second.file_keys.find(attribname); + auto ts2 = out_entry->second.tilestats.find(attribname); - if (fk2 == out_entry->second.file_keys.end()) { - out_entry->second.file_keys.insert(std::pair(attribname, fk->second)); + if (ts2 == out_entry->second.tilestats.end()) { + out_entry->second.tilestats.insert(std::pair(attribname, ts->second)); } else { - for (auto val : fk->second.sample_values) { - auto pt = std::lower_bound(fk2->second.sample_values.begin(), fk2->second.sample_values.end(), val); - if (pt == fk2->second.sample_values.end() || *pt != val) { // not found - fk2->second.sample_values.insert(pt, val); + for (auto val : ts->second.sample_values) { + auto pt = std::lower_bound(ts2->second.sample_values.begin(), ts2->second.sample_values.end(), val); + if (pt == ts2->second.sample_values.end() || *pt != val) { // not found + ts2->second.sample_values.insert(pt, val); - if (fk2->second.sample_values.size() > max_tilestats_sample_values) { - fk2->second.sample_values.pop_back(); + if (ts2->second.sample_values.size() > max_tilestats_sample_values) { + ts2->second.sample_values.pop_back(); } } } - fk2->second.type |= fk->second.type; + ts2->second.type |= ts->second.type; - if (fk->second.min < fk2->second.min) { - fk2->second.min = fk->second.min; + if (ts->second.min < ts2->second.min) { + ts2->second.min = ts->second.min; } - if (fk->second.max > fk2->second.max) { - fk2->second.max = fk->second.max; + if (ts->second.max > ts2->second.max) { + ts2->second.max = ts->second.max; } } } @@ -901,41 +901,41 @@ std::map merge_layermaps(std::vector &file_keys, std::string const &attrib, type_and_string const &val) { +void add_to_tilestats(std::map &tilestats, std::string const &attrib, serial_val const &val) { if (val.type == mvt_null) { return; } - auto fka = file_keys.find(attrib); - if (fka == file_keys.end()) { - file_keys.insert(std::pair(attrib, type_and_string_stats())); - fka = file_keys.find(attrib); + auto tsa = tilestats.find(attrib); + if (tsa == tilestats.end()) { + tilestats.insert(std::pair(attrib, tilestat())); + tsa = tilestats.find(attrib); } - if (fka == file_keys.end()) { + if (tsa == tilestats.end()) { fprintf(stderr, "Can't happen (tilestats)\n"); exit(EXIT_IMPOSSIBLE); } if (val.type == mvt_double) { - double d = atof(val.string.c_str()); + double d = atof(val.s.c_str()); - if (d < fka->second.min) { - fka->second.min = d; + if (d < tsa->second.min) { + tsa->second.min = d; } - if (d > fka->second.max) { - fka->second.max = d; + if (d > tsa->second.max) { + tsa->second.max = d; } } - auto pt = std::lower_bound(fka->second.sample_values.begin(), fka->second.sample_values.end(), val); - if (pt == fka->second.sample_values.end() || *pt != val) { // not found - fka->second.sample_values.insert(pt, val); + auto pt = std::lower_bound(tsa->second.sample_values.begin(), tsa->second.sample_values.end(), val); + if (pt == tsa->second.sample_values.end() || *pt != val) { // not found + tsa->second.sample_values.insert(pt, val); - if (fka->second.sample_values.size() > max_tilestats_sample_values) { - fka->second.sample_values.pop_back(); + if (tsa->second.sample_values.size() > max_tilestats_sample_values) { + tsa->second.sample_values.pop_back(); } } - fka->second.type |= (1 << val.type); + tsa->second.type |= (1 << val.type); } diff --git a/mbtiles.hpp b/mbtiles.hpp index 956b0550e..b18376686 100644 --- a/mbtiles.hpp +++ b/mbtiles.hpp @@ -9,16 +9,8 @@ extern size_t max_tilestats_attributes; extern size_t max_tilestats_sample_values; extern size_t max_tilestats_values; -struct type_and_string { - int type = 0; - std::string string = ""; - - bool operator<(const type_and_string &o) const; - bool operator!=(const type_and_string &o) const; -}; - -struct type_and_string_stats { - std::vector sample_values = std::vector(); // sorted +struct tilestat { + std::vector sample_values = std::vector(); // sorted double min = INFINITY; double max = -INFINITY; int type = 0; @@ -26,7 +18,7 @@ struct type_and_string_stats { struct layermap_entry { size_t id = 0; - std::map file_keys{}; + std::map tilestats{}; int minzoom = 0; int maxzoom = 0; std::string description = ""; @@ -84,7 +76,7 @@ void mbtiles_close(sqlite3 *outdb, const char *pgm); std::map merge_layermaps(std::vector > const &maps); std::map merge_layermaps(std::vector > const &maps, bool trunc); -void add_to_file_keys(std::map &file_keys, std::string const &layername, type_and_string const &val); +void add_to_tilestats(std::map &tilestats, std::string const &layername, serial_val const &val); unsigned long long fnv1a(std::string const &s); diff --git a/plugin.cpp b/plugin.cpp index 33407eb94..71eebc55f 100644 --- a/plugin.cpp +++ b/plugin.cpp @@ -241,24 +241,24 @@ std::vector parse_layers(int fd, int z, unsigned x, unsigned y, std:: } } - auto fk = layermap.find(layername); - if (fk == layermap.end()) { + auto ts = layermap.find(layername); + if (ts == layermap.end()) { fprintf(stderr, "Internal error: layer %s not found\n", layername.c_str()); exit(EXIT_IMPOSSIBLE); } - if (z < fk->second.minzoom) { - fk->second.minzoom = z; + if (z < ts->second.minzoom) { + ts->second.minzoom = z; } - if (z > fk->second.maxzoom) { - fk->second.maxzoom = z; + if (z > ts->second.maxzoom) { + ts->second.maxzoom = z; } if (feature.type == mvt_point) { - fk->second.points++; + ts->second.points++; } else if (feature.type == mvt_linestring) { - fk->second.lines++; + ts->second.lines++; } else if (feature.type == mvt_polygon) { - fk->second.polygons++; + ts->second.polygons++; } for (size_t i = 0; i < properties->value.object.length; i++) { @@ -274,11 +274,11 @@ std::vector parse_layers(int fd, int z, unsigned x, unsigned y, std:: mvt_value v = stringified_to_mvt_value(tp, s.c_str()); l->second.tag(feature, std::string(properties->value.object.keys[i]->value.string.string), v); - type_and_string attrib; + serial_val attrib; attrib.type = tp; - attrib.string = s; + attrib.s = s; - add_to_file_keys(fk->second.file_keys, std::string(properties->value.object.keys[i]->value.string.string), attrib); + add_to_tilestats(ts->second.tilestats, std::string(properties->value.object.keys[i]->value.string.string), attrib); } } @@ -478,27 +478,27 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: } } - auto fk = layermap.find(layername); - if (fk == layermap.end()) { + auto ts = layermap.find(layername); + if (ts == layermap.end()) { fprintf(stderr, "Internal error: layer %s not found\n", layername.c_str()); exit(EXIT_IMPOSSIBLE); } - sf.layer = fk->second.id; + sf.layer = ts->second.id; - if (z < fk->second.minzoom) { - fk->second.minzoom = z; + if (z < ts->second.minzoom) { + ts->second.minzoom = z; } - if (z > fk->second.maxzoom) { - fk->second.maxzoom = z; + if (z > ts->second.maxzoom) { + ts->second.maxzoom = z; } if (!postfilter) { if (sf.t == mvt_point) { - fk->second.points++; + ts->second.points++; } else if (sf.t == mvt_linestring) { - fk->second.lines++; + ts->second.lines++; } else if (sf.t == mvt_polygon) { - fk->second.polygons++; + ts->second.polygons++; } } @@ -515,12 +515,12 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: sf.full_keys.push_back(std::string(properties->value.object.keys[i]->value.string.string)); sf.full_values.push_back(v); - type_and_string attrib; - attrib.string = v.s; + serial_val attrib; + attrib.s = v.s; attrib.type = v.type; if (!postfilter) { - add_to_file_keys(fk->second.file_keys, std::string(properties->value.object.keys[i]->value.string.string), attrib); + add_to_tilestats(ts->second.tilestats, std::string(properties->value.object.keys[i]->value.string.string), attrib); } } } diff --git a/serial.cpp b/serial.cpp index 5a9d7ae50..cbb974a80 100644 --- a/serial.cpp +++ b/serial.cpp @@ -827,12 +827,8 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) { if (!sst->filters) { for (size_t i = 0; i < sf.full_keys.size(); i++) { - type_and_string attrib; - attrib.type = sf.full_values[i].type; - attrib.string = sf.full_values[i].s; - - auto fk = sst->layermap->find(sf.layername); - add_to_file_keys(fk->second.file_keys, sf.full_keys[i], attrib); + auto ts = sst->layermap->find(sf.layername); + add_to_tilestats(ts->second.tilestats, sf.full_keys[i], sf.full_values[i]); } } diff --git a/serial.hpp b/serial.hpp index 9467cd07d..05e6d4eaa 100644 --- a/serial.hpp +++ b/serial.hpp @@ -38,6 +38,9 @@ void deserialize_byte(char **f, signed char *n); struct serial_val { int type = 0; std::string s = ""; + + bool operator<(const serial_val &o) const; + bool operator!=(const serial_val &o) const; }; struct serial_feature { diff --git a/tile-join.cpp b/tile-join.cpp index 18652271a..0d0ba2657 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -149,7 +149,7 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map> attributes; + std::map> attributes; std::vector key_order; for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) { @@ -206,11 +206,11 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map>(key, std::pair(val, tas))); + attributes.insert(std::pair>(key, std::pair(val, sv))); key_order.push_back(key); } @@ -253,14 +253,14 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map>(joinkey, std::pair(outval, tas))); + attributes.insert(std::pair>(joinkey, std::pair(outval, sv))); key_order.push_back(joinkey); } } @@ -269,11 +269,11 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map(layer.name, layermap_entry(layermap.size()))); - file_keys = layermap.find(layer.name); - file_keys->second.minzoom = z; - file_keys->second.maxzoom = z; + tilestats = layermap.find(layer.name); + tilestats->second.minzoom = z; + tilestats->second.maxzoom = z; } // To keep attributes in their original order instead of alphabetical @@ -282,7 +282,7 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::mapsecond.first); - add_to_file_keys(file_keys->second.file_keys, k, fa->second.second); + add_to_tilestats(tilestats->second.tilestats, k, fa->second.second); attributes.erase(fa); } } @@ -300,19 +300,19 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::mapsecond.minzoom) { - file_keys->second.minzoom = z; + if (z < tilestats->second.minzoom) { + tilestats->second.minzoom = z; } - if (z > file_keys->second.maxzoom) { - file_keys->second.maxzoom = z; + if (z > tilestats->second.maxzoom) { + tilestats->second.maxzoom = z; } if (feat.type == mvt_point) { - file_keys->second.points++; + tilestats->second.points++; } else if (feat.type == mvt_linestring) { - file_keys->second.lines++; + tilestats->second.lines++; } else if (feat.type == mvt_polygon) { - file_keys->second.polygons++; + tilestats->second.polygons++; } } } diff --git a/tile.cpp b/tile.cpp index bdd9df56e..8eb8bd5cc 100644 --- a/tile.cpp +++ b/tile.cpp @@ -1816,17 +1816,13 @@ void add_tilestats(std::string const &layername, int z, std::vectorsecond.file_keys, key, attrib); + add_to_tilestats(ts->second.tilestats, key, val); } void preserve_attribute(attribute_op op, serial_feature &, char *stringpool, long long *pool_off, std::string &key, serial_val &val, partial &p) {