Skip to content

Commit

Permalink
Clean up internal naming related to tilestats (#190)
Browse files Browse the repository at this point in the history 
* Get rid of the type_and_string near-synonym for serial_val

* Rename file_keys to the more familiar tilestats

* "tas" (type_and_string) => "sv" (serial_val)

* "fk" (tile_keys) => "ts" (tilestats)

* Revert ""tas" (type_and_string) => "sv" (serial_val)"

This reverts commit 4854c57.

* More carefully this time: "tas" (type_and_string) => "sv" (serial_val)
  • Loading branch information
@e-n-f
e-n-f authored Jan 23, 2024
1 parent 679a0d6 commit f957f30
Show file tree
Hide file tree
Showing 8 changed files with 110 additions and 123 deletions.
4 changes: 2 additions & 2 deletions main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -479,10 +479,10 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const
sst.resize(CPUS);

pthread_t pthreads[CPUS];
std::vector<std::set<type_and_string> > file_subkeys;
std::vector<std::set<serial_val> > file_subkeys;

for (size_t i = 0; i < CPUS; i++) {
file_subkeys.push_back(std::set<type_and_string>());
file_subkeys.push_back(std::set<serial_val>());
}

for (size_t i = 0; i < CPUS; i++) {
Expand Down
98 changes: 49 additions & 49 deletions mbtiles.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,21 +196,21 @@ void mbtiles_erase_zoom(sqlite3 *outdb, int z) {
}
}

bool type_and_string::operator<(const type_and_string &o) const {
if (string < o.string) {
bool serial_val::operator<(const serial_val &o) const {
if (s < o.s) {
return true;
}
if (string == o.string && type < o.type) {
if (s == o.s && type < o.type) {
return true;
}
return false;
}

bool type_and_string::operator!=(const type_and_string &o) const {
bool serial_val::operator!=(const serial_val &o) const {
if (type != o.type) {
return true;
}
if (string != o.string) {
if (s != o.s) {
return true;
}
return false;
Expand Down Expand Up @@ -261,7 +261,7 @@ void tilestats(std::map<std::string, layermap_entry> const &layermap1, size_t el
state.nospace = true;
state.json_write_string(geomtype);

size_t attrib_count = layer.second.file_keys.size();
size_t attrib_count = layer.second.tilestats.size();
if (attrib_count > max_tilestats_attributes) {
attrib_count = max_tilestats_attributes;
}
Expand All @@ -277,7 +277,7 @@ void tilestats(std::map<std::string, layermap_entry> const &layermap1, size_t el
state.json_write_array();

size_t attrs = 0;
for (auto attribute : layer.second.file_keys) {
for (auto attribute : layer.second.tilestats) {
if (attrs == elements) {
break;
}
Expand Down Expand Up @@ -338,15 +338,15 @@ void tilestats(std::map<std::string, layermap_entry> const &layermap1, size_t el
vals++;

state.nospace = true;
state.json_write_stringified(value.string);
state.json_write_stringified(value.s);
} else {
std::string trunc = truncate16(value.string, 256);
std::string trunc = truncate16(value.s, 256);

if (trunc.size() == value.string.size()) {
if (trunc.size() == value.s.size()) {
vals++;

state.nospace = true;
state.json_write_string(value.string);
state.json_write_string(value.s);
}
}
}
Expand Down Expand Up @@ -714,7 +714,7 @@ metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minla
}

for (size_t i = 0; i < lnames.size(); i++) {
auto fk = layermap.find(lnames[i]);
auto ts = layermap.find(lnames[i]);
state.nospace = true;
state.json_write_hash();

Expand All @@ -726,17 +726,17 @@ metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minla
state.nospace = true;
state.json_write_string("description");
state.nospace = true;
state.json_write_string(fk->second.description);
state.json_write_string(ts->second.description);

state.nospace = true;
state.json_write_string("minzoom");
state.nospace = true;
state.json_write_signed(fk->second.minzoom);
state.json_write_signed(ts->second.minzoom);

state.nospace = true;
state.json_write_string("maxzoom");
state.nospace = true;
state.json_write_signed(fk->second.maxzoom);
state.json_write_signed(ts->second.maxzoom);

state.nospace = true;
state.json_write_string("fields");
Expand All @@ -745,7 +745,7 @@ metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minla

bool first = true;
size_t attribute_count = 0;
for (auto j = fk->second.file_keys.begin(); j != fk->second.file_keys.end(); ++j) {
for (auto j = ts->second.tilestats.begin(); j != ts->second.tilestats.end(); ++j) {
if (first) {
first = false;
}
Expand Down Expand Up @@ -852,35 +852,35 @@ std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::
exit(EXIT_IMPOSSIBLE);
}

for (auto fk = map->second.file_keys.begin(); fk != map->second.file_keys.end(); ++fk) {
std::string attribname = fk->first;
for (auto ts = map->second.tilestats.begin(); ts != map->second.tilestats.end(); ++ts) {
std::string attribname = ts->first;
if (trunc) {
attribname = truncate16(attribname, 256);
}

auto fk2 = out_entry->second.file_keys.find(attribname);
auto ts2 = out_entry->second.tilestats.find(attribname);

if (fk2 == out_entry->second.file_keys.end()) {
out_entry->second.file_keys.insert(std::pair<std::string, type_and_string_stats>(attribname, fk->second));
if (ts2 == out_entry->second.tilestats.end()) {
out_entry->second.tilestats.insert(std::pair<std::string, tilestat>(attribname, ts->second));
} else {
for (auto val : fk->second.sample_values) {
auto pt = std::lower_bound(fk2->second.sample_values.begin(), fk2->second.sample_values.end(), val);
if (pt == fk2->second.sample_values.end() || *pt != val) { // not found
fk2->second.sample_values.insert(pt, val);
for (auto val : ts->second.sample_values) {
auto pt = std::lower_bound(ts2->second.sample_values.begin(), ts2->second.sample_values.end(), val);
if (pt == ts2->second.sample_values.end() || *pt != val) { // not found
ts2->second.sample_values.insert(pt, val);

if (fk2->second.sample_values.size() > max_tilestats_sample_values) {
fk2->second.sample_values.pop_back();
if (ts2->second.sample_values.size() > max_tilestats_sample_values) {
ts2->second.sample_values.pop_back();
}
}
}

fk2->second.type |= fk->second.type;
ts2->second.type |= ts->second.type;

if (fk->second.min < fk2->second.min) {
fk2->second.min = fk->second.min;
if (ts->second.min < ts2->second.min) {
ts2->second.min = ts->second.min;
}
if (fk->second.max > fk2->second.max) {
fk2->second.max = fk->second.max;
if (ts->second.max > ts2->second.max) {
ts2->second.max = ts->second.max;
}
}
}
Expand All @@ -901,41 +901,41 @@ std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::
return out;
}

void add_to_file_keys(std::map<std::string, type_and_string_stats> &file_keys, std::string const &attrib, type_and_string const &val) {
void add_to_tilestats(std::map<std::string, tilestat> &tilestats, std::string const &attrib, serial_val const &val) {
if (val.type == mvt_null) {
return;
}

auto fka = file_keys.find(attrib);
if (fka == file_keys.end()) {
file_keys.insert(std::pair<std::string, type_and_string_stats>(attrib, type_and_string_stats()));
fka = file_keys.find(attrib);
auto tsa = tilestats.find(attrib);
if (tsa == tilestats.end()) {
tilestats.insert(std::pair<std::string, tilestat>(attrib, tilestat()));
tsa = tilestats.find(attrib);
}

if (fka == file_keys.end()) {
if (tsa == tilestats.end()) {
fprintf(stderr, "Can't happen (tilestats)\n");
exit(EXIT_IMPOSSIBLE);
}

if (val.type == mvt_double) {
double d = atof(val.string.c_str());
double d = atof(val.s.c_str());

if (d < fka->second.min) {
fka->second.min = d;
if (d < tsa->second.min) {
tsa->second.min = d;
}
if (d > fka->second.max) {
fka->second.max = d;
if (d > tsa->second.max) {
tsa->second.max = d;
}
}

auto pt = std::lower_bound(fka->second.sample_values.begin(), fka->second.sample_values.end(), val);
if (pt == fka->second.sample_values.end() || *pt != val) { // not found
fka->second.sample_values.insert(pt, val);
auto pt = std::lower_bound(tsa->second.sample_values.begin(), tsa->second.sample_values.end(), val);
if (pt == tsa->second.sample_values.end() || *pt != val) { // not found
tsa->second.sample_values.insert(pt, val);

if (fka->second.sample_values.size() > max_tilestats_sample_values) {
fka->second.sample_values.pop_back();
if (tsa->second.sample_values.size() > max_tilestats_sample_values) {
tsa->second.sample_values.pop_back();
}
}

fka->second.type |= (1 << val.type);
tsa->second.type |= (1 << val.type);
}
16 changes: 4 additions & 12 deletions mbtiles.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,24 +9,16 @@ extern size_t max_tilestats_attributes;
extern size_t max_tilestats_sample_values;
extern size_t max_tilestats_values;

struct type_and_string {
int type = 0;
std::string string = "";

bool operator<(const type_and_string &o) const;
bool operator!=(const type_and_string &o) const;
};

struct type_and_string_stats {
std::vector<type_and_string> sample_values = std::vector<type_and_string>(); // sorted
struct tilestat {
std::vector<serial_val> sample_values = std::vector<serial_val>(); // sorted
double min = INFINITY;
double max = -INFINITY;
int type = 0;
};

struct layermap_entry {
size_t id = 0;
std::map<std::string, type_and_string_stats> file_keys{};
std::map<std::string, tilestat> tilestats{};
int minzoom = 0;
int maxzoom = 0;
std::string description = "";
Expand Down Expand Up @@ -84,7 +76,7 @@ void mbtiles_close(sqlite3 *outdb, const char *pgm);
std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::string, layermap_entry> > const &maps);
std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::string, layermap_entry> > const &maps, bool trunc);

void add_to_file_keys(std::map<std::string, type_and_string_stats> &file_keys, std::string const &layername, type_and_string const &val);
void add_to_tilestats(std::map<std::string, tilestat> &tilestats, std::string const &layername, serial_val const &val);

unsigned long long fnv1a(std::string const &s);

Expand Down
50 changes: 25 additions & 25 deletions plugin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,24 +241,24 @@ std::vector<mvt_layer> parse_layers(int fd, int z, unsigned x, unsigned y, std::
}
}

auto fk = layermap.find(layername);
if (fk == layermap.end()) {
auto ts = layermap.find(layername);
if (ts == layermap.end()) {
fprintf(stderr, "Internal error: layer %s not found\n", layername.c_str());
exit(EXIT_IMPOSSIBLE);
}
if (z < fk->second.minzoom) {
fk->second.minzoom = z;
if (z < ts->second.minzoom) {
ts->second.minzoom = z;
}
if (z > fk->second.maxzoom) {
fk->second.maxzoom = z;
if (z > ts->second.maxzoom) {
ts->second.maxzoom = z;
}

if (feature.type == mvt_point) {
fk->second.points++;
ts->second.points++;
} else if (feature.type == mvt_linestring) {
fk->second.lines++;
ts->second.lines++;
} else if (feature.type == mvt_polygon) {
fk->second.polygons++;
ts->second.polygons++;
}

for (size_t i = 0; i < properties->value.object.length; i++) {
Expand All @@ -274,11 +274,11 @@ std::vector<mvt_layer> parse_layers(int fd, int z, unsigned x, unsigned y, std::
mvt_value v = stringified_to_mvt_value(tp, s.c_str());
l->second.tag(feature, std::string(properties->value.object.keys[i]->value.string.string), v);

type_and_string attrib;
serial_val attrib;
attrib.type = tp;
attrib.string = s;
attrib.s = s;

add_to_file_keys(fk->second.file_keys, std::string(properties->value.object.keys[i]->value.string.string), attrib);
add_to_tilestats(ts->second.tilestats, std::string(properties->value.object.keys[i]->value.string.string), attrib);
}
}

Expand Down Expand Up @@ -478,27 +478,27 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::
}
}

auto fk = layermap.find(layername);
if (fk == layermap.end()) {
auto ts = layermap.find(layername);
if (ts == layermap.end()) {
fprintf(stderr, "Internal error: layer %s not found\n", layername.c_str());
exit(EXIT_IMPOSSIBLE);
}
sf.layer = fk->second.id;
sf.layer = ts->second.id;

if (z < fk->second.minzoom) {
fk->second.minzoom = z;
if (z < ts->second.minzoom) {
ts->second.minzoom = z;
}
if (z > fk->second.maxzoom) {
fk->second.maxzoom = z;
if (z > ts->second.maxzoom) {
ts->second.maxzoom = z;
}

if (!postfilter) {
if (sf.t == mvt_point) {
fk->second.points++;
ts->second.points++;
} else if (sf.t == mvt_linestring) {
fk->second.lines++;
ts->second.lines++;
} else if (sf.t == mvt_polygon) {
fk->second.polygons++;
ts->second.polygons++;
}
}

Expand All @@ -515,12 +515,12 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::
sf.full_keys.push_back(std::string(properties->value.object.keys[i]->value.string.string));
sf.full_values.push_back(v);

type_and_string attrib;
attrib.string = v.s;
serial_val attrib;
attrib.s = v.s;
attrib.type = v.type;

if (!postfilter) {
add_to_file_keys(fk->second.file_keys, std::string(properties->value.object.keys[i]->value.string.string), attrib);
add_to_tilestats(ts->second.tilestats, std::string(properties->value.object.keys[i]->value.string.string), attrib);
}
}
}
Expand Down
8 changes: 2 additions & 6 deletions serial.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -827,12 +827,8 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {

if (!sst->filters) {
for (size_t i = 0; i < sf.full_keys.size(); i++) {
type_and_string attrib;
attrib.type = sf.full_values[i].type;
attrib.string = sf.full_values[i].s;

auto fk = sst->layermap->find(sf.layername);
add_to_file_keys(fk->second.file_keys, sf.full_keys[i], attrib);
auto ts = sst->layermap->find(sf.layername);
add_to_tilestats(ts->second.tilestats, sf.full_keys[i], sf.full_values[i]);
}
}

Expand Down
3 changes: 3 additions & 0 deletions serial.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ void deserialize_byte(char **f, signed char *n);
struct serial_val {
int type = 0;
std::string s = "";

bool operator<(const serial_val &o) const;
bool operator!=(const serial_val &o) const;
};

struct serial_feature {
Expand Down
Loading

0 comments on commit f957f30

Please sign in to comment.