Extend tippecanoe-overzoom to accept multiple input tiles

CHANGELOG.md

@@ -1,3 +1,8 @@
+# 2.38.0
+
+* `tippecanoe-overzoom` can now merge multiple tiles when overzooming them
+* Add `--stop-after` option to `tile-join` to limit the total number of tiles copied
+
 # 2.37.1
 
 * Reduce maximum memory used for vertex sorting

Makefile

@@ -278,6 +278,16 @@ overzoom-test: tippecanoe-overzoom
 	./tippecanoe-decode tests/pbf/13-1310-3166.pbf 13 1310 3166 > tests/pbf/13-1310-3166.pbf.json.check
 	cmp tests/pbf/13-1310-3166.pbf.json.check tests/pbf/13-1310-3166.pbf.json
 	rm tests/pbf/13-1310-3166.pbf tests/pbf/13-1310-3166.pbf.json.check
+	# Basic operation, multiple input form
+	./tippecanoe-overzoom -o tests/pbf/13-1310-3166.pbf -t 13/1310/3166 tests/pbf/11-327-791.pbf 11/327/791
+	./tippecanoe-decode tests/pbf/13-1310-3166.pbf 13 1310 3166 > tests/pbf/13-1310-3166.pbf.json.check
+	cmp tests/pbf/13-1310-3166.pbf.json.check tests/pbf/13-1310-3166.pbf.json
+	rm tests/pbf/13-1310-3166.pbf tests/pbf/13-1310-3166.pbf.json.check
+	# Multiple inputs
+	./tippecanoe-overzoom -o tests/pbf/13-1310-3166-ne.pbf -t 13/1310/3166 tests/pbf/11-327-791.pbf 11/327/791 tests/pbf/0-0-0.pbf 0/0/0
+	./tippecanoe-decode tests/pbf/13-1310-3166-ne.pbf 13 1310 3166 > tests/pbf/13-1310-3166-ne.pbf.json.check
+	cmp tests/pbf/13-1310-3166-ne.pbf.json.check tests/pbf/13-1310-3166-ne.pbf.json
+	rm tests/pbf/13-1310-3166-ne.pbf tests/pbf/13-1310-3166-ne.pbf.json.check
 	# Different detail and buffer, and attribute stripping
 	./tippecanoe-overzoom -d8 -b30 -y NAME -y name -y scalerank -o tests/pbf/13-1310-3166-8-30.pbf tests/pbf/11-327-791.pbf 11/327/791 13/1310/3166
 	./tippecanoe-decode tests/pbf/13-1310-3166-8-30.pbf 13 1310 3166 > tests/pbf/13-1310-3166-8-30.pbf.json.check
@@ -421,7 +431,13 @@ join-test: tippecanoe tippecanoe-decode tile-join
 	./tile-join --overzoom -f -o tests/ne_110m_ocean/join/joined.mbtiles tests/ne_110m_ocean/join/ocean.mbtiles tests/ne_110m_ocean/join/countries.mbtiles
 	./tippecanoe-decode -x generator tests/ne_110m_ocean/join/joined.mbtiles > tests/ne_110m_ocean/join/joined.mbtiles.json.check
 	cmp tests/ne_110m_ocean/join/joined.mbtiles.json.check tests/ne_110m_ocean/join/joined.mbtiles.json
-	rm -f tests/ne_110m_ocean/join/ocean.mbtiles tests/ne_110m_ocean/join/countries.mbtiles tests/ne_110m_ocean/join/joined.mbtiles tests/ne_110m_ocean/join/joined.mbtiles.json.check
+	#
+	# Test overzooming with tile count limit
+	#
+	./tile-join --overzoom -f -o tests/ne_110m_ocean/join/joined.mbtiles --stop-after 50 tests/ne_110m_ocean/join/ocean.mbtiles tests/ne_110m_ocean/join/countries.mbtiles
+	./tippecanoe-decode -x generator tests/ne_110m_ocean/join/joined.mbtiles > tests/ne_110m_ocean/join/joined.mbtiles-50.json.check
+	cmp tests/ne_110m_ocean/join/joined.mbtiles-50.json.check tests/ne_110m_ocean/join/joined.mbtiles-50.json
+	rm -f tests/ne_110m_ocean/join/ocean.mbtiles tests/ne_110m_ocean/join/countries.mbtiles tests/ne_110m_ocean/join/joined.mbtiles tests/ne_110m_ocean/join/joined.mbtiles.json.check tests/ne_110m_ocean/join/joined.mbtiles-50.json.check
 
 join-filter-test: tippecanoe tippecanoe-decode tile-join
 	# Comes out different from the direct tippecanoe run because null attributes are lost

README.md

@@ -793,6 +793,7 @@ The options are:
 
  * `-z` _zoom_ or `--maximum-zoom=`_zoom_: Don't copy tiles from higher zoom levels than the specified zoom
  * `-Z` _zoom_ or `--minimum-zoom=`_zoom_: Don't copy tiles from lower zoom levels than the specified zoom
+ * `-a` _count_ or `--stop-after=`_count_: Stop copying tile from additional zoom levels after the specified _count_ of tiles has been written to the output.
 
 ### Merging attributes from a CSV file
 
@@ -984,10 +985,15 @@ to turn those into moderate detail tiles at high zoom levels, for the benefit of
 renderers that cannot internally overzoom high-resolution tiles without losing
 some of the precision. Running:
 
-    tippecanoe-overzoom -o out.mvt.gz inz/inx/iny outz/outx/outy in.mvt.gz
+    tippecanoe-overzoom -o out.mvt.gz in.mvt.gz inz/inx/iny outz/outx/outy
 
 reads tile `inz/inx/iny` of `in.mvt.gz` and produces tile `outz/outx/outy` of `out.mvt.gz`.
 
+    tippecanoe-overzoom -o out.mvt.gz -t outz/outx/outy in.mvt.gz inz/inx/iny in2.mvt.gz in2z/in2x/in2y in3.mvt.gz in3z/in3x/in3y
+
+reads tile `inz/inx/iny` of `in.mvt.gz`, tile `in2z/in2x/in2y` of `in2.mvt.gz`, and tile `in3z/in3x/in3y` of `in3.mvt.gz`,
+and produces tile `outz/outx/outy` of `out.mvt.gz` from them.
+
 ### Options
 
  * `-b` *buffer*: Set the tile buffer in the output tile (default 5)

clip.cpp

@@ -752,143 +752,172 @@ static std::vector<std::pair<double, double>> clip_poly1(std::vector<std::pair<d
 	return out;
 }
 
-std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int ny,
+std::string overzoom(std::vector<input_tile> tiles, int nz, int nx, int ny,
 		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
 		     std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles) {
-	mvt_tile tile;
+	std::vector<source_tile> decoded;
 
-	try {
-		bool was_compressed;
-		if (!tile.decode(s, was_compressed)) {
-			fprintf(stderr, "Couldn't parse tile %d/%u/%u\n", oz, ox, oy);
-			exit(EXIT_MVT);
+	for (auto const &t : tiles) {
+		mvt_tile tile;
+
+		try {
+			bool was_compressed;
+			if (!tile.decode(t.tile, was_compressed)) {
+				fprintf(stderr, "Couldn't parse tile %d/%u/%u\n", t.z, t.x, t.y);
+				exit(EXIT_MVT);
+			}
+		} catch (std::exception const &e) {
+			fprintf(stderr, "PBF decoding error in tile %d/%u/%u\n", t.z, t.x, t.y);
+			exit(EXIT_PROTOBUF);
 		}
-	} catch (std::exception const &e) {
-		fprintf(stderr, "PBF decoding error in tile %d/%u/%u\n", oz, ox, oy);
-		exit(EXIT_PROTOBUF);
+
+		source_tile out;
+		out.tile = tile;
+		out.z = t.z;
+		out.x = t.x;
+		out.y = t.y;
+
+		decoded.push_back(out);
 	}
 
-	return overzoom(tile, oz, ox, oy, nz, nx, ny, detail, buffer, keep, do_compress, next_overzoomed_tiles);
+	return overzoom(decoded, nz, nx, ny, detail, buffer, keep, do_compress, next_overzoomed_tiles);
 }
 
-std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int ny,
+std::string overzoom(std::vector<source_tile> tiles, int nz, int nx, int ny,
 		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
 		     std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles) {
 	mvt_tile outtile;
 
-	for (auto const &layer : tile.layers) {
-		mvt_layer outlayer = mvt_layer();
+	for (auto const &tile : tiles) {
+		for (auto const &layer : tile.tile.layers) {
+			mvt_layer *outlayer = NULL;
 
-		int det = detail;
-		if (det <= 0) {
-			det = std::round(log(layer.extent) / log(2));
-		}
+			int det = detail;
+			if (det <= 0) {
+				det = std::round(log(layer.extent) / log(2));
+			}
 
-		outlayer.name = layer.name;
-		outlayer.version = layer.version;
-		outlayer.extent = 1LL << det;
+			for (size_t i = 0; i < outtile.layers.size(); i++) {
+				if (outtile.layers[i].name == layer.name) {
+					outlayer = &outtile.layers[i];
+				}
+			}
 
-		for (auto const &feature : layer.features) {
-			mvt_feature outfeature;
-			drawvec geom;
-			int t = feature.type;
+			if (outlayer == NULL) {
+				mvt_layer newlayer = mvt_layer();
 
-			// Convert feature geometry to world coordinates
+				newlayer.name = layer.name;
+				newlayer.version = layer.version;
+				newlayer.extent = 1LL << det;
 
-			long long tilesize = 1LL << (32 - oz);	// source tile size in world coordinates
-			draw ring_closure(0, 0, 0);
+				outtile.layers.push_back(newlayer);
+				outlayer = &outtile.layers.back();
+			}
 
-			for (auto const &g : feature.geometry) {
-				if (g.op == mvt_closepath) {
-					geom.push_back(ring_closure);
-				} else {
-					geom.emplace_back(g.op,
-							  g.x * tilesize / layer.extent + ox * tilesize,
-							  g.y * tilesize / layer.extent + oy * tilesize);
+			for (auto const &feature : layer.features) {
+				mvt_feature outfeature;
+				drawvec geom;
+				int t = feature.type;
 
-					if (g.op == mvt_moveto) {
-						ring_closure = geom.back();
-						ring_closure.op = mvt_lineto;
+				// Convert feature geometry to world coordinates
+
+				long long tilesize = 1LL << (32 - tile.z);  // source tile size in world coordinates
+				draw ring_closure(0, 0, 0);
+
+				for (auto const &g : feature.geometry) {
+					if (g.op == mvt_closepath) {
+						geom.push_back(ring_closure);
+					} else {
+						geom.emplace_back(g.op,
+								  g.x * tilesize / layer.extent + tile.x * tilesize,
+								  g.y * tilesize / layer.extent + tile.y * tilesize);
+
+						if (g.op == mvt_moveto) {
+							ring_closure = geom.back();
+							ring_closure.op = mvt_lineto;
+						}
 					}
 				}
-			}
 
-			// Now offset from world coordinates to output tile coordinates,
-			// but retain world scale, because that is what tippecanoe clipping expects
+				// Now offset from world coordinates to output tile coordinates,
+				// but retain world scale, because that is what tippecanoe clipping expects
 
-			long long outtilesize = 1LL << (32 - nz);  // destination tile size in world coordinates
-			for (auto &g : geom) {
-				g.x -= nx * outtilesize;
-				g.y -= ny * outtilesize;
-			}
+				long long outtilesize = 1LL << (32 - nz);  // destination tile size in world coordinates
+				for (auto &g : geom) {
+					g.x -= nx * outtilesize;
+					g.y -= ny * outtilesize;
+				}
 
-			// Clip to output tile
+				// Clip to output tile
 
-			long long xmin = LLONG_MAX;
-			long long ymin = LLONG_MAX;
-			long long xmax = LLONG_MIN;
-			long long ymax = LLONG_MIN;
+				long long xmin = LLONG_MAX;
+				long long ymin = LLONG_MAX;
+				long long xmax = LLONG_MIN;
+				long long ymax = LLONG_MIN;
 
-			for (auto const &g : geom) {
-				xmin = std::min(xmin, g.x);
-				ymin = std::min(ymin, g.y);
-				xmax = std::max(xmax, g.x);
-				ymax = std::max(ymax, g.y);
-			}
+				for (auto const &g : geom) {
+					xmin = std::min(xmin, g.x);
+					ymin = std::min(ymin, g.y);
+					xmax = std::max(xmax, g.x);
+					ymax = std::max(ymax, g.y);
+				}
 
-			long long b = outtilesize * buffer / 256;
-			if (xmax < -b || ymax < -b || xmin > outtilesize + b || ymin > outtilesize + b) {
-				continue;
-			}
+				long long b = outtilesize * buffer / 256;
+				if (xmax < -b || ymax < -b || xmin > outtilesize + b || ymin > outtilesize + b) {
+					continue;
+				}
 
-			if (t == VT_LINE) {
-				geom = clip_lines(geom, nz, buffer);
-			} else if (t == VT_POLYGON) {
-				drawvec dv;
-				geom = simple_clip_poly(geom, nz, buffer, dv, false);
-			} else if (t == VT_POINT) {
-				geom = clip_point(geom, nz, buffer);
-			}
+				if (t == VT_LINE) {
+					geom = clip_lines(geom, nz, buffer);
+				} else if (t == VT_POLYGON) {
+					drawvec dv;
+					geom = simple_clip_poly(geom, nz, buffer, dv, false);
+				} else if (t == VT_POINT) {
+					geom = clip_point(geom, nz, buffer);
+				}
 
-			// Scale to output tile extent
+				// Scale to output tile extent
 
-			to_tile_scale(geom, nz, det);
+				to_tile_scale(geom, nz, det);
 
-			// Clean geometries
+				// Clean geometries
 
-			geom = remove_noop(geom, t, 0);
-			if (t == VT_POLYGON) {
-				geom = clean_or_clip_poly(geom, 0, 0, false, false);
-				geom = close_poly(geom);
-			}
+				geom = remove_noop(geom, t, 0);
+				if (t == VT_POLYGON) {
+					geom = clean_or_clip_poly(geom, 0, 0, false, false);
+					geom = close_poly(geom);
+				}
 
-			// Add geometry to output feature
+				// Add geometry to output feature
 
-			outfeature.type = t;
-			for (auto const &g : geom) {
-				outfeature.geometry.emplace_back(g.op, g.x, g.y);
-			}
+				outfeature.type = t;
+				for (auto const &g : geom) {
+					outfeature.geometry.emplace_back(g.op, g.x, g.y);
+				}
 
-			// ID and attributes, if it didn't get clipped away
+				// ID and attributes, if it didn't get clipped away
 
-			if (outfeature.geometry.size() > 0) {
-				if (feature.has_id) {
-					outfeature.has_id = true;
-					outfeature.id = feature.id;
-				}
+				if (outfeature.geometry.size() > 0) {
+					if (feature.has_id) {
+						outfeature.has_id = true;
+						outfeature.id = feature.id;
+					}
 
-				for (size_t i = 0; i + 1 < feature.tags.size(); i += 2) {
-					if (keep.size() == 0 || keep.find(layer.keys[feature.tags[i]]) != keep.end()) {
-						outlayer.tag(outfeature, layer.keys[feature.tags[i]], layer.values[feature.tags[i + 1]]);
+					for (size_t i = 0; i + 1 < feature.tags.size(); i += 2) {
+						if (keep.size() == 0 || keep.find(layer.keys[feature.tags[i]]) != keep.end()) {
+							outlayer->tag(outfeature, layer.keys[feature.tags[i]], layer.values[feature.tags[i + 1]]);
+						}
 					}
-				}
 
-				outlayer.features.push_back(outfeature);
+					outlayer->features.push_back(outfeature);
+				}
 			}
 		}
+	}
 
-		if (outlayer.features.size() > 0) {
-			outtile.layers.push_back(outlayer);
+	for (ssize_t i = outtile.layers.size() - 1; i >= 0; i--) {
+		if (outtile.layers[i].features.size() == 0) {
+			outtile.layers.erase(outtile.layers.begin() + i);
 		}
 	}
 
@@ -903,7 +932,16 @@ std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int
 		if (outtile.layers.size() > 0) {
 			for (size_t x = 0; x < 2; x++) {
 				for (size_t y = 0; y < 2; y++) {
-					std::string child = overzoom(outtile, nz, nx, ny,
+					source_tile st;
+					st.tile = outtile;
+					st.z = nz;
+					st.x = nx;
+					st.y = ny;
+
+					std::vector<source_tile> sts;
+					sts.push_back(st);
+
+					std::string child = overzoom(sts,
 								     nz + 1, nx * 2 + x, ny * 2 + y,
 								     detail, buffer, keep, false, NULL);
 					if (child.size() > 0) {

geometry.hpp

@@ -98,11 +98,25 @@ void visvalingam(drawvec &ls, size_t start, size_t end, double threshold, size_t
 int pnpoly(const drawvec &vert, size_t start, size_t nvert, long long testx, long long testy);
 double distance_from_line(long long point_x, long long point_y, long long segA_x, long long segA_y, long long segB_x, long long segB_y);
 
-std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int ny,
+struct input_tile {
+	std::string tile;
+	int z;
+	int x;
+	int y;
+};
+
+struct source_tile {
+	mvt_tile tile;
+	int z;
+	int x;
+	int y;
+};
+
+std::string overzoom(std::vector<source_tile> tiles, int nz, int nx, int ny,
 		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
 		     std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles);
 
-std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int ny,
+std::string overzoom(std::vector<input_tile> tiles, int nz, int nx, int ny,
 		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
 		     std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles);