Skip to content

Commit

Permalink
Add whole-tile postprocess hook (#802)
Browse files Browse the repository at this point in the history
  • Loading branch information
@msbarry
msbarry authored Jan 23, 2024
1 parent 6331934 commit fa7bffb
Show file tree
Hide file tree
Showing 4 changed files with 170 additions and 25 deletions.
34 changes: 32 additions & 2 deletions planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.onthegomap.planetiler;

import com.onthegomap.planetiler.geo.GeometryException;
import com.onthegomap.planetiler.geo.TileCoord;
import com.onthegomap.planetiler.mbtiles.Mbtiles;
import com.onthegomap.planetiler.reader.SourceFeature;
import com.onthegomap.planetiler.reader.osm.OsmElement;
Expand Down Expand Up @@ -103,13 +104,40 @@ default void release() {}
* @return the new list of output features or {@code null} to not change anything. Set any elements of the list to
* {@code null} if they should be ignored.
* @throws GeometryException for any recoverable geometric operation failures - the framework will log the error, emit
* the original input features, and continue processing other tiles
* the original input features, and continue processing other layers
*/
default List<VectorTile.Feature> postProcessLayerFeatures(String layer, int zoom,
List<VectorTile.Feature> items) throws GeometryException {
return items;
}

/**
* Apply any post-processing to layers in an output tile before writing it to the output.
* <p>
* This is called before {@link #postProcessLayerFeatures(String, int, List)} gets called for each layer. Use this
* method if features in one layer should influence features in another layer, to create new layers from existing
* ones, or if you need to remove a layer entirely from the output.
* <p>
* These transformations may add, remove, or change the tags, geometry, or ordering of output features based on other
* features present in this tile. See {@link FeatureMerge} class for a set of common transformations that merge
* linestrings/polygons.
* <p>
* Many threads invoke this method concurrently so ensure thread-safe access to any shared data structures.
* <p>
* The default implementation passes through input features unaltered
*
* @param tileCoord the tile being post-processed
* @param layers all the output features in each layer on this tile
* @return the new map from layer to features or {@code null} to not change anything. Set any elements of the lists to
* {@code null} if they should be ignored.
* @throws GeometryException for any recoverable geometric operation failures - the framework will log the error, emit
* the original input features, and continue processing other tiles
*/
default Map<String, List<VectorTile.Feature>> postProcessTileFeatures(TileCoord tileCoord,
Map<String, List<VectorTile.Feature>> layers) throws GeometryException {
return layers;
}

/**
* Returns the name of the generated tileset to put into {@link Mbtiles} metadata
*
Expand Down Expand Up @@ -158,7 +186,9 @@ default boolean isOverlay() {
return false;
}

default Map<String,String> extraArchiveMetadata() { return Map.of(); }
default Map<String, String> extraArchiveMetadata() {
return Map.of();
}

/**
* Defines whether {@link Wikidata} should fetch wikidata translations for the input element.
Expand Down
54 changes: 39 additions & 15 deletions planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import javax.annotation.concurrent.NotThreadSafe;
Expand Down Expand Up @@ -449,28 +450,47 @@ public VectorTile getVectorTile(LayerAttrStats.Updater layerStats) {
if (layerStats != null) {
tile.trackLayerStats(layerStats.forZoom(tileCoord.z()));
}
List<VectorTile.Feature> items = new ArrayList<>(entries.size());
List<VectorTile.Feature> items = new ArrayList<>();
String currentLayer = null;
Map<String, List<VectorTile.Feature>> layerFeatures = new TreeMap<>();
for (SortableFeature entry : entries) {
var feature = decodeVectorTileFeature(entry);
String layer = feature.layer();

if (currentLayer == null) {
currentLayer = layer;
layerFeatures.put(currentLayer, items);
} else if (!currentLayer.equals(layer)) {
postProcessAndAddLayerFeatures(tile, currentLayer, items);
currentLayer = layer;
items.clear();
items = new ArrayList<>();
layerFeatures.put(layer, items);
}

items.add(feature);
}
postProcessAndAddLayerFeatures(tile, currentLayer, items);
// first post-process entire tile by invoking postProcessTileFeatures to allow for post-processing that combines
// features across different layers, infers new layers, or removes layers
try {
var initialFeatures = layerFeatures;
layerFeatures = profile.postProcessTileFeatures(tileCoord, layerFeatures);
if (layerFeatures == null) {
layerFeatures = initialFeatures;
}
} catch (Throwable e) { // NOSONAR - OK to catch Throwable since we re-throw Errors
handlePostProcessFailure(e, "entire tile");
}
// then let profiles post-process each layer in isolation with postProcessLayerFeatures
for (var entry : layerFeatures.entrySet()) {
postProcessAndAddLayerFeatures(tile, entry.getKey(), entry.getValue());
}
return tile;
}

private void postProcessAndAddLayerFeatures(VectorTile encoder, String layer,
List<VectorTile.Feature> features) {
if (features == null || features.isEmpty()) {
return;
}
try {
List<VectorTile.Feature> postProcessed = profile
.postProcessLayerFeatures(layer, tileCoord.z(), features);
Expand All @@ -482,21 +502,25 @@ private void postProcessAndAddLayerFeatures(VectorTile encoder, String layer,
// also remove points more than --max-point-buffer pixels outside the tile if the
// user has requested a narrower buffer than the profile provides by default
} catch (Throwable e) { // NOSONAR - OK to catch Throwable since we re-throw Errors
// failures in tile post-processing happen very late so err on the side of caution and
// log failures, only throwing when it's a fatal error
if (e instanceof GeometryException geoe) {
geoe.log(stats, "postprocess_layer",
"Caught error postprocessing features for " + layer + " layer on " + tileCoord, config.logJtsExceptions());
} else if (e instanceof Error err) {
LOGGER.error("Caught fatal error postprocessing features {} {}", layer, tileCoord, e);
throw err;
} else {
LOGGER.error("Caught error postprocessing features {} {}", layer, tileCoord, e);
}
handlePostProcessFailure(e, layer);
}
encoder.addLayerFeatures(layer, features);
}

private void handlePostProcessFailure(Throwable e, String entity) {
// failures in tile post-processing happen very late so err on the side of caution and
// log failures, only throwing when it's a fatal error
if (e instanceof GeometryException geoe) {
geoe.log(stats, "postprocess_layer",
"Caught error postprocessing features for " + entity + " on " + tileCoord, config.logJtsExceptions());
} else if (e instanceof Error err) {
LOGGER.error("Caught fatal error postprocessing features {} {}", entity, tileCoord, e);
throw err;
} else {
LOGGER.error("Caught error postprocessing features {} {}", entity, tileCoord, e);
}
}

void add(SortableFeature entry) {
numFeaturesProcessed.incrementAndGet();
long key = entry.key();
Expand Down
80 changes: 75 additions & 5 deletions planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1316,6 +1316,55 @@ void testMergeLineStrings(boolean connectEndpoints) throws Exception {
)), sortListValues(results.tiles));
}

@ParameterizedTest
@ValueSource(booleans = {false, true})
void postProcessTileFeatures(boolean postProcessLayersToo) throws Exception {
double y = 0.5 + Z15_WIDTH / 2;
double lat = GeoUtils.getWorldLat(y);

double x1 = 0.5 + Z15_WIDTH / 4;
double lng1 = GeoUtils.getWorldLon(x1);
double lng2 = GeoUtils.getWorldLon(x1 + Z15_WIDTH * 10d / 256);

List<SimpleFeature> features1 = List.of(
newReaderFeature(newPoint(lng1, lat), Map.of("from", "a")),
newReaderFeature(newPoint(lng2, lat), Map.of("from", "b"))
);
var testProfile = new Profile.NullProfile() {
@Override
public void processFeature(SourceFeature sourceFeature, FeatureCollector features) {
features.point(sourceFeature.getString("from"))
.inheritAttrFromSource("from")
.setMinZoom(15);
}

@Override
public Map<String, List<VectorTile.Feature>> postProcessTileFeatures(TileCoord tileCoord,
Map<String, List<VectorTile.Feature>> layers) {
List<VectorTile.Feature> features = new ArrayList<>();
features.addAll(layers.get("a"));
features.addAll(layers.get("b"));
return Map.of("c", features);
}

@Override
public List<VectorTile.Feature> postProcessLayerFeatures(String layer, int zoom, List<VectorTile.Feature> items) {
return postProcessLayersToo ? items.reversed() : items;
}
};
var results = run(
Map.of("threads", "1", "maxzoom", "15"),
(featureGroup, profile, config) -> processReaderFeatures(featureGroup, profile, config, features1),
testProfile);

assertSubmap(sortListValues(Map.of(
TileCoord.ofXYZ(Z15_TILES / 2, Z15_TILES / 2, 15), List.of(
feature(newPoint(64, 128), "c", Map.of("from", "a")),
feature(newPoint(74, 128), "c", Map.of("from", "b"))
)
)), sortListValues(results.tiles));
}

@Test
void testMergeLineStringsIgnoresRoundingIntersections() throws Exception {
double y = 0.5 + Z14_WIDTH / 2;
Expand Down Expand Up @@ -1681,6 +1730,12 @@ List<VectorTile.Feature> process(String layer, int zoom, List<VectorTile.Feature
throws GeometryException;
}

private interface TilePostprocessFunction {

Map<String, List<VectorTile.Feature>> process(TileCoord tileCoord, Map<String, List<VectorTile.Feature>> layers)
throws GeometryException;
}

private record PlanetilerResults(
Map<TileCoord, List<TestUtils.ComparableFeature>> tiles, Map<String, String> metadata, int tileDataCount
) {}
Expand All @@ -1692,7 +1747,8 @@ private record TestProfile(
@Override String version,
BiConsumer<SourceFeature, FeatureCollector> processFeature,
Function<OsmElement.Relation, List<OsmRelationInfo>> preprocessOsmRelation,
LayerPostprocessFunction postprocessLayerFeatures
LayerPostprocessFunction postprocessLayerFeatures,
TilePostprocessFunction postprocessTileFeatures
) implements Profile {

TestProfile(
Expand All @@ -1701,8 +1757,16 @@ private record TestProfile(
LayerPostprocessFunction postprocessLayerFeatures
) {
this(TEST_PROFILE_NAME, TEST_PROFILE_DESCRIPTION, TEST_PROFILE_ATTRIBUTION, TEST_PROFILE_VERSION, processFeature,
preprocessOsmRelation,
postprocessLayerFeatures);
preprocessOsmRelation, postprocessLayerFeatures, null);
}

TestProfile(
BiConsumer<SourceFeature, FeatureCollector> processFeature,
Function<OsmElement.Relation, List<OsmRelationInfo>> preprocessOsmRelation,
TilePostprocessFunction postprocessTileFeatures
) {
this(TEST_PROFILE_NAME, TEST_PROFILE_DESCRIPTION, TEST_PROFILE_ATTRIBUTION, TEST_PROFILE_VERSION, processFeature,
preprocessOsmRelation, null, postprocessTileFeatures);
}

static TestProfile processSourceFeatures(BiConsumer<SourceFeature, FeatureCollector> processFeature) {
Expand All @@ -1712,7 +1776,7 @@ static TestProfile processSourceFeatures(BiConsumer<SourceFeature, FeatureCollec
@Override
public List<OsmRelationInfo> preprocessOsmRelation(
OsmElement.Relation relation) {
return preprocessOsmRelation.apply(relation);
return preprocessOsmRelation == null ? null : preprocessOsmRelation.apply(relation);
}

@Override
Expand All @@ -1726,7 +1790,13 @@ public void release() {}
@Override
public List<VectorTile.Feature> postProcessLayerFeatures(String layer, int zoom,
List<VectorTile.Feature> items) throws GeometryException {
return postprocessLayerFeatures.process(layer, zoom, items);
return postprocessLayerFeatures == null ? null : postprocessLayerFeatures.process(layer, zoom, items);
}

@Override
public Map<String, List<VectorTile.Feature>> postProcessTileFeatures(TileCoord tileCoord,
Map<String, List<VectorTile.Feature>> layers) throws GeometryException {
return postprocessTileFeatures == null ? null : postprocessTileFeatures.process(tileCoord, layers);
}
}

Expand Down
27 changes: 24 additions & 3 deletions planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,7 @@ public static Map<TileCoord, List<ComparableFeature>> getTileMap(ReadableTileArc
case UNKNOWN -> throw new IllegalArgumentException("cannot decompress \"UNKNOWN\"");
};
var decoded = VectorTile.decode(bytes).stream()
.map(feature -> feature(decodeSilently(feature.geometry()), feature.attrs())).toList();
.map(feature -> feature(decodeSilently(feature.geometry()), feature.layer(), feature.attrs())).toList();
tiles.put(tile.coord(), decoded);
}
return tiles;
Expand Down Expand Up @@ -466,11 +466,32 @@ public int hashCode() {

public record ComparableFeature(
GeometryComparision geometry,
String layer,
Map<String, Object> attrs
) {}
) {

@Override
public boolean equals(Object o) {
return o == this || (o instanceof ComparableFeature other &&
geometry.equals(other.geometry) &&
attrs.equals(other.attrs) &&
(layer == null || other.layer == null || Objects.equals(layer, other.layer)));
}

@Override
public int hashCode() {
int result = geometry.hashCode();
result = 31 * result + attrs.hashCode();
return result;
}
}

public static ComparableFeature feature(Geometry geom, String layer, Map<String, Object> attrs) {
return new ComparableFeature(new NormGeometry(geom), layer, attrs);
}

public static ComparableFeature feature(Geometry geom, Map<String, Object> attrs) {
return new ComparableFeature(new NormGeometry(geom), attrs);
return new ComparableFeature(new NormGeometry(geom), null, attrs);
}

public static Map<String, Object> toMap(FeatureCollector.Feature feature, int zoom) {
Expand Down

0 comments on commit fa7bffb

Please sign in to comment.