diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/TransportNetworkConfig.java b/src/main/java/com/conveyal/r5/analyst/cluster/TransportNetworkConfig.java index 54f11ee8c..012e3204a 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/TransportNetworkConfig.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/TransportNetworkConfig.java @@ -4,11 +4,13 @@ import com.conveyal.r5.analyst.scenario.Modification; import com.conveyal.r5.analyst.scenario.RasterCost; import com.conveyal.r5.analyst.scenario.ShapefileLts; +import com.conveyal.r5.profile.StreetMode; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.util.List; +import java.util.Set; /** * All inputs and options that describe how to build a particular transport network (except the serialization version). @@ -39,4 +41,17 @@ public class TransportNetworkConfig { /** A list of _R5_ modifications to apply during network build. May be null. */ public List modifications; + /** + * Additional modes other than walk for which to pre-build large data structures (grid linkage and cost tables). + * When building a network, by default we build distance tables from transit stops to street vertices, to which we + * connect a grid covering the entire street network at the default zoom level. By default we do this only for the + * walk mode. Pre-building and serializing equivalent data structures for other modes allows workers to start up + * much faster in regional analyses. The work need only be done once when the first single-point worker to builds + * the network. Otherwise, hundreds of workers will each have to build these tables every time they start up. + * Some scenarios, such as those that affect the street layer, may still be slower to apply for modes listed here + * because some intermediate data (stop-to-vertex tables) are only retained for the walk mode. If this proves to be + * a problem it is a candidate for future optimization. + */ + public Set buildGridsForModes; + } diff --git a/src/main/java/com/conveyal/r5/streets/EgressCostTable.java b/src/main/java/com/conveyal/r5/streets/EgressCostTable.java index cb72a65e1..60bf5e336 100644 --- a/src/main/java/com/conveyal/r5/streets/EgressCostTable.java +++ b/src/main/java/com/conveyal/r5/streets/EgressCostTable.java @@ -209,7 +209,7 @@ public EgressCostTable (LinkedPointSet linkedPointSet, ProgressListener progress rebuildZone = linkedPointSet.streetLayer.scenarioEdgesBoundingGeometry(linkingDistanceLimitMeters); } - LOG.info("Creating EgressCostTables from each transit stop to PointSet points."); + LOG.info("Creating EgressCostTables from each transit stop to PointSet points for mode {}.", streetMode); if (rebuildZone != null) { LOG.info("Selectively computing tables for only those stops that might be affected by the scenario."); } @@ -232,9 +232,9 @@ public EgressCostTable (LinkedPointSet linkedPointSet, ProgressListener progress progressListener.beginTask(taskDescription, nStops); final LambdaCounter computeCounter = new LambdaCounter(LOG, nStops, computeLogFrequency, - "Computed new stop -> point tables for {} of {} transit stops."); + String.format("Computed new stop-to-point tables from {} of {} transit stops for mode %s.", streetMode)); final LambdaCounter copyCounter = new LambdaCounter(LOG, nStops, copyLogFrequency, - "Copied unchanged stop -> point tables for {} of {} transit stops."); + String.format("Copied unchanged stop-to-point tables from {} of {} transit stops for mode %s.", streetMode)); // Create a distance table from each transit stop to the points in this PointSet in parallel. // Each table is a flattened 2D array. Two values for each point reachable from this stop: (pointIndex, cost) // When applying a scenario, keep the existing distance table for those stops that could not be affected. @@ -262,16 +262,16 @@ public EgressCostTable (LinkedPointSet linkedPointSet, ProgressListener progress GeometryUtils.expandEnvelopeFixed(envelopeAroundStop, linkingDistanceLimitMeters); if (streetMode == StreetMode.WALK) { - // Walking distances from stops to street vertices are saved in the TransitLayer. - // Get the pre-computed walking distance table from the stop to the street vertices, - // then extend that table out from the street vertices to the points in this PointSet. - // TODO reuse the code that computes the walk tables at TransitLayer.buildOneDistanceTable() rather than - // duplicating it below for other modes. + // Distances from stops to street vertices are saved in the TransitLayer, but only for the walk mode. + // Get the pre-computed walking distance table from the stop to the street vertices, then extend that + // table out from the street vertices to the points in this PointSet. It may be possible to reuse the + // code that pre-computes walk tables at TransitLayer.buildOneDistanceTable() rather than duplicating + // it below for other (non-walk) modes. TIntIntMap distanceTableToVertices = transitLayer.stopToVertexDistanceTables.get(stopIndex); return distanceTableToVertices == null ? null : linkedPointSet.extendDistanceTableToPoints(distanceTableToVertices, envelopeAroundStop); } else { - + // For non-walk modes perform a search from each stop, as stop-to-vertex tables are not precomputed. Geometry egressArea = null; // If a pickup delay modification is present for this street mode, egressStopDelaysSeconds is @@ -301,14 +301,14 @@ public EgressCostTable (LinkedPointSet linkedPointSet, ProgressListener progress LOG.warn("Stop unlinked, cannot build distance table: {}", stopIndex); return null; } - // TODO setting the origin point of the router to the stop vertex does not work. - // This is probably because link edges do not allow car traversal. We could traverse them. - // As a stopgap we perform car linking at the geographic coordinate of the stop. + // Setting the origin point of the router to the stop vertex (as follows) does not work. // sr.setOrigin(vertexId); + // This is probably because link edges do not allow car traversal. We could traverse them. + // As a workaround we perform car linking at the geographic coordinate of the stop. VertexStore.Vertex vertex = linkedPointSet.streetLayer.vertexStore.getCursor(vertexId); sr.setOrigin(vertex.getLat(), vertex.getLon()); - // WALK is handled above, this block is exhaustively handling all other modes. + // WALK is handled in the if clause above, this else block is exhaustively handling all other modes. if (streetMode == StreetMode.BICYCLE) { sr.distanceLimitMeters = linkingDistanceLimitMeters; } else if (streetMode == StreetMode.CAR) { diff --git a/src/main/java/com/conveyal/r5/streets/LinkedPointSet.java b/src/main/java/com/conveyal/r5/streets/LinkedPointSet.java index c529ced10..01a51a43d 100644 --- a/src/main/java/com/conveyal/r5/streets/LinkedPointSet.java +++ b/src/main/java/com/conveyal/r5/streets/LinkedPointSet.java @@ -135,7 +135,7 @@ public class LinkedPointSet implements Serializable { * the same pointSet and streetMode as the preceding arguments. */ public LinkedPointSet (PointSet pointSet, StreetLayer streetLayer, StreetMode streetMode, LinkedPointSet baseLinkage) { - LOG.info("Linking pointset to street network..."); + LOG.info("Linking pointset to street network for mode {}...", streetMode); this.pointSet = pointSet; this.streetLayer = streetLayer; this.streetMode = streetMode; @@ -301,7 +301,7 @@ public synchronized EgressCostTable getEgressCostTable () { */ private void linkPointsToStreets (boolean all) { LambdaCounter linkCounter = new LambdaCounter(LOG, pointSet.featureCount(), 10000, - "Linked {} of {} PointSet points to streets."); + String.format("Linked {} of {} PointSet points to streets for mode %s.", streetMode)); // Construct a geometry around any edges added by the scenario, or null if there are no added edges. // As it is derived from edge geometries this is a fixed-point geometry and must be intersected with the same. diff --git a/src/main/java/com/conveyal/r5/transit/TransitLayer.java b/src/main/java/com/conveyal/r5/transit/TransitLayer.java index e140b0d89..871491ff2 100644 --- a/src/main/java/com/conveyal/r5/transit/TransitLayer.java +++ b/src/main/java/com/conveyal/r5/transit/TransitLayer.java @@ -535,16 +535,18 @@ public void rebuildTransientIndexes () { } /** - * Run a distance-constrained street search from every transit stop in the graph. + * Run a distance-constrained street search from every transit stop in the graph using the walk mode. * Store the distance to every reachable street vertex for each of these origin stops. * If a scenario has been applied, we need to build tables for any newly created stops and any stops within * transfer distance or access/egress distance of those new stops. In that case a rebuildZone geometry should be * supplied. If rebuildZone is null, a complete rebuild of all tables will occur for all stops. + * Note, this rebuilds for the WALK MODE ONLY. The network only has a field for retaining walk distance tables. + * This is a candidate for optimization if car or bicycle scenarios are slow to apply. * @param rebuildZone the zone within which to rebuild tables in FIXED-POINT DEGREES, or null to build all tables. */ public void buildDistanceTables(Geometry rebuildZone) { - LOG.info("Finding distances from transit stops to street vertices."); + LOG.info("Pre-computing distances from transit stops to street vertices (WALK mode only)."); if (rebuildZone != null) { LOG.info("Selectively finding distances for only those stops potentially affected by scenario application."); } diff --git a/src/main/java/com/conveyal/r5/transit/TransportNetwork.java b/src/main/java/com/conveyal/r5/transit/TransportNetwork.java index c68a8f166..3e3cb3720 100644 --- a/src/main/java/com/conveyal/r5/transit/TransportNetwork.java +++ b/src/main/java/com/conveyal/r5/transit/TransportNetwork.java @@ -25,6 +25,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Stream; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -263,14 +264,14 @@ public static InputFileType forFile(File file) { } /** - * For Analysis purposes, build an efficient implicit grid PointSet for this TransportNetwork. Then, for any modes - * supplied, we also build a linkage that is held permanently in the GridPointSet. This method is called when a - * network is first built. - * The resulting grid PointSet will cover the entire street network layer of this TransportNetwork, which should - * include every point we can route from or to. Any other destination grid (for the same mode, walking) can be made - * as a subset of this one since it includes every potentially accessible point. + * Build a grid PointSet covering the entire street network layer of this TransportNetwork, which should include + * every point we can route from or to. Then for all requested modes build a linkage that is held in the + * GridPointSet. This method is called when a network is first built so these linkages are serialized with it. + * Any other destination grid (at least for the same modes) can be made as a subset of this one since it includes + * every potentially accessible point. Destination grids for other modes will be made on demand, which is a slow + * operation that can occupy hundreds of workers for long periods of time when a regional analysis starts up. */ - public void rebuildLinkedGridPointSet(StreetMode... modes) { + public void rebuildLinkedGridPointSet(Iterable modes) { if (fullExtentGridPointSet != null) { throw new RuntimeException("Linked grid pointset was built more than once."); } @@ -280,6 +281,10 @@ public void rebuildLinkedGridPointSet(StreetMode... modes) { } } + public void rebuildLinkedGridPointSet(StreetMode... modes) { + rebuildLinkedGridPointSet(Set.of(modes)); + } + //TODO: add transit stops to envelope public Envelope getEnvelope() { return streetLayer.getEnvelope(); diff --git a/src/main/java/com/conveyal/r5/transit/TransportNetworkCache.java b/src/main/java/com/conveyal/r5/transit/TransportNetworkCache.java index d93891ba1..b7f1f1d8f 100644 --- a/src/main/java/com/conveyal/r5/transit/TransportNetworkCache.java +++ b/src/main/java/com/conveyal/r5/transit/TransportNetworkCache.java @@ -19,6 +19,7 @@ import com.conveyal.r5.streets.StreetLayer; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.LoadingCache; +import com.google.common.collect.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -158,28 +159,52 @@ private static FileStorageKey getR5NetworkFileStorageKey (String networkId) { return new FileStorageKey(BUNDLES, getR5NetworkFilename(networkId)); } + /** @return the network configuration (AKA manifest) for the given network ID, or null if no config file exists. */ + private TransportNetworkConfig loadNetworkConfig (String networkId) { + FileStorageKey configFileKey = new FileStorageKey(BUNDLES, getNetworkConfigFilename(networkId)); + if (!fileStorage.exists(configFileKey)) { + return null; + } + File configFile = fileStorage.getFile(configFileKey); + try { + // Use lenient mapper to mimic behavior in objectFromRequestBody. + return JsonUtilities.lenientObjectMapper.readValue(configFile, TransportNetworkConfig.class); + } catch (IOException e) { + throw new RuntimeException("Error reading TransportNetworkConfig. Does it contain new unrecognized fields?", e); + } + } + /** * If we did not find a cached network, build one from the input files. Should throw an exception rather than * returning null if for any reason it can't finish building one. */ private @Nonnull TransportNetwork buildNetwork (String networkId) { TransportNetwork network; - FileStorageKey networkConfigKey = new FileStorageKey(BUNDLES, GTFSCache.cleanId(networkId) + ".json"); - if (fileStorage.exists(networkConfigKey)) { - network = buildNetworkFromConfig(networkId); - } else { - LOG.warn("Detected old-format bundle stored as single ZIP file"); + TransportNetworkConfig networkConfig = loadNetworkConfig(networkId); + if (networkConfig == null) { + // The switch to use JSON manifests instead of zips occurred in 32a1aebe in July 2016. + // Over six years have passed, buildNetworkFromBundleZip is deprecated and could probably be removed. + LOG.warn("No network config (aka manifest) found. Assuming old-format network inputs bundle stored as a single ZIP file."); network = buildNetworkFromBundleZip(networkId); + } else { + network = buildNetworkFromConfig(networkConfig); } network.scenarioId = networkId; - // Networks created in TransportNetworkCache are going to be used for analysis work. Pre-compute distance tables - // from stops to street vertices, then pre-build a linked grid pointset for the whole region. These linkages - // should be serialized along with the network, which avoids building them when an analysis worker starts. - // The linkage we create here will never be used directly, but serves as a basis for scenario linkages, making - // analysis much faster to start up. + // Pre-compute distance tables from stops out to street vertices, then pre-build a linked grid pointset for the + // whole region covered by the street network. These tables and linkages will be serialized along with the + // network, which avoids building them when every analysis worker starts. The linkage we create here will never + // be used directly, but serves as a basis for scenario linkages, making analyses much faster to start up. + // Note, this retains stop-to-vertex distances for the WALK MODE ONLY, even when they are produced as + // intermediate results while building linkages for other modes. + // This is a candidate for optimization if car or bicycle scenarios are slow to apply. network.transitLayer.buildDistanceTables(null); - network.rebuildLinkedGridPointSet(StreetMode.WALK); + + Set buildGridsForModes = Sets.newHashSet(StreetMode.WALK); + if (networkConfig != null && networkConfig.buildGridsForModes != null) { + buildGridsForModes.addAll(networkConfig.buildGridsForModes); + } + network.rebuildLinkedGridPointSet(buildGridsForModes); // Cache the serialized network on the local filesystem and mirror it to any remote storage. try { @@ -247,17 +272,7 @@ private TransportNetwork buildNetworkFromBundleZip (String networkId) { * This describes the locations of files used to create a bundle, as well as options applied at network build time. * It contains the unique IDs of the GTFS feeds and OSM extract. */ - private TransportNetwork buildNetworkFromConfig (String networkId) { - FileStorageKey configFileKey = new FileStorageKey(BUNDLES, getNetworkConfigFilename(networkId)); - File configFile = fileStorage.getFile(configFileKey); - TransportNetworkConfig config; - - try { - // Use lenient mapper to mimic behavior in objectFromRequestBody. - config = JsonUtilities.lenientObjectMapper.readValue(configFile, TransportNetworkConfig.class); - } catch (IOException e) { - throw new RuntimeException("Error reading TransportNetworkConfig. Does it contain new unrecognized fields?", e); - } + private TransportNetwork buildNetworkFromConfig (TransportNetworkConfig config) { // FIXME duplicate code. All internal building logic should be encapsulated in a method like // TransportNetwork.build(osm, gtfs1, gtfs2...) // We currently have multiple copies of it, in buildNetworkFromConfig and buildNetworkFromBundleZip so you've @@ -265,7 +280,7 @@ private TransportNetwork buildNetworkFromConfig (String networkId) { // Maybe we should just completely deprecate bundle ZIPs and remove those code paths. TransportNetwork network = new TransportNetwork(); - network.scenarioId = networkId; + network.streetLayer = new StreetLayer(); network.streetLayer.loadFromOsm(osmCache.get(config.osmId));