From eb145f6266d69f5ffc919ecb902c3a37809f5595 Mon Sep 17 00:00:00 2001 From: Anson Stewart Date: Sat, 11 Mar 2023 00:59:38 -0500 Subject: [PATCH 1/7] Use gradle-build-action and let it manage its own caching rather than using actiones/cache, per https://github.com/marketplace/actions/gradle-build-action#why-use-the-gradle-build-action --- .github/workflows/gradle.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 22691e6d1..150535ab7 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -31,16 +31,14 @@ jobs: - uses: actions/checkout@v2.3.2 with: fetch-depth: 0 + - uses: gradle/gradle-build-action@2.4.0 + with: + gradle-version: 7.6.1 # Java setup step completes very fast, no need to run in a preconfigured docker container. - name: Set up JDK 11 uses: actions/setup-java@v1 with: java-version: 11 - - uses: actions/cache@v1 - id: cache - with: - path: ~/.gradle/caches - key: gradle-caches - name: Show version string run: gradle -q printVersion | head -n1 - name: Build and Test From 5824ba77e5d329c9ca4ebdd0ef545dede8dbe312 Mon Sep 17 00:00:00 2001 From: Anson Stewart Date: Sat, 11 Mar 2023 01:01:58 -0500 Subject: [PATCH 2/7] Properly specify action version number --- .github/workflows/gradle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 150535ab7..810809987 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -31,7 +31,7 @@ jobs: - uses: actions/checkout@v2.3.2 with: fetch-depth: 0 - - uses: gradle/gradle-build-action@2.4.0 + - uses: gradle/gradle-build-action@v2.4.0 with: gradle-version: 7.6.1 # Java setup step completes very fast, no need to run in a preconfigured docker container. From 9799e927655f40c5cd885baf9d7bddf7b6c94121 Mon Sep 17 00:00:00 2001 From: Anson Stewart Date: Sat, 11 Mar 2023 01:07:20 -0500 Subject: [PATCH 3/7] Use gradle-build-action in cypress-integration --- .github/workflows/cypress-integration.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cypress-integration.yml b/.github/workflows/cypress-integration.yml index 0fd16c257..150a1d209 100644 --- a/.github/workflows/cypress-integration.yml +++ b/.github/workflows/cypress-integration.yml @@ -22,13 +22,10 @@ jobs: with: fetch-depth: 0 path: r5 - # Build .jar and copy to ./ui directory (along with config file) - # Cache Gradle dependencies to speed up the shadow jar build - - uses: actions/cache@v1 - id: cache + - uses: gradle/gradle-build-action@v2.4.0 with: - path: ~/.gradle/caches - key: gradle-caches + gradle-version: 7.6.1 + # Build .jar and copy to ./ui directory (along with config file) - uses: actions/setup-java@v1 with: java-version: 11 From 95497de936c6a072dfbddad0852e08a6e879435a Mon Sep 17 00:00:00 2001 From: ansons Date: Sat, 11 Mar 2023 00:27:21 -0500 Subject: [PATCH 4/7] experimental: halve speed if lts exceeded by 1 --- .../com/conveyal/r5/streets/EdgeStore.java | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/conveyal/r5/streets/EdgeStore.java b/src/main/java/com/conveyal/r5/streets/EdgeStore.java index bc2719c58..b34fb9832 100644 --- a/src/main/java/com/conveyal/r5/streets/EdgeStore.java +++ b/src/main/java/com/conveyal/r5/streets/EdgeStore.java @@ -658,6 +658,8 @@ public StreetRouter.State traverse ( // two link edges in a row, in other words a shortcut. Disallow this. return null; } + // Used for experimental speed adjustments based on LTS + boolean LTS_ONE_LEVEL_HIGHER = false; // Check whether this edge allows the selected mode, considering the request settings. if (streetMode == StreetMode.WALK) { @@ -668,12 +670,19 @@ public StreetRouter.State traverse ( return null; } } else if (streetMode == StreetMode.BICYCLE) { - // If biking is not allowed on this edge, or if the traffic stress is too high, walk the bike. + // If biking is not allowed on this edge, try to walk the bike instead. boolean tryWalking = !getFlag(EdgeFlag.ALLOWS_BIKE); - if (req.bikeTrafficStress > 0 && req.bikeTrafficStress < 4) { - if (getFlag(EdgeFlag.BIKE_LTS_4)) tryWalking = true; - if (req.bikeTrafficStress < 3 && getFlag(EdgeFlag.BIKE_LTS_3)) tryWalking = true; - if (req.bikeTrafficStress < 2 && getFlag(EdgeFlag.BIKE_LTS_2)) tryWalking = true; + // Experimental: if the edge's LTS exceeds the requested limit by 1 level, double the traversal time. + // If the edge's LTS exceeds the requested limit by more than 1 level, try to walk the bike instead. + if (req.bikeTrafficStress == 3) { + if(getFlag(EdgeFlag.BIKE_LTS_4)) LTS_ONE_LEVEL_HIGHER = true; + } else if (req.bikeTrafficStress == 2) { + if (getFlag(EdgeFlag.BIKE_LTS_3)) LTS_ONE_LEVEL_HIGHER = true; + else if (getFlag(BIKE_LTS_4)) tryWalking = true; + } else if (req.bikeTrafficStress == 1) { + if (getFlag(EdgeFlag.BIKE_LTS_2)) LTS_ONE_LEVEL_HIGHER = true; + else if (getFlag(BIKE_LTS_3)) tryWalking = true; + else if (getFlag(BIKE_LTS_4)) tryWalking = true; } if (tryWalking) { if (!getFlag(EdgeFlag.ALLOWS_PEDESTRIAN)) { @@ -689,6 +698,7 @@ public StreetRouter.State traverse ( s1.streetMode = streetMode; int traverseTimeSeconds = timeCalculator.traversalTimeSeconds(this, streetMode, req); + if (LTS_ONE_LEVEL_HIGHER) traverseTimeSeconds *= 2; // This was rounding up, now truncating ... maybe change back for consistency? // int roundedTime = (int) Math.ceil(time); From 7689347ce38394c1f986e2d915b212f045baf609 Mon Sep 17 00:00:00 2001 From: ansons Date: Fri, 18 Aug 2023 16:04:42 -0400 Subject: [PATCH 5/7] Match only features within distance limit Without this change an edge can be matched to a feature if their bounding boxes overlap, even if they are far apart. --- .../r5/analyst/scenario/ShapefileLts.java | 7 ++--- .../r5/shapefile/ShapefileMatcher.java | 31 +++++++++++++------ .../r5/shapefile/ShapefileMatcherMain.java | 2 +- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/conveyal/r5/analyst/scenario/ShapefileLts.java b/src/main/java/com/conveyal/r5/analyst/scenario/ShapefileLts.java index 10279b002..e4086df5d 100644 --- a/src/main/java/com/conveyal/r5/analyst/scenario/ShapefileLts.java +++ b/src/main/java/com/conveyal/r5/analyst/scenario/ShapefileLts.java @@ -4,7 +4,6 @@ import com.conveyal.analysis.datasource.DataSourceException; import com.conveyal.file.FileStorageFormat; import com.conveyal.file.FileStorageKey; -import com.conveyal.r5.rastercost.ElevationLoader; import com.conveyal.r5.shapefile.ShapefileMatcher; import com.conveyal.r5.transit.TransportNetwork; import com.conveyal.r5.transit.TransportNetworkCache; @@ -24,8 +23,6 @@ */ public class ShapefileLts extends Modification { - public String ltsDataSource; - /** * ID of the linear shapefile DataSource containing bicycle LTS to be matched to streets. * We must assume its type because the workers don't have access to the DataStore metadata. @@ -35,6 +32,8 @@ public class ShapefileLts extends Modification { /** The name of the numeric attribute within the ltsDataSource containing LTS values from 1-4. */ public String ltsAttribute = "lts"; + public double matchLimitMeters = 3.0; + private FileStorageKey fileStorageKey; private File localFile; @@ -61,7 +60,7 @@ public boolean apply (TransportNetwork network) { network.streetLayer.edgeStore.flags = new TIntArrayList(network.streetLayer.edgeStore.flags); ShapefileMatcher shapefileMatcher = new ShapefileMatcher(network.streetLayer); try { - shapefileMatcher.match(localFile.getAbsolutePath(), ltsAttribute); + shapefileMatcher.match(localFile.getAbsolutePath(), ltsAttribute, matchLimitMeters); } catch (Exception e) { addError(ExceptionUtils.shortAndLongString(e)); } diff --git a/src/main/java/com/conveyal/r5/shapefile/ShapefileMatcher.java b/src/main/java/com/conveyal/r5/shapefile/ShapefileMatcher.java index d3486377a..d69fa4050 100644 --- a/src/main/java/com/conveyal/r5/shapefile/ShapefileMatcher.java +++ b/src/main/java/com/conveyal/r5/shapefile/ShapefileMatcher.java @@ -1,14 +1,17 @@ package com.conveyal.r5.shapefile; +import com.conveyal.r5.common.SphericalDistanceLibrary; import com.conveyal.r5.streets.EdgeStore; import com.conveyal.r5.streets.StreetLayer; import com.conveyal.r5.util.LambdaCounter; import com.conveyal.r5.util.ShapefileReader; import org.locationtech.jts.algorithm.distance.DiscreteHausdorffDistance; +import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.LineString; import org.locationtech.jts.geom.MultiLineString; import org.locationtech.jts.index.strtree.STRtree; +import org.locationtech.jts.operation.distance.DistanceOp; import org.opengis.feature.simple.SimpleFeature; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,7 +21,6 @@ import java.util.List; import java.util.stream.IntStream; -import static com.conveyal.r5.streets.EdgeStore.intToLts; import static com.conveyal.r5.streets.EdgeStore.EdgeFlag.BIKE_LTS_EXPLICIT; /** @@ -59,6 +61,8 @@ public class ShapefileMatcher { private StreetLayer streets; private int ltsAttributeIndex = -1; + private double matchLimitMeters; + public ShapefileMatcher (StreetLayer streets) { this.streets = streets; } @@ -72,12 +76,15 @@ public ShapefileMatcher (StreetLayer streets) { * OSM_INFERRED, OSM_EXPLICIT, SHAPEFILE_MATCH etc. This could also apply to things like speeds and slopes. * The values could be retained only for the duration of network building unless we have a reason to keep them. */ - public void match (String shapefileName, String attributeName) { + public void match (String shapefileName, String attributeName, double matchLimitMeters) { try { indexFeatures(shapefileName, attributeName); } catch (Throwable t) { throw new RuntimeException("Could not load and index shapefile.", t); } + + this.matchLimitMeters = matchLimitMeters; + LOG.info("Matching edges and setting bike LTS flags..."); // Even single-threaded this is pretty fast for small extracts, but it's readily paralellized. final LambdaCounter edgePairCounter = @@ -107,19 +114,23 @@ public void match (String shapefileName, String attributeName) { streets.edgeStore.nEdgePairs() - edgePairCounter.getCount()); } - // Match metric is currently Hausdorff distance, eventually replace with something that accounts for overlap length. + // For features within the given matchLimitMeters, choose the closest based on Hausdorff distance. + // This could eventually be replaced with something that accounts for overlap length. private SimpleFeature findBestMatch (LineString edgeGeometry) { SimpleFeature bestFeature = null; double bestDistance = Double.POSITIVE_INFINITY; List features = featureIndex.query(edgeGeometry.getEnvelopeInternal()); for (SimpleFeature feature : features) { - // Note that we're using unprojected coordinates so x distance is exaggerated realtive to y. - DiscreteHausdorffDistance dhd = new DiscreteHausdorffDistance(extractLineString(feature), edgeGeometry); - double distance = dhd.distance(); - // distance = overlap(extractLineString(feature), edgeGeometry); - if (bestDistance > distance) { - bestDistance = distance; - bestFeature = feature; + Coordinate[] nearestPoints = new DistanceOp(extractLineString(feature), edgeGeometry).nearestPoints(); + if (SphericalDistanceLibrary.fastDistance(nearestPoints[0], nearestPoints[1]) < matchLimitMeters) { + // Note that we're using unprojected coordinates so x distance is exaggerated relative to y. + DiscreteHausdorffDistance dhd = new DiscreteHausdorffDistance(extractLineString(feature), edgeGeometry); + double distance = dhd.distance(); + // distance = overlap(extractLineString(feature), edgeGeometry); + if (bestDistance > distance) { + bestDistance = distance; + bestFeature = feature; + } } } return bestFeature; diff --git a/src/main/java/com/conveyal/r5/shapefile/ShapefileMatcherMain.java b/src/main/java/com/conveyal/r5/shapefile/ShapefileMatcherMain.java index df86b3f9d..8e5aea982 100644 --- a/src/main/java/com/conveyal/r5/shapefile/ShapefileMatcherMain.java +++ b/src/main/java/com/conveyal/r5/shapefile/ShapefileMatcherMain.java @@ -21,7 +21,7 @@ public class ShapefileMatcherMain { public static void main (String[] args) throws Throwable { StreetLayer streetLayer = loadStreetLayer(); ShapefileMatcher shapefileMatcher = new ShapefileMatcher(streetLayer); - shapefileMatcher.match(SHAPE_FILE, SHAPE_FILE_ATTRIBUTE); + shapefileMatcher.match(SHAPE_FILE, SHAPE_FILE_ATTRIBUTE, 2.0); } private static StreetLayer loadStreetLayer () { From a566196b36c7170cfd5d5b467ca6fb0ad1a64249 Mon Sep 17 00:00:00 2001 From: ansons Date: Fri, 18 Aug 2023 16:05:46 -0400 Subject: [PATCH 6/7] Set NETWORK_FORMAT_VERSION to previous commit To trigger network rebuild when we have baked in modifications --- src/main/java/com/conveyal/r5/kryo/KryoNetworkSerializer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/r5/kryo/KryoNetworkSerializer.java b/src/main/java/com/conveyal/r5/kryo/KryoNetworkSerializer.java index 78fd0ca05..8bd884246 100644 --- a/src/main/java/com/conveyal/r5/kryo/KryoNetworkSerializer.java +++ b/src/main/java/com/conveyal/r5/kryo/KryoNetworkSerializer.java @@ -44,7 +44,7 @@ public abstract class KryoNetworkSerializer { * the serialization format itself does not change. This will ensure newer workers will not load cached older files. * We considered using an ISO date string as the version but that could get confusing when seen in filenames. */ - public static final String NETWORK_FORMAT_VERSION = "nv2"; + public static final String NETWORK_FORMAT_VERSION = "7689347c"; public static final byte[] HEADER = "R5NETWORK".getBytes(); From 7896ab1984ff3716dfa7a4bc82f775eda6427a4e Mon Sep 17 00:00:00 2001 From: ansons Date: Fri, 18 Aug 2023 17:24:46 -0400 Subject: [PATCH 7/7] Build changes Bringing up to date with dev --- .github/workflows/cypress-integration.yml | 9 ++++++--- .github/workflows/gradle.yml | 3 --- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cypress-integration.yml b/.github/workflows/cypress-integration.yml index 150a1d209..0fd16c257 100644 --- a/.github/workflows/cypress-integration.yml +++ b/.github/workflows/cypress-integration.yml @@ -22,10 +22,13 @@ jobs: with: fetch-depth: 0 path: r5 - - uses: gradle/gradle-build-action@v2.4.0 - with: - gradle-version: 7.6.1 # Build .jar and copy to ./ui directory (along with config file) + # Cache Gradle dependencies to speed up the shadow jar build + - uses: actions/cache@v1 + id: cache + with: + path: ~/.gradle/caches + key: gradle-caches - uses: actions/setup-java@v1 with: java-version: 11 diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 6c6b0911f..24f08b055 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -31,9 +31,6 @@ jobs: - uses: actions/checkout@v2.3.2 with: fetch-depth: 0 - - uses: gradle/gradle-build-action@v2.4.0 - with: - gradle-version: 7.6.1 # Java setup step completes very fast, no need to run in a preconfigured docker container. - name: Set up JDK 11 uses: actions/setup-java@v3