From 782adc1acf4a5ea11c407ada7d2ef48cb85036bf Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Mon, 30 Nov 2020 15:49:22 -0500 Subject: [PATCH 1/6] feat(graphql): add graphql fetcher for shapes as encoded polylines re ibi-group/datatools-ui#627 --- .../gtfs/graphql/GraphQLGtfsSchema.java | 23 +- .../graphql/fetchers/PolylineFetcher.java | 87 +++++ .../com/conveyal/gtfs/util/PolylineUtils.java | 325 ++++++++++++++++++ .../gtfs/graphql/GTFSGraphQLTest.java | 6 + .../conveyal/gtfs/util/PolylineUtilsTest.java | 109 ++++++ src/test/resources/graphql/feedPolylines.txt | 9 + .../GTFSGraphQLTest/canFetchPolylines-0.json | 11 + 7 files changed, 568 insertions(+), 2 deletions(-) create mode 100644 src/main/java/com/conveyal/gtfs/graphql/fetchers/PolylineFetcher.java create mode 100644 src/main/java/com/conveyal/gtfs/util/PolylineUtils.java create mode 100644 src/test/java/com/conveyal/gtfs/util/PolylineUtilsTest.java create mode 100644 src/test/resources/graphql/feedPolylines.txt create mode 100644 src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchPolylines-0.json diff --git a/src/main/java/com/conveyal/gtfs/graphql/GraphQLGtfsSchema.java b/src/main/java/com/conveyal/gtfs/graphql/GraphQLGtfsSchema.java index 4bde76b85..f99a9e5d4 100644 --- a/src/main/java/com/conveyal/gtfs/graphql/GraphQLGtfsSchema.java +++ b/src/main/java/com/conveyal/gtfs/graphql/GraphQLGtfsSchema.java @@ -5,6 +5,7 @@ import com.conveyal.gtfs.graphql.fetchers.JDBCFetcher; import com.conveyal.gtfs.graphql.fetchers.MapFetcher; import com.conveyal.gtfs.graphql.fetchers.NestedJDBCFetcher; +import com.conveyal.gtfs.graphql.fetchers.PolylineFetcher; import com.conveyal.gtfs.graphql.fetchers.RowCountFetcher; import com.conveyal.gtfs.graphql.fetchers.SQLColumnFetcher; import com.conveyal.gtfs.graphql.fetchers.SourceObjectFetcher; @@ -164,6 +165,12 @@ public class GraphQLGtfsSchema { .field(MapFetcher.field("point_type", GraphQLInt)) .build(); + // Represents a set of rows from shapes.txt joined by shape_id + public static final GraphQLObjectType shapeType = newObject().name("shape") + .field(string("shape_id")) + .field(string("polyline")) + .build(); + // Represents rows from frequencies.txt public static final GraphQLObjectType frequencyType = newObject().name("frequency") @@ -644,6 +651,20 @@ public class GraphQLGtfsSchema { // DataFetchers can either be class instances implementing the interface, or a static function reference .dataFetcher(new JDBCFetcher("patterns")) .build()) + .field(newFieldDefinition() + .name("shapes") + .type(new GraphQLList(shapeType)) + .argument(intArg(ID_ARG)) + .argument(intArg(LIMIT_ARG)) + .argument(intArg(OFFSET_ARG)) + .argument(floatArg(MIN_LAT)) + .argument(floatArg(MIN_LON)) + .argument(floatArg(MAX_LAT)) + .argument(floatArg(MAX_LON)) + .argument(multiStringArg("shape_id")) + // DataFetchers can either be class instances implementing the interface, or a static function reference + .dataFetcher(new PolylineFetcher()) + .build()) // Then the fields for the sub-tables within the feed (loaded directly from GTFS). .field(newFieldDefinition() .name("agency") @@ -798,8 +819,6 @@ public class GraphQLGtfsSchema { .newSchema() .query(feedQuery) // .query(patternsForStopQuery) - // TODO: Add mutations. - // .mutation(someMutation) .build(); diff --git a/src/main/java/com/conveyal/gtfs/graphql/fetchers/PolylineFetcher.java b/src/main/java/com/conveyal/gtfs/graphql/fetchers/PolylineFetcher.java new file mode 100644 index 000000000..eb015e309 --- /dev/null +++ b/src/main/java/com/conveyal/gtfs/graphql/fetchers/PolylineFetcher.java @@ -0,0 +1,87 @@ +package com.conveyal.gtfs.graphql.fetchers; + +import com.conveyal.gtfs.graphql.GTFSGraphQL; +import com.conveyal.gtfs.util.PolylineUtils; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import org.apache.commons.dbutils.DbUtils; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.Point; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * GraphQL fetcher to get encoded polylines for all shapes in a GTFS feed. + */ +public class PolylineFetcher implements DataFetcher { + public static final Logger LOG = LoggerFactory.getLogger(PolylineFetcher.class); + + @Override + public Object get(DataFetchingEnvironment environment) { + GeometryFactory gf = new GeometryFactory(); + List shapes = new ArrayList<>(); + Map parentFeedMap = environment.getSource(); + String namespace = (String) parentFeedMap.get("namespace"); + Connection connection = null; + try { + connection = GTFSGraphQL.getConnection(); + Statement statement = connection.createStatement(); + String sql = String.format( + "select shape_id, shape_pt_lon, shape_pt_lat from %s.shapes order by shape_id", + namespace + ); + LOG.info("SQL: {}", sql); + if (statement.execute(sql)) { + ResultSet result = statement.getResultSet(); + String currentShapeId = null; + String nextShapeId; + List shapePoints = new ArrayList<>(); + while (result.next()) { + // Get values from SQL row. + nextShapeId = result.getString(1); + double lon = result.getDouble(2); + double lat = result.getDouble(3); + if (currentShapeId != null && !nextShapeId.equals(currentShapeId)) { + // Finish current shape if new shape_id is encountered. + shapes.add(new Shape(currentShapeId, shapePoints)); + // Start building new shape. + shapePoints = new ArrayList<>(); + } + // Update current shape_id and add shape point to list. + currentShapeId = nextShapeId; + shapePoints.add(gf.createPoint(new Coordinate(lon, lat))); + } + // Add the final shape when result iteration is finished. + shapes.add(new Shape(currentShapeId, shapePoints)); + } + } catch (SQLException e) { + throw new RuntimeException(e); + } finally { + DbUtils.closeQuietly(connection); + } + return shapes; + } + + /** + * Simple class to return shapes for GraphQL response format. + */ + public static class Shape { + public String shape_id; + public String polyline; + + public Shape(String shape_id, List shapePoints) { + this.shape_id = shape_id; + // Encode the shapepoints as a polyline + this.polyline = PolylineUtils.encode(shapePoints, 6); + } + } +} diff --git a/src/main/java/com/conveyal/gtfs/util/PolylineUtils.java b/src/main/java/com/conveyal/gtfs/util/PolylineUtils.java new file mode 100644 index 000000000..bd7fa627e --- /dev/null +++ b/src/main/java/com/conveyal/gtfs/util/PolylineUtils.java @@ -0,0 +1,325 @@ +package com.conveyal.gtfs.util; + +import com.sun.istack.internal.NotNull; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.Point; + +import java.util.ArrayList; +import java.util.List; + +/** + * Polyline utils class contains method that can decode/encode a polyline, simplify a line, and + * more. This code is taken/derived from the mapbox-java project (MIT license): + * + * https://github.com/mapbox/mapbox-java/blob/master/services-geojson/src/main/java/com/mapbox/geojson/utils/PolylineUtils.java + */ +public final class PolylineUtils { + + private PolylineUtils() { + // Prevent initialization of this class + } + + // 1 by default (in the same metric as the point coordinates) + private static final double SIMPLIFY_DEFAULT_TOLERANCE = 1; + + // False by default (excludes distance-based preprocessing step which leads to highest quality + // simplification but runs slower) + private static final boolean SIMPLIFY_DEFAULT_HIGHEST_QUALITY = false; + + /** + * Decodes an encoded path string into a sequence of {@link Point}. + * + * @param encodedPath a String representing an encoded path string + * @param precision OSRMv4 uses 6, OSRMv5 and Google uses 5 + * @return list of {@link Point} making up the line + * @see Part of algorithm came from this source + * @see Part of algorithm came from this source. + * @since 1.0.0 + */ + @NotNull + public static List decode(@NotNull final String encodedPath, int precision) { + GeometryFactory gf = new GeometryFactory(); + int len = encodedPath.length(); + + // OSRM uses precision=6, the default Polyline spec divides by 1E5, capping at precision=5 + double factor = Math.pow(10, precision); + + // For speed we preallocate to an upper bound on the final length, then + // truncate the array before returning. + final List path = new ArrayList<>(); + int index = 0; + int lat = 0; + int lng = 0; + + while (index < len) { + int result = 1; + int shift = 0; + int temp; + do { + temp = encodedPath.charAt(index++) - 63 - 1; + result += temp << shift; + shift += 5; + } + while (temp >= 0x1f); + lat += (result & 1) != 0 ? ~(result >> 1) : (result >> 1); + + result = 1; + shift = 0; + do { + temp = encodedPath.charAt(index++) - 63 - 1; + result += temp << shift; + shift += 5; + } + while (temp >= 0x1f); + lng += (result & 1) != 0 ? ~(result >> 1) : (result >> 1); + + path.add(gf.createPoint(new Coordinate(lng / factor, lat / factor))); + } + + return path; + } + + /** + * Encodes a sequence of Points into an encoded path string. + * + * @param path list of {@link Point}s making up the line + * @param precision OSRMv4 uses 6, OSRMv5 and Google uses 5 + * @return a String representing a path string + * @since 1.0.0 + */ + @NotNull + public static String encode(@NotNull final List path, int precision) { + long lastLat = 0; + long lastLng = 0; + + final StringBuilder result = new StringBuilder(); + + // OSRM uses precision=6, the default Polyline spec divides by 1E5, capping at precision=5 + double factor = Math.pow(10, precision); + + for (final Point point : path) { + long lat = Math.round(point.getY() * factor); + long lng = Math.round(point.getX() * factor); + + long varLat = lat - lastLat; + long varLng = lng - lastLng; + + encode(varLat, result); + encode(varLng, result); + + lastLat = lat; + lastLng = lng; + } + return result.toString(); + } + + private static void encode(long variable, StringBuilder result) { + variable = variable < 0 ? ~(variable << 1) : variable << 1; + while (variable >= 0x20) { + result.append(Character.toChars((int) ((0x20 | (variable & 0x1f)) + 63))); + variable >>= 5; + } + result.append(Character.toChars((int) (variable + 63))); + } + + /* + * Polyline simplification method. It's a direct port of simplify.js to Java. + * See: https://github.com/mourner/simplify-js/blob/master/simplify.js + */ + + /** + * Reduces the number of points in a polyline while retaining its shape, giving a performance + * boost when processing it and also reducing visual noise. + * + * @param points an array of points + * @return an array of simplified points + * @see JavaScript implementation + * @since 1.2.0 + */ + @NotNull + public static List simplify(@NotNull List points) { + return simplify(points, SIMPLIFY_DEFAULT_TOLERANCE, SIMPLIFY_DEFAULT_HIGHEST_QUALITY); + } + + /** + * Reduces the number of points in a polyline while retaining its shape, giving a performance + * boost when processing it and also reducing visual noise. + * + * @param points an array of points + * @param tolerance affects the amount of simplification (in the same metric as the point + * coordinates) + * @return an array of simplified points + * @see JavaScript implementation + * @since 1.2.0 + */ + @NotNull + public static List simplify(@NotNull List points, double tolerance) { + return simplify(points, tolerance, SIMPLIFY_DEFAULT_HIGHEST_QUALITY); + } + + /** + * Reduces the number of points in a polyline while retaining its shape, giving a performance + * boost when processing it and also reducing visual noise. + * + * @param points an array of points + * @param highestQuality excludes distance-based preprocessing step which leads to highest quality + * simplification + * @return an array of simplified points + * @see JavaScript implementation + * @since 1.2.0 + */ + @NotNull + public static List simplify(@NotNull List points, boolean highestQuality) { + return simplify(points, SIMPLIFY_DEFAULT_TOLERANCE, highestQuality); + } + + /** + * Reduces the number of points in a polyline while retaining its shape, giving a performance + * boost when processing it and also reducing visual noise. + * + * @param points an array of points + * @param tolerance affects the amount of simplification (in the same metric as the point + * coordinates) + * @param highestQuality excludes distance-based preprocessing step which leads to highest quality + * simplification + * @return an array of simplified points + * @see JavaScript implementation + * @since 1.2.0 + */ + @NotNull + public static List simplify(@NotNull List points, double tolerance, + boolean highestQuality) { + if (points.size() <= 2) { + return points; + } + + double sqTolerance = tolerance * tolerance; + + points = highestQuality ? points : simplifyRadialDist(points, sqTolerance); + points = simplifyDouglasPeucker(points, sqTolerance); + + return points; + } + + /** + * Square distance between 2 points. + * + * @param p1 first {@link Point} + * @param p2 second Point + * @return square of the distance between two input points + */ + private static double getSqDist(Point p1, Point p2) { + double dx = p1.getX() - p2.getX(); + double dy = p1.getY() - p2.getY(); + return dx * dx + dy * dy; + } + + /** + * Square distance from a point to a segment. + * + * @param point {@link Point} whose distance from segment needs to be determined + * @param p1,p2 points defining the segment + * @return square of the distance between first input point and segment defined by + * other two input points + */ + private static double getSqSegDist(Point point, Point p1, Point p2) { + double horizontal = p1.getX(); + double vertical = p1.getY(); + double diffHorizontal = p2.getX() - horizontal; + double diffVertical = p2.getY() - vertical; + + if (diffHorizontal != 0 || diffVertical != 0) { + double total = ((point.getX() - horizontal) * diffHorizontal + (point.getY() + - vertical) * diffVertical) / (diffHorizontal * diffHorizontal + diffVertical + * diffVertical); + if (total > 1) { + horizontal = p2.getX(); + vertical = p2.getY(); + + } else if (total > 0) { + horizontal += diffHorizontal * total; + vertical += diffVertical * total; + } + } + + diffHorizontal = point.getX() - horizontal; + diffVertical = point.getY() - vertical; + + return diffHorizontal * diffHorizontal + diffVertical * diffVertical; + } + + /** + * Basic distance-based simplification. + * + * @param points a list of points to be simplified + * @param sqTolerance square of amount of simplification + * @return a list of simplified points + */ + private static List simplifyRadialDist(List points, double sqTolerance) { + Point prevPoint = points.get(0); + ArrayList newPoints = new ArrayList<>(); + newPoints.add(prevPoint); + Point point = null; + + for (int i = 1, len = points.size(); i < len; i++) { + point = points.get(i); + + if (getSqDist(point, prevPoint) > sqTolerance) { + newPoints.add(point); + prevPoint = point; + } + } + + if (!prevPoint.equals(point)) { + newPoints.add(point); + } + return newPoints; + } + + private static List simplifyDpStep( + List points, int first, int last, double sqTolerance, List simplified) { + double maxSqDist = sqTolerance; + int index = 0; + + ArrayList stepList = new ArrayList<>(); + + for (int i = first + 1; i < last; i++) { + double sqDist = getSqSegDist(points.get(i), points.get(first), points.get(last)); + if (sqDist > maxSqDist) { + index = i; + maxSqDist = sqDist; + } + } + + if (maxSqDist > sqTolerance) { + if (index - first > 1) { + stepList.addAll(simplifyDpStep(points, first, index, sqTolerance, simplified)); + } + + stepList.add(points.get(index)); + + if (last - index > 1) { + stepList.addAll(simplifyDpStep(points, index, last, sqTolerance, simplified)); + } + } + + return stepList; + } + + /** + * Simplification using Ramer-Douglas-Peucker algorithm. + * + * @param points a list of points to be simplified + * @param sqTolerance square of amount of simplification + * @return a list of simplified points + */ + private static List simplifyDouglasPeucker(List points, double sqTolerance) { + int last = points.size() - 1; + ArrayList simplified = new ArrayList<>(); + simplified.add(points.get(0)); + simplified.addAll(simplifyDpStep(points, 0, last, sqTolerance, simplified)); + simplified.add(points.get(last)); + return simplified; + } +} \ No newline at end of file diff --git a/src/test/java/com/conveyal/gtfs/graphql/GTFSGraphQLTest.java b/src/test/java/com/conveyal/gtfs/graphql/GTFSGraphQLTest.java index 2273f6bc9..59581e42f 100644 --- a/src/test/java/com/conveyal/gtfs/graphql/GTFSGraphQLTest.java +++ b/src/test/java/com/conveyal/gtfs/graphql/GTFSGraphQLTest.java @@ -118,6 +118,12 @@ public void canFetchPatterns() throws IOException { assertThat(queryGraphQL("feedPatterns.txt"), matchesSnapshot()); } + /** Tests that the patterns of a feed can be fetched. */ + @Test(timeout=5000) + public void canFetchPolylines() throws IOException { + assertThat(queryGraphQL("feedPolylines.txt"), matchesSnapshot()); + } + /** Tests that the agencies of a feed can be fetched. */ @Test(timeout=5000) public void canFetchAgencies() throws IOException { diff --git a/src/test/java/com/conveyal/gtfs/util/PolylineUtilsTest.java b/src/test/java/com/conveyal/gtfs/util/PolylineUtilsTest.java new file mode 100644 index 000000000..403ca5e91 --- /dev/null +++ b/src/test/java/com/conveyal/gtfs/util/PolylineUtilsTest.java @@ -0,0 +1,109 @@ +package com.conveyal.gtfs.util; + +import org.junit.Test; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.Point; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static com.conveyal.gtfs.util.PolylineUtils.decode; +import static com.conveyal.gtfs.util.PolylineUtils.encode; +import static com.conveyal.gtfs.util.PolylineUtils.simplify; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class PolylineUtilsTest { + + private static final int PRECISION_6 = 6; + private static final int PRECISION_5 = 5; + + // Delta for Coordinates comparison + private static final double DELTA = 0.000001; + + private static final String SIMPLIFICATION_INPUT = "simplification-input"; + private static final String SIMPLIFICATION_EXPECTED_OUTPUT = "simplification-expected-output"; + + private static final String TEST_LINE + = "_cqeFf~cjVf@p@fA}AtAoB`ArAx@hA`GbIvDiFv@gAh@t@X\\|@z@`@Z\\Xf@Vf@VpA\\tATJ@NBBkC"; + + private static final String TEST_LINE6 = + "qn_iHgp}LzCy@xCsAsC}PoEeD_@{A@uD_@Sg@Je@a@I_@FcAoFyGcCqFgQ{L{CmD"; + + private static final GeometryFactory gf = new GeometryFactory(); + + @Test + public void testDecodePath() { + List latLngs = decode(TEST_LINE, PRECISION_5); + + int expectedLength = 21; + assertEquals("Wrong length.", expectedLength, latLngs.size()); + + Point lastPoint = latLngs.get(expectedLength - 1); +// expectNearNumber(37.76953, lastPoint.getY(), 1e-6); +// expectNearNumber(-122.41488, lastPoint.getX(), 1e-6); + } + + @Test + public void testEncodePath5() { + List path = decode(TEST_LINE, PRECISION_5); + String encoded = encode(path, PRECISION_5); + assertEquals(TEST_LINE, encoded); + } + + @Test + public void testDecodeEncodePath6() { + List path = decode(TEST_LINE6, PRECISION_6); + String encoded = encode(path, PRECISION_6); + assertEquals(TEST_LINE6, encoded); + } + + @Test + public void testEncodeDecodePath6() { + List originalPath = Arrays.asList( + gf.createPoint(new Coordinate(2.2862036, 48.8267868)), + gf.createPoint(new Coordinate(2.4, 48.9)) + ); + + String encoded = encode(originalPath, PRECISION_6); + List path = decode(encoded, PRECISION_6); + assertEquals(originalPath.size(), path.size()); + + for (int i = 0; i < originalPath.size(); i++) { + assertEquals(originalPath.get(i).getY(), path.get(i).getY(), DELTA); + assertEquals(originalPath.get(i).getX(), path.get(i).getX(), DELTA); + } + } + + + @Test + public void decode_neverReturnsNullButRatherAnEmptyList() throws Exception { + List path = decode("", PRECISION_5); + assertNotNull(path); + assertEquals(0, path.size()); + } + + @Test + public void encode_neverReturnsNull() throws Exception { + String encodedString = encode(new ArrayList(), PRECISION_6); + assertNotNull(encodedString); + } + + @Test + public void simplify_neverReturnsNullButRatherAnEmptyList() throws Exception { + List simplifiedPath = simplify(new ArrayList(), PRECISION_6); + assertNotNull(simplifiedPath); + } + + @Test + public void simplify_returnSameListWhenListSizeIsLessThanOrEqualToTwo(){ + final List path = new ArrayList<>(); + path.add(gf.createPoint(new Coordinate(0, 0))); + path.add(gf.createPoint(new Coordinate(10, 0))); + List simplifiedPath = simplify(path, PRECISION_6, true); + assertTrue("Returned list is different from input list", path == simplifiedPath); + } +} \ No newline at end of file diff --git a/src/test/resources/graphql/feedPolylines.txt b/src/test/resources/graphql/feedPolylines.txt new file mode 100644 index 000000000..87a234166 --- /dev/null +++ b/src/test/resources/graphql/feedPolylines.txt @@ -0,0 +1,9 @@ +query ($namespace: String) { + feed(namespace: $namespace) { + feed_version + shapes { + shape_id + polyline + } + } +} diff --git a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchPolylines-0.json b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchPolylines-0.json new file mode 100644 index 000000000..af295dc69 --- /dev/null +++ b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchPolylines-0.json @@ -0,0 +1,11 @@ +{ + "data" : { + "feed" : { + "feed_version" : "1.0", + "shapes" : [ { + "polyline" : "qoeaFlqtgVFLe@b@~AvBfBlBvBfBhCpB", + "shape_id" : "5820f377-f947-4728-ac29-ac0102cbc34e" + } ] + } + } +} \ No newline at end of file From ceb1b1179a5b65e1d3a0a28303d5659b775e6438 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Wed, 2 Dec 2020 09:32:35 -0500 Subject: [PATCH 2/6] refactor: address PR #297 comments --- .../gtfs/graphql/GraphQLGtfsSchema.java | 14 +++----------- .../com/conveyal/gtfs/util/PolylineUtils.java | 19 ++++++------------- .../conveyal/gtfs/util/PolylineUtilsTest.java | 5 +++++ 3 files changed, 14 insertions(+), 24 deletions(-) diff --git a/src/main/java/com/conveyal/gtfs/graphql/GraphQLGtfsSchema.java b/src/main/java/com/conveyal/gtfs/graphql/GraphQLGtfsSchema.java index f99a9e5d4..e752660cb 100644 --- a/src/main/java/com/conveyal/gtfs/graphql/GraphQLGtfsSchema.java +++ b/src/main/java/com/conveyal/gtfs/graphql/GraphQLGtfsSchema.java @@ -166,7 +166,7 @@ public class GraphQLGtfsSchema { .build(); // Represents a set of rows from shapes.txt joined by shape_id - public static final GraphQLObjectType shapeType = newObject().name("shape") + public static final GraphQLObjectType shapeEncodedPolylineType = newObject().name("shapeEncodedPolyline") .field(string("shape_id")) .field(string("polyline")) .build(); @@ -652,16 +652,8 @@ public class GraphQLGtfsSchema { .dataFetcher(new JDBCFetcher("patterns")) .build()) .field(newFieldDefinition() - .name("shapes") - .type(new GraphQLList(shapeType)) - .argument(intArg(ID_ARG)) - .argument(intArg(LIMIT_ARG)) - .argument(intArg(OFFSET_ARG)) - .argument(floatArg(MIN_LAT)) - .argument(floatArg(MIN_LON)) - .argument(floatArg(MAX_LAT)) - .argument(floatArg(MAX_LON)) - .argument(multiStringArg("shape_id")) + .name("shapes_as_polylines") + .type(new GraphQLList(shapeEncodedPolylineType)) // DataFetchers can either be class instances implementing the interface, or a static function reference .dataFetcher(new PolylineFetcher()) .build()) diff --git a/src/main/java/com/conveyal/gtfs/util/PolylineUtils.java b/src/main/java/com/conveyal/gtfs/util/PolylineUtils.java index bd7fa627e..d45d4692c 100644 --- a/src/main/java/com/conveyal/gtfs/util/PolylineUtils.java +++ b/src/main/java/com/conveyal/gtfs/util/PolylineUtils.java @@ -1,6 +1,5 @@ package com.conveyal.gtfs.util; -import com.sun.istack.internal.NotNull; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.Point; @@ -37,8 +36,7 @@ private PolylineUtils() { * @see Part of algorithm came from this source. * @since 1.0.0 */ - @NotNull - public static List decode(@NotNull final String encodedPath, int precision) { + public static List decode(final String encodedPath, int precision) { GeometryFactory gf = new GeometryFactory(); int len = encodedPath.length(); @@ -88,8 +86,7 @@ public static List decode(@NotNull final String encodedPath, int precisio * @return a String representing a path string * @since 1.0.0 */ - @NotNull - public static String encode(@NotNull final List path, int precision) { + public static String encode(final List path, int precision) { long lastLat = 0; long lastLng = 0; @@ -137,8 +134,7 @@ private static void encode(long variable, StringBuilder result) { * @see JavaScript implementation * @since 1.2.0 */ - @NotNull - public static List simplify(@NotNull List points) { + public static List simplify(List points) { return simplify(points, SIMPLIFY_DEFAULT_TOLERANCE, SIMPLIFY_DEFAULT_HIGHEST_QUALITY); } @@ -153,8 +149,7 @@ public static List simplify(@NotNull List points) { * @see JavaScript implementation * @since 1.2.0 */ - @NotNull - public static List simplify(@NotNull List points, double tolerance) { + public static List simplify(List points, double tolerance) { return simplify(points, tolerance, SIMPLIFY_DEFAULT_HIGHEST_QUALITY); } @@ -169,8 +164,7 @@ public static List simplify(@NotNull List points, double tolerance * @see JavaScript implementation * @since 1.2.0 */ - @NotNull - public static List simplify(@NotNull List points, boolean highestQuality) { + public static List simplify(List points, boolean highestQuality) { return simplify(points, SIMPLIFY_DEFAULT_TOLERANCE, highestQuality); } @@ -187,8 +181,7 @@ public static List simplify(@NotNull List points, boolean highestQ * @see JavaScript implementation * @since 1.2.0 */ - @NotNull - public static List simplify(@NotNull List points, double tolerance, + public static List simplify(List points, double tolerance, boolean highestQuality) { if (points.size() <= 2) { return points; diff --git a/src/test/java/com/conveyal/gtfs/util/PolylineUtilsTest.java b/src/test/java/com/conveyal/gtfs/util/PolylineUtilsTest.java index 403ca5e91..e6f91b3c3 100644 --- a/src/test/java/com/conveyal/gtfs/util/PolylineUtilsTest.java +++ b/src/test/java/com/conveyal/gtfs/util/PolylineUtilsTest.java @@ -16,6 +16,11 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +/** + * Contains tests for {@link PolylineUtils}. This code is taken/derived from the mapbox-java project (MIT license): + * + * https://github.com/mapbox/mapbox-java/blob/master/services-geojson/src/test/java/com/mapbox/geojson/utils/PolylineUtilsTest.java + */ public class PolylineUtilsTest { private static final int PRECISION_6 = 6; From d4c3b5916eb6bf1a58f7a52b6d40e4480ffea925 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Wed, 2 Dec 2020 09:41:53 -0500 Subject: [PATCH 3/6] refactor(graphql): update feedPolylines query --- src/test/resources/graphql/feedPolylines.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/resources/graphql/feedPolylines.txt b/src/test/resources/graphql/feedPolylines.txt index 87a234166..3280df385 100644 --- a/src/test/resources/graphql/feedPolylines.txt +++ b/src/test/resources/graphql/feedPolylines.txt @@ -1,7 +1,7 @@ query ($namespace: String) { feed(namespace: $namespace) { feed_version - shapes { + shapes_as_polylines { shape_id polyline } From c67f4b727820bb767b168fb0f0bb093c75c4d1a4 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Wed, 2 Dec 2020 09:49:08 -0500 Subject: [PATCH 4/6] test(graphql): update feedPolylines snapshot --- .../gtfs/graphql/GTFSGraphQLTest/canFetchPolylines-0.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchPolylines-0.json b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchPolylines-0.json index af295dc69..033f1030e 100644 --- a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchPolylines-0.json +++ b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchPolylines-0.json @@ -2,8 +2,8 @@ "data" : { "feed" : { "feed_version" : "1.0", - "shapes" : [ { - "polyline" : "qoeaFlqtgVFLe@b@~AvBfBlBvBfBhCpB", + "shapes_as_polylines" : [ { + "polyline" : "yd`ueApwvugFpAdCuJlJ`]dd@l_@ja@jd@z_@lj@hb@", "shape_id" : "5820f377-f947-4728-ac29-ac0102cbc34e" } ] } From d19b281d0ffae3fca307dfc924af442ad1a9e9b1 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Wed, 2 Dec 2020 14:16:03 -0500 Subject: [PATCH 5/6] refactor(PolylineFetcher): add javadoc about response time --- .../gtfs/graphql/fetchers/PolylineFetcher.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/gtfs/graphql/fetchers/PolylineFetcher.java b/src/main/java/com/conveyal/gtfs/graphql/fetchers/PolylineFetcher.java index eb015e309..c36892631 100644 --- a/src/main/java/com/conveyal/gtfs/graphql/fetchers/PolylineFetcher.java +++ b/src/main/java/com/conveyal/gtfs/graphql/fetchers/PolylineFetcher.java @@ -21,6 +21,15 @@ /** * GraphQL fetcher to get encoded polylines for all shapes in a GTFS feed. + * + * Note: this fetcher was developed to prevent out of memory errors associated with requesting all trip patterns with + * their associated shapes. Previously, attempting to fetch all shapes for a large feed would require a separate SQL + * query for each shape (to join on shape_id) and result in many duplicated shapes in the response. + * + * Running a shapes polyline query on a 2017 Macbook Pro (2.5 GHz Dual-Core Intel Core i7) results in the following: + * - NL feed (11291 shapes): ~12 seconds (but nobody should be editing such a large feed in the editor) + * - TriMet (1302 shapes): ~8 seconds + * - MBTA (1272 shapes): ~4 seconds */ public class PolylineFetcher implements DataFetcher { public static final Logger LOG = LoggerFactory.getLogger(PolylineFetcher.class); @@ -36,7 +45,7 @@ public Object get(DataFetchingEnvironment environment) { connection = GTFSGraphQL.getConnection(); Statement statement = connection.createStatement(); String sql = String.format( - "select shape_id, shape_pt_lon, shape_pt_lat from %s.shapes order by shape_id", + "select shape_id, shape_pt_lon, shape_pt_lat from %s.shapes order by shape_id, shape_pt_sequence", namespace ); LOG.info("SQL: {}", sql); From bbc7ae00e7795ebce57a5ec8ba9beeb9dd797de6 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Thu, 3 Dec 2020 10:15:58 -0500 Subject: [PATCH 6/6] perf(PolylineFetcher): speed up shapes query by splitting into two queries --- .../graphql/fetchers/PolylineFetcher.java | 79 +++++++++++-------- 1 file changed, 46 insertions(+), 33 deletions(-) diff --git a/src/main/java/com/conveyal/gtfs/graphql/fetchers/PolylineFetcher.java b/src/main/java/com/conveyal/gtfs/graphql/fetchers/PolylineFetcher.java index c36892631..f176185c4 100644 --- a/src/main/java/com/conveyal/gtfs/graphql/fetchers/PolylineFetcher.java +++ b/src/main/java/com/conveyal/gtfs/graphql/fetchers/PolylineFetcher.java @@ -12,12 +12,15 @@ import org.slf4j.LoggerFactory; import java.sql.Connection; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; /** * GraphQL fetcher to get encoded polylines for all shapes in a GTFS feed. @@ -27,65 +30,75 @@ * query for each shape (to join on shape_id) and result in many duplicated shapes in the response. * * Running a shapes polyline query on a 2017 Macbook Pro (2.5 GHz Dual-Core Intel Core i7) results in the following: - * - NL feed (11291 shapes): ~12 seconds (but nobody should be editing such a large feed in the editor) - * - TriMet (1302 shapes): ~8 seconds - * - MBTA (1272 shapes): ~4 seconds + * - NL feed (11291 shapes, 4.2M rows): ~13 seconds (but hopefully nobody is editing such a large feed in the editor) + * - TriMet (1909 shapes, 719K rows): ~3 seconds + * - MBTA (1272 shapes, 268K rows): 1-2 seconds + * + * This was originally handled in a single query to the shapes table that sorted on shape_id and shape_pt_sequence, but + * per Evan Siroky rec, we've split into two queries to reduce the number of sort operations significantly. For example: + * + * There are 719k lines in the TriMet shapes.txt file with 1,909 unique shape IDs (so about 377 points per + * shape on average). With that, the numbers become: + * + * 1 * 719k * log2(719k) = 13.9m operations + * 1909 * 377 * log2(377) = 6.1m operations + * + * NOTE: This doesn't provide much of a benefit for the NL feed (in fact, it appears to have a disbenefit/slowdown + * of about 1-2 seconds on average), but for the other feeds tested the double query approach was about twice as fast. */ public class PolylineFetcher implements DataFetcher { - public static final Logger LOG = LoggerFactory.getLogger(PolylineFetcher.class); + private static final Logger LOG = LoggerFactory.getLogger(PolylineFetcher.class); + private static final GeometryFactory gf = new GeometryFactory(); @Override public Object get(DataFetchingEnvironment environment) { - GeometryFactory gf = new GeometryFactory(); - List shapes = new ArrayList<>(); Map parentFeedMap = environment.getSource(); String namespace = (String) parentFeedMap.get("namespace"); Connection connection = null; try { + List shapes = new ArrayList<>(); connection = GTFSGraphQL.getConnection(); - Statement statement = connection.createStatement(); - String sql = String.format( - "select shape_id, shape_pt_lon, shape_pt_lat from %s.shapes order by shape_id, shape_pt_sequence", + // First, collect all shape ids. + Set shapeIds = new HashSet<>(); + String getShapeIdsSql = String.format("select distinct shape_id from %s.shapes", namespace); + LOG.info(getShapeIdsSql); + ResultSet shapeIdsResult = connection.createStatement().executeQuery(getShapeIdsSql); + while (shapeIdsResult.next()) { + shapeIds.add(shapeIdsResult.getString(1)); + } + // Next, iterate over shape ids and build shapes progressively. + PreparedStatement shapePointsStatement = connection.prepareStatement(String.format( + "select shape_pt_lon, shape_pt_lat from %s.shapes where shape_id = ? order by shape_pt_sequence", namespace - ); - LOG.info("SQL: {}", sql); - if (statement.execute(sql)) { - ResultSet result = statement.getResultSet(); - String currentShapeId = null; - String nextShapeId; + )); + for (String shapeId : shapeIds) { + shapePointsStatement.setString(1, shapeId); + ResultSet result = shapePointsStatement.executeQuery(); List shapePoints = new ArrayList<>(); while (result.next()) { - // Get values from SQL row. - nextShapeId = result.getString(1); - double lon = result.getDouble(2); - double lat = result.getDouble(3); - if (currentShapeId != null && !nextShapeId.equals(currentShapeId)) { - // Finish current shape if new shape_id is encountered. - shapes.add(new Shape(currentShapeId, shapePoints)); - // Start building new shape. - shapePoints = new ArrayList<>(); - } - // Update current shape_id and add shape point to list. - currentShapeId = nextShapeId; + // Get lon/lat values from SQL row. + double lon = result.getDouble(1); + double lat = result.getDouble(2); shapePoints.add(gf.createPoint(new Coordinate(lon, lat))); } - // Add the final shape when result iteration is finished. - shapes.add(new Shape(currentShapeId, shapePoints)); + // Construct/add shape once all points have been gathered. + shapes.add(new Shape(shapeId, shapePoints)); } + // Finally, return the shapes with encoded polylines. + return shapes; } catch (SQLException e) { throw new RuntimeException(e); } finally { DbUtils.closeQuietly(connection); } - return shapes; } /** * Simple class to return shapes for GraphQL response format. */ public static class Shape { - public String shape_id; - public String polyline; + public final String shape_id; + public final String polyline; public Shape(String shape_id, List shapePoints) { this.shape_id = shape_id;