From 08449488aeded5f7f7c7e8fb34992f8d8331f590 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 5 Jun 2015 18:50:37 +0200 Subject: [PATCH 1/4] moved classes representing requests and responses over to OTP this prevents a circular dependency --- app/utils/AnalystClusterRequest.java | 52 ---- app/utils/ClusterGraphService.java | 327 ------------------------- app/utils/OneToManyProfileRequest.java | 33 --- app/utils/OneToManyRequest.java | 23 -- app/utils/PointSetDatastore.java | 160 ------------ app/utils/ResultEnvelope.java | 135 ---------- 6 files changed, 730 deletions(-) delete mode 100644 app/utils/AnalystClusterRequest.java delete mode 100644 app/utils/ClusterGraphService.java delete mode 100644 app/utils/OneToManyProfileRequest.java delete mode 100644 app/utils/OneToManyRequest.java delete mode 100644 app/utils/PointSetDatastore.java delete mode 100644 app/utils/ResultEnvelope.java diff --git a/app/utils/AnalystClusterRequest.java b/app/utils/AnalystClusterRequest.java deleted file mode 100644 index f3cbfba..0000000 --- a/app/utils/AnalystClusterRequest.java +++ /dev/null @@ -1,52 +0,0 @@ -package utils; - -import org.opentripplanner.analyst.PointFeature; - -import java.io.Serializable; - -/** - * Marker interface for requests sent to an SPTWorker. - * @author matthewc - * - */ -public abstract class AnalystClusterRequest implements Serializable { - /** The ID of the destinations pointset */ - public String destinationPointsetId; - - /** The ID of the graph against which to calculate this request */ - public String graphId; - - /** The job ID this is associated with */ - public String jobId; - - /** The id of this particular origin */ - public String id; - - /** To where should the result be POSTed */ - public String directOutputUrl; - - /** - * To what queue should the notification of the result be delivered? - */ - public String outputQueue; - - /** - * Where should the job be saved? - */ - public String outputLocation; - - /** Should times be included in the results (i.e. ResultSetWithTimes rather than ResultSet) */ - public boolean includeTimes = false; - - /** Is this a profile request? */ - public boolean profile; - - public AnalystClusterRequest(String destinationPointsetId, String graphId, boolean profile) { - this.destinationPointsetId = destinationPointsetId; - this.graphId = graphId; - this.profile = profile; - } - - /** used for deserialization from JSON */ - public AnalystClusterRequest () { /* do nothing */ } -} diff --git a/app/utils/ClusterGraphService.java b/app/utils/ClusterGraphService.java deleted file mode 100644 index f7599ab..0000000 --- a/app/utils/ClusterGraphService.java +++ /dev/null @@ -1,327 +0,0 @@ -package utils; - -import com.amazonaws.AmazonServiceException; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.profile.ProfileCredentialsProvider; -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.model.S3Object; -import com.google.common.collect.Maps; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.opentripplanner.graph_builder.GraphBuilder; -import org.opentripplanner.routing.graph.Graph; -import org.opentripplanner.routing.impl.DefaultStreetVertexIndexFactory; -import org.opentripplanner.routing.services.GraphService; -import org.opentripplanner.routing.services.GraphSource; -import org.opentripplanner.routing.services.GraphSource.Factory; -import org.opentripplanner.standalone.CommandLineParameters; -import org.opentripplanner.standalone.Router; - -import java.io.*; -import java.util.Collection; -import java.util.Enumeration; -import java.util.Map; -import java.util.zip.ZipEntry; -import java.util.zip.ZipException; -import java.util.zip.ZipFile; -import java.util.zip.ZipOutputStream; - - -public class ClusterGraphService extends GraphService { - - static File GRAPH_DIR = new File("cache", "graphs"); - - private String graphBucket; - - private Boolean workOffline = false; - private AmazonS3Client s3; - - // don't use more than 60% of free memory to cache graphs - private Map graphMap = Maps.newConcurrentMap(); - - @Override - public synchronized Router getRouter(String graphId) { - - GRAPH_DIR.mkdirs(); - - if(!graphMap.containsKey(graphId)) { - - try { - if (!bucketCached(graphId)) { - if(!workOffline) { - downloadGraphSourceFiles(graphId, GRAPH_DIR); - } - } - } catch (IOException e) { - e.printStackTrace(); - } - - CommandLineParameters params = new CommandLineParameters(); - params.build = new File(GRAPH_DIR, graphId); - params.inMemory = true; - GraphBuilder gbt = GraphBuilder.forDirectory(params, params.build); - gbt.run(); - - Graph g = gbt.getGraph(); - - g.routerId = graphId; - - g.index(new DefaultStreetVertexIndexFactory()); - - g.index.clusterStopsAsNeeded(); - - Router r = new Router(graphId, g); - - // temporarily disable graph caching so we don't run out of RAM. - // Long-term we will use an actual cache for this. - //graphMap.put(graphId,r); - - return r; - - } - else { - return graphMap.get(graphId); - } - } - - public ClusterGraphService(String s3CredentialsFilename, Boolean workOffline, String bucket) { - - if(!workOffline) { - if (s3CredentialsFilename != null) { - AWSCredentials creds = new ProfileCredentialsProvider(s3CredentialsFilename, "default").getCredentials(); - s3 = new AmazonS3Client(creds); - } - else { - // S3 credentials propagated to EC2 instances via IAM roles - s3 = new AmazonS3Client(); - } - - this.graphBucket = bucket; - } - - this.workOffline = workOffline; - } - - // adds either a zip file or graph directory to S3, or local cache for offline use - public void addGraphFile(File graphFile) throws IOException { - - String graphId = graphFile.getName(); - - if(graphId.endsWith(".zip")) - graphId = graphId.substring(0, graphId.length() - 4); - - File graphDir = new File(GRAPH_DIR, graphId); - - if (graphDir.exists()) { - if (graphDir.list().length == 0) { - graphDir.delete(); - } - else { - return; - } - } - - // if we're here the directory has either been deleted or never existed - graphDir.mkdirs(); - - File graphDataZip = new File(GRAPH_DIR, graphId + ".zip"); - - // if directory zip contents store as zip - // either way ensure there is an extracted copy in the local cache - if(graphFile.isDirectory()) { - FileUtils.copyDirectory(graphFile, graphDir); - - zipGraphDir(graphDir, graphDataZip); - } - else if(graphFile.getName().endsWith(".zip")) { - FileUtils.copyFile(graphFile, graphDataZip); - unpackGraphZip(graphDataZip, graphDir, false); - } - else { - graphDataZip = null; - } - - if(!workOffline && graphDataZip != null) { - // only upload if it's not there already - try { - s3.getObject(graphBucket, graphId + ".zip"); - } catch (AmazonServiceException e) { - s3.putObject(graphBucket, graphId+".zip", graphDataZip); - } - } - - graphDataZip.delete(); - - } - - public synchronized File getZippedGraph(String graphId) throws IOException { - - File graphDataDir = new File(GRAPH_DIR, graphId); - - File graphZipFile = new File(GRAPH_DIR, graphId + ".zip"); - - if(!graphDataDir.exists() && graphDataDir.isDirectory()) { - - FileOutputStream fileOutputStream = new FileOutputStream(graphZipFile); - ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream); - - byte[] buffer = new byte[1024]; - - for(File f : graphDataDir.listFiles()) { - ZipEntry zipEntry = new ZipEntry(f.getName()); - zipOutputStream.putNextEntry(zipEntry); - FileInputStream fileInput = new FileInputStream(f); - - int len; - while ((len = fileInput.read(buffer)) > 0) { - zipOutputStream.write(buffer, 0, len); - } - - fileInput.close(); - zipOutputStream.closeEntry(); - } - - zipOutputStream.close(); - - return graphZipFile; - - } - - return null; - - } - - private static boolean bucketCached(String graphId) throws IOException { - File graphData = new File(GRAPH_DIR, graphId); - - // check if cached but only as zip - if(!graphData.exists()) { - File graphDataZip = new File(GRAPH_DIR, graphId + ".zip"); - - if(graphDataZip.exists()) { - zipGraphDir(graphData, graphDataZip); - } - } - - - return graphData.exists() && graphData.isDirectory(); - } - - private void downloadGraphSourceFiles(String graphId, File dir) throws IOException { - - File graphCacheDir = dir; - if (!graphCacheDir.exists()) - graphCacheDir.mkdirs(); - - File graphZipFile = new File(graphCacheDir, graphId + ".zip"); - - File extractedGraphDir = new File(graphCacheDir, graphId); - - if (extractedGraphDir.exists()) { - FileUtils.deleteDirectory(extractedGraphDir); - } - - extractedGraphDir.mkdirs(); - - S3Object graphZip = s3.getObject(graphBucket, graphId+".zip"); - - InputStream zipFileIn = graphZip.getObjectContent(); - - OutputStream zipFileOut = new FileOutputStream(graphZipFile); - - IOUtils.copy(zipFileIn, zipFileOut); - IOUtils.closeQuietly(zipFileIn); - IOUtils.closeQuietly(zipFileOut); - - unpackGraphZip(graphZipFile, extractedGraphDir); - } - - private static void unpackGraphZip(File graphZipFile, File extractedGraphDir) throws ZipException, IOException { - // delete by default - unpackGraphZip(graphZipFile, extractedGraphDir, true); - } - - private static void unpackGraphZip(File graphZipFile, File extractedGraphDir, boolean delete) throws ZipException, IOException { - - ZipFile zipFile = new ZipFile(graphZipFile); - - Enumeration entries = zipFile.entries(); - - while (entries.hasMoreElements()) { - - ZipEntry entry = entries.nextElement(); - File entryDestination = new File(extractedGraphDir, entry.getName()); - - entryDestination.getParentFile().mkdirs(); - - if (entry.isDirectory()) - entryDestination.mkdirs(); - else { - InputStream entryFileIn = zipFile.getInputStream(entry); - OutputStream entryFileOut = new FileOutputStream(entryDestination); - IOUtils.copy(entryFileIn, entryFileOut); - IOUtils.closeQuietly(entryFileIn); - IOUtils.closeQuietly(entryFileOut); - } - } - - zipFile.close(); - - if (delete) { - graphZipFile.delete(); - } - } - - private static void zipGraphDir(File graphDirectory, File zipGraphFile) throws IOException { - - FileOutputStream fileOutputStream = new FileOutputStream(zipGraphFile); - ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream); - - byte[] buffer = new byte[1024]; - - for(File f : graphDirectory.listFiles()) { - if (f.isDirectory()) - continue; - - ZipEntry zipEntry = new ZipEntry(f.getName()); - zipOutputStream.putNextEntry(zipEntry); - FileInputStream fileInput = new FileInputStream(f); - - int len; - while ((len = fileInput.read(buffer)) > 0) { - zipOutputStream.write(buffer, 0, len); - } - - fileInput.close(); - zipOutputStream.closeEntry(); - } - - zipOutputStream.close(); - } - - - @Override - public int evictAll() { - graphMap.clear(); - return 0; - } - - @Override - public Collection getRouterIds() { - return graphMap.keySet(); - } - - @Override - public Factory getGraphSourceFactory() { - return null; - } - - @Override - public boolean registerGraph(String arg0, GraphSource arg1) { - return false; - } - - @Override - public void setDefaultRouterId(String arg0) { - } -} diff --git a/app/utils/OneToManyProfileRequest.java b/app/utils/OneToManyProfileRequest.java deleted file mode 100644 index d20c8ef..0000000 --- a/app/utils/OneToManyProfileRequest.java +++ /dev/null @@ -1,33 +0,0 @@ -package utils; - -import org.opentripplanner.analyst.PointFeature; -import org.opentripplanner.profile.ProfileRequest; - -import java.io.Serializable; - -/** - * Represents a request for a profile search from a given point to all vertices in the graph. - * @author mattwigway - * - */ -public class OneToManyProfileRequest extends AnalystClusterRequest implements Serializable { - public ProfileRequest options; - - /** used in single point mode with origin specified by options */ - public OneToManyProfileRequest(String to, ProfileRequest options, String graphId) { - super(to, graphId, true); - try { - this.options = options.clone(); - } catch (CloneNotSupportedException e) { - e.printStackTrace(); - throw new RuntimeException(e); - } - this.options.analyst = true; - - this.options.toLat = this.options.fromLat; - this.options.toLon = this.options.fromLon; - } - - /** used for deserialization from JSON */ - public OneToManyProfileRequest() { /* empty */ } -} diff --git a/app/utils/OneToManyRequest.java b/app/utils/OneToManyRequest.java deleted file mode 100644 index 26daed3..0000000 --- a/app/utils/OneToManyRequest.java +++ /dev/null @@ -1,23 +0,0 @@ -package utils; - -import org.opentripplanner.analyst.PointFeature; -import org.opentripplanner.common.model.GenericLocation; -import org.opentripplanner.routing.core.RoutingRequest; - -import java.io.Serializable; - -public class OneToManyRequest extends AnalystClusterRequest implements Serializable { - public RoutingRequest options; - - /** used for single point requests with from specified by options */ - public OneToManyRequest(String to, RoutingRequest options, String graphId) { - super(to, graphId, false); - - this.options = options.clone(); - this.options.batch = true; - this.options.rctx = null; - } - - /** used for deserialization from JSON */ - public OneToManyRequest () { /* nothing */ } -} diff --git a/app/utils/PointSetDatastore.java b/app/utils/PointSetDatastore.java deleted file mode 100644 index fce3b4a..0000000 --- a/app/utils/PointSetDatastore.java +++ /dev/null @@ -1,160 +0,0 @@ -package utils; - -import com.amazonaws.AmazonServiceException; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.profile.ProfileCredentialsProvider; -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.S3Object; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.io.ByteStreams; -import org.apache.commons.io.FileUtils; -import org.opentripplanner.analyst.PointSet; -import org.opentripplanner.analyst.PointSetCache; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.List; -import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; - -public class PointSetDatastore extends PointSetCache { - - static private File POINT_DIR = new File("cache", "pointsets"); - private String pointsetBucket; - - private AmazonS3Client s3; - private final Boolean workOffline; - - public PointSetDatastore(Integer maxCacheSize, String s3CredentialsFilename, - Boolean workOffline, String pointsetBucket){ - - super(); - - // allow the data store to work offline with cached data and skip S3 connection - this.workOffline = workOffline; - - this.pointsetBucket = pointsetBucket; - - if(!this.workOffline) { - if (s3CredentialsFilename != null) { - AWSCredentials creds = new ProfileCredentialsProvider(s3CredentialsFilename, "default").getCredentials(); - s3 = new AmazonS3Client(creds); - } - else { - // default credentials providers, e.g. IAM role - s3 = new AmazonS3Client(); - } - } - - // set up the cache - this.pointSets = CacheBuilder.newBuilder() - .maximumSize(maxCacheSize) - .build(new S3PointSetLoader(workOffline, s3, pointsetBucket)); - } - - // adds file to S3 Data store or offline cache (if working offline) - public String addPointSet(File pointSetFile, String pointSetId) throws IOException { - if (pointSetId == null) - throw new NullPointerException("null point set id"); - - File renamedPointSetFile = new File(POINT_DIR, pointSetId + ".json"); - - if (renamedPointSetFile.exists()) - return pointSetId; - - FileUtils.copyFile(pointSetFile, renamedPointSetFile); - - if(!this.workOffline) { - // only upload if it doesn't exist - try { - s3.getObjectMetadata(pointsetBucket, pointSetId + ".json.gz"); - } catch (AmazonServiceException e) { - // gzip compression in storage, not because we're worried about file size but to speed file transfer - FileInputStream fis = new FileInputStream(pointSetFile); - File tempFile = File.createTempFile(pointSetId, ".json.gz"); - FileOutputStream fos = new FileOutputStream(tempFile); - GZIPOutputStream gos = new GZIPOutputStream(fos); - - try { - ByteStreams.copy(fis, gos); - } finally { - gos.close(); - fis.close(); - } - - s3.putObject(pointsetBucket, pointSetId + ".json.gz", tempFile); - tempFile.delete(); - } - } - - return pointSetId; - - } - - /** does this pointset exist in local cache? */ - public boolean isCached (String pointsetId) { - return new File(POINT_DIR, pointsetId + ".json").exists(); - } - - /** - * Load pointsets from S3. - */ - protected static class S3PointSetLoader extends CacheLoader { - - private Boolean workOffline; - private AmazonS3Client s3; - private String pointsetBucket; - - /** - * Construct a new point set loader. S3 clients are generally threadsafe, so it's fine to share them. - */ - public S3PointSetLoader(Boolean workOffline, AmazonS3Client s3, String pointsetBucket) { - this.workOffline = workOffline; - this.s3 = s3; - this.pointsetBucket = pointsetBucket; - } - - @Override - public PointSet load (String pointSetId) throws Exception { - - File cachedFile; - - if(!workOffline) { - // get pointset metadata from S3 - cachedFile = new File(POINT_DIR, pointSetId + ".json"); - if(!cachedFile.exists()){ - POINT_DIR.mkdirs(); - - S3Object obj = s3.getObject(pointsetBucket, pointSetId + ".json.gz"); - ObjectMetadata objMet = obj.getObjectMetadata(); - FileOutputStream fos = new FileOutputStream(cachedFile); - GZIPInputStream gis = new GZIPInputStream(obj.getObjectContent()); - try { - ByteStreams.copy(gis, fos); - } finally { - fos.close(); - gis.close(); - } - } - } - else - cachedFile = new File(POINT_DIR, pointSetId + ".json"); - - - - // grab it from the cache - - return PointSet.fromGeoJson(cachedFile); - } - } - - @Override - public List getPointSetIds() { - // we have no clue what is in the S3 bucket. - throw new UnsupportedOperationException("S3-backed point set datastore does not know what pointsets are available."); - } -} diff --git a/app/utils/ResultEnvelope.java b/app/utils/ResultEnvelope.java deleted file mode 100644 index ecc3473..0000000 --- a/app/utils/ResultEnvelope.java +++ /dev/null @@ -1,135 +0,0 @@ -package utils; - -import org.opentripplanner.analyst.ResultSet; - -import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; - -/** - * This is a class that stores several result sets: an upper bound, a lower bound, and a central tendency. - * - * @author mattwigway - */ -public class ResultEnvelope implements Serializable { - /** - * The best case/upper bound (e.g. number of jobs reachable under best-case travel time) - */ - public ResultSet bestCase; - - /** - * The lower bound (e.g. number of jobs reachable under worst-case travel time) - */ - public ResultSet worstCase; - - /** - * The average case - */ - public ResultSet avgCase; - - /** - * The point estimate of the accessibility. If profile = false, this is the journey - * time returned by OTP. - */ - public ResultSet pointEstimate; - - /** - * The spread of results (future use). This could be the standard deviation or - * interquartile range, for example. - */ - public ResultSet spread; - - /** - * Is this a profile request? - * If so, upperBound and lowerBound will be defined, and pointEstimate not. - * Otherwise, only pointEstimate will be defined. - */ - public boolean profile; - - /** The ID of the job of which this resultenvelope is a part */ - public String jobId; - - /** The ID of the feature from whence this result envelope came */ - public String id; - - /** The ID of the shapefile/pointset from whence this result envelope came */ - public String destinationPointsetId; - - public ResultSet get (Which key) { - switch (key) { - case BEST_CASE: - return this.bestCase; - case WORST_CASE: - return this.worstCase; - case POINT_ESTIMATE: - return this.pointEstimate; - case SPREAD: - return this.spread; - case AVERAGE: - return this.avgCase; - default: - throw new IllegalStateException("Invalid result type!"); - } - } - - public void put (Which key, ResultSet val) { - switch (key) { - case BEST_CASE: - this.bestCase = val; - break; - case WORST_CASE: - this.worstCase = val; - break; - case POINT_ESTIMATE: - this.pointEstimate = val; - break; - case SPREAD: - this.spread = val; - break; - case AVERAGE: - this.avgCase = val; - break; - } - } - - /** - * Explode this result envelope into a result envelope for each contained attribute - * We do this because we need to retrieve all of the values for a particular variable quickly, in order to display the map, - * and looping over 10GB of data to do this is not tractable. - */ - public Map explode () { - Map exploded = new HashMap(); - - // find a non-null resultset - for (String attr : (pointEstimate != null ? pointEstimate : avgCase).histograms.keySet()) { - ResultEnvelope env = new ResultEnvelope(); - - for (Which which : Which.values()) { - ResultSet orig = this.get(which); - - if (orig != null) { - ResultSet rs = new ResultSet(); - rs.id = orig.id; - rs.histograms.put(attr, orig.histograms.get(attr)); - env.put(which, rs); - env.id = this.id; - } - } - - exploded.put(attr, env); - } - - return exploded; - } - - /** - * Build an empty result envelope. - */ - public ResultEnvelope () { - // do nothing, restore default constructor - } - - public static enum Which { - BEST_CASE, WORST_CASE, POINT_ESTIMATE, SPREAD, AVERAGE - } -} From 83fe63ab7d6ff89f902ebdf8427a512bd63f06e8 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 5 Jun 2015 20:16:32 +0200 Subject: [PATCH 2/4] update imports --- app/controllers/Api.java | 2 +- app/controllers/Gis.java | 2 +- app/controllers/QueueReceiver.java | 2 +- app/controllers/SinglePointTiles.java | 2 +- app/controllers/TilesImpl.java | 2 +- app/migrations/MigrateQueryResults.java | 2 +- app/tiles/AnalystTileRequest.java | 2 +- app/tiles/SurfaceComparisonTile.java | 2 +- app/tiles/SurfaceTile.java | 2 +- test/QueryResultStoreTest.java | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/app/controllers/Api.java b/app/controllers/Api.java index 755bc67..dea4732 100644 --- a/app/controllers/Api.java +++ b/app/controllers/Api.java @@ -19,7 +19,7 @@ import utils.JsonUtil; import utils.QueryResults; import utils.QueueManager; -import utils.ResultEnvelope; +import org.opentripplanner.analyst.cluster.ResultEnvelope; import java.io.IOException; import java.io.StringWriter; diff --git a/app/controllers/Gis.java b/app/controllers/Gis.java index 2fe8123..6555b48 100644 --- a/app/controllers/Gis.java +++ b/app/controllers/Gis.java @@ -29,7 +29,7 @@ import utils.DirectoryZip; import utils.HashUtils; import utils.QueryResults; -import utils.ResultEnvelope; +import org.opentripplanner.analyst.cluster.ResultEnvelope; import utils.ResultEnvelope.Which; import java.io.File; diff --git a/app/controllers/QueueReceiver.java b/app/controllers/QueueReceiver.java index 591da1e..08c110d 100644 --- a/app/controllers/QueueReceiver.java +++ b/app/controllers/QueueReceiver.java @@ -7,7 +7,7 @@ import play.mvc.Result; import utils.JsonUtil; import utils.QueueManager; -import utils.ResultEnvelope; +import org.opentripplanner.analyst.cluster.ResultEnvelope; import java.io.ByteArrayInputStream; import java.io.FileInputStream; diff --git a/app/controllers/SinglePointTiles.java b/app/controllers/SinglePointTiles.java index 81cb4b3..607c3e3 100644 --- a/app/controllers/SinglePointTiles.java +++ b/app/controllers/SinglePointTiles.java @@ -7,7 +7,7 @@ import tiles.AnalystTileRequest; import tiles.SurfaceComparisonTile; import tiles.SurfaceTile; -import utils.ResultEnvelope; +import org.opentripplanner.analyst.cluster.ResultEnvelope; /** * Tiles for single point. Intentionally not annotated with @Security.Authenticated; diff --git a/app/controllers/TilesImpl.java b/app/controllers/TilesImpl.java index a92dae0..03c13d2 100644 --- a/app/controllers/TilesImpl.java +++ b/app/controllers/TilesImpl.java @@ -6,7 +6,7 @@ import play.mvc.Security; import tiles.AnalystTileRequest; import tiles.AnalystTileRequest.*; -import utils.ResultEnvelope; +import org.opentripplanner.analyst.cluster.ResultEnvelope; /** These are tiles which require authentication in all cases */ @Security.Authenticated(Secured.class) diff --git a/app/migrations/MigrateQueryResults.java b/app/migrations/MigrateQueryResults.java index 30a1c27..8ee33a4 100644 --- a/app/migrations/MigrateQueryResults.java +++ b/app/migrations/MigrateQueryResults.java @@ -4,7 +4,7 @@ import utils.DataStore; import utils.QueryResultStore; -import utils.ResultEnvelope; +import org.opentripplanner.analyst.cluster.ResultEnvelope; /** Migrate query results from the old, slow storage format to the new, slightly faster, storage format */ public class MigrateQueryResults { diff --git a/app/tiles/AnalystTileRequest.java b/app/tiles/AnalystTileRequest.java index cd0d6bd..6f3e8ce 100644 --- a/app/tiles/AnalystTileRequest.java +++ b/app/tiles/AnalystTileRequest.java @@ -17,7 +17,7 @@ import utils.NaturalBreaksClassifier; import utils.QueryResults; import utils.QueryResults.QueryResultItem; -import utils.ResultEnvelope; +import org.opentripplanner.analyst.cluster.ResultEnvelope; import java.awt.*; import java.io.IOException; diff --git a/app/tiles/SurfaceComparisonTile.java b/app/tiles/SurfaceComparisonTile.java index 1d41d7e..f06032b 100644 --- a/app/tiles/SurfaceComparisonTile.java +++ b/app/tiles/SurfaceComparisonTile.java @@ -13,7 +13,7 @@ import org.opentripplanner.analyst.PointSet; import org.opentripplanner.analyst.ResultSet; import utils.HaltonPoints; -import utils.ResultEnvelope; +import org.opentripplanner.analyst.cluster.ResultEnvelope; import utils.ResultEnvelope.Which; import java.awt.*; diff --git a/app/tiles/SurfaceTile.java b/app/tiles/SurfaceTile.java index 15a2380..e590155 100644 --- a/app/tiles/SurfaceTile.java +++ b/app/tiles/SurfaceTile.java @@ -8,7 +8,7 @@ import org.opentripplanner.analyst.PointSet; import org.opentripplanner.analyst.ResultSet; import utils.HaltonPoints; -import utils.ResultEnvelope; +import org.opentripplanner.analyst.cluster.ResultEnvelope; import utils.ResultEnvelope.Which; import java.awt.*; diff --git a/test/QueryResultStoreTest.java b/test/QueryResultStoreTest.java index f85374d..5916ea1 100644 --- a/test/QueryResultStoreTest.java +++ b/test/QueryResultStoreTest.java @@ -5,7 +5,7 @@ import org.opentripplanner.analyst.Histogram; import org.opentripplanner.analyst.ResultSet; import utils.QueryResultStore; -import utils.ResultEnvelope; +import org.opentripplanner.analyst.cluster.ResultEnvelope; import java.io.File; import java.util.Arrays; From dca78be332996e230c064d3ce130c979c4d6c332 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Mon, 8 Jun 2015 14:15:41 +0200 Subject: [PATCH 3/4] use OTP 0.19 snapshot. force httpcomponents, fixes #106 --- build.sbt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 74c2ece..29baffa 100644 --- a/build.sbt +++ b/build.sbt @@ -9,11 +9,12 @@ libraryDependencies ++= Seq( javaEbean, cache, "org.mapdb" % "mapdb" % "1.0.6", + "org.apache.httpcomponents" % "httpclient" % "4.3.1", "org.julienrf" %% "play-jsmessages" % "1.6.2", "commons-io" % "commons-io" % "2.4", "com.amazonaws" % "aws-java-sdk" % "1.9.25", "com.typesafe.akka" % "akka-remote_2.10" % "2.3.5", - "org.opentripplanner" % "otp" % "0.18.0-SNAPSHOT", + "org.opentripplanner" % "otp" % "0.19.0-SNAPSHOT", "com.conveyal" % "gtfs-lib" % "0.1-SNAPSHOT", "com.fasterxml.jackson.datatype" % "jackson-datatype-joda" % "2.4.0", "com.logentries" % "logentries-appender" % "1.1.30" From ac50117b507916d3f16dd801d5fadd6638f2c577 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Mon, 8 Jun 2015 14:17:43 +0200 Subject: [PATCH 4/4] Update imports to use classes that were moved over to OTP related to #106 --- app/controllers/Gis.java | 11 ++++++++--- app/controllers/SinglePoint.java | 10 ++++++++-- app/models/Bundle.java | 25 +++++++++++++++++++++---- app/models/Query.java | 10 +++++++++- app/models/Shapefile.java | 27 ++++++++++++++++++++++----- app/tiles/SurfaceComparisonTile.java | 2 +- app/tiles/SurfaceTile.java | 2 +- app/utils/QueryResultStore.java | 27 ++++++++++++++++++++------- app/utils/QueryResults.java | 1 + app/utils/QueueManager.java | 15 ++++++++++++++- 10 files changed, 105 insertions(+), 25 deletions(-) diff --git a/app/controllers/Gis.java b/app/controllers/Gis.java index 6555b48..305122a 100644 --- a/app/controllers/Gis.java +++ b/app/controllers/Gis.java @@ -22,6 +22,8 @@ import org.opengis.feature.simple.SimpleFeatureType; import org.opentripplanner.analyst.PointSet; import org.opentripplanner.analyst.ResultSet; +import org.opentripplanner.analyst.cluster.ResultEnvelope; +import org.opentripplanner.analyst.cluster.ResultEnvelope.Which; import play.Logger; import play.mvc.Controller; import play.mvc.Result; @@ -29,12 +31,15 @@ import utils.DirectoryZip; import utils.HashUtils; import utils.QueryResults; -import org.opentripplanner.analyst.cluster.ResultEnvelope; -import utils.ResultEnvelope.Which; import java.io.File; import java.io.Serializable; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; @Security.Authenticated(Secured.class) public class Gis extends Controller { diff --git a/app/controllers/SinglePoint.java b/app/controllers/SinglePoint.java index ff6c073..a1c29d3 100644 --- a/app/controllers/SinglePoint.java +++ b/app/controllers/SinglePoint.java @@ -10,12 +10,18 @@ import models.Shapefile; import org.opentripplanner.analyst.Histogram; import org.opentripplanner.analyst.ResultSet; +import org.opentripplanner.analyst.cluster.AnalystClusterRequest; +import org.opentripplanner.analyst.cluster.OneToManyProfileRequest; +import org.opentripplanner.analyst.cluster.OneToManyRequest; +import org.opentripplanner.analyst.cluster.ResultEnvelope; +import org.opentripplanner.analyst.cluster.ResultEnvelope.Which; import play.Play; import play.libs.F; import play.mvc.Controller; import play.mvc.Result; -import utils.*; -import utils.ResultEnvelope.Which; +import utils.IdUtils; +import utils.JsonUtil; +import utils.QueueManager; import java.io.ByteArrayOutputStream; import java.io.IOException; diff --git a/app/models/Bundle.java b/app/models/Bundle.java index 6b133b5..d584843 100644 --- a/app/models/Bundle.java +++ b/app/models/Bundle.java @@ -1,7 +1,12 @@ package models; import com.conveyal.gtfs.GTFSFeed; -import com.conveyal.gtfs.model.*; +import com.conveyal.gtfs.model.Agency; +import com.conveyal.gtfs.model.Route; +import com.conveyal.gtfs.model.Shape; +import com.conveyal.gtfs.model.Stop; +import com.conveyal.gtfs.model.StopTime; +import com.conveyal.gtfs.model.Trip; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.google.common.base.Function; @@ -15,19 +20,31 @@ import jobs.ProcessTransitBundleJob; import org.apache.commons.io.FileUtils; import org.geotools.geometry.Envelope2D; -import org.mapdb.*; +import org.mapdb.Atomic; +import org.mapdb.BTreeMap; +import org.mapdb.DB; +import org.mapdb.DBMaker; +import org.mapdb.Fun; import org.mapdb.Fun.Tuple2; +import org.opentripplanner.analyst.cluster.ClusterGraphService; import play.Logger; import play.Play; import play.libs.Akka; import scala.concurrent.ExecutionContext; import scala.concurrent.duration.Duration; -import utils.*; +import utils.Bounds; +import utils.ClassLoaderSerializer; +import utils.DataStore; +import utils.HashUtils; import java.io.File; import java.io.IOException; import java.io.Serializable; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.TimeUnit; import java.util.zip.ZipException; diff --git a/app/models/Query.java b/app/models/Query.java index 5a2e22a..cc4c38d 100644 --- a/app/models/Query.java +++ b/app/models/Query.java @@ -8,12 +8,20 @@ import org.joda.time.LocalDate; import org.opentripplanner.analyst.PointFeature; import org.opentripplanner.analyst.PointSet; +import org.opentripplanner.analyst.cluster.AnalystClusterRequest; +import org.opentripplanner.analyst.cluster.OneToManyProfileRequest; +import org.opentripplanner.analyst.cluster.OneToManyRequest; +import org.opentripplanner.analyst.cluster.ResultEnvelope; import org.opentripplanner.common.model.GenericLocation; import org.opentripplanner.routing.core.TraverseModeSet; import otp.Analyst; import play.Logger; import play.Play; -import utils.*; +import utils.DataStore; +import utils.IdUtils; +import utils.JsonUtil; +import utils.QueryResultStore; +import utils.QueueManager; import java.io.IOException; import java.io.Serializable; diff --git a/app/models/Shapefile.java b/app/models/Shapefile.java index 592c793..a753d89 100644 --- a/app/models/Shapefile.java +++ b/app/models/Shapefile.java @@ -6,6 +6,7 @@ import com.vividsolutions.jts.geom.Geometry; import com.vividsolutions.jts.geom.MultiPolygon; import com.vividsolutions.jts.geom.Polygon; +import com.vividsolutions.jts.geom.Polygonal; import com.vividsolutions.jts.geom.prep.PreparedPolygon; import com.vividsolutions.jts.index.strtree.STRtree; import controllers.Application; @@ -30,14 +31,30 @@ import org.opentripplanner.analyst.PointFeature; import org.opentripplanner.analyst.PointSet; import org.opentripplanner.analyst.UnsupportedGeometryException; +import org.opentripplanner.analyst.cluster.PointSetDatastore; import play.Logger; import play.Play; import play.libs.Akka; import scala.concurrent.ExecutionContext; -import utils.*; - -import java.io.*; -import java.util.*; +import utils.Bounds; +import utils.DataStore; +import utils.HaltonPoints; +import utils.HashUtils; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.zip.ZipEntry; import java.util.zip.ZipException; @@ -122,7 +139,7 @@ public List getPreparedPolygons() { if(geom instanceof Polygon) { - PreparedPolygon pp = new PreparedPolygon((Polygon)geom); + PreparedPolygon pp = new PreparedPolygon((Polygonal)geom); preparedPolygons.add(pp); } diff --git a/app/tiles/SurfaceComparisonTile.java b/app/tiles/SurfaceComparisonTile.java index f06032b..e1b3f61 100644 --- a/app/tiles/SurfaceComparisonTile.java +++ b/app/tiles/SurfaceComparisonTile.java @@ -14,7 +14,7 @@ import org.opentripplanner.analyst.ResultSet; import utils.HaltonPoints; import org.opentripplanner.analyst.cluster.ResultEnvelope; -import utils.ResultEnvelope.Which; +import org.opentripplanner.analyst.cluster.ResultEnvelope.Which; import java.awt.*; import java.io.IOException; diff --git a/app/tiles/SurfaceTile.java b/app/tiles/SurfaceTile.java index e590155..2d35cc7 100644 --- a/app/tiles/SurfaceTile.java +++ b/app/tiles/SurfaceTile.java @@ -9,7 +9,7 @@ import org.opentripplanner.analyst.ResultSet; import utils.HaltonPoints; import org.opentripplanner.analyst.cluster.ResultEnvelope; -import utils.ResultEnvelope.Which; +import org.opentripplanner.analyst.cluster.ResultEnvelope.Which; import java.awt.*; import java.io.IOException; diff --git a/app/utils/QueryResultStore.java b/app/utils/QueryResultStore.java index b0cb468..03ebc3b 100644 --- a/app/utils/QueryResultStore.java +++ b/app/utils/QueryResultStore.java @@ -3,20 +3,33 @@ import com.beust.jcommander.internal.Sets; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import controllers.Application; import models.Query; -import models.Shapefile; -import org.mapdb.*; +import org.mapdb.Fun; import org.opentripplanner.analyst.Histogram; import org.opentripplanner.analyst.ResultSet; +import org.opentripplanner.analyst.cluster.ResultEnvelope; import play.Logger; import play.Play; -import java.io.*; -import java.util.*; -import java.util.Map.Entry; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.EOFException; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; import java.util.function.Consumer; -import java.util.stream.Stream; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; diff --git a/app/utils/QueryResults.java b/app/utils/QueryResults.java index 9625e2e..8acf324 100644 --- a/app/utils/QueryResults.java +++ b/app/utils/QueryResults.java @@ -9,6 +9,7 @@ import models.Shapefile.ShapeFeature; import org.mapdb.Fun.Tuple3; import org.opentripplanner.analyst.ResultSet; +import org.opentripplanner.analyst.cluster.ResultEnvelope; import java.awt.*; import java.util.Iterator; diff --git a/app/utils/QueueManager.java b/app/utils/QueueManager.java index c167c5d..00cc2c8 100644 --- a/app/utils/QueueManager.java +++ b/app/utils/QueueManager.java @@ -6,13 +6,26 @@ import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.sqs.AmazonSQSClient; -import com.amazonaws.services.sqs.model.*; +import com.amazonaws.services.sqs.model.CreateQueueRequest; +import com.amazonaws.services.sqs.model.CreateQueueResult; +import com.amazonaws.services.sqs.model.DeleteMessageBatchRequest; +import com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry; +import com.amazonaws.services.sqs.model.DeleteQueueRequest; +import com.amazonaws.services.sqs.model.GetQueueUrlResult; +import com.amazonaws.services.sqs.model.Message; +import com.amazonaws.services.sqs.model.QueueDoesNotExistException; +import com.amazonaws.services.sqs.model.ReceiveMessageRequest; +import com.amazonaws.services.sqs.model.ReceiveMessageResult; +import com.amazonaws.services.sqs.model.SendMessageBatchRequest; +import com.amazonaws.services.sqs.model.SendMessageBatchRequestEntry; import com.beust.jcommander.internal.Lists; import com.beust.jcommander.internal.Sets; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import models.Query; import models.TransportScenario; +import org.opentripplanner.analyst.cluster.AnalystClusterRequest; +import org.opentripplanner.analyst.cluster.ResultEnvelope; import play.Play; import play.libs.Akka; import scala.concurrent.duration.Duration;