Merge (current/future) feed versions for MTC

src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java

@@ -57,7 +57,7 @@ public enum JobType {
         EXPORT_SNAPSHOT_TO_GTFS,
         CONVERT_EDITOR_MAPDB_TO_SQL,
         VALIDATE_ALL_FEEDS,
-        MERGE_PROJECT_FEEDS
+        MERGE_FEED_VERSIONS
     }
 
     public MonitorableJob(String owner, String name, JobType type) {

src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java

@@ -1,8 +1,11 @@
 package com.conveyal.datatools.manager.controllers.api;
 
+import com.conveyal.datatools.common.utils.SparkUtils;
 import com.conveyal.datatools.manager.DataManager;
 import com.conveyal.datatools.manager.auth.Auth0UserProfile;
 import com.conveyal.datatools.manager.jobs.CreateFeedVersionFromSnapshotJob;
+import com.conveyal.datatools.manager.jobs.MergeFeedsJob;
+import com.conveyal.datatools.manager.jobs.MergeFeedsType;
 import com.conveyal.datatools.manager.jobs.ProcessSingleFeedJob;
 import com.conveyal.datatools.manager.models.FeedDownloadToken;
 import com.conveyal.datatools.manager.models.FeedSource;
@@ -13,6 +16,7 @@
 import com.conveyal.datatools.manager.persistence.Persistence;
 import com.conveyal.datatools.manager.utils.HashUtils;
 import com.conveyal.datatools.manager.utils.json.JsonManager;
+
 import com.fasterxml.jackson.databind.JsonNode;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -23,13 +27,17 @@
 import java.io.File;
 import java.util.Collection;
 import java.util.Date;
+import java.util.HashSet;
+import java.util.Set;
 
 import static com.conveyal.datatools.common.utils.S3Utils.downloadFromS3;
 import static com.conveyal.datatools.common.utils.SparkUtils.copyRequestStreamIntoFile;
 import static com.conveyal.datatools.common.utils.SparkUtils.downloadFile;
 import static com.conveyal.datatools.common.utils.SparkUtils.formatJobMessage;
 import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt;
 import static com.conveyal.datatools.manager.controllers.api.FeedSourceController.checkFeedSourcePermissions;
+import static com.conveyal.datatools.manager.jobs.MergeFeedsType.MTC;
+import static com.conveyal.datatools.manager.jobs.MergeFeedsType.REGIONAL;
 import static spark.Spark.delete;
 import static spark.Spark.get;
 import static spark.Spark.post;
@@ -262,6 +270,47 @@ private static FeedVersion publishToExternalResource (Request req, Response res)
         }
     }
 
+    /**
+     * HTTP controller that handles merging multiple feed versions for a given feed source, with version IDs specified
+     * in a comma-separated string in the feedVersionIds query parameter and merge type specified in mergeType query
+     * parameter. NOTE: REGIONAL merge type should only be handled through {@link ProjectController#downloadMergedFeed(Request, Response)}.
+     */
+    private static String mergeFeedVersions(Request req, Response res) {
+        String[] versionIds = req.queryParams("feedVersionIds").split(",");
+        // Try to parse merge type (null or bad value throws IllegalArgumentException).
+        MergeFeedsType mergeType;
+        try {
+            mergeType = MergeFeedsType.valueOf(req.queryParams("mergeType"));
+            if (mergeType.equals(REGIONAL)) throw new IllegalArgumentException();
+        } catch (IllegalArgumentException e) {
+            logMessageAndHalt(req, 400, "Must provide valid merge type.", e);
+            return null;
+        }
+        // Collect versions to merge (must belong to same feed source).
+        Set<FeedVersion> versions = new HashSet<>();
+        String feedSourceId = null;
+        for (String id : versionIds) {
+            FeedVersion v = Persistence.feedVersions.getById(id);
+            if (v == null) {
+                logMessageAndHalt(req, 400, "Must provide valid version ID");
+            }
+            // Store feed source id and check other versions for matching.
+            if (feedSourceId == null) feedSourceId = v.feedSourceId;
+            else if (!v.feedSourceId.equals(feedSourceId)) {
+                logMessageAndHalt(req, 400, "Cannot merge versions with different parent feed sources.");
+            }
+            versions.add(v);
+        }
+        if (versionIds.length != 2) {
+            logMessageAndHalt(req, 400, "Merging more than two versions is not currently supported.");
+        }
+        // Kick off merge feeds job.
+        Auth0UserProfile userProfile = req.attribute("user");
+        MergeFeedsJob mergeFeedsJob = new MergeFeedsJob(userProfile.getUser_id(), versions, "merged", mergeType);
+        DataManager.heavyExecutor.execute(mergeFeedsJob);
+        return SparkUtils.formatJobMessage(mergeFeedsJob.jobId, "Merging feed versions...");
+    }
+
     /**
      * Download locally stored feed version with token supplied by this application. This method is only used when
      * useS3 is set to false. Otherwise, a direct download from s3 should be used.
@@ -300,6 +349,7 @@ public static void register (String apiPrefix) {
         post(apiPrefix + "secure/feedversion", FeedVersionController::createFeedVersionViaUpload, json::write);
         post(apiPrefix + "secure/feedversion/fromsnapshot", FeedVersionController::createFeedVersionFromSnapshot, json::write);
         put(apiPrefix + "secure/feedversion/:id/rename", FeedVersionController::renameFeedVersion, json::write);
+        put(apiPrefix + "secure/feedversion/merge", FeedVersionController::mergeFeedVersions, json::write);
         post(apiPrefix + "secure/feedversion/:id/publish", FeedVersionController::publishToExternalResource, json::write);
         delete(apiPrefix + "secure/feedversion/:id", FeedVersionController::deleteFeedVersion, json::write);
 

src/main/java/com/conveyal/datatools/manager/controllers/api/ProjectController.java

@@ -5,8 +5,9 @@
 import com.conveyal.datatools.manager.auth.Auth0UserProfile;
 import com.conveyal.datatools.manager.jobs.FetchProjectFeedsJob;
 import com.conveyal.datatools.manager.jobs.MakePublicJob;
-import com.conveyal.datatools.manager.jobs.MergeProjectFeedsJob;
+import com.conveyal.datatools.manager.jobs.MergeFeedsJob;
 import com.conveyal.datatools.manager.models.FeedDownloadToken;
+import com.conveyal.datatools.manager.models.FeedSource;
 import com.conveyal.datatools.manager.models.FeedVersion;
 import com.conveyal.datatools.manager.models.JsonViews;
 import com.conveyal.datatools.manager.models.Project;
@@ -20,13 +21,16 @@
 import spark.Response;
 
 import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 import static com.conveyal.datatools.common.utils.S3Utils.downloadFromS3;
 import static com.conveyal.datatools.common.utils.SparkUtils.downloadFile;
 import static com.conveyal.datatools.common.utils.SparkUtils.formatJobMessage;
 import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt;
 import static com.conveyal.datatools.manager.DataManager.publicPath;
+import static com.conveyal.datatools.manager.jobs.MergeFeedsType.REGIONAL;
 import static spark.Spark.delete;
 import static spark.Spark.get;
 import static spark.Spark.post;
@@ -216,14 +220,28 @@ private static Project checkProjectPermissions(Request req, Project project, Str
      * to getFeedDownloadCredentials with the project ID to obtain either temporary S3 credentials or a download token
      * (depending on application configuration "application.data.use_s3_storage") to download the zip file.
      */
-    private static String downloadMergedFeed(Request req, Response res) {
+    static String downloadMergedFeed(Request req, Response res) {
         Project project = requestProjectById(req, "view");
         Auth0UserProfile userProfile = req.attribute("user");
         // TODO: make this an authenticated call?
-        MergeProjectFeedsJob mergeProjectFeedsJob = new MergeProjectFeedsJob(project, userProfile.getUser_id());
-        DataManager.heavyExecutor.execute(mergeProjectFeedsJob);
+        Set<FeedVersion> feedVersions = new HashSet<>();
+        // Get latest version for each feed source in project
+        Collection<FeedSource> feedSources = project.retrieveProjectFeedSources();
+        for (FeedSource fs : feedSources) {
+            // check if feed version exists
+            FeedVersion version = fs.retrieveLatest();
+            if (version == null) {
+                LOG.warn("Skipping {} because it has no feed versions", fs.name);
+                continue;
+            }
+            // modify feed version to use prepended feed id
+            LOG.info("Adding {} feed to merged zip", fs.name);
+            feedVersions.add(version);
+        }
+        MergeFeedsJob mergeFeedsJob = new MergeFeedsJob(userProfile.getUser_id(), feedVersions, project.id, REGIONAL);
+        DataManager.heavyExecutor.execute(mergeFeedsJob);
         // Return job ID to requester for monitoring job status.
-        return formatJobMessage(mergeProjectFeedsJob.jobId, "Merge operation is processing.");
+        return formatJobMessage(mergeFeedsJob.jobId, "Merge operation is processing.");
     }
 
     /**

src/main/java/com/conveyal/datatools/manager/gtfsplus/CalendarAttribute.java

@@ -0,0 +1,20 @@
+package com.conveyal.datatools.manager.gtfsplus;
+
+import com.conveyal.gtfs.model.Entity;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+public class CalendarAttribute extends Entity {
+
+    private static final long serialVersionUID = 1L;
+
+    public String service_id;
+    public String service_description;
+
+    // TODO
+    @Override
+    public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException {
+
+    }
+}

src/main/java/com/conveyal/datatools/manager/gtfsplus/Direction.java

@@ -0,0 +1,22 @@
+package com.conveyal.datatools.manager.gtfsplus;
+
+import com.conveyal.gtfs.model.Entity;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+public class Direction extends Entity {
+
+    private static final long serialVersionUID = 1L;
+
+    public String route_id;
+    public int direction_id;
+    public String direction;
+
+
+    // TODO
+    @Override
+    public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException {
+
+    }
+}

src/main/java/com/conveyal/datatools/manager/gtfsplus/FareRiderCategory.java

@@ -0,0 +1,24 @@
+package com.conveyal.datatools.manager.gtfsplus;
+
+import com.conveyal.gtfs.model.Entity;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.time.LocalDate;
+
+public class FareRiderCategory extends Entity {
+
+    private static final long serialVersionUID = 1L;
+
+    public String fare_id;
+    public int rider_category_id;
+    public double price;
+    public LocalDate expiration_date;
+    public LocalDate commencement_date;
+
+    // TODO
+    @Override
+    public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException {
+
+    }
+}

src/main/java/com/conveyal/datatools/manager/gtfsplus/FareZoneAttribute.java

@@ -0,0 +1,20 @@
+package com.conveyal.datatools.manager.gtfsplus;
+
+import com.conveyal.gtfs.model.Entity;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+public class FareZoneAttribute extends Entity {
+
+    private static final long serialVersionUID = 1L;
+
+    public String zone_id;
+    public String zone_name;
+
+    // TODO
+    @Override
+    public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException {
+
+    }
+}

src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusTable.java

@@ -0,0 +1,87 @@
+package com.conveyal.datatools.manager.gtfsplus;
+
+import com.conveyal.gtfs.loader.DateField;
+import com.conveyal.gtfs.loader.DoubleField;
+import com.conveyal.gtfs.loader.IntegerField;
+import com.conveyal.gtfs.loader.ShortField;
+import com.conveyal.gtfs.loader.StringField;
+import com.conveyal.gtfs.loader.Table;
+
+import static com.conveyal.gtfs.loader.Requirement.OPTIONAL;
+import static com.conveyal.gtfs.loader.Requirement.PROPRIETARY;
+import static com.conveyal.gtfs.loader.Requirement.REQUIRED;
+
+public class GtfsPlusTable {
+    public static final Table REALTIME_ROUTES = new Table("realtime_routes", RealtimeRoute.class, PROPRIETARY,
+        new StringField("route_id", REQUIRED).isReferenceTo(Table.ROUTES),
+        new ShortField("realtime_enabled", REQUIRED, 1),
+        new StringField("realtime_routename",  REQUIRED),
+        new StringField("realtime_routecode", REQUIRED)
+    );
+
+    public static final Table REALTIME_STOPS = new Table("realtime_stops", RealtimeStop.class, PROPRIETARY,
+        new StringField("trip_id", REQUIRED).isReferenceTo(Table.TRIPS),
+        new StringField("stop_id", REQUIRED).isReferenceTo(Table.STOPS),
+        new StringField("realtime_stop_id",  REQUIRED)
+    ).keyFieldIsNotUnique();
+
+    public static final Table DIRECTIONS = new Table("directions", Direction.class, PROPRIETARY,
+        new StringField("route_id", REQUIRED).isReferenceTo(Table.ROUTES),
+        new ShortField("direction_id", REQUIRED, 1),
+        new StringField("direction", REQUIRED)
+    ).keyFieldIsNotUnique();
+
+    public static final Table REALTIME_TRIPS = new Table("realtime_trips", RealtimeTrip.class, PROPRIETARY,
+        new StringField("trip_id", REQUIRED).isReferenceTo(Table.TRIPS),
+        new StringField("realtime_trip_id", REQUIRED)
+    );
+
+    public static final Table STOP_ATTRIBUTES = new Table("stop_attributes", StopAttribute.class, PROPRIETARY,
+        new StringField("stop_id", REQUIRED).isReferenceTo(Table.STOPS),
+        new ShortField("accessibility_id", REQUIRED, 8),
+        new StringField("cardinal_direction", OPTIONAL),
+        new StringField("relative_position", OPTIONAL),
+        new StringField("stop_city", REQUIRED)
+    );
+
+    public static final Table TIMEPOINTS = new Table("timepoints", TimePoint.class, PROPRIETARY,
+        new StringField("trip_id", REQUIRED).isReferenceTo(Table.TRIPS),
+        new StringField("stop_id", REQUIRED).isReferenceTo(Table.STOPS)
+    ).keyFieldIsNotUnique();
+
+    public static final Table RIDER_CATEGORIES = new Table("rider_categories", RiderCategory.class, PROPRIETARY,
+        new IntegerField("rider_category_id", REQUIRED, 1, 25),
+        new StringField("rider_category_description", REQUIRED)
+    );
+
+    public static final Table FARE_RIDER_CATEGORIES = new Table("fare_rider_categories", FareRiderCategory.class, PROPRIETARY,
+        new StringField("fare_id", REQUIRED),
+        new IntegerField("rider_category_id", REQUIRED, 2, 25).isReferenceTo(RIDER_CATEGORIES),
+        new DoubleField("price", REQUIRED, 0, Double.MAX_VALUE, 2),
+        new DateField("expiration_date", OPTIONAL),
+        new DateField("commencement_date", OPTIONAL)
+    ).keyFieldIsNotUnique();
+
+    public static final Table CALENDAR_ATTRIBUTES = new Table("calendar_attributes", CalendarAttribute.class, PROPRIETARY,
+        new StringField("service_id", REQUIRED).isReferenceTo(Table.CALENDAR),
+        new StringField("service_description", REQUIRED)
+    );
+
+    public static final Table FAREZONE_ATTRIBUTES = new Table("farezone_attributes", FareZoneAttribute.class, PROPRIETARY,
+        new StringField("zone_id", REQUIRED),
+        new StringField("zone_name", REQUIRED)
+    );
+
+    public static final Table[] tables = new Table[] {
+        REALTIME_ROUTES,
+        REALTIME_STOPS,
+        REALTIME_TRIPS,
+        DIRECTIONS,
+        STOP_ATTRIBUTES,
+        TIMEPOINTS,
+        RIDER_CATEGORIES,
+        FARE_RIDER_CATEGORIES,
+        CALENDAR_ATTRIBUTES,
+        FAREZONE_ATTRIBUTES
+    };
+}

src/main/java/com/conveyal/datatools/manager/gtfsplus/RealtimeRoute.java

@@ -0,0 +1,21 @@
+package com.conveyal.datatools.manager.gtfsplus;
+
+import com.conveyal.gtfs.model.Entity;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+public class RealtimeRoute extends Entity {
+    private static final long serialVersionUID = 1L;
+
+    public String route_id;
+    public int realtime_enabled;
+    public String realtime_routename;
+    public String realtime_routecode;
+
+    // TODO
+    @Override
+    public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException {
+
+    }
+}

src/main/java/com/conveyal/datatools/manager/gtfsplus/RealtimeStop.java

@@ -0,0 +1,21 @@
+package com.conveyal.datatools.manager.gtfsplus;
+
+import com.conveyal.gtfs.model.Entity;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+public class RealtimeStop extends Entity {
+
+    private static final long serialVersionUID = 1L;
+
+    public String trip_id;
+    public String stop_id;
+    public String realtime_stop_id;
+
+    // TODO
+    @Override
+    public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException {
+
+    }
+}