Merge pull request #159 from wsp-sag/bicounty_emme

merge `bicounty_emme` to `bicounty_dev`

lasso/mtc.py

@@ -103,6 +103,7 @@ def calculate_facility_type(
     join_gdf["oneWay"].fillna("", inplace = True)
     join_gdf["oneWay"] = join_gdf["oneWay"].apply(lambda x: "NA" if x in [None, np.nan, float('nan')] else x)
     join_gdf["oneWay"] = join_gdf["oneWay"].apply(lambda x: str(x) if type(x) == bool else x)
+    join_gdf["oneWay"] = join_gdf["oneWay"].apply(lambda x: str(x) if type(x) == int else x)
     join_gdf["oneWay"] = join_gdf["oneWay"].apply(lambda x: x if type(x) == str else ','.join(map(str, x)))
     join_gdf["oneWay_binary"] = join_gdf["oneWay"].apply(lambda x: 0 if "False" in x else 1)
 
@@ -1710,7 +1711,7 @@ def roadway_standard_to_mtc_network(
         roadway_network.links_df["assignable"]
     )
 
-    roadway_network = calculate_cntype(roadway_network, parameters)
+    roadway_network = calculate_cntype(roadway_network, parameters, overwrite=True)
     roadway_network = calculate_transit(roadway_network, parameters)
     roadway_network = calculate_useclass(roadway_network, parameters)
     roadway_network = calculate_facility_type(roadway_network, parameters, update_network_variable = True)
@@ -1735,6 +1736,8 @@ def roadway_standard_to_mtc_network(
         on = "id"
     )
 
+    roadway_network.links_mtc_df = gpd.GeoDataFrame(roadway_network.links_mtc_df, geometry=roadway_network.links_mtc_df.geometry)
+    roadway_network.nodes_mtc_df = gpd.GeoDataFrame(roadway_network.nodes_mtc_df, geometry=roadway_network.nodes_mtc_df.geometry)
     roadway_network.links_mtc_df.crs = roadway_network.crs
     roadway_network.nodes_mtc_df.crs = roadway_network.crs
     WranglerLogger.info("Setting Coordinate Reference System to {}".format(output_proj))
@@ -1748,8 +1751,11 @@ def roadway_standard_to_mtc_network(
         lambda g: g.y
     )
 
+    roadway_network.nodes_mtc_df["pnr"] = np.where(roadway_network.nodes_mtc_df['pnr']==0, '0.0', '1.0')
+
     # CUBE expect node id to be N
     roadway_network.nodes_mtc_df.rename(columns={"model_node_id": "N"}, inplace=True)
+    # roadway_network.nodes_mtc_df['model_node_id']=roadway_network.nodes_mtc_df['N']
 
     return roadway_network
 
@@ -1860,17 +1866,19 @@ def route_properties_gtfs_to_cube(
 
     trip_df["agency_id"].fillna("", inplace = True)
 
+    trip_df['dir_shp_index'] = trip_df.groupby(["TM2_operator", "route_id", "tod_name"]).cumcount()
+
     trip_df["NAME"] = trip_df.apply(
         lambda x: str(x.TM2_operator)
         + "_"
         + str(x.route_id)
         + "_"
-        + x.tod_name
+        + str(x.tod_name)
         + "_"
         + "d"
-        + str(int(x.direction_id))
+        + str(int(x.dir_shp_index))
         + "_s"
-        + x.shape_id,
+        + str(x.shape_id),       
         axis=1,
     )
 
@@ -1933,7 +1941,10 @@ def cube_format(transit_network, row):
         add_nntime = True
     else:
         add_nntime = False
-    s += "\n N={}".format(transit_network.shape_gtfs_to_cube(row, add_nntime))
+    nodes, runtime = transit_network.shape_gtfs_to_cube(row, add_nntime)
+    if add_nntime:
+        s += '\n RUNTIME={},'.format(runtime)
+    s += "\n N={}".format(nodes)
 
     # TODO: need NNTIME, ACCESS_C
 
@@ -1952,6 +1963,19 @@ def write_as_cube_lin(
         outpath: File location for output cube line file.
 
     """
+
+    transit_network.feed.trips['trip_id'] = transit_network.feed.trips['trip_id'].astype(int)
+    transit_network.feed.trips['shape_id'] = transit_network.feed.trips['shape_id'].astype(int)
+
+    transit_network.feed.stop_times['trip_id'] = transit_network.feed.stop_times['trip_id'].astype(int)
+    transit_network.feed.stop_times['stop_id'] = transit_network.feed.stop_times['stop_id'].astype(float).astype(int)
+
+    transit_network.feed.shapes['shape_id'] = transit_network.feed.shapes['shape_id'].astype(int)
+
+    transit_network.feed.stops['stop_id'] =  transit_network.feed.stops['stop_id'].astype(float).astype(int)
+
+    transit_network.feed.frequencies['trip_id'] =  transit_network.feed.frequencies['trip_id'].astype(int)
+
     if not outpath:
         outpath  = os.path.join(parameters.scratch_location,"outtransit.lin")
     trip_cube_df = route_properties_gtfs_to_cube(transit_network, parameters, outpath)
@@ -2151,7 +2175,9 @@ def _is_express_bus(x):
         if (x.route_short_name.startswith("J")) | (x.route_short_name.startswith("Lynx")):
             return 1
     if x.agency_name == "SolTrans":
-        if (x.route_short_name in ["80", "92", "78"]) | (x.route_long_name in ["80", "92", "78"]):
+        if ((x.route_short_name in ["80", "92", "78","Green","Blue","Red"]) | 
+            (x.route_long_name in ["80", "92", "78","Green","Blue","Red"])
+        ):
             return 1
     if x.agency_name == "Vine (Napa County)":
         if x.route_short_name in ["29"]:

lasso/parameters.py

@@ -1,4 +1,5 @@
 import os
+import pyproj
 from .logger import WranglerLogger
 
 
@@ -740,7 +741,7 @@ def __init__(self, **kwargs):
             #MTC
             'name',
             "distance",
-            #"roadway",
+            "roadway",
             #"name",
             #MC
             #"shape_id",
@@ -769,6 +770,7 @@ def __init__(self, **kwargs):
             "managed",
             "bus_only",
             "rail_only",
+            "pnr",      
             #MTC
             "assignable",
             "cntype",
@@ -795,6 +797,8 @@ def __init__(self, **kwargs):
             #bi-county
             "nmt2010",
             "nmt2020",
+            "BRT",
+            "has_transit"
         ]
 
         self.output_link_shp = os.path.join(self.scratch_location, "links.shp")
@@ -820,7 +824,7 @@ def __init__(self, **kwargs):
 
         self.fare_matrix_output_variables = ["faresystem", "origin_farezone", "destination_farezone", "price"]
 
-        self.zones = 4756
+        self.zones = 6593
         """
         Create all the possible headway variable combinations based on the cube time periods setting
         """
@@ -906,6 +910,8 @@ def __init__(self, **kwargs):
             #bi-county
             "nmt2010",
             "nmt2020",
+            "BRT",
+            "has_transit"               
         ]
 
         self.float_col = [
@@ -921,6 +927,7 @@ def __init__(self, **kwargs):
         self.string_col = [
             "osm_node_id",
             "name",
+            "pnr",      
             "roadway",
             "shstGeometryId",
             "access_AM",
@@ -937,4 +944,10 @@ def __init__(self, **kwargs):
 
         self.drive_buffer =  6
 
+        #self.network_build_crs = CRS("EPSG:2875")
+        #self.project_card_crs = CRS("EPSG:4326")
+        #self.transformer = pyproj.Transformer.from_crs(
+        #    self.network_build_crs, self.project_card_crs, always_xy=True
+        #)
+
         self.__dict__.update(kwargs)

lasso/project.py

@@ -165,6 +165,7 @@ def create_project(
         roadway_node_changes: Optional[DataFrame] = None,
         transit_changes: Optional[CubeTransit] = None,
         base_roadway_network: Optional[RoadwayNetwork] = None,
+        base_transit_network: Optional[StandardTransit] = None,
         base_cube_transit_network: Optional[CubeTransit] = None,
         build_cube_transit_network: Optional[CubeTransit] = None,
         project_name: Optional[str] = None,
@@ -403,6 +404,8 @@ def create_project(
                 gtfs_feed_dir=base_transit_dir,
                 parameters=parameters
             )
+        elif base_transit_network:
+            base_transit_network = base_transit_network
         else:
             msg = "No base transit network."
             WranglerLogger.info(msg)
@@ -695,16 +698,23 @@ def emme_id_to_wrangler_id(emme_link_change_df, emme_node_change_df, emme_transi
 
             # get new emme nodes
             new_emme_node_id_list = [
-                n for n in emme_node_change_df['emme_id'] if n not in emme_node_id_crosswalk_df['emme_node_id']
+                n for n in emme_node_change_df['emme_id'].to_list() if n not in emme_node_id_crosswalk_df['emme_node_id'].to_list()
             ]
             WranglerLogger.info('New emme node id list {}'.format(new_emme_node_id_list))
             new_wrangler_node = emme_node_id_crosswalk_df['model_node_id'].max()
 
             # add crosswalk for new emme nodes
             for new_emme_node in new_emme_node_id_list:
-                new_wrangler_node = new_wrangler_node + 1
-                emme_node_id_dict.update({new_emme_node : new_wrangler_node})
-
+                if new_emme_node in emme_node_id_dict.keys():
+                    msg = "new node id {} has already been added to the crosswalk".format(new_emme_node)
+                    WranglerLogger.error(msg)
+                    raise ValueError(msg)
+                else:
+                    new_wrangler_node = new_wrangler_node + 1
+                    emme_node_id_dict.update({new_emme_node : new_wrangler_node})
+                    new_emme_node_id_crosswalk_df = pd.DataFrame(emme_node_id_dict.items(), columns=['emme_node_id', 'model_node_id'])
+                    new_emme_node_id_crosswalk_df.to_csv(emme_node_id_crosswalk_file, index=False)
+
             # for nodes update model_node_id
             emme_node_change_df['model_node_id'] = emme_node_change_df['emme_id'].map(emme_node_id_dict).fillna(0)
 
@@ -873,7 +883,7 @@ def determine_roadway_network_changes_compatibility(
         roadway_link_changes.rename(columns=dbf_to_net_dict, inplace=True)
 
         for c in roadway_node_changes.columns:
-            if (c not in log_to_net_df["log"].tolist() + log_to_net_df["net"].tolist()) & (c not in ["A", "B"]):
+            if (c not in log_to_net_df["log"].tolist() + log_to_net_df["net"].tolist()) & (c not in ["A", "B", "X", "Y"]):
                 roadway_node_changes.rename(columns={c : c.lower()}, inplace=True)
         roadway_node_changes.rename(columns=log_to_net_dict, inplace=True)
         roadway_node_changes.rename(columns=dbf_to_net_dict, inplace=True)
@@ -1083,6 +1093,8 @@ def _process_node_additions(node_add_df):
 
             node_add_df = node_add_df.drop(["operation_final"], axis=1)
 
+            node_add_df = node_add_df.apply(_reproject_coordinates, axis=1)
+
             for x in node_add_df.columns:
                 node_add_df[x] = node_add_df[x].astype(self.base_roadway_network.nodes_df[x].dtype)
 
@@ -1093,6 +1105,12 @@ def _process_node_additions(node_add_df):
 
             return add_nodes_dict_list
 
+        def _reproject_coordinates(row):
+            reprojected_x, reprojected_y = self.parameters.transformer.transform(row['X'], row['Y'])
+            row['X'] = reprojected_x
+            row['Y'] = reprojected_y
+            return row
+
         def _process_single_link_change(change_row, changeable_col):
             """"""
 

lasso/roadway.py

@@ -1674,6 +1674,9 @@ def dataframe_to_fixed_width(df):
             dict: dictionary with columns names as keys, column width as values.
         """
         WranglerLogger.info("Starting fixed width conversion")
+        if 'name' in df.columns:
+            df['name']=df['name'].apply(lambda x: x.strip().split(',')[0].replace("[",'').replace("'nan'","").replace("nan","").replace("'",""))
+
 
         # get the max length for each variable column
         max_width_dict = dict(

lasso/transit.py

@@ -1167,8 +1167,11 @@ def shape_gtfs_to_cube(self, row, add_nntime = False):
 
         trip_node_df.sort_values(by = ["shape_pt_sequence"], inplace = True)
 
+        stops_df = self.feed.stops.copy()
+        stops_df['stop_id'] = stops_df['stop_id'].astype(float).astype(int)
+        trip_stop_times_df['stop_id'] = trip_stop_times_df['stop_id'].astype(float).astype(int)
+
         if 'trip_id' in self.feed.stops.columns:
-            stops_df = self.feed.stops.copy()
             if agency_raw_name != 'sjrtd_2015_0127':
                 stops_df = stops_df[stops_df.agency_raw_name != 'sjrtd_2015_0127']
                 trip_stop_times_df = pd.merge(
@@ -1177,12 +1180,13 @@ def shape_gtfs_to_cube(self, row, add_nntime = False):
             else:
                 stops_df = stops_df[stops_df.agency_raw_name == 'sjrtd_2015_0127']
                 stops_df['trip_id'] = stops_df['trip_id'].astype(float).astype(int).astype(str)
+                trip_stop_times_df['trip_id'] = trip_stop_times_df['trip_id'].astype(float).astype(int).astype(str)
                 trip_stop_times_df = pd.merge(
                     trip_stop_times_df, stops_df, how="left", on=['agency_raw_name', 'trip_id',"stop_id"]
                 )
         else:
             trip_stop_times_df = pd.merge(
-                trip_stop_times_df, self.feed.stops, how="left", on="stop_id"
+                trip_stop_times_df, stops_df, how="left", on="stop_id"
             )
 
         trip_stop_times_df["model_node_id"] = pd.to_numeric(trip_stop_times_df["model_node_id"]).astype(int)
@@ -1213,6 +1217,8 @@ def _access_type(x):
 
         trip_stop_times_df["ACCESS"] = trip_stop_times_df.apply(lambda x: _access_type(x), axis = 1)
 
+        trip_runtime = round(trip_stop_times_df[trip_stop_times_df['NNTIME'] > 0]['NNTIME'].sum(),2)
+
         # node list
         node_list_str = ""
         stop_seq = 0
@@ -1260,7 +1266,7 @@ def _access_type(x):
         node_list_str = node_list_str.replace(" NNTIME=0.0, N=", "")
         node_list_str = node_list_str.replace(" NNTIME=0.0,", "")
 
-        return node_list_str
+        return node_list_str, trip_runtime
 
 
     def cube_format(self, row):

mtc_data/lookups/faresystem_crosswalk.txt

@@ -319,3 +319,4 @@ Vine_GTFS_PLUS_2015,42,636,4
 Vine_GTFS_PLUS_2015,43,635,3
 Vine_GTFS_PLUS_2015,44,633,15
 Wheels_2016_7_13,45,0,
+SMART,101,,

mtc_data/lookups/gtfs_to_tm2_mode_crosswalk.csv

@@ -22,8 +22,8 @@ MarinTransit_2015_8_31,Marin Transit,MCTD,39,3,71,Local bus,24,0,Motor Standard
 Petaluma_2016_5_22,Petaluma Transit,267,21,3,68,Local bus,22,0,Motor Standard Bus
 RioVista_2015_8_20,Rio Vista Delta Breeze,5,13,3,52,Local bus,14,0,Motor Standard Bus
 SFMTA_2015_8_11,San Francisco Municipal Transportation Agency,SFMTA,22,0,110,Light rail,39,0,LRV2
-SFMTA_2015_8_11,San Francisco Municipal Transportation Agency,SFMTA,22,3,20,Local bus,2,0,Motor Standard Bus
-SFMTA_2015_8_11,San Francisco Municipal Transportation Agency,SFMTA,22,5,21,Local bus,3,0,Motor Standard Bus
+SFMTA_2015_8_11,San Francisco Municipal Transportation Agency,SFMTA,22,3,21,Local bus,2,0,Motor Standard Bus
+SFMTA_2015_8_11,San Francisco Municipal Transportation Agency,SFMTA,22,5,20,Local bus,3,0,Motor Standard Bus
 SF_Bay_Ferry2016_07_01,San Francisco Bay Ferry,SB,25,4,101,Ferry service,36,0,Ferry
 SamTrans_2015_8_20,SamTrans,samtrans-ca-us,6,3,24,Local bus,4,0,Motor Standard Bus
 SamTrans_2015_8_20,SamTrans,samtrans-ca-us,6,3,80,Express bus,25,1,SamTrans Plus Bus
@@ -50,3 +50,4 @@ Wheels_2016_7_13,Wheels Bus,LAVTA,24,3,17,Local bus,7,0,Motor Standard Bus
 commuteDOTorg_GTFSImportExport_20160127_final_mj,Commute.org Shuttle,alliance,17,3,14,Local bus,0,0,Motor Small Bus
 sjrtd_2015_0127,San Joaquin Regional Transit District (RTD),,100,3,96,Local bus,0,0,Motor Standard Bus
 sjrtd_2015_0127,San Joaquin Regional Transit District (RTD),,100,3,97,Express bus,0,1,Motor Standard Bus
+SMART,Sonoma Marin Area Rail Transit,SMART,101,2,135,Commuter rail,101,0,Unknown Train