diff --git a/README.md b/README.md index c73e5845..ddf7cea0 100644 --- a/README.md +++ b/README.md @@ -12,38 +12,28 @@ A python package to run the San Francisco Bay Area's Travel Model. ## Installation -Recommended install in a virtual environment. +It is recommended that tm2install in a virtual environment. Stable (to come - use bleeding edge for now): - ```bash pip install tm2py ``` -Bleeding edge: -TODO: Which environment is this? Does it still work for anyone? +For Developers, it is recomended that the following instructions are used to install +*Note: The Python Environment has recently been updated to python 3.11, there may be some instabilites with current build* +```bat +git clone --branch develop https://github.com/BayAreaMetro/tm2py.git -```bash -conda env create -f environment.yml +conda create -n tm2py python=3.11.9 conda activate tm2py -pip install git+https://github.com/bayareametro/tm2py@develop -``` - -The above directions didn't work for the MTC Windows environment. The following method did work, on a machine with Emme-4.6.0 installed. This required a compiled GDAL/Fiona package set for python 3.7, this can be found in the [lib directory](/lib/) , consisting of the following: - -1. GDAL-3.3.2-cp37-cp37m-win_amd64.whl -2. pyproj-3.2.1-cp37-cp37m-win_amd64.whl -3. Fiona-1.8.20-cp37-cp37m-win_amd64.whl -4. Shapely-1.8.1-cp37-cp37m-win_amd64.whl -5. geopandas-0.10.2-py2.py3-none-any.whl - -With these files in hand, the following installation instructions work: +conda install gdal +conda install pyproj +conda install fiona +conda install shapely +conda install geopandas -```bat -conda create -n tm2py python=3.7.6 -conda activate tm2py -pip install [the packages listed above, in that order] cd +git pip install -e . conda env config vars set GDAL_VERSION=3.3.2 ``` @@ -51,9 +41,17 @@ Finally, install the Emme python packages using the Emme GUI. This effectively c `C:\Users\%USERNAME%\.conda\envs\tm2py\Lib\site-packages\emme.pth` with the following contents, so you could create the file yourself. ```python -import os, site; site.addsitedir("C:/Program Files/INRO/Emme/Emme 4/Emme-4.6.0/Python37/Lib/site-packages") +import os, site; site.addsitedir(os.path.join(os.environ["EMMEPATH"], "Python311/Lib/site-packages")) +``` + +*This should start Emme OpenPath, if it does not you should be able to manually set the correct version of emme such as below* +```python +import os, site +os.environ["EMMEPATH"] = r"C:\Program Files\Bentley\OpenPaths\EMME 24.00.00" +site.addsitedir(os.path.join(os.environ["EMMEPATH"], "Python311/Lib/site-packages")) ``` + In troubleshooting, sometimes DLL load failure errors would occur which may be resolved by importing gdal before importing emme packages. Emme support explained this thusly: At load time, the EMME API will always load the geos_c co-located with the EMME API, unless it was already loaded from some other location, which is the case when you import GDAL first. EMME API seems to be compatible with the newer GDAL/geos_c (reminder: not tested!). But this does not appear to be the case the other way around (newer GDAL is not compatible with older geos_c). diff --git a/requirements.txt b/requirements.txt index 18b275f7..0e100c6a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,9 +25,9 @@ osmnx >= 0.12 pandas > 1.0 pydantic < 2.0 pyproj > 2.2.0 -pywin32==224 ; sys_platform == 'win32' +pywin32==306 ; sys_platform == 'win32' pyyaml -pywin32==224 ; sys_platform == 'win32' +pywin32==306 ; sys_platform == 'win32' rtree scipy shapely diff --git a/scripts/compare_skims.py b/scripts/compare_skims.py index 82c95ca6..ad1cd8dd 100644 --- a/scripts/compare_skims.py +++ b/scripts/compare_skims.py @@ -25,10 +25,10 @@ def read_matrix_as_long_df(path: Path, run_name): ) -a = read_matrix_as_long_df( - r"D:\TEMP\TM2.2.1.1-New_network_rerun\TM2.2.1.1_new_taz\skim_matrices\highway\HWYSKMAM_taz.omx", - "test", -) +# a = read_matrix_as_long_df( +# r"D:\TEMP\TM2.2.1.1-New_network_rerun\TM2.2.1.1_new_taz\skim_matrices\highway\HWYSKMAM_taz.omx", +# "test", +# ) # %% all_skims = [] for skim_matrix_path in network_fid_path.rglob("*AM_taz.omx"): diff --git a/scripts/compile_model_runs.py b/scripts/compile_model_runs.py index 553a3656..4d3c74ac 100644 --- a/scripts/compile_model_runs.py +++ b/scripts/compile_model_runs.py @@ -19,8 +19,9 @@ # print("writing") # input[["#link_id", "geometry"]].to_file(output_dir / "test_geom.geojson") -scenarios_to_consolidate = (11, 12, 13, 14, 15) -runs_to_consolidate = (3, 4) +scenarios_to_consolidate = (12,) +runs_to_consolidate = (15, 22, 26, 27) +# runs_to_consolidate = (15, 22) # %% @@ -42,7 +43,7 @@ def read_file_and_tag( return None run = file.parent.parent.stem - run_number = int(run.split("_")[-1]) + run_number = int(run.split("_")[1]) if run_number not in runs_to_consolidate: return None @@ -153,12 +154,140 @@ def combine_tables(dfs, columns_same): links_wide_table["direction"] = links_wide_table["geometry"].apply( get_linestring_direction ) +#%% +# a little side quest + +rename_dict = { + 15: "run 15 old code, emme 4.6.1", + 22: "run 22 previous version of pr, open paths", + 26: "run 26 final version of pr, emme 4.6.1", + 27: "run 27, PR with dropping maz no drive network", +} +ft_map = { + 1.: "Freeway", + 2.: "Expressway", + 3.: "Ramp", + 4.: "Divided Arterial", + 5.: "Undivided Arterial", + 6.: "Collector", + 7.: "Local", + 8.: "Connector", + 99.: "Service Road " +} + +all_tables = pd.concat(all_links_no_none) +print(pd.crosstab(all_tables["run_number"].map(rename_dict), all_tables["@ft"]).rename(columns=ft_map).to_markdown()) + # %% ft_cols = [col for col in links_wide_table.columns if "ft_" in col] links_wide_table["ft"] = links_wide_table[ft_cols].max(axis=1) links_wide_table = links_wide_table.drop(columns=ft_cols) +#%% +plotting_table = links_wide_table.head(10_000_000).dropna() +print(plotting_table.shape) +import matplotlib.pyplot as plt +from scipy import stats +for i in range(1, 7): + slicer = plotting_table["ft"] == i + x = plotting_table.loc[slicer, "@volau_run15_scenAM"] + y = plotting_table.loc[slicer, "@volau_run27_scenAM"] + plt.scatter(x, y, label=f'ft = {i}') + plt.xlabel("run 15") + plt.ylabel("run 27") + print(stats.linregress(x, y)) + print(i) + # plt.show() +plt.legend() +#%% + +no_vol_links = links_wide_table[(links_wide_table["@volau_run27_scenAM"] < 1) & (links_wide_table["ft"] == 1)] +slicer = (no_vol_links["@volau_run15_scenAM"] > 1) +no_vol_links[slicer] +#%% +vol_pairs_to_compare = [ + ("@volau_run15_scenAM", "@volau_run27_scenAM"), + # ("@volau_run22_scenAM", "@volau_run26_scenAM"), + # ("@volau_run23_scenAM", "@volau_run25_scenAM"), +] +rename_dict = { + "@volau_run15_scenAM": "run 15 old code, emme 4.6.1", + "@volau_run22_scenAM": "run 22 previous version of pr, open paths", + "@volau_run23_scenAM": "emme Open Paths", + "@volau_run24_scenAM": "emme Open Paths Network Accelerate", + "@volau_run25_scenAM": "remove Cosmetic Nodes", + "@volau_run26_scenAM": "run 26 final version of pr, emme 4.6.1" +} + +# vs 22 +# add rmse +# add total vmt for vmt +# check number fo links +ft_map = { + 1: "Freeway", + 2: "Expressway", + 3: "Ramp", + 4: "Divided Arterial", + 5: "Undivided Arterial", + 6: "Collector", + 7: "Local", + 8: "Connector" +} +# vol_pairs_to_compare = [ +# ("@volau_run15_scenAM", "@volau_run22_scenAM"), +# ] + +bases = [] +comparisons = [] +facility_type = [] +slopes = [] +r_vals = [] + +rmse = [] +base_vmt = [] +comparison_vmt = [] + + +for base, compare in vol_pairs_to_compare: + print(base, compare) + stats_table = links_wide_table.dropna() + for i in range(1, 7): + slicer = stats_table["ft"] == i + x = stats_table.loc[slicer, base] + y = stats_table.loc[slicer, compare] + lingress = stats.linregress(x, y) + + bases.append(base) + comparisons.append(compare) + facility_type.append(i) + slopes.append(lingress.slope) + r_vals.append(lingress.rvalue) + rmse.append(round(((x-y)**2).mean()**0.5,2)) + + base_vmt.append(x.sum()) + comparison_vmt.append(y.sum()) + +df = pd.DataFrame.from_dict( + dict( + base = bases, + comparison = comparisons, + facility_type = facility_type, + slopes = slopes, + r_vals = r_vals, + rmse = rmse, + base_vmt = base_vmt, + comparison_vmt = comparison_vmt, + ) +) +df["base"] = df["base"].map(rename_dict) +df["comparison"] = df["comparison"].map(rename_dict) +df["slopes"] = df["slopes"].round(2) +df["r_vals"] = df["r_vals"].round(2) +df["facility_type"] = df["facility_type"].map(ft_map) +print(df.to_markdown()) +#%% +import pyperclip # %% links_wide_table.to_file( Path( diff --git a/tm2py/components/network/highway/highway_assign.py b/tm2py/components/network/highway/highway_assign.py index fa2091c2..7035249d 100644 --- a/tm2py/components/network/highway/highway_assign.py +++ b/tm2py/components/network/highway/highway_assign.py @@ -332,7 +332,8 @@ def _get_assignment_spec( "normalized_gap": 0.0, }, "performance_settings": { - "number_of_processors": self.controller.num_processors + "number_of_processors": self.controller.num_processors, + "network_acceleration": self.config.network_acceleration, }, } if not path_analysis: diff --git a/tm2py/config.py b/tm2py/config.py index fed43e3b..cf00ada3 100644 --- a/tm2py/config.py +++ b/tm2py/config.py @@ -957,6 +957,7 @@ class HighwayConfig(ConfigItem): generic_highway_mode_code: str = Field(min_length=1, max_length=1) relative_gaps: Tuple[HighwayRelativeGapConfig, ...] = Field() max_iterations: int = Field(ge=0) + network_acceleration: bool = Field() area_type_buffer_dist_miles: float = Field(gt=0) drive_access_output_skim_path: Optional[str] = Field(default=None) output_skim_path: pathlib.Path = Field()