From 10b79b71184e79c509f812a2ed160022186d5623 Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Mon, 31 Oct 2022 10:57:03 -0700 Subject: [PATCH 1/3] Put import gdal statement in inro_context fixture - ran pre-commit - PLEASE TEST ON EMME ENVIRONMENT to see if it resolves the gdal issue @lmz @i-am-sijia --- tests/conftest.py | 36 ++++++++++++++++++++---------------- tests/test_trucks.py | 2 ++ 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index f80238da..01765b30 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,9 +2,8 @@ import os import sys from pathlib import Path + import pytest -# why import gdal first: https://github.com/BayAreaMetro/tm2py/blob/7a563f0c5cea2125f28bfaedc50205e70c532094/README.md?plain=1#L57 -import gdal print("CONFTEST LOADED") @@ -50,20 +49,21 @@ def temp_dir(): def pytest_addoption(parser): """Parse command line arguments.""" parser.addoption("--inro", action="store", default="notmock") - print('pytest_addoption') + print("pytest_addoption") def mocked_inro_context(): import unittest.mock + """Mocking of modules which need to be mocked for tests.""" - sys.modules["inro.emme.database.emmebank"] = unittest.mock.MagicMock() - sys.modules["inro.emme.network"] = unittest.mock.MagicMock() - sys.modules["inro.emme.database.scenario"] = unittest.mock.MagicMock() - sys.modules["inro.emme.database.matrix"] = unittest.mock.MagicMock() - sys.modules["inro.emme.network.node"] = unittest.mock.MagicMock() - sys.modules["inro.emme.desktop.app"] = unittest.mock.MagicMock() - sys.modules["inro"] = unittest.mock.MagicMock() - sys.modules["inro.modeller"] = unittest.mock.MagicMock() + sys.modules["inro.emme.database.emmebank"] = unittest.mock.MagicMock() + sys.modules["inro.emme.network"] = unittest.mock.MagicMock() + sys.modules["inro.emme.database.scenario"] = unittest.mock.MagicMock() + sys.modules["inro.emme.database.matrix"] = unittest.mock.MagicMock() + sys.modules["inro.emme.network.node"] = unittest.mock.MagicMock() + sys.modules["inro.emme.desktop.app"] = unittest.mock.MagicMock() + sys.modules["inro"] = unittest.mock.MagicMock() + sys.modules["inro.modeller"] = unittest.mock.MagicMock() @pytest.fixture(scope="session") @@ -73,21 +73,25 @@ def inro_context(pytestconfig): try: # obey command line option _inro = pytestconfig.getoption("inro") - print('_inro = [{}]'.format(_inro)) + print("_inro = [{}]".format(_inro)) if _inro.lower() == "mock": print("Mocking inro environment.") mocked_inro_context() else: + # why import gdal first: https://github.com/BayAreaMetro/tm2py/blob/7a563f0c5cea2125f28bfaedc50205e70c532094/README.md?plain=1#L57 + import gdal import inro.emme.database.emmebank print("Using inro environment.") - except Exception as inst: - print(type(inst)) # the exception instance - print(inst.args) # arguments stored in .args - print(inst) # __str__ allows args to be printed directly, + except Exception as inst: + print(type(inst)) # the exception instance + print(inst.args) # arguments stored in .args + print(inst) # __str__ allows args to be printed directly, # if commandline option fails, try using Emme and then failing that, using Mock try: + # why import gdal first: https://github.com/BayAreaMetro/tm2py/blob/7a563f0c5cea2125f28bfaedc50205e70c532094/README.md?plain=1#L57 + import gdal import inro.emme.database.emmebank print("Using inro environment.") diff --git a/tests/test_trucks.py b/tests/test_trucks.py index 098ba2d1..3ec0f025 100644 --- a/tests/test_trucks.py +++ b/tests/test_trucks.py @@ -1,7 +1,9 @@ "Test commercial vehicle model." import os + import pytest + from tm2py.examples import get_example From 606c6b6268586b8a421d84ea8a87e649272cc575 Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Mon, 31 Oct 2022 12:52:10 -0700 Subject: [PATCH 2/3] Get tests working - remove/update dependencies - create ctramp context - inro_context returns values to use for determinations - use inro_context to determine which parts of tests can be run --- bin/tm2py | 2 +- docs/api.md | 2 +- mkdocs.yml | 4 ---- requirements.txt | 2 +- tests/conftest.py | 50 ++++++++++++++++++++++++++++++++++++++++ tests/test_air_access.py | 5 +++- tests/test_externals.py | 3 --- tests/test_highway.py | 17 ++++++++++++++ tests/test_household.py | 5 +++- tests/test_network.py | 5 ++++ tests/test_run_flow.py | 21 ----------------- tests/test_trucks.py | 3 +++ 12 files changed, 86 insertions(+), 33 deletions(-) diff --git a/bin/tm2py b/bin/tm2py index db2cf8d2..2169f9ca 100644 --- a/bin/tm2py +++ b/bin/tm2py @@ -2,7 +2,7 @@ import argparse -# this is mysterious but appears to resolve DLL load errors that otherwise occur +# why import gdal first: https://github.com/BayAreaMetro/tm2py/blob/7a563f0c5cea2125f28bfaedc50205e70c532094/README.md?plain=1#L57 import gdal from tm2py import RunController diff --git a/docs/api.md b/docs/api.md index b92b8127..765dbbc2 100644 --- a/docs/api.md +++ b/docs/api.md @@ -25,7 +25,7 @@ Personal travel demand generated by household members. ::: tm2py.components.demand.household -::: tm2py.config.HouseholdConfig +::: tm2py.HouseholdConfig #### Air Passenger Demand diff --git a/mkdocs.yml b/mkdocs.yml index a8557749..88d6d4be 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -70,14 +70,10 @@ extra: extra_javascript: - https://unpkg.com/mermaid@8.5.0/dist/mermaid.min.js -extra_css: - - https://use.fontawesome.com/releases/v5.13.0/css/all.css - markdown_extensions: - admonition - codehilite: linenums: true - - fontawesome_markdown - meta - pymdownx.inlinehilite - pymdownx.tasklist: diff --git a/requirements.txt b/requirements.txt index 63af6215..f885c62d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ mkdocs-awesome-pages-plugin mkdocs-jupyter mkdocs-macros-plugin mkdocs-material -mkdocstrings +mkdocstrings >= 0.18 mkdocstrings-python notebook numpy diff --git a/tests/conftest.py b/tests/conftest.py index 01765b30..43620b3c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -77,12 +77,14 @@ def inro_context(pytestconfig): if _inro.lower() == "mock": print("Mocking inro environment.") mocked_inro_context() + return "mock" else: # why import gdal first: https://github.com/BayAreaMetro/tm2py/blob/7a563f0c5cea2125f28bfaedc50205e70c532094/README.md?plain=1#L57 import gdal import inro.emme.database.emmebank print("Using inro environment.") + return "inro" except Exception as inst: print(type(inst)) # the exception instance print(inst.args) # arguments stored in .args @@ -95,6 +97,54 @@ def inro_context(pytestconfig): import inro.emme.database.emmebank print("Using inro environment.") + return "inro" except ModuleNotFoundError: print("Mocking inro environment.") mocked_inro_context() + return "mock" + +@pytest.fixture(scope="session") +def ctramp_context(pytestconfig): + """Identifies if ctramp is available.""" + import subprocess + try: + from tm2py.tools import run_process + commands = [ + "CALL CTRAMP\\runtime\\CTRampEnv.bat", + "set PATH=%CD%\\CTRAMP\runtime;C:\\Windows\\System32;%JAVA_PATH%\bin;" + "%TPP_PATH%;%PYTHON_PATH%;%PYTHON_PATH%\\condabin;%PYTHON_PATH%\\envs", + 'CALL CTRAMP\runtime\runMtxMgr.cmd %HOST_IP_ADDRESS% "%JAVA_PATH%"', + ] + run_process(commands, name="start_matrix_manager") + + print("Using ctramp environment.") + return "ctramp" + except subprocess.CalledProcessError: + print("No ctramp environment.") + return None + + +@pytest.fixture(scope="session") +def union_city(examples_dir, root_dir, inro_context): + """Union City model run testing fixture.""" + from tm2py.controller import RunController + from tm2py.examples import get_example + + EXAMPLE = "UnionCity" + _example_root = examples_dir / EXAMPLE + + get_example(example_name="UnionCity", root_dir=root_dir) + controller = RunController( + [ + os.path.join(examples_dir, "scenario_config.toml"), + os.path.join(examples_dir, "model_config.toml"), + ], + run_dir=_example_root, + ) + + #------Below this line, need Inro's Emme installed----- + if inro_context != "inro": return + #------------------------------------------------------ + + controller.run() + return controller diff --git a/tests/test_air_access.py b/tests/test_air_access.py index 9ecceb1f..0108717f 100644 --- a/tests/test_air_access.py +++ b/tests/test_air_access.py @@ -7,7 +7,6 @@ from tm2py.examples import get_example -@pytest.mark.menow def test_air_pax_model(inro_context, examples_dir, root_dir): "Tests that airport access model be run." from tools import test_component @@ -15,6 +14,10 @@ def test_air_pax_model(inro_context, examples_dir, root_dir): get_example(example_name="UnionCity", root_dir=root_dir) my_run = test_component(examples_dir, "air_passenger") + + #------Below this line, need Inro's Emme installed----- + if inro_context != "inro": return + my_run.run_next() # TODO write assert diff --git a/tests/test_externals.py b/tests/test_externals.py index e3d5fb1f..fc9bf9d4 100644 --- a/tests/test_externals.py +++ b/tests/test_externals.py @@ -6,8 +6,6 @@ from tm2py.examples import get_example - -# @pytest.mark.menow def test_external_travel(examples_dir, root_dir): "Tests that internal/external travel component can be run." from tools import test_component @@ -15,6 +13,5 @@ def test_external_travel(examples_dir, root_dir): get_example(example_name="UnionCity", root_dir=root_dir) my_run = test_component(examples_dir, "internal_external") - my_run.run_next() # TODO write assert diff --git a/tests/test_highway.py b/tests/test_highway.py index d4fbbbbf..3ce61c51 100644 --- a/tests/test_highway.py +++ b/tests/test_highway.py @@ -11,6 +11,9 @@ def test_prepare_highway_network(inro_context, examples_dir): from tools import test_component my_run = test_component(examples_dir, "prepare_network_highway") + + #------Below this line, need Inro's Emme installed----- + if inro_context != "inro": return my_run.run_next() # TODO write assert @@ -21,6 +24,10 @@ def test_highway_assign(inro_context, examples_dir): from tools import test_component my_run = test_component(examples_dir, "highway") + + #------Below this line, need Inro's Emme installed----- + if inro_context != "inro": return + my_run.run_next() # TODO write assert @@ -31,6 +38,10 @@ def test_highway_maz_assign(inro_context, examples_dir): from tools import test_component my_run = test_component(examples_dir, "highway_maz_assign") + + #------Below this line, need Inro's Emme installed----- + if inro_context != "inro": return + my_run.run_next() # TODO write assert @@ -48,6 +59,9 @@ def test_highway_maz_skim(inro_context, examples_dir): def test_highway_skims(inro_context, union_city): """Test that the OMX highway skims match the reference.""" + #------Below this line, need Inro's Emme installed----- + if inro_context != "inro": return + run_dir = union_city.run_dir ref_dir_hwy_skims = os.path.join(run_dir, "ref_skim_matrices", "highway") @@ -79,6 +93,9 @@ def test_highway_skims(inro_context, union_city): def test_maz_da_skims(inro_context, union_city): """Test that the DA MAZ skims match the reference.""" + #------Below this line, need Inro's Emme installed----- + if inro_context != "inro": return + run_dir = union_city.run_dir ref_dir_hwy_skims = os.path.join(run_dir, "ref_skim_matrices", "highway") diff --git a/tests/test_household.py b/tests/test_household.py index 74b9b1e4..8bdb73f4 100644 --- a/tests/test_household.py +++ b/tests/test_household.py @@ -6,11 +6,14 @@ @pytest.mark.skipci -def test_household_travel(examples_dir): +def test_household_travel(ctramp_context,examples_dir): "Tests that household travel component can be run." from tools import test_component my_run = test_component(examples_dir, "household") + + #------Below this line, need Inro's Emme installed----- + if ctramp_context != "ctramp": return my_run.run_next() # TODO write assert diff --git a/tests/test_network.py b/tests/test_network.py index 5136664b..26104b80 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -12,3 +12,8 @@ def test_get_blended_skim(inro_context): # TODO pass + + + #------Below this line, need Inro's Emme installed----- + if inro_context != "inro": return + diff --git a/tests/test_run_flow.py b/tests/test_run_flow.py index 75ead6a1..8d2d1014 100644 --- a/tests/test_run_flow.py +++ b/tests/test_run_flow.py @@ -42,27 +42,6 @@ def test_example_download(examples_dir, root_dir, inro_context): assert not (os.path.exists(os.path.join(example_root, "test_data.zip"))) -@pytest.fixture(scope="session") -def union_city(examples_dir, root_dir, inro_context): - """Union City model run testing fixture.""" - from tm2py.controller import RunController - from tm2py.examples import get_example - - EXAMPLE = "UnionCity" - _example_root = os.path.join(examples_dir, EXAMPLE) - - get_example(example_name="UnionCity", root_dir=root_dir) - controller = RunController( - [ - os.path.join(examples_dir, "scenario_config.toml"), - os.path.join(examples_dir, "model_config.toml"), - ], - run_dir=_example_root, - ) - controller.run() - return controller - - def test_validate_input_fail(examples_dir, inro_context, temp_dir): """Test that validate_input fails when required inputs are missing.""" import toml diff --git a/tests/test_trucks.py b/tests/test_trucks.py index 3ec0f025..86b16334 100644 --- a/tests/test_trucks.py +++ b/tests/test_trucks.py @@ -14,6 +14,9 @@ def test_commercial_vehicle(inro_context, examples_dir, root_dir): get_example(example_name="UnionCity", root_dir=root_dir) my_run = test_component(examples_dir, "truck") + + #------Below this line, need Inro's Emme installed----- + if inro_context != "inro": return my_run.run_next() # TODO write assert From fcf25fb070567b8afbf719cb1bc870640982e75e Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Mon, 31 Oct 2022 13:19:45 -0700 Subject: [PATCH 3/3] Update test_highway.py --- tests/test_highway.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_highway.py b/tests/test_highway.py index 3ce61c51..bcab9eeb 100644 --- a/tests/test_highway.py +++ b/tests/test_highway.py @@ -27,6 +27,7 @@ def test_highway_assign(inro_context, examples_dir): #------Below this line, need Inro's Emme installed----- if inro_context != "inro": return + print("!!!!!!!INRO CONTEXT ",inro_context) my_run.run_next() @@ -56,7 +57,7 @@ def test_highway_maz_skim(inro_context, examples_dir): # TODO write assert - +@pytest.mark.menow def test_highway_skims(inro_context, union_city): """Test that the OMX highway skims match the reference.""" #------Below this line, need Inro's Emme installed----- @@ -90,7 +91,7 @@ def test_highway_skims(inro_context, union_city): assert len(missing_skims) == 0, f"Missing skims: {missing_skims}" assert len(different_skims) == 0, f"Different skims: {different_skims}" - +@pytest.mark.menow def test_maz_da_skims(inro_context, union_city): """Test that the DA MAZ skims match the reference.""" #------Below this line, need Inro's Emme installed-----