Add running examples snakemake

.github/workflows/staging-snt.yml

@@ -19,7 +19,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: '3.8'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

Jenkinsfile

@@ -59,8 +59,8 @@ podTemplate(
 				def wheelFile = sh(returnStdout: true, script: "find ./dist -name '*.whl'").toString().trim()
 				//def wheelFile = sh(returnStdout: true, script: "python3 ./.github/scripts/get_wheel_filename.py --package-file package_setup.py").toString().trim()
 				echo "This is the package file: ${wheelFile}"
-				sh "pip3 install $wheelFile --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple"
-				 
+				//sh "pip3 install $wheelFile --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple"
+				sh "pip3 install $wheelFile --index-url=https://[email protected]:[email protected]/api/pypi/pypi-staging/simple --pre"
 				//sh "pip3 install dataclasses"
 				sh 'pip3 install keyrings.alt'
 				sh "pip3 freeze"
@@ -108,7 +108,7 @@ podTemplate(
 			echo "Running examples"
 				dir('examples') {
 					sh 'pip3 install snakemake'
-					sh 'snakemake --cores=10 --config python_version=python3'
+					sh 'snakemake --cores=1 --config python_version=python3'
 				}
 			}
 

examples/ben/example/run_1960-2004/run_simulations.py

@@ -42,9 +42,10 @@ def _post_run(experiment: Experiment, **kwargs):
     Return:
         None
     """
-    with open("ben\\example\\run_1960-2004\\experiment_id.txt", "w") as fd:
-        fd.write(experiment.uid.hex)
-    pass
+    wait_until_done = kwargs.get('wait_until_done', None)
+    if wait_until_done and experiment.succeeded:
+        with open("ben/example/run_1960-2004/experiment_id.txt", "w") as fd:
+            fd.write(experiment.uid.hex)
 
 
 def _config_experiment(**kwargs):
@@ -87,7 +88,7 @@ def run_experiment(**kwargs):
 
     experiment = _config_experiment(**kwargs)
     _pre_run(experiment, **kwargs)
-    experiment.run(wait_until_done=True, wait_on_done=False)
+    experiment.run(**kwargs)
     _post_run(experiment, **kwargs)
 
 
@@ -110,4 +111,4 @@ def run_experiment(**kwargs):
     # dtk.setup(pathlib.Path(manifest.eradication_path).parent)
     # os.chdir(os.path.dirname(__file__))
     # print("...done.")
-    run_experiment(show_warnings_once=True)
+    run_experiment(show_warnings_once=True, wait_until_done=True)

examples/ben/example/run_2005-2022/run_simulations.py

@@ -42,9 +42,10 @@ def _post_run(experiment: Experiment, **kwargs):
     Return:
         None
     """
-    with open("ben\\example\\run_2005-2022\\experiment_id.txt", "w") as fd:
-        fd.write(experiment.uid.hex)
-    pass
+    wait_until_done = kwargs.get('wait_until_done', None)
+    if wait_until_done and experiment.succeeded:
+        with open("ben/example/run_2005-2022/experiment_id.txt", "w") as fd:
+            fd.write(experiment.uid.hex)
 
 
 def _config_experiment(**kwargs):
@@ -87,7 +88,7 @@ def run_experiment(**kwargs):
 
     experiment = _config_experiment(**kwargs)
     _pre_run(experiment, **kwargs)
-    experiment.run(wait_until_done=True, wait_on_done=False)
+    experiment.run(**kwargs)
     _post_run(experiment, **kwargs)
 
 
@@ -110,4 +111,4 @@ def run_experiment(**kwargs):
     # dtk.setup(pathlib.Path(manifest.eradication_path).parent)
     # os.chdir(os.path.dirname(__file__))
     # print("...done.")
-    run_experiment(show_warnings_once=True)
+    run_experiment(show_warnings_once=True, wait_until_done=True)

examples/monique/calibration/baseline_calibration/01_serialize_transmission_sweep/run_simulations.py

@@ -40,11 +40,11 @@ def _post_run(experiment: Experiment, **kwargs):
     Return:
         None
     """
-    # Save experiment id to file to be used by snakefile
-    with open("monique\\calibration\\baseline_calibration\\01_serialize_transmission_sweep\\experiment_id.txt", "w") as fd:
-        fd.write(experiment.uid.hex)
-
-    pass
+    wait_until_done = kwargs.get('wait_until_done', None)
+    if wait_until_done and experiment.succeeded:
+        # Save experiment id to file to be used by snakefile
+        with open("monique/calibration/baseline_calibration/01_serialize_transmission_sweep/experiment_id.txt", "w") as fd:
+            fd.write(experiment.uid.hex)
 
 
 def _config_experiment(**kwargs):
@@ -87,7 +87,7 @@ def run_experiment(**kwargs):
 
     experiment = _config_experiment(**kwargs)
     _pre_run(experiment, **kwargs)
-    experiment.run(wait_until_done=True, wait_on_done=False)
+    experiment.run(**kwargs)
     _post_run(experiment, **kwargs)
 
 
@@ -106,4 +106,4 @@ def run_experiment(**kwargs):
     # dtk.setup(pathlib.Path(manifest.eradication_path).parent)
     # os.chdir(os.path.dirname(__file__))
     # print("...done.")
-    run_experiment(show_warnings_once=True)
+    run_experiment(show_warnings_once=True, wait_until_done=True)

examples/monique/calibration/baseline_calibration/02_run_transmission_sweep/run_simulations.py

@@ -39,10 +39,11 @@ def _post_run(experiment: Experiment, **kwargs):
     Return:
         None
     """
-    # Save experiment id to file to be used by snakefile
-    with open(r"monique\\calibration\\baseline_calibration\\02_run_transmission_sweep\\experiment_id.txt", "w") as fd:
-        fd.write(experiment.uid.hex)
-    pass
+    wait_until_done = kwargs.get('wait_until_done', None)
+    if wait_until_done and experiment.succeeded:
+        # Save experiment id to file to be used by snakefile
+        with open("monique/calibration/baseline_calibration/02_run_transmission_sweep/experiment_id.txt", "w") as fd:
+            fd.write(experiment.uid.hex)
 
 
 def _config_experiment(**kwargs):
@@ -85,7 +86,7 @@ def run_experiment(**kwargs):
 
     experiment = _config_experiment(**kwargs)
     _pre_run(experiment, **kwargs)
-    experiment.run(wait_until_done=True, wait_on_done=False)
+    experiment.run(**kwargs)
     _post_run(experiment, **kwargs)
 
 
@@ -104,4 +105,4 @@ def run_experiment(**kwargs):
     # dtk.setup(pathlib.Path(manifest.eradication_path).parent)
     # os.chdir(os.path.dirname(__file__))
     # print("...done.")
-    run_experiment(show_warnings=False)
+    run_experiment(show_warnings=False, wait_until_done=True)

examples/monique/calibration/seasonality_calibration/01_burnin_for_seasonalityCalib/run_simulations.py

@@ -38,10 +38,10 @@ def _post_run(experiment: Experiment, **kwargs):
     Return:
         None
     """
-    with open("monique\\calibration\\seasonality_calibration\\01_burnin_for_seasonalityCalib\\experiment_id.txt", "w") as fd:
-        fd.write(experiment.uid.hex)
-
-    pass
+    wait_until_done = kwargs.get('wait_until_done', None)
+    if wait_until_done and experiment.succeeded:
+        with open("monique/calibration/seasonality_calibration/01_burnin_for_seasonalityCalib/experiment_id.txt", "w") as fd:
+            fd.write(experiment.uid.hex)
 
 
 def _config_experiment(**kwargs):
@@ -84,7 +84,7 @@ def run_experiment(**kwargs):
 
     experiment = _config_experiment(**kwargs)
     _pre_run(experiment, **kwargs)
-    experiment.run(wait_until_done=True, wait_on_done=False)
+    experiment.run(**kwargs)
     _post_run(experiment, **kwargs)
 
 
@@ -103,4 +103,4 @@ def run_experiment(**kwargs):
     # dtk.setup(pathlib.Path(manifest.eradication_path).parent)
     # os.chdir(os.path.dirname(__file__))
     # print("...done.")
-    run_experiment(show_warnings_once=True)
+    run_experiment(show_warnings_once=True, wait_until_done=True)

examples/monique/calibration/seasonality_calibration/02_seasonality_calibration/run_calibration.py

@@ -158,6 +158,23 @@ def get_manager(**kwargs):
     return calib_manager
 
 
+def _post_run(suite_id, **kwargs):
+    """
+    Add extra work after run calibration.
+    Args:
+        suite_id:
+        kwargs: additional parameters
+    Return:
+        None
+    """
+    from idmtools.core import ItemType
+    suite = platform.get_item(suite_id, ItemType.SUITE)
+
+    if all([exp.succeeded for exp in suite.experiments]):
+        with open("monique/calibration/seasonality_calibration/02_seasonality_calibration/experiment_id.txt", "w") as fd:
+            fd.write("Calibration good.")
+
+
 def run_calibration(directory: str = '.', **kwargs):
     """
     Get configured calibration and run.
@@ -176,6 +193,7 @@ def run_calibration(directory: str = '.', **kwargs):
     calib_manager = get_manager(**kwargs)
     _pre_run(**kwargs)
     calib_manager.run_calibration(**kwargs)
+    _post_run(calib_manager.suite_id, **kwargs)
 
 
 if __name__ == "__main__":

examples/monique/run_future_scenarios/run_simulations.py

@@ -39,9 +39,10 @@ def _post_run(experiment: Experiment, **kwargs):
     Return:
         None
     """
-    with open("monique\\run_future_scenarios\\experiment_id.txt", "w") as fd:
-        fd.write(experiment.uid.hex)
-    pass
+    wait_until_done = kwargs.get('wait_until_done', None)
+    if wait_until_done and experiment.succeeded:
+        with open("monique/run_future_scenarios/experiment_id.txt", "w") as fd:
+            fd.write(experiment.uid.hex)
 
 
 def _config_experiment(**kwargs):
@@ -103,4 +104,4 @@ def run_experiment(**kwargs):
     # dtk.setup(pathlib.Path(manifest.eradication_path).parent)
     # os.chdir(os.path.dirname(__file__))
     # print("...done.")
-    run_experiment(show_warnings_once=True)
+    run_experiment(show_warnings_once=True, wait_until_done=True)

examples/monique/run_to_present/run_simulations.py

@@ -39,8 +39,10 @@ def _post_run(experiment: Experiment, **kwargs):
     Return:
         None
     """
-    with open("monique\\run_to_present\\experiment_id.txt", "w") as fd:
-        fd.write(experiment.uid.hex)
+    wait_until_done = kwargs.get('wait_until_done', None)
+    if wait_until_done and experiment.succeeded:
+        with open("monique\\run_to_present\\experiment_id.txt", "w") as fd:
+            fd.write(experiment.uid.hex)
     pass
 
 
@@ -103,4 +105,4 @@ def run_experiment(**kwargs):
     # dtk.setup(pathlib.Path(manifest.eradication_path).parent)
     # os.chdir(os.path.dirname(__file__))
     # print("...done.")
-    run_experiment(show_warnings_once=True)
+    run_experiment(show_warnings_once=True, wait_until_done=True)

examples/update_lines.py

@@ -7,12 +7,12 @@ def update_parameters_in_file(filepath, parameters: dict = None):
         Helper function that updates file parameters to run examples in snakemake
     Args:
         filepath:
-        parameters: a dictionary formatted as: {"what to find in line": "what to replace the line with"}
+        parameters: a dictionary formatted as: {"what to find in line": "what to replace the entire line with"}
 
     Returns:
 
     """
-    temp = "temp.ptxt"
+    temp = "temp.txt"
     with open(temp, 'w') as new_file:
         with open(filepath) as old_file:
             for line in old_file:

requirements.txt

@@ -1,5 +1,6 @@
 -i https://packages.idmod.org/api/pypi/pypi-production/simple
-emodpy-malaria~=3.1.1
+emodpy-malaria
+requests==2.29.0
 idmtools_calibra~=1.0.9
 idmtools-platform-comps~=1.7.7
 scipy