From 699e98d52b3f4f1eeed75dd66764fe44666c4409 Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Wed, 31 Jan 2024 15:58:22 -0500 Subject: [PATCH 01/23] feat: tests for steps This commit updates the test-B01_SL_load_single_file.yml workflow file. It changes the name of the workflow to "Test steps" and updates the runs-on value to "ubuntu-latest". Additionally, it adds a step to enable Git LFS and installs pytest and its dependencies. Finally, it updates the run command for the first step and adds a new step to run tests using pytest. --- .../test-B01_SL_load_single_file.yml | 14 +- tests/test_steps.py | 309 ++++++++++++++++++ 2 files changed, 317 insertions(+), 6 deletions(-) create mode 100644 tests/test_steps.py diff --git a/.github/workflows/test-B01_SL_load_single_file.yml b/.github/workflows/test-B01_SL_load_single_file.yml index e56b6389..c8330863 100644 --- a/.github/workflows/test-B01_SL_load_single_file.yml +++ b/.github/workflows/test-B01_SL_load_single_file.yml @@ -1,4 +1,4 @@ -name: Test B01_SL_load_single_file +name: Test steps on: pull_request: {} push: @@ -9,11 +9,13 @@ jobs: fail-fast: false matrix: version: ['3.11'] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: install mpi run: sudo apt update && sudo apt-get install openmpi-bin openmpi-doc libopenmpi-dev - uses: actions/checkout@v3 + with: + lfs: true - name: setup python uses: actions/setup-python@v4 with: @@ -21,10 +23,10 @@ jobs: cache: "pip" - name: install icesat2-tracks using pip run: pip install . - - name: List dependencies - run: pip list - - name: first step B01_SL_load_single_file - run: python src/icesat2_tracks/analysis_db/B01_SL_load_single_file.py --track-name 20190502052058_05180312_005_01 --batch-key SH_testSLsinglefile2 --output-dir ./work + - name: install pytest + run: pip install pytest pytest-xdist pytest-sugar pytest-durations + - name: Run tests + run: pytest --capture=sys --verbose --showlocals --tb=long --numprocesses=auto tests/test_steps.py - name: second step make_spectra run: python src/icesat2_tracks/analysis_db/B02_make_spectra_gFT.py SH_20190502_05180312 SH_testSLsinglefile2 True - name: third step plot_spectra diff --git a/tests/test_steps.py b/tests/test_steps.py new file mode 100644 index 00000000..006cb2c4 --- /dev/null +++ b/tests/test_steps.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python +from datetime import datetime +from pathlib import Path +import shutil +import subprocess +import tarfile +from tempfile import mkdtemp + + +def checkpaths(paths): + result = [Path(pth).is_file() for pth in paths] + print("\n") + # report which files are missing (in red) and which are present (in green) + for pth, res in zip(paths, result): + if res: + print(f"\033[92m{pth} is present\033[0m") + else: + print(f"\033[91m{pth} is missing\033[0m") + return all(result) + + +def get_all_filenames(directory): + """ + Get a list of all file names in a directory + """ + return [p.name for p in Path(directory).rglob("*")] + + +def check_file_exists(directory, prefix, stepno: int = 4): + # Get a list of all files in the directory + files = get_all_filenames(targetdirs[str(stepno)] / directory) + # Check if there is a file with the specified prefix + file_exists = any(file.startswith(prefix) for file in files) + return file_exists + + +def delete_pdf_files(directory): + files = [file for file in Path(directory).iterdir() if file.suffix == ".pdf"] + delete_files(files) + + +def delete_files(file_paths): + for file_path in file_paths: + delete_file(file_path) + + +def delete_file(file_path): + path = Path(file_path) + if path.exists(): + path.unlink() + + +def getoutputdir(script): + outputdir = script.index("--output-dir") + 1 + return script[outputdir] + + +def extract_tarball(outputdir, tarball_path): + tar = tarfile.open(Path(tarball_path)) + tar.extractall(Path(outputdir), filter="data") + tar.close() + + +def run_test(script, paths, delete_paths=True, suppress_output=True): + # configuration + outputdir = getoutputdir(script) + + # update paths to check + paths = [Path(outputdir, pth) for pth in paths] + + if delete_paths: + delete_files(paths) + + kwargs = {"check": True} + if suppress_output: + kwargs["stdout"] = subprocess.DEVNULL + kwargs["stderr"] = subprocess.DEVNULL + + # run the script + subprocess.run(script, **kwargs) + + # run the tests + result = checkpaths(paths) + + return result + + +def makepathlist(dir, files): + return [Path(dir, f) for f in files] + + +# The scriptx variables are the scripts to be tested. The pathsx variables are the paths to the files that should be produced by the scripts. The scripts are run and the paths are checked to see whether the expected files were produced. If the files were produced, the test passes. If not, the test fails. + + +script1 = [ + "python", + "src/icesat2_tracks/analysis_db/B01_SL_load_single_file.py", + "--track-name", + "20190502052058_05180312_005_01", + "--batch-key", + "SH_testSLsinglefile2", + "--output-dir", +] +paths1 = [ + "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312/B01_track.png.png", + "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312/B01b_ATL06_corrected.png.png", + "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312/B01b_beam_statistics.png.png", + "work/SH_testSLsinglefile2/A01b_ID/A01b_ID_SH_20190502_05180312.json", + "work/SH_testSLsinglefile2/B01_regrid/SH_20190502_05180312_B01_binned.h5", +] + + +script2 = [ + "python", + "src/icesat2_tracks/analysis_db/B02_make_spectra_gFT.py", + "--track-name", + "SH_20190502_05180312", + "--batch-key", + "SH_testSLsinglefile2", + "--output-dir", +] +paths2 = [ + "work/SH_testSLsinglefile2/B02_spectra/B02_SH_20190502_05180312_params.h5", + "work/SH_testSLsinglefile2/B02_spectra/B02_SH_20190502_05180312_gFT_x.nc", + "work/SH_testSLsinglefile2/B02_spectra/B02_SH_20190502_05180312_gFT_k.nc", + "work/SH_testSLsinglefile2/B02_spectra/B02_SH_20190502_05180312_FFT.nc", +] + +script3 = [ + "python", + "src/icesat2_tracks/analysis_db/B03_plot_spectra_ov.py", + "--track-name", + "SH_20190502_05180312", + "--batch-key", + "SH_testSLsinglefile2", + "--id-flag", + "--output-dir", +] + +_root = "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312" +_paths3 = [ + "B03_specs_L25000.0.png", + "B03_specs_coord_check.png", + "B03_success.json", +] +paths3 = makepathlist(_root, _paths3) + +script4 = [ + "python", + "src/icesat2_tracks/analysis_db/A02c_IOWAGA_thredds_prior.py", + "--track-name", + "SH_20190502_05180312", + "--batch-key", + "SH_testSLsinglefile2", + "--id-flag", + "--output-dir", +] +paths4 = [ + "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312/A02_SH_2_hindcast_data.pdf", + "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312/A02_SH_2_hindcast_prior.pdf", +] +dir4, prefix4 = ( + "work/SH_testSLsinglefile2/A02_prior/", + "A02_SH_20190502_05180312_hindcast", +) + +# TODO: step 5 +script5 = [ + "python", + "src/icesat2_tracks/analysis_db/B04_angle.py", + "SH_20190502_05180312", + "SH_testSLsinglefile2", + "True", +] +paths5 = [ + "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312/B04_success.json", + "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312/B04_prior_angle.png", + "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312/B04_marginal_distributions.pdf", + "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312/B04_data_avail.pdf", + "work/SH_testSLsinglefile2/B04_angle/B04_SH_20190502_05180312_res_table.h5", + "work/SH_testSLsinglefile2/B04_angle/B04_SH_20190502_05180312_marginals.nc", +] + +scripts = [script1, script2, script3, script4, script5] +targetdirs = ( + dict() +) # to be populated in setup_module with the target dirs for each step +__outdir = [] # to be populated in setup_module with the temp dir for all steps + + +def setup_module(): + """ + Set up the module for testing. + + This function makes a temporary directory with subdirectories with the required input data for each step. + + ```shell + $ tree -L 1 tests/tmpcpn_6tne + tests/tmpcpn_6tne + ├── step1 + ├── step2 + ├── step3 + ├── step4 + └── step5 + + 5 directories, 0 files + ``` + + When running in parallel using the xdist plugin, each worker will have its own copy of all the input data. This is necessary because the tests are run in parallel and the input data is modified by the tests. This way, the teardown function can delete the temporary directory for each worker without affecting the other workers. + """ + + homedir = Path(__file__).parent + input_data_dir = homedir / "testdata" + timestamp = datetime.now().strftime("%Y%m%d%H%M%S") + _outdir = mkdtemp(dir=homedir, suffix=timestamp) # temp dir for all steps + __outdir.append(_outdir) + + # make temp dirs for each step in _outdir + # the one for step1 with no input data + tmpstep1 = Path(_outdir) / "step1" + tmpstep1.mkdir() + script1.append(tmpstep1) + + # make temp dirs for steps 2,... with input data + for tarball in input_data_dir.glob("*.tar.gz"): + source_script, for_step_num = tarball.stem.split("-for-step-") + for_step_num = for_step_num.split(".")[0] + targetdir = Path(_outdir) / f"step{for_step_num}" + targetdirs[for_step_num] = targetdir + extract_tarball(targetdir, tarball) + + # Extracted files are in targetdir/script_name. Move them to its parent targetdir. Delete the script_name dir. + parent = targetdir / "work" + + # Rename and update parent to targetdir / script_name + new_parent = Path(targetdir, source_script) + parent.rename(new_parent) + + for child in new_parent.iterdir(): + if child.is_dir(): + shutil.move(str(child), Path(child.parent).parent) + shutil.rmtree(new_parent) + + # add the target dirs to the scripts + for i, script in enumerate(scripts[1:], start=2): + script.append(targetdirs[str(i)]) + + # throw in tmpstep1 in targetdirs to clean up later + targetdirs["1"] = tmpstep1 + + +def teardown_module(): + """ + Clean up after testing is complete. + """ + shutil.rmtree(__outdir[-1]) + + +def test_step1(): + # Step 1: B01_SL_load_single_file.py ~ 2 minutes + assert run_test(script1, paths1, delete_paths=False) # passing + + +# def test_step2(): +# # Step 2: B02_make_spectra_gFT.py ~ 2 min +# assert run_test(script2, paths2) # passing + + +# def check_B03_freq_reconst_x(): +# outputdir = getoutputdir(script3) +# directory = Path( +# outputdir, "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312B03_spectra/" +# ) +# files = get_all_filenames(directory) + +# # Check there are 5 pdf files +# return len([f for f in files if f.endswith("pdf")]) == 5 + + +# def test_step3(): +# # Step 3: B03_plot_spectra_ov.py ~ 11 sec +# # This script has stochastic behavior, so the files produced don't always have the same names but the count of pdf files is constant for the test input data. +# t1 = run_test(script3, paths3) +# t2 = check_B03_freq_reconst_x() +# assert t1 +# assert t2 + + +# def test_step4(): +# # Step 4: A02c_IOWAGA_thredds_prior.py ~ 23 sec +# t1 = run_test(script4, paths4) +# t2 = check_file_exists(dir4, prefix4) +# assert t1 +# assert t2 + + +# TODO: step 5 after first merge of PR +# def test_step5(): +# # Step 5: B04_angle.py ~ 9 min +# assert run_test(script5, paths5) + +if __name__ == "__main__": + setup_module() + test_step1() # passing + # test_step2() # passing + # test_step3() # passing + # test_step4() # passing + # test_step5() + teardown_module() From 413484553d5c4e3a33b24595c1373a0ba2e89812 Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Thu, 1 Feb 2024 09:24:17 -0500 Subject: [PATCH 02/23] Add new files and directories to .gitignore --- .gitignore | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a6f0b6af..fe9f3c10 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,10 @@ analysis_db/makefile data_prehandling/ track_lists/ explorations/ +/tests/tests/inputdata +/tests/input-data +__explorations +logs analysis_db/support_files/ *.egg-info/ .installed.cfg @@ -45,6 +49,7 @@ analysis_db/support_files/ # Environments .env .venv +.venv39/ env/ venv/ ENV/ @@ -57,4 +62,4 @@ dist/ #visual code .vscode/ -*.h5 \ No newline at end of file +*.h5 From a937cbc71b23c47c77441dac1b89825d056bb8d5 Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Thu, 1 Feb 2024 09:37:26 -0500 Subject: [PATCH 03/23] feat: handle tar.gz via lfs --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..f087b429 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.tar.gz filter=lfs diff=lfs merge=lfs -text From f870e1b5b4a224bd35c11046f69101990bbd08c6 Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Thu, 1 Feb 2024 09:39:36 -0500 Subject: [PATCH 04/23] feat: add test data files for steps 2-5 --- tests/testdata/A02c_IOWAGA_thredds_prior-for-step-5.tar.gz | 3 +++ tests/testdata/B01_SL_load_single_file-for-step-2.tar.gz | 3 +++ tests/testdata/B02_make_spectra_gFT-for-step-3.tar.gz | 3 +++ tests/testdata/B03_plot_spectra_ov-for-step-4.tar.gz | 3 +++ 4 files changed, 12 insertions(+) create mode 100644 tests/testdata/A02c_IOWAGA_thredds_prior-for-step-5.tar.gz create mode 100644 tests/testdata/B01_SL_load_single_file-for-step-2.tar.gz create mode 100644 tests/testdata/B02_make_spectra_gFT-for-step-3.tar.gz create mode 100644 tests/testdata/B03_plot_spectra_ov-for-step-4.tar.gz diff --git a/tests/testdata/A02c_IOWAGA_thredds_prior-for-step-5.tar.gz b/tests/testdata/A02c_IOWAGA_thredds_prior-for-step-5.tar.gz new file mode 100644 index 00000000..02358666 --- /dev/null +++ b/tests/testdata/A02c_IOWAGA_thredds_prior-for-step-5.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:025d7d31692273fec660fe9402e34dc62ba2ded613e19616dda7c8e87b4bcef1 +size 23512596 diff --git a/tests/testdata/B01_SL_load_single_file-for-step-2.tar.gz b/tests/testdata/B01_SL_load_single_file-for-step-2.tar.gz new file mode 100644 index 00000000..b6b48cce --- /dev/null +++ b/tests/testdata/B01_SL_load_single_file-for-step-2.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dac3b06877b9301191d5eeb30eaad69ebc4cde26d8f96da1ed3c99e41d0a3b0e +size 11682469 diff --git a/tests/testdata/B02_make_spectra_gFT-for-step-3.tar.gz b/tests/testdata/B02_make_spectra_gFT-for-step-3.tar.gz new file mode 100644 index 00000000..6b5041f4 --- /dev/null +++ b/tests/testdata/B02_make_spectra_gFT-for-step-3.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:327110408aea28e61a691a7e7e5df0db13f8e73801c7ac9ddec118c27e66e0e3 +size 22822613 diff --git a/tests/testdata/B03_plot_spectra_ov-for-step-4.tar.gz b/tests/testdata/B03_plot_spectra_ov-for-step-4.tar.gz new file mode 100644 index 00000000..a168e8e1 --- /dev/null +++ b/tests/testdata/B03_plot_spectra_ov-for-step-4.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ef492d4b31f5ef22e33fc6831d255bbfdfac70ce504f4f09a031c8e7a041fde +size 23456276 From b7db6e7c081e496e020e05ac34c02a3d8f2e1bcc Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Thu, 1 Feb 2024 10:11:09 -0500 Subject: [PATCH 05/23] refactor: test_steps --- tests/test_steps.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_steps.py b/tests/test_steps.py index 006cb2c4..441bae09 100644 --- a/tests/test_steps.py +++ b/tests/test_steps.py @@ -26,9 +26,9 @@ def get_all_filenames(directory): return [p.name for p in Path(directory).rglob("*")] -def check_file_exists(directory, prefix, stepno: int = 4): +def check_file_exists(directory, prefix, stepnum: int = 4): # Get a list of all files in the directory - files = get_all_filenames(targetdirs[str(stepno)] / directory) + files = get_all_filenames(targetdirs[str(stepnum)] / directory) # Check if there is a file with the specified prefix file_exists = any(file.startswith(prefix) for file in files) return file_exists @@ -89,7 +89,7 @@ def makepathlist(dir, files): return [Path(dir, f) for f in files] -# The scriptx variables are the scripts to be tested. The pathsx variables are the paths to the files that should be produced by the scripts. The scripts are run and the paths are checked to see whether the expected files were produced. If the files were produced, the test passes. If not, the test fails. +# The `scriptx` variables are the scripts to be tested. The `pathsx` variables are the paths to the files that should be produced by the scripts. The scripts are run and the paths are checked to see whether the expected files were produced. If the files were produced, the test passes. If not, the test fails. script1 = [ @@ -195,8 +195,8 @@ def setup_module(): This function makes a temporary directory with subdirectories with the required input data for each step. ```shell - $ tree -L 1 tests/tmpcpn_6tne - tests/tmpcpn_6tne + $ tree -L 1 tests/tempdir + tests/tempdir ├── step1 ├── step2 ├── step3 @@ -261,6 +261,7 @@ def test_step1(): assert run_test(script1, paths1, delete_paths=False) # passing +# TODO: for steps 2-5 after their respective prs are merged # def test_step2(): # # Step 2: B02_make_spectra_gFT.py ~ 2 min # assert run_test(script2, paths2) # passing @@ -294,7 +295,6 @@ def test_step1(): # assert t2 -# TODO: step 5 after first merge of PR # def test_step5(): # # Step 5: B04_angle.py ~ 9 min # assert run_test(script5, paths5) From 52046b94dc537c4c67f892214007fa1c15858b7a Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Thu, 1 Feb 2024 10:41:01 -0500 Subject: [PATCH 06/23] fix: remove old script runs outside test suite --- .github/workflows/test-B01_SL_load_single_file.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/test-B01_SL_load_single_file.yml b/.github/workflows/test-B01_SL_load_single_file.yml index c8330863..811e85db 100644 --- a/.github/workflows/test-B01_SL_load_single_file.yml +++ b/.github/workflows/test-B01_SL_load_single_file.yml @@ -27,11 +27,3 @@ jobs: run: pip install pytest pytest-xdist pytest-sugar pytest-durations - name: Run tests run: pytest --capture=sys --verbose --showlocals --tb=long --numprocesses=auto tests/test_steps.py - - name: second step make_spectra - run: python src/icesat2_tracks/analysis_db/B02_make_spectra_gFT.py SH_20190502_05180312 SH_testSLsinglefile2 True - - name: third step plot_spectra - run: python src/icesat2_tracks/analysis_db/B03_plot_spectra_ov.py SH_20190502_05180312 SH_testSLsinglefile2 True - - name: fourth step IOWAGA thredds - run: python src/icesat2_tracks/analysis_db/A02c_IOWAGA_thredds_prior.py SH_20190502_05180312 SH_testSLsinglefile2 True - - name: Fifth step B04_angle - run: python src/icesat2_tracks/analysis_db/B04_angle.py SH_20190502_05180312 SH_testSLsinglefile2 True From 4d2f223633496773dd34765abdd2f3220b2dd25d Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Thu, 1 Feb 2024 11:21:53 -0500 Subject: [PATCH 07/23] fix: remove unnecessary steps from and rename to tests --- .../{test-B01_SL_load_single_file.yml => tests.yml} | 8 -------- 1 file changed, 8 deletions(-) rename .github/workflows/{test-B01_SL_load_single_file.yml => tests.yml} (58%) diff --git a/.github/workflows/test-B01_SL_load_single_file.yml b/.github/workflows/tests.yml similarity index 58% rename from .github/workflows/test-B01_SL_load_single_file.yml rename to .github/workflows/tests.yml index c8330863..811e85db 100644 --- a/.github/workflows/test-B01_SL_load_single_file.yml +++ b/.github/workflows/tests.yml @@ -27,11 +27,3 @@ jobs: run: pip install pytest pytest-xdist pytest-sugar pytest-durations - name: Run tests run: pytest --capture=sys --verbose --showlocals --tb=long --numprocesses=auto tests/test_steps.py - - name: second step make_spectra - run: python src/icesat2_tracks/analysis_db/B02_make_spectra_gFT.py SH_20190502_05180312 SH_testSLsinglefile2 True - - name: third step plot_spectra - run: python src/icesat2_tracks/analysis_db/B03_plot_spectra_ov.py SH_20190502_05180312 SH_testSLsinglefile2 True - - name: fourth step IOWAGA thredds - run: python src/icesat2_tracks/analysis_db/A02c_IOWAGA_thredds_prior.py SH_20190502_05180312 SH_testSLsinglefile2 True - - name: Fifth step B04_angle - run: python src/icesat2_tracks/analysis_db/B04_angle.py SH_20190502_05180312 SH_testSLsinglefile2 True From c97573ddea55a7f03de4ce3909dd00b0802fb6fd Mon Sep 17 00:00:00 2001 From: Camilo Diaz Date: Tue, 13 Feb 2024 05:47:03 -0500 Subject: [PATCH 08/23] added test dependecies --- pyproject.toml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 479de43b..d27ef6f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,8 +122,6 @@ dependencies = [ # Optional "siphon >=0.9, <1.0.0", "h5py >=3.5.0, < 4.0.0", "termcolor >=2.4.0, < 3.0.0", - "pytest >=7.4.4, < 8.0.0", - "pytest-xdist >=3.5.0, < 4.0.0", "typer >=0.9.0, < 1.0.0", ] @@ -137,7 +135,11 @@ dependencies = [ # Optional # projects. [project.optional-dependencies] # Optional dev = ["check-manifest","black","icesat2-tracks[test]"] -test = ["coverage"] +test = [ "pytest >=7.4.4, < 8.0.0", + "pytest-xdist >=3.5.0, < 4.0.0", + "coverage", + "pytest-sugar", + "pytest-durations"] # List URLs that are relevant to your project # From 0d1dbf2fc2c9db75338635b210d2b7883100cece Mon Sep 17 00:00:00 2001 From: Camilo Diaz Date: Thu, 15 Feb 2024 23:17:44 -0500 Subject: [PATCH 09/23] renamed github workflow file --- ...1_SL_load_single_file.yml => test_icesat2_tracks_pipeline.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{test-B01_SL_load_single_file.yml => test_icesat2_tracks_pipeline.yml} (100%) diff --git a/.github/workflows/test-B01_SL_load_single_file.yml b/.github/workflows/test_icesat2_tracks_pipeline.yml similarity index 100% rename from .github/workflows/test-B01_SL_load_single_file.yml rename to .github/workflows/test_icesat2_tracks_pipeline.yml From 2eb706323905f1cf57c8b88b688fad699ebe5689 Mon Sep 17 00:00:00 2001 From: Camilo Diaz Date: Thu, 15 Feb 2024 23:30:22 -0500 Subject: [PATCH 10/23] restoring test.yml file --- .github/workflows/tests.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..811e85db --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,29 @@ +name: Test steps +on: + pull_request: {} + push: + branches: [ main ] +jobs: + python_run_scripts: + strategy: + fail-fast: false + matrix: + version: ['3.11'] + runs-on: ubuntu-latest + steps: + - name: install mpi + run: sudo apt update && sudo apt-get install openmpi-bin openmpi-doc libopenmpi-dev + - uses: actions/checkout@v3 + with: + lfs: true + - name: setup python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.version }} # install the python version needed + cache: "pip" + - name: install icesat2-tracks using pip + run: pip install . + - name: install pytest + run: pip install pytest pytest-xdist pytest-sugar pytest-durations + - name: Run tests + run: pytest --capture=sys --verbose --showlocals --tb=long --numprocesses=auto tests/test_steps.py From 980d30be7c422f42172bf543a38b2ffc9250c832 Mon Sep 17 00:00:00 2001 From: Camilo Diaz Date: Fri, 16 Feb 2024 23:37:26 -0500 Subject: [PATCH 11/23] fixed test step 3 --- tests/test_steps.py | 85 +++++++++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 37 deletions(-) diff --git a/tests/test_steps.py b/tests/test_steps.py index 441bae09..bf46f027 100644 --- a/tests/test_steps.py +++ b/tests/test_steps.py @@ -168,9 +168,12 @@ def makepathlist(dir, files): script5 = [ "python", "src/icesat2_tracks/analysis_db/B04_angle.py", + "--track-name", "SH_20190502_05180312", + "--batch-key", "SH_testSLsinglefile2", - "True", + "--id-flag", + "--output-dir", ] paths5 = [ "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312/B04_success.json", @@ -225,15 +228,19 @@ def setup_module(): for tarball in input_data_dir.glob("*.tar.gz"): source_script, for_step_num = tarball.stem.split("-for-step-") for_step_num = for_step_num.split(".")[0] - targetdir = Path(_outdir) / f"step{for_step_num}" - targetdirs[for_step_num] = targetdir - extract_tarball(targetdir, tarball) + target_output_dir = Path(_outdir) / f"step{for_step_num}" + targetdirs[for_step_num] = target_output_dir + print("Target Dir") + print(target_output_dir) + print("TarBall") + print(tarball) + extract_tarball(target_output_dir, tarball) # Extracted files are in targetdir/script_name. Move them to its parent targetdir. Delete the script_name dir. - parent = targetdir / "work" + parent = target_output_dir / "work" # Rename and update parent to targetdir / script_name - new_parent = Path(targetdir, source_script) + new_parent = Path(target_output_dir, source_script) parent.rename(new_parent) for child in new_parent.iterdir(): @@ -262,48 +269,52 @@ def test_step1(): # TODO: for steps 2-5 after their respective prs are merged -# def test_step2(): -# # Step 2: B02_make_spectra_gFT.py ~ 2 min -# assert run_test(script2, paths2) # passing +def test_step2(): + # Step 2: B02_make_spectra_gFT.py ~ 2 min + assert run_test(script2, paths2) # passing -# def check_B03_freq_reconst_x(): -# outputdir = getoutputdir(script3) -# directory = Path( -# outputdir, "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312B03_spectra/" -# ) -# files = get_all_filenames(directory) +def check_B03_freq_reconst_x(): + outputdir = getoutputdir(script3) + directory = Path( + outputdir, "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312/B03_spectra/" + ) + print("directory") + print(directory) + files = get_all_filenames(directory) + print("files") + print(files) -# # Check there are 5 pdf files -# return len([f for f in files if f.endswith("pdf")]) == 5 + # Check there are 5 pdf files + return len([f for f in files if f.endswith("pdf")]) == 5 -# def test_step3(): -# # Step 3: B03_plot_spectra_ov.py ~ 11 sec -# # This script has stochastic behavior, so the files produced don't always have the same names but the count of pdf files is constant for the test input data. -# t1 = run_test(script3, paths3) -# t2 = check_B03_freq_reconst_x() -# assert t1 -# assert t2 +def test_step3(): + # Step 3: B03_plot_spectra_ov.py ~ 11 sec + # This script has stochastic behavior, so the files produced don't always have the same names but the count of pdf files is constant for the test input data. + t1 = run_test(script3, paths3) + t2 = check_B03_freq_reconst_x() + assert t1 + assert t2 -# def test_step4(): -# # Step 4: A02c_IOWAGA_thredds_prior.py ~ 23 sec -# t1 = run_test(script4, paths4) -# t2 = check_file_exists(dir4, prefix4) -# assert t1 -# assert t2 +def test_step4(): + # Step 4: A02c_IOWAGA_thredds_prior.py ~ 23 sec + t1 = run_test(script4, paths4) + t2 = check_file_exists(dir4, prefix4) + assert t1 + assert t2 -# def test_step5(): -# # Step 5: B04_angle.py ~ 9 min -# assert run_test(script5, paths5) +def test_step5(): + # Step 5: B04_angle.py ~ 9 min + assert run_test(script5, paths5) if __name__ == "__main__": setup_module() test_step1() # passing - # test_step2() # passing - # test_step3() # passing - # test_step4() # passing - # test_step5() + test_step2() # passing + test_step3() # passing + test_step4() # passing + test_step5() teardown_module() From 830bbc73077e4bceee080d7e7010b48729e61f0f Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Fri, 23 Feb 2024 15:19:13 -0500 Subject: [PATCH 12/23] test: tests for steps 5-7 --- tests/test_steps.py | 183 +++++++++++------- ...02c_IOWAGA_thredds_prior-for-step-5.tar.gz | 4 +- tests/testdata/B04_angle-for-step-6.tar.gz | 3 + .../B05_define_angle-for-step-7.tar.gz | 3 + 4 files changed, 123 insertions(+), 70 deletions(-) create mode 100644 tests/testdata/B04_angle-for-step-6.tar.gz create mode 100644 tests/testdata/B05_define_angle-for-step-7.tar.gz diff --git a/tests/test_steps.py b/tests/test_steps.py index bf46f027..6c96584a 100644 --- a/tests/test_steps.py +++ b/tests/test_steps.py @@ -27,6 +27,9 @@ def get_all_filenames(directory): def check_file_exists(directory, prefix, stepnum: int = 4): + """ + This is needed because step 4 produces files with different names even when using the same input data. + """ # Get a list of all files in the directory files = get_all_filenames(targetdirs[str(stepnum)] / directory) # Check if there is a file with the specified prefix @@ -61,6 +64,19 @@ def extract_tarball(outputdir, tarball_path): tar.close() +def create_script(script_name): + head = ["python"] + tail = [ + "--track-name", + "SH_20190502_05180312", + "--batch-key", + "SH_testSLsinglefile2", + "--output-dir", + ] + base_path = "src/icesat2_tracks/analysis_db/" + return head + [f"{base_path}{script_name}.py"] + tail + + def run_test(script, paths, delete_paths=True, suppress_output=True): # configuration outputdir = getoutputdir(script) @@ -91,7 +107,7 @@ def makepathlist(dir, files): # The `scriptx` variables are the scripts to be tested. The `pathsx` variables are the paths to the files that should be produced by the scripts. The scripts are run and the paths are checked to see whether the expected files were produced. If the files were produced, the test passes. If not, the test fails. - +# Script 1 is different than the others because it doesn't have any input data and uses a different track name. script1 = [ "python", "src/icesat2_tracks/analysis_db/B01_SL_load_single_file.py", @@ -109,34 +125,30 @@ def makepathlist(dir, files): "work/SH_testSLsinglefile2/B01_regrid/SH_20190502_05180312_B01_binned.h5", ] - -script2 = [ - "python", - "src/icesat2_tracks/analysis_db/B02_make_spectra_gFT.py", - "--track-name", - "SH_20190502_05180312", - "--batch-key", - "SH_testSLsinglefile2", - "--output-dir", +script_names_2_to_7 = [ + "B02_make_spectra_gFT", + "B03_plot_spectra_ov", + "A02c_IOWAGA_thredds_prior", + "B04_angle", + "B05_define_angle", + "B06_correct_separate_var", ] -paths2 = [ - "work/SH_testSLsinglefile2/B02_spectra/B02_SH_20190502_05180312_params.h5", - "work/SH_testSLsinglefile2/B02_spectra/B02_SH_20190502_05180312_gFT_x.nc", - "work/SH_testSLsinglefile2/B02_spectra/B02_SH_20190502_05180312_gFT_k.nc", - "work/SH_testSLsinglefile2/B02_spectra/B02_SH_20190502_05180312_FFT.nc", + +script2, script3, script4, script5, script6, script7 = [ + create_script(name) for name in script_names_2_to_7 ] -script3 = [ - "python", - "src/icesat2_tracks/analysis_db/B03_plot_spectra_ov.py", - "--track-name", - "SH_20190502_05180312", - "--batch-key", - "SH_testSLsinglefile2", - "--id-flag", - "--output-dir", +# Define the paths for the second script +_root = "work/SH_testSLsinglefile2/B02_spectra/" +_paths2 = [ + "B02_SH_20190502_05180312_params.h5", + "B02_SH_20190502_05180312_gFT_x.nc", + "B02_SH_20190502_05180312_gFT_k.nc", + "B02_SH_20190502_05180312_FFT.nc", ] +paths2 = makepathlist(_root, _paths2) +# Define the paths for the third script _root = "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312" _paths3 = [ "B03_specs_L25000.0.png", @@ -145,46 +157,75 @@ def makepathlist(dir, files): ] paths3 = makepathlist(_root, _paths3) -script4 = [ - "python", - "src/icesat2_tracks/analysis_db/A02c_IOWAGA_thredds_prior.py", - "--track-name", - "SH_20190502_05180312", - "--batch-key", - "SH_testSLsinglefile2", - "--id-flag", - "--output-dir", -] + paths4 = [ "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312/A02_SH_2_hindcast_data.pdf", "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312/A02_SH_2_hindcast_prior.pdf", -] +] # deterministic paths dir4, prefix4 = ( "work/SH_testSLsinglefile2/A02_prior/", "A02_SH_20190502_05180312_hindcast", -) +) # stochastic paths + +# Define the paths for the fifth script +_plotroot = "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312" +_plotnames = [ + "B04_success.json", + "B04_prior_angle.png", + "B04_marginal_distributions.pdf", + "B04_data_avail.pdf", +] +_workroot = "work/SH_testSLsinglefile2/B04_angle" +_worknames = [ + "B04_SH_20190502_05180312_res_table.h5", + "B04_SH_20190502_05180312_marginals.nc", +] +paths5 = makepathlist(_plotroot, _plotnames) + makepathlist(_workroot, _worknames) -# TODO: step 5 -script5 = [ - "python", - "src/icesat2_tracks/analysis_db/B04_angle.py", - "--track-name", - "SH_20190502_05180312", - "--batch-key", - "SH_testSLsinglefile2", - "--id-flag", - "--output-dir", +# Define the paths for the sixth script +base_path = "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312" +paths6 = [ + f"{base_path}/B05_angle/B05_weighted_marginals_x{i}.pdf" + for i in range(612, 788, 25) ] -paths5 = [ - "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312/B04_success.json", - "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312/B04_prior_angle.png", - "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312/B04_marginal_distributions.pdf", - "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312/B04_data_avail.pdf", - "work/SH_testSLsinglefile2/B04_angle/B04_SH_20190502_05180312_res_table.h5", - "work/SH_testSLsinglefile2/B04_angle/B04_SH_20190502_05180312_marginals.nc", +other_files = [ + f"{base_path}/B05_dir_ov.pdf", + f"{base_path}/B05_success.json", + "work/SH_testSLsinglefile2/B04_angle/B05_SH_20190502_05180312_angle_pdf.nc", ] +paths6.extend(other_files) + + +# Paths for the seventh step +root_plots = "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312/" +root_work = "work/SH_testSLsinglefile2/" +b06_correction = f"{root_plots}B06_correction/" +b06_corrected_separated = f"{root_work}B06_corrected_separated/" + +b06_correction_files = [ + "B06_angle_def.png", + "SH_20190502_05180312_B06_atten_ov.pdf", + "SH_20190502_05180312_B06_atten_ov.png", + "SH_20190502_05180312_B06_atten_ov_simple.pdf", + "SH_20190502_05180312_B06_atten_ov_simple.png", +] + +b06_corrected_separated_files = [ + "B06_SH_20190502_05180312_B06_corrected_resid.h5", + "B06_SH_20190502_05180312_binned_resid.h5", + "B06_SH_20190502_05180312_gFT_k_corrected.nc", + "B06_SH_20190502_05180312_gFT_x_corrected.nc", +] + +paths7 = [f"{b06_correction}{file}" for file in b06_correction_files] +paths7.extend( + f"{b06_corrected_separated}{file}" for file in b06_corrected_separated_files +) +paths7.append(f"{root_plots}B06_success.json") + +# These are the scripts to be tested and appended with the target directories for each step +scripts = [script1, script2, script3, script4, script5, script6, script7] -scripts = [script1, script2, script3, script4, script5] targetdirs = ( dict() ) # to be populated in setup_module with the target dirs for each step @@ -230,10 +271,6 @@ def setup_module(): for_step_num = for_step_num.split(".")[0] target_output_dir = Path(_outdir) / f"step{for_step_num}" targetdirs[for_step_num] = target_output_dir - print("Target Dir") - print(target_output_dir) - print("TarBall") - print(tarball) extract_tarball(target_output_dir, tarball) # Extracted files are in targetdir/script_name. Move them to its parent targetdir. Delete the script_name dir. @@ -268,7 +305,6 @@ def test_step1(): assert run_test(script1, paths1, delete_paths=False) # passing -# TODO: for steps 2-5 after their respective prs are merged def test_step2(): # Step 2: B02_make_spectra_gFT.py ~ 2 min assert run_test(script2, paths2) # passing @@ -279,11 +315,7 @@ def check_B03_freq_reconst_x(): directory = Path( outputdir, "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312/B03_spectra/" ) - print("directory") - print(directory) files = get_all_filenames(directory) - print("files") - print(files) # Check there are 5 pdf files return len([f for f in files if f.endswith("pdf")]) == 5 @@ -300,7 +332,9 @@ def test_step3(): def test_step4(): # Step 4: A02c_IOWAGA_thredds_prior.py ~ 23 sec + # check deterministic paths t1 = run_test(script4, paths4) + # check stochastic paths t2 = check_file_exists(dir4, prefix4) assert t1 assert t2 @@ -310,11 +344,24 @@ def test_step5(): # Step 5: B04_angle.py ~ 9 min assert run_test(script5, paths5) + +def test_step6(): + # Step 5: B05_define_angle.py ~ + assert run_test(script6, paths6) + + +def test_step7(): + # Step 7: B06_correct_separate_var.py ~ + assert run_test(script7, paths7) + + if __name__ == "__main__": setup_module() - test_step1() # passing - test_step2() # passing - test_step3() # passing - test_step4() # passing - test_step5() + # test_step1() # passing + # test_step2() # passing + # test_step3() # passing + # test_step4() # passing + # test_step5() # passing + # test_step6() # passing + # test_step7() # passing teardown_module() diff --git a/tests/testdata/A02c_IOWAGA_thredds_prior-for-step-5.tar.gz b/tests/testdata/A02c_IOWAGA_thredds_prior-for-step-5.tar.gz index 02358666..765a34a7 100644 --- a/tests/testdata/A02c_IOWAGA_thredds_prior-for-step-5.tar.gz +++ b/tests/testdata/A02c_IOWAGA_thredds_prior-for-step-5.tar.gz @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:025d7d31692273fec660fe9402e34dc62ba2ded613e19616dda7c8e87b4bcef1 -size 23512596 +oid sha256:e2f85180db57be8762c874e0a6956523af8815d01d96f5186b01819d4835b087 +size 28942200 diff --git a/tests/testdata/B04_angle-for-step-6.tar.gz b/tests/testdata/B04_angle-for-step-6.tar.gz new file mode 100644 index 00000000..beba4ef6 --- /dev/null +++ b/tests/testdata/B04_angle-for-step-6.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7da28afa41dbbc115baaafcfaedd33d615ddd8a80ff56189cc90241b89f245e0 +size 29401005 diff --git a/tests/testdata/B05_define_angle-for-step-7.tar.gz b/tests/testdata/B05_define_angle-for-step-7.tar.gz new file mode 100644 index 00000000..407a61d5 --- /dev/null +++ b/tests/testdata/B05_define_angle-for-step-7.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01cc2b1479d57c001608f0532bc90c00f7a8fab1bcf46153488304ac63fb6401 +size 30190599 From de3201320397c0206ecc9365aa3966b6597a4da4 Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Fri, 23 Feb 2024 15:26:31 -0500 Subject: [PATCH 13/23] test: update CI workflow to use test suite --- .../test_icesat2_tracks_pipeline.yml | 19 ++++-------- .github/workflows/tests.yml | 29 ------------------- 2 files changed, 5 insertions(+), 43 deletions(-) delete mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/test_icesat2_tracks_pipeline.yml b/.github/workflows/test_icesat2_tracks_pipeline.yml index 216f2b9c..1ae037f3 100644 --- a/.github/workflows/test_icesat2_tracks_pipeline.yml +++ b/.github/workflows/test_icesat2_tracks_pipeline.yml @@ -34,17 +34,8 @@ jobs: icesat2waves define-angle --help icesat2waves correct-separate --help # prelim name - - name: first step B01_SL_load_single_file - run: load-file --track-name 20190502052058_05180312_005_01 --batch-key SH_testSLsinglefile2 --output-dir ./work - - name: second step make_spectra - run: make-spectra --track-name SH_20190502_05180312 --batch-key SH_testSLsinglefile2 --output-dir ./work - - name: third step plot_spectra - run: plot-spectra --track-name SH_20190502_05180312 --batch-key SH_testSLsinglefile2 --output-dir ./work - - name: fouth step IOWAGA threads - run: make-iowaga-threads-prior --track-name SH_20190502_05180312 --batch-key SH_testSLsinglefile2 --output-dir ./work - - name: fifth step B04_angle - run: make-b04-angle --track-name SH_20190502_05180312 --batch-key SH_testSLsinglefile2 --output-dir ./work - - name: sixth step B04_define_angle - run: define-angle --track-name SH_20190502_05180312 --batch-key SH_testSLsinglefile2 --output-dir ./work - - name: seventh step B06_correct_separate - run: correct-separate --track-name SH_20190502_05180312 --batch-key SH_testSLsinglefile2 --output-dir ./work \ No newline at end of file + - name: install pytest + run: pip install pytest pytest-xdist pytest-sugar pytest-durations + - name: Run tests + run: pytest --capture=sys --verbose --showlocals --tb=long --numprocesses=auto tests/test_steps.py + \ No newline at end of file diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index 811e85db..00000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Test steps -on: - pull_request: {} - push: - branches: [ main ] -jobs: - python_run_scripts: - strategy: - fail-fast: false - matrix: - version: ['3.11'] - runs-on: ubuntu-latest - steps: - - name: install mpi - run: sudo apt update && sudo apt-get install openmpi-bin openmpi-doc libopenmpi-dev - - uses: actions/checkout@v3 - with: - lfs: true - - name: setup python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.version }} # install the python version needed - cache: "pip" - - name: install icesat2-tracks using pip - run: pip install . - - name: install pytest - run: pip install pytest pytest-xdist pytest-sugar pytest-durations - - name: Run tests - run: pytest --capture=sys --verbose --showlocals --tb=long --numprocesses=auto tests/test_steps.py From 4d1f98ed59ad2c55b193583683afe259f23fa065 Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Fri, 23 Feb 2024 16:05:08 -0500 Subject: [PATCH 14/23] fix: add lfs --- .github/workflows/test_icesat2_tracks_pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test_icesat2_tracks_pipeline.yml b/.github/workflows/test_icesat2_tracks_pipeline.yml index 1ae037f3..22f07dad 100644 --- a/.github/workflows/test_icesat2_tracks_pipeline.yml +++ b/.github/workflows/test_icesat2_tracks_pipeline.yml @@ -14,6 +14,8 @@ jobs: - name: install mpi run: sudo apt update && sudo apt-get install openmpi-bin openmpi-doc libopenmpi-dev - uses: actions/checkout@v3 + with: + lfs: true - name: setup python uses: actions/setup-python@v4 with: From 6fe6b3859b757825144ca5ecdfcaf23465fb07ed Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Mon, 26 Feb 2024 02:47:26 -0500 Subject: [PATCH 15/23] fix: delete entry point --- tests/test_steps.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tests/test_steps.py b/tests/test_steps.py index 6c96584a..d552188f 100644 --- a/tests/test_steps.py +++ b/tests/test_steps.py @@ -353,15 +353,3 @@ def test_step6(): def test_step7(): # Step 7: B06_correct_separate_var.py ~ assert run_test(script7, paths7) - - -if __name__ == "__main__": - setup_module() - # test_step1() # passing - # test_step2() # passing - # test_step3() # passing - # test_step4() # passing - # test_step5() # passing - # test_step6() # passing - # test_step7() # passing - teardown_module() From df023e6dc63252ddfba91e9ae89b4c955e1e8103 Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Mon, 26 Feb 2024 11:39:09 -0500 Subject: [PATCH 16/23] fix: restore original smoke test workflow with verbose --- .../test_icesat2_tracks_pipeline.yml | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test_icesat2_tracks_pipeline.yml b/.github/workflows/test_icesat2_tracks_pipeline.yml index 22f07dad..9912c646 100644 --- a/.github/workflows/test_icesat2_tracks_pipeline.yml +++ b/.github/workflows/test_icesat2_tracks_pipeline.yml @@ -1,4 +1,4 @@ -name: Test Steps +name: Smoke Tests on: pull_request: {} push: @@ -14,8 +14,6 @@ jobs: - name: install mpi run: sudo apt update && sudo apt-get install openmpi-bin openmpi-doc libopenmpi-dev - uses: actions/checkout@v3 - with: - lfs: true - name: setup python uses: actions/setup-python@v4 with: @@ -36,8 +34,17 @@ jobs: icesat2waves define-angle --help icesat2waves correct-separate --help # prelim name - - name: install pytest - run: pip install pytest pytest-xdist pytest-sugar pytest-durations - - name: Run tests - run: pytest --capture=sys --verbose --showlocals --tb=long --numprocesses=auto tests/test_steps.py - \ No newline at end of file + - name: first step B01_SL_load_single_file + run: load-file --track-name 20190502052058_05180312_005_01 --batch-key SH_testSLsinglefile2 --output-dir ./work --verbose + - name: second step make_spectra + run: make-spectra --track-name SH_20190502_05180312 --batch-key SH_testSLsinglefile2 --output-dir ./work --verbose + - name: third step plot_spectra + run: plot-spectra --track-name SH_20190502_05180312 --batch-key SH_testSLsinglefile2 --output-dir ./work --verbose + - name: fouth step IOWAGA threads + run: make-iowaga-threads-prior --track-name SH_20190502_05180312 --batch-key SH_testSLsinglefile2 --output-dir ./work --verbose + - name: fifth step B04_angle + run: make-b04-angle --track-name SH_20190502_05180312 --batch-key SH_testSLsinglefile2 --output-dir ./work --verbose + - name: sixth step B04_define_angle + run: define-angle --track-name SH_20190502_05180312 --batch-key SH_testSLsinglefile2 --output-dir ./work --verbose + - name: seventh step B06_correct_separate + run: correct-separate --track-name SH_20190502_05180312 --batch-key SH_testSLsinglefile2 --output-dir ./work --verbose \ No newline at end of file From 276b1b0f0e755d4cc832c35e40168c19fb0a00ad Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Mon, 26 Feb 2024 11:41:07 -0500 Subject: [PATCH 17/23] ci: workflow with test suite --- .github/workflows/test_test_suite.yml | 30 +++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/test_test_suite.yml diff --git a/.github/workflows/test_test_suite.yml b/.github/workflows/test_test_suite.yml new file mode 100644 index 00000000..3ab9c0f3 --- /dev/null +++ b/.github/workflows/test_test_suite.yml @@ -0,0 +1,30 @@ +name: Test Steps +on: + pull_request: {} + push: + branches: [ main ] +jobs: + python_run_scripts: + strategy: + fail-fast: false + matrix: + version: ['3.11'] + runs-on: ubuntu-22.04 + steps: + - name: install mpi + run: sudo apt update && sudo apt-get install openmpi-bin openmpi-doc libopenmpi-dev + - uses: actions/checkout@v3 + with: + lfs: true + - name: setup python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.version }} # install the python version needed + cache: "pip" + - name: install icesat2-tracks using pip + run: pip install . + - name: install pytest + run: pip install pytest pytest-xdist pytest-sugar pytest-durations + - name: Run tests + run: pytest --capture=sys --verbose --showlocals --tb=long --numprocesses=auto tests/test_steps.py + \ No newline at end of file From fe27354a17b5391460166cc758a080cc1050b9dd Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Mon, 26 Feb 2024 14:37:10 -0500 Subject: [PATCH 18/23] refactor: create_script function to accept track_name parameter Use it to define script1 --- tests/test_steps.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/tests/test_steps.py b/tests/test_steps.py index d552188f..0a378569 100644 --- a/tests/test_steps.py +++ b/tests/test_steps.py @@ -64,13 +64,14 @@ def extract_tarball(outputdir, tarball_path): tar.close() -def create_script(script_name): +def create_script(script_name, track_name="SH_20190502_05180312"): head = ["python"] tail = [ "--track-name", - "SH_20190502_05180312", + track_name, "--batch-key", "SH_testSLsinglefile2", + "--verbose", "--output-dir", ] base_path = "src/icesat2_tracks/analysis_db/" @@ -107,16 +108,10 @@ def makepathlist(dir, files): # The `scriptx` variables are the scripts to be tested. The `pathsx` variables are the paths to the files that should be produced by the scripts. The scripts are run and the paths are checked to see whether the expected files were produced. If the files were produced, the test passes. If not, the test fails. -# Script 1 is different than the others because it doesn't have any input data and uses a different track name. -script1 = [ - "python", - "src/icesat2_tracks/analysis_db/B01_SL_load_single_file.py", - "--track-name", - "20190502052058_05180312_005_01", - "--batch-key", - "SH_testSLsinglefile2", - "--output-dir", -] +# Script 1 is different from the others because it doesn't have any input data and uses a different track name. +script1 = create_script( + script_name="B01_SL_load_single_file", track_name="20190502052058_05180312_005_01" +) paths1 = [ "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312/B01_track.png.png", "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312/B01b_ATL06_corrected.png.png", From f2fe3479c2881114e053fa155d179a00fe0ce6f1 Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Mon, 26 Feb 2024 15:51:26 -0500 Subject: [PATCH 19/23] fix: restore .gitignore --- .gitignore | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.gitignore b/.gitignore index 0301cfb8..3574230a 100644 --- a/.gitignore +++ b/.gitignore @@ -34,10 +34,6 @@ analysis_db/makefile data_prehandling/ track_lists/ explorations/ -/tests/tests/inputdata -/tests/input-data -__explorations -logs analysis_db/support_files/ *.egg-info/ .installed.cfg From 7a37d2616421e8fa611682fac6b4fa3b6eba34cd Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Mon, 26 Feb 2024 16:24:23 -0500 Subject: [PATCH 20/23] docs: add module level docstring and some comments --- tests/test_steps.py | 50 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/tests/test_steps.py b/tests/test_steps.py index 0a378569..70bb3413 100644 --- a/tests/test_steps.py +++ b/tests/test_steps.py @@ -1,4 +1,32 @@ #!/usr/bin/env python +""" +This module contains a test suite for the following commands: + - name: Step 1 B01_SL_load_single_file + cmd: load-file --track-name 20190502052058_05180312_005_01 --batch-key SH_testSLsinglefile2 --output-dir ./work --verbose + - name: second step make_spectra + cmd: make-spectra --track-name SH_20190502_05180312 --batch-key SH_testSLsinglefile2 --output-dir ./work --verbose + - name: third step plot_spectra + cmd: plot-spectra --track-name SH_20190502_05180312 --batch-key SH_testSLsinglefile2 --output-dir ./work --verbose + - name: fourth step IOWAGA threads + cmd: make-iowaga-threads-prior --track-name SH_20190502_05180312 --batch-key SH_testSLsinglefile2 --output-dir ./work --verbose + - name: fifth step B04_angle + cmd: make-b04-angle --track-name SH_20190502_05180312 --batch-key SH_testSLsinglefile2 --output-dir ./work --verbose + - name: sixth step B04_define_angle + cmd: define-angle --track-name SH_20190502_05180312 --batch-key SH_testSLsinglefile2 --output-dir ./work --verbose + - name: seventh step B06_correct_separate + cmd: correct-separate --track-name SH_20190502_05180312 --batch-key SH_testSLsinglefile2 --output-dir ./work --verbose + +To this end, it sets up a temporary directory within the tests/ directory with subdirectories containing the required input data for each step. The tests are run in parallel using the xdist plugin, with each worker having its own copy of the input data. This allows the tests to modify the input data without affecting other workers. The `setup_module` function (fixture) is responsible for creating the temporary directory and setting up the input data for each step. It also prepares the target directories for each step by extracting the necessary files from tarballs in the `tests/testdata` directory and organizing them in the appropriate directory structure. + +After the tests are completed, the `teardown_module` function is called to clean up the temporary directory. + +The metadata for each script is stored in the `scripts` list, and the paths to the files that should be produced by the scripts are stored in the `paths` list. The `run_test` function is used to run the scripts and check whether the expected files were produced. If the files were produced, the test passes. If not, the test fails. + +With the input data in place, pytest runs each of test_stepX functions, which call the `run_test(script, paths_to_check)` to run the scripts and check whether the expected files were produced. If the files were produced, the test passes. If not, the test fails. + +To create similar tests for other steps, you can use the `create_script` function to create the script and the `makepathlist` function to create the paths to the files that should be produced by the script. The `create_script` function takes the name of the script as an argument and returns a list containing the command to run the script using the `subprocess.run` function. The `makepathlist` function takes a directory and a list of file names as arguments and returns a list of paths to the files in the directory. If the test requires input data, you can use the `extract_tarball` function to extract the input data from a tarball and organize it in the appropriate directory structure within the `setup_module` fixture. +""" + from datetime import datetime from pathlib import Path import shutil @@ -191,7 +219,7 @@ def makepathlist(dir, files): paths6.extend(other_files) -# Paths for the seventh step +# Paths for the seventh script root_plots = "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312/" root_work = "work/SH_testSLsinglefile2/" b06_correction = f"{root_plots}B06_correction/" @@ -306,6 +334,22 @@ def test_step2(): def check_B03_freq_reconst_x(): + # The script3 produces plots in the `/plots/SH//SH_/B03_spectra` directory. + + # ```shell + # (.venv) $ tree /plots/SH//SH_/B03_spectra + # /plots/SH//SH_/B03_spectra + # ├── B03_freq_reconst_x28.pdf + # ├── B03_freq_reconst_x47.pdf + # ├── B03_freq_reconst_x56.pdf + # ├── B03_freq_reconst_x66.pdf + # └── B03_freq_reconst_x75.pdf + + # 0 directories, 5 files + # ``` + + # File names in this directory all have the `B03_freq_reconst_x` prefix but with different numbers at the end. The numbers are not deterministic, so we can't check for specific file names. Instead, we can check that there are 5 pdf files in the directory. + outputdir = getoutputdir(script3) directory = Path( outputdir, "plots/SH/SH_testSLsinglefile2/SH_20190502_05180312/B03_spectra/" @@ -320,8 +364,8 @@ def test_step3(): # Step 3: B03_plot_spectra_ov.py ~ 11 sec # This script has stochastic behavior, so the files produced don't always have the same names but the count of pdf files is constant for the test input data. t1 = run_test(script3, paths3) - t2 = check_B03_freq_reconst_x() assert t1 + t2 = check_B03_freq_reconst_x() assert t2 @@ -329,9 +373,9 @@ def test_step4(): # Step 4: A02c_IOWAGA_thredds_prior.py ~ 23 sec # check deterministic paths t1 = run_test(script4, paths4) + assert t1 # check stochastic paths t2 = check_file_exists(dir4, prefix4) - assert t1 assert t2 From b362171c606c92c17a250beaea3a2041b0c7c62b Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Wed, 28 Feb 2024 13:56:24 -0500 Subject: [PATCH 21/23] Apply suggestions from code review Co-authored-by: Timothy Divoll --- tests/test_steps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_steps.py b/tests/test_steps.py index 70bb3413..a7fb596a 100644 --- a/tests/test_steps.py +++ b/tests/test_steps.py @@ -16,7 +16,7 @@ - name: seventh step B06_correct_separate cmd: correct-separate --track-name SH_20190502_05180312 --batch-key SH_testSLsinglefile2 --output-dir ./work --verbose -To this end, it sets up a temporary directory within the tests/ directory with subdirectories containing the required input data for each step. The tests are run in parallel using the xdist plugin, with each worker having its own copy of the input data. This allows the tests to modify the input data without affecting other workers. The `setup_module` function (fixture) is responsible for creating the temporary directory and setting up the input data for each step. It also prepares the target directories for each step by extracting the necessary files from tarballs in the `tests/testdata` directory and organizing them in the appropriate directory structure. +To this end, it sets up a temporary directory within the `tests/` directory with subdirectories containing the required input data for each step. The tests are run in parallel using the `xdist` plugin, with each worker having its own copy of the input data. This allows the tests to modify the input data without affecting other workers. The `setup_module` function (fixture) is responsible for creating the temporary directory and setting up the input data for each step. It also prepares the target directories for each step by extracting the necessary files from tarballs in the `tests/testdata` directory and organizing them in the appropriate directory structure. After the tests are completed, the `teardown_module` function is called to clean up the temporary directory. From c9a02df40cd9884774679ccfdcbbf7e6180eb68d Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Thu, 29 Feb 2024 12:37:43 -0500 Subject: [PATCH 22/23] Apply suggestions from code review Co-authored-by: Timothy Divoll --- tests/test_steps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_steps.py b/tests/test_steps.py index a7fb596a..5af0ea67 100644 --- a/tests/test_steps.py +++ b/tests/test_steps.py @@ -273,7 +273,7 @@ def setup_module(): 5 directories, 0 files ``` - When running in parallel using the xdist plugin, each worker will have its own copy of all the input data. This is necessary because the tests are run in parallel and the input data is modified by the tests. This way, the teardown function can delete the temporary directory for each worker without affecting the other workers. + When running in parallel using the `xdist` plugin, each worker will have its own copy of all the input data. This is necessary because the tests are run in parallel and the input data is modified by the tests. This way, the teardown function can delete the temporary directory for each worker without affecting the other workers. """ homedir = Path(__file__).parent From b1da226ac58dc5daedbe712d1d2b6f18ceaec99a Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Thu, 29 Feb 2024 13:49:23 -0500 Subject: [PATCH 23/23] Apply suggestions from code review Co-authored-by: Timothy Divoll --- .github/workflows/test_icesat2_tracks_pipeline.yml | 2 +- tests/test_steps.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_icesat2_tracks_pipeline.yml b/.github/workflows/test_icesat2_tracks_pipeline.yml index 9912c646..d8b435c1 100644 --- a/.github/workflows/test_icesat2_tracks_pipeline.yml +++ b/.github/workflows/test_icesat2_tracks_pipeline.yml @@ -40,7 +40,7 @@ jobs: run: make-spectra --track-name SH_20190502_05180312 --batch-key SH_testSLsinglefile2 --output-dir ./work --verbose - name: third step plot_spectra run: plot-spectra --track-name SH_20190502_05180312 --batch-key SH_testSLsinglefile2 --output-dir ./work --verbose - - name: fouth step IOWAGA threads + - name: fourth step IOWAGA threads run: make-iowaga-threads-prior --track-name SH_20190502_05180312 --batch-key SH_testSLsinglefile2 --output-dir ./work --verbose - name: fifth step B04_angle run: make-b04-angle --track-name SH_20190502_05180312 --batch-key SH_testSLsinglefile2 --output-dir ./work --verbose diff --git a/tests/test_steps.py b/tests/test_steps.py index 5af0ea67..2cb6203f 100644 --- a/tests/test_steps.py +++ b/tests/test_steps.py @@ -49,7 +49,7 @@ def checkpaths(paths): def get_all_filenames(directory): """ - Get a list of all file names in a directory + Get a list of all file names in a directory. """ return [p.name for p in Path(directory).rglob("*")]