fix tests

.github/workflows/forcingprocessor.yml

@@ -43,7 +43,9 @@ jobs:
     - name: Test with pytest
       run: |
         cd forcingprocessor
-        python -m pytest -vv --deselect="tests/test_forcingprocessor.py::test_google_cloud_storage" --deselect="tests/test_forcingprocessor.py::test_gcs" --deselect="tests/test_forcingprocessor.py::test_gs" --deselect="tests/test_forcingprocessor.py::test_ciroh_zarr" --deselect="tests/test_forcingprocessor.py::test_nomads_post_processed" --deselect="tests/test_forcingprocessor.py::test_retro_ciroh_zarr"
+        python -m pytest -vv --deselect="tests/test_forcingprocessor.py::test_google_cloud_storage" --deselect="tests/test_forcingprocessor.py::test_gcs" --deselect="tests/test_forcingprocessor.py::test_gs" --deselect="tests/test_forcingprocessor.py::test_ciroh_zarr" --deselect="tests/test_forcingprocessor.py::test_nomads_post_processed" --deselect="tests/test_forcingprocessor.py::test_retro_ciroh_zarr" python -m pytest -vv --deselect="tests/test_forcingprocessor.py::test_google_cloud_storage" --deselect="tests/test_forcingprocessor.py::test_gcs" --deselect="tests/test_forcingprocessor.py::test_gs" --deselect="tests/test_forcingprocessor.py::test_ciroh_zarr" --deselect="tests/test_forcingprocessor.py::test_nomads_post_processed" --deselect="tests/test_forcingprocessor.py::test_retro_ciroh_zarr" --deselect="tests/test_hf2ds.py" --deselect="tests/test_plotter.py"
+        python -m pytest -vv -k tests/test_hf2ds.py
+        python -m pytest -vv -k tests/test_plotter.py
         python -m pytest -vv -k test_google_cloud_storage 
         python -m pytest -vv -k test_gs 
         python -m pytest -vv -k test_gcs 

forcingprocessor/src/forcingprocessor/processor.py

@@ -763,7 +763,7 @@ def prep_ngen_data(conf):
         metaf_path   = bucket_path + '/metadata/forcings_metadata'
         bucket, key  = convert_url2key(metaf_path,storage_type)
         conf_path    = f"{key}/conf_fp.json"
-        filenamelist_path = f"{key}{os.path.basename(nwm_file)}"
+        filenamelist_path = f"{key}/{os.path.basename(nwm_file)}"
         s3 = boto3.client("s3")          
         s3.put_object(
                 Body=json.dumps(conf),

forcingprocessor/tests/test_forcingprocessor.py

@@ -218,7 +218,10 @@ def test_s3_output():
     generate_nwmfiles(nwmurl_conf_retro)
     prep_ngen_data(conf)     
     conf['storage']['output_path'] = str(data_dir)
-    os.system(f'aws s3api delete-object --bucket {test_bucket} --key pytest_fp/1_forcings.nc')
+    os.system(f'aws s3api delete-object --bucket {test_bucket} --key pytest_fp/forcings/1_forcings.nc')
+    os.system(f'aws s3api delete-object --bucket {test_bucket} --key pytest_fp/metadata/forcings_metadata/conf_fp.json')
+    os.system(f'aws s3api delete-object --bucket {test_bucket} --key pytest_fp/metadata/forcings_metadata/retro_filenamelist.txt')
+    os.system(f'aws s3api delete-object --bucket {test_bucket} --key pytest_fp/metadata/forcings_metadata/profile_fp.txt')
 
 def test_muliple_weights():
     conf['forcing']['nwm_file'] = retro_filenamelist

forcingprocessor/tests/test_hf2ds.py

@@ -1,19 +1,22 @@
 from forcingprocessor.weights_hf2ds import hf2ds
 from pathlib import Path
 import os
+import time
+import pytest
 
 HF_VERSION="v2.1.1"
 test_dir = Path(__file__).parent
 data_dir = (test_dir/'data').resolve()
+os.system(f'mkdir {data_dir}')
 out_parq = os.path.join(data_dir,"out.parquet")
-if os.path.exists(data_dir):
-    os.system(f"rm -rf {data_dir}")
-os.system(f"mkdir {data_dir}")
+parq_name = "09_weights.parquet"
+parq_path = os.path.join(data_dir,parq_name)
+os.system(f"curl -o {parq_path} -L -O https://lynker-spatial.s3-us-west-2.amazonaws.com/hydrofabric/{HF_VERSION}/nextgen/conus_forcing-weights/vpuid%3D09/part-0.parquet")
+geopackage_name = "palisade.gpkg"
+gpkg_path = os.path.join(data_dir,geopackage_name)
+os.system(f"curl -o {gpkg_path} -L -O https://ngen-datastream.s3.us-east-2.amazonaws.com/{geopackage_name}")
 
 def test_parquet():
-    parq_name = "09_weights.parquet"
-    parq_path = os.path.join(data_dir,parq_name)
-    os.system(f"curl -o {parq_path} -L -O https://lynker-spatial.s3-us-west-2.amazonaws.com/hydrofabric/{HF_VERSION}/nextgen/conus_forcing-weights/vpuid%3D09/part-0.parquet")
     weights,_ = hf2ds([parq_path])
     assert len(weights) > 0
 
@@ -22,9 +25,6 @@ def test_parquet_lynker_spatial():
     assert len(weights) > 0    
 
 def test_gpkg():
-    geopackage_name = "palisade.gpkg"
-    gpkg_path = os.path.join(data_dir,geopackage_name)
-    os.system(f"curl -o {gpkg_path} -L -O https://ngen-datastream.s3.us-east-2.amazonaws.com/{geopackage_name}")
     weights,_ = hf2ds([gpkg_path])
     assert len(weights) > 0
 

forcingprocessor/tests/test_plotter.py

@@ -6,7 +6,6 @@
 from forcingprocessor.plot_forcings import nc_to_3darray, plot_ngen_forcings, csvs_to_3darray, get_nwm_data_array
 from forcingprocessor.nwm_filenames_generator import generate_nwmfiles
 import requests
-import pytest
 
 date = datetime.now()
 date = date.strftime('%Y%m%d')
@@ -64,11 +63,11 @@
         "lead_time"    : [1]
     }
 
-@pytest.fixture
-def clean_dir(autouse=True):
-    if os.path.exists(DATA_DIR):
-        os.system(f'rm -rf {str(DATA_DIR)}')
-    os.system(f'mkdir {str(DATA_DIR)}')
+# @pytest.fixture
+# def clean_dir(autouse=True):
+#     if os.path.exists(DATA_DIR):
+#         os.system(f'rm -rf {str(DATA_DIR)}')
+#     os.system(f'mkdir {str(DATA_DIR)}')
 
 def test_forcings_plot():
     nwmurl_conf['start_date'] = date + hourminute