remove tests for analysis_assim

CIROH-UA · Oct 9, 2024 · b8d142a · b8d142a
1 parent 515830f
commit b8d142a
Show file tree

Hide file tree

Showing 5 changed files with 84 additions and 28 deletions.
diff --git a/forcingprocessor/tests/test_forcingprocessor.py b/forcingprocessor/tests/test_forcingprocessor.py
@@ -142,14 +142,57 @@ def test_noaa_nwm_pds_https():
     assert assert_file.exists()
     os.remove(assert_file)     
 
-def test_noaa_nwm_pds_s3():
+def test_noaa_nwm_pds_https_short_range():
+    nwmurl_conf['start_date'] = date + hourminute
+    nwmurl_conf['end_date']   = date + hourminute    
+    nwmurl_conf["urlbaseinput"] = 7
+    nwmurl_conf["runinput"] = 1
+    generate_nwmfiles(nwmurl_conf)          
+    prep_ngen_data(conf)
+    assert assert_file.exists()
+    os.remove(assert_file) 
+
+def test_noaa_nwm_pds_https_medium_range():
+    nwmurl_conf['start_date'] = date + hourminute
+    nwmurl_conf['end_date']   = date + hourminute    
+    nwmurl_conf["urlbaseinput"] = 7
+    nwmurl_conf["runinput"] = 2
+    generate_nwmfiles(nwmurl_conf)          
+    prep_ngen_data(conf)
+    assert assert_file.exists()
+    os.remove(assert_file)         
+
+def test_noaa_nwm_pds_https_analysis_assim():
+    assert False, f'test_nomads_post_processed() is BROKEN - https://github.com/CIROH-UA/nwmurl/issues/36'
     nwmurl_conf['start_date'] = date + hourminute
     nwmurl_conf['end_date']   = date + hourminute    
+    nwmurl_conf["urlbaseinput"] = 7
+    nwmurl_conf["runinput"] = 5
+    generate_nwmfiles(nwmurl_conf)          
+    prep_ngen_data(conf)
+    assert assert_file.exists()
+    os.remove(assert_file)  
+
+def test_noaa_nwm_pds_https_analysis_assim_extend():
+    assert False, f'test_nomads_post_processed() is BROKEN - https://github.com/CIROH-UA/nwmurl/issues/36'
+    nwmurl_conf['start_date'] = date + hourminute
+    nwmurl_conf['end_date']   = date + hourminute    
+    nwmurl_conf["urlbaseinput"] = 7
+    nwmurl_conf["runinput"] = 7
+    generate_nwmfiles(nwmurl_conf)          
+    prep_ngen_data(conf)
+    assert assert_file.exists()
+    os.remove(assert_file)    
+
+def test_noaa_nwm_pds_s3():
+    nwmurl_conf['start_date'] = date + hourminute
+    nwmurl_conf['end_date']   = date + hourminute   
+    nwmurl_conf["runinput"] = 1 
     nwmurl_conf["urlbaseinput"] = 8
     generate_nwmfiles(nwmurl_conf)          
     prep_ngen_data(conf)
     assert assert_file.exists()
-    os.remove(assert_file)       
+    os.remove(assert_file)            
 
 def test_ciroh_zarr():
     assert False, "Not implemented"

diff --git a/python_tools/src/python_tools/configure_datastream.py b/python_tools/src/python_tools/configure_datastream.py
@@ -95,8 +95,10 @@ def create_conf_nwm(args):
 
     start_dt = start_dt.replace(hour=1,minute=0,second=0,microsecond=0)
     end_dt   = end_dt.replace(hour=1,minute=0,second=0,microsecond=0)
-    start_str = start_dt.strftime('%Y-%m-%d %H:%M:%S')    
-    end_str   = end_dt.strftime('%Y-%m-%d %H:%M:%S')            
+    start_str_real = start_dt.strftime('%Y-%m-%d %H:%M:%S')    
+    end_str_real   = end_dt.strftime('%Y-%m-%d %H:%M:%S')      
+    start_str_nwm = start_dt.strftime('%Y%m%d%H%M') 
+    end_str_nwm = start_dt.strftime('%Y%m%d%H%M')   
 
     if "RETRO" in args.forcing_source:                    
         if "V2" in args.forcing_source:
@@ -149,11 +151,12 @@ def create_conf_nwm(args):
                 num_hrs=3
         else:
             runinput=2
+            num_hrs = 24
 
         nwm_conf = {
             "forcing_type" : "operational_archive",
-            "start_date"   : start,
-            "end_date"     : end,
+            "start_date"   : start_str_nwm,
+            "end_date"     : end_str_nwm,
             "runinput"     : runinput,
             "varinput"     : varinput,
             "geoinput"     : geoinput,
@@ -163,7 +166,7 @@ def create_conf_nwm(args):
             "lead_time"    : [x+dt for x in range(num_hrs)]
         }
 
-    return nwm_conf, start_str, end_str
+    return nwm_conf, start_str_real, end_str_real
 
 def create_conf_fp(args):
     geo_base = args.gpkg.split('/')[-1]      
@@ -286,5 +289,10 @@ def create_confs(args):
     parser.add_argument("--s3_bucket", type=str,help="s3 bucket to write to",default="")
     parser.add_argument("--s3_prefix", type=str,help="s3 prefix to prepend to files",required="")    
 
-    args = parser.parse_args()      
+    args = parser.parse_args() 
+
+    if args.forcing_source == "": args.forcing_source="NWM_V3"
+    if args.hydrofabric_version == "": args.hydrofabric_version="v2.1.1"
+    if args.nprocs == "": args.nprocs=os.cpu_count()
+
     create_confs(args)
diff --git a/research_datastream/terraform/test/execution_gp_arm_docker_buildNtester.json b/research_datastream/terraform/test/execution_gp_arm_docker_buildNtester.json
@@ -2,7 +2,7 @@
   "commands"  : [
     "runuser -l ec2-user -c 'rm -rf /home/ec2-user/ngen-datastream && docker rmi -f $(docker images -aq)'",
     "runuser -l ec2-user -c 'git clone https://github.com/CIROH-UA/ngen-datastream.git'",
-    "runuser -l ec2-user -c '/home/ec2-user/ngen-datastream/scripts/docker_builds.sh -b -s'",
+    "runuser -l ec2-user -c '/home/ec2-user/ngen-datastream/scripts/docker_builds.sh -f -d'",
     "runuser -l ec2-user -c 'curl -L -O https://ngen-datastream.s3.us-east-2.amazonaws.com/palisade.gpkg'",
     "runuser -l ec2-user -c '/home/ec2-user/ngen-datastream/scripts/stream.sh -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d /home/ec2-user/ngen-datastream/data/datastream_test -g /home/ec2-user/palisade.gpkg -R /home/ec2-user/ngen-datastream/configs/ngen/realization_sloth_nom_cfe_pet.json -n 4'",
     "runuser -l ec2-user -c '/home/ec2-user/ngen-datastream/research_datastream/terraform/test/docker_loginNpush.sh >> /home/ec2-user/ngen-datastream/docker_login_log.txt'",

diff --git a/scripts/docker_builds.sh b/scripts/docker_builds.sh
@@ -11,45 +11,48 @@ if [ $PLATFORM = "x86_64" ]; then
 fi
 TAG="latest$PLATORM_TAG"
 
-BUILD="no"
+BUILD_FORCINGPROCESSOR="no"
+BUILD_DATASTREAM="no"
 PUSH="no"
-SKIP_DEPS="no"
-while getopts "bps" flag; do
+BUILD_DEPS="no"
+while getopts "pefd" flag; do
  case $flag in
-   b) BUILD="yes"
-   ;;
    p) PUSH="yes"
    ;;
-   s) SKIP_DEPS="yes"
-   ;;   
+   e) BUILD_DEPS="yes"
+   ;; 
+   f) BUILD_FORCINGPROCESSOR="yes"
+   ;; 
+   d) BUILD_DATASTREAM="yes"
+   ;;         
    \?)
    ;;
  esac
 done
 
-if [ "$BUILD" = "yes" ]; then
-
-  cd $DOCKER_DIR
-  echo "Building docker from "$DOCKER_DIR
-  if [ "$SKIP_DEPS" = "no" ]; then
-    docker build -t awiciroh/datastream-deps:$TAG -f Dockerfile.datastream-deps . --no-cache --build-arg TAG_NAME=$TAG --build-arg ARCH=$PLATFORM --platform linux/$PLATFORM
-  fi
-
+cd $DOCKER_DIR
+if [ "$BUILD_DEPS" = "yes" ]; then
+  docker build -t awiciroh/datastream-deps:$TAG -f Dockerfile.datastream-deps . --no-cache --build-arg TAG_NAME=$TAG --build-arg ARCH=$PLATFORM --platform linux/$PLATFORM
   if [ -d "$DOCKER_DATASTREAM" ]; then
     rm -rf $DOCKER_DATASTREAM
   fi
+fi
+if [ "$BUILD_FORCINGPROCESSOR" = "yes" ]; then
   mkdir $DOCKER_DATASTREAM
   cp -r $DATASTREAM_PATH/forcingprocessor $DOCKER_DATASTREAM/forcingprocessor
   docker build -t awiciroh/forcingprocessor:$TAG -f Dockerfile.forcingprocessor . --no-cache --build-arg TAG_NAME=$TAG --platform linux/$PLATFORM
-
   if [ -d "$DOCKER_DATASTREAM" ]; then
     rm -rf $DOCKER_DATASTREAM
   fi
+fi
+if [ "$BUILD_DATASTREAM" = "yes" ]; then
   mkdir $DOCKER_DATASTREAM
   cp -r $DATASTREAM_PATH/python_tools $DOCKER_DATASTREAM/python_tools
   cp -r $DATASTREAM_PATH/configs $DOCKER_DATASTREAM/configs
   docker build -t awiciroh/datastream:$TAG -f Dockerfile.datastream . --no-cache --build-arg TAG_NAME=$TAG --platform linux/$PLATFORM
-  echo "Docker containers have been built!"
+  if [ -d "$DOCKER_DATASTREAM" ]; then
+    rm -rf $DOCKER_DATASTREAM
+  fi
 fi
 
 if [ "$PUSH" = "yes" ]; then

diff --git a/scripts/stream.sh b/scripts/stream.sh
@@ -511,13 +511,15 @@ else
 fi
 log_time "VALIDATION_END" $DATASTREAM_PROFILING
 
+# NIGAB_TAG="latest$PLATORM_TAG"
+NIGAB_TAG="1.2.1"
 log_time "NGEN_START" $DATASTREAM_PROFILING
 echo "Running NextGen in AUTO MODE from CIROH-UA/NGIAB-CloudInfra"
 if [ "$DRYRUN" == "True" ]; then
     echo "DRYRUN - NEXTGEN EXECUTION SKIPPED"
-    echo "COMMAND: docker run --rm -v "$NGEN_RUN":"$DOCKER_MOUNT" awiciroh/ciroh-ngen-image:latest$PLATORM_TAG "$DOCKER_MOUNT" auto $NPROCS"
+    echo "COMMAND: docker run --rm -v "$NGEN_RUN":"$DOCKER_MOUNT" awiciroh/ciroh-ngen-image:$NIGAB_TAG "$DOCKER_MOUNT" auto $NPROCS"
 else
-    docker run --rm -v "$NGEN_RUN":"$DOCKER_MOUNT" awiciroh/ciroh-ngen-image:latest$PLATORM_TAG "$DOCKER_MOUNT" auto $NPROCS
+    docker run --rm -v "$NGEN_RUN":"$DOCKER_MOUNT" awiciroh/ciroh-ngen-image:$NIGAB_TAG "$DOCKER_MOUNT" auto $NPROCS
     cp -r $NGEN_RUN/*partitions* $DATASTREAM_RESOURCES_NGENCONF/
 fi
 log_time "NGEN_END" $DATASTREAM_PROFILING