diff --git a/forcingprocessor/tests/test_forcingprocessor.py b/forcingprocessor/tests/test_forcingprocessor.py index 6c060854..2b3ceac4 100644 --- a/forcingprocessor/tests/test_forcingprocessor.py +++ b/forcingprocessor/tests/test_forcingprocessor.py @@ -142,14 +142,57 @@ def test_noaa_nwm_pds_https(): assert assert_file.exists() os.remove(assert_file) -def test_noaa_nwm_pds_s3(): +def test_noaa_nwm_pds_https_short_range(): + nwmurl_conf['start_date'] = date + hourminute + nwmurl_conf['end_date'] = date + hourminute + nwmurl_conf["urlbaseinput"] = 7 + nwmurl_conf["runinput"] = 1 + generate_nwmfiles(nwmurl_conf) + prep_ngen_data(conf) + assert assert_file.exists() + os.remove(assert_file) + +def test_noaa_nwm_pds_https_medium_range(): + nwmurl_conf['start_date'] = date + hourminute + nwmurl_conf['end_date'] = date + hourminute + nwmurl_conf["urlbaseinput"] = 7 + nwmurl_conf["runinput"] = 2 + generate_nwmfiles(nwmurl_conf) + prep_ngen_data(conf) + assert assert_file.exists() + os.remove(assert_file) + +def test_noaa_nwm_pds_https_analysis_assim(): + assert False, f'test_nomads_post_processed() is BROKEN - https://github.com/CIROH-UA/nwmurl/issues/36' nwmurl_conf['start_date'] = date + hourminute nwmurl_conf['end_date'] = date + hourminute + nwmurl_conf["urlbaseinput"] = 7 + nwmurl_conf["runinput"] = 5 + generate_nwmfiles(nwmurl_conf) + prep_ngen_data(conf) + assert assert_file.exists() + os.remove(assert_file) + +def test_noaa_nwm_pds_https_analysis_assim_extend(): + assert False, f'test_nomads_post_processed() is BROKEN - https://github.com/CIROH-UA/nwmurl/issues/36' + nwmurl_conf['start_date'] = date + hourminute + nwmurl_conf['end_date'] = date + hourminute + nwmurl_conf["urlbaseinput"] = 7 + nwmurl_conf["runinput"] = 7 + generate_nwmfiles(nwmurl_conf) + prep_ngen_data(conf) + assert assert_file.exists() + os.remove(assert_file) + +def test_noaa_nwm_pds_s3(): + nwmurl_conf['start_date'] = date + hourminute + nwmurl_conf['end_date'] = date + hourminute + nwmurl_conf["runinput"] = 1 nwmurl_conf["urlbaseinput"] = 8 generate_nwmfiles(nwmurl_conf) prep_ngen_data(conf) assert assert_file.exists() - os.remove(assert_file) + os.remove(assert_file) def test_ciroh_zarr(): assert False, "Not implemented" diff --git a/python_tools/src/python_tools/configure_datastream.py b/python_tools/src/python_tools/configure_datastream.py index 7fe25425..dc6832ff 100644 --- a/python_tools/src/python_tools/configure_datastream.py +++ b/python_tools/src/python_tools/configure_datastream.py @@ -95,8 +95,10 @@ def create_conf_nwm(args): start_dt = start_dt.replace(hour=1,minute=0,second=0,microsecond=0) end_dt = end_dt.replace(hour=1,minute=0,second=0,microsecond=0) - start_str = start_dt.strftime('%Y-%m-%d %H:%M:%S') - end_str = end_dt.strftime('%Y-%m-%d %H:%M:%S') + start_str_real = start_dt.strftime('%Y-%m-%d %H:%M:%S') + end_str_real = end_dt.strftime('%Y-%m-%d %H:%M:%S') + start_str_nwm = start_dt.strftime('%Y%m%d%H%M') + end_str_nwm = start_dt.strftime('%Y%m%d%H%M') if "RETRO" in args.forcing_source: if "V2" in args.forcing_source: @@ -149,11 +151,12 @@ def create_conf_nwm(args): num_hrs=3 else: runinput=2 + num_hrs = 24 nwm_conf = { "forcing_type" : "operational_archive", - "start_date" : start, - "end_date" : end, + "start_date" : start_str_nwm, + "end_date" : end_str_nwm, "runinput" : runinput, "varinput" : varinput, "geoinput" : geoinput, @@ -163,7 +166,7 @@ def create_conf_nwm(args): "lead_time" : [x+dt for x in range(num_hrs)] } - return nwm_conf, start_str, end_str + return nwm_conf, start_str_real, end_str_real def create_conf_fp(args): geo_base = args.gpkg.split('/')[-1] @@ -286,5 +289,10 @@ def create_confs(args): parser.add_argument("--s3_bucket", type=str,help="s3 bucket to write to",default="") parser.add_argument("--s3_prefix", type=str,help="s3 prefix to prepend to files",required="") - args = parser.parse_args() + args = parser.parse_args() + + if args.forcing_source == "": args.forcing_source="NWM_V3" + if args.hydrofabric_version == "": args.hydrofabric_version="v2.1.1" + if args.nprocs == "": args.nprocs=os.cpu_count() + create_confs(args) \ No newline at end of file diff --git a/research_datastream/terraform/test/execution_gp_arm_docker_buildNtester.json b/research_datastream/terraform/test/execution_gp_arm_docker_buildNtester.json index a8913252..198ffb47 100644 --- a/research_datastream/terraform/test/execution_gp_arm_docker_buildNtester.json +++ b/research_datastream/terraform/test/execution_gp_arm_docker_buildNtester.json @@ -2,7 +2,7 @@ "commands" : [ "runuser -l ec2-user -c 'rm -rf /home/ec2-user/ngen-datastream && docker rmi -f $(docker images -aq)'", "runuser -l ec2-user -c 'git clone https://github.com/CIROH-UA/ngen-datastream.git'", - "runuser -l ec2-user -c '/home/ec2-user/ngen-datastream/scripts/docker_builds.sh -b -s'", + "runuser -l ec2-user -c '/home/ec2-user/ngen-datastream/scripts/docker_builds.sh -f -d'", "runuser -l ec2-user -c 'curl -L -O https://ngen-datastream.s3.us-east-2.amazonaws.com/palisade.gpkg'", "runuser -l ec2-user -c '/home/ec2-user/ngen-datastream/scripts/stream.sh -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d /home/ec2-user/ngen-datastream/data/datastream_test -g /home/ec2-user/palisade.gpkg -R /home/ec2-user/ngen-datastream/configs/ngen/realization_sloth_nom_cfe_pet.json -n 4'", "runuser -l ec2-user -c '/home/ec2-user/ngen-datastream/research_datastream/terraform/test/docker_loginNpush.sh >> /home/ec2-user/ngen-datastream/docker_login_log.txt'", diff --git a/scripts/docker_builds.sh b/scripts/docker_builds.sh index c2a78833..93e6074d 100755 --- a/scripts/docker_builds.sh +++ b/scripts/docker_builds.sh @@ -11,45 +11,48 @@ if [ $PLATFORM = "x86_64" ]; then fi TAG="latest$PLATORM_TAG" -BUILD="no" +BUILD_FORCINGPROCESSOR="no" +BUILD_DATASTREAM="no" PUSH="no" -SKIP_DEPS="no" -while getopts "bps" flag; do +BUILD_DEPS="no" +while getopts "pefd" flag; do case $flag in - b) BUILD="yes" - ;; p) PUSH="yes" ;; - s) SKIP_DEPS="yes" - ;; + e) BUILD_DEPS="yes" + ;; + f) BUILD_FORCINGPROCESSOR="yes" + ;; + d) BUILD_DATASTREAM="yes" + ;; \?) ;; esac done -if [ "$BUILD" = "yes" ]; then - - cd $DOCKER_DIR - echo "Building docker from "$DOCKER_DIR - if [ "$SKIP_DEPS" = "no" ]; then - docker build -t awiciroh/datastream-deps:$TAG -f Dockerfile.datastream-deps . --no-cache --build-arg TAG_NAME=$TAG --build-arg ARCH=$PLATFORM --platform linux/$PLATFORM - fi - +cd $DOCKER_DIR +if [ "$BUILD_DEPS" = "yes" ]; then + docker build -t awiciroh/datastream-deps:$TAG -f Dockerfile.datastream-deps . --no-cache --build-arg TAG_NAME=$TAG --build-arg ARCH=$PLATFORM --platform linux/$PLATFORM if [ -d "$DOCKER_DATASTREAM" ]; then rm -rf $DOCKER_DATASTREAM fi +fi +if [ "$BUILD_FORCINGPROCESSOR" = "yes" ]; then mkdir $DOCKER_DATASTREAM cp -r $DATASTREAM_PATH/forcingprocessor $DOCKER_DATASTREAM/forcingprocessor docker build -t awiciroh/forcingprocessor:$TAG -f Dockerfile.forcingprocessor . --no-cache --build-arg TAG_NAME=$TAG --platform linux/$PLATFORM - if [ -d "$DOCKER_DATASTREAM" ]; then rm -rf $DOCKER_DATASTREAM fi +fi +if [ "$BUILD_DATASTREAM" = "yes" ]; then mkdir $DOCKER_DATASTREAM cp -r $DATASTREAM_PATH/python_tools $DOCKER_DATASTREAM/python_tools cp -r $DATASTREAM_PATH/configs $DOCKER_DATASTREAM/configs docker build -t awiciroh/datastream:$TAG -f Dockerfile.datastream . --no-cache --build-arg TAG_NAME=$TAG --platform linux/$PLATFORM - echo "Docker containers have been built!" + if [ -d "$DOCKER_DATASTREAM" ]; then + rm -rf $DOCKER_DATASTREAM + fi fi if [ "$PUSH" = "yes" ]; then diff --git a/scripts/stream.sh b/scripts/stream.sh index 0b8f0a76..bba88386 100755 --- a/scripts/stream.sh +++ b/scripts/stream.sh @@ -511,13 +511,15 @@ else fi log_time "VALIDATION_END" $DATASTREAM_PROFILING +# NIGAB_TAG="latest$PLATORM_TAG" +NIGAB_TAG="1.2.1" log_time "NGEN_START" $DATASTREAM_PROFILING echo "Running NextGen in AUTO MODE from CIROH-UA/NGIAB-CloudInfra" if [ "$DRYRUN" == "True" ]; then echo "DRYRUN - NEXTGEN EXECUTION SKIPPED" - echo "COMMAND: docker run --rm -v "$NGEN_RUN":"$DOCKER_MOUNT" awiciroh/ciroh-ngen-image:latest$PLATORM_TAG "$DOCKER_MOUNT" auto $NPROCS" + echo "COMMAND: docker run --rm -v "$NGEN_RUN":"$DOCKER_MOUNT" awiciroh/ciroh-ngen-image:$NIGAB_TAG "$DOCKER_MOUNT" auto $NPROCS" else - docker run --rm -v "$NGEN_RUN":"$DOCKER_MOUNT" awiciroh/ciroh-ngen-image:latest$PLATORM_TAG "$DOCKER_MOUNT" auto $NPROCS + docker run --rm -v "$NGEN_RUN":"$DOCKER_MOUNT" awiciroh/ciroh-ngen-image:$NIGAB_TAG "$DOCKER_MOUNT" auto $NPROCS cp -r $NGEN_RUN/*partitions* $DATASTREAM_RESOURCES_NGENCONF/ fi log_time "NGEN_END" $DATASTREAM_PROFILING