From 1f6985bd768c6b95b179834b8f10de298997dec7 Mon Sep 17 00:00:00 2001 From: Jordan Laser Date: Tue, 8 Oct 2024 21:35:31 -0600 Subject: [PATCH] fix s3 --- .github/workflows/test_datastream_options.yml | 6 ++++-- scripts/stream.sh | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_datastream_options.yml b/.github/workflows/test_datastream_options.yml index af210d28..55996cf6 100644 --- a/.github/workflows/test_datastream_options.yml +++ b/.github/workflows/test_datastream_options.yml @@ -27,6 +27,8 @@ jobs: aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws configure set region us-east-1 + export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} + export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} - name: Build docker containers run : | @@ -107,7 +109,7 @@ jobs: if: always() run: | sudo rm -rf $(pwd)/data/datastream_test - ./scripts/stream.sh -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json -S ngen-datastream -o git_actions_test/ + ./scripts/stream.sh -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json -S ciroh-community-ngen-datastream -o git_actions_test/ aws s3api wait object-exists --bucket ngen-datastream --key git_actions_test/ngen-run.tar.gz aws s3api delete-object --bucket ngen-datastream --key git_actions_test/ngen-run.tar.gz @@ -121,7 +123,7 @@ jobs: if: always() run: | sudo rm -rf $(pwd)/data/datastream_test - ./scripts/stream.sh -s DAILY -C NWM_V3 -e 202006200000 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json + ./scripts/stream.sh -s DAILY -C NWM_V3 -e 202410030000 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json - name: DAILY short_range today test if: always() diff --git a/scripts/stream.sh b/scripts/stream.sh index f135e51e..8281942f 100755 --- a/scripts/stream.sh +++ b/scripts/stream.sh @@ -482,6 +482,8 @@ else python3 "$DOCKER_FP"processor.py "$DOCKER_META"/conf_fp.json" else docker run --rm -v "$DATA_DIR:"$DOCKER_MOUNT"" \ + -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ + -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ -u $(id -u):$(id -g) \ -w "$DOCKER_RESOURCES" $DOCKER_TAG \ python3 "$DOCKER_FP"processor.py "$DOCKER_META"/conf_fp.json