Skip to content

Commit

Permalink
fix copy into mount
Browse files Browse the repository at this point in the history
  • Loading branch information
JordanLaserGit committed Feb 6, 2024
1 parent f1a76d0 commit 6276ae8
Showing 1 changed file with 8 additions and 3 deletions.
11 changes: 8 additions & 3 deletions scripts/stream.sh
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ fi
DATE=$(env TZ=US/Eastern date +'%Y%m%d')
if [ $START_DATE == "DAILY" ]; then
DATA_PATH="${PACAKGE_DIR%/}/data/$DATE"
S3_OUT="$S3_MOUNT/daily"
fi

if [ ${#RELATIVE_TO} -gt 0 ] ; then
Expand Down Expand Up @@ -264,12 +265,16 @@ echo "Running NextGen in AUTO MODE from CIROH-UA/NGIAB-CloudInfra"
docker run --rm -v "$NGEN_RUN_PATH":"$DOCKER_MOUNT" awiciroh/ciroh-ngen-image:latest-local "$DOCKER_MOUNT" auto

# hashing
docker run --rm -v "$NGEN_RUN_PATH":"$DOCKER_MOUNT" zwills/merkdir /merkdir/merkdir gen -o $DOCKER_MOUNT/merkdir.file $DOCKER_MOUNT
docker run --rm -v "$DATA_PATH":"$DOCKER_MOUNT" zwills/merkdir /merkdir/merkdir gen -o $DOCKER_MOUNT/merkdir.file $DOCKER_MOUNT

TAR_NAME="ngen-run.tar.gz"
TAR_PATH="${DATA_PATH%/}/$TAR_NAME"
tar -cf - $NGEN_RUN_PATH | pigz > $TAR_PATH

mv $DATASTREAM_RESOURCES "../datastream-resources-$DATE"

if [ -z $S3_MOUNT ]; then
mkdir $S3_MOUNT/$DATE
cp -r $NGEN_RUN_PATH $S3_MOUNT/$DATE
mkdir -p $S3_MOUNT
mount-s3 ngen-datastream $S3_MOUNT
cp -r $DATA_PATH $S3_OUT

0 comments on commit 6276ae8

Please sign in to comment.