From f40323cb8728f145e5551eec165d368de77182c4 Mon Sep 17 00:00:00 2001 From: Richard Lau Date: Mon, 22 Jul 2024 13:19:12 +0100 Subject: [PATCH] ansible: fix promotion to R2 dist-prod for releases Due to R2 limitations, `aws s3 cp` and `aws s3 sync` only succeed in copying different sets of files across. Fortunately the sets are disjoint, so running both commands (ignoring errors) will copy all the files across. Update `resha_release.sh` to redirect to a log file the output from the `upload_to_cloudflare.sh` to prevent extra output interfering with the `release.sh` script. --- ansible/www-standalone/tools/promote/resha_release.sh | 7 ++++++- .../www-standalone/tools/promote/upload_to_cloudflare.sh | 6 +++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/ansible/www-standalone/tools/promote/resha_release.sh b/ansible/www-standalone/tools/promote/resha_release.sh index 86fd92694..97325fe43 100755 --- a/ansible/www-standalone/tools/promote/resha_release.sh +++ b/ansible/www-standalone/tools/promote/resha_release.sh @@ -25,6 +25,11 @@ fi ${__dirname}/_resha.sh $site $dstdir $2 -. ${__dirname}/upload_to_cloudflare.sh $site $2 +# https://github.com/nodejs/build/issues/3508 +# Output from upload_to_cloudflare.sh must not go to stdout to prevent +# breaking the release.sh script. +logfile="${__dirname}/logs/cloudflare.log" +date >> "${logfile}" +. ${__dirname}/upload_to_cloudflare.sh $site $2 >> "${logfile}" 2>&1 /home/nodejs/queue-cdn-purge.sh $site resha_release diff --git a/ansible/www-standalone/tools/promote/upload_to_cloudflare.sh b/ansible/www-standalone/tools/promote/upload_to_cloudflare.sh index d91438b9a..bc2add9e6 100755 --- a/ansible/www-standalone/tools/promote/upload_to_cloudflare.sh +++ b/ansible/www-standalone/tools/promote/upload_to_cloudflare.sh @@ -42,6 +42,10 @@ fi relativedir=${dstdir/$dist_rootdir/"$site/"} tmpversion=$2 -aws s3 cp $staging_bucket/$relativedir/$tmpversion/ $dist_bucket/$relativedir/$tmpversion/ --endpoint-url=$cloudflare_endpoint --profile $cloudflare_profile --recursive --no-follow-symlinks +# Due to R2 limitations, `aws s3 cp` and `aws s3 sync` only succeed in copying +# different sets of files across. Fortunately the sets are disjoint, so running +# both commands (ignoring errors) will copy all the files across. +aws s3 cp $staging_bucket/$relativedir/$tmpversion/ $dist_bucket/$relativedir/$tmpversion/ --endpoint-url=$cloudflare_endpoint --profile $cloudflare_profile --recursive --no-follow-symlinks --copy-props none || true +aws s3 sync $staging_bucket/$relativedir/$tmpversion/ $dist_bucket/$relativedir/$tmpversion/ --endpoint-url=$cloudflare_endpoint --profile $cloudflare_profile --no-follow-symlinks || true aws s3 cp $staging_bucket/$relativedir/index.json $dist_bucket/$relativedir/index.json --endpoint-url=$cloudflare_endpoint --profile $cloudflare_profile aws s3 cp $staging_bucket/$relativedir/index.tab $dist_bucket/$relativedir/index.tab --endpoint-url=$cloudflare_endpoint --profile $cloudflare_profile