From 9049e71e90f7acbfbe2e757e7d2b069c521c702f Mon Sep 17 00:00:00 2001
From: Sean Holung <sean.holung@gmail.com>
Date: Fri, 6 Oct 2023 12:28:30 -0400
Subject: [PATCH] Batch process redirects async (#10005)

* process redirects async in batches

* remove aws redirect file
---
 scripts/make-s3-redirects.sh           | 28 ++++++++++++---
 scripts/redirects/aws-v6-redirects.txt | 47 --------------------------
 2 files changed, 24 insertions(+), 51 deletions(-)
 delete mode 100644 scripts/redirects/aws-v6-redirects.txt

diff --git a/scripts/make-s3-redirects.sh b/scripts/make-s3-redirects.sh
index 0cbff2b9469d..9e68de1a529b 100755
--- a/scripts/make-s3-redirects.sh
+++ b/scripts/make-s3-redirects.sh
@@ -13,13 +13,25 @@ destination_bucket="$(cat "$(origin_bucket_metadata_filepath)" | jq -r ".bucket"
 redirects_file="./redirects.txt"
 aws s3 cp "s3://${destination_bucket}/redirects.txt" "$redirects_file" --region "$(aws_region)"
 
-echo "Processing S3 redirects for ${destination_bucket}..."
+echo "Processing S3 redirects ${destination_bucket}..."
+batch_size=50
+command_count=0
 IFS="|"
 while read key location; do
     echo "Redirecting $key to $location"
-    aws s3api put-object --key "$key" --website-redirect-location "$location" --bucket "$destination_bucket" --acl public-read --region "$(aws_region)"
+    aws s3api put-object --key "$key" --website-redirect-location "$location" --bucket "$destination_bucket" --acl public-read --region "$(aws_region)" &
+
+    ((command_count++))
+    # Process aws s3 commands async in batches of 50. Once 50 is reached, wait until batch completes.
+    if [ "$command_count" -eq "$batch_size" ]; then
+        wait
+        command_count=0
+    fi
 done < "$redirects_file"
 
+wait
+command_count=0
+
 rm "$redirects_file"
 
 # Apply custom redirects supplied in the `scripts/redirects` directory.
@@ -29,7 +41,15 @@ ls -l "./scripts/redirects/" | tail -n +2 | awk '{print $9}' | while read line;
         # skip empty lines
         if [[ ! -z "$key" ]]; then
             echo "Redirecting $key to $location"
-            aws s3api put-object --key "$key" --website-redirect-location "$location" --bucket "$destination_bucket" --acl public-read --region "$(aws_region)"
+            aws s3api put-object --key "$key" --website-redirect-location "$location" --bucket "$destination_bucket" --acl public-read --region "$(aws_region)" &
+
+            ((command_count++))
+            # Process s3 commands async in batches of 50. Once 50 is reached, wait until batch completes.
+            if [ "$command_count" -eq "$batch_size" ]; then
+                wait
+                command_count=0
+            fi
         fi
     done < "$redirect_file"
-done
+    wait
+done
\ No newline at end of file
diff --git a/scripts/redirects/aws-v6-redirects.txt b/scripts/redirects/aws-v6-redirects.txt
deleted file mode 100644
index b85fece29248..000000000000
--- a/scripts/redirects/aws-v6-redirects.txt
+++ /dev/null
@@ -1,47 +0,0 @@
-registry/packages/aws/api-docs/applicationloadbalancing/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/alb/
-registry/packages/aws/api-docs/applicationloadbalancing/getlistener/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/alb/getlistener/
-registry/packages/aws/api-docs/applicationloadbalancing/getloadbalancer/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/alb/getloadbalancer/
-registry/packages/aws/api-docs/applicationloadbalancing/gettargetgroup/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/alb/gettargetgroup/
-registry/packages/aws/api-docs/applicationloadbalancing/listener/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/alb/listener/
-registry/packages/aws/api-docs/applicationloadbalancing/listenercertificate/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/alb/listenercertificate/
-registry/packages/aws/api-docs/applicationloadbalancing/listenerrule/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/alb/listenerrule/
-registry/packages/aws/api-docs/applicationloadbalancing/loadbalancer/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/alb/loadbalancer/
-registry/packages/aws/api-docs/applicationloadbalancing/targetgroup/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/alb/targetgroup/
-registry/packages/aws/api-docs/applicationloadbalancing/targetgroupattachment/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/alb/targetgroupattachment/
-registry/packages/aws/api-docs/cloudtrail/getfunction/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/cloudfront/getfunction/
-registry/packages/aws/api-docs/ec2/getsubnetids/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/ec2/getsubnets/
-registry/packages/aws/api-docs/ec2/transitgatewaypeeringattachmentaccepter/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/ec2transitgateway/peeringattachmentaccepter/
-registry/packages/aws/api-docs/elasticache/securitygroup/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/elasticache/
-registry/packages/aws/api-docs/elasticloadbalancing/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/elb/
-registry/packages/aws/api-docs/elasticloadbalancing/appcookiestickinesspolicy/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/elb/appcookiestickinesspolicy/
-registry/packages/aws/api-docs/elasticloadbalancing/attachment/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/elb/attachment/
-registry/packages/aws/api-docs/elasticloadbalancing/gethostedzoneid/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/elb/gethostedzoneid/
-registry/packages/aws/api-docs/elasticloadbalancing/getloadbalancer/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/elb/getloadbalancer/
-registry/packages/aws/api-docs/elasticloadbalancing/getserviceaccount/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/elb/getserviceaccount/
-registry/packages/aws/api-docs/elasticloadbalancing/listenerpolicy/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/elb/listenerpolicy/
-registry/packages/aws/api-docs/elasticloadbalancing/loadbalancer/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/elb/loadbalancer/
-registry/packages/aws/api-docs/elasticloadbalancing/loadbalancerbackendserverpolicy/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/elb/loadbalancerbackendserverpolicy/
-registry/packages/aws/api-docs/elasticloadbalancing/loadbalancercookiestickinesspolicy/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/elb/loadbalancercookiestickinesspolicy/
-registry/packages/aws/api-docs/elasticloadbalancing/loadbalancerpolicy/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/elb/loadbalancerpolicy/
-registry/packages/aws/api-docs/elasticloadbalancing/sslnegotiationpolicy/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/elb/sslnegotiationpolicy/
-registry/packages/aws/api-docs/elasticloadbalancingv2/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/lb/
-registry/packages/aws/api-docs/elasticloadbalancingv2/getlistener/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/lb/getlistener/
-registry/packages/aws/api-docs/elasticloadbalancingv2/getloadbalancer/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/lb/getloadbalancer/
-registry/packages/aws/api-docs/elasticloadbalancingv2/gettargetgroup/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/lb/gettargetgroup/
-registry/packages/aws/api-docs/elasticloadbalancingv2/listener/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/lb/listener/
-registry/packages/aws/api-docs/elasticloadbalancingv2/listenercertificate/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/lb/listenercertificate/
-registry/packages/aws/api-docs/elasticloadbalancingv2/listenerrule/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/lb/listenerrule/
-registry/packages/aws/api-docs/elasticloadbalancingv2/loadbalancer/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/lb/loadbalancer/
-registry/packages/aws/api-docs/elasticloadbalancingv2/targetgroup/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/lb/targetgroup/
-registry/packages/aws/api-docs/elasticloadbalancingv2/targetgroupattachment/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/lb/targetgroupattachment/
-registry/packages/aws/api-docs/getami/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/ec2/getami/
-registry/packages/aws/api-docs/getamiids/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/ec2/getamiids/
-registry/packages/aws/api-docs/getautoscalinggroups/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/autoscaling/getamiids/
-registry/packages/aws/api-docs/getcanonicaluserid/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/s3/getcanonicaluserid/
-registry/packages/aws/api-docs/getelasticip/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/ec2/getelasticip/
-registry/packages/aws/api-docs/getprefixlist/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/ec2/getprefixlist/
-registry/packages/aws/api-docs/macie/memberaccountassociation/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/macie/
-registry/packages/aws/api-docs/macie/s3bucketassociation/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/macie/
-registry/packages/aws/api-docs/rds/securitygroup/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/rds/
-registry/packages/aws/api-docs/redshift/securitygroup/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/redshift/
-registry/packages/aws/api-docs/ses/confgurationset/index.html|https://www.pulumi.com/registry/packages/aws/api-docs/ses/configurationset/
\ No newline at end of file