Batch process redirects async (#10005)

* process redirects async in batches

* remove aws redirect file

scripts/make-s3-redirects.sh

@@ -13,13 +13,25 @@ destination_bucket="$(cat "$(origin_bucket_metadata_filepath)" | jq -r ".bucket"
 redirects_file="./redirects.txt"
 aws s3 cp "s3://${destination_bucket}/redirects.txt" "$redirects_file" --region "$(aws_region)"
 
-echo "Processing S3 redirects for ${destination_bucket}..."
+echo "Processing S3 redirects ${destination_bucket}..."
+batch_size=50
+command_count=0
 IFS="|"
 while read key location; do
     echo "Redirecting $key to $location"
-    aws s3api put-object --key "$key" --website-redirect-location "$location" --bucket "$destination_bucket" --acl public-read --region "$(aws_region)"
+    aws s3api put-object --key "$key" --website-redirect-location "$location" --bucket "$destination_bucket" --acl public-read --region "$(aws_region)" &
+
+    ((command_count++))
+    # Process aws s3 commands async in batches of 50. Once 50 is reached, wait until batch completes.
+    if [ "$command_count" -eq "$batch_size" ]; then
+        wait
+        command_count=0
+    fi
 done < "$redirects_file"
 
+wait
+command_count=0
+
 rm "$redirects_file"
 
 # Apply custom redirects supplied in the `scripts/redirects` directory.
@@ -29,7 +41,15 @@ ls -l "./scripts/redirects/" | tail -n +2 | awk '{print $9}' | while read line;
         # skip empty lines
         if [[ ! -z "$key" ]]; then
             echo "Redirecting $key to $location"
-            aws s3api put-object --key "$key" --website-redirect-location "$location" --bucket "$destination_bucket" --acl public-read --region "$(aws_region)"
+            aws s3api put-object --key "$key" --website-redirect-location "$location" --bucket "$destination_bucket" --acl public-read --region "$(aws_region)" &
+
+            ((command_count++))
+            # Process s3 commands async in batches of 50. Once 50 is reached, wait until batch completes.
+            if [ "$command_count" -eq "$batch_size" ]; then
+                wait
+                command_count=0
+            fi
         fi
     done < "$redirect_file"
-done
+    wait
+done