Process S3 redirects in parallel

package.json

@@ -6,6 +6,7 @@
     "minify-css": "node scripts/minify-css.js"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.427.0",
     "@fullhuman/postcss-purgecss": "^4.0.3",
     "@octokit/auth-action": "^1.3.2",
     "@octokit/graphql": "^4.6.2",

scripts/make-s3-redirects.js

@@ -0,0 +1,82 @@
+const { PutObjectCommand, S3Client } = require("@aws-sdk/client-s3");
+const fs = require("fs");
+
+const bucket = process.argv[2];
+const redirectsFile = process.argv[3] || "./public/redirects.txt";
+const region = process.argv[4] || "us-west-2"
+
+async function doRedirects(bucket, region) {
+    const redirects = fs.readFileSync(redirectsFile, "utf-8").trim().split("\n");
+    console.log(`Processing ${redirects.length} redirects...`);
+
+    const client = new S3Client({ region, endpoint: `https://s3.${region}.amazonaws.com` });
+
+    // Chunk requests into groups of a thousand to process them more efficiently.
+    const chunkSize = 1000;
+    const chunks = [];
+    const results = [];
+
+    for (let i = 0; i < redirects.length; i += chunkSize) {
+        chunks.push({ chunk: i / chunkSize, lines: redirects.slice(i, i + chunkSize) });
+    }
+
+    for await (const chunk of chunks) {
+        console.log(` ↳ Processing group ${chunk.chunk + 1} of ${chunks.length} (${chunk.lines.length} URLs)...`);
+
+        const result = await Promise.allSettled(chunk.lines.map(line => {
+            const [ key, location ] = line.split("|");
+
+            return new Promise(async (resolve, reject) => {
+                try {
+                    console.log(` ↳ Redirecting ${key} to ${location}`);
+
+                    const command = new PutObjectCommand({
+                        Bucket: bucket,
+                        Key: key,
+                        WebsiteRedirectLocation: location,
+                        ACL: "public-read",
+                        Body: "",
+                        ContentLength: 0,
+                    });
+
+                    const res = await client.send(command);
+                    const status = res.$metadata.httpStatusCode;
+
+                    if (status < 400) {
+                        resolve(status);
+                    } else {
+                        reject(status);
+                    }
+                }
+                catch (error) {
+                    reject(`Error redirecting ${key}: ${error}`);
+                }
+            });
+        }));
+
+        results.push(...result);
+    }
+
+    return results;
+}
+
+doRedirects(bucket, region)
+    .then(results => {
+
+        const summary = {
+            checked: results.length,
+            fulfilled: results.filter(r => r.status === "fulfilled").map(r => r.value) || [],
+            rejected: results.filter(r => r.status === "rejected").map(r => r.reason) || [],
+        };
+
+        console.log(" ↳ Done. ✨\n");
+
+        if (summary.rejected.length > 0) {
+            throw new Error(`One or more redirects failed: \n\n${summary.rejected.join("\n")}\n`);
+        }
+    });
+
+// Exit non-zero when something goes wrong in the promise chain.
+process.on("unhandledRejection", error => {
+    throw new Error(error);
+});

scripts/make-s3-redirects.sh

@@ -14,22 +14,10 @@ redirects_file="./redirects.txt"
 aws s3 cp "s3://${destination_bucket}/redirects.txt" "$redirects_file" --region "$(aws_region)"
 
 echo "Processing S3 redirects for ${destination_bucket}..."
-IFS="|"
-while read key location; do
-    echo "Redirecting $key to $location"
-    aws s3api put-object --key "$key" --website-redirect-location "$location" --bucket "$destination_bucket" --acl public-read --region "$(aws_region)"
-done < "$redirects_file"
+node scripts/make-s3-redirects.js "${destination_bucket}" "${build_dir}/redirects.txt" "$(aws_region)"
 
-rm "$redirects_file"
-
-# Apply custom redirects supplied in the `scripts/redirects` directory.
+echo "Processing custom redirects in scripts/redirects..."
 ls -l "./scripts/redirects/" | tail -n +2 | awk '{print $9}' | while read line; do
     redirect_file="./scripts/redirects/$line"
-    while read key location; do
-        # skip empty lines
-        if [[ ! -z "$key" ]]; then
-            echo "Redirecting $key to $location"
-            aws s3api put-object --key "$key" --website-redirect-location "$location" --bucket "$destination_bucket" --acl public-read --region "$(aws_region)"
-        fi
-    done < "$redirect_file"
+    node scripts/make-s3-redirects.js "${destination_bucket}" "${redirect_file}" "$(aws_region)"
 done