diff --git a/devTools/syncGraphersToR2/syncGraphersToR2.ts b/devTools/syncGraphersToR2/syncGraphersToR2.ts index e4abd461c23..e0fe4fe3e71 100644 --- a/devTools/syncGraphersToR2/syncGraphersToR2.ts +++ b/devTools/syncGraphersToR2/syncGraphersToR2.ts @@ -35,6 +35,7 @@ import { } from "@ourworldindata/utils" import { string } from "ts-pattern/dist/patterns.js" import { chunk, take } from "lodash" +import ProgressBar from "progress" type HashAndId = Pick @@ -86,6 +87,13 @@ async function syncWithR2( console.log("Number of files to upsert", hashesOfFilesToToUpsert.size) console.log("Number of files to delete", hashesOfFilesToDelete.size) + let progressBar = new ProgressBar( + "--- Deleting obsolote configs [:bar] :current/:total :elapseds\n", + { + total: hashesOfFilesToDelete.size, + } + ) + for (const [key, _] of hashesOfFilesToDelete.entries()) { const deleteObjectCommandInput: DeleteObjectCommandInput = { Bucket: GRAPHER_CONFIG_R2_BUCKET, @@ -96,8 +104,20 @@ async function syncWithR2( new DeleteObjectCommand(deleteObjectCommandInput) ) else console.log("Would have deleted", key) + progressBar.tick() } + console.log("Finished deletes") + + progressBar = new ProgressBar( + "--- Storing missing configs [:bar] :current/:total :elapseds\n", + { + total: hashesOfFilesToToUpsert.size, + } + ) + + const errors = [] + // Chunk the inserts so that we don't need to keep all the full configs in memory for (const batch of chunk([...hashesOfFilesToToUpsert.entries()], 100)) { const fullConfigs = await knexRaw< @@ -116,19 +136,33 @@ async function syncWithR2( console.error(`Full config not found for id ${id}`) return } - const putObjectCommandInput: PutObjectCommandInput = { - Bucket: GRAPHER_CONFIG_R2_BUCKET, - Key: key, - Body: full, - ContentMD5: fullMd5, + try { + const putObjectCommandInput: PutObjectCommandInput = { + Bucket: GRAPHER_CONFIG_R2_BUCKET, + Key: key, + Body: full, + ContentMD5: fullMd5, + } + if (!dryRun) + await s3Client.send( + new PutObjectCommand(putObjectCommandInput) + ) + else console.log("Would have upserted", key) + } catch (err) { + return err } - if (!dryRun) - return s3Client.send( - new PutObjectCommand(putObjectCommandInput) - ) - else console.log("Would have upserted", key) + progressBar.tick() }) - await Promise.all(uploadPromises) + const promiseResults = await Promise.allSettled(uploadPromises) + const batchErrors = promiseResults + .filter((result) => result.status === "rejected") + .map((result) => result.reason) + errors.push(...batchErrors) + } + + console.log("Finished upserts") + if (errors.length > 0) { + console.error("Errors during upserts", errors) } }