add pMap

owid · Feb 19, 2024 · 2ee009e · 2ee009e
1 parent caf932c
commit 2ee009e
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 7 deletions.
diff --git a/baker/GrapherBaker.tsx b/baker/GrapherBaker.tsx
@@ -61,6 +61,7 @@ import { parseFaqs } from "../db/model/Gdoc/rawToEnriched.js"
 import { GdocPost } from "../db/model/Gdoc/GdocPost.js"
 import { getShortPageCitation } from "../site/gdocs/utils.js"
 import { getSlugForTopicTag, getTagToSlugMap } from "./GrapherBakingUtils.js"
+import pMap from "p-map"
 
 const renderDatapageIfApplicable = async (
     grapher: GrapherInterface,
@@ -491,11 +492,13 @@ export const bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers =
             }
         )
 
-        await Promise.all(
-            jobs.map(async (job) => {
+        await pMap(
+            jobs,
+            async (job) => {
                 await bakeSingleGrapherChart(job)
                 progressBar.tick({ name: `slug ${job.slug}` })
-            })
+            },
+            { concurrency: 20 }
         )
 
         await deleteOldGraphers(bakedSiteDir, excludeUndefined(newSlugs))

diff --git a/db/model/Variable.ts b/db/model/Variable.ts
@@ -415,10 +415,10 @@ export const fetchS3DataValuesByPath = async (
     const resp = await retryPromise(
         () =>
             fetch(dataPath, { keepalive: true }).then((response) => {
-                if (!resp.ok) {
+                if (!response.ok) {
                     // Trigger retry
                     throw new Error(
-                        `Error fetching data from S3 for ${dataPath}: ${resp.status} ${resp.statusText}`
+                        `Error fetching data from S3 for ${dataPath}: ${response.status} ${response.statusText}`
                     )
                 }
                 return response
@@ -448,10 +448,10 @@ export const fetchS3MetadataByPath = async (
     const resp = await retryPromise(
         () =>
             fetch(metadataPath, { keepalive: true }).then((response) => {
-                if (!resp.ok) {
+                if (!response.ok) {
                     // Trigger retry
                     throw new Error(
-                        `Error fetching metadata from S3 for ${metadataPath}: ${resp.status} ${resp.statusText}`
+                        `Error fetching metadata from S3 for ${metadataPath}: ${response.status} ${response.statusText}`
                     )
                 }
                 return response