Parallelize records creation and fetch all Contentful entries

package.json

@@ -1,6 +1,6 @@
 {
   "name": "datocms-client",
-  "version": "3.0.39",
+  "version": "3.0.40",
   "description": "DatoCMS API client and CLI tool",
   "browser": "dist/client.js",
   "main": "lib/index.js",
@@ -110,6 +110,7 @@
     "ora": "4.0.3",
     "pluralize": "^8.0.0",
     "pretty-error": "^2.1.1",
+    "promise-limit": "^2.7.0",
     "proxy-polyfill": "^0.3.0",
     "pusher-js": "^5.0.3",
     "qs": "^6.9.1",

src/contentfulImport/createRecords.js

@@ -1,4 +1,5 @@
 import ora from 'ora';
+import promiseLimit from 'promise-limit';
 import Progress from './progress';
 import { toItemApiKey, toFieldApiKey } from './toApiKey';
 
@@ -21,6 +22,9 @@ export default async ({
 
     spinner.text = progress.tick();
 
+    const limit = promiseLimit(5);
+    const jobs = [];
+
     for (const entry of entries) {
       const { contentType } = entry.sys;
       const contentTypeApiKey = toItemApiKey(contentType.sys.id);
@@ -125,20 +129,26 @@ export default async ({
           emptyFieldValues,
         );
 
-        const record = await datoClient.items.create({
-          ...recordAttributes,
-          itemType: itemType.id.toString(),
-        });
+        jobs.push(
+          limit(async () => {
+            const record = await datoClient.items.create({
+              ...recordAttributes,
+              itemType: itemType.id.toString(),
+            });
 
-        if (entry.sys.publishedVersion) {
-          recordsToPublish.push(record.id);
-        }
+            if (entry.sys.publishedVersion) {
+              recordsToPublish.push(record.id);
+            }
 
-        spinner.text = progress.tick();
-        contentfulRecordMap[entry.sys.id] = record.id;
+            spinner.text = progress.tick();
+            contentfulRecordMap[entry.sys.id] = record.id;
+          }),
+        );
       }
     }
 
+    await Promise.all(jobs);
+
     spinner.succeed();
 
     return { contentfulRecordMap, recordsToPublish };

src/contentfulImport/getContentfulData.js

@@ -1,5 +1,28 @@
 import ora from 'ora';
 
+async function allPages(apiCall) {
+  const limit = 100;
+  let items = [];
+  let skip = 0;
+
+  // eslint-disable-next-line no-constant-condition
+  while (true) {
+    const response = await apiCall({ skip, limit, order: 'sys.createdAt' });
+
+    const { items: pageItems, total } = response;
+
+    items = [].concat(items, pageItems);
+
+    if (items.length >= total) {
+      break;
+    } else {
+      skip += limit;
+    }
+  }
+
+  return items;
+}
+
 export default async ({ client, skipContent, contentType }) => {
   const spinner = ora('Downloading Contentful data structure').start();
   const environments = await client.getEnvironments();
@@ -17,18 +40,20 @@ export default async ({ client, skipContent, contentType }) => {
   let assets;
 
   if (!skipContent) {
-    const rawEntries = await environment.getEntries();
-    const rawAssets = await environment.getAssets();
+    const rawEntries = await allPages(environment.getEntries.bind(environment));
+    const rawAssets = await allPages(environment.getAssets.bind(environment));
 
     entries = contentType
-      ? rawEntries.items.filter(entry =>
+      ? rawEntries.filter(entry =>
           includeTypes.has(entry.sys.contentType.sys.id),
         )
-      : rawEntries.items;
-    assets = rawAssets.items;
+      : rawEntries;
+    assets = rawAssets;
   }
 
-  spinner.succeed();
+  spinner.succeed(
+    `Found ${entries.length} entries and ${assets.length} assets in Contentful project`,
+  );
 
   return {
     defaultLocale,

src/contentfulImport/progress.js

@@ -2,10 +2,11 @@ import { Progress } from 'clui';
 
 export default function(size, text) {
   let currentValue = 0;
-  const progress = new Progress(size);
+  const progress = new Progress(30);
 
   this.tick = () => {
-    const progressText = progress.update((currentValue += 1), size);
-    return `${text} ${progressText}`;
+    currentValue += 1;
+    const progressText = progress.update(currentValue / size);
+    return `${text.padEnd(70, ' ')}${progressText}`;
   };
 }

yarn.lock

@@ -5861,6 +5861,11 @@ promise-inflight@^1.0.1:
   resolved "https://registry.yarnpkg.com/promise-inflight/-/promise-inflight-1.0.1.tgz#98472870bf228132fcbdd868129bad12c3c029e3"
   integrity sha1-mEcocL8igTL8vdhoEputEsPAKeM=
 
+promise-limit@^2.7.0:
+  version "2.7.0"
+  resolved "https://registry.yarnpkg.com/promise-limit/-/promise-limit-2.7.0.tgz#eb5737c33342a030eaeaecea9b3d3a93cb592b26"
+  integrity sha512-7nJ6v5lnJsXwGprnGXga4wx6d1POjvi5Qmf1ivTRxTjH4Z/9Czja/UCMLVmB9N93GeWOU93XaFaEt6jbuoagNw==
+
 propagate@^2.0.0:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/propagate/-/propagate-2.0.1.tgz#40cdedab18085c792334e64f0ac17256d38f9a45"