diff --git a/package.json b/package.json index fa469283..9a2c3e92 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "datocms-client", - "version": "3.0.39", + "version": "3.0.40", "description": "DatoCMS API client and CLI tool", "browser": "dist/client.js", "main": "lib/index.js", @@ -110,6 +110,7 @@ "ora": "4.0.3", "pluralize": "^8.0.0", "pretty-error": "^2.1.1", + "promise-limit": "^2.7.0", "proxy-polyfill": "^0.3.0", "pusher-js": "^5.0.3", "qs": "^6.9.1", diff --git a/src/contentfulImport/createRecords.js b/src/contentfulImport/createRecords.js index 357cc97e..6c41272b 100644 --- a/src/contentfulImport/createRecords.js +++ b/src/contentfulImport/createRecords.js @@ -1,4 +1,5 @@ import ora from 'ora'; +import promiseLimit from 'promise-limit'; import Progress from './progress'; import { toItemApiKey, toFieldApiKey } from './toApiKey'; @@ -21,6 +22,9 @@ export default async ({ spinner.text = progress.tick(); + const limit = promiseLimit(5); + const jobs = []; + for (const entry of entries) { const { contentType } = entry.sys; const contentTypeApiKey = toItemApiKey(contentType.sys.id); @@ -125,20 +129,26 @@ export default async ({ emptyFieldValues, ); - const record = await datoClient.items.create({ - ...recordAttributes, - itemType: itemType.id.toString(), - }); + jobs.push( + limit(async () => { + const record = await datoClient.items.create({ + ...recordAttributes, + itemType: itemType.id.toString(), + }); - if (entry.sys.publishedVersion) { - recordsToPublish.push(record.id); - } + if (entry.sys.publishedVersion) { + recordsToPublish.push(record.id); + } - spinner.text = progress.tick(); - contentfulRecordMap[entry.sys.id] = record.id; + spinner.text = progress.tick(); + contentfulRecordMap[entry.sys.id] = record.id; + }), + ); } } + await Promise.all(jobs); + spinner.succeed(); return { contentfulRecordMap, recordsToPublish }; diff --git a/src/contentfulImport/getContentfulData.js b/src/contentfulImport/getContentfulData.js index 0104c89a..15d5e9a9 100644 --- a/src/contentfulImport/getContentfulData.js +++ b/src/contentfulImport/getContentfulData.js @@ -1,5 +1,28 @@ import ora from 'ora'; +async function allPages(apiCall) { + const limit = 100; + let items = []; + let skip = 0; + + // eslint-disable-next-line no-constant-condition + while (true) { + const response = await apiCall({ skip, limit, order: 'sys.createdAt' }); + + const { items: pageItems, total } = response; + + items = [].concat(items, pageItems); + + if (items.length >= total) { + break; + } else { + skip += limit; + } + } + + return items; +} + export default async ({ client, skipContent, contentType }) => { const spinner = ora('Downloading Contentful data structure').start(); const environments = await client.getEnvironments(); @@ -17,18 +40,20 @@ export default async ({ client, skipContent, contentType }) => { let assets; if (!skipContent) { - const rawEntries = await environment.getEntries(); - const rawAssets = await environment.getAssets(); + const rawEntries = await allPages(environment.getEntries.bind(environment)); + const rawAssets = await allPages(environment.getAssets.bind(environment)); entries = contentType - ? rawEntries.items.filter(entry => + ? rawEntries.filter(entry => includeTypes.has(entry.sys.contentType.sys.id), ) - : rawEntries.items; - assets = rawAssets.items; + : rawEntries; + assets = rawAssets; } - spinner.succeed(); + spinner.succeed( + `Found ${entries.length} entries and ${assets.length} assets in Contentful project`, + ); return { defaultLocale, diff --git a/src/contentfulImport/progress.js b/src/contentfulImport/progress.js index 6028e38a..ae6cfd5a 100644 --- a/src/contentfulImport/progress.js +++ b/src/contentfulImport/progress.js @@ -2,10 +2,11 @@ import { Progress } from 'clui'; export default function(size, text) { let currentValue = 0; - const progress = new Progress(size); + const progress = new Progress(30); this.tick = () => { - const progressText = progress.update((currentValue += 1), size); - return `${text} ${progressText}`; + currentValue += 1; + const progressText = progress.update(currentValue / size); + return `${text.padEnd(70, ' ')}${progressText}`; }; } diff --git a/yarn.lock b/yarn.lock index ae2a63f0..0cd56e97 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5861,6 +5861,11 @@ promise-inflight@^1.0.1: resolved "https://registry.yarnpkg.com/promise-inflight/-/promise-inflight-1.0.1.tgz#98472870bf228132fcbdd868129bad12c3c029e3" integrity sha1-mEcocL8igTL8vdhoEputEsPAKeM= +promise-limit@^2.7.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/promise-limit/-/promise-limit-2.7.0.tgz#eb5737c33342a030eaeaecea9b3d3a93cb592b26" + integrity sha512-7nJ6v5lnJsXwGprnGXga4wx6d1POjvi5Qmf1ivTRxTjH4Z/9Czja/UCMLVmB9N93GeWOU93XaFaEt6jbuoagNw== + propagate@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/propagate/-/propagate-2.0.1.tgz#40cdedab18085c792334e64f0ac17256d38f9a45"