feat: obey the rate limit

Wait as long as specified in the retry-after header, send out by GitLab Fixes #73
jdalrymple · Jul 4, 2018 · 9b46250 · 9b46250
1 parent 4458bea
commit 9b46250
Show file tree

Hide file tree

Showing 2 changed files with 50 additions and 33 deletions.
diff --git a/src/infrastructure/RequestHelper.js b/src/infrastructure/RequestHelper.js
@@ -54,45 +54,59 @@ function getStream(service, endpoint, options = {}) {
   return StreamableRequest.get(requestOptions);
 }
 
-async function getPaginated(service, endpoint, options = {}) {
+async function wait(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+async function getPaginated(service, endpoint, options = {}, sleepOnRateLimit = true) {
   const { showPagination, maxPages, ...queryOptions } = options;
   const requestOptions = defaultRequest(service, endpoint, {
     headers: service.headers,
     qs: queryOptions,
     resolveWithFullResponse: true,
   });
 
-  const response = await service.requester.get(requestOptions);
-  const links = LinkParser(response.headers.link) || {};
-  const page = response.headers['x-page'];
-  const underMaxPageLimit = maxPages ? page < maxPages : true;
-  let more = [];
-  let data;
-
-  // If not looking for a singular page and still under the max pages limit
-  // AND their is a next page, paginate
-  if (!queryOptions.page && underMaxPageLimit && links.next) {
-    more = await getPaginated(service, links.next.url.replace(service.url, ''), options);
-    data = [...response.body, ...more];
-  } else {
-    data = response.body;
-  }
+  try {
+    const response = await service.requester.get(requestOptions);
+    const links = LinkParser(response.headers.link) || {};
+    const page = response.headers['x-page'];
+    const underMaxPageLimit = maxPages ? page < maxPages : true;
+    let more = [];
+    let data;
+
+    // If not looking for a singular page and still under the max pages limit
+    // AND their is a next page, paginate
+    if (!queryOptions.page && underMaxPageLimit && links.next) {
+      more = await getPaginated(service, links.next.url.replace(service.url, ''), options);
+      data = [...response.body, ...more];
+    } else {
+      data = response.body;
+    }
 
-  if (queryOptions.page && showPagination) {
-    return {
-      data,
-      pagination: {
-        total: response.headers['x-total'],
-        next: response.headers['x-next-page'] || null,
-        current: response.headers['x-page'] || null,
-        previous: response.headers['x-prev-page'] || null,
-        perPage: response.headers['x-per-page'],
-        totalPages: response.headers['x-total-pages'],
-      },
-    };
-  }
+    if (queryOptions.page && showPagination) {
+      return {
+        data,
+        pagination: {
+          total: response.headers['x-total'],
+          next: response.headers['x-next-page'] || null,
+          current: response.headers['x-page'] || null,
+          previous: response.headers['x-prev-page'] || null,
+          perPage: response.headers['x-per-page'],
+          totalPages: response.headers['x-total-pages'],
+        },
+      };
+    }
 
-  return data;
+    return data;
+  } catch (err) {
+    const sleepTime = parseInt(err.response.headers['retry-after'], 10);
+    if (sleepOnRateLimit && parseInt(err.statusCode, 10) === 429
+         && sleepTime) {
+      await wait(sleepTime * 1000);
+      return getPaginated(service, endpoint, options, sleepOnRateLimit);
+    }
+    throw err;
+  }
 }
 
 class RequestHelper {

diff --git a/src/infrastructure/XMLHttpRequester.js b/src/infrastructure/XMLHttpRequester.js
@@ -1,14 +1,17 @@
 import { StatusCodeError } from 'request-promise-core/errors';
 import Promisify from 'util.promisify';
 import XHR from 'xhr';
+import wait from './RequestHelper';
 
 function promisifyFn(fn) {
   const promisifiedFn = Promisify(fn);
 
-  return async function (opts) {
+  return async function getResponse(opts) {
     const response = await promisifiedFn(opts);
-
-    if (response.statusCode >= 400 && response.statusCode <= 599) {
+    const sleepTime = parseInt(response.headers['retry-after'], 10);
+    if (response.statusCode === 429 && sleepTime) {
+      await wait(sleepTime * 1000);
+    } else if (response.statusCode >= 400 && response.statusCode <= 599) {
       throw new StatusCodeError(response.statusCode, response.body, {}, null);
     }