From 47f1975df041dbd8aaca83e127d2b72a99fee2ee Mon Sep 17 00:00:00 2001 From: samogot Date: Sun, 13 May 2018 23:36:56 +0300 Subject: [PATCH] Turn throttle into config option instead of instance property As was documented in command comments. That allows to set global throttle as well as dynamically change throttle command-wise --- index.js | 7 ++-- lib/commands/throttle.js | 4 +- test/throttle.js | 85 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index 02a59d3..bd3d970 100644 --- a/index.js +++ b/index.js @@ -62,7 +62,6 @@ function Osmosis(url, params) { this.queue = new Queue(this); this.command = new Command(this); this.id = ++instanceId; - this.throttle = new RateLimiter(999, 1, true); } @@ -74,6 +73,7 @@ function Osmosis(url, params) { * @property {string} accept - HTTP Accept header * @property {bool} compressed - Compress HTTP requests * @property {number} concurrency - Number of simultaneous HTTP requests + * @property {RateLimiter} throttle - RateLimiter object to throttle following requests * @property {bool} decode_response - Decode compressed HTTP responses * @property {number} follow - Number of redirects to follow * @property {bool} follow_set_cookies - Set cookies for redirects @@ -93,6 +93,7 @@ Osmosis.prototype.opts = { 'application/xml;q=0.9,*/*;q=0.8', compressed: true, concurrency: 5, + throttle: new RateLimiter(999, 1, true), decode_response: true, follow: 3, follow_set_cookies: true, @@ -182,7 +183,7 @@ Osmosis.prototype.request = function (url, opts, callback, tries) { this.queue.requests++; this.queue.push(); - this.throttle.removeTokens(1, function(err, remainingRequests) { + opts.throttle.removeTokens(1, function(err, remainingRequests) { request(url.method, url, url.params, @@ -325,7 +326,7 @@ Osmosis.prototype.resources = function () { 'requests: ' + this.requests + ' (' + this.queue.requests + ' queued), ' + - 'throttled: ' + parseInt(this.throttle.getTokensRemaining()) + ', ' + + 'throttled: ' + parseInt(this.opts.throttle.getTokensRemaining()) + ', ' + 'RAM: ' + toMB(mem.rss) + ' (' + memDiff + '), ' + diff --git a/lib/commands/throttle.js b/lib/commands/throttle.js index efeddce..910db84 100644 --- a/lib/commands/throttle.js +++ b/lib/commands/throttle.js @@ -13,7 +13,9 @@ var RateLimiter = require('limiter').RateLimiter; module.exports = function (tokensPerInterval, interval) { - this.instance.throttle = new RateLimiter( + var opts = this.getOpts(); + + opts.throttle = new RateLimiter( tokensPerInterval || 1000, interval || 1 ); diff --git a/test/throttle.js b/test/throttle.js index e2fe96c..89bc6e0 100644 --- a/test/throttle.js +++ b/test/throttle.js @@ -1,5 +1,6 @@ var osmosis = require('../index'), server = require('./server'), + RateLimiter = require('limiter').RateLimiter, url = server.host + ':' + server.port; @@ -26,6 +27,90 @@ module.exports.on = function (assert) { }); }; + +module.exports.global_throttle = function (assert) { + var date0 = +Date.now(), interval = 2000; + osmosis.config('throttle', new RateLimiter(2, interval)); + osmosis.get(url + '/get') + .then(function () { + var date = +Date.now(); + assert.ok(date - date0 < interval); + }) + .set({ + test: osmosis + .get(url + '/get') + .then(function () { + var date = +Date.now(); + assert.ok(date - date0 < interval); + }) + .get(url + '/get') + .then(function () { + var date = +Date.now(); + assert.ok(date - date0 >= interval); + }) + }) + .done(function () { + assert.done(); + // Turn global throttling off, so it won't affect other tests + osmosis.config('throttle', new RateLimiter(999, 1)); + }); +}; + + +module.exports.dynamic_change = function (assert) { + var date0 = +Date.now(), interval = 2000, date1, date2; + osmosis.get(url + '/get') + .then(function () { + var date = +Date.now(); + assert.ok(date - date0 < interval); + }) + .get(url + '/get') + .then(function () { + var date = +Date.now(); + assert.ok(date - date0 < interval); + }) + .get(url + '/get') + .then(function () { + var date = date1 = +Date.now(); + assert.ok(date - date0 < interval); + }) + .throttle(2, interval) + .get(url + '/get') + .then(function () { + var date = +Date.now(); + assert.ok(date - date1 < interval); + }) + .get(url + '/get') + .then(function () { + var date = +Date.now(); + assert.ok(date - date1 < interval); + }) + .get(url + '/get') + .then(function () { + var date = date2 = +Date.now(); + assert.ok(date - date1 >= interval); + }) + .throttle() // relax + .get(url + '/get') + .then(function () { + var date = +Date.now(); + assert.ok(date - date2 < interval); + }) + .get(url + '/get') + .then(function () { + var date = +Date.now(); + assert.ok(date - date2 < interval); + }) + .get(url + '/get') + .then(function () { + var date = +Date.now(); + assert.ok(date - date2 < interval); + }) + .done(function () { + assert.done(); + }); +}; + module.exports.off = function (assert) { var date0 = +Date.now(), interval = 2000; osmosis.get(url + '/get')