const queue = require('async').queue;
const request = require('request');
/** Class control free keys. */
function ApiKeys(apiKeys) {
this.apiKeys = apiKeys;
this.locked = {};
}
/**
* Get a free key and lock it
* @return {string} Key.
*/
ApiKeys.prototype.getKey = function () {
for (const key of this.apiKeys) {
if (!this.locked[key]) {
this.locked[key] = true;
return key;
}
}
}
/**
* Method free key for next request
* @param {string} key - Locked key.
*/
ApiKeys.prototype.freeKey = function (key) {
this.locked[key] = false;
}
const API_KEYS = ['key1', 'key2', 'key3', 'key4', 'key5', 'key6', 'ke7', 'key8'];
const MAX_WORKERS_IN_QUEUE = API_KEYS.length;
const apiKeys = new ApiKeys(API_KEYS);
const requestQueue = queue(function (task, callback) {
const key = apiKeys.getKey();
task.options.url += '&key=' + key;
request(task.options, (err, res, body) => {
apiKeys.freeKey(key);
callback(err, res, body);
})
}, MAX_WORKERS_IN_QUEUE);
const requestUrls = [];
for (const url of requestUrls) {
const options = {
url: url,
method: 'GET'
};
requestQueue.push({ options: options }, function (err, res, body) {
// process response here
});
}
requestQueue.drain = function () {
console.log('all items have been processed');
};