From 8446792214db99c17fc503d688f23a2bcfbfb21c Mon Sep 17 00:00:00 2001 From: Brad Vogel Date: Mon, 16 Mar 2015 16:54:58 -0700 Subject: [PATCH 1/3] Change API so it can be instantiated. Fixes https://github.com/wapisasa/batchelor/issues/4 --- lib/batchelor.js | 399 +++++++++++++++++++++++------------------------ 1 file changed, 199 insertions(+), 200 deletions(-) diff --git a/lib/batchelor.js b/lib/batchelor.js index 4e84c25..dc777c3 100644 --- a/lib/batchelor.js +++ b/lib/batchelor.js @@ -25,255 +25,254 @@ 'use strict'; var request = require('request'), - Dicer = require('dicer'), - parser = require('http-string-parser'), - hat = require('hat'); - -(function (Batchelor) { - - /** - * Initialise Batchelor - * - * @options {object} options for the parent request - * @return {object} Batchelor object - */ - Batchelor.init = function (options) { - - // URI is only required option - if (!options.uri) { - throw new Error('options.uri is required'); + Dicer = require('dicer'), + parser = require('http-string-parser'), + hat = require('hat'); + + +/** + * Initialise Batchelor + * + * @options {object} options for the parent request + * @return {object} Batchelor object + */ +var Batchelor = function(options) { + + // URI is only required option + if (!options.uri) { + throw new Error('options.uri is required'); + } + + // Defaults options + options.method = options.method || 'POST'; // Default is POST as this was built specifically for Google + options.headers['Content-Type'] = options.headers['Content-Type'] || 'multipart/mixed;'; + + // Globalise the options + this._globalOptions = options; + + // Ready internal variables + this._requests = []; + this._requestSpecificCallbacks = []; + this._requestExtensionData = []; +}; + +/** + * Add a request into the requests array + * + * @options {object/array} options for individual requests/array of requests + * @return {object} Batchelor object + */ +Batchelor.prototype.add = function(options) { + + var _self = this; + + // Check if adding multiple requests and loop through array + if (!!Array.isArray(options)) { + options.forEach(_self.add); + + return _self; + } + + // Check for required options + if (!options.path) { + throw new Error('options.path is a required argument'); + } + + // Give each request an id so we can identify the response + var rack = hat.rack(), + requestId = options.requestId || 'Batchelor_' + rack(); + + // Save out request specific callback + if (!!options.callback) { + this._requestSpecificCallbacks[requestId] = options.callback; + } + + // Any data thay needs to be passed through to the callback + if (!!options.extend) { + this._requestExtensionData[requestId] = options.extend; + } + + // Set Defaults + options.method = options.method || 'GET'; + options.requestId = requestId; + if (options.method === 'POST') { + // It's not a GET so we need something to push + if (!options.parameters) { + throw new Error('when using POST: options.parameters is required'); } - // Defaults options - options.method = options.method || 'POST'; // Default is POST as this was built specifically for Google - options.headers['Content-Type'] = options.headers['Content-Type'] || 'multipart/mixed;'; + // We will asume if it's not set, we are sending a body of JSON + options.parameters['Content-Type'] = options.parameters['Content-Type'] || 'application/json;'; - // Globalise the options - this._globalOptions = options; - - // Ready internal variables - this._requests = []; - this._requestSpecificCallbacks = []; - this._requestExtensionData = []; - - // Chaining baby - return this; - - }; + // Body exists? + if (!options.parameters.body) { + throw new Error('when using POST: options.parameters.body is required'); + } + } - /** - * Add a request into the requests array - * - * @options {object/array} options for individual requests/array of requests - * @return {object} Batchelor object - */ - Batchelor.add = function (options) { + // Push to _requests Array + this._requests.push(options); - var _self = this; + // Chaining baby + return this; - // Check if adding multiple requests and loop through array - if (!!Array.isArray(options)) { - options.forEach(_self.add); +}; - return _self; - } +/** + * Run the batch requests + * + * @callback {function} function to run when request is complete + * @return {object} API response via callback + */ +Batchelor.prototype.run = function(callback) { - // Check for required options - if (!options.path) { - throw new Error('options.path is a required argument'); - } + var _multiparts = [], + _self = this; - // Give each request an id so we can identify the response - var rack = hat.rack(), - requestId = options.requestId || 'Batchelor_' + rack(); + // Build multipart request + this._requests.forEach(function(requestPart) { - // Save out request specific callback - if (!!options.callback) { - this._requestSpecificCallbacks[requestId] = options.callback; - } + var requestSettings = { + 'Content-Type': 'application/http', + 'Content-ID': requestPart.requestId, + 'body': requestPart.method + ' ' + requestPart.path + '\n' + }; - // Any data thay needs to be passed through to the callback - if (!!options.extend) { - this._requestExtensionData[requestId] = options.extend; + // Check if this part needs it's own auth + if ((!!requestPart.auth) && (!!requestPart.auth.bearer)) { + requestSettings.body += 'Authorization: Bearer ' + requestPart.auth.bearer + '\n'; } - // Set Defaults - options.method = options.method || 'GET'; - options.requestId = requestId; - if (options.method === 'POST') { - // It's not a GET so we need something to push - if (!options.parameters) { - throw new Error('when using POST: options.parameters is required'); - } - - // We will asume if it's not set, we are sending a body of JSON - options.parameters['Content-Type'] = options.parameters['Content-Type'] || 'application/json;'; - - // Body exists? - if (!options.parameters.body) { - throw new Error('when using POST: options.parameters.body is required'); - } + // Replace body with a request if this isn't a GET + if (requestPart.method !== 'GET') { + requestSettings.body = requestPart.method + ' ' + requestPart.path + '\n' + + 'Content-Type: ' + requestPart.parameters['Content-Type'] + '\n\n' + + JSON.stringify(requestPart.parameters.body); } - // Push to _requests Array - this._requests.push(options); - - // Chaining baby - return this; - - }; + // Push into array for the request + _multiparts.push(requestSettings); - /** - * Run the batch requests - * - * @callback {function} function to run when request is complete - * @return {object} API response via callback - */ - Batchelor.run = function (callback) { + }); - var _multiparts = [], - _self = this; + // Collate objects + var requestObj = this._globalOptions; + requestObj.multipart = _multiparts; - // Build multipart request - this._requests.forEach(function (requestPart) { + // Run Requests + request(requestObj).on('response', function(responseObj) { - var requestSettings = { - 'Content-Type': 'application/http', - 'Content-ID': requestPart.requestId, - 'body': requestPart.method + ' ' + requestPart.path + '\n' + // Get the boundary + var boundaryRegex = /^multipart\/.+?(?:; boundary=(?:(?:"(.+)")|(?:([^\s]+))))$/i, + boundary = boundaryRegex.exec(responseObj.headers['content-type'])[2], + responseBody = { + 'parts': [], + 'errors': 0 }; - // Check if this part needs it's own auth - if ((!!requestPart.auth) && (!!requestPart.auth.bearer)) { - requestSettings.body += 'Authorization: Bearer ' + requestPart.auth.bearer + '\n'; - } - - // Replace body with a request if this isn't a GET - if (requestPart.method !== 'GET') { - requestSettings.body = requestPart.method + ' ' + requestPart.path + '\n' + - 'Content-Type: ' + requestPart.parameters['Content-Type'] + '\n\n' + - JSON.stringify(requestPart.parameters.body); - } - - // Push into array for the request - _multiparts.push(requestSettings); + // Cannot find our boundary + if (!boundary) { + throw new Error('Problem getting boundary data'); + } + // Now we read our body and output + var d = new Dicer({ + boundary: boundary }); - // Collate objects - var requestObj = this._globalOptions; - requestObj.multipart = _multiparts; + // Roll through our parts and build the bigger picture + d.on('part', function(p) { - // Run Requests - request(requestObj).on('response', function (responseObj) { - - // Get the boundary - var boundaryRegex = /^multipart\/.+?(?:; boundary=(?:(?:"(.+)")|(?:([^\s]+))))$/i, - boundary = boundaryRegex.exec(responseObj.headers['content-type'])[2], - responseBody = { 'parts': [], 'errors': 0 }; - - // Cannot find our boundary - if (!boundary) { - throw new Error('Problem getting boundary data'); - } + var part = { + data: [], + bodylen: 0, + header: undefined + }; - // Now we read our body and output - var d = new Dicer({ boundary: boundary }); + p.on('header', function(h) { - // Roll through our parts and build the bigger picture - d.on('part', function (p) { + // Save header data + part.header = h; - var part = { - data: [], - bodylen: 0, - header: undefined - }; + }).on('data', function(data) { - p.on('header', function (h) { + // Build this part's body + part.data.push(data); + part.bodylen += data.length; - // Save header data - part.header = h; + }).on('end', function() { - }).on('data', function (data) { + // Bring body parts together + if (part.data) { + part.data = Buffer.concat(part.data, part.bodylen); + } - // Build this part's body - part.data.push(data); - part.bodylen += data.length; + // Parse the raw http response + part.data = parser.parseResponse(part.data.toString()); - }).on('end', function () { + // Parse response JSON + if (part.data.body) { + part.data.body = JSON.parse(part.data.body); + } - // Bring body parts together - if (part.data) { - part.data = Buffer.concat(part.data, part.bodylen); - } + // Get the response id if exists + var returnedContentId = part.header['content-id'], + currentRequestCallback, + currentRequestExtendData; - // Parse the raw http response - part.data = parser.parseResponse(part.data.toString()); + if (!!returnedContentId) { - // Parse response JSON - if (part.data.body) { - part.data.body = JSON.parse(part.data.body); + // In some cases (Google API) this is a single item array, flatten it and remove "response" identifier. + if (!!Array.isArray(returnedContentId) && returnedContentId.length < 2) { + returnedContentId = returnedContentId.join('').replace(/response-/, ''); } - // Get the response id if exists - var returnedContentId = part.header['content-id'], - currentRequestCallback, - currentRequestExtendData; - - if (!!returnedContentId) { - - // In some cases (Google API) this is a single item array, flatten it and remove "response" identifier. - if (!!Array.isArray(returnedContentId) && returnedContentId.length < 2) { - returnedContentId = returnedContentId.join('').replace(/response-/, ''); - } + part.data.headers['Content-ID'] = returnedContentId; - part.data.headers['Content-ID'] = returnedContentId; + // As we have a content id, we can link to individual callback (if exists) + currentRequestCallback = _self._requestSpecificCallbacks[returnedContentId]; - // As we have a content id, we can link to individual callback (if exists) - currentRequestCallback = _self._requestSpecificCallbacks[returnedContentId]; + // If this request has extend data, retrieve it for callback + currentRequestExtendData = _self._requestExtensionData[returnedContentId]; + } - // If this request has extend data, retrieve it for callback - currentRequestExtendData = _self._requestExtensionData[returnedContentId]; - } - - // If this individual request has a callback, run it - if (!!currentRequestCallback) { - - currentRequestCallback(part.data, currentRequestExtendData || null); + // If this individual request has a callback, run it + if (!!currentRequestCallback) { - } else { - // Anything that doesn't have a specific callback get's pushed into main body - responseBody.parts.push(part.data); - } - - }); - }).on('finish', function () { + currentRequestCallback(part.data, currentRequestExtendData || null); - // All remaining responses get output - // Extend data sent through as is for parse in callback - callback(responseBody, _self._requestExtensionData); + } else { + // Anything that doesn't have a specific callback get's pushed into main body + responseBody.parts.push(part.data); + } }); + }).on('finish', function() { + + // All remaining responses get output + // Extend data sent through as is for parse in callback + callback(responseBody, _self._requestExtensionData); - // Pipe our response to Dicer - responseObj.pipe(d); }); - }; - /** - * Reset all internal options if a re-run is required - * - * @return {object} Batchelor object - */ - Batchelor.reset = function () { + // Pipe our response to Dicer + responseObj.pipe(d); + }); +}; - this._requests = []; - this._requestSpecificCallbacks = []; - this._requestExtensionData = []; - this._globalOptions.multipart = []; +/** + * Reset all internal options if a re-run is required + * + * @return {object} Batchelor object + */ +Batchelor.prototype.reset = function() { - return this; + this._requests = []; + this._requestSpecificCallbacks = []; + this._requestExtensionData = []; + this._globalOptions.multipart = []; - }; + return this; +}; -}(exports)); +module.exports = Batchelor; From c6a944e52edd499658c61cb6e90615ebfd2ae23e Mon Sep 17 00:00:00 2001 From: Brad Vogel Date: Wed, 18 Mar 2015 10:25:18 -0700 Subject: [PATCH 2/3] Update readme with the new v1.0 API. Refs https://github.com/wapisasa/batchelor/issues/4 --- README.md | 54 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 0792869..e4c3cd7 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,26 @@ Feel free to get involved in development. ## Issues, Features and Bugs This has been created for an internal project of [wapisasa](https://github.com/wapisasa) so it might not fit everyone's requirements. It might also be buggy as it's an alpha release. Any issues, feature requests or bugs that need attention please [let us know](https://github.com/wapisasa/batchelor/issues). +## Upgrading to 1.0 + +The API was changed in 1.0 to move from a singleton instance to a constructor. So before where you used `Batchelor` directly: + +``` node +var Batchelor = require('batchelor') +Batchelor.init(...) +Batchelor.add(...) +Batchelor.run(...) +``` + +you now need to create an instance: + +``` node +var Batchelor = require('batchelor') +var batch = new Batchelor(...) +batch.add(...) +batch.run(...) +``` + ## Installation This library has also been distributed on `npm`. Install it with the following command: @@ -21,6 +41,8 @@ This library has also been distributed on `npm`. Install it with the following c $ npm install batchelor --save ``` +See for why this change was made. + ## How to Use #### GET Requests ``` node @@ -28,7 +50,7 @@ var Batchelor = require('batchelor'); ``` Once the module has been included, we initialise it with all our default options: ``` node -Batchelor.init({ +var batch = new Batchelor({ 'uri':'https://www.googleapis.com/batch', 'method':'POST', 'auth': { @@ -43,14 +65,14 @@ We can then start adding requests to our batch. This can be done 2 ways: As a one-off object: ``` node -Batchelor.add({ +batch.add({ 'method':'GET', 'path':'/plusDomains/v1/people/me/activities/user' }) ``` Or an Array of objects: ``` node -Batchelor.add([ +batch.add([ { 'method':'GET', 'path':'/plusDomains/v1/people/me/activities/user' @@ -67,14 +89,14 @@ Batchelor.add([ ``` Once you have added all of the requests you need, call `.run()`: ``` node -Batchelor.run(function(response){ +batch.run(function(response){ res.json(response); }); ``` #### POST Requests The above examples show `GET` requests. To perform a `POST` requires a few more settings: ``` node -Batchelor.add({ +batch.add({ 'method':'POST', 'path':'/plusDomains/v1/people/me/activities', 'parameters':{ @@ -84,9 +106,9 @@ Batchelor.add({ }); ``` #### Callbacks -By default, all responses are returned through the callback function in the `Batchelor.run()` call. Alternatively, a callback can be supplied for each individual calls: +By default, all responses are returned through the callback function in the `batch.run()` call. Alternatively, a callback can be supplied for each individual calls: ``` node -Batchelor.add({ +batch.add({ 'method':'POST', 'path':'/plusDomains/v1/people/me/activities', 'parameters':{ @@ -99,9 +121,9 @@ Batchelor.add({ }); ``` #### Request and Response IDs -The module will assign a request a randomly generated unique `Content-ID` by default, but this can be supplied as part of the options to supply `Batchelor.add()`: +The module will assign a request a randomly generated unique `Content-ID` by default, but this can be supplied as part of the options to supply `batch.add()`: ``` node -Batchelor.add({ +batch.add({ 'method':'GET', 'path':'/plusDomains/v1/people/me/activities/user', 'requestId':'Batch_UniqueID_1' @@ -109,12 +131,10 @@ Batchelor.add({ ``` #### A Couple of Little Gifts ###### Method Chaining -All methods return the `Batchelor` object. So you can chain calls together. +All methods return the `Batchelor` instance. So you can chain calls together. ``` node -Batchelor.init({ - ... -}).add([ +batch.add([ ... ]).run(function(data){ ... @@ -123,7 +143,7 @@ Batchelor.init({ ###### Data Pass-through When passing options to the `.add()` you can include an Object called `extend`. In the case of providing a callback, this will be passed back as a second parameter. When using the default callback on the `.run()` call, an array of all data passed through will be added as a second parameter with the requestId as the key: ``` node -Batchelor.add({ +batch.add({ ... 'extend':{ ... @@ -136,12 +156,12 @@ Batchelor.add({ This could be required, for example, when making multiple requests with different Auth data and then needing to make further requests with the same Auth data. ###### Resetting and Re-using -Once Batchelor has been run, there are certain use-cases for running futher batch requests on response. This requires the variables in the module to be reset. This can be done using the `.reset()` call: +Once Batchelor has been run, there are certain use-cases for running futher batch requests on response. This requires the variables in the instance to be reset. This can be done using the `.reset()` call: ``` node -Batchelor.run(function(response){ +batch.run(function(response){ // Reset Batchelor for further use - Batchelor.reset(); + batch.reset(); ... }); From 79438da02173670d64377eb80fd779c9b55ba9e9 Mon Sep 17 00:00:00 2001 From: James Haley Date: Thu, 19 Mar 2015 14:07:58 +0000 Subject: [PATCH 3/3] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e4c3cd7..e58aa75 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Batchelor.add(...) Batchelor.run(...) ``` -you now need to create an instance: +You now need to create an instance of Batchelor: ``` node var Batchelor = require('batchelor')