Merge pull request #72 from FunnelEnvy/master

Pagination for Asset Type URL

.gitignore

@@ -1,5 +1,6 @@
 /node_modules/
 /.idea/
+/.history
 *.iml
 test/helper/config.bk.js
 package-lock.json

lib/connection.js

@@ -3,7 +3,9 @@ var _ = require('lodash'),
     Promise = require('bluebird'),
     Retry = require('./retry'),
     errors = require('./errors'),
+    getNextPageFn = require('./nextPageFn'),
     util = require('util'),
+    helper = require('./util'),
     EventEmitter = require('events').EventEmitter,
     log = require('./util').logger(),
     restlerMethodArgCount = {
@@ -43,24 +45,6 @@ function Connection(options) {
 
 util.inherits(Connection, EventEmitter);
 
-function getNextPageFn(conn, method, args) {
-  var options = _.clone(_.last(args) || {});
-  args = _.clone(args);
-  args.pop();
-
-  return function nextPage(nextPageToken) {
-    var params = options;
-    if (method === 'get') {
-      params = options.query = options.query || {};
-    } else if (method === 'post' || method === 'put') {
-      params = options.data = options.data || {};
-    }
-    params.nextPageToken = nextPageToken;
-
-    return conn._request.apply(conn, _.flatten([method, args, options], true));
-  }
-}
-
 /**
  * This function is just a helper function that delegates everything to
  * restler, but returns a Promise instead of going with the existing event
@@ -98,12 +82,22 @@ Connection.prototype._request = function(method) {
               log.debug('Request failed: ', data);
               defer.reject(new errors.HttpError(resp.statusCode, data));
             } else {
-              if (_.has(data, 'nextPageToken')) {
-                Object.defineProperty(data, 'nextPage', {
-                  enumerable: false,
-                  value: _.partial(nextPageFn, data.nextPageToken)
-                });
+              if(helper.nextPageType(args[0]) === 'offset') {
+                if(!_.isEmpty(data.result)) {
+                  Object.defineProperty(data, 'nextPage', {
+                    enumerable: false,
+                    value: _.partial(nextPageFn)
+                  });
+                }
+              } else {
+                if (_.has(data, 'nextPageToken')) {
+                  Object.defineProperty(data, 'nextPage', {
+                    enumerable: false,
+                    value: _.partial(nextPageFn, data.nextPageToken)
+                  });
+                }
               }
+
               defer.resolve(data);
             }
           })

lib/nextPageFn/index.js

@@ -0,0 +1,20 @@
+var offset = require('./offset');
+var token = require('./token');
+var util = require('../util');
+var _ = require('lodash');
+
+module.exports = getNextPageFn;
+
+function getNextPageFn(conn, method, args) {
+    var assetStrategy = {
+        offset,
+        token
+    };
+
+    var options = _.clone(_.last(args) || {});
+    args = _.clone(args);
+    args.pop();
+
+    var selectedStrategy = assetStrategy[util.nextPageType(args[0])]
+    return selectedStrategy(conn, method, args, options);
+}

lib/nextPageFn/offset.js

@@ -0,0 +1,16 @@
+var _ = require('lodash');
+
+module.exports =  (conn, method, args, options) => {
+    return () => {
+        var params = options;
+        if (method === 'get') {
+            params = options.query = options.query || {};
+        } else if (method === 'post' || method === 'put') {
+            params = options.data = options.data || {};
+        }
+
+        params.offset = (params.offset || 0) + (params.maxReturn || 20);
+
+        return conn._request.apply(conn, _.flatten([method, args, options], true));
+    }
+};

lib/nextPageFn/token.js

@@ -0,0 +1,16 @@
+var _ = require('lodash');
+
+module.exports =  (conn, method, args, options) => {
+    return (nextPageToken) => {
+        var params = options;
+        if (method === 'get') {
+            params = options.query = options.query || {};
+        } else if (method === 'post' || method === 'put') {
+            params = options.data = options.data || {};
+        }
+
+        params.nextPageToken = nextPageToken;
+
+        return conn._request.apply(conn, _.flatten([method, args, options], true));
+    }
+};

lib/stream.js

@@ -67,7 +67,7 @@ MarketoStream.prototype._read = function () {
     this.fetch()
   } else if (this._pushNext()) {
     return;
-  } else if (this._data.moreResult || this._data.nextPageToken) {
+  } else if (this._data.moreResult || this._data.nextPage) {
     this.fetch(this._data.nextPage())
   } else { // No data left in the batch and no more data from marketo, end the stream
     this.push(null);

lib/util.js

@@ -64,5 +64,13 @@ module.exports = {
     }
 
     return this.arrayToCSV(options, ['fields', 'filterValues']);
-  }
+  },
+
+  nextPageType: function(requestedUrl) {
+    var type = 'token';
+    if(requestedUrl.indexOf('asset') !== -1) {
+        type = 'offset';
+    }
+    return type;
+  },
 };

package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-marketo-rest",
-  "version": "0.7.7",
+  "version": "0.7.8",
   "description": "marketo rest client",
   "repository": {
     "type": "git",
@@ -23,6 +23,8 @@
   },
   "devDependencies": {
     "mocha": "5.0.x",
-    "replay": "2.1.x"
+    "nock": "^11.7.0",
+    "replay": "2.1.x",
+    "rewire": "^4.0.1"
   }
 }

test/connection.test.js

@@ -0,0 +1,101 @@
+const _ = require('lodash'),
+    Promise = require('bluebird'),
+    assert = require('assert'),
+    util = require('../lib/util'),
+    nock = require('nock'),
+    rewire = require('rewire');
+
+const Connection = rewire("../lib/connection.js")
+Connection.__set__("getNextPageFn", getNextPageFn)
+function getNextPageFn(conn, method, args) {
+    var assetStrategy = {
+        offset: () => {
+            return () => 'offset'
+        },
+        token: () => {
+            return () => 'token'
+        },
+    };
+
+    var options = _.clone(_.last(args) || {});
+    args = _.clone(args);
+    args.pop();
+
+    var selectedStrategy = assetStrategy[util.nextPageType(args[0])]
+    return selectedStrategy(conn, method, args, options);
+}
+
+const assetScope = nock('http://localhost-mock')
+    .filteringRequestBody(body => {
+        console.log(body);
+        return true;
+    })
+    .post('/asset')
+    .reply(200, {
+        "success": true,
+        "errors": [],
+        "requestId": "6efc#16c8967a21f",
+        "warnings": [],
+        "result": [
+            {
+                "id": 4363,
+                "name": "Smart List Test 01"
+            }
+        ]
+    });
+
+const tokenScope = nock('http://localhost-mock')
+    .filteringRequestBody(body => {
+        console.log(body);
+        return true;
+    })
+    .post('/rest/v1')
+    .reply(200, {
+        "moreResult": true,
+        "nextPageToken": "string",
+        "requestId": "string",
+        "result": [
+            {
+                "id": 0,
+                "status": "string"
+            }
+        ],
+        "success": true
+    });
+
+const ASSET_URL = 'asset',
+      TOKEN_URL = 'rest/v1',
+      IDENTITY_URL = 'identity';
+
+Connection.prototype.getOAuthToken = function() {
+    return Promise.resolve({access_token: 'test_token'});
+}
+
+function getUrl(path) {
+    return 'http://localhost-mock/' + path;
+}
+
+function getConnection() {
+    var options = {
+        endpoint: getUrl(''),
+        identity: getUrl(IDENTITY_URL),
+        clientId: 'someId',
+        clientSecret: 'someSecret'
+    };
+    return new Connection(options);
+}
+
+describe('Connection', function() {
+    it('token type pagination', function() {
+        return getConnection().post(TOKEN_URL, {data: {_method: 'GET', maxReturn: 200}}).then(resp => {
+            assert.equal(resp.nextPage(), 'token')
+        });
+    });
+
+    it('offset type pagination', function() {
+        return getConnection().post(ASSET_URL, {data: {_method: 'GET', maxReturn: 200}}).then(resp => {
+            assert.equal(resp.nextPage(), 'offset')
+        });
+    });
+});
+