Add db paginate

.eslintrc

@@ -51,7 +51,6 @@
     "no-trailing-spaces": 2,
     "no-underscore-dangle": 0,
     "no-unneeded-ternary": 1,
-    "one-var": 2,
     "quotes": [2, "single", "avoid-escape"],
     "semi": [2, "always"],
     "keyword-spacing": 2,

lib/responses.js

@@ -408,6 +408,41 @@ function expectJSON(fn, res) {
   }).catch(handleError(res));
 }
 
+/**
+ * Checks the page size requested to make sure it's a Number and within the maximum.
+ * Returns the default or 0 if there's no default or size is not a Number.
+ * @param {Object} req
+ * @returns {number}
+ */
+function checkPageSize(req) {
+  const size = req.query.size || process.env.CLAY_STORAGE_PAGE_SIZE || 0;
+  const num = Math.abs(Number(size));
+  const max = process.env.CLAY_STORAGE_MAX_PAGE || null;
+
+  if (isNaN(num)) return 0;
+
+  if (max) {
+    return num > max ? max : num;
+  } else return num;
+}
+
+/**
+ * Checks whether the previous value for a page request matches up with the uri,
+ * ie. all the ids returned should contain the request uri.
+ * @param {Object} req
+ * @returns {string|undefined}
+ */
+function checkPrevious(req) {
+  const prefix = req.uri || null;
+  const previous = req.query.prev || null;
+
+  if (previous && previous.includes(prefix)) {
+    return previous;
+  } else {
+    return;
+  }
+}
+
 /**
  * List all things in the db
  * @param {object} [options]
@@ -421,7 +456,9 @@ function list(options) {
     let list,
       listOptions = _.assign({
         prefix: req.uri,
-        values: false
+        values: false,
+        previous: checkPrevious(req),
+        size: checkPageSize(req)
       }, options);
 
     list = db.list(listOptions);
@@ -441,7 +478,7 @@ function listWithoutVersions() {
       return [filter({wantStrings: true}, function (str) {
         return str.indexOf('@') === -1;
       })];
-    }
+    },
   };
 
   return list(options);

lib/services/db.js

@@ -80,7 +80,7 @@ function list(options = {}) {
     transformOptions.isArray = true;
   }
 
-  readStream = module.exports.createReadStream(options);
+  readStream = module.exports.paginate(options);
 
   if (_.isFunction(options.transforms)) {
     options.transforms = options.transforms();
@@ -136,6 +136,7 @@ module.exports.patchMeta;
 module.exports.getMeta;
 module.exports.raw;
 module.exports.createReadStream;
+module.exports.paginate;
 
 // For testing
 module.exports.defer = defer;

test/fixtures/mocks/storage.js

@@ -20,6 +20,7 @@ class Storage {
     this.clearMem         = this.clear;
     this.pipeToPromise    = db.pipeToPromise;
     this.createReadStream = (ops) => this.inMem.createReadStream(ops);
+    this.paginate = (ops) => this.inMem.createReadStream(ops);
     this.getLatestData    = sinon.stub();
 
     db.registerStorage(this);