stream-router: default autoPaginate: true

Author: stephenplusplus

lib/bigquery/index.js

@@ -163,31 +163,33 @@ BigQuery.prototype.dataset = function(id) {
  * @param {object=} query - Configuration object.
  * @param {boolean} query.all - List all datasets, including hidden ones.
  * @param {boolean} query.autoPaginate - Have pagination handled automatically.
- *     Default: false.
+ *     Default: true.
  * @param {number} query.maxResults - Maximum number of results to return.
  * @param {string} query.pageToken - Token returned from a previous call, to
  *     request the next page of results.
  * @param {function} callback - The callback function.
  *
  * @example
+ * bigquery.getDatasets(function(err, datasets) {
+ *   if (!err) {
+ *     // datasets is an array of Dataset objects.
+ *   }
+ * });
+ *
+ * //-
+ * // To control how many API requests are made and page through the results
+ * // manually, set `autoPaginate` to `false`.
+ * //-
  * var callback = function(err, datasets, nextQuery, apiResponse) {
  *   if (nextQuery) {
  *     // More results exist.
  *     bigquery.getDatasets(nextQuery, callback);
  *   }
  * };
  *
- * bigquery.getDatasets(callback);
- *
- * //-
- * // To have pagination handled for you, set `autoPaginate`. Note the changed
- * // callback parameters.
- * //-
  * bigquery.getDatasets({
- *   autoPaginate: true
- * }, function(err, datasets) {
- *   // Called after all datasets have been retrieved.
- * });
+ *   autoPaginate: false
+ * }, callback);
  *
  * //-
  * // Get the datasets from your project as a readable object stream.
@@ -251,7 +253,7 @@ BigQuery.prototype.getDatasets = function(query, callback) {
  * @param {boolean=} options.allUsers - Display jobs owned by all users in the
  *     project.
  * @param {boolean} options.autoPaginate - Have pagination handled
- *     automatically. Default: false.
+ *     automatically. Default: true.
  * @param {number=} options.maxResults - Maximum number of results to return.
  * @param {string=} options.pageToken - Token returned from a previous call, to
  *     request the next page of results.
@@ -263,19 +265,27 @@ BigQuery.prototype.getDatasets = function(query, callback) {
  * @param {function} callback - The callback function.
  *
  * @example
- * bigquery.getJobs(function(err, jobs, nextQuery, apiResponse) {
- *   // If `nextQuery` is non-null, there are more results to fetch.
+ * bigquery.getJobs(function(err, jobs) {
+ *   if (!err) {
+ *     // jobs is an array of Job objects.
+ *   }
  * });
  *
  * //-
- * // To have pagination handled for you, set `autoPaginate`. Note the changed
- * // callback parameters.
+ * // To control how many API requests are made and page through the results
+ * // manually, set `autoPaginate` to `false`.
  * //-
+ * var callback = function(err, jobs, nextQuery, apiRespose) {
+ *   if (nextQuery) {
+ *     // More results exist.
+ *     bigquery.getJobs(nextQuery, callback);
+ *   }
+ * };
+ *
  * bigquery.getJobs({
- *   autoPaginate: true
- * }, function(err, jobs) {
- *   // Called after all jobs have been retrieved.
- * });
+ *   autoPaginate: false
+ * }, callback);
+ *
  * //-
  * // Get the jobs from your project as a readable object stream.
  * //-
@@ -360,7 +370,7 @@ BigQuery.prototype.job = function(id) {
  *
  * @param {string|object} options - A string SQL query or configuration object.
  * @param {boolean} options.autoPaginate - Have pagination handled
- *     automatically. Default: false.
+ *     automatically. Default: true.
  * @param {number} options.maxResults - Maximum number of results to read.
  * @param {string} options.query - A query string, following the BigQuery query
  *     syntax, of the query to execute.
@@ -373,29 +383,26 @@ BigQuery.prototype.job = function(id) {
  * @example
  * var query = 'SELECT url FROM [publicdata:samples.github_nested] LIMIT 100';
  *
+ * bigquery.query(query, function(err, rows) {
+ *   if (!err) {
+ *     // Handle results here.
+ *   }
+ * });
+ *
  * //-
- * // You can run a query against your data in a serial manner.
+ * // To control how many API requests are made and page through the results
+ * // manually, set `autoPaginate` to `false`.
  * //-
  * var callback = function(err, rows, nextQuery, apiResponse) {
- *   // Handle results here.
- *
  *   if (nextQuery) {
  *     bigquery.query(nextQuery, callback);
  *   }
  * };
  *
- * bigquery.query(query, callback);
- *
- * //-
- * // To have pagination handled for you, set `autoPaginate`. Note the changed
- * // callback parameters.
- * //-
  * bigquery.query({
  *   query: query,
- *   autoPaginate: true
- * }, function(err, rows) {
- *   // Called after all rows have been retrieved.
- * });
+ *   autoPaginate: false
+ * }, callback);
  *
  * //-
  * // You can also use the `query` method as a readable object stream by

lib/bigquery/job.js

@@ -92,7 +92,7 @@ Job.prototype.getMetadata = function(callback) {
  *
  * @param {object=} options - Configuration object.
  * @param {boolean} options.autoPaginate - Have pagination handled
- *     automatically. Default: false.
+ *     automatically. Default: true.
  * @param {number} options.maxResults - Maximum number of results to read.
  * @param {string} options.pageToken - Page token, returned by a previous call,
  *     to request the next page of results. Note: This is automatically added to

lib/bigquery/table.js

@@ -484,33 +484,31 @@ Table.prototype.getMetadata = function(callback) {
  *
  * @param {object=} options - The configuration object.
  * @param {boolean} options.autoPaginate - Have pagination handled
- *     automatically. Default: false.
+ *     automatically. Default: true.
  * @param {number} options.maxResults - Maximum number of results to return.
  * @param {function} callback - The callback function.
  *
  * @example
- * var options = {
- *   maxResults: 100
- * };
+ * table.getRows(function(err, rows) {
+ *   if (!err) {
+ *     // Handle results here.
+ *   }
+ * });
  *
+ * //-
+ * // To control how many API requests are made and page through the results
+ * // manually, set `autoPaginate` to `false`.
+ * //-
  * var callback = function(err, rows, nextQuery, apiResponse) {
  *   if (nextQuery) {
  *     // More results exist.
  *     table.getRows(nextQuery, callback);
  *   }
  * };
  *
- * table.getRows(options, callback);
- *
- * //-
- * // To have pagination handled for you, set `autoPaginate`. Note the changed
- * // callback parameters.
- * //-
  * table.getRows({
- *   autoPaginate: true
- * }, function(err, rows) {
- *   // Called after all rows have been retrieved.
- * });
+ *   autoPaginate: false
+ * }, callback);
  *
  * //-
  * // Get the rows as a readable object stream.

lib/common/stream-router.js

@@ -74,22 +74,51 @@ streamRouter.extend = function(Class, methodNames) {
  *     method received.
  */
 streamRouter.parseArguments_ = function(args) {
-  var parsedArguments = {};
+  var query;
+  var callback;
+  var maxResults = -1;
+  var autoPaginate = true;
 
   var firstArgument = args[0];
   var lastArgument = args[args.length - 1];
 
   if (util.is(firstArgument, 'function')) {
-    parsedArguments.callback = firstArgument;
+    callback = firstArgument;
   } else {
-    parsedArguments.query = firstArgument;
+    query = firstArgument;
   }
 
   if (util.is(lastArgument, 'function')) {
-    parsedArguments.callback = lastArgument;
+    callback = lastArgument;
   }
 
-  return parsedArguments;
+  if (util.is(query, 'object')) {
+    // Check if the user only asked for a certain amount of results.
+    if (util.is(query.maxResults, 'number')) {
+      // `maxResults` is used API-wide.
+      maxResults = query.maxResults;
+    } else if (util.is(query.limitVal, 'number')) {
+      // `limitVal` is part of a Datastore query.
+      maxResults = query.limitVal;
+    } else if (util.is(query.pageSize, 'number')) {
+      // `pageSize` is Pub/Sub's `maxResults`.
+      maxResults = query.pageSize;
+    }
+
+    if (callback &&
+        (maxResults !== -1 || // The user specified a limit.
+        query.autoPaginate === false ||
+        query.autoPaginateVal === false)) {
+      autoPaginate = false;
+    }
+  }
+
+  return {
+    query: query || {},
+    callback: callback,
+    maxResults: maxResults,
+    autoPaginate: autoPaginate
+  };
 };
 
 /**
@@ -103,20 +132,20 @@ streamRouter.parseArguments_ = function(args) {
  *     commonly an object, but to make the API more simple, it can also be a
  *     string in some places.
  * @param {function=} parsedArguments.callback - Callback function.
+ * @param {boolean} parsedArguments.autoPaginate - Auto-pagination enabled.
+ * @param {number} parsedArguments.maxResults - Maximum results to return.
  * @param {function} originalMethod - The cached method that accepts a callback
  *     and returns `nextQuery` to receive more results.
  * @return {undefined|stream}
  */
 streamRouter.router_ = function(parsedArguments, originalMethod) {
-  var query = parsedArguments.query || {};
+  var query = parsedArguments.query;
   var callback = parsedArguments.callback;
+  var autoPaginate = parsedArguments.autoPaginate;
 
   if (callback) {
-    if (query.autoPaginate === true || query.autoPaginateVal === true) {
-      delete query.autoPaginate;
-      delete query.autoPaginateVal;
-
-      this.runAsStream_(query, originalMethod)
+    if (autoPaginate) {
+      this.runAsStream_(parsedArguments, originalMethod)
         .on('error', callback)
         .pipe(concat(function(results) {
           callback(null, results);
@@ -125,7 +154,7 @@ streamRouter.router_ = function(parsedArguments, originalMethod) {
       originalMethod(query, callback);
     }
   } else {
-    return this.runAsStream_(query, originalMethod);
+    return this.runAsStream_(parsedArguments, originalMethod);
   }
 };
 
@@ -136,26 +165,19 @@ streamRouter.router_ = function(parsedArguments, originalMethod) {
  * `maxResults` and `limitVal` (from Datastore) will act as a cap for how many
  * results are fetched and emitted to the stream.
  *
- * @param {object=|string=} query - Query object. This is most
+ * @param {object=|string=} parsedArguments.query - Query object. This is most
  *     commonly an object, but to make the API more simple, it can also be a
  *     string in some places.
+ * @param {function=} parsedArguments.callback - Callback function.
+ * @param {boolean} parsedArguments.autoPaginate - Auto-pagination enabled.
+ * @param {number} parsedArguments.maxResults - Maximum results to return.
  * @param {function} originalMethod - The cached method that accepts a callback
  *     and returns `nextQuery` to receive more results.
  * @return {stream} - Readable object stream.
  */
-streamRouter.runAsStream_ = function(query, originalMethod) {
-  query = query || {};
-
-  var resultsToSend = -1;
-
-  // Check if the user only asked for a certain amount of results.
-  if (util.is(query.maxResults, 'number')) {
-    // `maxResults` is used API-wide.
-    resultsToSend = query.maxResults;
-  } else if (util.is(query.limitVal, 'number')) {
-    // `limitVal` is part of a Datastore query.
-    resultsToSend = query.limitVal;
-  }
+streamRouter.runAsStream_ = function(parsedArguments, originalMethod) {
+  var query = parsedArguments.query;
+  var resultsToSend = parsedArguments.maxResults;
 
   var stream = streamEvents(through.obj());
 
@@ -201,7 +223,7 @@ streamRouter.runAsStream_ = function(query, originalMethod) {
   }
 
   stream.once('reading', function() {
-    originalMethod.call(null, query, onResultSet);
+    originalMethod(query, onResultSet);
   });
 
   return stream;

lib/common/util.js

@@ -674,13 +674,12 @@ function makeRequest(reqOpts, config, callback) {
 
   config = config || {};
 
+  reqOpts = util.decorateRequest(reqOpts);
+
   var MAX_RETRIES = config.maxRetries || 3;
   var autoRetry = config.autoRetry !== false ? true : false;
   var attemptedRetries = 0;
 
-  reqOpts.headers = reqOpts.headers || {};
-  reqOpts.headers['User-Agent'] = USER_AGENT;
-
   function shouldRetry(err) {
     return autoRetry &&
       MAX_RETRIES > attemptedRetries &&
@@ -704,3 +703,28 @@ function makeRequest(reqOpts, config, callback) {
 }
 
 util.makeRequest = makeRequest;
+
+/**
+ * Decorate the options about to be made in a request.
+ *
+ * @param {object} reqOpts - The options to be passed to `request`.
+ * @return {object} reqOpts - The decorated reqOpts.
+ */
+function decorateRequest(reqOpts) {
+  reqOpts.headers = reqOpts.headers || {};
+  reqOpts.headers['User-Agent'] = USER_AGENT;
+
+  if (util.is(reqOpts.qs, 'object')) {
+    delete reqOpts.qs.autoPaginate;
+    delete reqOpts.qs.autoPaginateVal;
+  }
+
+  if (util.is(reqOpts.json, 'object')) {
+    delete reqOpts.json.autoPaginate;
+    delete reqOpts.json.autoPaginateVal;
+  }
+
+  return reqOpts;
+}
+
+util.decorateRequest = decorateRequest;

lib/datastore/query.js

@@ -73,7 +73,7 @@ function Query(namespace, kinds) {
   this.selectVal = [];
 
   // pagination
-  this.autoPaginateVal = false;
+  this.autoPaginateVal = true;
   this.startVal = null;
   this.endVal = null;
   this.limitVal = -1;