Merge pull request #1068 from stephenplusplus/vlad--patch-1

bigquery: remove automatic `insertId` generation & allow specifying raw format

Author: stephenplusplus

lib/bigquery/table.js

@@ -21,7 +21,6 @@
 'use strict';
 
 var arrify = require('arrify');
-var crypto = require('crypto');
 var duplexify = require('duplexify');
 var extend = require('extend');
 var format = require('string-format-obj');
@@ -849,6 +848,21 @@ Table.prototype.import = function(source, metadata, callback) {
  * @resource [Tabledata: insertAll API Documentation]{@link https://cloud.google.com/bigquery/docs/reference/v2/tabledata/insertAll}
  *
  * @param {object|object[]} rows - The rows to insert into the table.
+ * @param {object=} options - Configuration object.
+ * @param {boolean} options.ignoreUnknownValues - Accept rows that contain
+ *     values that do not match the schema. The unknown values are ignored.
+ *     Default: `false`.
+ * @param {boolean} options.raw - If `true`, the `rows` argument is expected to
+ *     be formatted as according to the
+ *     [specification](https://cloud.google.com/bigquery/docs/reference/v2/tabledata/insertAll).
+ * @param {boolean} options.skipInvalidRows - Insert all valid rows of a
+ *     request, even if invalid rows exist. Default: `false`.
+ * @param {string} options.templateSuffix - Treat the destination table as a
+ *     base template, and insert the rows into an instance table named
+ *     "{destination}{templateSuffix}". BigQuery will manage creation of
+ *     the instance table, using the schema of the base template table. See
+ *     [Automatic table creation using template tables](https://cloud.google.com/bigquery/streaming-data-into-bigquery#template-tables)
+ *     for considerations when working with templates tables.
  * @param {function} callback - The callback function.
  * @param {?error} callback.err - An error returned while making this request.
  * @param {array} callback.insertErrors - A list of errors for insert failures.
@@ -879,6 +893,25 @@ Table.prototype.import = function(source, metadata, callback) {
  * table.insert(rows, insertHandler);
  *
  * //-
+ * // Insert a row as according to the <a href="https://cloud.google.com/bigquery/docs/reference/v2/tabledata/insertAll">
+ * // specification</a>.
+ * //-
+ * var row = {
+ *   insertId: '1',
+ *   json: {
+ *     INSTNM: 'Motion Picture Institute of Michigan',
+ *     CITY: 'Troy',
+ *     STABBR: 'MI'
+ *   }
+ * };
+ *
+ * var options = {
+ *   raw: true
+ * };
+ *
+ * table.insert(row, options, insertHandler);
+ *
+ * //-
  * // Handling the response.
  * //-
  * function insertHandler(err, insertErrors, apiResponse) {
@@ -897,23 +930,30 @@ Table.prototype.import = function(source, metadata, callback) {
  *   // recommendations on handling errors.
  * }
  */
-Table.prototype.insert = function(rows, callback) {
-  var body = {
-    rows: arrify(rows).map(function(row) {
-      var rowObject = {};
-      // Use the stringified contents of the row as a unique insert ID.
-      var md5 = crypto.createHash('md5');
-      md5.update(JSON.stringify(row));
-      rowObject.insertId = md5.digest('hex');
-      rowObject.json = row;
-      return rowObject;
-    })
-  };
+Table.prototype.insert = function(rows, options, callback) {
+  if (is.fn(options)) {
+    callback = options;
+    options = {};
+  }
+
+  var json = extend(true, options, {
+    rows: arrify(rows)
+  });
+
+  if (!options.raw) {
+    json.rows = arrify(rows).map(function(row) {
+      return {
+        json: row
+      };
+    });
+  }
+
+  delete options.raw;
 
   this.request({
     method: 'POST',
     uri: '/insertAll',
-    json: body
+    json: json
   }, function(err, resp) {
     if (err) {
       callback(err, null, resp);
@@ -928,7 +968,7 @@ Table.prototype.insert = function(rows, callback) {
             reason: error.reason
           };
         }),
-        row: body.rows[insertError.index].json
+        row: json.rows[insertError.index].json
       };
     });
 

test/bigquery/table.js

@@ -18,7 +18,6 @@
 
 var arrify = require('arrify');
 var assert = require('assert');
-var crypto = require('crypto');
 var extend = require('extend');
 var mockery = require('mockery');
 var nodeutil = require('util');
@@ -918,14 +917,19 @@ describe('BigQuery/Table', function() {
       { state: 'MI', gender: 'M', year: '2015', name: 'Berkley', count: '0' }
     ];
 
+    var rawData = [
+      { insertId: 1, json: data[0] },
+      { insertId: 2, json: data[1] },
+      { insertId: 3, json: data[2] },
+      { insertId: 4, json: data[3] },
+      { insertId: 5, json: data[4] },
+    ];
+
     var dataApiFormat = {
       rows: data.map(function(row) {
-        var rowObject = {};
-        var md5 = crypto.createHash('md5');
-        md5.update(JSON.stringify(row));
-        rowObject.insertId = md5.digest('hex');
-        rowObject.json = row;
-        return rowObject;
+        return {
+          json: row
+        };
       })
     };
 
@@ -995,6 +999,50 @@ describe('BigQuery/Table', function() {
         done();
       });
     });
+
+    it('should insert raw data', function(done) {
+      table.request = function(reqOpts) {
+        assert.equal(reqOpts.method, 'POST');
+        assert.equal(reqOpts.uri, '/insertAll');
+        assert.deepEqual(reqOpts.json, { rows: rawData });
+        assert.strictEqual(reqOpts.json.raw, undefined);
+        done();
+      };
+
+      var opts = { raw: true };
+      table.insert(rawData, opts, done);
+    });
+
+    it('should accept options', function(done) {
+      var opts = {
+        ignoreUnknownValues: true,
+        skipInvalidRows: true,
+        templateSuffix: 'test'
+      };
+
+      table.request = function(reqOpts) {
+        assert.equal(reqOpts.method, 'POST');
+        assert.equal(reqOpts.uri, '/insertAll');
+
+        assert.strictEqual(
+          reqOpts.json.ignoreUnknownValues,
+          opts.ignoreUnknownValues
+        );
+        assert.strictEqual(
+          reqOpts.json.skipInvalidRows,
+          opts.skipInvalidRows
+        );
+        assert.strictEqual(
+          reqOpts.json.templateSuffix,
+          opts.templateSuffix
+        );
+
+        assert.deepEqual(reqOpts.json.rows, dataApiFormat.rows);
+        done();
+      };
+
+      table.insert(data, opts, done);
+    });
   });
 
   describe('query', function() {