bigquery: convert nested objects to their native types

packages/bigquery/src/table.js

@@ -177,6 +177,39 @@ Table.createSchemaFromString_ = function(str) {
   });
 };
 
+/**
+ * Convert a row entry from native types to their encoded types that the API
+ * expects.
+ *
+ * @static
+ * @private
+ *
+ * @param {*} value - The value to be converted.
+ * @return {*} The converted value.
+ */
+Table.encodeValue_ = function(value) {
+  if (value instanceof Buffer) {
+    return value.toString('base64');
+  }
+
+  if (is.date(value)) {
+    return value.toJSON();
+  }
+
+  if (is.array(value)) {
+    return value.map(Table.encodeValue_);
+  }
+
+  if (is.object(value)) {
+    return Object.keys(value).reduce(function(acc, key) {
+      acc[key] = Table.encodeValue_(value[key]);
+      return acc;
+    }, {});
+  }
+
+  return value;
+};
+
 /**
  * Merge a rowset returned from the API with a table schema.
  *
@@ -188,34 +221,19 @@ Table.createSchemaFromString_ = function(str) {
  * @return {array} Fields using their matching names from the table's schema.
  */
 Table.mergeSchemaWithRows_ = function(schema, rows) {
-  return rows.map(mergeSchema).map(flattenRows);
+  return arrify(rows).map(mergeSchema).map(flattenRows);
 
   function mergeSchema(row) {
     return row.f.map(function(field, index) {
       var schemaField = schema.fields[index];
       var value = field.v;
 
-      switch (schemaField.type) {
-        case 'BOOLEAN': {
-          value = value === 'true';
-          break;
-        }
-        case 'FLOAT': {
-          if (!is.nil(value)) {
-            value = parseFloat(value);
-          }
-          break;
-        }
-        case 'INTEGER': {
-          if (!is.nil(value)) {
-            value = parseInt(value, 10);
-          }
-          break;
-        }
-        case 'TIMESTAMP': {
-          value = new Date(value * 1000);
-          break;
-        }
+      if (schemaField.mode === 'REPEATED') {
+        value = value.map(function(val) {
+          return convert(schemaField, val.v);
+        });
+      } else {
+        value = convert(schemaField, value);
       }
 
       var fieldObject = {};
@@ -224,6 +242,41 @@ Table.mergeSchemaWithRows_ = function(schema, rows) {
     });
   }
 
+  function convert(schemaField, value) {
+    if (is.nil(value)) {
+      return value;
+    }
+
+    switch (schemaField.type) {
+      case 'BOOLEAN': {
+        value = value === 'true';
+        break;
+      }
+      case 'BYTES': {
+        value = new Buffer(value, 'base64');
+        break;
+      }
+      case 'FLOAT': {
+        value = parseFloat(value);
+        break;
+      }
+      case 'INTEGER': {
+        value = parseInt(value, 10);
+        break;
+      }
+      case 'RECORD': {
+        value = Table.mergeSchemaWithRows_(schemaField, value).pop();
+        break;
+      }
+      case 'TIMESTAMP': {
+        value = new Date(value * 1000);
+        break;
+      }
+    }
+
+    return value;
+  }
+
   function flattenRows(rows) {
     return rows.reduce(function(acc, row) {
       var key = Object.keys(row)[0];
@@ -925,7 +978,7 @@ Table.prototype.insert = function(rows, options, callback) {
   if (!options.raw) {
     json.rows = arrify(rows).map(function(row) {
       return {
-        json: row
+        json: Table.encodeValue_(row)
       };
     });
   }
@@ -977,11 +1030,12 @@ Table.prototype.query = function(query, callback) {
  *     table.
  * @param {string} metadata.name - A descriptive name for the table.
  * @param {string|object} metadata.schema - A comma-separated list of name:type
- *     pairs. Valid types are "string", "integer", "float", "boolean", and
- *     "timestamp". If the type is omitted, it is assumed to be "string".
- *     Example: "name:string, age:integer". Schemas can also be specified as a
- *     JSON array of fields, which allows for nested and repeated fields. See
- *     a [Table resource](http://goo.gl/sl8Dmg) for more detailed information.
+ *     pairs. Valid types are "string", "integer", "float", "boolean", "bytes",
+ *     "record", and "timestamp". If the type is omitted, it is assumed to be
+ *     "string". Example: "name:string, age:integer". Schemas can also be
+ *     specified as a JSON array of fields, which allows for nested and repeated
+ *     fields. See a [Table resource](http://goo.gl/sl8Dmg) for more detailed
+ *     information.
  * @param {function} callback - The callback function.
  * @param {?error} callback.err - An error returned while making this request.
   * @param {object} callback.apiResponse - The full API response.

packages/bigquery/system-test/bigquery.js

@@ -38,6 +38,55 @@ describe('BigQuery', function() {
 
   var query = 'SELECT url FROM [publicdata:samples.github_nested] LIMIT 100';
 
+  var SCHEMA = [
+    {
+      name: 'id',
+      type: 'INTEGER'
+    },
+    {
+      name: 'breed',
+      type: 'STRING'
+    },
+    {
+      name: 'name',
+      type: 'STRING'
+    },
+    {
+      name: 'dob',
+      type: 'TIMESTAMP'
+    },
+    {
+      name: 'around',
+      type: 'BOOLEAN'
+    },
+    {
+      name: 'buffer',
+      type: 'BYTES'
+    },
+    {
+      name: 'arrayOfInts',
+      type: 'INTEGER',
+      mode: 'REPEATED'
+    },
+    {
+      name: 'recordOfRecords',
+      type: 'RECORD',
+      fields: [
+        {
+          name: 'records',
+          type: 'RECORD',
+          mode: 'REPEATED',
+          fields: [
+            {
+              name: 'record',
+              type: 'BOOLEAN'
+            }
+          ]
+        }
+      ]
+    }
+  ];
+
   before(function(done) {
     async.series([
       // Remove buckets created for the tests.
@@ -51,7 +100,7 @@ describe('BigQuery', function() {
 
       // Create the test table.
       table.create.bind(table, {
-        schema: 'id:integer,breed,name,dob:timestamp,around:boolean'
+        schema: SCHEMA
       }),
 
       // Create a Bucket.
@@ -312,15 +361,7 @@ describe('BigQuery', function() {
     var TEST_DATA_JSON_PATH = require.resolve('./data/kitten-test-data.json');
 
     it('should have created the correct schema', function() {
-      assert.deepEqual(table.metadata.schema, {
-        fields: [
-          { name: 'id', type: 'INTEGER' },
-          { name: 'breed', type: 'STRING' },
-          { name: 'name', type: 'STRING' },
-          { name: 'dob', type: 'TIMESTAMP' },
-          { name: 'around', type: 'BOOLEAN' }
-        ]
-      });
+      assert.deepEqual(table.metadata.schema.fields, SCHEMA);
     });
 
     it('should get the rows in a table', function(done) {
@@ -388,14 +429,21 @@ describe('BigQuery', function() {
       });
 
       it('should convert values to their schema types', function(done) {
-        var now = new Date();
-
         var data = {
           name: 'dave',
           breed: 'british shorthair',
           id: 99,
-          dob: now.toJSON(),
-          around: true
+          dob: new Date(),
+          around: true,
+          buffer: new Buffer('test'),
+          arrayOfInts: [1, 3, 5],
+          recordOfRecords: {
+            records: [
+              {
+                record: true
+              }
+            ]
+          }
         };
 
         table.insert(data, function(err, insertErrors) {
@@ -409,7 +457,11 @@ describe('BigQuery', function() {
           function query(callback) {
             var row;
 
-            table.query('SELECT * FROM ' + table.id + ' WHERE id = ' + data.id)
+            table
+              .query({
+                query: 'SELECT * FROM ' + table.id + ' WHERE id = ' + data.id,
+                useLegacySql: false
+              })
               .on('error', callback)
               .once('data', function(row_) { row = row_; })
               .on('end', function() {
@@ -418,10 +470,7 @@ describe('BigQuery', function() {
                   return;
                 }
 
-                assert.strictEqual(row.name, data.name);
-                assert.strictEqual(row.breed, data.breed);
-                assert.strictEqual(row.id, data.id);
-                assert.deepEqual(row.dob, now);
+                assert.deepEqual(row, data);
                 callback();
               });
           }

packages/bigquery/system-test/data/kitten-test-data.json

@@ -1,3 +1,3 @@
-{ "name": "silvano", "breed": "the cat kind", "id": 1, "dob": 1414634759011, "around": false }
-{ "name": "ryan", "breed": "golden retriever?", "id": 2, "dob": 1414634759012, "around": false }
-{ "name": "stephen", "breed": "idkanycatbreeds", "id": 3, "dob": 1414634759013, "around": true }
+{ "name": "silvano", "breed": "the cat kind", "id": 1, "dob": 1414634759011, "around": false, "buffer": "dGVzdA==", "arrayOfInts": [1, 3, 5], "recordOfRecords": { "records": [{ "record": true }] } }
+{ "name": "ryan", "breed": "golden retriever?", "id": 2, "dob": 1414634759012, "around": false, "buffer": "dGVzdA==", "arrayOfInts": [1, 3, 5], "recordOfRecords": { "records": [{ "record": true }] } }
+{ "name": "stephen", "breed": "idkanycatbreeds", "id": 3, "dob": 1414634759013, "around": true, "buffer": "dGVzdA==", "arrayOfInts": [1, 3, 5], "recordOfRecords": { "records": [{ "record": true }] } }