Merge pull request #2806 from tseaver/2229-bigquery-marshal-bytes-time-types

Marshal 'BYTES' and 'TIME' column / paramter types

Author: tseaver

bigquery/google/cloud/bigquery/_helpers.py

@@ -14,6 +14,7 @@
 
 """Shared helper functions for BigQuery API classes."""
 
+import base64
 from collections import OrderedDict
 import datetime
 
@@ -22,6 +23,7 @@
 from google.cloud._helpers import _datetime_to_rfc3339
 from google.cloud._helpers import _microseconds_from_datetime
 from google.cloud._helpers import _RFC3339_NO_FRACTION
+from google.cloud._helpers import _time_from_iso8601_time_naive
 
 
 def _not_null(value, field):
@@ -47,6 +49,17 @@ def _bool_from_json(value, field):
         return value.lower() in ['t', 'true', '1']
 
 
+def _string_from_json(value, _):
+    """NOOP string -> string coercion"""
+    return value
+
+
+def _bytes_from_json(value, field):
+    """Base64-decode value"""
+    if _not_null(value, field):
+        return base64.decodestring(value)
+
+
 def _timestamp_from_json(value, field):
     """Coerce 'value' to a datetime, if set or not nullable."""
     if _not_null(value, field):
@@ -64,9 +77,17 @@ def _datetime_from_json(value, field):
 def _date_from_json(value, field):
     """Coerce 'value' to a datetime date, if set or not nullable"""
     if _not_null(value, field):
+        # value will be a string, in YYYY-MM-DD form.
         return _date_from_iso8601_date(value)
 
 
+def _time_from_json(value, field):
+    """Coerce 'value' to a datetime date, if set or not nullable"""
+    if _not_null(value, field):
+        # value will be a string, in HH:MM:SS form.
+        return _time_from_iso8601_time_naive(value)
+
+
 def _record_from_json(value, field):
     """Coerce 'value' to a mapping, if set or not nullable."""
     if _not_null(value, field):
@@ -82,23 +103,20 @@ def _record_from_json(value, field):
         return record
 
 
-def _string_from_json(value, _):
-    """NOOP string -> string coercion"""
-    return value
-
-
 _CELLDATA_FROM_JSON = {
     'INTEGER': _int_from_json,
     'INT64': _int_from_json,
     'FLOAT': _float_from_json,
     'FLOAT64': _float_from_json,
     'BOOLEAN': _bool_from_json,
     'BOOL': _bool_from_json,
+    'STRING': _string_from_json,
+    'BYTES': _bytes_from_json,
     'TIMESTAMP': _timestamp_from_json,
     'DATETIME': _datetime_from_json,
     'DATE': _date_from_json,
+    'TIME': _time_from_json,
     'RECORD': _record_from_json,
-    'STRING': _string_from_json,
 }
 
 
@@ -121,6 +139,13 @@ def _bool_to_json(value):
     return value
 
 
+def _bytes_to_json(value):
+    """Coerce 'value' to an JSON-compatible representation."""
+    if isinstance(value, bytes):
+        value = base64.encodestring(value)
+    return value
+
+
 def _timestamp_to_json(value):
     """Coerce 'value' to an JSON-compatible representation."""
     if isinstance(value, datetime.datetime):
@@ -142,16 +167,25 @@ def _date_to_json(value):
     return value
 
 
+def _time_to_json(value):
+    """Coerce 'value' to an JSON-compatible representation."""
+    if isinstance(value, datetime.time):
+        value = value.isoformat()
+    return value
+
+
 _SCALAR_VALUE_TO_JSON = {
     'INTEGER': _int_to_json,
     'INT64': _int_to_json,
     'FLOAT': _float_to_json,
     'FLOAT64': _float_to_json,
     'BOOLEAN': _bool_to_json,
     'BOOL': _bool_to_json,
+    'BYTES': _bytes_to_json,
     'TIMESTAMP': _timestamp_to_json,
     'DATETIME': _datetime_to_json,
     'DATE': _date_to_json,
+    'TIME': _time_to_json,
 }
 
 

bigquery/unit_tests/test__helpers.py

@@ -105,6 +105,44 @@ def test_w_value_other(self):
         self.assertFalse(coerced)
 
 
+class Test_string_from_json(unittest.TestCase):
+
+    def _call_fut(self, value, field):
+        from google.cloud.bigquery._helpers import _string_from_json
+        return _string_from_json(value, field)
+
+    def test_w_none_nullable(self):
+        self.assertIsNone(self._call_fut(None, _Field('NULLABLE')))
+
+    def test_w_none_required(self):
+        self.assertIsNone(self._call_fut(None, _Field('REQUIRED')))
+
+    def test_w_string_value(self):
+        coerced = self._call_fut('Wonderful!', object())
+        self.assertEqual(coerced, 'Wonderful!')
+
+
+class Test_bytes_from_json(unittest.TestCase):
+
+    def _call_fut(self, value, field):
+        from google.cloud.bigquery._helpers import _bytes_from_json
+        return _bytes_from_json(value, field)
+
+    def test_w_none_nullable(self):
+        self.assertIsNone(self._call_fut(None, _Field('NULLABLE')))
+
+    def test_w_none_required(self):
+        with self.assertRaises(TypeError):
+            self._call_fut(None, _Field('REQUIRED'))
+
+    def test_w_base64_encoded_value(self):
+        import base64
+        expected = b'Wonderful!'
+        encoded = base64.encodestring(expected)
+        coerced = self._call_fut(encoded, object())
+        self.assertEqual(coerced, expected)
+
+
 class Test_timestamp_from_json(unittest.TestCase):
 
     def _call_fut(self, value, field):
@@ -177,6 +215,27 @@ def test_w_string_value(self):
             datetime.date(1987, 9, 22))
 
 
+class Test_time_from_json(unittest.TestCase):
+
+    def _call_fut(self, value, field):
+        from google.cloud.bigquery._helpers import _time_from_json
+        return _time_from_json(value, field)
+
+    def test_w_none_nullable(self):
+        self.assertIsNone(self._call_fut(None, _Field('NULLABLE')))
+
+    def test_w_none_required(self):
+        with self.assertRaises(TypeError):
+            self._call_fut(None, _Field('REQUIRED'))
+
+    def test_w_string_value(self):
+        import datetime
+        coerced = self._call_fut('12:12:27', object())
+        self.assertEqual(
+            coerced,
+            datetime.time(12, 12, 27))
+
+
 class Test_record_from_json(unittest.TestCase):
 
     def _call_fut(self, value, field):
@@ -238,23 +297,6 @@ def test_w_record_subfield(self):
         self.assertEqual(coerced, expected)
 
 
-class Test_string_from_json(unittest.TestCase):
-
-    def _call_fut(self, value, field):
-        from google.cloud.bigquery._helpers import _string_from_json
-        return _string_from_json(value, field)
-
-    def test_w_none_nullable(self):
-        self.assertIsNone(self._call_fut(None, _Field('NULLABLE')))
-
-    def test_w_none_required(self):
-        self.assertIsNone(self._call_fut(None, _Field('RECORD')))
-
-    def test_w_string_value(self):
-        coerced = self._call_fut('Wonderful!', object())
-        self.assertEqual(coerced, 'Wonderful!')
-
-
 class Test_row_from_json(unittest.TestCase):
 
     def _call_fut(self, row, schema):
@@ -471,6 +513,23 @@ def test_w_string(self):
         self.assertEqual(self._call_fut('false'), 'false')
 
 
+class Test_bytes_to_json(unittest.TestCase):
+
+    def _call_fut(self, value):
+        from google.cloud.bigquery._helpers import _bytes_to_json
+        return _bytes_to_json(value)
+
+    def test_w_non_bytes(self):
+        non_bytes = object()
+        self.assertIs(self._call_fut(non_bytes), non_bytes)
+
+    def test_w_bytes(self):
+        import base64
+        source = b'source'
+        expected = base64.encodestring(source)
+        self.assertEqual(self._call_fut(source), expected)
+
+
 class Test_timestamp_to_json(unittest.TestCase):
 
     def _call_fut(self, value):
@@ -522,6 +581,22 @@ def test_w_datetime(self):
         self.assertEqual(self._call_fut(when), '2016-12-03')
 
 
+class Test_time_to_json(unittest.TestCase):
+
+    def _call_fut(self, value):
+        from google.cloud.bigquery._helpers import _time_to_json
+        return _time_to_json(value)
+
+    def test_w_string(self):
+        RFC3339 = '12:13:41'
+        self.assertEqual(self._call_fut(RFC3339), RFC3339)
+
+    def test_w_datetime(self):
+        import datetime
+        when = datetime.time(12, 13, 41)
+        self.assertEqual(self._call_fut(when), '12:13:41')
+
+
 class Test_ConfigurationProperty(unittest.TestCase):
 
     @staticmethod

core/google/cloud/_helpers.py

@@ -251,6 +251,19 @@ def _date_from_iso8601_date(value):
     return datetime.datetime.strptime(value, '%Y-%m-%d').date()
 
 
+def _time_from_iso8601_time_naive(value):
+    """Convert a zoneless ISO8601 time string to naive datetime time
+
+    :type value: str
+    :param value: The time string to convert
+
+    :rtype: :class:`datetime.time`
+    :returns: A datetime time object created from the string
+
+    """
+    return datetime.datetime.strptime(value, '%H:%M:%S').time()
+
+
 def _rfc3339_to_datetime(dt_str):
     """Convert a microsecond-precision timetamp to a native datetime.
 

core/unit_tests/test__helpers.py

@@ -265,6 +265,18 @@ def test_todays_date(self):
         self.assertEqual(self._call_fut(TODAY.strftime("%Y-%m-%d")), TODAY)
 
 
+class Test___time_from_iso8601_time_naive(unittest.TestCase):
+
+    def _call_fut(self, value):
+        from google.cloud._helpers import _time_from_iso8601_time_naive
+        return _time_from_iso8601_time_naive(value)
+
+    def test_todays_date(self):
+        import datetime
+        WHEN = datetime.time(12, 9, 42)
+        self.assertEqual(self._call_fut(("12:09:42")), WHEN)
+
+
 class Test__rfc3339_to_datetime(unittest.TestCase):
 
     def _call_fut(self, dt_str):

system_tests/bigquery.py

@@ -479,12 +479,49 @@ def _job_done(instance):
         # raise an error, and that the job completed (in the `retry()`
         # above).
 
-    def test_sync_query_w_nested_arrays_and_structs(self):
+    def test_sync_query_w_standard_sql_types(self):
+        import datetime
+        from google.cloud._helpers import UTC
+        naive = datetime.datetime(2016, 12, 5, 12, 41, 9)
+        stamp = "%s %s" % (naive.date().isoformat(), naive.time().isoformat())
+        zoned = naive.replace(tzinfo=UTC)
         EXAMPLES = [
             {
                 'sql': 'SELECT 1',
                 'expected': 1,
             },
+            {
+                'sql': 'SELECT 1.3',
+                'expected': 1.3,
+            },
+            {
+                'sql': 'SELECT TRUE',
+                'expected': True,
+            },
+            {
+                'sql': 'SELECT "ABC"',
+                'expected': 'ABC',
+            },
+            {
+                'sql': 'SELECT CAST("foo" AS BYTES)',
+                'expected': b'foo',
+            },
+            {
+                'sql': 'SELECT TIMESTAMP "%s"' % (stamp,),
+                'expected': zoned,
+            },
+            {
+                'sql': 'SELECT DATETIME(TIMESTAMP "%s")' % (stamp,),
+                'expected': naive,
+            },
+            {
+                'sql': 'SELECT DATE(TIMESTAMP "%s")' % (stamp,),
+                'expected': naive.date(),
+            },
+            {
+                'sql': 'SELECT TIME(TIMESTAMP "%s")' % (stamp,),
+                'expected': naive.time(),
+            },
             {
                 'sql': 'SELECT (1, 2)',
                 'expected': {'_field_1': 1, '_field_2': 2},