Merge branch 'main' into feat/expose-cursor-query-job

Author: r1b

.github/.OwlBot.lock.yaml

@@ -13,4 +13,4 @@
 # limitations under the License.
 docker:
   image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest
-  digest: sha256:8555f0e37e6261408f792bfd6635102d2da5ad73f8f09bcb24f25e6afb5fac97
+  digest: sha256:2e247c7bf5154df7f98cce087a20ca7605e236340c7d6d1a14447e5c06791bd6

.kokoro/requirements.in

@@ -5,6 +5,6 @@ typing-extensions
 twine
 wheel
 setuptools
-nox
+nox>=2022.11.21 # required to remove dependency on py
 charset-normalizer<3
 click<8.1.0

.kokoro/requirements.txt

@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
 #
 #    pip-compile --allow-unsafe --generate-hashes requirements.in
 #
@@ -335,9 +335,9 @@ more-itertools==9.0.0 \
     --hash=sha256:250e83d7e81d0c87ca6bd942e6aeab8cc9daa6096d12c5308f3f92fa5e5c1f41 \
     --hash=sha256:5a6257e40878ef0520b1803990e3e22303a41b5714006c32a3fd8304b26ea1ab
     # via jaraco-classes
-nox==2022.8.7 \
-    --hash=sha256:1b894940551dc5c389f9271d197ca5d655d40bdc6ccf93ed6880e4042760a34b \
-    --hash=sha256:96cca88779e08282a699d672258ec01eb7c792d35bbbf538c723172bce23212c
+nox==2022.11.21 \
+    --hash=sha256:0e41a990e290e274cb205a976c4c97ee3c5234441a8132c8c3fd9ea3c22149eb \
+    --hash=sha256:e21c31de0711d1274ca585a2c5fde36b1aa962005ba8e9322bf5eeed16dcd684
     # via -r requirements.in
 packaging==21.3 \
     --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \
@@ -380,10 +380,6 @@ protobuf==3.20.3 \
     #   gcp-docuploader
     #   gcp-releasetool
     #   google-api-core
-py==1.11.0 \
-    --hash=sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719 \
-    --hash=sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378
-    # via nox
 pyasn1==0.4.8 \
     --hash=sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d \
     --hash=sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba

CHANGELOG.md

@@ -5,6 +5,19 @@
 [1]: https://pypi.org/project/google-cloud-bigquery/#history
 
 
+## [3.7.0](https://github.com/googleapis/python-bigquery/compare/v3.6.0...v3.7.0) (2023-03-06)
+
+
+### Features
+
+* Add `connection_properties` and `create_session` to `LoadJobConfig` ([#1509](https://github.com/googleapis/python-bigquery/issues/1509)) ([cd0aaa1](https://github.com/googleapis/python-bigquery/commit/cd0aaa15960e9ca7a0aaf411c8e4990f95421816))
+* Add default_query_job_config property and property setter to BQ client ([#1511](https://github.com/googleapis/python-bigquery/issues/1511)) ([a23092c](https://github.com/googleapis/python-bigquery/commit/a23092cad834c6a016f455d46fefa13bb6cdbf0f))
+
+
+### Documentation
+
+* Remove < 3.11 reference from README ([#1502](https://github.com/googleapis/python-bigquery/issues/1502)) ([c7417f4](https://github.com/googleapis/python-bigquery/commit/c7417f43563e20a3e6f1a57f46925fb274b28b07))
+
 ## [3.6.0](https://github.com/googleapis/python-bigquery/compare/v3.5.0...v3.6.0) (2023-02-22)
 
 

google/cloud/bigquery/client.py

@@ -210,6 +210,9 @@ class Client(ClientWithProject):
         default_query_job_config (Optional[google.cloud.bigquery.job.QueryJobConfig]):
             Default ``QueryJobConfig``.
             Will be merged into job configs passed into the ``query`` method.
+        default_load_job_config (Optional[google.cloud.bigquery.job.LoadJobConfig]):
+            Default ``LoadJobConfig``.
+            Will be merged into job configs passed into the ``load_table_*`` methods.
         client_info (Optional[google.api_core.client_info.ClientInfo]):
             The client info used to send a user-agent string along with API
             requests. If ``None``, then default info will be used. Generally,
@@ -235,6 +238,7 @@ def __init__(
         _http=None,
         location=None,
         default_query_job_config=None,
+        default_load_job_config=None,
         client_info=None,
         client_options=None,
     ) -> None:
@@ -260,6 +264,7 @@ def __init__(
         self._connection = Connection(self, **kw_args)
         self._location = location
         self._default_query_job_config = copy.deepcopy(default_query_job_config)
+        self._default_load_job_config = copy.deepcopy(default_load_job_config)
 
     @property
     def location(self):
@@ -277,6 +282,17 @@ def default_query_job_config(self):
     def default_query_job_config(self, value: QueryJobConfig):
         self._default_query_job_config = copy.deepcopy(value)
 
+    @property
+    def default_load_job_config(self):
+        """Default ``LoadJobConfig``.
+        Will be merged into job configs passed into the ``load_table_*`` methods.
+        """
+        return self._default_load_job_config
+
+    @default_load_job_config.setter
+    def default_load_job_config(self, value: LoadJobConfig):
+        self._default_load_job_config = copy.deepcopy(value)
+
     def close(self):
         """Close the underlying transport objects, releasing system resources.
 
@@ -1976,15 +1992,8 @@ def create_job(
             )
             destination = _get_sub_prop(job_config, ["copy", "destinationTable"])
             destination = TableReference.from_api_repr(destination)
-            sources = []
-            source_configs = _get_sub_prop(job_config, ["copy", "sourceTables"])
-            if source_configs is None:
-                source_configs = [_get_sub_prop(job_config, ["copy", "sourceTable"])]
-            for source_config in source_configs:
-                table_ref = TableReference.from_api_repr(source_config)
-                sources.append(table_ref)
             return self.copy_table(
-                sources,
+                [],  # Source table(s) already in job_config resource.
                 destination,
                 job_config=typing.cast(CopyJobConfig, copy_job_config),
                 retry=retry,
@@ -2337,8 +2346,8 @@ def load_table_from_uri(
 
         Raises:
             TypeError:
-                If ``job_config`` is not an instance of :class:`~google.cloud.bigquery.job.LoadJobConfig`
-                class.
+                If ``job_config`` is not an instance of
+                :class:`~google.cloud.bigquery.job.LoadJobConfig` class.
         """
         job_id = _make_job_id(job_id, job_id_prefix)
 
@@ -2355,11 +2364,14 @@ def load_table_from_uri(
 
         destination = _table_arg_to_table_ref(destination, default_project=self.project)
 
-        if job_config:
-            job_config = copy.deepcopy(job_config)
-            _verify_job_config_type(job_config, google.cloud.bigquery.job.LoadJobConfig)
+        if job_config is not None:
+            _verify_job_config_type(job_config, LoadJobConfig)
+        else:
+            job_config = job.LoadJobConfig()
 
-        load_job = job.LoadJob(job_ref, source_uris, destination, self, job_config)
+        new_job_config = job_config._fill_from_default(self._default_load_job_config)
+
+        load_job = job.LoadJob(job_ref, source_uris, destination, self, new_job_config)
         load_job._begin(retry=retry, timeout=timeout)
 
         return load_job
@@ -2431,8 +2443,8 @@ def load_table_from_file(
                 mode.
 
             TypeError:
-                If ``job_config`` is not an instance of :class:`~google.cloud.bigquery.job.LoadJobConfig`
-                class.
+                If ``job_config`` is not an instance of
+                :class:`~google.cloud.bigquery.job.LoadJobConfig` class.
         """
         job_id = _make_job_id(job_id, job_id_prefix)
 
@@ -2444,10 +2456,15 @@ def load_table_from_file(
 
         destination = _table_arg_to_table_ref(destination, default_project=self.project)
         job_ref = job._JobReference(job_id, project=project, location=location)
-        if job_config:
-            job_config = copy.deepcopy(job_config)
-            _verify_job_config_type(job_config, google.cloud.bigquery.job.LoadJobConfig)
-        load_job = job.LoadJob(job_ref, None, destination, self, job_config)
+
+        if job_config is not None:
+            _verify_job_config_type(job_config, LoadJobConfig)
+        else:
+            job_config = job.LoadJobConfig()
+
+        new_job_config = job_config._fill_from_default(self._default_load_job_config)
+
+        load_job = job.LoadJob(job_ref, None, destination, self, new_job_config)
         job_resource = load_job.to_api_repr()
 
         if rewind:
@@ -2571,43 +2588,40 @@ def load_table_from_dataframe(
                 If a usable parquet engine cannot be found. This method
                 requires :mod:`pyarrow` to be installed.
             TypeError:
-                If ``job_config`` is not an instance of :class:`~google.cloud.bigquery.job.LoadJobConfig`
-                class.
+                If ``job_config`` is not an instance of
+                :class:`~google.cloud.bigquery.job.LoadJobConfig` class.
         """
         job_id = _make_job_id(job_id, job_id_prefix)
 
-        if job_config:
-            _verify_job_config_type(job_config, google.cloud.bigquery.job.LoadJobConfig)
-            # Make a copy so that the job config isn't modified in-place.
-            job_config_properties = copy.deepcopy(job_config._properties)
-            job_config = job.LoadJobConfig()
-            job_config._properties = job_config_properties
-
+        if job_config is not None:
+            _verify_job_config_type(job_config, LoadJobConfig)
         else:
             job_config = job.LoadJobConfig()
 
+        new_job_config = job_config._fill_from_default(self._default_load_job_config)
+
         supported_formats = {job.SourceFormat.CSV, job.SourceFormat.PARQUET}
-        if job_config.source_format is None:
+        if new_job_config.source_format is None:
             # default value
-            job_config.source_format = job.SourceFormat.PARQUET
+            new_job_config.source_format = job.SourceFormat.PARQUET
 
         if (
-            job_config.source_format == job.SourceFormat.PARQUET
-            and job_config.parquet_options is None
+            new_job_config.source_format == job.SourceFormat.PARQUET
+            and new_job_config.parquet_options is None
         ):
             parquet_options = ParquetOptions()
             # default value
             parquet_options.enable_list_inference = True
-            job_config.parquet_options = parquet_options
+            new_job_config.parquet_options = parquet_options
 
-        if job_config.source_format not in supported_formats:
+        if new_job_config.source_format not in supported_formats:
             raise ValueError(
                 "Got unexpected source_format: '{}'. Currently, only PARQUET and CSV are supported".format(
-                    job_config.source_format
+                    new_job_config.source_format
                 )
             )
 
-        if pyarrow is None and job_config.source_format == job.SourceFormat.PARQUET:
+        if pyarrow is None and new_job_config.source_format == job.SourceFormat.PARQUET:
             # pyarrow is now the only supported parquet engine.
             raise ValueError("This method requires pyarrow to be installed")
 
@@ -2618,8 +2632,8 @@ def load_table_from_dataframe(
         # schema, and check if dataframe schema is compatible with it - except
         # for WRITE_TRUNCATE jobs, the existing schema does not matter then.
         if (
-            not job_config.schema
-            and job_config.write_disposition != job.WriteDisposition.WRITE_TRUNCATE
+            not new_job_config.schema
+            and new_job_config.write_disposition != job.WriteDisposition.WRITE_TRUNCATE
         ):
             try:
                 table = self.get_table(destination)
@@ -2630,7 +2644,7 @@ def load_table_from_dataframe(
                     name
                     for name, _ in _pandas_helpers.list_columns_and_indexes(dataframe)
                 )
-                job_config.schema = [
+                new_job_config.schema = [
                     # Field description and policy tags are not needed to
                     # serialize a data frame.
                     SchemaField(
@@ -2644,11 +2658,11 @@ def load_table_from_dataframe(
                     if field.name in columns_and_indexes
                 ]
 
-        job_config.schema = _pandas_helpers.dataframe_to_bq_schema(
-            dataframe, job_config.schema
+        new_job_config.schema = _pandas_helpers.dataframe_to_bq_schema(
+            dataframe, new_job_config.schema
         )
 
-        if not job_config.schema:
+        if not new_job_config.schema:
             # the schema could not be fully detected
             warnings.warn(
                 "Schema could not be detected for all columns. Loading from a "
@@ -2659,13 +2673,13 @@ def load_table_from_dataframe(
             )
 
         tmpfd, tmppath = tempfile.mkstemp(
-            suffix="_job_{}.{}".format(job_id[:8], job_config.source_format.lower())
+            suffix="_job_{}.{}".format(job_id[:8], new_job_config.source_format.lower())
         )
         os.close(tmpfd)
 
         try:
 
-            if job_config.source_format == job.SourceFormat.PARQUET:
+            if new_job_config.source_format == job.SourceFormat.PARQUET:
                 if _PYARROW_VERSION in _PYARROW_BAD_VERSIONS:
                     msg = (
                         "Loading dataframe data in PARQUET format with pyarrow "
@@ -2676,13 +2690,13 @@ def load_table_from_dataframe(
                     )
                     warnings.warn(msg, category=RuntimeWarning)
 
-                if job_config.schema:
+                if new_job_config.schema:
                     if parquet_compression == "snappy":  # adjust the default value
                         parquet_compression = parquet_compression.upper()
 
                     _pandas_helpers.dataframe_to_parquet(
                         dataframe,
-                        job_config.schema,
+                        new_job_config.schema,
                         tmppath,
                         parquet_compression=parquet_compression,
                         parquet_use_compliant_nested_type=True,
@@ -2722,7 +2736,7 @@ def load_table_from_dataframe(
                     job_id_prefix=job_id_prefix,
                     location=location,
                     project=project,
-                    job_config=job_config,
+                    job_config=new_job_config,
                     timeout=timeout,
                 )
 
@@ -2798,22 +2812,22 @@ def load_table_from_json(
 
         Raises:
             TypeError:
-                If ``job_config`` is not an instance of :class:`~google.cloud.bigquery.job.LoadJobConfig`
-                class.
+                If ``job_config`` is not an instance of
+                :class:`~google.cloud.bigquery.job.LoadJobConfig` class.
         """
         job_id = _make_job_id(job_id, job_id_prefix)
 
-        if job_config:
-            _verify_job_config_type(job_config, google.cloud.bigquery.job.LoadJobConfig)
-            # Make a copy so that the job config isn't modified in-place.
-            job_config = copy.deepcopy(job_config)
+        if job_config is not None:
+            _verify_job_config_type(job_config, LoadJobConfig)
         else:
             job_config = job.LoadJobConfig()
 
-        job_config.source_format = job.SourceFormat.NEWLINE_DELIMITED_JSON
+        new_job_config = job_config._fill_from_default(self._default_load_job_config)
+
+        new_job_config.source_format = job.SourceFormat.NEWLINE_DELIMITED_JSON
 
-        if job_config.schema is None:
-            job_config.autodetect = True
+        if new_job_config.schema is None:
+            new_job_config.autodetect = True
 
         if project is None:
             project = self.project
@@ -2835,7 +2849,7 @@ def load_table_from_json(
             job_id_prefix=job_id_prefix,
             location=location,
             project=project,
-            job_config=job_config,
+            job_config=new_job_config,
             timeout=timeout,
         )