Skip to content

Commit ecb2162

Browse files
tswasttswast
authored
feat(bigquery): add range partitioning to tables, load jobs, and query jobs (#9477)
* feat(bigquery): add range partitioning to tables, load jobs, and query jobs These classes and properties add support for the integer range partitioning feature. These offer more flexibility in partitioning options than time-based partitioning. * Add integer range partitioning classes to bigquery module * Rename PartitionRange to RangeDefinition * Revert "Rename PartitionRange to RangeDefinition" This reverts commit 9bb5d8b. * Add Beta disclaimer to range partitioning features.
1 parent 32eda79 commit ecb2162

File tree

6 files changed

+487
-11
lines changed

6 files changed

+487
-11
lines changed

‎bigquery/docs/reference.rst

+4-2
Original file line numberDiff line numberDiff line change
@@ -83,11 +83,13 @@ Table
8383
.. autosummary::
8484
:toctree: generated
8585

86+
table.PartitionRange
87+
table.RangePartitioning
88+
table.Row
89+
table.RowIterator
8690
table.Table
8791
table.TableListItem
8892
table.TableReference
89-
table.Row
90-
table.RowIterator
9193
table.TimePartitioning
9294
table.TimePartitioningType
9395

‎bigquery/google/cloud/bigquery/__init__.py

+8-3
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,11 @@
7373
from google.cloud.bigquery.routine import RoutineArgument
7474
from google.cloud.bigquery.routine import RoutineReference
7575
from google.cloud.bigquery.schema import SchemaField
76+
from google.cloud.bigquery.table import PartitionRange
77+
from google.cloud.bigquery.table import RangePartitioning
78+
from google.cloud.bigquery.table import Row
7679
from google.cloud.bigquery.table import Table
7780
from google.cloud.bigquery.table import TableReference
78-
from google.cloud.bigquery.table import Row
7981
from google.cloud.bigquery.table import TimePartitioningType
8082
from google.cloud.bigquery.table import TimePartitioning
8183
from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration
@@ -96,16 +98,19 @@
9698
# Tables
9799
"Table",
98100
"TableReference",
101+
"PartitionRange",
102+
"RangePartitioning",
99103
"Row",
104+
"TimePartitioning",
105+
"TimePartitioningType",
106+
# Jobs
100107
"CopyJob",
101108
"CopyJobConfig",
102109
"ExtractJob",
103110
"ExtractJobConfig",
104111
"LoadJob",
105112
"LoadJobConfig",
106113
"UnknownJob",
107-
"TimePartitioningType",
108-
"TimePartitioning",
109114
# Models
110115
"Model",
111116
"ModelReference",

‎bigquery/google/cloud/bigquery/job.py

+101-4
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,9 @@
2727
from google.cloud.bigquery.dataset import Dataset
2828
from google.cloud.bigquery.dataset import DatasetListItem
2929
from google.cloud.bigquery.dataset import DatasetReference
30+
from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration
3031
from google.cloud.bigquery.external_config import ExternalConfig
32+
from google.cloud.bigquery import _helpers
3133
from google.cloud.bigquery.query import _query_param_from_api_repr
3234
from google.cloud.bigquery.query import ArrayQueryParameter
3335
from google.cloud.bigquery.query import ScalarQueryParameter
@@ -37,12 +39,11 @@
3739
from google.cloud.bigquery.routine import RoutineReference
3840
from google.cloud.bigquery.schema import SchemaField
3941
from google.cloud.bigquery.table import _EmptyRowIterator
42+
from google.cloud.bigquery.table import RangePartitioning
4043
from google.cloud.bigquery.table import _table_arg_to_table_ref
4144
from google.cloud.bigquery.table import TableReference
4245
from google.cloud.bigquery.table import Table
4346
from google.cloud.bigquery.table import TimePartitioning
44-
from google.cloud.bigquery import _helpers
45-
from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration
4647

4748
_DONE_STATE = "DONE"
4849
_STOPPED_REASON = "stopped"
@@ -1180,6 +1181,40 @@ def quote_character(self):
11801181
def quote_character(self, value):
11811182
self._set_sub_prop("quote", value)
11821183

1184+
@property
1185+
def range_partitioning(self):
1186+
"""Optional[google.cloud.bigquery.table.RangePartitioning]:
1187+
Configures range-based partitioning for destination table.
1188+
1189+
.. note::
1190+
**Beta**. The integer range partitioning feature is in a
1191+
pre-release state and might change or have limited support.
1192+
1193+
Only specify at most one of
1194+
:attr:`~google.cloud.bigquery.job.LoadJobConfig.time_partitioning` or
1195+
:attr:`~google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.
1196+
1197+
Raises:
1198+
ValueError:
1199+
If the value is not
1200+
:class:`~google.cloud.bigquery.table.RangePartitioning` or
1201+
:data:`None`.
1202+
"""
1203+
resource = self._get_sub_prop("rangePartitioning")
1204+
if resource is not None:
1205+
return RangePartitioning(_properties=resource)
1206+
1207+
@range_partitioning.setter
1208+
def range_partitioning(self, value):
1209+
resource = value
1210+
if isinstance(value, RangePartitioning):
1211+
resource = value._properties
1212+
elif value is not None:
1213+
raise ValueError(
1214+
"Expected value to be RangePartitioning or None, got {}.".format(value)
1215+
)
1216+
self._set_sub_prop("rangePartitioning", resource)
1217+
11831218
@property
11841219
def schema(self):
11851220
"""List[google.cloud.bigquery.schema.SchemaField]: Schema of the
@@ -1249,6 +1284,10 @@ def source_format(self, value):
12491284
def time_partitioning(self):
12501285
"""google.cloud.bigquery.table.TimePartitioning: Specifies time-based
12511286
partitioning for the destination table.
1287+
1288+
Only specify at most one of
1289+
:attr:`~google.cloud.bigquery.job.LoadJobConfig.time_partitioning` or
1290+
:attr:`~google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.
12521291
"""
12531292
prop = self._get_sub_prop("timePartitioning")
12541293
if prop is not None:
@@ -1463,6 +1502,13 @@ def destination_table_friendly_name(self):
14631502
"""
14641503
return self._configuration.destination_table_friendly_name
14651504

1505+
@property
1506+
def range_partitioning(self):
1507+
"""See
1508+
:attr:`google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.
1509+
"""
1510+
return self._configuration.range_partitioning
1511+
14661512
@property
14671513
def time_partitioning(self):
14681514
"""See
@@ -2242,6 +2288,40 @@ def query_parameters(self):
22422288
def query_parameters(self, values):
22432289
self._set_sub_prop("queryParameters", _to_api_repr_query_parameters(values))
22442290

2291+
@property
2292+
def range_partitioning(self):
2293+
"""Optional[google.cloud.bigquery.table.RangePartitioning]:
2294+
Configures range-based partitioning for destination table.
2295+
2296+
.. note::
2297+
**Beta**. The integer range partitioning feature is in a
2298+
pre-release state and might change or have limited support.
2299+
2300+
Only specify at most one of
2301+
:attr:`~google.cloud.bigquery.job.LoadJobConfig.time_partitioning` or
2302+
:attr:`~google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.
2303+
2304+
Raises:
2305+
ValueError:
2306+
If the value is not
2307+
:class:`~google.cloud.bigquery.table.RangePartitioning` or
2308+
:data:`None`.
2309+
"""
2310+
resource = self._get_sub_prop("rangePartitioning")
2311+
if resource is not None:
2312+
return RangePartitioning(_properties=resource)
2313+
2314+
@range_partitioning.setter
2315+
def range_partitioning(self, value):
2316+
resource = value
2317+
if isinstance(value, RangePartitioning):
2318+
resource = value._properties
2319+
elif value is not None:
2320+
raise ValueError(
2321+
"Expected value to be RangePartitioning or None, got {}.".format(value)
2322+
)
2323+
self._set_sub_prop("rangePartitioning", resource)
2324+
22452325
@property
22462326
def udf_resources(self):
22472327
"""List[google.cloud.bigquery.query.UDFResource]: user
@@ -2318,8 +2398,18 @@ def table_definitions(self, values):
23182398

23192399
@property
23202400
def time_partitioning(self):
2321-
"""google.cloud.bigquery.table.TimePartitioning: Specifies time-based
2322-
partitioning for the destination table.
2401+
"""Optional[google.cloud.bigquery.table.TimePartitioning]: Specifies
2402+
time-based partitioning for the destination table.
2403+
2404+
Only specify at most one of
2405+
:attr:`~google.cloud.bigquery.job.LoadJobConfig.time_partitioning` or
2406+
:attr:`~google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.
2407+
2408+
Raises:
2409+
ValueError:
2410+
If the value is not
2411+
:class:`~google.cloud.bigquery.table.TimePartitioning` or
2412+
:data:`None`.
23232413
"""
23242414
prop = self._get_sub_prop("timePartitioning")
23252415
if prop is not None:
@@ -2552,6 +2642,13 @@ def maximum_bytes_billed(self):
25522642
"""
25532643
return self._configuration.maximum_bytes_billed
25542644

2645+
@property
2646+
def range_partitioning(self):
2647+
"""See
2648+
:attr:`google.cloud.bigquery.job.QueryJobConfig.range_partitioning`.
2649+
"""
2650+
return self._configuration.range_partitioning
2651+
25552652
@property
25562653
def table_definitions(self):
25572654
"""See

0 commit comments

Comments
 (0)