Skip to content

Commit 371c4f2

Browse files
emar-kartswast
emar-kar
authored and
tswast
committed
refactor(bigquery): update code samples to use strings for table and dataset IDs (#9495)
* refactor(bigquery): update code samples * kms_key_name update * refactor(bigquery): kms_key_name update * samples update * docs session fix * docs session fix * update docs * assertion update with re * docs(bigquery): remove location parameter from samples The parameter can sometimes confuse new BigQuery developers. Since location autodetection now works pretty well, the parameter can be removed from code samples for better clarity, except where the samples want to explicitly demonstrate its usage. * Pass location where auto-detection not supported * update QueryJobConfig * unify undelete_table sample * refactor: update test files with the new conditions * refactor: remove extra blank line in test files
1 parent 9360057 commit 371c4f2

File tree

61 files changed

+729
-500
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

61 files changed

+729
-500
lines changed

‎bigquery/docs/snippets.py

-399
Large diffs are not rendered by default.

‎bigquery/docs/usage/encryption.rst

+2-2
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ Cloud KMS for the destination table.
3636
Copy a table, using a customer-managed encryption key from Cloud KMS for the
3737
destination table.
3838

39-
.. literalinclude:: ../snippets.py
39+
.. literalinclude:: ../samples/copy_table_cmek.py
4040
:language: python
4141
:dedent: 4
4242
:start-after: [START bigquery_copy_table_cmek]
@@ -45,7 +45,7 @@ destination table.
4545
Write query results to a table, using a customer-managed encryption key from
4646
Cloud KMS for the destination table.
4747

48-
.. literalinclude:: ../snippets.py
48+
.. literalinclude:: ../samples/client_query_destination_table_cmek.py
4949
:language: python
5050
:dedent: 4
5151
:start-after: [START bigquery_query_destination_table_cmek]

‎bigquery/docs/usage/queries.rst

+2-2
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ Run a query and wait for it to finish with the
1717
Run a dry run query
1818
^^^^^^^^^^^^^^^^^^^
1919

20-
.. literalinclude:: ../snippets.py
20+
.. literalinclude:: ../samples/client_query_dry_run.py
2121
:language: python
2222
:dedent: 4
2323
:start-after: [START bigquery_query_dry_run]
@@ -30,7 +30,7 @@ Writing query results to a destination table
3030
See BigQuery documentation for more information on
3131
`writing query results <https://cloud.google.com/bigquery/docs/writing-results>`_.
3232

33-
.. literalinclude:: ../snippets.py
33+
.. literalinclude:: ../samples/client_query_destination_table.py
3434
:language: python
3535
:dedent: 4
3636
:start-after: [START bigquery_query_destination_table]

‎bigquery/samples/client_query.py

+1-3
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,7 @@ def client_query(client):
3030
ORDER BY total_people DESC
3131
LIMIT 20
3232
"""
33-
query_job = client.query(
34-
query, location="US" # Must match the destination dataset(s) location.
35-
) # Make an API request.
33+
query_job = client.query(query) # Make an API request.
3634

3735
print("The query data:")
3836
for row in query_job:

‎bigquery/samples/client_query_add_column.py

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# Copyright 2019 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
def client_query_add_column(client, table_id):
17+
18+
# [START bigquery_add_column_query_append]
19+
from google.cloud import bigquery
20+
21+
# TODO(developer): Construct a BigQuery client object.
22+
# client = bigquery.Client()
23+
24+
# TODO(developer): Set table_id to the ID of the destination table.
25+
# table_id = "your-project.your_dataset.your_table_name"
26+
27+
# Retrieves the destination table and checks the length of the schema.
28+
table = client.get_table(table_id) # Make an API request.
29+
print("Table {} contains {} columns".format(table_id, len(table.schema)))
30+
31+
# Configures the query to append the results to a destination table,
32+
# allowing field addition.
33+
job_config = bigquery.QueryJobConfig(destination=table_id)
34+
job_config.schema_update_options = [
35+
bigquery.SchemaUpdateOption.ALLOW_FIELD_ADDITION
36+
]
37+
job_config.write_disposition = bigquery.WriteDisposition.WRITE_APPEND
38+
39+
# Start the query, passing in the extra configuration.
40+
query_job = client.query(
41+
# In this example, the existing table contains only the 'full_name' and
42+
# 'age' columns, while the results of this query will contain an
43+
# additional 'favorite_color' column.
44+
'SELECT "Timmy" as full_name, 85 as age, "Blue" as favorite_color;',
45+
job_config=job_config,
46+
) # Make an API request.
47+
query_job.result() # Wait for the job to complete.
48+
49+
# Checks the updated length of the schema.
50+
table = client.get_table(table_id) # Make an API request.
51+
print("Table {} now contains {} columns".format(table_id, len(table.schema)))
52+
# [END bigquery_add_column_query_append]

‎bigquery/samples/client_query_batch.py

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# Copyright 2019 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
def client_query_batch(client):
17+
18+
# [START bigquery_query_batch]
19+
from google.cloud import bigquery
20+
21+
# TODO(developer): Construct a BigQuery client object.
22+
# client = bigquery.Client()
23+
24+
job_config = bigquery.QueryJobConfig(
25+
# Run at batch priority, which won't count toward concurrent rate limit.
26+
priority=bigquery.QueryPriority.BATCH
27+
)
28+
29+
sql = """
30+
SELECT corpus
31+
FROM `bigquery-public-data.samples.shakespeare`
32+
GROUP BY corpus;
33+
"""
34+
35+
# Start the query, passing in the extra configuration.
36+
query_job = client.query(sql, job_config=job_config) # Make an API request.
37+
38+
# Check on the progress by getting the job's updated state. Once the state
39+
# is `DONE`, the results are ready.
40+
query_job = client.get_job(
41+
query_job.job_id, location=query_job.location
42+
) # Make an API request.
43+
44+
print("Job {} is currently in state {}".format(query_job.job_id, query_job.state))
45+
# [END bigquery_query_batch]
46+
return query_job

‎bigquery/samples/client_query_destination_table.py

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Copyright 2019 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
def client_query_destination_table(client, table_id):
17+
18+
# [START bigquery_query_destination_table]
19+
from google.cloud import bigquery
20+
21+
# TODO(developer): Construct a BigQuery client object.
22+
# client = bigquery.Client()
23+
24+
# TODO(developer): Set table_id to the ID of the destination table.
25+
# table_id = "your-project.your_dataset.your_table_name"
26+
27+
job_config = bigquery.QueryJobConfig(destination=table_id)
28+
29+
sql = """
30+
SELECT corpus
31+
FROM `bigquery-public-data.samples.shakespeare`
32+
GROUP BY corpus;
33+
"""
34+
35+
# Start the query, passing in the extra configuration.
36+
query_job = client.query(sql, job_config=job_config) # Make an API request.
37+
query_job.result() # Wait for the job to complete.
38+
39+
print("Query results loaded to the table {}".format(table_id))
40+
# [END bigquery_query_destination_table]

‎bigquery/samples/client_query_destination_table_cmek.py

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Copyright 2019 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
def client_query_destination_table_cmek(client, table_id, kms_key_name):
17+
18+
# [START bigquery_query_destination_table_cmek]
19+
from google.cloud import bigquery
20+
21+
# TODO(developer): Construct a BigQuery client object.
22+
# client = bigquery.Client()
23+
24+
# TODO(developer): Set table_id to the ID of the destination table.
25+
# table_id = "your-project.your_dataset.your_table_name"
26+
27+
# Set the encryption key to use for the destination.
28+
# TODO(developer): Replace this key with a key you have created in KMS.
29+
# kms_key_name = "projects/{}/locations/{}/keyRings/{}/cryptoKeys/{}".format(
30+
# your-project, location, your-ring, your-key
31+
# )
32+
33+
job_config = bigquery.QueryJobConfig(
34+
destination=table_id,
35+
destination_encryption_configuration=bigquery.EncryptionConfiguration(
36+
kms_key_name=kms_key_name
37+
),
38+
)
39+
40+
# Start the query, passing in the extra configuration.
41+
query_job = client.query(
42+
"SELECT 17 AS my_col;", job_config=job_config
43+
) # Make an API request.
44+
query_job.result() # Wait for the job to complete.
45+
46+
table = client.get_table(table_id) # Make an API request.
47+
if table.encryption_configuration.kms_key_name == kms_key_name:
48+
print("The destination table is written using the encryption configuration")
49+
# [END bigquery_query_destination_table_cmek]

‎bigquery/samples/client_query_destination_table_legacy.py

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# Copyright 2019 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
def client_query_destination_table_legacy(client, table_id):
17+
18+
# [START bigquery_query_legacy_large_results]
19+
from google.cloud import bigquery
20+
21+
# TODO(developer): Construct a BigQuery client object.
22+
# client = bigquery.Client()
23+
24+
# TODO(developer): Set table_id to the ID of the destination table.
25+
# table_id = "your-project.your_dataset.your_table_name"
26+
27+
# Set the destination table and use_legacy_sql to True to use
28+
# legacy SQL syntax.
29+
job_config = bigquery.QueryJobConfig(
30+
allow_large_results=True, destination=table_id, use_legacy_sql=True
31+
)
32+
33+
sql = """
34+
SELECT corpus
35+
FROM [bigquery-public-data:samples.shakespeare]
36+
GROUP BY corpus;
37+
"""
38+
39+
# Start the query, passing in the extra configuration.
40+
query_job = client.query(sql, job_config=job_config) # Make an API request.
41+
query_job.result() # Wait for the job to complete.
42+
43+
print("Query results loaded to the table {}".format(table_id))
44+
# [END bigquery_query_legacy_large_results]

‎bigquery/samples/client_query_dry_run.py

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Copyright 2019 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
def client_query_dry_run(client):
17+
18+
# [START bigquery_query_dry_run]
19+
from google.cloud import bigquery
20+
21+
# TODO(developer): Construct a BigQuery client object.
22+
# client = bigquery.Client()
23+
24+
job_config = bigquery.QueryJobConfig(dry_run=True, use_query_cache=False)
25+
26+
# Start the query, passing in the extra configuration.
27+
query_job = client.query(
28+
(
29+
"SELECT name, COUNT(*) as name_count "
30+
"FROM `bigquery-public-data.usa_names.usa_1910_2013` "
31+
"WHERE state = 'WA' "
32+
"GROUP BY name"
33+
),
34+
job_config=job_config,
35+
) # Make an API request.
36+
37+
# A dry run query completes immediately.
38+
print("This query will process {} bytes.".format(query_job.total_bytes_processed))
39+
# [END bigquery_query_dry_run]
40+
return query_job

‎bigquery/samples/client_query_legacy_sql.py

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Copyright 2019 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
def client_query_legacy_sql(client):
17+
18+
# [START bigquery_query_legacy]
19+
from google.cloud import bigquery
20+
21+
# TODO(developer): Construct a BigQuery client object.
22+
# client = bigquery.Client()
23+
24+
query = (
25+
"SELECT name FROM [bigquery-public-data:usa_names.usa_1910_2013] "
26+
'WHERE state = "TX" '
27+
"LIMIT 100"
28+
)
29+
30+
# Set use_legacy_sql to True to use legacy SQL syntax.
31+
job_config = bigquery.QueryJobConfig(use_legacy_sql=True)
32+
33+
# Start the query, passing in the extra configuration.
34+
query_job = client.query(query, job_config=job_config) # Make an API request.
35+
36+
print("The query data:")
37+
for row in query_job:
38+
print(row)
39+
# [END bigquery_query_legacy]

0 commit comments

Comments
 (0)