Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(bigquery): update code samples to use strings for table and dataset IDs #9495

Merged
merged 18 commits into from
Dec 2, 2019
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion bigquery/samples/client_query_batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,4 +42,4 @@ def client_query_batch(client):

print("Job {} is currently in state {}".format(query_job.job_id, query_job.state))
# [END bigquery_query_batch]
return query_job.job_id
return query_job
14 changes: 7 additions & 7 deletions bigquery/samples/client_query_destination_table_cmek.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
# limitations under the License.


def client_query_destination_table_cmek(client, table_id):
def client_query_destination_table_cmek(client, table_id, kms_key_name):

# [START bigquery_query_destination_table_cmek]
from google.cloud import bigquery
Expand All @@ -24,14 +24,14 @@ def client_query_destination_table_cmek(client, table_id):
# TODO(developer): Set table_id to the ID of the destination table.
# table_id = "your-project.your_dataset.your_table_name"

job_config = bigquery.QueryJobConfig()
job_config.destination = table_id

# Set the encryption key to use for the destination.
# TODO(developer): Replace this key with a key you have created in KMS.
kms_key_name = "projects/{}/locations/{}/keyRings/{}/cryptoKeys/{}".format(
"cloud-samples-tests", "us", "test", "test"
)
# kms_key_name = "projects/{}/locations/{}/keyRings/{}/cryptoKeys/{}".format(
# your-project, location, your-ring, your-key
# )

job_config = bigquery.QueryJobConfig()
job_config.destination = table_id

encryption_config = bigquery.EncryptionConfiguration(kms_key_name=kms_key_name)
job_config.destination_encryption_configuration = encryption_config
Expand Down
8 changes: 4 additions & 4 deletions bigquery/samples/copy_table_cmek.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
# limitations under the License.


def copy_table_cmek(client, dest_table_id, orig_table_id):
def copy_table_cmek(client, dest_table_id, orig_table_id, kms_key_name):

# [START bigquery_copy_table_cmek]
from google.cloud import bigquery
Expand All @@ -29,9 +29,9 @@ def copy_table_cmek(client, dest_table_id, orig_table_id):

# Set the encryption key to use for the destination.
# TODO(developer): Replace this key with a key you have created in KMS.
kms_key_name = "projects/{}/locations/{}/keyRings/{}/cryptoKeys/{}".format(
"cloud-samples-tests", "us", "test", "test"
)
# kms_key_name = "projects/{}/locations/{}/keyRings/{}/cryptoKeys/{}".format(
# your-project, location, your-ring, your-key
# )

encryption_config = bigquery.EncryptionConfiguration(kms_key_name=kms_key_name)
job_config = bigquery.CopyJobConfig()
Expand Down
10 changes: 5 additions & 5 deletions bigquery/samples/copy_table_multiple_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
# limitations under the License.


def copy_table_multiple_source(client, dest_table_id, tables_ids):
def copy_table_multiple_source(client, dest_table_id, table_ids):

# [START bigquery_copy_table_multiple_source]
# TODO(developer): Import the client library.
Expand All @@ -25,15 +25,15 @@ def copy_table_multiple_source(client, dest_table_id, tables_ids):
# TODO(developer): Set dest_table_id to the ID of the destination table.
# dest_table_id = "your-project.your_dataset.your_table_name"

# TODO(developer): Set tables_ids to the list of the IDs of the original tables.
# tables_ids = ["your-project.your_dataset.your_table_name", ...]
# TODO(developer): Set table_ids to the list of the IDs of the original tables.
# table_ids = ["your-project.your_dataset.your_table_name", ...]

job = client.copy_table(
tables_ids,
table_ids,
dest_table_id,
location="US", # Must match the source and the destination dataset(s) location.
) # Make an API request.
job.result() # Wait for the job to complete.

print("A copy of {} tables has been created".format(len(tables_ids)))
print("The tables {} have been appended to {}".format(table_ids, dest_table_id))
# [END bigquery_copy_table_multiple_source]
5 changes: 5 additions & 0 deletions bigquery/samples/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,3 +132,8 @@ def model_id(client, dataset_id):

client.query(sql).result()
return model_id


@pytest.fixture
def kms_key_name():
return "projects/cloud-samples-tests/locations/us/keyRings/test/cryptoKeys/test"
4 changes: 2 additions & 2 deletions bigquery/samples/tests/test_client_query_batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,6 @@

def test_client_query_batch(capsys, client):

job_id = client_query_batch.client_query_batch(client)
job = client_query_batch.client_query_batch(client)
out, err = capsys.readouterr()
assert "Job {} is currently in state DONE".format(job_id) in out
assert "Job {} is currently in state {}".format(job.job_id, job.state) in out
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,12 @@
from .. import client_query_destination_table_cmek


def test_client_query_destination_table_cmek(capsys, client, random_table_id):
def test_client_query_destination_table_cmek(
capsys, client, random_table_id, kms_key_name
):

client_query_destination_table_cmek.client_query_destination_table_cmek(
client, random_table_id
client, random_table_id, kms_key_name
)
out, err = capsys.readouterr()
assert "The destination table is written using the encryption configuration" in out
2 changes: 1 addition & 1 deletion bigquery/samples/tests/test_client_query_dry_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,6 @@ def test_client_query_dry_run(capsys, client):

query_job = client_query_dry_run.client_query_dry_run(client)
out, err = capsys.readouterr()
assert "This query will process 65935918 bytes." in out
assert "This query will process 0 bytes." not in out
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I got confused by this test.

I'd very much like to see a positive assertion that a non-zero number is present. not in could happen for many different reasons.

Suggested change
assert "This query will process 0 bytes." not in out
assert "This query will process" in out
assert query_job.total_bytes_processed > 0

assert query_job.state == "DONE"
assert query_job.dry_run
2 changes: 1 addition & 1 deletion bigquery/samples/tests/test_client_query_legacy_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,4 @@ def test_client_query_legacy_sql(capsys, client):

client_query_legacy_sql.client_query_legacy_sql(client)
out, err = capsys.readouterr()
assert "Row(('Frances',), {'name': 0})" in out
assert "Row((" in out
8 changes: 6 additions & 2 deletions bigquery/samples/tests/test_copy_table_cmek.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,12 @@
from .. import copy_table_cmek


def test_copy_table_cmek(capsys, client, random_table_id, table_with_data_id):
def test_copy_table_cmek(
capsys, client, random_table_id, table_with_data_id, kms_key_name
):

copy_table_cmek.copy_table_cmek(client, random_table_id, table_with_data_id)
copy_table_cmek.copy_table_cmek(
client, random_table_id, table_with_data_id, kms_key_name
)
out, err = capsys.readouterr()
assert "A copy of the table created" in out
9 changes: 6 additions & 3 deletions bigquery/samples/tests/test_copy_table_multiple_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,16 @@ def test_copy_table_multiple_source(capsys, client, random_table_id, random_data
body, table_ref, location="US", job_config=job_config
).result()

tables_ids = [
table_ids = [
"{}.table1".format(random_dataset_id),
"{}.table2".format(random_dataset_id),
]

copy_table_multiple_source.copy_table_multiple_source(
client, random_table_id, tables_ids
client, random_table_id, table_ids
)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's call dest_table = client.get_table(random_table_id) and then assert dest_table.num_rows > 0 so that we know that the job did what we expected it to do.

out, err = capsys.readouterr()
assert "A copy of 2 tables has been created" in out
assert (
"The tables {} have been appended to {}".format(table_ids, random_table_id)
in out
)