Skip to content
This repository was archived by the owner on Oct 3, 2023. It is now read-only.

fix(samples): removed env variables and buckets from creating bq. #526

Merged
merged 8 commits into from
Sep 16, 2022
Merged

fix(samples): removed env variables and buckets from creating bq. #526

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
b9f8aa9
Removed env variables and buckets from creating BQ.
dfirova Aug 31, 2022
548b287
Fix: removed buckets from BQ import
dfirova Aug 31, 2022
79409a8
Merge remote-tracking branch 'origin/GRST-258-remove-buckets' into GR…
dfirova Sep 1, 2022
622d0c1
pr fix: imports.
dfirova Sep 5, 2022
ad1e7dc
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Sep 12, 2022
d7113c9
Merge branch 'main' into GRST-258-remove-buckets
dfirova Sep 14, 2022
e3bad7f
pr fix: fixed test.
dfirova Sep 15, 2022
ee334ac
pr fix: added comment.
dfirova Sep 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@

import com.google.cloud.bigquery.Field;
import com.google.cloud.bigquery.Schema;
import product.setup.ProductsCreateBigqueryTable;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
Expand All @@ -35,11 +37,9 @@ public static void main(String... args) throws IOException {
String validEventsTable = "events";
String invalidEventsTable = "events_some_invalid";
String eventsSchemaFilePath = "src/main/resources/events_schema.json";
String validEventsSourceFile =
String.format("gs://%s/user_events.json", EventsCreateGcsBucket.getBucketName());
String validEventsSourceFile = ProductsCreateBigqueryTable.class.getResource("/user_events.json").getPath();
String invalidEventsSourceFile =
String.format(
"gs://%s/user_events_some_invalid.json", EventsCreateGcsBucket.getBucketName());
ProductsCreateBigqueryTable.class.getResource("/user_events_some_invalid.json").getPath();

BufferedReader bufferedReader = new BufferedReader(new FileReader(eventsSchemaFilePath));
String jsonToString = bufferedReader.lines().collect(Collectors.joining());
Expand All @@ -49,8 +49,8 @@ public static void main(String... args) throws IOException {

createBqDataset(dataset);
createBqTable(dataset, validEventsTable, eventsSchema);
uploadDataToBqTable(dataset, validEventsTable, validEventsSourceFile, eventsSchema);
uploadDataToBqTable(dataset, validEventsTable, validEventsSourceFile);
createBqTable(dataset, invalidEventsTable, eventsSchema);
uploadDataToBqTable(dataset, invalidEventsTable, invalidEventsSourceFile, eventsSchema);
uploadDataToBqTable(dataset, invalidEventsTable, invalidEventsSourceFile);
}
}
10 changes: 4 additions & 6 deletions samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,9 @@ public static void main(String... args) throws IOException {
String validProductsTable = "products";
String invalidProductsTable = "products_some_invalid";
String productSchemaFilePath = "src/main/resources/product_schema.json";
String validProductsSourceFile =
String.format("gs://%s/products.json", ProductsCreateGcsBucket.getBucketName());
String validProductsSourceFile = ProductsCreateBigqueryTable.class.getResource("/products.json").getPath();
String invalidProductsSourceFile =
String.format(
"gs://%s/products_some_invalid.json", ProductsCreateGcsBucket.getBucketName());
ProductsCreateBigqueryTable.class.getResource("products_some_invalid.json").getPath();

BufferedReader bufferedReader = new BufferedReader(new FileReader(productSchemaFilePath));
String jsonToString = bufferedReader.lines().collect(Collectors.joining());
Expand All @@ -49,8 +47,8 @@ public static void main(String... args) throws IOException {

createBqDataset(dataset);
createBqTable(dataset, validProductsTable, productSchema);
uploadDataToBqTable(dataset, validProductsTable, validProductsSourceFile, productSchema);
uploadDataToBqTable(dataset, validProductsTable, validProductsSourceFile);
createBqTable(dataset, invalidProductsTable, productSchema);
uploadDataToBqTable(dataset, invalidProductsTable, invalidProductsSourceFile, productSchema);
uploadDataToBqTable(dataset, invalidProductsTable, invalidProductsSourceFile);
}
}
53 changes: 24 additions & 29 deletions samples/interactive-tutorials/src/main/java/setup/SetupCleanup.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,25 +22,8 @@
import com.google.api.gax.paging.Page;
import com.google.api.gax.rpc.NotFoundException;
import com.google.cloud.ServiceOptions;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.*;
import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption;
import com.google.cloud.bigquery.BigQueryException;
import com.google.cloud.bigquery.BigQueryOptions;
import com.google.cloud.bigquery.Dataset;
import com.google.cloud.bigquery.DatasetId;
import com.google.cloud.bigquery.DatasetInfo;
import com.google.cloud.bigquery.Field;
import com.google.cloud.bigquery.FieldList;
import com.google.cloud.bigquery.FormatOptions;
import com.google.cloud.bigquery.Job;
import com.google.cloud.bigquery.JobInfo;
import com.google.cloud.bigquery.LegacySQLTypeName;
import com.google.cloud.bigquery.LoadJobConfiguration;
import com.google.cloud.bigquery.Schema;
import com.google.cloud.bigquery.StandardTableDefinition;
import com.google.cloud.bigquery.TableDefinition;
import com.google.cloud.bigquery.TableId;
import com.google.cloud.bigquery.TableInfo;
import com.google.cloud.retail.v2.CreateProductRequest;
import com.google.cloud.retail.v2.DeleteProductRequest;
import com.google.cloud.retail.v2.FulfillmentInfo;
Expand Down Expand Up @@ -71,7 +54,10 @@
import com.google.protobuf.Int32Value;
import com.google.protobuf.Timestamp;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.channels.Channels;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Instant;
import java.util.Arrays;
Expand Down Expand Up @@ -350,26 +336,35 @@ public static void createBqTable(String datasetName, String tableName, Schema sc
}

public static void uploadDataToBqTable(
String datasetName, String tableName, String sourceUri, Schema schema) {
String datasetName, String tableName, String sourceUri) {
try {
BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService();
TableId tableId = TableId.of(datasetName, tableName);
LoadJobConfiguration loadConfig =
LoadJobConfiguration.newBuilder(tableId, sourceUri)
.setFormatOptions(FormatOptions.json())
.setSchema(schema)
.build();
Job job = bigquery.create(JobInfo.of(loadConfig));
job = job.waitFor();

WriteChannelConfiguration writeChannelConfiguration =
WriteChannelConfiguration.newBuilder(tableId).setFormatOptions(FormatOptions.json()).build();

String jobName = "jobId_" + UUID.randomUUID();
JobId jobId = JobId.newBuilder().setLocation("us").setJob(jobName).build();

try (TableDataWriteChannel writer = bigquery.writer(jobId, writeChannelConfiguration);
OutputStream stream = Channels.newOutputStream(writer)) {
Files.copy(Paths.get(sourceUri), stream);
}

Job job = bigquery.getJob(jobId);
Job completedJob = job.waitFor();
if (job.isDone()) {
System.out.printf("Json from GCS successfully loaded in a table '%s'.%n", tableName);
System.out.printf("Json successfully loaded in a table '%s'.%n", tableName);
} else {
System.out.println(
"BigQuery was unable to load into the table due to an error:"
+ job.getStatus().getError());
"BigQuery was unable to load into the table due to an error:"
+ job.getStatus().getError());
}
} catch (BigQueryException | InterruptedException e) {
System.out.printf("Column not added during load append: %s%n", e.getMessage());
} catch (IOException e) {
System.out.printf("Error copying file: %s%n", e.getMessage());
}
}

Expand Down