Skip to content

Commit 7de32ef

Browse files
sborisenkoxgcf-owl-bot[bot]tetiana-karasovakweinmeister
authored and
Shabirmean
committed
chore(samples): Retail Tutorials. Product Setup/Cleanup test resources (#291)
* Configure modules settings. * Add resources files. * Product setup/cleanup impl. * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * Format code. * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * Replace PROJECT_NUMBER with PROJECT_ID * kokoro files updated * Change branch. * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com> Co-authored-by: tetiana-karasova <[email protected]> Co-authored-by: Karl Weinmeister <[email protected]>
1 parent 5bcd3c0 commit 7de32ef

16 files changed

+1448
-0
lines changed

‎retail/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* Copyright 2022 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package events.setup;
18+
19+
import static events.setup.EventsCreateGcsBucket.eventsCreateGcsBucketAndUploadJsonFiles;
20+
import static setup.SetupCleanup.createBqDataset;
21+
import static setup.SetupCleanup.createBqTable;
22+
import static setup.SetupCleanup.getGson;
23+
import static setup.SetupCleanup.uploadDataToBqTable;
24+
25+
import com.google.cloud.bigquery.Field;
26+
import com.google.cloud.bigquery.Schema;
27+
import java.io.BufferedReader;
28+
import java.io.FileReader;
29+
import java.io.IOException;
30+
import java.util.stream.Collectors;
31+
32+
public class EventsCreateBigQueryTable {
33+
34+
public static void createBqTableWithEvents() throws IOException {
35+
eventsCreateGcsBucketAndUploadJsonFiles();
36+
37+
String dataset = "user_events";
38+
String validEventsTable = "events";
39+
String invalidEventsTable = "events_some_invalid";
40+
String eventsSchemaFilePath = "src/main/resources/events_schema.json";
41+
String validEventsSourceFile =
42+
String.format("gs://%s/user_events.json", EventsCreateGcsBucket.getBucketName());
43+
String invalidEventsSourceFile =
44+
String.format(
45+
"gs://%s/user_events_some_invalid.json", EventsCreateGcsBucket.getBucketName());
46+
47+
BufferedReader bufferedReader = new BufferedReader(new FileReader(eventsSchemaFilePath));
48+
String jsonToString = bufferedReader.lines().collect(Collectors.joining());
49+
jsonToString = jsonToString.replace("\"fields\"", "\"subFields\"");
50+
Field[] fields = getGson().fromJson(jsonToString, Field[].class);
51+
Schema eventsSchema = Schema.of(fields);
52+
53+
createBqDataset(dataset);
54+
createBqTable(dataset, validEventsTable, eventsSchema);
55+
uploadDataToBqTable(dataset, validEventsTable, validEventsSourceFile, eventsSchema);
56+
createBqTable(dataset, invalidEventsTable, eventsSchema);
57+
uploadDataToBqTable(dataset, invalidEventsTable, invalidEventsSourceFile, eventsSchema);
58+
}
59+
}

‎retail/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Copyright 2022 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package events.setup;
18+
19+
import static setup.SetupCleanup.createBucket;
20+
import static setup.SetupCleanup.uploadObject;
21+
22+
import java.io.IOException;
23+
24+
public class EventsCreateGcsBucket {
25+
26+
private static final String BUCKET_NAME = System.getenv("BUCKET_NAME");
27+
28+
public static String getBucketName() {
29+
return BUCKET_NAME;
30+
}
31+
32+
public static void eventsCreateGcsBucketAndUploadJsonFiles() throws IOException {
33+
createBucket(BUCKET_NAME);
34+
uploadObject(BUCKET_NAME, "user_events.json", "src/main/resources/user_events.json");
35+
uploadObject(
36+
BUCKET_NAME,
37+
"user_events_some_invalid.json",
38+
"src/main/resources/user_events_some_invalid.json");
39+
}
40+
}

‎retail/interactive-tutorials/src/main/java/init/CreateTestResources.java

+114
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
/*
2+
* Copyright 2022 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package init;
18+
19+
import static events.setup.EventsCreateBigQueryTable.createBqTableWithEvents;
20+
import static events.setup.EventsCreateGcsBucket.eventsCreateGcsBucketAndUploadJsonFiles;
21+
import static product.setup.ProductsCreateBigqueryTable.createBqTableWithProducts;
22+
import static product.setup.ProductsCreateGcsBucket.productsCreateGcsBucketAndUploadJsonFiles;
23+
24+
import com.google.cloud.retail.v2.GcsSource;
25+
import com.google.cloud.retail.v2.ImportErrorsConfig;
26+
import com.google.cloud.retail.v2.ImportMetadata;
27+
import com.google.cloud.retail.v2.ImportProductsRequest;
28+
import com.google.cloud.retail.v2.ImportProductsRequest.ReconciliationMode;
29+
import com.google.cloud.retail.v2.ImportProductsResponse;
30+
import com.google.cloud.retail.v2.ProductInputConfig;
31+
import com.google.cloud.retail.v2.ProductServiceClient;
32+
import com.google.longrunning.Operation;
33+
import com.google.longrunning.OperationsClient;
34+
import java.io.IOException;
35+
import java.util.Collections;
36+
37+
public class CreateTestResources {
38+
private static final String PROJECT_ID = System.getenv("PROJECT_ID");
39+
private static final String BUCKET_NAME = System.getenv("BUCKET_NAME");
40+
private static final String GCS_BUCKET = String.format("gs://%s", System.getenv("BUCKET_NAME"));
41+
private static final String GCS_ERROR_BUCKET = String.format("%s/errors", GCS_BUCKET);
42+
private static final String DEFAULT_CATALOG =
43+
String.format(
44+
"projects/%s/locations/global/catalogs/default_catalog/" + "branches/0", PROJECT_ID);
45+
46+
public static void main(String[] args) throws IOException, InterruptedException {
47+
productsCreateGcsBucketAndUploadJsonFiles();
48+
eventsCreateGcsBucketAndUploadJsonFiles();
49+
importProductsFromGcs();
50+
createBqTableWithProducts();
51+
createBqTableWithEvents();
52+
}
53+
54+
public static ImportProductsRequest getImportProductsGcsRequest(String gcsObjectName) {
55+
GcsSource gcsSource =
56+
GcsSource.newBuilder()
57+
.addAllInputUris(
58+
Collections.singleton(String.format("gs://%s/%s", BUCKET_NAME, gcsObjectName)))
59+
.build();
60+
ProductInputConfig inputConfig =
61+
ProductInputConfig.newBuilder().setGcsSource(gcsSource).build();
62+
System.out.println("GRS source: " + gcsSource.getInputUrisList());
63+
64+
ImportErrorsConfig errorsConfig =
65+
ImportErrorsConfig.newBuilder().setGcsPrefix(GCS_ERROR_BUCKET).build();
66+
ImportProductsRequest importRequest =
67+
ImportProductsRequest.newBuilder()
68+
.setParent(DEFAULT_CATALOG)
69+
.setReconciliationMode(ReconciliationMode.INCREMENTAL)
70+
.setInputConfig(inputConfig)
71+
.setErrorsConfig(errorsConfig)
72+
.build();
73+
System.out.println("Import products from google cloud source request: " + importRequest);
74+
75+
return importRequest;
76+
}
77+
78+
public static void importProductsFromGcs() throws IOException, InterruptedException {
79+
ImportProductsRequest importGcsRequest = getImportProductsGcsRequest("products.json");
80+
81+
try (ProductServiceClient serviceClient = ProductServiceClient.create()) {
82+
String operationName =
83+
serviceClient.importProductsCallable().call(importGcsRequest).getName();
84+
System.out.printf("OperationName = %s\n", operationName);
85+
86+
OperationsClient operationsClient = serviceClient.getOperationsClient();
87+
Operation operation = operationsClient.getOperation(operationName);
88+
89+
while (!operation.getDone()) {
90+
System.out.println("Please wait till operation is completed.");
91+
// Keep polling the operation periodically until the import task is done.
92+
int awaitDuration = 30000;
93+
Thread.sleep(awaitDuration);
94+
operation = operationsClient.getOperation(operationName);
95+
}
96+
97+
System.out.println("Import products operation is completed.");
98+
99+
if (operation.hasMetadata()) {
100+
ImportMetadata metadata = operation.getMetadata().unpack(ImportMetadata.class);
101+
System.out.printf(
102+
"Number of successfully imported products: %s\n", metadata.getSuccessCount());
103+
System.out.printf(
104+
"Number of failures during the importing: %s\n", metadata.getFailureCount());
105+
}
106+
107+
if (operation.hasResponse()) {
108+
ImportProductsResponse response =
109+
operation.getResponse().unpack(ImportProductsResponse.class);
110+
System.out.printf("Operation result: %s", response);
111+
}
112+
}
113+
}
114+
}

‎retail/interactive-tutorials/src/main/java/init/RemoveTestResources.java

+72
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
/*
2+
* Copyright 2022 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package init;
18+
19+
import static setup.SetupCleanup.deleteBucket;
20+
import static setup.SetupCleanup.deleteDataset;
21+
22+
import com.google.api.gax.rpc.PermissionDeniedException;
23+
import com.google.cloud.retail.v2.DeleteProductRequest;
24+
import com.google.cloud.retail.v2.ListProductsRequest;
25+
import com.google.cloud.retail.v2.Product;
26+
import com.google.cloud.retail.v2.ProductServiceClient;
27+
import com.google.cloud.retail.v2.ProductServiceClient.ListProductsPagedResponse;
28+
import java.io.IOException;
29+
30+
public class RemoveTestResources {
31+
32+
private static final String PROJECT_ID = System.getenv("PROJECT_ID");
33+
private static final String BUCKET_NAME = System.getenv("BUCKET_NAME");
34+
private static final String DEFAULT_CATALOG =
35+
String.format(
36+
"projects/%s/locations/global/catalogs/default_catalog/" + "branches/0", PROJECT_ID);
37+
38+
public static void main(String[] args) throws IOException {
39+
deleteBucket(BUCKET_NAME);
40+
deleteAllProducts();
41+
deleteDataset(PROJECT_ID, "products");
42+
deleteDataset(PROJECT_ID, "user_events");
43+
}
44+
45+
public static void deleteAllProducts() throws IOException {
46+
System.out.println("Deleting products in process, please wait...");
47+
48+
try (ProductServiceClient productServiceClient = ProductServiceClient.create()) {
49+
ListProductsRequest listRequest =
50+
ListProductsRequest.newBuilder().setParent(DEFAULT_CATALOG).build();
51+
ListProductsPagedResponse products = productServiceClient.listProducts(listRequest);
52+
53+
int deleteCount = 0;
54+
55+
for (Product product : products.iterateAll()) {
56+
DeleteProductRequest deleteRequest =
57+
DeleteProductRequest.newBuilder().setName(product.getName()).build();
58+
59+
try {
60+
productServiceClient.deleteProduct(deleteRequest);
61+
deleteCount++;
62+
} catch (PermissionDeniedException e) {
63+
System.out.println(
64+
"Ignore PermissionDenied in case the product does not exist "
65+
+ "at time of deletion.");
66+
}
67+
}
68+
69+
System.out.printf("%s products were deleted from %s%n", deleteCount, DEFAULT_CATALOG);
70+
}
71+
}
72+
}

‎retail/interactive-tutorials/src/main/java/init/TEST_RESOURCES_SETUP_CLEANUP.md

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# How to set up/ tear down the test resources
2+
3+
## Required environment variables
4+
5+
To successfully import the catalog data for tests, the following environment variables should be
6+
set:
7+
8+
- PROJECT_ID
9+
- PROJECT_NUMBER
10+
- BUCKET_NAME
11+
12+
The Secret Manager name is set in .kokoro/presubmit/common.cfg file, SECRET_MANAGER_KEYS variable.
13+
14+
## Import catalog data
15+
16+
There is a JSON file with valid products prepared in the `product` directory:
17+
`resources/products.json`.
18+
19+
Run the `CreateTestResources` to perform the following actions:
20+
21+
- create the GCS bucket <BUCKET_NAME>;
22+
- upload the product data from `resources/products.json` file to the bucket;
23+
- import products to the default branch of the Retail catalog;
24+
- upload the product data from `resources/user_events.json` file to the bucket;
25+
- create a BigQuery dataset and table `products`;
26+
- insert products from resources/products.json to the created products table;
27+
- create a BigQuery dataset and table `events`;
28+
- insert user events from resources/user_events.json to the created events table;
29+
30+
```
31+
mvn compile exec:java -Dexec.mainClass="init.CreateTestResources"
32+
```
33+
34+
In the result 316 products should be created in the test project catalog.
35+
36+
## Remove catalog data
37+
38+
Run the `RemoveTestResources` to perform the following actions:
39+
40+
- remove all objects from the GCS bucket <BUCKET_NAME>;
41+
- remove the <BUCKET_NAME> bucket;
42+
- delete all products from the Retail catalog;
43+
- remove all objects from the GCS bucket <BUCKET_NAME>;
44+
- remove dataset `products` along with tables;
45+
- remove dataset `user_events` along with tables;
46+
47+
```
48+
mvn compile exec:java -Dexec.mainClass="init.RemoveTestResources"
49+
```

‎retail/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* Copyright 2022 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package product.setup;
18+
19+
import static product.setup.ProductsCreateGcsBucket.productsCreateGcsBucketAndUploadJsonFiles;
20+
import static setup.SetupCleanup.createBqDataset;
21+
import static setup.SetupCleanup.createBqTable;
22+
import static setup.SetupCleanup.getGson;
23+
import static setup.SetupCleanup.uploadDataToBqTable;
24+
25+
import com.google.cloud.bigquery.Field;
26+
import com.google.cloud.bigquery.Schema;
27+
import java.io.BufferedReader;
28+
import java.io.FileReader;
29+
import java.io.IOException;
30+
import java.util.stream.Collectors;
31+
32+
public class ProductsCreateBigqueryTable {
33+
34+
public static void createBqTableWithProducts() throws IOException {
35+
productsCreateGcsBucketAndUploadJsonFiles();
36+
37+
String dataset = "products";
38+
String validProductsTable = "products";
39+
String invalidProductsTable = "products_some_invalid";
40+
String productSchemaFilePath = "src/main/resources/product_schema.json";
41+
String validProductsSourceFile =
42+
String.format("gs://%s/products.json", ProductsCreateGcsBucket.getBucketName());
43+
String invalidProductsSourceFile =
44+
String.format(
45+
"gs://%s/products_some_invalid.json", ProductsCreateGcsBucket.getBucketName());
46+
47+
BufferedReader bufferedReader = new BufferedReader(new FileReader(productSchemaFilePath));
48+
String jsonToString = bufferedReader.lines().collect(Collectors.joining());
49+
jsonToString = jsonToString.replace("\"fields\"", "\"subFields\"");
50+
Field[] fields = getGson().fromJson(jsonToString, Field[].class);
51+
Schema productSchema = Schema.of(fields);
52+
53+
createBqDataset(dataset);
54+
createBqTable(dataset, validProductsTable, productSchema);
55+
uploadDataToBqTable(dataset, validProductsTable, validProductsSourceFile, productSchema);
56+
createBqTable(dataset, invalidProductsTable, productSchema);
57+
uploadDataToBqTable(dataset, invalidProductsTable, invalidProductsSourceFile, productSchema);
58+
}
59+
}

0 commit comments

Comments
 (0)