Skip to content

Commit 00790ad

Browse files
sborisenkoxgcf-owl-bot[bot]tetiana-karasovapmichalskiGDkweinmeister
authored and
Shabirmean
committed
chore(samples): Retail Tutorials. Import products (#297)
* Configure modules settings. * Add import products samples. * Minor fixes. * Replace PROJECT_NUMBER with PROJECT_ID * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * kokoro config files are updated * * common base class for import products created to remove code duplication * inline import products now how different ids * cleaned imports * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * * removed base class and made examples self-contained * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com> Co-authored-by: tetiana-karasova <[email protected]> Co-authored-by: Piotr Michalski <[email protected]> Co-authored-by: Karl Weinmeister <[email protected]>
1 parent 7de32ef commit 00790ad

File tree

6 files changed

+597
-0
lines changed

6 files changed

+597
-0
lines changed

‎retail/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java

+114
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
/*
2+
* Copyright 2022 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
// [START retail_import_products_from_big_query]
18+
19+
/*
20+
* Import products into a catalog from big query table using Retail API
21+
*/
22+
23+
package product;
24+
25+
import com.google.cloud.retail.v2.BigQuerySource;
26+
import com.google.cloud.retail.v2.ImportMetadata;
27+
import com.google.cloud.retail.v2.ImportProductsRequest;
28+
import com.google.cloud.retail.v2.ImportProductsRequest.ReconciliationMode;
29+
import com.google.cloud.retail.v2.ImportProductsResponse;
30+
import com.google.cloud.retail.v2.ProductInputConfig;
31+
import com.google.cloud.retail.v2.ProductServiceClient;
32+
import com.google.longrunning.Operation;
33+
import com.google.longrunning.OperationsClient;
34+
import java.io.IOException;
35+
36+
public class ImportProductsBigQueryTable {
37+
38+
private static final String PROJECT_ID = System.getenv("PROJECT_ID");
39+
private static final String DEFAULT_CATALOG =
40+
String.format(
41+
"projects/%s/locations/global/catalogs/default_catalog/" + "branches/default_branch",
42+
PROJECT_ID);
43+
private static final String DATASET_ID = "products";
44+
private static final String TABLE_ID = "products";
45+
// TO CHECK ERROR HANDLING USE THE TABLE WITH INVALID PRODUCTS:
46+
// TABLE_ID = "products_some_invalid"
47+
private static final String DATA_SCHEMA = "product";
48+
49+
public static void main(String[] args) throws IOException, InterruptedException {
50+
// TRY THE FULL RECONCILIATION MODE HERE:
51+
ReconciliationMode reconciliationMode = ReconciliationMode.INCREMENTAL;
52+
ImportProductsRequest importBigQueryRequest =
53+
getImportProductsBigQueryRequest(reconciliationMode);
54+
waitForOperationCompletion(importBigQueryRequest);
55+
}
56+
57+
public static ImportProductsRequest getImportProductsBigQueryRequest(
58+
ReconciliationMode reconciliationMode) {
59+
BigQuerySource bigQuerySource =
60+
BigQuerySource.newBuilder()
61+
.setProjectId(PROJECT_ID)
62+
.setDatasetId(DATASET_ID)
63+
.setTableId(TABLE_ID)
64+
.setDataSchema(DATA_SCHEMA)
65+
.build();
66+
67+
ProductInputConfig inputConfig =
68+
ProductInputConfig.newBuilder().setBigQuerySource(bigQuerySource).build();
69+
70+
ImportProductsRequest importRequest =
71+
ImportProductsRequest.newBuilder()
72+
.setParent(DEFAULT_CATALOG)
73+
.setReconciliationMode(reconciliationMode)
74+
.setInputConfig(inputConfig)
75+
.build();
76+
System.out.printf("Import products from big query table request: %s%n", importRequest);
77+
78+
return importRequest;
79+
}
80+
81+
private static void waitForOperationCompletion(ImportProductsRequest importRequest)
82+
throws IOException, InterruptedException {
83+
try (ProductServiceClient serviceClient = ProductServiceClient.create()) {
84+
String operationName = serviceClient.importProductsCallable().call(importRequest).getName();
85+
System.out.printf("OperationName = %s\n", operationName);
86+
87+
OperationsClient operationsClient = serviceClient.getOperationsClient();
88+
Operation operation = operationsClient.getOperation(operationName);
89+
90+
while (!operation.getDone()) {
91+
// Keep polling the operation periodically until the import task is done.
92+
int awaitDuration = 30000;
93+
Thread.sleep(awaitDuration);
94+
operation = operationsClient.getOperation(operationName);
95+
}
96+
97+
if (operation.hasMetadata()) {
98+
ImportMetadata metadata = operation.getMetadata().unpack(ImportMetadata.class);
99+
System.out.printf(
100+
"Number of successfully imported products: %s\n", metadata.getSuccessCount());
101+
System.out.printf(
102+
"Number of failures during the importing: %s\n", metadata.getFailureCount());
103+
}
104+
105+
if (operation.hasResponse()) {
106+
ImportProductsResponse response =
107+
operation.getResponse().unpack(ImportProductsResponse.class);
108+
System.out.printf("Operation result: %s%n", response);
109+
}
110+
}
111+
}
112+
}
113+
114+
// [END retail_import_products_from_big_query]

‎retail/interactive-tutorials/src/main/java/product/ImportProductsGcs.java

+117
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
/*
2+
* Copyright 2022 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
// [START retail_import_products_from_gcs]
18+
19+
/*
20+
* Import products into a catalog from gcs using Retail API
21+
*/
22+
23+
package product;
24+
25+
import com.google.cloud.retail.v2.GcsSource;
26+
import com.google.cloud.retail.v2.ImportErrorsConfig;
27+
import com.google.cloud.retail.v2.ImportMetadata;
28+
import com.google.cloud.retail.v2.ImportProductsRequest;
29+
import com.google.cloud.retail.v2.ImportProductsRequest.ReconciliationMode;
30+
import com.google.cloud.retail.v2.ImportProductsResponse;
31+
import com.google.cloud.retail.v2.ProductInputConfig;
32+
import com.google.cloud.retail.v2.ProductServiceClient;
33+
import com.google.longrunning.Operation;
34+
import com.google.longrunning.OperationsClient;
35+
import java.io.IOException;
36+
import java.util.Collections;
37+
38+
public class ImportProductsGcs {
39+
40+
private static final String PROJECT_ID = System.getenv("PROJECT_ID");
41+
private static final String DEFAULT_CATALOG =
42+
String.format(
43+
"projects/%s/locations/global/catalogs/default_catalog/" + "branches/default_branch",
44+
PROJECT_ID);
45+
private static final String GCS_BUCKET = String.format("gs://%s", System.getenv("BUCKET_NAME"));
46+
private static final String GCS_ERROR_BUCKET = String.format("%s/errors", GCS_BUCKET);
47+
private static final String GCS_PRODUCTS_OBJECT = "products.json";
48+
// TO CHECK ERROR HANDLING USE THE JSON WITH INVALID PRODUCT
49+
// GCS_PRODUCTS_OBJECT = "products_some_invalid.json"
50+
51+
public static void main(String[] args) throws IOException, InterruptedException {
52+
ImportProductsRequest importGcsRequest = getImportProductsGcsRequest(GCS_PRODUCTS_OBJECT);
53+
waitForOperationCompletion(importGcsRequest);
54+
}
55+
56+
public static ImportProductsRequest getImportProductsGcsRequest(String gcsObjectName) {
57+
GcsSource gcsSource =
58+
GcsSource.newBuilder()
59+
.addAllInputUris(
60+
Collections.singleton(String.format("%s/%s", GCS_BUCKET, gcsObjectName)))
61+
.build();
62+
63+
ProductInputConfig inputConfig =
64+
ProductInputConfig.newBuilder().setGcsSource(gcsSource).build();
65+
66+
System.out.println("GRS source: " + gcsSource.getInputUrisList());
67+
68+
ImportErrorsConfig errorsConfig =
69+
ImportErrorsConfig.newBuilder().setGcsPrefix(GCS_ERROR_BUCKET).build();
70+
71+
ImportProductsRequest importRequest =
72+
ImportProductsRequest.newBuilder()
73+
.setParent(DEFAULT_CATALOG)
74+
.setReconciliationMode(ReconciliationMode.INCREMENTAL)
75+
.setInputConfig(inputConfig)
76+
.setErrorsConfig(errorsConfig)
77+
.build();
78+
79+
System.out.println("Import products from google cloud source request: " + importRequest);
80+
81+
return importRequest;
82+
}
83+
84+
private static void waitForOperationCompletion(ImportProductsRequest importRequest)
85+
throws IOException, InterruptedException {
86+
try (ProductServiceClient serviceClient = ProductServiceClient.create()) {
87+
String operationName = serviceClient.importProductsCallable().call(importRequest).getName();
88+
System.out.printf("OperationName = %s\n", operationName);
89+
90+
OperationsClient operationsClient = serviceClient.getOperationsClient();
91+
Operation operation = operationsClient.getOperation(operationName);
92+
93+
while (!operation.getDone()) {
94+
// Keep polling the operation periodically until the import task is done.
95+
int awaitDuration = 30000;
96+
Thread.sleep(awaitDuration);
97+
operation = operationsClient.getOperation(operationName);
98+
}
99+
100+
if (operation.hasMetadata()) {
101+
ImportMetadata metadata = operation.getMetadata().unpack(ImportMetadata.class);
102+
System.out.printf(
103+
"Number of successfully imported products: %s\n", metadata.getSuccessCount());
104+
System.out.printf(
105+
"Number of failures during the importing: %s\n", metadata.getFailureCount());
106+
}
107+
108+
if (operation.hasResponse()) {
109+
ImportProductsResponse response =
110+
operation.getResponse().unpack(ImportProductsResponse.class);
111+
System.out.printf("Operation result: %s%n", response);
112+
}
113+
}
114+
}
115+
}
116+
117+
// [END retail_import_products_from_gcs]

0 commit comments

Comments
 (0)