Skip to content

Commit f8a2c60

Browse files
nirupa-kumaraverikitsch
nirupa-kumar
authored and
averikitsch
committed
samples: Datalabeling beta samples (#1365)
* Datalabeling beta samples * Fixing tests * Fixing tests * Ignoring tests for now due to time duration. * Ignoring tests for now due to time duration. * Ignoring tests for now due to time duration. * debugging * debugging
1 parent 7906ac2 commit f8a2c60

15 files changed

+1433
-0
lines changed

‎datalabeling/snippets/src/main/java/com/example/datalabeling/CreateAnnotationSpecSet.java

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
/*
2+
* Copyright 2019 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example.datalabeling;
18+
19+
// [START datalabeling_create_annotation_spec_set_beta]
20+
import com.google.cloud.datalabeling.v1beta1.AnnotationSpec;
21+
import com.google.cloud.datalabeling.v1beta1.AnnotationSpecSet;
22+
import com.google.cloud.datalabeling.v1beta1.CreateAnnotationSpecSetRequest;
23+
import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceClient;
24+
import com.google.cloud.datalabeling.v1beta1.ProjectName;
25+
import java.io.IOException;
26+
import java.util.ArrayList;
27+
import java.util.HashMap;
28+
import java.util.List;
29+
import java.util.Map;
30+
import java.util.Map.Entry;
31+
32+
class CreateAnnotationSpecSet {
33+
34+
// Create an annotation spec set.
35+
static void createAnnotationSpecSet(String projectId) {
36+
// String projectId = "YOUR_PROJECT_ID";
37+
38+
Map<String, String> annotationLabels = new HashMap<>();
39+
annotationLabels.put("label_1", "label_1_description");
40+
annotationLabels.put("label_2", "label_2_description");
41+
42+
try (DataLabelingServiceClient dataLabelingServiceClient = DataLabelingServiceClient.create()) {
43+
44+
ProjectName projectName = ProjectName.of(projectId);
45+
46+
List<AnnotationSpec> annotationSpecs = new ArrayList<>();
47+
for (Entry<String, String> entry : annotationLabels.entrySet()) {
48+
AnnotationSpec annotationSpec = AnnotationSpec.newBuilder()
49+
.setDisplayName(entry.getKey())
50+
.setDescription(entry.getValue())
51+
.build();
52+
annotationSpecs.add(annotationSpec);
53+
}
54+
55+
AnnotationSpecSet annotationSpecSet = AnnotationSpecSet.newBuilder()
56+
.setDisplayName("YOUR_ANNOTATION_SPEC_SET_DISPLAY_NAME")
57+
.setDescription("YOUR_DESCRIPTION")
58+
.addAllAnnotationSpecs(annotationSpecs)
59+
.build();
60+
61+
CreateAnnotationSpecSetRequest request = CreateAnnotationSpecSetRequest.newBuilder()
62+
.setAnnotationSpecSet(annotationSpecSet)
63+
.setParent(projectName.toString())
64+
.build();
65+
66+
AnnotationSpecSet result = dataLabelingServiceClient.createAnnotationSpecSet(request);
67+
68+
System.out.format("Name: %s\n", result.getName());
69+
System.out.format("DisplayName: %s\n", result.getDisplayName());
70+
System.out.format("Description: %s\n", result.getDescription());
71+
System.out.format("Annotation Count: %d\n", result.getAnnotationSpecsCount());
72+
73+
for (AnnotationSpec annotationSpec : result.getAnnotationSpecsList()) {
74+
System.out.format("\tDisplayName: %s\n", annotationSpec.getDisplayName());
75+
System.out.format("\tDescription: %s\n\n", annotationSpec.getDescription());
76+
}
77+
} catch (IOException e) {
78+
e.printStackTrace();
79+
}
80+
}
81+
}
82+
// [END datalabeling_create_annotation_spec_set_beta]

‎datalabeling/snippets/src/main/java/com/example/datalabeling/CreateDataset.java

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*
2+
* Copyright 2019 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example.datalabeling;
18+
19+
// [START datalabeling_create_dataset_beta]
20+
import com.google.cloud.datalabeling.v1beta1.CreateDatasetRequest;
21+
import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceClient;
22+
import com.google.cloud.datalabeling.v1beta1.Dataset;
23+
import com.google.cloud.datalabeling.v1beta1.ProjectName;
24+
import java.io.IOException;
25+
26+
class CreateDataset {
27+
28+
// Create a dataset that is initially empty.
29+
static void createDataset(String projectId, String datasetName) {
30+
// String projectId = "YOUR_PROJECT_ID";
31+
// String datasetName = "YOUR_DATASET_DISPLAY_NAME";
32+
33+
try (DataLabelingServiceClient dataLabelingServiceClient = DataLabelingServiceClient.create()) {
34+
ProjectName projectName = ProjectName.of(projectId);
35+
36+
Dataset dataset = Dataset.newBuilder()
37+
.setDisplayName(datasetName)
38+
.setDescription("YOUR_DESCRIPTION")
39+
.build();
40+
41+
CreateDatasetRequest createDatasetRequest = CreateDatasetRequest.newBuilder()
42+
.setParent(projectName.toString())
43+
.setDataset(dataset)
44+
.build();
45+
46+
Dataset createdDataset = dataLabelingServiceClient.createDataset(createDatasetRequest);
47+
48+
System.out.format("Name: %s\n", createdDataset.getName());
49+
System.out.format("DisplayName: %s\n", createdDataset.getDisplayName());
50+
System.out.format("Description: %s\n", createdDataset.getDescription());
51+
} catch (IOException e) {
52+
e.printStackTrace();
53+
}
54+
}
55+
}
56+
// [END datalabeling_create_dataset_beta]

‎datalabeling/snippets/src/main/java/com/example/datalabeling/CreateInstruction.java

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
* Copyright 2019 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example.datalabeling;
18+
19+
// [START datalabeling_create_instruction_beta]
20+
import com.google.api.gax.longrunning.OperationFuture;
21+
import com.google.cloud.datalabeling.v1beta1.CreateInstructionMetadata;
22+
import com.google.cloud.datalabeling.v1beta1.CreateInstructionRequest;
23+
import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceClient;
24+
import com.google.cloud.datalabeling.v1beta1.DataType;
25+
import com.google.cloud.datalabeling.v1beta1.Instruction;
26+
import com.google.cloud.datalabeling.v1beta1.PdfInstruction;
27+
import com.google.cloud.datalabeling.v1beta1.ProjectName;
28+
import java.io.IOException;
29+
import java.util.concurrent.ExecutionException;
30+
31+
class CreateInstruction {
32+
33+
// Create a instruction for a dataset.
34+
static void createInstruction(String projectId, String pdfUri) {
35+
// String projectId = "YOUR_PROJECT_ID";
36+
// String pdfUri = "gs://YOUR_BUCKET_ID/path_to_pdf_or_csv";
37+
38+
try (DataLabelingServiceClient dataLabelingServiceClient = DataLabelingServiceClient.create()) {
39+
40+
ProjectName projectName = ProjectName.of(projectId);
41+
42+
// There are two types of instructions: CSV (CsvInstruction) or PDF (PdfInstruction)
43+
PdfInstruction pdfInstruction = PdfInstruction.newBuilder()
44+
.setGcsFileUri(pdfUri)
45+
.build();
46+
47+
Instruction instruction = Instruction.newBuilder()
48+
.setDisplayName("YOUR_INSTRUCTION_DISPLAY_NAME")
49+
.setDescription("YOUR_DESCRIPTION")
50+
.setDataType(DataType.IMAGE) // DataTypes: AUDIO, IMAGE, VIDEO, TEXT
51+
.setPdfInstruction(pdfInstruction) // .setCsvInstruction() or .setPdfInstruction()
52+
.build();
53+
54+
CreateInstructionRequest createInstructionRequest = CreateInstructionRequest.newBuilder()
55+
.setInstruction(instruction)
56+
.setParent(projectName.toString())
57+
.build();
58+
59+
OperationFuture<Instruction, CreateInstructionMetadata> operation =
60+
dataLabelingServiceClient.createInstructionAsync(createInstructionRequest);
61+
62+
Instruction result = operation.get();
63+
64+
System.out.format("Name: %s\n", result.getName());
65+
System.out.format("DisplayName: %s\n", result.getDisplayName());
66+
System.out.format("Description: %s\n", result.getDescription());
67+
System.out.format("GCS SOURCE URI: %s\n", result.getPdfInstruction().getGcsFileUri());
68+
} catch (IOException | InterruptedException | ExecutionException e) {
69+
e.printStackTrace();
70+
}
71+
}
72+
}
73+
// [END datalabeling_create_instruction_beta]

‎datalabeling/snippets/src/main/java/com/example/datalabeling/ExportData.java

+78
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/*
2+
* Copyright 2019 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example.datalabeling;
18+
19+
// [START datalabeling_export_data_beta]
20+
import com.google.api.gax.longrunning.OperationFuture;
21+
import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceClient;
22+
import com.google.cloud.datalabeling.v1beta1.ExportDataOperationMetadata;
23+
import com.google.cloud.datalabeling.v1beta1.ExportDataOperationResponse;
24+
import com.google.cloud.datalabeling.v1beta1.ExportDataRequest;
25+
import com.google.cloud.datalabeling.v1beta1.GcsDestination;
26+
import com.google.cloud.datalabeling.v1beta1.LabelStats;
27+
import com.google.cloud.datalabeling.v1beta1.OutputConfig;
28+
import java.io.IOException;
29+
import java.util.Map.Entry;
30+
import java.util.Set;
31+
import java.util.concurrent.ExecutionException;
32+
33+
class ExportData {
34+
35+
// Export data from an annotated dataset.
36+
static void exportData(String datasetName, String annotatedDatasetName, String gcsOutputUri) {
37+
// String datasetName = DataLabelingServiceClient.formatDatasetName(
38+
// "YOUR_PROJECT_ID", "YOUR_DATASETS_UUID");
39+
// String annotatedDatasetName = DataLabelingServiceClient.formatAnnotatedDatasetName(
40+
// "YOUR_PROJECT_ID",
41+
// "YOUR_DATASET_UUID",
42+
// "YOUR_ANNOTATED_DATASET_UUID");
43+
// String gcsOutputUri = "gs://YOUR_BUCKET_ID/export_path";
44+
45+
try (DataLabelingServiceClient dataLabelingServiceClient = DataLabelingServiceClient.create()) {
46+
GcsDestination gcsDestination = GcsDestination.newBuilder()
47+
.setOutputUri(gcsOutputUri)
48+
.setMimeType("text/csv")
49+
.build();
50+
51+
OutputConfig outputConfig = OutputConfig.newBuilder()
52+
.setGcsDestination(gcsDestination)
53+
.build();
54+
55+
ExportDataRequest exportDataRequest = ExportDataRequest.newBuilder()
56+
.setName(datasetName)
57+
.setOutputConfig(outputConfig)
58+
.setAnnotatedDataset(annotatedDatasetName)
59+
.build();
60+
61+
OperationFuture<ExportDataOperationResponse, ExportDataOperationMetadata> operation =
62+
dataLabelingServiceClient.exportDataAsync(exportDataRequest);
63+
64+
ExportDataOperationResponse response = operation.get();
65+
66+
System.out.format("Exported item count: %d\n", response.getExportCount());
67+
LabelStats labelStats = response.getLabelStats();
68+
Set<Entry<String, Long>> entries = labelStats.getExampleCountMap().entrySet();
69+
for (Entry<String, Long> entry : entries) {
70+
System.out.format("\tLabel: %s\n", entry.getKey());
71+
System.out.format("\tCount: %d\n\n", entry.getValue());
72+
}
73+
} catch (IOException | InterruptedException | ExecutionException e) {
74+
e.printStackTrace();
75+
}
76+
}
77+
}
78+
// [END datalabeling_export_data_beta]

‎datalabeling/snippets/src/main/java/com/example/datalabeling/ImportData.java

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
/*
2+
* Copyright 2019 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example.datalabeling;
18+
19+
// [START datalabeling_import_data_beta]
20+
import com.google.api.gax.longrunning.OperationFuture;
21+
import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceClient;
22+
import com.google.cloud.datalabeling.v1beta1.DataType;
23+
import com.google.cloud.datalabeling.v1beta1.GcsSource;
24+
import com.google.cloud.datalabeling.v1beta1.ImportDataOperationMetadata;
25+
import com.google.cloud.datalabeling.v1beta1.ImportDataOperationResponse;
26+
import com.google.cloud.datalabeling.v1beta1.ImportDataRequest;
27+
import com.google.cloud.datalabeling.v1beta1.InputConfig;
28+
import java.io.IOException;
29+
import java.util.concurrent.ExecutionException;
30+
31+
class ImportData {
32+
33+
// Import data to an existing dataset.
34+
static void importData(String datasetName, String gcsSourceUri) {
35+
// String datasetName = DataLabelingServiceClient.formatDatasetName(
36+
// "YOUR_PROJECT_ID", "YOUR_DATASETS_UUID");
37+
// String gcsSourceUri = "gs://YOUR_BUCKET_ID/path_to_data";
38+
39+
try (DataLabelingServiceClient dataLabelingServiceClient = DataLabelingServiceClient.create()) {
40+
GcsSource gcsSource = GcsSource.newBuilder()
41+
.setInputUri(gcsSourceUri)
42+
.setMimeType("text/csv")
43+
.build();
44+
45+
InputConfig inputConfig = InputConfig.newBuilder()
46+
.setDataType(DataType.IMAGE) // DataTypes: AUDIO, IMAGE, VIDEO, TEXT
47+
.setGcsSource(gcsSource)
48+
.build();
49+
50+
ImportDataRequest importDataRequest = ImportDataRequest.newBuilder()
51+
.setName(datasetName)
52+
.setInputConfig(inputConfig)
53+
.build();
54+
55+
OperationFuture<ImportDataOperationResponse, ImportDataOperationMetadata> operation =
56+
dataLabelingServiceClient.importDataAsync(importDataRequest);
57+
58+
ImportDataOperationResponse response = operation.get();
59+
60+
System.out.format("Imported items: %d\n", response.getImportCount());
61+
} catch (IOException | InterruptedException | ExecutionException e) {
62+
e.printStackTrace();
63+
}
64+
}
65+
}
66+
// [END datalabeling_import_data_beta]

0 commit comments

Comments
 (0)