Skip to content

Commit 6640a47

Browse files
etsybaevetsybaev
authored
[888]Destination-Bigquery: fixed check method to capture mismatch dataset location (#19358)
* [888]Destination-Bigquery: fixed check method to capture mismatch dataset location
1 parent 705dd3e commit 6640a47

File tree

8 files changed

+19
-12
lines changed

8 files changed

+19
-12
lines changed

‎airbyte-config/init/src/main/resources/seed/destination_definitions.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
- name: BigQuery
2828
destinationDefinitionId: 22f6c74f-5699-40ff-833c-4a879ea40133
2929
dockerRepository: airbyte/destination-bigquery
30-
dockerImageTag: 1.2.6
30+
dockerImageTag: 1.2.7
3131
documentationUrl: https://docs.airbyte.com/integrations/destinations/bigquery
3232
icon: bigquery.svg
3333
normalizationRepository: airbyte/normalization
@@ -43,7 +43,7 @@
4343
- name: BigQuery (denormalized typed struct)
4444
destinationDefinitionId: 079d5540-f236-4294-ba7c-ade8fd918496
4545
dockerRepository: airbyte/destination-bigquery-denormalized
46-
dockerImageTag: 1.2.6
46+
dockerImageTag: 1.2.7
4747
documentationUrl: https://docs.airbyte.com/integrations/destinations/bigquery
4848
icon: bigquery.svg
4949
normalizationRepository: airbyte/normalization

‎airbyte-config/init/src/main/resources/seed/destination_specs.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -285,7 +285,7 @@
285285
supported_destination_sync_modes:
286286
- "overwrite"
287287
- "append"
288-
- dockerImage: "airbyte/destination-bigquery:1.2.6"
288+
- dockerImage: "airbyte/destination-bigquery:1.2.7"
289289
spec:
290290
documentationUrl: "https://docs.airbyte.com/integrations/destinations/bigquery"
291291
connectionSpecification:
@@ -495,7 +495,7 @@
495495
- "overwrite"
496496
- "append"
497497
- "append_dedup"
498-
- dockerImage: "airbyte/destination-bigquery-denormalized:1.2.6"
498+
- dockerImage: "airbyte/destination-bigquery-denormalized:1.2.7"
499499
spec:
500500
documentationUrl: "https://docs.airbyte.com/integrations/destinations/bigquery"
501501
connectionSpecification:

‎airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true
1717

1818
COPY --from=build /airbyte /airbyte
1919

20-
LABEL io.airbyte.version=1.2.6
20+
LABEL io.airbyte.version=1.2.7
2121
LABEL io.airbyte.name=airbyte/destination-bigquery-denormalized

‎airbyte-integrations/connectors/destination-bigquery/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true
1717

1818
COPY --from=build /airbyte /airbyte
1919

20-
LABEL io.airbyte.version=1.2.6
20+
LABEL io.airbyte.version=1.2.7
2121
LABEL io.airbyte.name=airbyte/destination-bigquery

‎airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import com.google.auth.oauth2.GoogleCredentials;
1010
import com.google.cloud.bigquery.BigQuery;
1111
import com.google.cloud.bigquery.BigQueryOptions;
12+
import com.google.cloud.bigquery.Dataset;
1213
import com.google.cloud.bigquery.Job;
1314
import com.google.cloud.bigquery.QueryJobConfiguration;
1415
import com.google.cloud.storage.Storage;
@@ -89,7 +90,10 @@ public AirbyteConnectionStatus check(final JsonNode config) {
8990

9091
BigQueryUtils.checkHasCreateAndDeleteDatasetRole(bigquery, datasetId, datasetLocation);
9192

92-
BigQueryUtils.createDataset(bigquery, datasetId, datasetLocation);
93+
final Dataset dataset = BigQueryUtils.getOrCreateDataset(bigquery, datasetId, datasetLocation);
94+
if (!dataset.getLocation().equals(datasetLocation)) {
95+
throw new ConfigErrorException("Actual dataset location doesn't match to location from config");
96+
}
9397
final QueryJobConfiguration queryConfig = QueryJobConfiguration
9498
.newBuilder(String.format("SELECT * FROM `%s.INFORMATION_SCHEMA.TABLES` LIMIT 1;", datasetId))
9599
.setUseLegacySql(false)

‎airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ public String getStagingFullPath(final String datasetId, final String stream) {
8484
public void createSchemaIfNotExists(final String datasetId, final String datasetLocation) {
8585
if (!existingSchemas.contains(datasetId)) {
8686
LOGGER.info("Creating dataset {}", datasetId);
87-
BigQueryUtils.createDataset(bigQuery, datasetId, datasetLocation);
87+
BigQueryUtils.getOrCreateDataset(bigQuery, datasetId, datasetLocation);
8888
existingSchemas.add(datasetId);
8989
}
9090
}

‎airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -102,18 +102,19 @@ public static void createSchemaAndTableIfNeeded(final BigQuery bigquery,
102102
final String datasetLocation,
103103
final Schema schema) {
104104
if (!existingSchemas.contains(schemaName)) {
105-
createDataset(bigquery, schemaName, datasetLocation);
105+
getOrCreateDataset(bigquery, schemaName, datasetLocation);
106106
existingSchemas.add(schemaName);
107107
}
108108
BigQueryUtils.createPartitionedTable(bigquery, tmpTableId, schema);
109109
}
110110

111-
public static void createDataset(final BigQuery bigquery, final String datasetId, final String datasetLocation) {
112-
final Dataset dataset = bigquery.getDataset(datasetId);
111+
public static Dataset getOrCreateDataset(final BigQuery bigquery, final String datasetId, final String datasetLocation) {
112+
Dataset dataset = bigquery.getDataset(datasetId);
113113
if (dataset == null || !dataset.exists()) {
114114
final DatasetInfo datasetInfo = DatasetInfo.newBuilder(datasetId).setLocation(datasetLocation).build();
115-
bigquery.create(datasetInfo);
115+
dataset = bigquery.create(datasetInfo);
116116
}
117+
return dataset;
117118
}
118119

119120
public static void checkHasCreateAndDeleteDatasetRole(final BigQuery bigquery, final String datasetId, final String datasetLocation) {

‎docs/integrations/destinations/bigquery.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@ Now that you have set up the BigQuery destination connector, check out the follo
136136

137137
| Version | Date | Pull Request | Subject |
138138
|:--------|:-----------|:----------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------|
139+
| 1.2.7 | 2022-11-11 | [#19358](https://github.com/airbytehq/airbyte/pull/19358) | fixed check method to capture mismatch dataset location |
139140
| 1.2.6 | 2022-11-10 | [#18554](https://github.com/airbytehq/airbyte/pull/18554) | Improve check connection method to handle more errors |
140141
| 1.2.5 | 2022-10-19 | [#18162](https://github.com/airbytehq/airbyte/pull/18162) | Improve error logs |
141142
| 1.2.4 | 2022-09-26 | [#16890](https://github.com/airbytehq/airbyte/pull/16890) | Add user-agent header |
@@ -188,6 +189,7 @@ Now that you have set up the BigQuery destination connector, check out the follo
188189

189190
| Version | Date | Pull Request | Subject |
190191
|:--------|:-----------|:----------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------|
192+
| 1.2.7 | 2022-11-11 | [#19358](https://github.com/airbytehq/airbyte/pull/19358) | fixed check method to capture mismatch dataset location |
191193
| 1.2.6 | 2022-11-10 | [#18554](https://github.com/airbytehq/airbyte/pull/18554) | Improve check connection method to handle more errors |
192194
| 1.2.5 | 2022-10-19 | [#18162](https://github.com/airbytehq/airbyte/pull/18162) | Improve error logs |
193195
| 1.2.4 | 2022-09-26 | [#16890](https://github.com/airbytehq/airbyte/pull/16890) | Add user-agent header |

0 commit comments

Comments
 (0)