Skip to content

Commit 6a68d1f

Browse files
rodireichoctavia-squidington-iii
rodireich
and
octavia-squidington-iii
authored
Map number(integer) into an integer rather than a float (#20730)
* Fix failing test * Bigquery-denormalized update spec plus destination_definitions.yaml to fix integration tests * sanity * Update version number and release note * auto-bump connector version Co-authored-by: Octavia Squidington III <[email protected]>
1 parent 06ef1e9 commit 6a68d1f

File tree

5 files changed

+20
-16
lines changed

5 files changed

+20
-16
lines changed

‎airbyte-config/init/src/main/resources/seed/destination_definitions.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@
5757
- name: BigQuery (denormalized typed struct)
5858
destinationDefinitionId: 079d5540-f236-4294-ba7c-ade8fd918496
5959
dockerRepository: airbyte/destination-bigquery-denormalized
60-
dockerImageTag: 1.2.9
60+
dockerImageTag: 1.2.10
6161
documentationUrl: https://docs.airbyte.com/integrations/destinations/bigquery
6262
icon: bigquery.svg
6363
resourceRequirements:

‎airbyte-config/init/src/main/resources/seed/destination_specs.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -831,7 +831,7 @@
831831
- "overwrite"
832832
- "append"
833833
- "append_dedup"
834-
- dockerImage: "airbyte/destination-bigquery-denormalized:1.2.9"
834+
- dockerImage: "airbyte/destination-bigquery-denormalized:1.2.10"
835835
spec:
836836
documentationUrl: "https://docs.airbyte.com/integrations/destinations/bigquery"
837837
connectionSpecification:
@@ -1025,7 +1025,7 @@
10251025
order: 5
10261026
supportsIncremental: true
10271027
supportsNormalization: false
1028-
supportsDBT: true
1028+
supportsDBT: false
10291029
supported_destination_sync_modes:
10301030
- "overwrite"
10311031
- "append"

‎airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true
1717

1818
COPY --from=build /airbyte /airbyte
1919

20-
LABEL io.airbyte.version=1.2.9
20+
LABEL io.airbyte.version=1.2.10
2121
LABEL io.airbyte.name=airbyte/destination-bigquery-denormalized

‎airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryDenormalizedRecordFormatter.java

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import java.util.concurrent.TimeUnit;
4141
import java.util.function.Consumer;
4242
import java.util.stream.Collectors;
43+
import org.apache.commons.lang3.StringUtils;
4344
import org.slf4j.Logger;
4445
import org.slf4j.LoggerFactory;
4546

@@ -68,15 +69,15 @@ private ArrayFormatter getArrayFormatter() {
6869
return arrayFormatter;
6970
}
7071

71-
public void setArrayFormatter(ArrayFormatter arrayFormatter) {
72+
public void setArrayFormatter(final ArrayFormatter arrayFormatter) {
7273
this.arrayFormatter = arrayFormatter;
7374
this.jsonSchema = formatJsonSchema(this.originalJsonSchema.deepCopy());
7475
this.bigQuerySchema = getBigQuerySchema(jsonSchema);
7576
}
7677

7778
@Override
7879
protected JsonNode formatJsonSchema(final JsonNode jsonSchema) {
79-
var modifiedJsonSchema = jsonSchema.deepCopy(); // Issue #5912 is reopened (PR #11166) formatAllOfAndAnyOfFields(namingResolver, jsonSchema);
80+
final var modifiedJsonSchema = jsonSchema.deepCopy(); // Issue #5912 is reopened (PR #11166) formatAllOfAndAnyOfFields(namingResolver, jsonSchema);
8081
getArrayFormatter().populateEmptyArrays(modifiedJsonSchema);
8182
getArrayFormatter().surroundArraysByObjects(modifiedJsonSchema);
8283
return modifiedJsonSchema;
@@ -117,7 +118,7 @@ private JsonNode formatData(final FieldList fields, final JsonNode root) {
117118
if (fields == null) {
118119
return root;
119120
}
120-
JsonNode formattedData;
121+
final JsonNode formattedData;
121122
if (root.isObject()) {
122123
formattedData = getObjectNode(fields, root);
123124
} else if (root.isArray()) {
@@ -151,7 +152,7 @@ private JsonNode getArrayNode(final FieldList fields, final JsonNode root) {
151152
} else {
152153
subFields = arrayField.getSubFields();
153154
}
154-
List<JsonNode> arrayItems = MoreIterators.toList(root.elements()).stream()
155+
final List<JsonNode> arrayItems = MoreIterators.toList(root.elements()).stream()
155156
.map(p -> formatData(subFields, p))
156157
.toList();
157158

@@ -245,15 +246,15 @@ private static JsonNode getFileDefinition(final JsonNode fieldDefinition) {
245246
}
246247

247248
private static JsonNode allOfAndAnyOfFieldProcessing(final String fieldName, final JsonNode fieldDefinition) {
248-
ObjectReader reader = mapper.readerFor(new TypeReference<List<JsonNode>>() {});
249-
List<JsonNode> list;
249+
final ObjectReader reader = mapper.readerFor(new TypeReference<List<JsonNode>>() {});
250+
final List<JsonNode> list;
250251
try {
251252
list = reader.readValue(fieldDefinition.get(fieldName));
252-
} catch (IOException e) {
253+
} catch (final IOException e) {
253254
throw new IllegalStateException(
254255
String.format("Failed to read and process the following field - %s", fieldDefinition));
255256
}
256-
ObjectNode objectNode = mapper.createObjectNode();
257+
final ObjectNode objectNode = mapper.createObjectNode();
257258
list.forEach(field -> {
258259
objectNode.set("big_query_" + field.get("type").asText(), field);
259260
});
@@ -268,8 +269,8 @@ private static JsonNode allOfAndAnyOfFieldProcessing(final String fieldName, fin
268269
private static Builder getField(final StandardNameTransformer namingResolver, final String key, final JsonNode fieldDefinition) {
269270
final String fieldName = namingResolver.getIdentifier(key);
270271
final Builder builder = Field.newBuilder(fieldName, StandardSQLTypeName.STRING);
271-
JsonNode updatedFileDefinition = getFileDefinition(fieldDefinition);
272-
JsonNode type = updatedFileDefinition.get(TYPE_FIELD);
272+
final JsonNode updatedFileDefinition = getFileDefinition(fieldDefinition);
273+
final JsonNode type = updatedFileDefinition.get(TYPE_FIELD);
273274
final JsonNode airbyteType = updatedFileDefinition.get(AIRBYTE_TYPE);
274275
final List<JsonSchemaType> fieldTypes = getTypes(fieldName, type);
275276
for (int i = 0; i < fieldTypes.size(); i++) {
@@ -288,7 +289,9 @@ private static Builder getField(final StandardNameTransformer namingResolver, fi
288289
builder.setType(primaryType.getBigQueryType());
289290
}
290291
case NUMBER -> {
291-
if (airbyteType != null && airbyteType.asText().equals("big_integer")) {
292+
if (airbyteType != null
293+
&& StringUtils.equalsAnyIgnoreCase(airbyteType.asText(),
294+
"big_integer", "integer")) {
292295
builder.setType(StandardSQLTypeName.INT64);
293296
} else {
294297
builder.setType(primaryType.getBigQueryType());

‎docs/integrations/destinations/bigquery.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ Airbyte converts any invalid characters into `_` characters when writing data. H
9898
|:------------------------------------|:--------------|:---------------------------|
9999
| DATE | DATE | DATE |
100100
| STRING (BASE64) | STRING | STRING |
101-
| NUMBER | FLOAT | FLOAT |
101+
| NUMBER | FLOAT | NUMBER |
102102
| OBJECT | STRING | RECORD |
103103
| STRING | STRING | STRING |
104104
| BOOLEAN | BOOLEAN | BOOLEAN |
@@ -191,6 +191,7 @@ Now that you have set up the BigQuery destination connector, check out the follo
191191

192192
| Version | Date | Pull Request | Subject |
193193
|:--------|:-----------|:----------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------|
194+
| 1.2.10 | 2023-01-04 | [#20730](https://github.com/airbytehq/airbyte/pull/20730) | An incoming source Number type will create a big query integer rather than a float. |
194195
| 1.2.9 | 2022-12-14 | [#20501](https://github.com/airbytehq/airbyte/pull/20501) | Report GCS staging failures that occur during connection check |
195196
| 1.2.8 | 2022-11-22 | [#19489](https://github.com/airbytehq/airbyte/pull/19489) | Added non-billable projects handle to check connection stage |
196197
| 1.2.7 | 2022-11-11 | [#19358](https://github.com/airbytehq/airbyte/pull/19358) | Fixed check method to capture mismatch dataset location |

0 commit comments

Comments
 (0)