Skip to content

Commit 2226a2e

Browse files
authored
Base Java S3: Update Avro TimeWithTimezone schema mapping (#21909)
* 21908 Base Java S3: Update Avro TimeWithTimezone schema mapping * 21908 Base Java S3: Formatting * 21908 Base Java S3: fix integration test gcs + S3 * 21908 Base Java S3: fix unit test * 21908 Base Java S3: fix format
1 parent f981668 commit 2226a2e

File tree

6 files changed

+16
-13
lines changed

6 files changed

+16
-13
lines changed

airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/JsonSchemaType.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ public enum JsonSchemaType {
2323
DATE_V1("WellKnownTypes.json#/definitions/Date", Schema.Type.INT),
2424
TIMESTAMP_WITH_TIMEZONE_V1("WellKnownTypes.json#/definitions/TimestampWithTimezone", Schema.Type.LONG),
2525
TIMESTAMP_WITHOUT_TIMEZONE_V1("WellKnownTypes.json#/definitions/TimestampWithoutTimezone", Schema.Type.LONG),
26-
TIME_WITH_TIMEZONE_V1("WellKnownTypes.json#/definitions/TimeWithTimezone", Schema.Type.LONG),
26+
TIME_WITH_TIMEZONE_V1("WellKnownTypes.json#/definitions/TimeWithTimezone", Schema.Type.STRING),
2727
TIME_WITHOUT_TIMEZONE_V1("WellKnownTypes.json#/definitions/TimeWithoutTimezone", Schema.Type.LONG),
2828
OBJECT("object", Schema.Type.RECORD),
2929
ARRAY("array", Schema.Type.ARRAY),

airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/JsonToAvroSchemaConverter.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -225,11 +225,12 @@ Schema parseSingleType(final String fieldName,
225225

226226
final Schema fieldSchema;
227227
switch (fieldType) {
228-
case INTEGER_V1, NUMBER_V1, BOOLEAN_V1, STRING_V1, BINARY_DATA_V1 -> fieldSchema = Schema.create(fieldType.getAvroType());
228+
case INTEGER_V1, NUMBER_V1, BOOLEAN_V1, STRING_V1, TIME_WITH_TIMEZONE_V1, BINARY_DATA_V1 -> fieldSchema =
229+
Schema.create(fieldType.getAvroType());
229230
case DATE_V1 -> fieldSchema = LogicalTypes.date().addToSchema(Schema.create(Schema.Type.INT));
230231
case TIMESTAMP_WITH_TIMEZONE_V1, TIMESTAMP_WITHOUT_TIMEZONE_V1 -> fieldSchema = LogicalTypes.timestampMicros()
231232
.addToSchema(Schema.create(Schema.Type.LONG));
232-
case TIME_WITH_TIMEZONE_V1, TIME_WITHOUT_TIMEZONE_V1 -> fieldSchema = LogicalTypes.timeMicros().addToSchema(Schema.create(Schema.Type.LONG));
233+
case TIME_WITHOUT_TIMEZONE_V1 -> fieldSchema = LogicalTypes.timeMicros().addToSchema(Schema.create(Schema.Type.LONG));
233234
case INTEGER_V0, NUMBER_V0, NUMBER_INT_V0, NUMBER_BIGINT_V0, NUMBER_FLOAT_V0, BOOLEAN_V0 -> fieldSchema =
234235
Schema.create(fieldType.getAvroType());
235236
case STRING_V0 -> {

airbyte-integrations/bases/base-java-s3/src/test/resources/parquet/json_schema_converter/type_conversion_test_cases_v1.json

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -204,11 +204,7 @@
204204
"jsonFieldSchema": {
205205
"$ref": "WellKnownTypes.json#/definitions/TimeWithTimezone"
206206
},
207-
"avroFieldType": [
208-
"null",
209-
{ "type": "long", "logicalType": "time-micros" },
210-
"string"
211-
]
207+
"avroFieldType": ["null", "string"]
212208
},
213209
{
214210
"fieldName": "array_field_without_items",

airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/comparator/AdvancedTestDataComparator.java

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -105,8 +105,10 @@ protected boolean compareJsonNodes(final JsonNode expectedValue, final JsonNode
105105
return compareDateTimeValues(expectedValue.asText(), actualValue.asText());
106106
} else if (isDateValue(expectedValue.asText())) {
107107
return compareDateValues(expectedValue.asText(), actualValue.asText());
108-
} else if (isTimeWithTimezone(expectedValue.asText()) || isTimeWithoutTimezone(expectedValue.asText())) {
109-
return compareTime(expectedValue.asText(), actualValue.asText());
108+
} else if (isTimeWithTimezone(expectedValue.asText())) {
109+
return compareTimeWithTimeZone(expectedValue.asText(), actualValue.asText());
110+
} else if (isTimeWithoutTimezone(expectedValue.asText())) {
111+
return compareTimeWithoutTimeZone(expectedValue.asText(), actualValue.asText());
110112
} else if (expectedValue.isArray()) {
111113
return compareArrays(expectedValue, actualValue);
112114
} else if (expectedValue.isObject()) {
@@ -215,7 +217,11 @@ protected boolean compareDateValues(final String airbyteMessageValue, final Stri
215217
return compareTextValues(airbyteMessageValue, destinationValue);
216218
}
217219

218-
protected boolean compareTime(final String airbyteMessageValue, final String destinationValue) {
220+
protected boolean compareTimeWithoutTimeZone(final String airbyteMessageValue, final String destinationValue) {
221+
return compareTextValues(airbyteMessageValue, destinationValue);
222+
}
223+
224+
protected boolean compareTimeWithTimeZone(final String airbyteMessageValue, final String destinationValue) {
219225
return compareTextValues(airbyteMessageValue, destinationValue);
220226
}
221227

airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroTestDataComparator.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ protected boolean compareDateTimeValues(String airbyteMessageValue, String desti
3737
}
3838

3939
@Override
40-
protected boolean compareTime(final String airbyteMessageValue, final String destinationValue) {
40+
protected boolean compareTimeWithoutTimeZone(final String airbyteMessageValue, final String destinationValue) {
4141
LocalTime destinationDate = LocalTime.ofInstant(getInstantFromEpoch(destinationValue), ZoneOffset.UTC);
4242
LocalTime expectedDate = LocalTime.parse(airbyteMessageValue, DateTimeFormatter.ISO_TIME);
4343
return expectedDate.equals(destinationDate);

airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3AvroParquetTestDataComparator.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ protected boolean compareDateTimeValues(String airbyteMessageValue, String desti
4141
}
4242

4343
@Override
44-
protected boolean compareTime(final String airbyteMessageValue, final String destinationValue) {
44+
protected boolean compareTimeWithoutTimeZone(final String airbyteMessageValue, final String destinationValue) {
4545
var destinationDate = LocalTime.ofInstant(getInstantFromEpoch(destinationValue), ZoneOffset.UTC);
4646
var expectedDate = LocalTime.parse(airbyteMessageValue, DateTimeFormatter.ISO_TIME);
4747
return expectedDate.equals(destinationDate);

0 commit comments

Comments
 (0)