Skip to content

Commit 3e609f1

Browse files
edgaoedgao
authored
Destination snowflake: Update to latest cdk (#34612)
1 parent 18c9ebc commit 3e609f1

28 files changed

+35
-33
lines changed

‎airbyte-integrations/connectors/destination-snowflake/build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ plugins {
33
}
44

55
airbyteJavaConnector {
6-
cdkVersionRequired = '0.31.7'
6+
cdkVersionRequired = '0.33.2'
77
features = ['db-destinations', 's3-destinations', 'typing-deduping']
88
useLocalCdk = false
99
}

‎airbyte-integrations/connectors/destination-snowflake/gradle.properties

+1-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,2 @@
1-
# currently limit the number of parallel threads until further investigation into the issues \
2-
# where Snowflake will fail to login using config credentials
31
testExecutionConcurrency=-1
4-
JunitMethodExecutionTimeout=15 m
2+
JunitMethodExecutionTimeout=30 m

‎airbyte-integrations/connectors/destination-snowflake/metadata.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ data:
55
connectorSubtype: database
66
connectorType: destination
77
definitionId: 424892c4-daac-4491-b35d-c6688ba547ba
8-
dockerImageTag: 3.7.2
8+
dockerImageTag: 3.7.3
99
dockerRepository: airbyte/destination-snowflake
1010
documentationUrl: https://docs.airbyte.com/integrations/destinations/snowflake
1111
githubIssueLabel: destination-snowflake

‎airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -238,19 +238,19 @@ private Set<String> getPks(final StreamConfig stream) {
238238
: Collections.emptySet();
239239
}
240240

241-
private boolean isAirbyteRawIdColumnMatch(final TableDefinition existingTable) {
241+
protected boolean isAirbyteRawIdColumnMatch(final TableDefinition existingTable) {
242242
final String abRawIdColumnName = COLUMN_NAME_AB_RAW_ID.toUpperCase();
243243
return existingTable.columns().containsKey(abRawIdColumnName) &&
244244
toJdbcTypeName(AirbyteProtocolType.STRING).equals(existingTable.columns().get(abRawIdColumnName).getType());
245245
}
246246

247-
private boolean isAirbyteExtractedAtColumnMatch(final TableDefinition existingTable) {
247+
protected boolean isAirbyteExtractedAtColumnMatch(final TableDefinition existingTable) {
248248
final String abExtractedAtColumnName = COLUMN_NAME_AB_EXTRACTED_AT.toUpperCase();
249249
return existingTable.columns().containsKey(abExtractedAtColumnName) &&
250250
toJdbcTypeName(AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE).equals(existingTable.columns().get(abExtractedAtColumnName).getType());
251251
}
252252

253-
private boolean isAirbyteMetaColumnMatch(TableDefinition existingTable) {
253+
protected boolean isAirbyteMetaColumnMatch(TableDefinition existingTable) {
254254
final String abMetaColumnName = COLUMN_NAME_AB_META.toUpperCase();
255255
return existingTable.columns().containsKey(abMetaColumnName) &&
256256
"VARIANT".equals(existingTable.columns().get(abMetaColumnName).getType());

‎airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorIntegrationTest.java

+7-2
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,11 @@ public class SnowflakeSqlGeneratorIntegrationTest extends BaseSqlGeneratorIntegr
5454
private static JdbcDatabase database;
5555
private static DataSource dataSource;
5656

57+
@Override
58+
protected boolean getSupportsSafeCast() {
59+
return true;
60+
}
61+
5762
@BeforeAll
5863
public static void setupSnowflake() {
5964
final JsonNode config = Jsons.deserialize(IOs.readFile(Path.of("secrets/1s1t_internal_staging_config.json")));
@@ -371,8 +376,8 @@ protected void migrationAssertions(final List<? extends JsonNode> v1RawRecords,
371376
record -> record.get(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID).asText(),
372377
Function.identity()));
373378
assertAll(
374-
() -> assertEquals(6, v1RawRecords.size()),
375-
() -> assertEquals(6, v2RawRecords.size()));
379+
() -> assertEquals(5, v1RawRecords.size()),
380+
() -> assertEquals(5, v2RawRecords.size()));
376381
v1RawRecords.forEach(v1Record -> {
377382
final var v1id = v1Record.get(JavaBaseConstants.COLUMN_NAME_AB_ID.toUpperCase()).asText();
378383
assertAll(

‎airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/ltz_extracted_at_sync1_expectedrecords_dedup_final.jsonl

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
// Note the -08:00 offset in extracted_at.
22
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-01T00:01:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "Los Angeles", "state": "CA"}}
33
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 201, "UPDATED_AT": "2000-01-01T00:02:00.000000000Z", "NAME": "Bob", "ADDRESS": {"city": "Boston", "state": "MA"}}
4-
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "ID1": 2, "ID2": 200, "UPDATED_AT": "2000-01-01T00:03:00.000000000Z", "NAME": "Charlie"}
4+
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 2, "ID2": 200, "UPDATED_AT": "2000-01-01T00:03:00.000000000Z", "NAME": "Charlie", "AGE": 42, "REGISTRATION_DATE": "2023-12-23"}
55
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 3, "ID2": 200, "UPDATED_AT": "2000-01-01T00:04:00.000000000Z", "NAME": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}

‎airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/ltz_extracted_at_sync1_expectedrecords_raw.jsonl

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}}
33
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}}
44
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}}
5-
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}}
5+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}}
66
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}}

‎airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 200, "OLD_CURSOR": 1, "NAME": "Alice", "ADDRESS": {"city": "Los Angeles", "state": "CA"}}
22
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 201, "OLD_CURSOR": 2, "NAME": "Bob", "ADDRESS": {"city": "Boston", "state": "MA"}}
3-
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "ID1": 2, "ID2": 200, "OLD_CURSOR": 3, "NAME": "Charlie"}
3+
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":[]}, "ID1": 2, "ID2": 200, "OLD_CURSOR": 3, "NAME": "Charlie", "AGE": 42, "REGISTRATION_DATE": "2023-12-23"}

‎airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name" :"Alice", "address": {"city": "San Francisco", "state": "CA"}}}
22
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}}
33
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}}
4-
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}}
4+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}}

‎airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
// Keep the Alice record with more recent UPDATED_AT
22
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-01T00:01:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "Los Angeles", "state": "CA"}}
33
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 201, "UPDATED_AT": "2000-01-01T00:02:00.000000000Z", "NAME": "Bob", "ADDRESS": {"city": "Boston", "state": "MA"}}
4-
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "ID1": 2, "ID2": 200, "UPDATED_AT": "2000-01-01T00:03:00.000000000Z", "NAME": "Charlie"}
4+
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":[]}, "ID1": 2, "ID2": 200, "UPDATED_AT": "2000-01-01T00:03:00.000000000Z", "NAME": "Charlie", "AGE": 42, "REGISTRATION_DATE": "2023-12-23"}
55
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":[]}, "ID1": 3, "ID2": 200, "UPDATED_AT": "2000-01-01T00:04:00.000000000Z", "NAME": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}

‎airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-01T00:01:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "Los Angeles", "state": "CA"}}
33
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 201, "UPDATED_AT": "2000-01-01T00:02:00.000000000Z", "NAME": "Bob", "ADDRESS": {"city": "Boston", "state": "MA"}}
44
// Invalid columns are nulled out (i.e. SQL null, not JSON null)
5-
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "ID1": 2, "ID2": 200, "UPDATED_AT": "2000-01-01T00:03:00.000000000Z", "NAME": "Charlie"}
5+
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":[]}, "ID1": 2, "ID2": 200, "UPDATED_AT": "2000-01-01T00:03:00.000000000Z", "NAME": "Charlie", "AGE": 42, "REGISTRATION_DATE": "2023-12-23"}
66
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":[]}, "ID1": 3, "ID2": 200, "UPDATED_AT": "2000-01-01T00:04:00.000000000Z", "NAME": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}

‎airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,5 +3,5 @@
33
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}}
44
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}}
55
// Invalid data is still allowed in the raw table.
6-
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}}
6+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}}
77
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}}

‎airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:02.000000000Z", "_AIRBYTE_META":{"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-02T00:00:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "Seattle", "state": "WA"}}
22
// Charlie wasn't reemitted with UPDATED_AT, so it still has a null cursor
3-
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "ID1": 2, "ID2": 200, "NAME": "Charlie"}
3+
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":[]}, "ID1": 2, "ID2": 200, "NAME": "Charlie", "AGE": 42, "REGISTRATION_DATE": "2023-12-23"}

‎airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name" :"Alice", "address": {"city": "San Francisco", "state": "CA"}}}
22
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}}
33
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}}
4-
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}}
4+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}}
55
{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}}
66
{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}}
77
{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}}

‎airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-01T00:00:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "San Francisco", "state": "CA"}}
22
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-01T00:01:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "Los Angeles", "state": "CA"}}
33
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 201, "UPDATED_AT": "2000-01-01T00:02:00.000000000Z", "NAME": "Bob", "ADDRESS": {"city": "Boston", "state": "MA"}}
4-
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "ID1": 2, "ID2": 200, "UPDATED_AT": "2000-01-01T00:03:00.000000000Z", "NAME": "Charlie"}
4+
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":[]}, "ID1": 2, "ID2": 200, "UPDATED_AT": "2000-01-01T00:03:00.000000000Z", "NAME": "Charlie", "AGE": 42, "REGISTRATION_DATE": "2023-12-23"}
55
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":[]}, "ID1": 3, "ID2": 200, "UPDATED_AT": "2000-01-01T00:04:00.000000000Z", "NAME": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}
66

77
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:02.000000000Z", "_AIRBYTE_META":{"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-02T00:00:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "Seattle", "state": "WA"}}

‎airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:02.000000000Z", "_AIRBYTE_META":{"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-02T00:00:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "Seattle", "state": "WA"}}
22
// Delete Bob, keep Charlie
3-
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "ID1": 2, "ID2": 200, "UPDATED_AT": "2000-01-01T00:03:00.000000000Z", "NAME": "Charlie"}
3+
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":[]}, "ID1": 2, "ID2": 200, "UPDATED_AT": "2000-01-01T00:03:00.000000000Z", "NAME": "Charlie", "AGE": 42, "REGISTRATION_DATE": "2023-12-23"}
44
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"errors":[]}, "ID1": 3, "ID2": 200, "UPDATED_AT": "2000-01-01T00:04:00.000000000Z", "NAME": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}

‎airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final_mixed_tzs.jsonl

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:02.000000000Z", "_AIRBYTE_META":{"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-02T00:00:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "Seattle", "state": "WA"}}
22
// Delete Bob, keep Charlie. We continue to keep old records in PST
3-
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "ID1": 2, "ID2": 200, "UPDATED_AT": "2000-01-01T00:03:00.000000000Z", "NAME": "Charlie"}
3+
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 2, "ID2": 200, "UPDATED_AT": "2000-01-01T00:03:00.000000000Z", "NAME": "Charlie", "AGE": 42, "REGISTRATION_DATE": "2023-12-23"}
44
{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 3, "ID2": 200, "UPDATED_AT": "2000-01-01T00:04:00.000000000Z", "NAME": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}

‎airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}}
33
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}}
44
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}}
5-
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}}
5+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}}
66
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}}
77
// And append the records from the second sync
88
{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}}

0 commit comments

Comments
 (0)