Skip to content

Commit d31f0dd

Browse files
edgaogisripa
edgao
and
gisripa
authored
Destination redshift: Update to latest cdk (#34613)
Co-authored-by: Gireesh Sreepathi <[email protected]>
1 parent 0da1499 commit d31f0dd

File tree

33 files changed

+57
-48
lines changed

33 files changed

+57
-48
lines changed

‎airbyte-integrations/connectors/destination-redshift/build.gradle

+2-2
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ plugins {
44
}
55

66
airbyteJavaConnector {
7-
cdkVersionRequired = '0.29.12'
7+
cdkVersionRequired = '0.33.0'
88
features = ['db-destinations', 's3-destinations', 'typing-deduping']
99
useLocalCdk = false
1010
}
@@ -29,7 +29,7 @@ dependencies {
2929
implementation 'com.amazonaws:aws-java-sdk-s3:1.11.978'
3030
// TODO: Verify no aws sdk code is pulled by this dependency causing classpath conflicts
3131
// https://docs.aws.amazon.com/redshift/latest/mgmt/jdbc20-jdbc10-driver-differences.html
32-
implementation 'com.amazon.redshift:redshift-jdbc42:2.1.0.23'
32+
implementation 'com.amazon.redshift:redshift-jdbc42:2.1.0.26'
3333
implementation 'org.apache.commons:commons-csv:1.4'
3434
implementation 'com.github.alexmojaki:s3-stream-upload:2.2.2'
3535

‎airbyte-integrations/connectors/destination-redshift/metadata.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ data:
55
connectorSubtype: database
66
connectorType: destination
77
definitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc
8-
dockerImageTag: 2.4.3
8+
dockerImageTag: 2.5.0
99
dockerRepository: airbyte/destination-redshift
1010
documentationUrl: https://docs.airbyte.com/integrations/destinations/redshift
1111
githubIssueLabel: destination-redshift

‎airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ public static JsonNode getJdbcConfig(final JsonNode redshiftConfig) {
128128
}
129129

130130
@Override
131-
protected JdbcSqlGenerator getSqlGenerator() {
131+
protected JdbcSqlGenerator getSqlGenerator(final JsonNode config) {
132132
return new RedshiftSqlGenerator(super.getNamingResolver());
133133
}
134134

‎airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,7 @@ public JsonNode toJdbcConfig(final JsonNode config) {
179179
}
180180

181181
@Override
182-
protected JdbcSqlGenerator getSqlGenerator() {
182+
protected JdbcSqlGenerator getSqlGenerator(final JsonNode config) {
183183
return new RedshiftSqlGenerator(getNamingResolver());
184184
}
185185

@@ -271,7 +271,7 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN
271271
typerDeduper,
272272
parsedCatalog,
273273
defaultNamespace,
274-
true)
274+
JavaBaseConstants.DestinationColumns.V2_WITH_META)
275275
.setDataTransformer(getDataTransformer(parsedCatalog, defaultNamespace))
276276
.build()
277277
.createAsync();

‎airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ public String uploadRecordsToStage(final JdbcDatabase database,
9999

100100
private String putManifest(final String manifestContents, final String stagingPath) {
101101
final String manifestFilePath = stagingPath + String.format("%s.manifest", UUID.randomUUID());
102-
s3StorageOperations.uploadManifest(s3Config.getBucketName(), manifestFilePath, manifestContents);
102+
s3StorageOperations.uploadManifest(manifestFilePath, manifestContents);
103103
return manifestFilePath;
104104
}
105105

‎airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftRawTableAirbyteMetaMigration.kt

+1-1
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ class RedshiftRawTableAirbyteMetaMigration(
6464
"Executing RawTableAirbyteMetaMigration for ${stream.id.originalNamespace}.${stream.id.originalName} for real"
6565
)
6666
destinationHandler.execute(
67-
getRawTableMetaColumnAddDdl(stream.id.rawNamespace!!, stream.id.rawName!!)
67+
getRawTableMetaColumnAddDdl(stream.id.rawNamespace, stream.id.rawName)
6868
)
6969

7070
// Update the state. We didn't modify the table in a relevant way, so don't invalidate the

‎airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGenerator.java

+8-9
Original file line numberDiff line numberDiff line change
@@ -98,17 +98,17 @@ protected SQLDialect getDialect() {
9898
*/
9999

100100
@Override
101-
protected Field<?> castedField(final Field<?> field, final AirbyteType type, final String alias, final boolean useExpensiveSaferCasting) {
101+
protected Field<?> castedField(final Field<?> field, final AirbyteType type, final boolean useExpensiveSaferCasting) {
102102
if (type instanceof final AirbyteProtocolType airbyteProtocolType) {
103103
switch (airbyteProtocolType) {
104104
case STRING -> {
105105
return field(CASE_STATEMENT_SQL_TEMPLATE,
106106
jsonTypeOf(field).ne("string").and(field.isNotNull()),
107107
jsonSerialize(field),
108-
castedField(field, airbyteProtocolType, useExpensiveSaferCasting)).as(quotedName(alias));
108+
castedField(field, airbyteProtocolType, useExpensiveSaferCasting));
109109
}
110110
default -> {
111-
return castedField(field, airbyteProtocolType, useExpensiveSaferCasting).as(quotedName(alias));
111+
return castedField(field, airbyteProtocolType, useExpensiveSaferCasting);
112112
}
113113
}
114114

@@ -117,12 +117,12 @@ protected Field<?> castedField(final Field<?> field, final AirbyteType type, fin
117117
return switch (type.getTypeName()) {
118118
case Struct.TYPE, UnsupportedOneOf.TYPE -> field(CASE_STATEMENT_NO_ELSE_SQL_TEMPLATE,
119119
jsonTypeOf(field).eq("object"),
120-
cast(field, getStructType())).as(quotedName(alias));
120+
cast(field, getStructType()));
121121
case Array.TYPE -> field(CASE_STATEMENT_NO_ELSE_SQL_TEMPLATE,
122122
jsonTypeOf(field).eq("array"),
123-
cast(field, getArrayType())).as(quotedName(alias));
123+
cast(field, getArrayType()));
124124
// No nested Unions supported so this will definitely not result in infinite recursion.
125-
case Union.TYPE -> castedField(field, ((Union) type).chooseType(), alias, useExpensiveSaferCasting);
125+
case Union.TYPE -> castedField(field, ((Union) type).chooseType(), useExpensiveSaferCasting);
126126
default -> throw new IllegalArgumentException("Unsupported AirbyteType: " + type);
127127
};
128128
}
@@ -135,8 +135,7 @@ protected List<Field<?>> extractRawDataFields(final LinkedHashMap<ColumnId, Airb
135135
.map(column -> castedField(
136136
field(quotedName(COLUMN_NAME_DATA, column.getKey().getOriginalName())),
137137
column.getValue(),
138-
column.getKey().getName(),
139-
useExpensiveSaferCasting))
138+
useExpensiveSaferCasting).as(column.getKey().getName()))
140139
.collect(Collectors.toList());
141140
}
142141

@@ -176,7 +175,7 @@ Field<?> toCastingErrorCaseStmt(final ColumnId column, final AirbyteType type) {
176175
// TODO: Timestamp format issues can result in null values when cast, add regex check if destination
177176
// supports regex functions.
178177
return field(CASE_STATEMENT_SQL_TEMPLATE,
179-
field.isNotNull().and(castedField(field, type, column.getName(), true).isNull()),
178+
field.isNotNull().and(castedField(field, type, true).as(column.getName()).isNull()),
180179
function("ARRAY", getSuperType(),
181180
function("JSON_PARSE", getSuperType(), val(
182181
"{\"field\": \"" + column.getName() + "\", "

‎airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/AbstractRedshiftTypingDedupingTest.java

+7
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.jooq.DSLContext;
2828
import org.jooq.conf.Settings;
2929
import org.jooq.impl.DSL;
30+
import org.junit.jupiter.api.Disabled;
3031
import org.junit.jupiter.api.Test;
3132

3233
public abstract class AbstractRedshiftTypingDedupingTest extends JdbcTypingDedupingTest {
@@ -63,6 +64,9 @@ protected DSLContext getDslContext() {
6364
}
6465

6566
@Test
67+
@Disabled("Redshift connector 2.4.3 and below are rendered useless with "
68+
+ "Redshift cluster version https://docs.aws.amazon.com/redshift/latest/mgmt/cluster-versions.html#cluster-version-181 "
69+
+ "due to metadata calls hanging. We cannot run this test anymore")
6670
public void testRawTableMetaMigration_append() throws Exception {
6771
final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of(
6872
new ConfiguredAirbyteStream()
@@ -86,6 +90,9 @@ public void testRawTableMetaMigration_append() throws Exception {
8690
}
8791

8892
@Test
93+
@Disabled("Redshift connector 2.4.3 and below are rendered useless with "
94+
+ "Redshift cluster version https://docs.aws.amazon.com/redshift/latest/mgmt/cluster-versions.html#cluster-version-181 "
95+
+ "due to metadata calls hanging. We cannot run this test anymore")
8996
public void testRawTableMetaMigration_incrementalDedupe() throws Exception {
9097
final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of(
9198
new ConfiguredAirbyteStream()

‎airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorIntegrationTest.java

+5
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,11 @@
4848

4949
public class RedshiftSqlGeneratorIntegrationTest extends JdbcSqlGeneratorIntegrationTest<RedshiftState> {
5050

51+
@Override
52+
protected boolean getSupportsSafeCast() {
53+
return true;
54+
}
55+
5156
/**
5257
* Redshift's JDBC driver doesn't map certain data types onto {@link java.sql.JDBCType} usefully.
5358
* This class adds special handling for those types.

‎airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 1, "id2": 200, "old_cursor": 1, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}
22
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}
3-
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[{"field":"age","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"registration_date","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"}]}, "id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie"}
3+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}

‎airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name" :"Alice", "address": {"city": "San Francisco", "state": "CA"}}, "_airbyte_meta": {"changes": []}}
22
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}, "_airbyte_meta": {"changes": []}}
33
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}, "_airbyte_meta": {"changes": []}}
4-
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}, "_airbyte_meta": {"changes": []}}
4+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}, "_airbyte_meta": {"changes": []}}

‎airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
// Keep the Alice record with more recent updated_at
22
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}
33
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}
4-
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[{"field":"age","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"registration_date","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"}
4+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}
55
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}

‎airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}
33
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}
44
// Invalid columns are nulled out (i.e. SQL null, not JSON null)
5-
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[{"field":"age","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"registration_date","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"}
5+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}
66
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}

‎airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}, "_airbyte_meta": {"changes": []}}
33
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}, "_airbyte_meta": {"changes": []}}
44
// Invalid data is still allowed in the raw table.
5-
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}, "_airbyte_meta": {"changes": [{"field": "address", "change": "NULLED", "reason": "SOURCE_RETRIEVAL_ERROR"}]}}
5+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}, "_airbyte_meta": {"changes": [{"field": "address", "change": "NULLED", "reason": "SOURCE_RETRIEVAL_ERROR"}]}}
66
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}, "_airbyte_meta": {"changes": []}}

‎airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_messages_before_meta.jsonl

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,6 @@
99
// Emit a record with no _ab_cdc_deleted_at field. CDC sources typically emit an explicit null, but we should handle both cases.
1010
{"type": "RECORD", "record": {"emitted_at": 1000, "data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}}}
1111
// Emit a record with an invalid age & address nulled at source.
12-
{"type": "RECORD", "record": {"emitted_at": 1000, "data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}, "meta": {"changes": [{"field": "address", "change": "NULLED", "reason": "SOURCE_RETRIEVAL_ERROR"}]}}}
12+
{"type": "RECORD", "record": {"emitted_at": 1000, "data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}, "meta": {"changes": [{"field": "address", "change": "NULLED", "reason": "SOURCE_RETRIEVAL_ERROR"}]}}}
1313
// Emit a record with interesting characters in one of the values.
1414
{"type": "RECORD", "record": {"emitted_at": 1000, "data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}}}

‎airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}
22
// Charlie wasn't re-emitted with updated_at, so it still has a null cursor
3-
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[{"field":"age","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"registration_date","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"}]}, "id1": 2, "id2": 200, "name": "Charlie"}
3+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 2, "id2": 200, "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}

‎airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name" :"Alice", "address": {"city": "San Francisco", "state": "CA"}}, "_airbyte_meta": {"changes": []}}
22
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}, "_airbyte_meta": {"changes": []}}
33
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}, "_airbyte_meta": {"changes": []}}
4-
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}, "_airbyte_meta": {"changes": []}}
4+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}, "_airbyte_meta": {"changes": []}}
55
{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}, "_airbyte_meta": {"changes": []}}
66
{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}, "_airbyte_meta": {"changes": []}}
77
{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}, "_airbyte_meta": {"changes": []}}

‎airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}
22
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}
33
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}
4-
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[{"field":"age","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"registration_date","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"}
4+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}
55
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}
66

77
{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}

0 commit comments

Comments
 (0)