Skip to content

Commit 24a2cef

Browse files
gisripaMarius Postaevantahler
authored and
jatinyadav-cc
committed
✨ Destination Postgres: DV2 GA (airbytehq#35042)
Co-authored-by: Marius Posta <[email protected]> Co-authored-by: Evan Tahler <[email protected]>
1 parent 279aa0f commit 24a2cef

File tree

49 files changed

+524
-264
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+524
-264
lines changed

‎airbyte-cdk/java/airbyte-cdk/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,7 @@ MavenLocal debugging steps:
166166

167167
| Version | Date | Pull Request | Subject |
168168
|:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------|
169+
| 0.20.4 | 2024-02-12 | [\#35042](https://github.com/airbytehq/airbyte/pull/35042) | Use delegate's isDestinationV2 invocation in SshWrappedDestination. |
169170
| 0.20.3 | 2024-02-09 | [\#34580](https://github.com/airbytehq/airbyte/pull/34580) | Support special chars in mysql/mssql database name. |
170171
| 0.20.2 | 2024-02-12 | [\#35111](https://github.com/airbytehq/airbyte/pull/35144) | Make state emission from async framework synchronized. |
171172
| 0.20.1 | 2024-02-11 | [\#35111](https://github.com/airbytehq/airbyte/pull/35111) | Fix GlobalAsyncStateManager stats counting logic. |

‎airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshWrappedDestination.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,4 +129,9 @@ protected SshTunnel getTunnelInstance(final JsonNode config) throws Exception {
129129
: getInstance(config, hostKey, portKey);
130130
}
131131

132+
@Override
133+
public boolean isV2Destination() {
134+
return delegate.isV2Destination();
135+
}
136+
132137
}

‎airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
version=0.20.3
1+
version=0.20.4

‎airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle

Lines changed: 2 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,48 +1,20 @@
11
plugins {
2-
id 'application'
32
id 'airbyte-java-connector'
43
}
54

65
airbyteJavaConnector {
7-
cdkVersionRequired = '0.16.3'
8-
features = [
9-
'db-sources', // required for tests
10-
'db-destinations'
11-
]
6+
cdkVersionRequired = '0.20.4'
7+
features = ['db-destinations', 'typing-deduping', 'datastore-postgres']
128
useLocalCdk = false
139
}
1410

15-
//remove once upgrading the CDK version to 0.4.x or later
16-
java {
17-
compileJava {
18-
options.compilerArgs.remove("-Werror")
19-
}
20-
}
21-
22-
airbyteJavaConnector.addCdkDependencies()
23-
2411
application {
2512
mainClass = 'io.airbyte.integrations.destination.postgres.PostgresDestinationStrictEncrypt'
2613
applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
2714
}
2815

2916
dependencies {
3017
implementation project(':airbyte-integrations:connectors:destination-postgres')
31-
// TODO: declare typing-deduping as a CDK feature instead of importing from source.
32-
implementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping')
33-
34-
testImplementation libs.testcontainers.jdbc
35-
testImplementation libs.testcontainers.postgresql
36-
37-
testFixturesImplementation libs.testcontainers.jdbc
38-
testFixturesImplementation libs.testcontainers.postgresql
3918

4019
integrationTestJavaImplementation testFixtures(project(':airbyte-integrations:connectors:destination-postgres'))
41-
integrationTestJavaImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:typing-deduping'))
42-
}
43-
44-
configurations.all {
45-
resolutionStrategy {
46-
force libs.jooq
47-
}
4820
}

‎airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,27 @@ data:
22
connectorSubtype: database
33
connectorType: destination
44
definitionId: 25c5221d-dce2-4163-ade9-739ef790f503
5-
dockerImageTag: 0.6.3
5+
dockerImageTag: 2.0.0
66
dockerRepository: airbyte/destination-postgres-strict-encrypt
77
documentationUrl: https://docs.airbyte.com/integrations/destinations/postgres
88
githubIssueLabel: destination-postgres
99
icon: postgresql.svg
1010
license: ELv2
1111
name: Postgres
12-
normalizationConfig:
13-
normalizationIntegrationType: postgres
14-
normalizationRepository: airbyte/normalization
15-
normalizationTag: 0.4.1
1612
registries:
1713
cloud:
1814
enabled: false
1915
oss:
2016
enabled: false
17+
releases:
18+
breakingChanges:
19+
2.0.0:
20+
message: >
21+
This version introduces [Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2), which provides better error handling, incremental delivery of data for large syncs, and improved final table structures.
22+
To review the breaking changes, and how to upgrade, see [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#quick-start-to-upgrading).
23+
These changes will likely require updates to downstream dbt / SQL models, which we walk through [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#updating-downstream-transformations).
24+
Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync. For more controlled upgrade [see instructions](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#upgrading-connections-one-by-one-with-dual-writing).
25+
upgradeDeadline: "2024-05-31"
2126
releaseStage: alpha
2227
supportsDbt: true
2328
tags:

‎airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncrypt.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,11 @@ public AirbyteConnectionStatus check(final JsonNode config) throws Exception {
6060
return super.check(config);
6161
}
6262

63+
@Override
64+
public boolean isV2Destination() {
65+
return true;
66+
}
67+
6368
public static void main(final String[] args) throws Exception {
6469
final Destination destination = new PostgresDestinationStrictEncrypt();
6570
LOGGER.info("starting destination: {}", PostgresDestinationStrictEncrypt.class);

‎airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import org.junit.jupiter.api.Disabled;
1717
import org.junit.jupiter.api.Test;
1818

19+
@Disabled("Disabled after DV2 migration. Re-enable with fixtures updated to DV2.")
1920
public class PostgresDestinationStrictEncryptAcceptanceTest extends AbstractPostgresDestinationAcceptanceTest {
2021

2122
private PostgresTestDatabase testDb;

‎airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresStrictEncryptTypingDedupingTest.java

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3+
*/
4+
5+
package io.airbyte.integrations.destination.postgres.typing_deduping;
6+
7+
import com.fasterxml.jackson.databind.JsonNode;
8+
import com.fasterxml.jackson.databind.node.ObjectNode;
9+
import com.google.common.collect.ImmutableMap;
10+
import io.airbyte.integrations.destination.postgres.PostgresDestination;
11+
import io.airbyte.integrations.destination.postgres.PostgresTestDatabase;
12+
import io.airbyte.integrations.destination.postgres.PostgresTestDatabase.BaseImage;
13+
import javax.sql.DataSource;
14+
import org.junit.jupiter.api.AfterAll;
15+
import org.junit.jupiter.api.BeforeAll;
16+
17+
// TODO: This test is added to ensure coverage missed by disabling DATs. Redundant when DATs
18+
// enabled.
19+
public class PostgresStrictEncryptTypingDedupingTest extends AbstractPostgresTypingDedupingTest {
20+
21+
protected static PostgresTestDatabase testContainer;
22+
23+
@BeforeAll
24+
public static void setupPostgres() {
25+
// Postgres-13 is alpine image and SSL conf is failing to load, intentionally using 12:bullseye
26+
testContainer = PostgresTestDatabase.in(BaseImage.POSTGRES_12, PostgresTestDatabase.ContainerModifier.CERT);
27+
}
28+
29+
@AfterAll
30+
public static void teardownPostgres() {
31+
testContainer.close();
32+
}
33+
34+
@Override
35+
protected ObjectNode getBaseConfig() {
36+
return (ObjectNode) testContainer.configBuilder()
37+
.with("schema", "public")
38+
.withDatabase()
39+
.withResolvedHostAndPort()
40+
.withCredentials()
41+
.withSsl(ImmutableMap.builder()
42+
.put("mode", "verify-ca") // verify-full will not work since the spawned container is only allowed for 127.0.0.1/32 CIDRs
43+
.put("ca_certificate", testContainer.getCertificates().caCertificate())
44+
.build())
45+
.build();
46+
}
47+
48+
@Override
49+
protected DataSource getDataSource(final JsonNode config) {
50+
// Intentionally ignore the config and rebuild it.
51+
// The config param has the resolved (i.e. in-docker) host/port.
52+
// We need the unresolved host/port since the test wrapper code is running from the docker host
53+
// rather than in a container.
54+
return new PostgresDestination().getDataSource(testContainer.configBuilder()
55+
.with("schema", "public")
56+
.withDatabase()
57+
.withHostAndPort()
58+
.withCredentials()
59+
.withoutSsl()
60+
.build());
61+
}
62+
63+
@Override
64+
protected String getImageName() {
65+
return "airbyte/destination-postgres-strict-encrypt:dev";
66+
}
67+
68+
}

‎airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "old_cursor": 1, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}
2+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}
3+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie"}

‎airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name" :"Alice", "address": {"city": "San Francisco", "state": "CA"}}}
2+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}}
3+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}}
4+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}}

‎airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
// Keep the Alice record with more recent updated_at
2+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}
3+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}
4+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"}
5+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}

‎airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00.000000Z", "name": "Someone completely different"}

‎airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}
2+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}
3+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}
4+
// Invalid columns are nulled out (i.e. SQL null, not JSON null)
5+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"}
6+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}

‎airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}}
2+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}}
3+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}}
4+
// Invalid data is still allowed in the raw table.
5+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}}
6+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}}

‎airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different"}}

‎airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}
2+
// Charlie wasn't reemitted with updated_at, so it still has a null cursor
3+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "name": "Charlie"}

‎airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name" :"Alice", "address": {"city": "San Francisco", "state": "CA"}}}
2+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}}
3+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}}
4+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}}
5+
{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}}
6+
{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}}
7+
{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}}

‎airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}
2+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}
3+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}
4+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"}
5+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}
6+
7+
{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}
8+
{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}}
9+
{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000Z"}

‎airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}
2+
{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}}
3+
{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000Z"}

‎airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}}
2+
{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}}
3+
{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}}

‎airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}
2+
// Delete Bob, keep Charlie
3+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"}
4+
{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}

‎airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2001-01-02T00:00:00.000000Z", "name": "Someone completely different v2"}

0 commit comments

Comments
 (0)