Skip to content

Commit beda34b

Browse files
gisripagisripa
authored
Destination Postgres: Adapt to kotlin CDK (#36620)
1 parent c5284c7 commit beda34b

File tree

14 files changed

+69
-67
lines changed

14 files changed

+69
-67
lines changed

‎airbyte-cdk/java/airbyte-cdk/README.md

+9-8
Original file line numberDiff line numberDiff line change
@@ -144,14 +144,15 @@ Maven and Gradle will automatically reference the correct (pinned) version of th
144144

145145
| Version | Date | Pull Request | Subject |
146146
|:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------|
147-
| 0.29.3 | 2024-04-04 | [\#36759](https://github.com/airbytehq/airbyte/pull/36759) | Minor fixes. |
148-
| 0.29.3 | 2024-04-04 | [\#36706](https://github.com/airbytehq/airbyte/pull/36706) | Enabling spotbugs for s3-destination. |
149-
| 0.29.3 | 2024-04-03 | [\#36705](https://github.com/airbytehq/airbyte/pull/36705) | Enabling spotbugs for db-sources. |
150-
| 0.29.3 | 2024-04-03 | [\#36704](https://github.com/airbytehq/airbyte/pull/36704) | Enabling spotbugs for datastore-postgres. |
151-
| 0.29.3 | 2024-04-03 | [\#36703](https://github.com/airbytehq/airbyte/pull/36703) | Enabling spotbugs for gcs-destination. |
152-
| 0.29.3 | 2024-04-03 | [\#36702](https://github.com/airbytehq/airbyte/pull/36702) | Enabling spotbugs for db-destinations. |
153-
| 0.29.3 | 2024-04-03 | [\#36701](https://github.com/airbytehq/airbyte/pull/36701) | Enabling spotbugs for typing_and_deduping. |
154-
| 0.29.3 | 2024-04-03 | [\#36612](https://github.com/airbytehq/airbyte/pull/36612) | Enabling spotbugs for dependencies. |
147+
| 0.29.5 | 2024-04-05 | [\#36620](https://github.com/airbytehq/airbyte/pull/36620) | Missed changes - open for extension for destination-postgres |
148+
| 0.29.3 | 2024-04-04 | [\#36759](https://github.com/airbytehq/airbyte/pull/36759) | Minor fixes. |
149+
| 0.29.3 | 2024-04-04 | [\#36706](https://github.com/airbytehq/airbyte/pull/36706) | Enabling spotbugs for s3-destination. |
150+
| 0.29.3 | 2024-04-03 | [\#36705](https://github.com/airbytehq/airbyte/pull/36705) | Enabling spotbugs for db-sources. |
151+
| 0.29.3 | 2024-04-03 | [\#36704](https://github.com/airbytehq/airbyte/pull/36704) | Enabling spotbugs for datastore-postgres. |
152+
| 0.29.3 | 2024-04-03 | [\#36703](https://github.com/airbytehq/airbyte/pull/36703) | Enabling spotbugs for gcs-destination. |
153+
| 0.29.3 | 2024-04-03 | [\#36702](https://github.com/airbytehq/airbyte/pull/36702) | Enabling spotbugs for db-destinations. |
154+
| 0.29.3 | 2024-04-03 | [\#36701](https://github.com/airbytehq/airbyte/pull/36701) | Enabling spotbugs for typing_and_deduping. |
155+
| 0.29.3 | 2024-04-03 | [\#36612](https://github.com/airbytehq/airbyte/pull/36612) | Enabling spotbugs for dependencies. |
155156
| 0.29.2 | 2024-04-04 | [\#36845](https://github.com/airbytehq/airbyte/pull/36772) | Changes to make source-mongo compileable |
156157
| 0.29.1 | 2024-04-03 | [\#36772](https://github.com/airbytehq/airbyte/pull/36772) | Changes to make source-mssql compileable |
157158
| 0.29.0 | 2024-04-02 | [\#36759](https://github.com/airbytehq/airbyte/pull/36759) | Build artifact publication changes and fixes. |

‎airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
version=0.29.4
1+
version=0.29.5

‎airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.kt

+1-1
Original file line numberDiff line numberDiff line change
@@ -243,7 +243,7 @@ abstract class DestinationAcceptanceTest {
243243
return supportsDbt != null && supportsDbt.asBoolean(false)
244244
}
245245

246-
protected val destinationDefinitionKey: String
246+
protected open val destinationDefinitionKey: String
247247
get() = imageNameWithoutTag
248248

249249
protected open fun getNormalizationIntegrationType(): String? {

‎airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ plugins {
33
}
44

55
airbyteJavaConnector {
6-
cdkVersionRequired = '0.27.3'
6+
cdkVersionRequired = '0.29.5'
77
features = ['db-destinations', 'typing-deduping', 'datastore-postgres']
88
useLocalCdk = false
99
}

‎airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ data:
22
connectorSubtype: database
33
connectorType: destination
44
definitionId: 25c5221d-dce2-4163-ade9-739ef790f503
5-
dockerImageTag: 2.0.5
5+
dockerImageTag: 2.0.6
66
dockerRepository: airbyte/destination-postgres-strict-encrypt
77
documentationUrl: https://docs.airbyte.com/integrations/destinations/postgres
88
githubIssueLabel: destination-postgres

‎airbyte-integrations/connectors/destination-postgres/build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ plugins {
33
}
44

55
airbyteJavaConnector {
6-
cdkVersionRequired = '0.27.3'
6+
cdkVersionRequired = '0.29.5'
77
features = ['db-destinations', 'datastore-postgres', 'typing-deduping']
88
useLocalCdk = false
99
}

‎airbyte-integrations/connectors/destination-postgres/metadata.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ data:
55
connectorSubtype: database
66
connectorType: destination
77
definitionId: 25c5221d-dce2-4163-ade9-739ef790f503
8-
dockerImageTag: 2.0.5
8+
dockerImageTag: 2.0.6
99
dockerRepository: airbyte/destination-postgres
1010
documentationUrl: https://docs.airbyte.com/integrations/destinations/postgres
1111
githubIssueLabel: destination-postgres

‎airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
import static io.airbyte.cdk.integrations.util.PostgresSslConnectionUtils.PARAM_MODE;
99
import static io.airbyte.cdk.integrations.util.PostgresSslConnectionUtils.PARAM_SSL;
1010
import static io.airbyte.cdk.integrations.util.PostgresSslConnectionUtils.PARAM_SSL_MODE;
11-
import static io.airbyte.cdk.integrations.util.PostgresSslConnectionUtils.obtainConnectionOptions;
1211

1312
import com.fasterxml.jackson.databind.JsonNode;
1413
import com.google.common.collect.ImmutableMap;
@@ -24,6 +23,7 @@
2423
import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination;
2524
import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcDestinationHandler;
2625
import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator;
26+
import io.airbyte.cdk.integrations.util.PostgresSslConnectionUtils;
2727
import io.airbyte.commons.json.Jsons;
2828
import io.airbyte.integrations.base.destination.typing_deduping.DestinationHandler;
2929
import io.airbyte.integrations.base.destination.typing_deduping.ParsedCatalog;
@@ -92,7 +92,7 @@ protected Map<String, String> getDefaultConnectionProperties(final JsonNode conf
9292
if (DISABLE.equals(config.get(PARAM_SSL_MODE).get(PARAM_MODE).asText())) {
9393
additionalParameters.put("sslmode", DISABLE);
9494
} else {
95-
additionalParameters.putAll(obtainConnectionOptions(config.get(PARAM_SSL_MODE)));
95+
additionalParameters.putAll(PostgresSslConnectionUtils.obtainConnectionOptions(config.get(PARAM_SSL_MODE)));
9696
}
9797
} else {
9898
additionalParameters.put(JdbcUtils.SSL_KEY, "true");

‎airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresSqlOperations.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88

99
import io.airbyte.cdk.db.jdbc.JdbcDatabase;
1010
import io.airbyte.cdk.integrations.base.TypingAndDedupingFlag;
11-
import io.airbyte.cdk.integrations.destination.async.partial_messages.PartialAirbyteMessage;
11+
import io.airbyte.cdk.integrations.destination.async.model.PartialAirbyteMessage;
1212
import io.airbyte.cdk.integrations.destination.jdbc.JdbcSqlOperations;
1313
import java.io.BufferedReader;
1414
import java.io.File;

‎airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresSqlGenerator.java

+21-21
Original file line numberDiff line numberDiff line change
@@ -71,11 +71,11 @@ public StreamId buildStreamId(final String namespace, final String name, final S
7171
// To keep it consistent when querying raw table in T+D query, convert it to lowercase.
7272
// TODO: This logic should be unified across Raw and final table operations in a single class
7373
// operating on a StreamId.
74-
final String streamName = namingTransformer.convertStreamName(StreamId.concatenateRawTableName(namespace, name)).toLowerCase();
74+
final String streamName = getNamingTransformer().convertStreamName(StreamId.concatenateRawTableName(namespace, name)).toLowerCase();
7575
return new StreamId(
76-
namingTransformer.getNamespace(namespace),
77-
namingTransformer.convertStreamName(name),
78-
namingTransformer.getNamespace(rawNamespaceOverride).toLowerCase(),
76+
getNamingTransformer().getNamespace(namespace),
77+
getNamingTransformer().convertStreamName(name),
78+
getNamingTransformer().getNamespace(rawNamespaceOverride).toLowerCase(),
7979
streamName.length() > 63 ? streamName.substring(0, 63) : streamName,
8080
namespace,
8181
name);
@@ -117,22 +117,22 @@ public DataType<?> toDialectType(AirbyteProtocolType airbyteProtocolType) {
117117
@Override
118118
public Sql createTable(final StreamConfig stream, final String suffix, final boolean force) {
119119
final List<Sql> statements = new ArrayList<>();
120-
final Name finalTableName = name(stream.id().finalNamespace(), stream.id().finalName() + suffix);
120+
final Name finalTableName = name(stream.getId().getFinalNamespace(), stream.getId().getFinalName() + suffix);
121121

122122
statements.add(super.createTable(stream, suffix, force));
123123

124-
if (stream.destinationSyncMode() == DestinationSyncMode.APPEND_DEDUP) {
124+
if (stream.getDestinationSyncMode() == DestinationSyncMode.APPEND_DEDUP) {
125125
// An index for our ROW_NUMBER() PARTITION BY pk ORDER BY cursor, extracted_at function
126-
final List<Name> pkNames = stream.primaryKey().stream()
127-
.map(pk -> quotedName(pk.name()))
126+
final List<Name> pkNames = stream.getPrimaryKey().stream()
127+
.map(pk -> quotedName(pk.getName()))
128128
.toList();
129129
statements.add(Sql.of(getDslContext().createIndex().on(
130130
finalTableName,
131131
Stream.of(
132132
pkNames.stream(),
133133
// if cursor is present, then a stream containing its name
134134
// but if no cursor, then empty stream
135-
stream.cursor().stream().map(cursor -> quotedName(cursor.name())),
135+
stream.getCursor().stream().map(cursor -> quotedName(cursor.getName())),
136136
Stream.of(name(COLUMN_NAME_AB_EXTRACTED_AT))).flatMap(Function.identity()).toList())
137137
.getSQL()));
138138
}
@@ -151,12 +151,12 @@ public Sql createTable(final StreamConfig stream, final String suffix, final boo
151151

152152
@Override
153153
protected List<String> createIndexSql(final StreamConfig stream, final String suffix) {
154-
if (stream.destinationSyncMode() == DestinationSyncMode.APPEND_DEDUP && !stream.primaryKey().isEmpty()) {
154+
if (stream.getDestinationSyncMode() == DestinationSyncMode.APPEND_DEDUP && !stream.getPrimaryKey().isEmpty()) {
155155
return List.of(
156156
getDslContext().createIndex().on(
157-
name(stream.id().finalNamespace(), stream.id().finalName() + suffix),
158-
stream.primaryKey().stream()
159-
.map(pk -> quotedName(pk.name()))
157+
name(stream.getId().getFinalNamespace(), stream.getId().getFinalName() + suffix),
158+
stream.getPrimaryKey().stream()
159+
.map(pk -> quotedName(pk.getName()))
160160
.toList())
161161
.getSQL());
162162
} else {
@@ -172,7 +172,7 @@ protected List<Field<?>> extractRawDataFields(final LinkedHashMap<ColumnId, Airb
172172
.map(column -> castedField(
173173
extractColumnAsJson(column.getKey()),
174174
column.getValue(),
175-
column.getKey().name(),
175+
column.getKey().getName(),
176176
useExpensiveSaferCasting))
177177
.collect(Collectors.toList());
178178
}
@@ -269,11 +269,11 @@ private Field<Object> toCastingErrorCaseStmt(final ColumnId column, final Airbyt
269269
return switch (type) {
270270
case Struct ignored -> field(CASE_STATEMENT_SQL_TEMPLATE,
271271
extract.isNotNull().and(jsonTypeof(extract).notIn("object", "null")),
272-
nulledChangeObject(column.originalName()),
272+
nulledChangeObject(column.getOriginalName()),
273273
cast(val((Object) null), JSONB_TYPE));
274274
case Array ignored -> field(CASE_STATEMENT_SQL_TEMPLATE,
275275
extract.isNotNull().and(jsonTypeof(extract).notIn("array", "null")),
276-
nulledChangeObject(column.originalName()),
276+
nulledChangeObject(column.getOriginalName()),
277277
cast(val((Object) null), JSONB_TYPE));
278278
// Unknown types require no casting, so there's never an error.
279279
// Similarly, everything can cast to string without error.
@@ -284,28 +284,28 @@ private Field<Object> toCastingErrorCaseStmt(final ColumnId column, final Airbyt
284284
extract.isNotNull()
285285
.and(jsonTypeof(extract).ne("null"))
286286
.and(castedField(extract, type, true).isNull()),
287-
nulledChangeObject(column.originalName()),
287+
nulledChangeObject(column.getOriginalName()),
288288
cast(val((Object) null), JSONB_TYPE));
289289
};
290290
}
291291

292292
@Override
293293
protected Condition cdcDeletedAtNotNullCondition() {
294294
return field(name(COLUMN_NAME_AB_LOADED_AT)).isNotNull()
295-
.and(jsonTypeof(extractColumnAsJson(cdcDeletedAtColumn)).ne("null"));
295+
.and(jsonTypeof(extractColumnAsJson(getCdcDeletedAtColumn())).ne("null"));
296296
}
297297

298298
@Override
299299
protected Field<Integer> getRowNumber(final List<ColumnId> primaryKeys, final Optional<ColumnId> cursor) {
300300
// literally identical to redshift's getRowNumber implementation, changes here probably should
301301
// be reflected there
302302
final List<Field<?>> primaryKeyFields =
303-
primaryKeys != null ? primaryKeys.stream().map(columnId -> field(quotedName(columnId.name()))).collect(Collectors.toList())
303+
primaryKeys != null ? primaryKeys.stream().map(columnId -> field(quotedName(columnId.getName()))).collect(Collectors.toList())
304304
: new ArrayList<>();
305305
final List<Field<?>> orderedFields = new ArrayList<>();
306306
// We can still use Jooq's field to get the quoted name with raw sql templating.
307307
// jooq's .desc returns SortField<?> instead of Field<?> and NULLS LAST doesn't work with it
308-
cursor.ifPresent(columnId -> orderedFields.add(field("{0} desc NULLS LAST", field(quotedName(columnId.name())))));
308+
cursor.ifPresent(columnId -> orderedFields.add(field("{0} desc NULLS LAST", field(quotedName(columnId.getName())))));
309309
orderedFields.add(field("{0} desc", quotedName(COLUMN_NAME_AB_EXTRACTED_AT)));
310310
return rowNumber()
311311
.over()
@@ -317,7 +317,7 @@ protected Field<Integer> getRowNumber(final List<ColumnId> primaryKeys, final Op
317317
* Extract a raw field, leaving it as jsonb
318318
*/
319319
private Field<Object> extractColumnAsJson(final ColumnId column) {
320-
return field("{0} -> {1}", name(COLUMN_NAME_DATA), val(column.originalName()));
320+
return field("{0} -> {1}", name(COLUMN_NAME_DATA), val(column.getOriginalName()));
321321
}
322322

323323
private Field<String> jsonTypeof(final Field<?> field) {

‎airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresSqlGeneratorIntegrationTest.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ protected JdbcSqlGenerator getSqlGenerator() {
7676

7777
@Override
7878
protected DestinationHandler<PostgresState> getDestinationHandler() {
79-
return new PostgresDestinationHandler(databaseName, database, namespace);
79+
return new PostgresDestinationHandler(databaseName, database, getNamespace());
8080
}
8181

8282
@Override
@@ -92,10 +92,10 @@ protected Field<?> toJsonValue(final String valueAsString) {
9292
@Test
9393
@Override
9494
public void testCreateTableIncremental() throws Exception {
95-
final Sql sql = generator.createTable(incrementalDedupStream, "", false);
96-
destinationHandler.execute(sql);
95+
final Sql sql = getGenerator().createTable(getIncrementalDedupStream(), "", false);
96+
getDestinationHandler().execute(sql);
9797

98-
List<DestinationInitialStatus<PostgresState>> initialStatuses = destinationHandler.gatherInitialState(List.of(incrementalDedupStream));
98+
List<DestinationInitialStatus<PostgresState>> initialStatuses = getDestinationHandler().gatherInitialState(List.of(getIncrementalDedupStream()));
9999
assertEquals(1, initialStatuses.size());
100100
final DestinationInitialStatus<PostgresState> initialStatus = initialStatuses.getFirst();
101101
assertTrue(initialStatus.isFinalTablePresent());

‎airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/PostgresTestDatabase.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -190,9 +190,9 @@ public PostgresConfigBuilder withCdcReplication(String LsnCommitBehaviour) {
190190
.with("is_test", true)
191191
.with("replication_method", Jsons.jsonNode(ImmutableMap.builder()
192192
.put("method", "CDC")
193-
.put("replication_slot", testDatabase.getReplicationSlotName())
194-
.put("publication", testDatabase.getPublicationName())
195-
.put("initial_waiting_seconds", DEFAULT_CDC_REPLICATION_INITIAL_WAIT.getSeconds())
193+
.put("replication_slot", getTestDatabase().getReplicationSlotName())
194+
.put("publication", getTestDatabase().getPublicationName())
195+
.put("initial_waiting_seconds", ConfigBuilder.DEFAULT_CDC_REPLICATION_INITIAL_WAIT.getSeconds())
196196
.put("lsn_commit_behaviour", LsnCommitBehaviour)
197197
.build()));
198198
}

0 commit comments

Comments
 (0)