Skip to content

Commit 13e16de

Browse files
authored
redshift_catalog_parser_update (#38126)
## What Adapting to CDK signature changes in `StreamConfig` ## Review guide <!-- 1. `x.py` 2. `y.py` --> ## User Impact <!-- * What is the end result perceived by the user? * If there are negative side effects, please list them. --> ## Can this PR be safely reverted and rolled back? <!-- * If unsure, leave it blank. --> - [x] YES 💚 - [ ] NO ❌
1 parent 1f1744f commit 13e16de

File tree

6 files changed

+13
-15
lines changed

6 files changed

+13
-15
lines changed

airbyte-integrations/connectors/destination-redshift/build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ plugins {
44
}
55

66
airbyteJavaConnector {
7-
cdkVersionRequired = '0.34.0'
7+
cdkVersionRequired = '0.35.0'
88
features = ['db-destinations', 's3-destinations', 'typing-deduping']
99
useLocalCdk = false
1010
}

airbyte-integrations/connectors/destination-redshift/metadata.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ data:
55
connectorSubtype: database
66
connectorType: destination
77
definitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc
8-
dockerImageTag: 2.6.0
8+
dockerImageTag: 2.6.1
99
dockerRepository: airbyte/destination-redshift
1010
documentationUrl: https://docs.airbyte.com/integrations/destinations/redshift
1111
githubIssueLabel: destination-redshift

airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,7 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN
241241
catalogParser = new CatalogParser(sqlGenerator, rawNamespace);
242242
} else {
243243
rawNamespace = JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE;
244-
catalogParser = new CatalogParser(sqlGenerator);
244+
catalogParser = new CatalogParser(sqlGenerator, rawNamespace);
245245
}
246246
final RedshiftDestinationHandler redshiftDestinationHandler = new RedshiftDestinationHandler(databaseName, database, rawNamespace);
247247
parsedCatalog = catalogParser.parseCatalog(catalog);

airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorTest.java

+6-7
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
import io.airbyte.integrations.base.destination.typing_deduping.Struct;
1818
import io.airbyte.integrations.destination.redshift.RedshiftSQLNameTransformer;
1919
import io.airbyte.protocol.models.v0.DestinationSyncMode;
20-
import io.airbyte.protocol.models.v0.SyncMode;
2120
import java.io.IOException;
2221
import java.time.Instant;
2322
import java.util.Arrays;
@@ -79,18 +78,19 @@ public void setup() {
7978
columns.put(redshiftSqlGenerator.buildColumnId("_ab_cdc_deleted_at"), AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE);
8079
incrementalDedupStream = new StreamConfig(
8180
streamId,
82-
SyncMode.INCREMENTAL,
8381
DestinationSyncMode.APPEND_DEDUP,
8482
primaryKey,
8583
Optional.of(cursor),
86-
columns);
84+
columns,
85+
0,
86+
0,
87+
0);
8788
incrementalAppendStream = new StreamConfig(
8889
streamId,
89-
SyncMode.INCREMENTAL,
9090
DestinationSyncMode.APPEND,
9191
primaryKey,
9292
Optional.of(cursor),
93-
columns);
93+
columns, 0, 0, 0);
9494
}
9595

9696
@Test
@@ -132,11 +132,10 @@ public void test2000ColumnSql() {
132132
}
133133
final Sql generatedSql = redshiftSqlGenerator.updateTable(new StreamConfig(
134134
streamId,
135-
SyncMode.INCREMENTAL,
136135
DestinationSyncMode.APPEND_DEDUP,
137136
primaryKey,
138137
Optional.of(cursor),
139-
columns), "unittest", Optional.of(Instant.parse("2023-02-15T18:35:24.00Z")), false);
138+
columns, 0, 0, 0), "unittest", Optional.of(Instant.parse("2023-02-15T18:35:24.00Z")), false);
140139
// This should not throw an exception.
141140
assertFalse(generatedSql.transactions().isEmpty());
142141
}

airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSuperLimitationTransformerTest.java

+1-3
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
import io.airbyte.protocol.models.v0.AirbyteRecordMessageMetaChange.Reason;
2727
import io.airbyte.protocol.models.v0.DestinationSyncMode;
2828
import io.airbyte.protocol.models.v0.StreamDescriptor;
29-
import io.airbyte.protocol.models.v0.SyncMode;
3029
import java.io.IOException;
3130
import java.nio.charset.StandardCharsets;
3231
import java.util.HashMap;
@@ -57,11 +56,10 @@ public void setup() {
5756
final StreamId streamId = new StreamId("test_schema", "users_final", "test_schema", "users_raw", "test_schema", "users_final");
5857
StreamConfig streamConfig = new StreamConfig(
5958
streamId,
60-
SyncMode.INCREMENTAL,
6159
DestinationSyncMode.APPEND_DEDUP,
6260
primaryKey,
6361
Optional.empty(),
64-
columns);
62+
columns, 0, 0, 0);
6563
final ParsedCatalog parsedCatalog = new ParsedCatalog(List.of(streamConfig));
6664
transformer = new RedshiftSuperLimitationTransformer(parsedCatalog, "test_schema");
6765
}

docs/integrations/destinations/redshift.md

+3-2
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,8 @@ Each stream will be output into its own raw table in Redshift. Each table will c
241241
## Changelog
242242

243243
| Version | Date | Pull Request | Subject |
244-
| :------ | :--------- | :--------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
244+
|:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
245+
| 2.6.1 | 2024-05-13 | [\#38126](https://github.com/airbytehq/airbyte/pull/38126) | Adapt to signature changes in `StreamConfig` |
245246
| 2.6.0 | 2024-05-08 | [\#37713](https://github.com/airbytehq/airbyte/pull/37713) | Remove option for incremental typing and deduping |
246247
| 2.5.0 | 2024-05-06 | [\#34613](https://github.com/airbytehq/airbyte/pull/34613) | Upgrade Redshift driver to work with Cluster patch 181; Adapt to CDK 0.33.0; Minor signature changes |
247248
| 2.4.3 | 2024-04-10 | [\#36973](https://github.com/airbytehq/airbyte/pull/36973) | Limit the Standard inserts SQL statement to less than 16MB |
@@ -308,7 +309,7 @@ Each stream will be output into its own raw table in Redshift. Each table will c
308309
| 0.3.55 | 2023-01-26 | [\#20631](https://github.com/airbytehq/airbyte/pull/20631) | Added support for destination checkpointing with staging |
309310
| 0.3.54 | 2023-01-18 | [\#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions |
310311
| 0.3.53 | 2023-01-03 | [\#17273](https://github.com/airbytehq/airbyte/pull/17273) | Flatten JSON arrays to fix maximum size check for SUPER field |
311-
| 0.3.52 | 2022-12-30 | [\#20879](https://github.com/airbytehq/airbyte/pull/20879) | Added configurable parameter for number of file buffers (⛔ this version has a bug and will not work; use `0.3.56` instead) |
312+
| 0.3.52 | 2022-12-30 | [\#20879](https://github.com/airbytehq/airbyte/pull/20879) | Added configurable parameter for number of file buffers (⛔ this version has a bug and will not work; use `0.3.56` instead) |
312313
| 0.3.51 | 2022-10-26 | [\#18434](https://github.com/airbytehq/airbyte/pull/18434) | Fix empty S3 bucket path handling |
313314
| 0.3.50 | 2022-09-14 | [\#15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage |
314315
| 0.3.49 | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243) | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields) |

0 commit comments

Comments
 (0)