diff --git a/airbyte-integrations/connectors/destination-redshift/build.gradle b/airbyte-integrations/connectors/destination-redshift/build.gradle index 035ed71d6de74..a6bb21f2f0dae 100644 --- a/airbyte-integrations/connectors/destination-redshift/build.gradle +++ b/airbyte-integrations/connectors/destination-redshift/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.34.0' + cdkVersionRequired = '0.35.0' features = ['db-destinations', 's3-destinations', 'typing-deduping'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/destination-redshift/metadata.yaml b/airbyte-integrations/connectors/destination-redshift/metadata.yaml index f7486913dfc3b..8f5d47535cd7b 100644 --- a/airbyte-integrations/connectors/destination-redshift/metadata.yaml +++ b/airbyte-integrations/connectors/destination-redshift/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: destination definitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc - dockerImageTag: 2.6.0 + dockerImageTag: 2.6.1 dockerRepository: airbyte/destination-redshift documentationUrl: https://docs.airbyte.com/integrations/destinations/redshift githubIssueLabel: destination-redshift diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java index 981a7cfdfa94b..2306f72ffe19c 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java @@ -241,7 +241,7 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN catalogParser = new CatalogParser(sqlGenerator, rawNamespace); } else { rawNamespace = JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE; - catalogParser = new CatalogParser(sqlGenerator); + catalogParser = new CatalogParser(sqlGenerator, rawNamespace); } final RedshiftDestinationHandler redshiftDestinationHandler = new RedshiftDestinationHandler(databaseName, database, rawNamespace); parsedCatalog = catalogParser.parseCatalog(catalog); diff --git a/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorTest.java b/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorTest.java index e40bbcd309f25..0238f74362a9b 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorTest.java @@ -17,7 +17,6 @@ import io.airbyte.integrations.base.destination.typing_deduping.Struct; import io.airbyte.integrations.destination.redshift.RedshiftSQLNameTransformer; import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.SyncMode; import java.io.IOException; import java.time.Instant; import java.util.Arrays; @@ -79,18 +78,19 @@ public void setup() { columns.put(redshiftSqlGenerator.buildColumnId("_ab_cdc_deleted_at"), AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); incrementalDedupStream = new StreamConfig( streamId, - SyncMode.INCREMENTAL, DestinationSyncMode.APPEND_DEDUP, primaryKey, Optional.of(cursor), - columns); + columns, + 0, + 0, + 0); incrementalAppendStream = new StreamConfig( streamId, - SyncMode.INCREMENTAL, DestinationSyncMode.APPEND, primaryKey, Optional.of(cursor), - columns); + columns, 0, 0, 0); } @Test @@ -132,11 +132,10 @@ public void test2000ColumnSql() { } final Sql generatedSql = redshiftSqlGenerator.updateTable(new StreamConfig( streamId, - SyncMode.INCREMENTAL, DestinationSyncMode.APPEND_DEDUP, primaryKey, Optional.of(cursor), - columns), "unittest", Optional.of(Instant.parse("2023-02-15T18:35:24.00Z")), false); + columns, 0, 0, 0), "unittest", Optional.of(Instant.parse("2023-02-15T18:35:24.00Z")), false); // This should not throw an exception. assertFalse(generatedSql.transactions().isEmpty()); } diff --git a/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSuperLimitationTransformerTest.java b/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSuperLimitationTransformerTest.java index 3f011d39780bb..e5516f84f0288 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSuperLimitationTransformerTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSuperLimitationTransformerTest.java @@ -26,7 +26,6 @@ import io.airbyte.protocol.models.v0.AirbyteRecordMessageMetaChange.Reason; import io.airbyte.protocol.models.v0.DestinationSyncMode; import io.airbyte.protocol.models.v0.StreamDescriptor; -import io.airbyte.protocol.models.v0.SyncMode; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.HashMap; @@ -57,11 +56,10 @@ public void setup() { final StreamId streamId = new StreamId("test_schema", "users_final", "test_schema", "users_raw", "test_schema", "users_final"); StreamConfig streamConfig = new StreamConfig( streamId, - SyncMode.INCREMENTAL, DestinationSyncMode.APPEND_DEDUP, primaryKey, Optional.empty(), - columns); + columns, 0, 0, 0); final ParsedCatalog parsedCatalog = new ParsedCatalog(List.of(streamConfig)); transformer = new RedshiftSuperLimitationTransformer(parsedCatalog, "test_schema"); } diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index a7460e4eb109c..2469cfc90294a 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -241,7 +241,8 @@ Each stream will be output into its own raw table in Redshift. Each table will c ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :--------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +|:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 2.6.1 | 2024-05-13 | [\#38126](https://github.com/airbytehq/airbyte/pull/38126) | Adapt to signature changes in `StreamConfig` | | 2.6.0 | 2024-05-08 | [\#37713](https://github.com/airbytehq/airbyte/pull/37713) | Remove option for incremental typing and deduping | | 2.5.0 | 2024-05-06 | [\#34613](https://github.com/airbytehq/airbyte/pull/34613) | Upgrade Redshift driver to work with Cluster patch 181; Adapt to CDK 0.33.0; Minor signature changes | | 2.4.3 | 2024-04-10 | [\#36973](https://github.com/airbytehq/airbyte/pull/36973) | Limit the Standard inserts SQL statement to less than 16MB | @@ -308,7 +309,7 @@ Each stream will be output into its own raw table in Redshift. Each table will c | 0.3.55 | 2023-01-26 | [\#20631](https://github.com/airbytehq/airbyte/pull/20631) | Added support for destination checkpointing with staging | | 0.3.54 | 2023-01-18 | [\#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions | | 0.3.53 | 2023-01-03 | [\#17273](https://github.com/airbytehq/airbyte/pull/17273) | Flatten JSON arrays to fix maximum size check for SUPER field | -| 0.3.52 | 2022-12-30 | [\#20879](https://github.com/airbytehq/airbyte/pull/20879) | Added configurable parameter for number of file buffers (⛔ this version has a bug and will not work; use `0.3.56` instead) | +| 0.3.52 | 2022-12-30 | [\#20879](https://github.com/airbytehq/airbyte/pull/20879) | Added configurable parameter for number of file buffers (⛔ this version has a bug and will not work; use `0.3.56` instead) | | 0.3.51 | 2022-10-26 | [\#18434](https://github.com/airbytehq/airbyte/pull/18434) | Fix empty S3 bucket path handling | | 0.3.50 | 2022-09-14 | [\#15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage | | 0.3.49 | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243) | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields) |