diff --git a/airbyte-integrations/connectors/source-mssql/metadata.yaml b/airbyte-integrations/connectors/source-mssql/metadata.yaml index f47fbae81d09d..92d8700bd4d26 100644 --- a/airbyte-integrations/connectors/source-mssql/metadata.yaml +++ b/airbyte-integrations/connectors/source-mssql/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 - dockerImageTag: 4.0.27 + dockerImageTag: 4.0.28 dockerRepository: airbyte/source-mssql documentationUrl: https://docs.airbyte.com/integrations/sources/mssql githubIssueLabel: source-mssql diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlDebeziumConverter.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlDebeziumConverter.java index 3625e0d8b8d3d..0a5653fe53e83 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlDebeziumConverter.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlDebeziumConverter.java @@ -17,7 +17,6 @@ import java.time.*; import java.time.format.DateTimeFormatter; import java.util.*; -import microsoft.sql.DateTimeOffset; import org.apache.kafka.connect.data.SchemaBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,7 +33,6 @@ public class MssqlDebeziumConverter implements CustomConverter registration) { - registration.register(SchemaBuilder.string(), input -> { - if (Objects.isNull(input)) { - return DebeziumConverterUtils.convertDefaultValue(field); - } - - if (input instanceof DateTimeOffset) { - return DataTypeUtils.toISO8601String( - OffsetDateTime.parse(input.toString(), - DateTimeFormatter.ofPattern(DEBEZIUM_DATETIMEOFFSET_FORMAT))); - } - - LOGGER.warn("Uncovered DateTimeOffSet class type '{}'. Use default converter", - input.getClass().getName()); - return input.toString(); - }); - } - private void registerTime(final RelationalColumn field, final ConverterRegistration registration) { registration.register(SchemaBuilder.string(), input -> { diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java index 4a85ce730a6f3..574b563f8f437 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java @@ -183,9 +183,11 @@ protected void initTests() { TestDataHolder.builder() .sourceType("datetimeoffset") .airbyteType(JsonSchemaType.STRING_TIMESTAMP_WITH_TIMEZONE) - .addInsertValues("'0001-01-10 00:00:00 +01:00'", "'9999-01-10 00:00:00 +01:00'", "null") + .addInsertValues("'0001-01-10 00:00:00 +01:00'", "'9999-01-10 00:00:00 +01:00'", "null", "'2024-05-10 19:00:01.604805 +03:00'", + "'2024-03-02 19:08:07.1234567 +09:00'", "'2024-03-02 19:08:07.12345678 +09:00'") .addExpectedValues("0001-01-10 00:00:00.0000000 +01:00", - "9999-01-10 00:00:00.0000000 +01:00", null) + "9999-01-10 00:00:00.0000000 +01:00", null, "2024-05-10 19:00:01.6048050 +03:00", "2024-03-02 19:08:07.1234567 +09:00", + "2024-03-02 19:08:07.1234568 +09:00") .createTablePatternSql(CREATE_TABLE_SQL) .build()); diff --git a/docs/integrations/sources/mssql.md b/docs/integrations/sources/mssql.md index cb4a42e9baf2e..ca42758fd314b 100644 --- a/docs/integrations/sources/mssql.md +++ b/docs/integrations/sources/mssql.md @@ -422,57 +422,58 @@ WHERE actor_definition_id ='b5ea17b1-f170-46dc-bc31-cc744ca984c1' AND (configura | Version | Date | Pull Request | Subject | |:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------| -| 4.0.27 | 2024-05-29 | [38584](https://github.com/airbytehq/airbyte/pull/38584) | Set is_resumable flag in discover. | -| 4.0.26 | 2024-05-16 | [38292](https://github.com/airbytehq/airbyte/pull/38292) | Improve cursor value query to return only one row | -| 4.0.25 | 2024-05-29 | [38775](https://github.com/airbytehq/airbyte/pull/38775) | Publish CDK | -| 4.0.24 | 2024-05-23 | [38640](https://github.com/airbytehq/airbyte/pull/38640) | Sync sending trace status messages indicating progress. | -| 4.0.23 | 2024-05-15 | [38208](https://github.com/airbytehq/airbyte/pull/38208) | disable counts in full refresh stream in state message. | -| 4.0.22 | 2024-05-14 | [38196](https://github.com/airbytehq/airbyte/pull/38196) | Bump jdbc driver version to 12.6.1.jre11 | -| 4.0.21 | 2024-05-07 | [38054](https://github.com/airbytehq/airbyte/pull/38054) | Resumeable refresh should run only if there is source defined pk. | -| 4.0.20 | 2024-05-07 | [38042](https://github.com/airbytehq/airbyte/pull/38042) | Bump debezium version to latest. | -| 4.0.19 | 2024-05-07 | [38029](https://github.com/airbytehq/airbyte/pull/38029) | Fix previous release. | -| 4.0.18 | 2024-04-30 | [37451](https://github.com/airbytehq/airbyte/pull/37451) | Resumable full refresh read of tables. | -| 4.0.17 | 2024-05-02 | [37781](https://github.com/airbytehq/airbyte/pull/37781) | Adopt latest CDK. | -| 4.0.16 | 2024-05-01 | [37742](https://github.com/airbytehq/airbyte/pull/37742) | Adopt latest CDK. Remove Debezium retries. | -| 4.0.15 | 2024-04-22 | [37541](https://github.com/airbytehq/airbyte/pull/37541) | Adopt latest CDK. reduce excessive logs. | -| 4.0.14 | 2024-04-22 | [37476](https://github.com/airbytehq/airbyte/pull/37476) | Adopt latest CDK. | -| 4.0.13 | 2024-04-16 | [37111](https://github.com/airbytehq/airbyte/pull/37111) | Populate null values in record message. | -| 4.0.12 | 2024-04-15 | [37326](https://github.com/airbytehq/airbyte/pull/37326) | Allow up to 60 minutes of wait for the an initial CDC record. | -| 4.0.11 | 2024-04-15 | [37325](https://github.com/airbytehq/airbyte/pull/37325) | Populate airbyte_meta.changes + error handling. | -| 4.0.10 | 2024-04-15 | [37110](https://github.com/airbytehq/airbyte/pull/37110) | Internal cleanup. | -| 4.0.9 | 2024-04-10 | [36919](https://github.com/airbytehq/airbyte/pull/36919) | Fix a bug in conversion of null values. | -| 4.0.8 | 2024-04-05 | [36872](https://github.com/airbytehq/airbyte/pull/36872) | Update to connector's metadat definition. | -| 4.0.7 | 2024-04-03 | [36772](https://github.com/airbytehq/airbyte/pull/36772) | Adopt latest CDK. | -| 4.0.6 | 2024-03-25 | [36333](https://github.com/airbytehq/airbyte/pull/36333) | Deprecate Dbz state iterator. | -| 4.0.5 | 2024-03-21 | [36364](https://github.com/airbytehq/airbyte/pull/36364) | Allow up to 40 minutes of wait for the an initial CDC record. | -| 4.0.4 | 2024-03-20 | [36325](https://github.com/airbytehq/airbyte/pull/36325) | [Refactor] : Remove mssql initial source operations . | -| 4.0.3 | 2024-03-19 | [36263](https://github.com/airbytehq/airbyte/pull/36263) | Fix a failure seen in CDC with tables containing default values. | -| 4.0.2 | 2024-03-06 | [35792](https://github.com/airbytehq/airbyte/pull/35792) | Initial sync will now send record count in state message. | -| 4.0.1 | 2024-03-12 | [36011](https://github.com/airbytehq/airbyte/pull/36011) | Read correctly null values of columns with default value in CDC. | -| 4.0.0 | 2024-03-06 | [35873](https://github.com/airbytehq/airbyte/pull/35873) | Terabyte-sized tables support, reliability improvements, bug fixes. | -| 3.7.7 | 2024-03-06 | [35816](https://github.com/airbytehq/airbyte/pull/35816) | Fix query that was failing on a case sensitive server. | -| 3.7.6 | 2024-03-04 | [35721](https://github.com/airbytehq/airbyte/pull/35721) | Fix tests | -| 3.7.5 | 2024-02-29 | [35739](https://github.com/airbytehq/airbyte/pull/35739) | Allow configuring the queue size used for cdc events. | -| 3.7.4 | 2024-02-26 | [35566](https://github.com/airbytehq/airbyte/pull/35566) | Add config to throw an error on invalid CDC position. | -| 3.7.3 | 2024-02-23 | [35596](https://github.com/airbytehq/airbyte/pull/35596) | Fix a logger issue | -| 3.7.2 | 2024-02-21 | [35368](https://github.com/airbytehq/airbyte/pull/35368) | Change query syntax to make it compatible with Azure SQL Managed Instance. | -| 3.7.1 | 2024-02-20 | [35405](https://github.com/airbytehq/airbyte/pull/35405) | Change query syntax to make it compatible with Azure Synapse. | -| 3.7.0 | 2024-01-30 | [33311](https://github.com/airbytehq/airbyte/pull/33311) | Source mssql with checkpointing initial sync. | -| 3.6.1 | 2024-01-26 | [34573](https://github.com/airbytehq/airbyte/pull/34573) | Adopt CDK v0.16.0. | -| 3.6.0 | 2024-01-10 | [33700](https://github.com/airbytehq/airbyte/pull/33700) | Remove CDC config options for data_to_sync and snapshot isolation. | -| 3.5.1 | 2024-01-05 | [33510](https://github.com/airbytehq/airbyte/pull/33510) | Test-only changes. | -| 3.5.0 | 2023-12-19 | [33071](https://github.com/airbytehq/airbyte/pull/33071) | Fix SSL configuration parameters | -| 3.4.1 | 2024-01-02 | [33755](https://github.com/airbytehq/airbyte/pull/33755) | Encode binary to base64 format | -| 3.4.0 | 2023-12-19 | [33481](https://github.com/airbytehq/airbyte/pull/33481) | Remove LEGACY state flag | -| 3.3.2 | 2023-12-14 | [33505](https://github.com/airbytehq/airbyte/pull/33505) | Using the released CDK. | -| 3.3.1 | 2023-12-12 | [33225](https://github.com/airbytehq/airbyte/pull/33225) | extracting MsSql specific files out of the CDK. | -| 3.3.0 | 2023-12-12 | [33018](https://github.com/airbytehq/airbyte/pull/33018) | Migrate to Per-stream/Global states and away from Legacy states | -| 3.2.1 | 2023-12-11 | [33330](https://github.com/airbytehq/airbyte/pull/33330) | Parse DatetimeOffset fields with the correct format when used as cursor | -| 3.2.0 | 2023-12-07 | [33225](https://github.com/airbytehq/airbyte/pull/33225) | CDC : Enable compression of schema history blob in state. | -| 3.1.0 | 2023-11-28 | [32882](https://github.com/airbytehq/airbyte/pull/32882) | Enforce SSL on Airbyte Cloud. | -| 3.0.2 | 2023-11-27 | [32573](https://github.com/airbytehq/airbyte/pull/32573) | Format Datetime and Datetime2 datatypes to 6-digit microsecond precision | -| 3.0.1 | 2023-11-22 | [32656](https://github.com/airbytehq/airbyte/pull/32656) | Adopt java CDK version 0.5.0. | -| 3.0.0 | 2023-11-07 | [31531](https://github.com/airbytehq/airbyte/pull/31531) | Remapped date, smalldatetime, datetime2, time, and datetimeoffset datatype to their correct Airbyte types | +| 4.0.28 | 2024-06-08 | [39342](https://github.com/airbytehq/airbyte/pull/39342) | Fix custom conversion in CDC for datetimeoffset type. | +| 4.0.27 | 2024-05-29 | [38584](https://github.com/airbytehq/airbyte/pull/38584) | Set is_resumable flag in discover. | +| 4.0.26 | 2024-05-16 | [38292](https://github.com/airbytehq/airbyte/pull/38292) | Improve cursor value query to return only one row | +| 4.0.25 | 2024-05-29 | [38775](https://github.com/airbytehq/airbyte/pull/38775) | Publish CDK | +| 4.0.24 | 2024-05-23 | [38640](https://github.com/airbytehq/airbyte/pull/38640) | Sync sending trace status messages indicating progress. | +| 4.0.23 | 2024-05-15 | [38208](https://github.com/airbytehq/airbyte/pull/38208) | disable counts in full refresh stream in state message. | +| 4.0.22 | 2024-05-14 | [38196](https://github.com/airbytehq/airbyte/pull/38196) | Bump jdbc driver version to 12.6.1.jre11 | +| 4.0.21 | 2024-05-07 | [38054](https://github.com/airbytehq/airbyte/pull/38054) | Resumeable refresh should run only if there is source defined pk. | +| 4.0.20 | 2024-05-07 | [38042](https://github.com/airbytehq/airbyte/pull/38042) | Bump debezium version to latest. | +| 4.0.19 | 2024-05-07 | [38029](https://github.com/airbytehq/airbyte/pull/38029) | Fix previous release. | +| 4.0.18 | 2024-04-30 | [37451](https://github.com/airbytehq/airbyte/pull/37451) | Resumable full refresh read of tables. | +| 4.0.17 | 2024-05-02 | [37781](https://github.com/airbytehq/airbyte/pull/37781) | Adopt latest CDK. | +| 4.0.16 | 2024-05-01 | [37742](https://github.com/airbytehq/airbyte/pull/37742) | Adopt latest CDK. Remove Debezium retries. | +| 4.0.15 | 2024-04-22 | [37541](https://github.com/airbytehq/airbyte/pull/37541) | Adopt latest CDK. reduce excessive logs. | +| 4.0.14 | 2024-04-22 | [37476](https://github.com/airbytehq/airbyte/pull/37476) | Adopt latest CDK. | +| 4.0.13 | 2024-04-16 | [37111](https://github.com/airbytehq/airbyte/pull/37111) | Populate null values in record message. | +| 4.0.12 | 2024-04-15 | [37326](https://github.com/airbytehq/airbyte/pull/37326) | Allow up to 60 minutes of wait for the an initial CDC record. | +| 4.0.11 | 2024-04-15 | [37325](https://github.com/airbytehq/airbyte/pull/37325) | Populate airbyte_meta.changes + error handling. | +| 4.0.10 | 2024-04-15 | [37110](https://github.com/airbytehq/airbyte/pull/37110) | Internal cleanup. | +| 4.0.9 | 2024-04-10 | [36919](https://github.com/airbytehq/airbyte/pull/36919) | Fix a bug in conversion of null values. | +| 4.0.8 | 2024-04-05 | [36872](https://github.com/airbytehq/airbyte/pull/36872) | Update to connector's metadat definition. | +| 4.0.7 | 2024-04-03 | [36772](https://github.com/airbytehq/airbyte/pull/36772) | Adopt latest CDK. | +| 4.0.6 | 2024-03-25 | [36333](https://github.com/airbytehq/airbyte/pull/36333) | Deprecate Dbz state iterator. | +| 4.0.5 | 2024-03-21 | [36364](https://github.com/airbytehq/airbyte/pull/36364) | Allow up to 40 minutes of wait for the an initial CDC record. | +| 4.0.4 | 2024-03-20 | [36325](https://github.com/airbytehq/airbyte/pull/36325) | [Refactor] : Remove mssql initial source operations . | +| 4.0.3 | 2024-03-19 | [36263](https://github.com/airbytehq/airbyte/pull/36263) | Fix a failure seen in CDC with tables containing default values. | +| 4.0.2 | 2024-03-06 | [35792](https://github.com/airbytehq/airbyte/pull/35792) | Initial sync will now send record count in state message. | +| 4.0.1 | 2024-03-12 | [36011](https://github.com/airbytehq/airbyte/pull/36011) | Read correctly null values of columns with default value in CDC. | +| 4.0.0 | 2024-03-06 | [35873](https://github.com/airbytehq/airbyte/pull/35873) | Terabyte-sized tables support, reliability improvements, bug fixes. | +| 3.7.7 | 2024-03-06 | [35816](https://github.com/airbytehq/airbyte/pull/35816) | Fix query that was failing on a case sensitive server. | +| 3.7.6 | 2024-03-04 | [35721](https://github.com/airbytehq/airbyte/pull/35721) | Fix tests | +| 3.7.5 | 2024-02-29 | [35739](https://github.com/airbytehq/airbyte/pull/35739) | Allow configuring the queue size used for cdc events. | +| 3.7.4 | 2024-02-26 | [35566](https://github.com/airbytehq/airbyte/pull/35566) | Add config to throw an error on invalid CDC position. | +| 3.7.3 | 2024-02-23 | [35596](https://github.com/airbytehq/airbyte/pull/35596) | Fix a logger issue | +| 3.7.2 | 2024-02-21 | [35368](https://github.com/airbytehq/airbyte/pull/35368) | Change query syntax to make it compatible with Azure SQL Managed Instance. | +| 3.7.1 | 2024-02-20 | [35405](https://github.com/airbytehq/airbyte/pull/35405) | Change query syntax to make it compatible with Azure Synapse. | +| 3.7.0 | 2024-01-30 | [33311](https://github.com/airbytehq/airbyte/pull/33311) | Source mssql with checkpointing initial sync. | +| 3.6.1 | 2024-01-26 | [34573](https://github.com/airbytehq/airbyte/pull/34573) | Adopt CDK v0.16.0. | +| 3.6.0 | 2024-01-10 | [33700](https://github.com/airbytehq/airbyte/pull/33700) | Remove CDC config options for data_to_sync and snapshot isolation. | +| 3.5.1 | 2024-01-05 | [33510](https://github.com/airbytehq/airbyte/pull/33510) | Test-only changes. | +| 3.5.0 | 2023-12-19 | [33071](https://github.com/airbytehq/airbyte/pull/33071) | Fix SSL configuration parameters | +| 3.4.1 | 2024-01-02 | [33755](https://github.com/airbytehq/airbyte/pull/33755) | Encode binary to base64 format | +| 3.4.0 | 2023-12-19 | [33481](https://github.com/airbytehq/airbyte/pull/33481) | Remove LEGACY state flag | +| 3.3.2 | 2023-12-14 | [33505](https://github.com/airbytehq/airbyte/pull/33505) | Using the released CDK. | +| 3.3.1 | 2023-12-12 | [33225](https://github.com/airbytehq/airbyte/pull/33225) | extracting MsSql specific files out of the CDK. | +| 3.3.0 | 2023-12-12 | [33018](https://github.com/airbytehq/airbyte/pull/33018) | Migrate to Per-stream/Global states and away from Legacy states | +| 3.2.1 | 2023-12-11 | [33330](https://github.com/airbytehq/airbyte/pull/33330) | Parse DatetimeOffset fields with the correct format when used as cursor | +| 3.2.0 | 2023-12-07 | [33225](https://github.com/airbytehq/airbyte/pull/33225) | CDC : Enable compression of schema history blob in state. | +| 3.1.0 | 2023-11-28 | [32882](https://github.com/airbytehq/airbyte/pull/32882) | Enforce SSL on Airbyte Cloud. | +| 3.0.2 | 2023-11-27 | [32573](https://github.com/airbytehq/airbyte/pull/32573) | Format Datetime and Datetime2 datatypes to 6-digit microsecond precision | +| 3.0.1 | 2023-11-22 | [32656](https://github.com/airbytehq/airbyte/pull/32656) | Adopt java CDK version 0.5.0. | +| 3.0.0 | 2023-11-07 | [31531](https://github.com/airbytehq/airbyte/pull/31531) | Remapped date, smalldatetime, datetime2, time, and datetimeoffset datatype to their correct Airbyte types | | 2.0.4 | 2023-11-06 | [#32193](https://github.com/airbytehq/airbyte/pull/32193) | Adopt java CDK version 0.4.1. | | 2.0.3 | 2023-10-31 | [32024](https://github.com/airbytehq/airbyte/pull/32024) | Upgrade to Debezium version 2.4.0. | | 2.0.2 | 2023-10-30 | [31960](https://github.com/airbytehq/airbyte/pull/31960) | Adopt java CDK version 0.2.0. |