Skip to content

Commit ed44c09

Browse files
theyuelitheyueli
authored
[source-mysql] Upgrade Debezium and mysql connector to 2.7.1.Final (#44013)
1 parent a0eb14e commit ed44c09

File tree

6 files changed

+28
-28
lines changed

6 files changed

+28
-28
lines changed

‎airbyte-integrations/connectors/source-mysql/build.gradle

+2-2
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@ application {
2424

2525
dependencies {
2626
implementation 'mysql:mysql-connector-java:8.0.30'
27-
implementation 'io.debezium:debezium-embedded:2.5.4.Final'
28-
implementation 'io.debezium:debezium-connector-mysql:2.5.4.Final'
27+
implementation 'io.debezium:debezium-embedded:2.7.1.Final'
28+
implementation 'io.debezium:debezium-connector-mysql:2.7.1.Final'
2929

3030
testFixturesImplementation 'org.testcontainers:mysql:1.19.0'
3131

‎airbyte-integrations/connectors/source-mysql/metadata.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ data:
99
connectorSubtype: database
1010
connectorType: source
1111
definitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad
12-
dockerImageTag: 3.6.9
12+
dockerImageTag: 3.7.0
1313
dockerRepository: airbyte/source-mysql
1414
documentationUrl: https://docs.airbyte.com/integrations/sources/mysql
1515
githubIssueLabel: source-mysql

‎airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/CustomMySQLTinyIntOneToBooleanConverter.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
package io.airbyte.integrations.source.mysql.cdc;
66

7-
import io.debezium.connector.mysql.converters.TinyIntOneToBooleanConverter;
7+
import io.debezium.connector.binlog.converters.TinyIntOneToBooleanConverter;
88
import io.debezium.spi.converter.RelationalColumn;
99
import org.apache.kafka.connect.data.SchemaBuilder;
1010

‎airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySqlDebeziumStateUtil.java

+13-13
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,11 @@
2323
import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog;
2424
import io.debezium.config.Configuration;
2525
import io.debezium.connector.common.OffsetReader;
26-
import io.debezium.connector.mysql.GtidSet;
2726
import io.debezium.connector.mysql.MySqlConnectorConfig;
2827
import io.debezium.connector.mysql.MySqlOffsetContext;
2928
import io.debezium.connector.mysql.MySqlOffsetContext.Loader;
3029
import io.debezium.connector.mysql.MySqlPartition;
31-
import io.debezium.connector.mysql.strategy.mysql.MySqlGtidSet;
30+
import io.debezium.connector.mysql.gtid.MySqlGtidSet;
3231
import io.debezium.engine.ChangeEvent;
3332
import io.debezium.pipeline.spi.Offsets;
3433
import io.debezium.pipeline.spi.Partition;
@@ -71,12 +70,12 @@ public boolean savedOffsetStillPresentOnServer(final JdbcDatabase database, fina
7170
if (gtidSetFromSavedState.isContainedWithin(availableGtidSet)) {
7271
LOGGER.info("MySQL server current GTID set {} does contain the GTID set required by the connector {}", availableGtidSet,
7372
gtidSetFromSavedState);
74-
final Optional<GtidSet> gtidSetToReplicate = subtractGtidSet(availableGtidSet, gtidSetFromSavedState, database);
73+
final Optional<MySqlGtidSet> gtidSetToReplicate = subtractGtidSet(availableGtidSet, gtidSetFromSavedState, database);
7574
if (gtidSetToReplicate.isPresent()) {
76-
final Optional<GtidSet> purgedGtidSet = purgedGtidSet(database);
75+
final Optional<MySqlGtidSet> purgedGtidSet = purgedGtidSet(database);
7776
if (purgedGtidSet.isPresent()) {
7877
LOGGER.info("MySQL server has already purged {} GTIDs", purgedGtidSet.get());
79-
final Optional<GtidSet> nonPurgedGtidSetToReplicate = subtractGtidSet(gtidSetToReplicate.get(), purgedGtidSet.get(), database);
78+
final Optional<MySqlGtidSet> nonPurgedGtidSetToReplicate = subtractGtidSet(gtidSetToReplicate.get(), purgedGtidSet.get(), database);
8079
if (nonPurgedGtidSetToReplicate.isPresent()) {
8180
LOGGER.info("GTIDs known by the MySQL server but not processed yet {}, for replication are available only {}", gtidSetToReplicate,
8281
nonPurgedGtidSetToReplicate);
@@ -116,16 +115,16 @@ private List<String> getExistingLogFiles(final JdbcDatabase database) {
116115
}
117116
}
118117

119-
private Optional<GtidSet> subtractGtidSet(final GtidSet set1, final GtidSet set2, final JdbcDatabase database) {
120-
try (final Stream<GtidSet> stream = database.unsafeResultSetQuery(
118+
private Optional<MySqlGtidSet> subtractGtidSet(final MySqlGtidSet set1, final MySqlGtidSet set2, final JdbcDatabase database) {
119+
try (final Stream<MySqlGtidSet> stream = database.unsafeResultSetQuery(
121120
connection -> {
122121
final PreparedStatement ps = connection.prepareStatement("SELECT GTID_SUBTRACT(?, ?)");
123122
ps.setString(1, set1.toString());
124123
ps.setString(2, set2.toString());
125124
return ps.executeQuery();
126125
},
127126
resultSet -> new MySqlGtidSet(resultSet.getString(1)))) {
128-
final List<GtidSet> gtidSets = stream.toList();
127+
final List<MySqlGtidSet> gtidSets = stream.toList();
129128
if (gtidSets.isEmpty()) {
130129
return Optional.empty();
131130
} else if (gtidSets.size() == 1) {
@@ -138,8 +137,8 @@ private Optional<GtidSet> subtractGtidSet(final GtidSet set1, final GtidSet set2
138137
}
139138
}
140139

141-
private Optional<GtidSet> purgedGtidSet(final JdbcDatabase database) {
142-
try (final Stream<Optional<GtidSet>> stream = database.unsafeResultSetQuery(
140+
private Optional<MySqlGtidSet> purgedGtidSet(final JdbcDatabase database) {
141+
try (final Stream<Optional<MySqlGtidSet>> stream = database.unsafeResultSetQuery(
143142
connection -> connection.createStatement().executeQuery("SELECT @@global.gtid_purged"),
144143
resultSet -> {
145144
if (resultSet.getMetaData().getColumnCount() > 0) {
@@ -150,7 +149,7 @@ private Optional<GtidSet> purgedGtidSet(final JdbcDatabase database) {
150149
}
151150
return Optional.empty();
152151
})) {
153-
final List<Optional<GtidSet>> gtidSet = stream.toList();
152+
final List<Optional<MySqlGtidSet>> gtidSet = stream.toList();
154153
if (gtidSet.isEmpty()) {
155154
return Optional.empty();
156155
} else if (gtidSet.size() == 1) {
@@ -237,9 +236,10 @@ public JsonNode constructInitialDebeziumState(final Properties properties,
237236
final ConfiguredAirbyteCatalog catalog,
238237
final JdbcDatabase database) {
239238
// https://debezium.io/documentation/reference/2.2/connectors/mysql.html#mysql-property-snapshot-mode
240-
// We use the schema_only_recovery property cause using this mode will instruct Debezium to
239+
// We use the recovery property cause using this mode will instruct Debezium to
241240
// construct the db schema history.
242-
properties.setProperty("snapshot.mode", "schema_only_recovery");
241+
// Note that we used to use schema_only_recovery mode, but this mode has been deprecated.
242+
properties.setProperty("snapshot.mode", "recovery");
243243
final String dbName = database.getSourceConfig().get(JdbcUtils.DATABASE_KEY).asText();
244244
// Topic.prefix is sanitized version of database name. At this stage properties does not have this
245245
// value - it's set in RelationalDbDebeziumPropertiesManager.

‎airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java

+4-5
Original file line numberDiff line numberDiff line change
@@ -780,11 +780,10 @@ public void testTwoStreamSync() throws Exception {
780780
}
781781

782782
/**
783-
* This test creates lots of tables increasing the schema history size above the limit of
784-
* {@link AirbyteSchemaHistoryStorage#SIZE_LIMIT_TO_COMPRESS_MB} forcing the
785-
* {@link AirbyteSchemaHistoryStorage#read()} method to compress the schema history blob as part of
786-
* the state message which allows us to test that the next sync is able to work fine when provided
787-
* with a compressed blob in the state.
783+
* This test creates lots of tables increasing the schema history size above the limit of forcing
784+
* the {@link AirbyteSchemaHistoryStorage#read()} method to compress the schema history blob as part
785+
* of the state message which allows us to test that the next sync is able to work fine when
786+
* provided with a compressed blob in the state.
788787
*/
789788
@Test
790789
@Timeout(value = 120)

‎docs/integrations/sources/mysql.md

+7-6
Original file line numberDiff line numberDiff line change
@@ -233,12 +233,13 @@ Any database or table encoding combination of charset and collation is supported
233233

234234
| Version | Date | Pull Request | Subject |
235235
|:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------|
236-
| 3.6.9 | 2024-08-08 | [43410](https://github.com/airbytehq/airbyte/pull/43410) | Adopt latest CDK. |
237-
| 3.6.8 | 2024-07-30 | [42869](https://github.com/airbytehq/airbyte/pull/42869) | Adopt latest CDK. |
238-
| 3.6.7 | 2024-07-30 | [42550](https://github.com/airbytehq/airbyte/pull/42550) | Correctly report stream states. |
239-
| 3.6.6 | 2024-07-29 | [42852](https://github.com/airbytehq/airbyte/pull/42852) | Bump CDK version to latest to use new bug fixes on error translation. |
240-
| 3.6.5 | 2024-07-24 | [42417](https://github.com/airbytehq/airbyte/pull/42417) | Handle null error message in ConnectorExceptionHandler. |
241-
| 3.6.4 | 2024-07-23 | [42421](https://github.com/airbytehq/airbyte/pull/42421) | Remove final transient error emitter iterators. |
236+
| 3.7.0 | 2024-08-13 | [44013](https://github.com/airbytehq/airbyte/pull/44013) | Upgrading to Debezium 2.7.1.Final |
237+
| 3.6.9 | 2024-08-08 | [43410](https://github.com/airbytehq/airbyte/pull/43410) | Adopt latest CDK. |
238+
| 3.6.8 | 2024-07-30 | [42869](https://github.com/airbytehq/airbyte/pull/42869) | Adopt latest CDK. |
239+
| 3.6.7 | 2024-07-30 | [42550](https://github.com/airbytehq/airbyte/pull/42550) | Correctly report stream states. |
240+
| 3.6.6 | 2024-07-29 | [42852](https://github.com/airbytehq/airbyte/pull/42852) | Bump CDK version to latest to use new bug fixes on error translation. |
241+
| 3.6.5 | 2024-07-24 | [42417](https://github.com/airbytehq/airbyte/pull/42417) | Handle null error message in ConnectorExceptionHandler. |
242+
| 3.6.4 | 2024-07-23 | [42421](https://github.com/airbytehq/airbyte/pull/42421) | Remove final transient error emitter iterators. |
242243
| 3.6.3 | 2024-07-22 | [42024](https://github.com/airbytehq/airbyte/pull/42024) | Fix a NPE bug on resuming from a failed attempt. |
243244
| 3.6.2 | 2024-07-17 | [42087](https://github.com/airbytehq/airbyte/pull/42087) | Adding more error translations for MySql source. |
244245
| 3.6.1 | 2024-07-19 | [42122](https://github.com/airbytehq/airbyte/pull/42122) | Improve wass error message + logging. |

0 commit comments

Comments
 (0)