Skip to content

Commit fcbfaa3

Browse files
akashkulkakashkulk
authored
[DB source errors] : Handle common transient errors (#38104)
1 parent 59cdc36 commit fcbfaa3

File tree

11 files changed

+29
-6
lines changed

11 files changed

+29
-6
lines changed

‎airbyte-cdk/java/airbyte-cdk/README.md

+1
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,7 @@ corresponds to that version.
174174

175175
| Version | Date | Pull Request | Subject |
176176
|:--------| :--------- | :--------------------------------------------------------- |:---------------------------------------------------------------------------------------------------------------------------------------------------------------|
177+
| 0.35.2 | 2024-05-13 | [\#38104](https://github.com/airbytehq/airbyte/pull/38104) | Handle transient error messages |
177178
| 0.35.0 | 2024-05-13 | [\#38127](https://github.com/airbytehq/airbyte/pull/38127) | Destinations: Populate generation/sync ID on StreamConfig |
178179
| 0.34.4 | 2024-05-10 | [\#37712](https://github.com/airbytehq/airbyte/pull/37712) | make sure the exceptionHandler always terminates |
179180
| 0.34.4 | 2024-05-10 | [\#37712](https://github.com/airbytehq/airbyte/pull/37712) | make sure the exceptionHandler always terminates |

‎airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/util/ConnectorExceptionUtil.kt

+19-1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import io.airbyte.commons.exceptions.ConfigErrorException
99
import io.airbyte.commons.exceptions.ConnectionErrorException
1010
import io.airbyte.commons.exceptions.TransientErrorException
1111
import io.airbyte.commons.functional.Either
12+
import java.io.EOFException
1213
import java.sql.SQLException
1314
import java.sql.SQLSyntaxErrorException
1415
import java.util.stream.Collectors
@@ -25,6 +26,8 @@ object ConnectorExceptionUtil {
2526
const val RECOVERY_CONNECTION_ERROR_MESSAGE: String =
2627
"We're having issues syncing from a Postgres replica that is configured as a hot standby server. " +
2728
"Please see https://go.airbyte.com/pg-hot-standby-error-message for options and workarounds"
29+
const val DATABASE_CONNECTION_ERROR: String =
30+
"Encountered an error while connecting to the database error"
2831

2932
@JvmField val HTTP_AUTHENTICATION_ERROR_CODES: List<Int> = ImmutableList.of(401, 403)
3033

@@ -35,7 +38,10 @@ object ConnectorExceptionUtil {
3538
}
3639

3740
fun isTransientError(e: Throwable?): Boolean {
38-
return isTransientErrorException(e) || isRecoveryConnectionException(e)
41+
return isTransientErrorException(e) ||
42+
isRecoveryConnectionException(e) ||
43+
isTransientEOFException(e) ||
44+
isTransientSQLException(e)
3945
}
4046

4147
fun getDisplayMessage(e: Throwable?): String? {
@@ -49,6 +55,8 @@ object ConnectorExceptionUtil {
4955
RECOVERY_CONNECTION_ERROR_MESSAGE
5056
} else if (isUnknownColumnInFieldListException(e)) {
5157
e!!.message
58+
} else if (isTransientError(e)) {
59+
DATABASE_CONNECTION_ERROR
5260
} else {
5361
String.format(
5462
COMMON_EXCEPTION_MESSAGE_TEMPLATE,
@@ -137,6 +145,16 @@ object ConnectorExceptionUtil {
137145
return e is ConnectionErrorException
138146
}
139147

148+
private fun isTransientEOFException(e: Throwable?): Boolean {
149+
return (e is EOFException) &&
150+
e.message!!.lowercase().contains("connection was unexpectedly lost")
151+
}
152+
153+
private fun isTransientSQLException(e: Throwable?): Boolean {
154+
return (e is SQLException) &&
155+
e.message!!.lowercase().contains("An I/O error occurred while sending to the backend")
156+
}
157+
140158
private fun isRecoveryConnectionException(e: Throwable?): Boolean {
141159
return e is SQLException &&
142160
e.message!!.lowercase().contains("due to conflict with recovery")

‎airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
version=0.35.1
1+
version=0.35.2

‎airbyte-integrations/connectors/source-mysql/build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ plugins {
66
}
77

88
airbyteJavaConnector {
9-
cdkVersionRequired = '0.33.1'
9+
cdkVersionRequired = '0.35.2'
1010
features = ['db-sources']
1111
useLocalCdk = false
1212
}

‎airbyte-integrations/connectors/source-mysql/metadata.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ data:
99
connectorSubtype: database
1010
connectorType: source
1111
definitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad
12-
dockerImageTag: 3.4.2
12+
dockerImageTag: 3.4.3
1313
dockerRepository: airbyte/source-mysql
1414
documentationUrl: https://docs.airbyte.com/integrations/sources/mysql
1515
githubIssueLabel: source-mysql

‎airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java

+1
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@
5959
import org.junit.jupiter.api.Test;
6060

6161
@Order(2)
62+
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "NP_NULL_ON_SOME_PATH")
6263
class MySqlJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest<MySqlSource, MySQLTestDatabase> {
6364

6465
protected static final String USERNAME_WITHOUT_PERMISSION = "new_user";

‎airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSslJdbcSourceAcceptanceTest.java

+1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import org.junit.jupiter.api.Order;
1010

1111
@Order(3)
12+
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "NP_NULL_ON_SOME_PATH")
1213
class MySqlSslJdbcSourceAcceptanceTest extends MySqlJdbcSourceAcceptanceTest {
1314

1415
@Override

‎airbyte-integrations/connectors/source-postgres/build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ java {
1212
}
1313

1414
airbyteJavaConnector {
15-
cdkVersionRequired = '0.35.1'
15+
cdkVersionRequired = '0.35.2'
1616
features = ['db-sources', 'datastore-postgres']
1717
useLocalCdk = false
1818
}

‎airbyte-integrations/connectors/source-postgres/metadata.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ data:
99
connectorSubtype: database
1010
connectorType: source
1111
definitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
12-
dockerImageTag: 3.4.2
12+
dockerImageTag: 3.4.3
1313
dockerRepository: airbyte/source-postgres
1414
documentationUrl: https://docs.airbyte.com/integrations/sources/postgres
1515
githubIssueLabel: source-postgres

‎docs/integrations/sources/mysql.md

+1
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,7 @@ Any database or table encoding combination of charset and collation is supported
230230

231231
| Version | Date | Pull Request | Subject |
232232
|:--------|:-----------| :--------------------------------------------------------- |:------------------------------------------------------------------------------------------------------------------------------------------------|
233+
| 3.4.3 | 2024-05-13 | [38104](https://github.com/airbytehq/airbyte/pull/38104) | Handle transient error messages. |
233234
| 3.4.2 | 2024-05-07 | [38046](https://github.com/airbytehq/airbyte/pull/38046) | Resumeable refresh should run only if there is source defined pk. |
234235
| 3.4.1 | 2024-05-03 | [37824](https://github.com/airbytehq/airbyte/pull/37824) | Fixed a bug on Resumeable full refresh where cursor based source throw NPE. |
235236
| 3.4.0 | 2024-05-02 | [36932](https://github.com/airbytehq/airbyte/pull/36932) | Resumeable full refresh. Note please upgrade your platform - minimum platform version is 0.58.0. |

‎docs/integrations/sources/postgres.md

+1
Original file line numberDiff line numberDiff line change
@@ -308,6 +308,7 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp
308308

309309
| Version | Date | Pull Request | Subject |
310310
|---------|------------|----------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
311+
| 3.4.3 | 2024-05-13 | [38104](https://github.com/airbytehq/airbyte/pull/38104) | Handle transient error messages. |
311312
| 3.4.2 | 2024-05-10 | [38171](https://github.com/airbytehq/airbyte/pull/38171) | Bug fix on final state setup. |
312313
| 3.4.1 | 2024-05-10 | [38130](https://github.com/airbytehq/airbyte/pull/38130) | Bug fix on old PG where ctid column not found when stream is a view. |
313314
| 3.4.0 | 2024-04-29 | [37112](https://github.com/airbytehq/airbyte/pull/37112) | resumeable full refresh. |

0 commit comments

Comments
 (0)