Skip to content

Commit 0296c43

Browse files
akashkulkakashkulk
authored
[Db analytics] : add message for data type serialization error (#36981)
1 parent c7e5d20 commit 0296c43

File tree

8 files changed

+39
-16
lines changed

8 files changed

+39
-16
lines changed

‎airbyte-cdk/java/airbyte-cdk/README.md

+1
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,7 @@ Maven and Gradle will automatically reference the correct (pinned) version of th
144144

145145
| Version | Date | Pull Request | Subject |
146146
|:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------|
147+
| 0.29.13 | 2024-04-10 | [\#36981](https://github.com/airbytehq/airbyte/pull/36981) | DB sources : Emit analytics for data type serialization errors. |
147148
| 0.29.11 | 2024-04-10 | [\#36865](https://github.com/airbytehq/airbyte/pull/36865) | Sources : Remove noisy log line. |
148149
| 0.29.10 | 2024-04-10 | [\#36805](https://github.com/airbytehq/airbyte/pull/36805) | Destinations: Enhance CatalogParser name collision handling; add DV2 tests for long identifiers |
149150
| 0.29.9 | 2024-04-09 | [\#36047](https://github.com/airbytehq/airbyte/pull/36047) | Destinations: CDK updates for raw-only destinations |

‎airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/db/DbAnalyticsUtils.kt

+8
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,17 @@ import io.airbyte.protocol.models.v0.AirbyteAnalyticsTraceMessage
1212
*/
1313
object DbAnalyticsUtils {
1414
const val CDC_CURSOR_INVALID_KEY: String = "db-sources-cdc-cursor-invalid"
15+
const val DATA_TYPES_SERIALIZATION_ERROR_KEY = "db-sources-data-serialization-error"
1516

1617
@JvmStatic
1718
fun cdcCursorInvalidMessage(): AirbyteAnalyticsTraceMessage {
1819
return AirbyteAnalyticsTraceMessage().withType(CDC_CURSOR_INVALID_KEY).withValue("1")
1920
}
21+
22+
@JvmStatic
23+
fun dataTypesSerializationErrorMessage(): AirbyteAnalyticsTraceMessage {
24+
return AirbyteAnalyticsTraceMessage()
25+
.withType(DATA_TYPES_SERIALIZATION_ERROR_KEY)
26+
.withValue("1")
27+
}
2028
}

‎airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/db/jdbc/AbstractJdbcCompatibleSourceOperations.kt

+20-12
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,9 @@ import com.fasterxml.jackson.databind.JsonNode
77
import com.fasterxml.jackson.databind.ObjectMapper
88
import com.fasterxml.jackson.databind.node.ObjectNode
99
import io.airbyte.cdk.db.DataTypeUtils
10+
import io.airbyte.cdk.db.DbAnalyticsUtils.dataTypesSerializationErrorMessage
1011
import io.airbyte.cdk.db.JdbcCompatibleSourceOperations
11-
import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations.Companion.LOGGER
12+
import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility
1213
import io.airbyte.commons.json.Jsons
1314
import io.airbyte.protocol.models.v0.AirbyteRecordMessageMeta
1415
import io.airbyte.protocol.models.v0.AirbyteRecordMessageMetaChange
@@ -40,18 +41,25 @@ abstract class AbstractJdbcCompatibleSourceOperations<Datatype> :
4041

4142
for (i in 1..columnCount) {
4243
val columnName = queryContext.metaData.getColumnName(i)
44+
val columnTypeName = queryContext.metaData.getColumnTypeName(i)
4345
try {
4446
// convert to java types that will convert into reasonable json.
4547
copyToJsonField(queryContext, i, jsonNode)
4648
} catch (e: java.lang.Exception) {
47-
LOGGER.info("Failed to serialize column: {}, with error {}", columnName, e.message)
49+
LOGGER.info(
50+
"Failed to serialize column: {}, of type {}, with error {}",
51+
columnName,
52+
columnTypeName,
53+
e.message
54+
)
55+
AirbyteTraceMessageUtility.emitAnalyticsTrace(dataTypesSerializationErrorMessage())
4856
metaChanges.add(
4957
AirbyteRecordMessageMetaChange()
5058
.withField(columnName)
5159
.withChange(AirbyteRecordMessageMetaChange.Change.NULLED)
5260
.withReason(
53-
AirbyteRecordMessageMetaChange.Reason.SOURCE_SERIALIZATION_ERROR
54-
)
61+
AirbyteRecordMessageMetaChange.Reason.SOURCE_SERIALIZATION_ERROR,
62+
),
5563
)
5664
}
5765
}
@@ -166,8 +174,8 @@ abstract class AbstractJdbcCompatibleSourceOperations<Datatype> :
166174
columnName,
167175
DataTypeUtils.returnNullIfInvalid(
168176
{ resultSet.getDouble(index) },
169-
{ d: Double? -> java.lang.Double.isFinite(d!!) }
170-
)
177+
{ d: Double? -> java.lang.Double.isFinite(d!!) },
178+
),
171179
)
172180
}
173181

@@ -182,8 +190,8 @@ abstract class AbstractJdbcCompatibleSourceOperations<Datatype> :
182190
columnName,
183191
DataTypeUtils.returnNullIfInvalid(
184192
{ resultSet.getFloat(index) },
185-
{ f: Float? -> java.lang.Float.isFinite(f!!) }
186-
)
193+
{ f: Float? -> java.lang.Float.isFinite(f!!) },
194+
),
187195
)
188196
}
189197

@@ -226,7 +234,7 @@ abstract class AbstractJdbcCompatibleSourceOperations<Datatype> :
226234
) {
227235
node.put(
228236
columnName,
229-
DateTimeConverter.convertToTime(getObject(resultSet, index, LocalTime::class.java))
237+
DateTimeConverter.convertToTime(getObject(resultSet, index, LocalTime::class.java)),
230238
)
231239
}
232240

@@ -241,8 +249,8 @@ abstract class AbstractJdbcCompatibleSourceOperations<Datatype> :
241249
node.put(
242250
columnName,
243251
DateTimeConverter.convertToTimestamp(
244-
getObject(resultSet, index, LocalDateTime::class.java)
245-
)
252+
getObject(resultSet, index, LocalDateTime::class.java),
253+
),
246254
)
247255
} catch (e: Exception) {
248256
// for backward compatibility
@@ -450,7 +458,7 @@ abstract class AbstractJdbcCompatibleSourceOperations<Datatype> :
450458
val localDate = timestamptz.toLocalDate()
451459
node.put(
452460
columnName,
453-
resolveEra(localDate, timestamptz.format(DataTypeUtils.TIMESTAMPTZ_FORMATTER))
461+
resolveEra(localDate, timestamptz.format(DataTypeUtils.TIMESTAMPTZ_FORMATTER)),
454462
)
455463
}
456464

‎airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
version=0.29.11
1+
version=0.29.13

‎airbyte-integrations/connectors/source-postgres/build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ java {
1212
}
1313

1414
airbyteJavaConnector {
15-
cdkVersionRequired = '0.29.11'
15+
cdkVersionRequired = '0.29.13'
1616
features = ['db-sources', 'datastore-postgres']
1717
useLocalCdk = false
1818
}

‎airbyte-integrations/connectors/source-postgres/metadata.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ data:
99
connectorSubtype: database
1010
connectorType: source
1111
definitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
12-
dockerImageTag: 3.3.24
12+
dockerImageTag: 3.3.25
1313
dockerRepository: airbyte/source-postgres
1414
documentationUrl: https://docs.airbyte.com/integrations/sources/postgres
1515
githubIssueLabel: source-postgres

‎airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidPostgresSourceOperations.java

+6-1
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,11 @@
44

55
package io.airbyte.integrations.source.postgres.ctid;
66

7+
import static io.airbyte.cdk.db.DbAnalyticsUtils.dataTypesSerializationErrorMessage;
8+
79
import com.fasterxml.jackson.databind.node.ObjectNode;
810
import io.airbyte.cdk.db.jdbc.AirbyteRecordData;
11+
import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility;
912
import io.airbyte.commons.json.Jsons;
1013
import io.airbyte.integrations.source.postgres.PostgresSourceOperations;
1114
import io.airbyte.integrations.source.postgres.cdc.PostgresCdcConnectorMetadataInjector;
@@ -45,6 +48,7 @@ public RowDataWithCtid recordWithCtid(final ResultSet queryContext) throws SQLEx
4548
final List<AirbyteRecordMessageMetaChange> metaChanges = new ArrayList<>();
4649
for (int i = 1; i <= columnCount; i++) {
4750
final String columnName = metadata.getColumnName(i);
51+
final String columnTypeName = metadata.getColumnTypeName(i);
4852
try {
4953
if (columnName.equalsIgnoreCase(CTID)) {
5054
ctid = queryContext.getString(i);
@@ -54,7 +58,8 @@ public RowDataWithCtid recordWithCtid(final ResultSet queryContext) throws SQLEx
5458
// convert to java types that will convert into reasonable json.
5559
copyToJsonField(queryContext, i, jsonNode);
5660
} catch (Exception e) {
57-
LOGGER.info("Failed to serialize column: {}, with error {}", columnName, e.getMessage());
61+
LOGGER.info("Failed to serialize column: {}, of type {}, with error {}", columnName, columnTypeName, e.getMessage());
62+
AirbyteTraceMessageUtility.emitAnalyticsTrace(dataTypesSerializationErrorMessage());
5863
metaChanges.add(
5964
new AirbyteRecordMessageMetaChange()
6065
.withField(columnName)

‎docs/integrations/sources/postgres.md

+1
Original file line numberDiff line numberDiff line change
@@ -292,6 +292,7 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp
292292

293293
| Version | Date | Pull Request | Subject |
294294
|---------|------------|----------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
295+
| 3.3.25 | 2024-04-10 | [36981](https://github.com/airbytehq/airbyte/pull/36981) | Track latest CDK |
295296
| 3.3.24 | 2024-04-10 | [36865](https://github.com/airbytehq/airbyte/pull/36865) | Track latest CDK |
296297
| 3.3.23 | 2024-04-02 | [36759](https://github.com/airbytehq/airbyte/pull/36759) | Track latest CDK |
297298
| 3.3.22 | 2024-04-01 | [36739](https://github.com/airbytehq/airbyte/pull/36739) | Fix useLocalCdk flag. |

0 commit comments

Comments
 (0)