🎉 Updated timestamp transformation with microseconds

airbyte-db/lib/src/main/java/io/airbyte/db/DataTypeUtils.java

@@ -14,9 +14,13 @@
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.function.Function;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class DataTypeUtils {
 
+  private static final Logger LOGGER = LoggerFactory.getLogger(DataTypeUtils.class);
+
   public static final String DATE_FORMAT_PATTERN = "yyyy-MM-dd'T'HH:mm:ss'Z'";
   public static final DateFormat DATE_FORMAT = new SimpleDateFormat(DATE_FORMAT_PATTERN); // Quoted "Z" to indicate UTC, no timezone offset
 
@@ -40,6 +44,12 @@ public static <T> T returnNullIfInvalid(final DataTypeSupplier<T> valueProducer,
     }
   }
 
+  public static String toISO8601StringWithMicroseconds(Instant instant) {
+    var microSeconds = (instant.getNano() / 1000) % 1000;
+    String dateWithMilliseconds = DATE_FORMAT_WITH_MILLISECONDS.format(Date.from(instant));
+    return dateWithMilliseconds.substring(0, 23) + microSeconds + dateWithMilliseconds.substring(23);
+  }
+
   public static String toISO8601StringWithMilliseconds(final long epochMillis) {
     return DATE_FORMAT_WITH_MILLISECONDS.format(Date.from(Instant.ofEpochMilli(epochMillis)));
   }

airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java

@@ -19,16 +19,21 @@
 import java.sql.SQLException;
 import java.sql.Timestamp;
 import java.text.ParseException;
+import java.time.Instant;
 import java.util.Collections;
 import java.util.List;
 import java.util.StringJoiner;
 import javax.xml.bind.DatatypeConverter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Source operation skeleton for JDBC compatible databases.
  */
 public abstract class AbstractJdbcCompatibleSourceOperations<Datatype> implements JdbcCompatibleSourceOperations<Datatype> {
 
+  private static final Logger LOGGER = LoggerFactory.getLogger(AbstractJdbcCompatibleSourceOperations.class);
+
   @Override
   public JsonNode rowToJson(final ResultSet queryContext) throws SQLException {
     // the first call communicates with the database. after that the result is cached.
@@ -120,9 +125,8 @@ protected void putTime(final ObjectNode node, final String columnName, final Res
 
   protected void putTimestamp(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException {
     // https://www.cis.upenn.edu/~bcpierce/courses/629/jdkdocs/guide/jdbc/getstart/mapping.doc.html
-    final Timestamp t = resultSet.getTimestamp(index);
-    final java.util.Date d = new java.util.Date(t.getTime() + (t.getNanos() / 1000000));
-    node.put(columnName, DataTypeUtils.toISO8601String(d));
+    final Instant instant = resultSet.getTimestamp(index).toInstant();
+    node.put(columnName, DataTypeUtils.toISO8601StringWithMicroseconds(instant));
   }
 
   protected void putBinary(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException {
@@ -143,8 +147,10 @@ protected void setTimestamp(final PreparedStatement preparedStatement, final int
     // Parsing TIME as a TIMESTAMP might potentially break for ClickHouse cause it doesn't expect TIME
     // value in the following format
     try {
+      LOGGER.error("===>> {}", value);
+      LOGGER.error("===>> {}", DataTypeUtils.DATE_FORMAT_WITH_MILLISECONDS.parse(value));
       preparedStatement.setTimestamp(parameterIndex, Timestamp
-          .from(DataTypeUtils.DATE_FORMAT.parse(value).toInstant()));
+          .from(DataTypeUtils.DATE_FORMAT_WITH_MILLISECONDS.parse(value).toInstant()));
     } catch (final ParseException e) {
       throw new RuntimeException(e);
     }

airbyte-db/lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java

@@ -144,8 +144,8 @@ void testSetStatementField() throws SQLException {
       sourceOperations.setStatementField(ps, 11, JDBCType.CHAR, "a");
       sourceOperations.setStatementField(ps, 12, JDBCType.VARCHAR, "a");
       sourceOperations.setStatementField(ps, 13, JDBCType.DATE, "2020-11-01T00:00:00Z");
-      sourceOperations.setStatementField(ps, 14, JDBCType.TIME, "1970-01-01T05:00:00Z");
-      sourceOperations.setStatementField(ps, 15, JDBCType.TIMESTAMP, "2001-09-29T03:00:00Z");
+      sourceOperations.setStatementField(ps, 14, JDBCType.TIME, "1970-01-01T05:00:00.000Z");
+      sourceOperations.setStatementField(ps, 15, JDBCType.TIMESTAMP, "2001-09-29T03:00:00.000Z");
       sourceOperations.setStatementField(ps, 16, JDBCType.BINARY, "61616161");
 
       ps.execute();

airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtils.java

@@ -39,7 +39,7 @@ public static String convertDate(final Object input) {
     } else if (input instanceof Duration) {
       return DataTypeUtils.toISO8601String((Duration) input);
     } else if (input instanceof Timestamp) {
-      return DataTypeUtils.toISO8601String(((Timestamp) input).toLocalDateTime());
+      return DataTypeUtils.toISO8601StringWithMicroseconds((((Timestamp) input).toInstant()));
     } else if (input instanceof Number) {
       return DataTypeUtils.toISO8601String(
           new Timestamp(((Number) input).longValue()).toLocalDateTime());

airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java

@@ -327,8 +327,8 @@ protected void initTests() {
         TestDataHolder.builder()
             .sourceType("timestamp")
             .airbyteType(JsonSchemaPrimitive.STRING)
-            .addInsertValues("TIMESTAMP '2004-10-19 10:23:54'", "null")
-            .addExpectedValues("2004-10-19T10:23:54Z", null)
+            .addInsertValues("TIMESTAMP '2004-10-19 10:23:54'", "TIMESTAMP '2004-10-19 10:23:54.123456'", "null")
+            .addExpectedValues("2004-10-19T10:23:54.0000Z", "2004-10-19T10:23:54.123456Z", null)
             .build());
 
     addDataTypeTestData(

airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java

@@ -312,8 +312,9 @@ protected void initTests() {
             .createTablePatternSql(CREATE_TABLE_SQL)
             .sourceType("TIMESTAMP")
             .airbyteType(JsonSchemaPrimitive.STRING)
-            .addInsertValues("null", "'2018-03-22-12.00.00.123'", "'20180322125959'", "'20180101 12:00:59 PM'")
-            .addExpectedValues(null, "2018-03-22T12:00:00Z", "2018-03-22T12:59:59Z", "2018-01-01T12:00:59Z") // milliseconds values are erased
+            .addInsertValues("null", "'2018-03-22-12.00.00.123'", "'2018-03-22-12.00.00.123456'", "'20180322125959'", "'20180101 12:00:59 PM'")
+            .addExpectedValues(null, "2018-03-22T12:00:00.1230Z", "2018-03-22T12:00:00.123456Z", "2018-03-22T12:59:59.0000Z",
+                "2018-01-01T12:00:59.0000Z") // milliseconds values are erased
             .build());
   }
 

airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java

@@ -723,11 +723,11 @@ void testReadMultipleTablesIncrementally() throws Exception {
   }
 
   // when initial and final cursor fields are the same.
-  private void incrementalCursorCheck(
-                                      final String cursorField,
-                                      final String initialCursorValue,
-                                      final String endCursorValue,
-                                      final List<AirbyteMessage> expectedRecordMessages)
+  protected void incrementalCursorCheck(
+                                        final String cursorField,
+                                        final String initialCursorValue,
+                                        final String endCursorValue,
+                                        final List<AirbyteMessage> expectedRecordMessages)
       throws Exception {
     incrementalCursorCheck(cursorField, cursorField, initialCursorValue, endCursorValue,
         expectedRecordMessages);

airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java

@@ -340,34 +340,35 @@ protected void initTests() {
             .sourceType("smalldatetime")
             .airbyteType(JsonSchemaPrimitive.STRING)
             .addInsertValues("'1900-01-01'", "'2079-06-06'", "null")
-            .addExpectedValues("1900-01-01T00:00:00Z", "2079-06-06T00:00:00Z", null)
+            .addExpectedValues("1900-01-01T00:00:00.0000Z", "2079-06-06T00:00:00.0000Z", null)
             .createTablePatternSql(CREATE_TABLE_SQL)
             .build());
 
     addDataTypeTestData(
         TestDataHolder.builder()
             .sourceType("datetime")
             .airbyteType(JsonSchemaPrimitive.STRING)
-            .addInsertValues("'1753-01-01'", "'9999-12-31'", "null")
-            .addExpectedValues("1753-01-01T00:00:00Z", "9999-12-31T00:00:00Z", null)
+            .addInsertValues("'1753-01-01'", "'9999-12-31'", "'9999-12-31T13:00:04Z'",
+                "'9999-12-31T13:00:04.123Z'", "null")
+            .addExpectedValues("1753-01-01T00:00:00.0000Z", "9999-12-31T00:00:00.0000Z", "9999-12-31T13:00:04.0000Z",
+                "9999-12-31T13:00:04.1230Z", null)
             .createTablePatternSql(CREATE_TABLE_SQL)
             .build());
 
     addDataTypeTestData(
         TestDataHolder.builder()
             .sourceType("datetime2")
             .airbyteType(JsonSchemaPrimitive.STRING)
-            .addInsertValues("'0001-01-01'", "'9999-12-31'", "null")
-            .addExpectedValues("0001-01-01T00:00:00Z", "9999-12-31T00:00:00Z", null)
+            .addInsertValues("'0001-01-01'", "'9999-12-31'", "'9999-12-31T13:00:04.123456Z'", "null")
+            .addExpectedValues("0001-01-01T00:00:00.0000Z", "9999-12-31T00:00:00.0000Z", "9999-12-31T13:00:04.123456Z", null)
             .createTablePatternSql(CREATE_TABLE_SQL)
             .build());
 
     addDataTypeTestData(
         TestDataHolder.builder()
             .sourceType("time")
             .airbyteType(JsonSchemaPrimitive.STRING)
-            .addInsertValues("'00:00:00.0000000'", "'23:59:59.9999999'", "'00:00:00'", "'23:58'",
-                "null")
+            .addInsertValues("'00:00:00.0000000'", "'23:59:59.9999999'", "'00:00:00'", "'23:58'", "null")
             .addExpectedValues("00:00:00", "23:59:59.9999999", "00:00:00", "23:58:00", null)
             .createTablePatternSql(CREATE_TABLE_SQL)
             .build());

airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceDatatypeTest.java

@@ -181,42 +181,42 @@ protected void initTests() {
         TestDataHolder.builder()
             .sourceType("date")
             .airbyteType(JsonSchemaPrimitive.STRING)
-            .addInsertValues("'0001-01-01'", "'9999-12-31'", "'1999-01-08'",
-                "null")
-            .addExpectedValues("0001-01-01T00:00:00Z", "9999-12-31T00:00:00Z",
-                "1999-01-08T00:00:00Z", null)
+            .addInsertValues("'0001-01-01'", "'9999-12-31'", "'1999-01-08'", "null")
+            .addExpectedValues("0001-01-01T00:00:00Z", "9999-12-31T00:00:00Z", "1999-01-08T00:00:00Z", null)
             .build());
 
     addDataTypeTestData(
         TestDataHolder.builder()
             .sourceType("smalldatetime")
             .airbyteType(JsonSchemaPrimitive.STRING)
             .addInsertValues("'1900-01-01'", "'2079-06-06'", "null")
-            .addExpectedValues("1900-01-01T00:00:00Z", "2079-06-06T00:00:00Z", null)
+            .addExpectedValues("1900-01-01T00:00:00.0000Z", "2079-06-06T00:00:00.0000Z", null)
             .build());
 
     addDataTypeTestData(
         TestDataHolder.builder()
             .sourceType("datetime")
             .airbyteType(JsonSchemaPrimitive.STRING)
-            .addInsertValues("'1753-01-01'", "'9999-12-31'", "null")
-            .addExpectedValues("1753-01-01T00:00:00Z", "9999-12-31T00:00:00Z", null)
+            .addInsertValues("'1753-01-01'", "'9999-12-31'", "'9999-12-31T13:00:04Z'",
+                "'9999-12-31T13:00:04.123Z'", "null")
+            .addExpectedValues("1753-01-01T00:00:00.0000Z", "9999-12-31T00:00:00.0000Z", "9999-12-31T13:00:04.0000Z",
+                "9999-12-31T13:00:04.1230Z", null)
             .build());
 
     addDataTypeTestData(
         TestDataHolder.builder()
             .sourceType("datetime2")
             .airbyteType(JsonSchemaPrimitive.STRING)
-            .addInsertValues("'0001-01-01'", "'9999-12-31'", "null")
-            .addExpectedValues("0001-01-01T00:00:00Z", "9999-12-31T00:00:00Z", null)
+            .addInsertValues("'0001-01-01'", "'9999-12-31'", "'9999-12-31T13:00:04.123456Z'", "null")
+            .addExpectedValues("0001-01-01T00:00:00.0000Z", "9999-12-31T00:00:00.0000Z", "9999-12-31T13:00:04.123456Z", null)
             .build());
 
     addDataTypeTestData(
         TestDataHolder.builder()
             .sourceType("time")
             .airbyteType(JsonSchemaPrimitive.STRING)
-            .addInsertValues("null", "'13:00:01'", "'13:00:04Z'")
-            .addExpectedValues(null, "13:00:01.0000000", "13:00:04.0000000")
+            .addInsertValues("null", "'13:00:01'", "'13:00:04Z'", "'13:00:04.123456Z'")
+            .addExpectedValues(null, "13:00:01.0000000", "13:00:04.0000000", "13:00:04.1234560")
             .build());
 
     addDataTypeTestData(

airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceDatatypeTest.java

@@ -245,7 +245,7 @@ protected void initTests() {
             .sourceType("datetime")
             .airbyteType(JsonSchemaPrimitive.STRING)
             .addInsertValues("null", "'2005-10-10 23:22:21'")
-            .addExpectedValues(null, "2005-10-10T23:22:21Z")
+            .addExpectedValues(null, "2005-10-10T23:22:21.0000Z")
             .build());
 
     addDataTypeTestData(
@@ -262,7 +262,7 @@ protected void initTests() {
             .airbyteType(JsonSchemaPrimitive.STRING)
             // JDBC driver can process only "clock"(00:00:00-23:59:59) values.
             // https://debezium.io/documentation/reference/connectors/mysql.html#mysql-temporal-types
-            .addInsertValues("null", "'-23:59:59.123456'", "'00:00:00'")
+            .addInsertValues("null", "'-23:59:59'", "'00:00:00'")
             .addExpectedValues(null, "1970-01-01T23:59:59Z", "1970-01-01T00:00:00Z")
             .build());
 

airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceDatatypeTest.java

@@ -290,7 +290,7 @@ protected void initTests() {
             .sourceType("datetime")
             .airbyteType(JsonSchemaPrimitive.STRING)
             .addInsertValues("null", "'2005-10-10 23:22:21'")
-            .addExpectedValues(null, "2005-10-10T23:22:21Z")
+            .addExpectedValues(null, "2005-10-10T23:22:21.0000Z")
             .build());
 
     addDataTypeTestData(
@@ -306,7 +306,7 @@ protected void initTests() {
             .sourceType("time")
             .airbyteType(JsonSchemaPrimitive.STRING)
             // JDBC driver can process only "clock"(00:00:00-23:59:59) values.
-            .addInsertValues("null", "'-23:59:59.123456'", "'00:00:00'")
+            .addInsertValues("null", "'-23:59:59'", "'00:00:00'")
             .addExpectedValues(null, "1970-01-01T23:59:59Z", "1970-01-01T00:00:00Z")
             .build());
 

airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java

@@ -182,8 +182,9 @@ protected void initTests() {
         TestDataHolder.builder()
             .sourceType("DATE")
             .airbyteType(JsonSchemaPrimitive.STRING)
-            .addInsertValues("to_date('-4700/01/01','syyyy/mm/dd')", "to_date('9999/12/31 23:59:59','yyyy/mm/dd hh24:mi:ss')", "null")
-            .addExpectedValues("4700-01-01T00:00:00Z", "9999-12-31T23:59:59Z", null)
+            .addInsertValues("to_date('-4700/01/01','syyyy/mm/dd')",
+                "to_date('9999/12/31 23:59:59','yyyy/mm/dd hh24:mi:ss')", "null")
+            .addExpectedValues("4700-01-01T00:00:00.0000Z", "9999-12-31T23:59:59.0000Z", null)
             // @TODO stream fails when gets Zero date value
             // .addInsertValues("'2021/01/00'", "'2021/00/00'", "'0000/00/00'")
             .build());
@@ -192,8 +193,9 @@ protected void initTests() {
         TestDataHolder.builder()
             .sourceType("TIMESTAMP")
             .airbyteType(JsonSchemaPrimitive.STRING)
-            .addInsertValues("to_timestamp('2020-06-10 06:14:00.742000000', 'YYYY-MM-DD HH24:MI:SS.FF')")
-            .addExpectedValues("2020-06-10T06:14:01Z")
+            .addInsertValues("to_timestamp('2020-06-10 06:14:00.742', 'YYYY-MM-DD HH24:MI:SS.FF')",
+                "to_timestamp('2020-06-10 06:14:00.742123', 'YYYY-MM-DD HH24:MI:SS.FF')")
+            .addExpectedValues("2020-06-10T06:14:00.7420Z", "2020-06-10T06:14:00.742123Z")
             .build());
 
     addDataTypeTestData(
@@ -202,29 +204,32 @@ protected void initTests() {
             .airbyteType(JsonSchemaPrimitive.STRING)
             .fullSourceDataType("TIMESTAMP WITH TIME ZONE")
             .addInsertValues("to_timestamp_tz('21-FEB-2009 18:00:00 EST', 'DD-MON-YYYY HH24:MI:SS TZR')",
-                "to_timestamp_tz('21-FEB-2009 18:00:00 -5:00', 'DD-MON-YYYY HH24:MI:SS TZH:TZM')")
-            .addExpectedValues("2009-02-21 18:00:00.0 EST", "2009-02-21 18:00:00.0 -5:00")
+                "to_timestamp_tz('21-FEB-2009 18:00:00.123456 EST', 'DD-MON-YYYY HH24:MI:SS.FF TZR')",
+                "to_timestamp_tz('21-FEB-2009 18:00:00 -5:00', 'DD-MON-YYYY HH24:MI:SS TZH:TZM')",
+                "to_timestamp_tz('21-FEB-2009 18:00:00.123456 -5:00', 'DD-MON-YYYY HH24:MI:SS.FF TZH:TZM')")
+            .addExpectedValues("2009-02-21 18:00:00.0 EST", "2009-02-21 18:00:00.123456 EST",
+                "2009-02-21 18:00:00.0 -5:00", "2009-02-21 18:00:00.123456 -5:00")
             .build());
 
-    final DateFormat utcFormat = new SimpleDateFormat("dd-MMM-yyyy HH:mm:ss");
+    final DateFormat utcFormat = new SimpleDateFormat("dd-MMM-yyyy HH:mm:ss.SSSSSS");
     utcFormat.setTimeZone(TimeZone.getTimeZone(Calendar.getInstance().getTimeZone().getID()));
     Date date = null;
     try {
-      date = utcFormat.parse("21-Feb-2009 18:00:00");
+      date = utcFormat.parse("21-Feb-2009 18:00:00.000456");
     } catch (final ParseException e) {
       LOGGER.error("Unparseable date");
-      date = Date.from(Instant.parse("2009-02-21T18:00:00.00Z"));
+      date = Date.from(Instant.parse("2009-02-21T18:00:00.000456Z"));
     }
-    final DateFormat currentTFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+    final DateFormat currentTFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSSSSS");
     currentTFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
     final String utc = currentTFormat.format(date);
     addDataTypeTestData(
         TestDataHolder.builder()
             .sourceType("TIMESTAMP")
             .airbyteType(JsonSchemaPrimitive.STRING)
             .fullSourceDataType("TIMESTAMP WITH LOCAL TIME ZONE")
-            .addInsertValues("to_timestamp_tz('21-FEB-2009 18:00:00', 'DD-MON-YYYY HH24:MI:SS')")
-            .addExpectedValues(utc + ".0 UTC")
+            .addInsertValues("to_timestamp_tz('21-FEB-2009 18:00:00.000456', 'DD-MON-YYYY HH24:MI:SS.FF')")
+            .addExpectedValues(utc + " UTC")
             .build());
 
     addDataTypeTestData(