diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index b4827791603a7..477a3caddbb6c 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -140,7 +140,7 @@ - name: Cockroachdb sourceDefinitionId: 9fa5862c-da7c-11eb-8d19-0242ac130003 dockerRepository: airbyte/source-cockroachdb - dockerImageTag: 0.1.7 + dockerImageTag: 0.1.8 documentationUrl: https://docs.airbyte.io/integrations/sources/cockroachdb icon: cockroachdb.svg sourceType: database @@ -321,7 +321,7 @@ - name: IBM Db2 sourceDefinitionId: 447e0381-3780-4b46-bb62-00a4e3c8b8e2 dockerRepository: airbyte/source-db2 - dockerImageTag: 0.1.7 + dockerImageTag: 0.1.8 documentationUrl: https://docs.airbyte.io/integrations/sources/db2 icon: db2.svg sourceType: database @@ -432,7 +432,7 @@ - name: Microsoft SQL Server (MSSQL) sourceDefinitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 dockerRepository: airbyte/source-mssql - dockerImageTag: 0.3.15 + dockerImageTag: 0.3.16 documentationUrl: https://docs.airbyte.io/integrations/sources/mssql icon: mssql.svg sourceType: database @@ -474,7 +474,7 @@ - name: MySQL sourceDefinitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad dockerRepository: airbyte/source-mysql - dockerImageTag: 0.5.4 + dockerImageTag: 0.5.5 documentationUrl: https://docs.airbyte.io/integrations/sources/mysql icon: mysql.svg sourceType: database @@ -508,7 +508,7 @@ - name: Oracle DB sourceDefinitionId: b39a7370-74c3-45a6-ac3a-380d48520a83 dockerRepository: airbyte/source-oracle - dockerImageTag: 0.3.12 + dockerImageTag: 0.3.13 documentationUrl: https://docs.airbyte.io/integrations/sources/oracle icon: oracle.svg sourceType: database @@ -578,7 +578,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 0.4.6 + dockerImageTag: 0.4.7 documentationUrl: https://docs.airbyte.io/integrations/sources/postgres icon: postgresql.svg sourceType: database @@ -704,7 +704,7 @@ - name: Snowflake sourceDefinitionId: e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2 dockerRepository: airbyte/source-snowflake - dockerImageTag: 0.1.7 + dockerImageTag: 0.1.8 documentationUrl: https://docs.airbyte.io/integrations/sources/snowflake icon: snowflake.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 8ac57614d197f..7ceb863378812 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1205,7 +1205,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-cockroachdb:0.1.7" +- dockerImage: "airbyte/source-cockroachdb:0.1.8" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/cockroachdb" connectionSpecification: @@ -3189,7 +3189,7 @@ - - "client_secret" oauthFlowOutputParameters: - - "refresh_token" -- dockerImage: "airbyte/source-db2:0.1.7" +- dockerImage: "airbyte/source-db2:0.1.8" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/db2" connectionSpecification: @@ -4283,7 +4283,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mssql:0.3.15" +- dockerImage: "airbyte/source-mssql:0.3.16" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/mssql" connectionSpecification: @@ -4982,7 +4982,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mysql:0.5.4" +- dockerImage: "airbyte/source-mysql:0.5.5" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/mysql" connectionSpecification: @@ -5371,7 +5371,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-oracle:0.3.12" +- dockerImage: "airbyte/source-oracle:0.3.13" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/oracle" connectionSpecification: @@ -5961,7 +5961,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:0.4.6" +- dockerImage: "airbyte/source-postgres:0.4.7" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: @@ -7410,7 +7410,7 @@ - - "client_secret" oauthFlowOutputParameters: - - "refresh_token" -- dockerImage: "airbyte/source-snowflake:0.1.7" +- dockerImage: "airbyte/source-snowflake:0.1.8" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/snowflake" connectionSpecification: diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/DataTypeUtils.java b/airbyte-db/lib/src/main/java/io/airbyte/db/DataTypeUtils.java index 873b65f364e0a..53396d0274251 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/DataTypeUtils.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/DataTypeUtils.java @@ -14,9 +14,13 @@ import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.function.Function; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class DataTypeUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(DataTypeUtils.class); + public static final String DATE_FORMAT_PATTERN = "yyyy-MM-dd'T'HH:mm:ss'Z'"; public static final DateFormat DATE_FORMAT = new SimpleDateFormat(DATE_FORMAT_PATTERN); // Quoted "Z" to indicate UTC, no timezone offset @@ -40,6 +44,25 @@ public static T returnNullIfInvalid(final DataTypeSupplier valueProducer, } } + public static String toISO8601StringWithMicroseconds(Instant instant) { + + String dateWithMilliseconds = DATE_FORMAT_WITH_MILLISECONDS.format(Date.from(instant)); + return dateWithMilliseconds.substring(0, 23) + calculateMicrosecondsString(instant.getNano()) + dateWithMilliseconds.substring(23); + } + + private static String calculateMicrosecondsString(int nano) { + var microSeconds = (nano / 1000) % 1000; + String result; + if (microSeconds < 10) { + result = "00" + microSeconds; + } else if (microSeconds < 100) { + result = "0" + microSeconds; + } else { + result = "" + microSeconds; + } + return result; + } + public static String toISO8601StringWithMilliseconds(final long epochMillis) { return DATE_FORMAT_WITH_MILLISECONDS.format(Date.from(Instant.ofEpochMilli(epochMillis))); } diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java b/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java index b56818c55ac69..38e86c2b4702e 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java @@ -19,6 +19,7 @@ import java.sql.SQLException; import java.sql.Timestamp; import java.text.ParseException; +import java.time.Instant; import java.util.Collections; import java.util.List; import java.util.StringJoiner; @@ -120,9 +121,8 @@ protected void putTime(final ObjectNode node, final String columnName, final Res protected void putTimestamp(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { // https://www.cis.upenn.edu/~bcpierce/courses/629/jdkdocs/guide/jdbc/getstart/mapping.doc.html - final Timestamp t = resultSet.getTimestamp(index); - final java.util.Date d = new java.util.Date(t.getTime() + (t.getNanos() / 1000000)); - node.put(columnName, DataTypeUtils.toISO8601String(d)); + final Instant instant = resultSet.getTimestamp(index).toInstant(); + node.put(columnName, DataTypeUtils.toISO8601StringWithMicroseconds(instant)); } protected void putBinary(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { @@ -143,8 +143,14 @@ protected void setTimestamp(final PreparedStatement preparedStatement, final int // Parsing TIME as a TIMESTAMP might potentially break for ClickHouse cause it doesn't expect TIME // value in the following format try { - preparedStatement.setTimestamp(parameterIndex, Timestamp - .from(DataTypeUtils.DATE_FORMAT.parse(value).toInstant())); + var micro = value.substring(value.lastIndexOf('.') + 1, value.length() - 1); + var nanos = micro + "000"; + var valueWithoutMicros = value.replace("." + micro, ""); + + var timestamp = Timestamp + .from(DataTypeUtils.DATE_FORMAT.parse(valueWithoutMicros).toInstant()); + timestamp.setNanos(Integer.parseInt(nanos)); + preparedStatement.setTimestamp(parameterIndex, timestamp); } catch (final ParseException e) { throw new RuntimeException(e); } diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java b/airbyte-db/lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java index d0481b258d516..181b458f6da1d 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java @@ -144,8 +144,8 @@ void testSetStatementField() throws SQLException { sourceOperations.setStatementField(ps, 11, JDBCType.CHAR, "a"); sourceOperations.setStatementField(ps, 12, JDBCType.VARCHAR, "a"); sourceOperations.setStatementField(ps, 13, JDBCType.DATE, "2020-11-01T00:00:00Z"); - sourceOperations.setStatementField(ps, 14, JDBCType.TIME, "1970-01-01T05:00:00Z"); - sourceOperations.setStatementField(ps, 15, JDBCType.TIMESTAMP, "2001-09-29T03:00:00Z"); + sourceOperations.setStatementField(ps, 14, JDBCType.TIME, "1970-01-01T05:00:00.000Z"); + sourceOperations.setStatementField(ps, 15, JDBCType.TIMESTAMP, "2001-09-29T03:00:00.000Z"); sourceOperations.setStatementField(ps, 16, JDBCType.BINARY, "61616161"); ps.execute(); diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtils.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtils.java index 7e978adbc8904..c2e9f00efdf99 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtils.java +++ b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtils.java @@ -39,7 +39,7 @@ public static String convertDate(final Object input) { } else if (input instanceof Duration) { return DataTypeUtils.toISO8601String((Duration) input); } else if (input instanceof Timestamp) { - return DataTypeUtils.toISO8601String(((Timestamp) input).toLocalDateTime()); + return DataTypeUtils.toISO8601StringWithMicroseconds((((Timestamp) input).toInstant())); } else if (input instanceof Number) { return DataTypeUtils.toISO8601String( new Timestamp(((Number) input).longValue()).toLocalDateTime()); diff --git a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile index eeeb39aaf5a73..74549b35b6ee0 100644 --- a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-cockroachdb-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=0.1.5 LABEL io.airbyte.name=airbyte/source-cockroachdb-strict-encrypt diff --git a/airbyte-integrations/connectors/source-cockroachdb/Dockerfile b/airbyte-integrations/connectors/source-cockroachdb/Dockerfile index f0c972ddcfc1b..53a566b148273 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/Dockerfile +++ b/airbyte-integrations/connectors/source-cockroachdb/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-cockroachdb COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.1.8 LABEL io.airbyte.name=airbyte/source-cockroachdb diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java index 52d252cd68f67..57abee37cb843 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java @@ -327,8 +327,8 @@ protected void initTests() { TestDataHolder.builder() .sourceType("timestamp") .airbyteType(JsonSchemaType.STRING) - .addInsertValues("TIMESTAMP '2004-10-19 10:23:54'", "null") - .addExpectedValues("2004-10-19T10:23:54Z", null) + .addInsertValues("TIMESTAMP '2004-10-19 10:23:54'", "TIMESTAMP '2004-10-19 10:23:54.123456'", "null") + .addExpectedValues("2004-10-19T10:23:54.000000Z", "2004-10-19T10:23:54.123456Z", null) .build()); addDataTypeTestData( diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile index ea38c9b2b4e6f..5da79c802501c 100644 --- a/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-db2-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=0.1.5 LABEL io.airbyte.name=airbyte/source-db2-strict-encrypt diff --git a/airbyte-integrations/connectors/source-db2/Dockerfile b/airbyte-integrations/connectors/source-db2/Dockerfile index 7a7dcb5a533f6..7b133fe44e4ac 100644 --- a/airbyte-integrations/connectors/source-db2/Dockerfile +++ b/airbyte-integrations/connectors/source-db2/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-db2 COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.1.8 LABEL io.airbyte.name=airbyte/source-db2 diff --git a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java index e10b7c2cc288f..0efdb43f756a2 100644 --- a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java @@ -312,8 +312,9 @@ protected void initTests() { .createTablePatternSql(CREATE_TABLE_SQL) .sourceType("TIMESTAMP") .airbyteType(JsonSchemaType.STRING) - .addInsertValues("null", "'2018-03-22-12.00.00.123'", "'20180322125959'", "'20180101 12:00:59 PM'") - .addExpectedValues(null, "2018-03-22T12:00:00Z", "2018-03-22T12:59:59Z", "2018-01-01T12:00:59Z") // milliseconds values are erased + .addInsertValues("null", "'2018-03-22-12.00.00.123'", "'2018-03-22-12.00.00.123456'", "'20180322125959'", "'20180101 12:00:59 PM'") + .addExpectedValues(null, "2018-03-22T12:00:00.123000Z", "2018-03-22T12:00:00.123456Z", "2018-03-22T12:59:59.000000Z", + "2018-01-01T12:00:59.000000Z") .build()); } diff --git a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java index f105185c4608a..60d11c7184499 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java +++ b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java @@ -147,7 +147,7 @@ protected List>> discoverInternal(final JdbcData } protected Predicate excludeNotAccessibleTables(final Set internalSchemas, - final Set tablesWithSelectGrantPrivilege) { + final Set tablesWithSelectGrantPrivilege) { return jsonNode -> { if (tablesWithSelectGrantPrivilege.isEmpty()) { return isNotInternalSchema(jsonNode, internalSchemas); @@ -160,7 +160,8 @@ protected Predicate excludeNotAccessibleTables(final Set inter }; } - // needs to override isNotInternalSchema for connectors that override getPrivilegesTableForCurrentUser() + // needs to override isNotInternalSchema for connectors that override + // getPrivilegesTableForCurrentUser() protected boolean isNotInternalSchema(JsonNode jsonNode, Set internalSchemas) { return !internalSchemas.contains(jsonNode.get(INTERNAL_SCHEMA_NAME).asText()); } diff --git a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java index fa380a2948e35..02ab8faef80bc 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java @@ -723,11 +723,11 @@ void testReadMultipleTablesIncrementally() throws Exception { } // when initial and final cursor fields are the same. - private void incrementalCursorCheck( - final String cursorField, - final String initialCursorValue, - final String endCursorValue, - final List expectedRecordMessages) + protected void incrementalCursorCheck( + final String cursorField, + final String initialCursorValue, + final String endCursorValue, + final List expectedRecordMessages) throws Exception { incrementalCursorCheck(cursorField, cursorField, initialCursorValue, endCursorValue, expectedRecordMessages); diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile index fc2d387dd5bc2..f4ed2ac15ac72 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mssql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.1.8 LABEL io.airbyte.name=airbyte/source-mssql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mssql/Dockerfile b/airbyte-integrations/connectors/source-mssql/Dockerfile index 2f6bd76aae4a4..0cfc0f4d8bd45 100644 --- a/airbyte-integrations/connectors/source-mssql/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mssql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.15 +LABEL io.airbyte.version=0.3.16 LABEL io.airbyte.name=airbyte/source-mssql diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java index 8c6030054e12a..5aec0eed0be65 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java @@ -340,7 +340,7 @@ protected void initTests() { .sourceType("smalldatetime") .airbyteType(JsonSchemaType.STRING) .addInsertValues("'1900-01-01'", "'2079-06-06'", "null") - .addExpectedValues("1900-01-01T00:00:00Z", "2079-06-06T00:00:00Z", null) + .addExpectedValues("1900-01-01T00:00:00.000000Z", "2079-06-06T00:00:00.000000Z", null) .createTablePatternSql(CREATE_TABLE_SQL) .build()); @@ -348,8 +348,10 @@ protected void initTests() { TestDataHolder.builder() .sourceType("datetime") .airbyteType(JsonSchemaType.STRING) - .addInsertValues("'1753-01-01'", "'9999-12-31'", "null") - .addExpectedValues("1753-01-01T00:00:00Z", "9999-12-31T00:00:00Z", null) + .addInsertValues("'1753-01-01'", "'9999-12-31'", "'9999-12-31T13:00:04Z'", + "'9999-12-31T13:00:04.123Z'", "null") + .addExpectedValues("1753-01-01T00:00:00.000000Z", "9999-12-31T00:00:00.000000Z", "9999-12-31T13:00:04.000000Z", + "9999-12-31T13:00:04.123000Z", null) .createTablePatternSql(CREATE_TABLE_SQL) .build()); @@ -357,8 +359,8 @@ protected void initTests() { TestDataHolder.builder() .sourceType("datetime2") .airbyteType(JsonSchemaType.STRING) - .addInsertValues("'0001-01-01'", "'9999-12-31'", "null") - .addExpectedValues("0001-01-01T00:00:00Z", "9999-12-31T00:00:00Z", null) + .addInsertValues("'0001-01-01'", "'9999-12-31'", "'9999-12-31T13:00:04.123456Z'", "null") + .addExpectedValues("0001-01-01T00:00:00.000000Z", "9999-12-31T00:00:00.000000Z", "9999-12-31T13:00:04.123456Z", null) .createTablePatternSql(CREATE_TABLE_SQL) .build()); @@ -366,8 +368,7 @@ protected void initTests() { TestDataHolder.builder() .sourceType("time") .airbyteType(JsonSchemaType.STRING) - .addInsertValues("'00:00:00.0000000'", "'23:59:59.9999999'", "'00:00:00'", "'23:58'", - "null") + .addInsertValues("'00:00:00.0000000'", "'23:59:59.9999999'", "'00:00:00'", "'23:58'", "null") .addExpectedValues("00:00:00", "23:59:59.9999999", "00:00:00", "23:58:00", null) .createTablePatternSql(CREATE_TABLE_SQL) .build()); diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceDatatypeTest.java index 89fa382bc62c9..a55bba73c8bbb 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceDatatypeTest.java @@ -181,10 +181,8 @@ protected void initTests() { TestDataHolder.builder() .sourceType("date") .airbyteType(JsonSchemaType.STRING) - .addInsertValues("'0001-01-01'", "'9999-12-31'", "'1999-01-08'", - "null") - .addExpectedValues("0001-01-01T00:00:00Z", "9999-12-31T00:00:00Z", - "1999-01-08T00:00:00Z", null) + .addInsertValues("'0001-01-01'", "'9999-12-31'", "'1999-01-08'", "null") + .addExpectedValues("0001-01-01T00:00:00Z", "9999-12-31T00:00:00Z", "1999-01-08T00:00:00Z", null) .build()); addDataTypeTestData( @@ -192,31 +190,33 @@ protected void initTests() { .sourceType("smalldatetime") .airbyteType(JsonSchemaType.STRING) .addInsertValues("'1900-01-01'", "'2079-06-06'", "null") - .addExpectedValues("1900-01-01T00:00:00Z", "2079-06-06T00:00:00Z", null) + .addExpectedValues("1900-01-01T00:00:00.000000Z", "2079-06-06T00:00:00.000000Z", null) .build()); addDataTypeTestData( TestDataHolder.builder() .sourceType("datetime") .airbyteType(JsonSchemaType.STRING) - .addInsertValues("'1753-01-01'", "'9999-12-31'", "null") - .addExpectedValues("1753-01-01T00:00:00Z", "9999-12-31T00:00:00Z", null) + .addInsertValues("'1753-01-01'", "'9999-12-31'", "'9999-12-31T13:00:04Z'", + "'9999-12-31T13:00:04.123Z'", "null") + .addExpectedValues("1753-01-01T00:00:00.000000Z", "9999-12-31T00:00:00.000000Z", "9999-12-31T13:00:04.000000Z", + "9999-12-31T13:00:04.123000Z", null) .build()); addDataTypeTestData( TestDataHolder.builder() .sourceType("datetime2") .airbyteType(JsonSchemaType.STRING) - .addInsertValues("'0001-01-01'", "'9999-12-31'", "null") - .addExpectedValues("0001-01-01T00:00:00Z", "9999-12-31T00:00:00Z", null) + .addInsertValues("'0001-01-01'", "'9999-12-31'", "'9999-12-31T13:00:04.123456Z'", "null") + .addExpectedValues("0001-01-01T00:00:00.000000Z", "9999-12-31T00:00:00.000000Z", "9999-12-31T13:00:04.123456Z", null) .build()); addDataTypeTestData( TestDataHolder.builder() .sourceType("time") .airbyteType(JsonSchemaType.STRING) - .addInsertValues("null", "'13:00:01'", "'13:00:04Z'") - .addExpectedValues(null, "13:00:01.0000000", "13:00:04.0000000") + .addInsertValues("null", "'13:00:01'", "'13:00:04Z'", "'13:00:04.123456Z'") + .addExpectedValues(null, "13:00:01.0000000", "13:00:04.0000000", "13:00:04.1234560") .build()); addDataTypeTestData( diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile index 8e18b0806e2be..d3a12f1c79d6c 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mysql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.10 +LABEL io.airbyte.version=0.1.11 LABEL io.airbyte.name=airbyte/source-mysql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mysql/Dockerfile b/airbyte-integrations/connectors/source-mysql/Dockerfile index 1b1f8950fd929..3d6087e2982c4 100644 --- a/airbyte-integrations/connectors/source-mysql/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mysql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.5.4 +LABEL io.airbyte.version=0.5.5 LABEL io.airbyte.name=airbyte/source-mysql diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceDatatypeTest.java index 231e6a03b8dc4..40453e2280106 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceDatatypeTest.java @@ -243,19 +243,19 @@ protected void initTests() { // Check Zero-date value for mandatory field addDataTypeTestData( TestDataHolder.builder() - .sourceType("date") - .fullSourceDataType("date not null") - .airbyteType(JsonSchemaType.STRING) - .addInsertValues("'0000-00-00'") - .addExpectedValues("1970-01-01T00:00:00Z") - .build()); + .sourceType("date") + .fullSourceDataType("date not null") + .airbyteType(JsonSchemaType.STRING) + .addInsertValues("'0000-00-00'") + .addExpectedValues("1970-01-01T00:00:00Z") + .build()); addDataTypeTestData( TestDataHolder.builder() .sourceType("datetime") .airbyteType(JsonSchemaType.STRING) .addInsertValues("null", "'2005-10-10 23:22:21'") - .addExpectedValues(null, "2005-10-10T23:22:21Z") + .addExpectedValues(null, "2005-10-10T23:22:21.000000Z") .build()); // Check Zero-date value for mandatory field @@ -278,7 +278,7 @@ protected void initTests() { // Check Zero-date value for mandatory field addDataTypeTestData( - TestDataHolder.builder() + TestDataHolder.builder() .sourceType("timestamp") .fullSourceDataType("timestamp not null") .airbyteType(JsonSchemaType.STRING) @@ -292,7 +292,7 @@ protected void initTests() { .airbyteType(JsonSchemaType.STRING) // JDBC driver can process only "clock"(00:00:00-23:59:59) values. // https://debezium.io/documentation/reference/connectors/mysql.html#mysql-temporal-types - .addInsertValues("null", "'-23:59:59.123456'", "'00:00:00'") + .addInsertValues("null", "'-23:59:59'", "'00:00:00'") .addExpectedValues(null, "1970-01-01T23:59:59Z", "1970-01-01T00:00:00Z") .build()); diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceDatatypeTest.java index 883ef59c15b1c..7333e689f64de 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceDatatypeTest.java @@ -290,7 +290,7 @@ protected void initTests() { .sourceType("datetime") .airbyteType(JsonSchemaType.STRING) .addInsertValues("null", "'2005-10-10 23:22:21'") - .addExpectedValues(null, "2005-10-10T23:22:21Z") + .addExpectedValues(null, "2005-10-10T23:22:21.000000Z") .build()); addDataTypeTestData( @@ -306,7 +306,7 @@ protected void initTests() { .sourceType("time") .airbyteType(JsonSchemaType.STRING) // JDBC driver can process only "clock"(00:00:00-23:59:59) values. - .addInsertValues("null", "'-23:59:59.123456'", "'00:00:00'") + .addInsertValues("null", "'-23:59:59'", "'00:00:00'") .addExpectedValues(null, "1970-01-01T23:59:59Z", "1970-01-01T00:00:00Z") .build()); diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile index 4bab366f00061..eb7a130d450a5 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile @@ -17,5 +17,5 @@ ENV TZ UTC COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=0.1.5 LABEL io.airbyte.name=airbyte/source-oracle-strict-encrypt diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptJdbcSourceAcceptanceTest.java index a00872c120695..437869f2de463 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptJdbcSourceAcceptanceTest.java @@ -5,18 +5,30 @@ package io.airbyte.integrations.source.oracle_strict_encrypt; import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; +import io.airbyte.commons.util.MoreIterators; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.integrations.source.oracle.OracleSource; +import io.airbyte.integrations.source.relationaldb.models.DbState; +import io.airbyte.integrations.source.relationaldb.models.DbStreamState; +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import io.airbyte.protocol.models.AirbyteStateMessage; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; +import io.airbyte.protocol.models.DestinationSyncMode; +import io.airbyte.protocol.models.SyncMode; import java.math.BigDecimal; import java.sql.Connection; import java.sql.DriverManager; @@ -24,9 +36,12 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.ArrayList; import java.util.List; +import java.util.Optional; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -225,4 +240,105 @@ void testSpec() throws Exception { assertEquals(expected, actual); } + @Override + protected List getTestMessages() { + return Lists.newArrayList( + new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName) + .withNamespace(getDefaultNamespace()) + .withData(Jsons.jsonNode(ImmutableMap + .of(COL_ID, ID_VALUE_1, + COL_NAME, "picard", + COL_UPDATED_AT, "2004-10-19T00:00:00.000000Z")))), + new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName) + .withNamespace(getDefaultNamespace()) + .withData(Jsons.jsonNode(ImmutableMap + .of(COL_ID, ID_VALUE_2, + COL_NAME, "crusher", + COL_UPDATED_AT, + "2005-10-19T00:00:00.000000Z")))), + new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName) + .withNamespace(getDefaultNamespace()) + .withData(Jsons.jsonNode(ImmutableMap + .of(COL_ID, ID_VALUE_3, + COL_NAME, "vash", + COL_UPDATED_AT, "2006-10-19T00:00:00.000000Z"))))); + } + + @Test + void testIncrementalTimestampCheckCursor() throws Exception { + incrementalCursorCheck( + COL_UPDATED_AT, + "2005-10-18T00:00:00.000000Z", + "2006-10-19T00:00:00.000000Z", + Lists.newArrayList(getTestMessages().get(1), getTestMessages().get(2))); + } + + @Test + void testReadOneTableIncrementallyTwice() throws Exception { + final String namespace = getDefaultNamespace(); + final ConfiguredAirbyteCatalog configuredCatalog = getConfiguredCatalogWithOneStream(namespace); + configuredCatalog.getStreams().forEach(airbyteStream -> { + airbyteStream.setSyncMode(SyncMode.INCREMENTAL); + airbyteStream.setCursorField(Lists.newArrayList(COL_ID)); + airbyteStream.setDestinationSyncMode(DestinationSyncMode.APPEND); + }); + + final DbState state = new DbState() + .withStreams(Lists.newArrayList(new DbStreamState().withStreamName(streamName).withStreamNamespace(namespace))); + final List actualMessagesFirstSync = MoreIterators + .toList(source.read(config, configuredCatalog, Jsons.jsonNode(state))); + + final Optional stateAfterFirstSyncOptional = actualMessagesFirstSync.stream() + .filter(r -> r.getType() == AirbyteMessage.Type.STATE).findFirst(); + assertTrue(stateAfterFirstSyncOptional.isPresent()); + + database.execute(connection -> { + connection.createStatement().execute( + String.format("INSERT INTO %s(id, name, updated_at) VALUES (4,'riker', '2006-10-19')", + getFullyQualifiedTableName(TABLE_NAME))); + connection.createStatement().execute( + String.format("INSERT INTO %s(id, name, updated_at) VALUES (5, 'data', '2006-10-19')", + getFullyQualifiedTableName(TABLE_NAME))); + }); + + final List actualMessagesSecondSync = MoreIterators + .toList(source.read(config, configuredCatalog, + stateAfterFirstSyncOptional.get().getState().getData())); + + Assertions.assertEquals(2, + (int) actualMessagesSecondSync.stream().filter(r -> r.getType() == AirbyteMessage.Type.RECORD).count()); + final List expectedMessages = new ArrayList<>(); + expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + .withData(Jsons.jsonNode(ImmutableMap + .of(COL_ID, ID_VALUE_4, + COL_NAME, "riker", + COL_UPDATED_AT, "2006-10-19T00:00:00.000000Z"))))); + expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + .withData(Jsons.jsonNode(ImmutableMap + .of(COL_ID, ID_VALUE_5, + COL_NAME, "data", + COL_UPDATED_AT, "2006-10-19T00:00:00.000000Z"))))); + expectedMessages.add(new AirbyteMessage() + .withType(AirbyteMessage.Type.STATE) + .withState(new AirbyteStateMessage() + .withData(Jsons.jsonNode(new DbState() + .withCdc(false) + .withStreams(Lists.newArrayList(new DbStreamState() + .withStreamName(streamName) + .withStreamNamespace(namespace) + .withCursorField(ImmutableList.of(COL_ID)) + .withCursor("5"))))))); + + setEmittedAtToNull(actualMessagesSecondSync); + + assertTrue(expectedMessages.size() == actualMessagesSecondSync.size()); + assertTrue(expectedMessages.containsAll(actualMessagesSecondSync)); + assertTrue(actualMessagesSecondSync.containsAll(expectedMessages)); + } + } diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test/resources/expected_spec.json index 0b4b703cb2910..0a7cdf7cdfdcb 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test/resources/expected_spec.json @@ -4,7 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Oracle Source Spec", "type": "object", - "required": ["host", "port", "sid", "username"], + "required": ["host", "port", "sid", "username", "encryption"], "additionalProperties": false, "properties": { "host": { diff --git a/airbyte-integrations/connectors/source-oracle/Dockerfile b/airbyte-integrations/connectors/source-oracle/Dockerfile index 985955003231b..a0e02bc7dea85 100644 --- a/airbyte-integrations/connectors/source-oracle/Dockerfile +++ b/airbyte-integrations/connectors/source-oracle/Dockerfile @@ -8,5 +8,5 @@ ENV TZ UTC COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.3.12 +LABEL io.airbyte.version=0.3.13 LABEL io.airbyte.name=airbyte/source-oracle diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java index a09f52f318334..b655321dc7378 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java @@ -182,8 +182,9 @@ protected void initTests() { TestDataHolder.builder() .sourceType("DATE") .airbyteType(JsonSchemaType.STRING) - .addInsertValues("to_date('-4700/01/01','syyyy/mm/dd')", "to_date('9999/12/31 23:59:59','yyyy/mm/dd hh24:mi:ss')", "null") - .addExpectedValues("4700-01-01T00:00:00Z", "9999-12-31T23:59:59Z", null) + .addInsertValues("to_date('-4700/01/01','syyyy/mm/dd')", + "to_date('9999/12/31 23:59:59','yyyy/mm/dd hh24:mi:ss')", "null") + .addExpectedValues("4700-01-01T00:00:00.000000Z", "9999-12-31T23:59:59.000000Z", null) // @TODO stream fails when gets Zero date value // .addInsertValues("'2021/01/00'", "'2021/00/00'", "'0000/00/00'") .build()); @@ -192,8 +193,9 @@ protected void initTests() { TestDataHolder.builder() .sourceType("TIMESTAMP") .airbyteType(JsonSchemaType.STRING) - .addInsertValues("to_timestamp('2020-06-10 06:14:00.742000000', 'YYYY-MM-DD HH24:MI:SS.FF')") - .addExpectedValues("2020-06-10T06:14:01Z") + .addInsertValues("to_timestamp('2020-06-10 06:14:00.742', 'YYYY-MM-DD HH24:MI:SS.FF')", + "to_timestamp('2020-06-10 06:14:00.742123', 'YYYY-MM-DD HH24:MI:SS.FF')") + .addExpectedValues("2020-06-10T06:14:00.742000Z", "2020-06-10T06:14:00.742123Z") .build()); addDataTypeTestData( @@ -202,20 +204,23 @@ protected void initTests() { .airbyteType(JsonSchemaType.STRING) .fullSourceDataType("TIMESTAMP WITH TIME ZONE") .addInsertValues("to_timestamp_tz('21-FEB-2009 18:00:00 EST', 'DD-MON-YYYY HH24:MI:SS TZR')", - "to_timestamp_tz('21-FEB-2009 18:00:00 -5:00', 'DD-MON-YYYY HH24:MI:SS TZH:TZM')") - .addExpectedValues("2009-02-21 18:00:00.0 EST", "2009-02-21 18:00:00.0 -5:00") + "to_timestamp_tz('21-FEB-2009 18:00:00.123456 EST', 'DD-MON-YYYY HH24:MI:SS.FF TZR')", + "to_timestamp_tz('21-FEB-2009 18:00:00 -5:00', 'DD-MON-YYYY HH24:MI:SS TZH:TZM')", + "to_timestamp_tz('21-FEB-2009 18:00:00.123456 -5:00', 'DD-MON-YYYY HH24:MI:SS.FF TZH:TZM')") + .addExpectedValues("2009-02-21 18:00:00.0 EST", "2009-02-21 18:00:00.123456 EST", + "2009-02-21 18:00:00.0 -5:00", "2009-02-21 18:00:00.123456 -5:00") .build()); - final DateFormat utcFormat = new SimpleDateFormat("dd-MMM-yyyy HH:mm:ss"); + final DateFormat utcFormat = new SimpleDateFormat("dd-MMM-yyyy HH:mm:ss.SSSSSS"); utcFormat.setTimeZone(TimeZone.getTimeZone(Calendar.getInstance().getTimeZone().getID())); Date date = null; try { - date = utcFormat.parse("21-Feb-2009 18:00:00"); + date = utcFormat.parse("21-Feb-2009 18:00:00.000456"); } catch (final ParseException e) { LOGGER.error("Unparseable date"); - date = Date.from(Instant.parse("2009-02-21T18:00:00.00Z")); + date = Date.from(Instant.parse("2009-02-21T18:00:00.000456Z")); } - final DateFormat currentTFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + final DateFormat currentTFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSSSSS"); currentTFormat.setTimeZone(TimeZone.getTimeZone("UTC")); final String utc = currentTFormat.format(date); addDataTypeTestData( @@ -223,8 +228,8 @@ protected void initTests() { .sourceType("TIMESTAMP") .airbyteType(JsonSchemaType.STRING) .fullSourceDataType("TIMESTAMP WITH LOCAL TIME ZONE") - .addInsertValues("to_timestamp_tz('21-FEB-2009 18:00:00', 'DD-MON-YYYY HH24:MI:SS')") - .addExpectedValues(utc + ".0 UTC") + .addInsertValues("to_timestamp_tz('21-FEB-2009 18:00:00.000456', 'DD-MON-YYYY HH24:MI:SS.FF')") + .addExpectedValues(utc + " UTC") .build()); addDataTypeTestData( diff --git a/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java index b6efc8a5148e7..0d5c7a604403f 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java @@ -5,15 +5,28 @@ package io.airbyte.integrations.source.oracle; import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; +import io.airbyte.commons.util.MoreIterators; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; +import io.airbyte.integrations.source.relationaldb.models.DbState; +import io.airbyte.integrations.source.relationaldb.models.DbStreamState; +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.AirbyteMessage.Type; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import io.airbyte.protocol.models.AirbyteStateMessage; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; +import io.airbyte.protocol.models.DestinationSyncMode; +import io.airbyte.protocol.models.SyncMode; import java.math.BigDecimal; import java.sql.Connection; import java.sql.DriverManager; @@ -21,9 +34,12 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.ArrayList; import java.util.List; +import java.util.Optional; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -115,6 +131,107 @@ void cleanUpTables() throws SQLException { conn.close(); } + @Override + protected List getTestMessages() { + return Lists.newArrayList( + new AirbyteMessage().withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName) + .withNamespace(getDefaultNamespace()) + .withData(Jsons.jsonNode(ImmutableMap + .of(COL_ID, ID_VALUE_1, + COL_NAME, "picard", + COL_UPDATED_AT, "2004-10-19T00:00:00.000000Z")))), + new AirbyteMessage().withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName) + .withNamespace(getDefaultNamespace()) + .withData(Jsons.jsonNode(ImmutableMap + .of(COL_ID, ID_VALUE_2, + COL_NAME, "crusher", + COL_UPDATED_AT, + "2005-10-19T00:00:00.000000Z")))), + new AirbyteMessage().withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName) + .withNamespace(getDefaultNamespace()) + .withData(Jsons.jsonNode(ImmutableMap + .of(COL_ID, ID_VALUE_3, + COL_NAME, "vash", + COL_UPDATED_AT, "2006-10-19T00:00:00.000000Z"))))); + } + + @Test + void testIncrementalTimestampCheckCursor() throws Exception { + incrementalCursorCheck( + COL_UPDATED_AT, + "2005-10-18T00:00:00.000000Z", + "2006-10-19T00:00:00.000000Z", + Lists.newArrayList(getTestMessages().get(1), getTestMessages().get(2))); + } + + @Test + void testReadOneTableIncrementallyTwice() throws Exception { + final String namespace = getDefaultNamespace(); + final ConfiguredAirbyteCatalog configuredCatalog = getConfiguredCatalogWithOneStream(namespace); + configuredCatalog.getStreams().forEach(airbyteStream -> { + airbyteStream.setSyncMode(SyncMode.INCREMENTAL); + airbyteStream.setCursorField(Lists.newArrayList(COL_ID)); + airbyteStream.setDestinationSyncMode(DestinationSyncMode.APPEND); + }); + + final DbState state = new DbState() + .withStreams(Lists.newArrayList(new DbStreamState().withStreamName(streamName).withStreamNamespace(namespace))); + final List actualMessagesFirstSync = MoreIterators + .toList(source.read(config, configuredCatalog, Jsons.jsonNode(state))); + + final Optional stateAfterFirstSyncOptional = actualMessagesFirstSync.stream() + .filter(r -> r.getType() == Type.STATE).findFirst(); + assertTrue(stateAfterFirstSyncOptional.isPresent()); + + database.execute(connection -> { + connection.createStatement().execute( + String.format("INSERT INTO %s(id, name, updated_at) VALUES (4,'riker', '2006-10-19')", + getFullyQualifiedTableName(TABLE_NAME))); + connection.createStatement().execute( + String.format("INSERT INTO %s(id, name, updated_at) VALUES (5, 'data', '2006-10-19')", + getFullyQualifiedTableName(TABLE_NAME))); + }); + + final List actualMessagesSecondSync = MoreIterators + .toList(source.read(config, configuredCatalog, + stateAfterFirstSyncOptional.get().getState().getData())); + + Assertions.assertEquals(2, + (int) actualMessagesSecondSync.stream().filter(r -> r.getType() == Type.RECORD).count()); + final List expectedMessages = new ArrayList<>(); + expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + .withData(Jsons.jsonNode(ImmutableMap + .of(COL_ID, ID_VALUE_4, + COL_NAME, "riker", + COL_UPDATED_AT, "2006-10-19T00:00:00.000000Z"))))); + expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + .withData(Jsons.jsonNode(ImmutableMap + .of(COL_ID, ID_VALUE_5, + COL_NAME, "data", + COL_UPDATED_AT, "2006-10-19T00:00:00.000000Z"))))); + expectedMessages.add(new AirbyteMessage() + .withType(Type.STATE) + .withState(new AirbyteStateMessage() + .withData(Jsons.jsonNode(new DbState() + .withCdc(false) + .withStreams(Lists.newArrayList(new DbStreamState() + .withStreamName(streamName) + .withStreamNamespace(namespace) + .withCursorField(ImmutableList.of(COL_ID)) + .withCursor("5"))))))); + + setEmittedAtToNull(actualMessagesSecondSync); + + assertTrue(expectedMessages.size() == actualMessagesSecondSync.size()); + assertTrue(expectedMessages.containsAll(actualMessagesSecondSync)); + assertTrue(actualMessagesSecondSync.containsAll(expectedMessages)); + } + @Override public boolean supportsSchemas() { // See https://www.oratable.com/oracle-user-schema-difference/ diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile index 5514d400576a6..b4e8db25575fc 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.8 +LABEL io.airbyte.version=0.1.9 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test/resources/expected_spec.json index c786b05f15be5..b12bbd51eed38 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test/resources/expected_spec.json @@ -1,5 +1,5 @@ { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/postgres", + "documentationUrl": "https://docs.airbyte.com/integrations/sources/postgres", "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Postgres Source Spec", @@ -78,7 +78,7 @@ { "title": "Logical Replication (CDC)", "additionalProperties": false, - "description": "Logical replication uses the Postgres write-ahead log (WAL) to detect inserts, updates, and deletes. This needs to be configured on the source database itself. Only available on Postgres 10 and above. Read the Postgres Source docs for more information.", + "description": "Logical replication uses the Postgres write-ahead log (WAL) to detect inserts, updates, and deletes. This needs to be configured on the source database itself. Only available on Postgres 10 and above. Read the Postgres Source docs for more information.", "required": ["method", "replication_slot", "publication"], "properties": { "method": { @@ -90,18 +90,21 @@ }, "plugin": { "type": "string", - "description": "A logical decoding plug-in installed on the PostgreSQL server. `pgoutput` plug-in is used by default.\nIf replication table contains a lot of big jsonb values it is recommended to use `wal2json` plug-in. For more information about `wal2json` plug-in read Postgres Source docs.", + "title": "Plugin", + "description": "A logical decoding plug-in installed on the PostgreSQL server. `pgoutput` plug-in is used by default.\nIf replication table contains a lot of big jsonb values it is recommended to use `wal2json` plug-in. For more information about `wal2json` plug-in read Postgres Source docs.", "enum": ["pgoutput", "wal2json"], "default": "pgoutput", "order": 1 }, "replication_slot": { "type": "string", + "title":"Replication Slot", "description": "A plug-in logical replication slot.", "order": 2 }, "publication": { "type": "string", + "title":"Publication", "description": "A Postgres publication used for consuming changes.", "order": 3 } diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index f0aab1646f2d8..8044530f9176f 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.6 +LABEL io.airbyte.version=0.4.7 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceDatatypeTest.java index e559a64dcdab6..83e68ce956a03 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceDatatypeTest.java @@ -383,16 +383,16 @@ protected void initTests() { TestDataHolder.builder() .sourceType("timestamp") .airbyteType(JsonSchemaType.STRING) - .addInsertValues("TIMESTAMP '2004-10-19 10:23:54'", "null") - .addExpectedValues("2004-10-19T10:23:54Z", null) + .addInsertValues("TIMESTAMP '2004-10-19 10:23:54'", "TIMESTAMP '2004-10-19 10:23:54.123456'", "null") + .addExpectedValues("2004-10-19T10:23:54.000000Z", "2004-10-19T10:23:54.123456Z", null) .build()); addDataTypeTestData( TestDataHolder.builder() .sourceType("timestamptz") .airbyteType(JsonSchemaType.STRING) - .addInsertValues("TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54+03'", "null") - .addExpectedValues("2004-10-19T07:23:54Z", null) + .addInsertValues("TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54+03'", "TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456+03'", "null") + .addExpectedValues("2004-10-19T07:23:54Z", "2004-10-19T07:23:54.123456Z", null) .build()); addDataTypeTestData( diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java index 2390270a4ecac..8d956897b6eb7 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java @@ -471,8 +471,8 @@ protected void initTests() { .sourceType("timestamp") .fullSourceDataType(fullSourceType) .airbyteType(JsonSchemaType.STRING) - .addInsertValues("TIMESTAMP '2004-10-19 10:23:54'", "null") - .addExpectedValues("2004-10-19T10:23:54Z", null) + .addInsertValues("TIMESTAMP '2004-10-19 10:23:54'", "TIMESTAMP '2004-10-19 10:23:54.123456'", "null") + .addExpectedValues("2004-10-19T10:23:54.000000Z", "2004-10-19T10:23:54.123456Z", null) .build()); } @@ -483,9 +483,9 @@ protected void initTests() { .sourceType("timestamptz") .fullSourceDataType(fullSourceType) .airbyteType(JsonSchemaType.STRING) - .addInsertValues("TIMESTAMP '2004-10-19 10:23:54-08'", "null") + .addInsertValues("TIMESTAMP '2004-10-19 10:23:54-08'", "TIMESTAMP '2004-10-19 10:23:54.123456-08'", "null") // 2004-10-19T10:23:54Z-8 = 2004-10-19T17:23:54Z - .addExpectedValues("2004-10-19T17:23:54Z", null) + .addExpectedValues("2004-10-19T17:23:54.000000Z", "2004-10-19T17:23:54.123456Z", null) .build()); } diff --git a/airbyte-integrations/connectors/source-snowflake/Dockerfile b/airbyte-integrations/connectors/source-snowflake/Dockerfile index 258381da62273..b593cfd5db01a 100644 --- a/airbyte-integrations/connectors/source-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/source-snowflake/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-snowflake COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.1.8 LABEL io.airbyte.name=airbyte/source-snowflake diff --git a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java index a1d95855e4aa1..39454d4b0b69c 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java @@ -247,8 +247,8 @@ protected void initTests() { TestDataHolder.builder() .sourceType("DATETIME") .airbyteType(JsonSchemaType.STRING) - .addInsertValues("null", "'0001-01-01 00:00:00'", "'9999-12-31 23:59:59'") - .addExpectedValues(null, "0001-01-01T00:00:00Z", "9999-12-31T23:59:59Z") + .addInsertValues("null", "'0001-01-01 00:00:00'", "'9999-12-31 23:59:59'", "'9999-12-31 23:59:59.123456'") + .addExpectedValues(null, "0001-01-01T00:00:00.000000Z", "9999-12-31T23:59:59.000000Z", "9999-12-31T23:59:59.123456Z") .build()); addDataTypeTestData( TestDataHolder.builder() @@ -262,29 +262,29 @@ protected void initTests() { TestDataHolder.builder() .sourceType("TIMESTAMP") .airbyteType(JsonSchemaType.STRING) - .addInsertValues("null", "'2018-03-22 12:00:00.123'") - .addExpectedValues(null, "2018-03-22T12:00:00Z") + .addInsertValues("null", "'2018-03-22 12:00:00.123'", "'2018-03-22 12:00:00.123456'") + .addExpectedValues(null, "2018-03-22T12:00:00.123000Z", "2018-03-22T12:00:00.123456Z") .build()); addDataTypeTestData( TestDataHolder.builder() .sourceType("TIMESTAMP_LTZ") .airbyteType(JsonSchemaType.STRING) - .addInsertValues("null", "'2018-03-22 12:00:00.123 +05:00'") - .addExpectedValues(null, "2018-03-22T07:00:00Z") + .addInsertValues("null", "'2018-03-22 12:00:00.123 +05:00'", "'2018-03-22 12:00:00.123456 +05:00'") + .addExpectedValues(null, "2018-03-22T07:00:00.123000Z", "2018-03-22T07:00:00.123456Z") .build()); addDataTypeTestData( TestDataHolder.builder() .sourceType("TIMESTAMP_NTZ") .airbyteType(JsonSchemaType.STRING) - .addInsertValues("null", "'2018-03-22 12:00:00.123 +05:00'") - .addExpectedValues(null, "2018-03-22T12:00:00Z") + .addInsertValues("null", "'2018-03-22 12:00:00.123 +05:00'", "'2018-03-22 12:00:00.123456 +05:00'") + .addExpectedValues(null, "2018-03-22T12:00:00.123000Z", "2018-03-22T12:00:00.123456Z") .build()); addDataTypeTestData( TestDataHolder.builder() .sourceType("TIMESTAMP_TZ") .airbyteType(JsonSchemaType.STRING) - .addInsertValues("null", "'2018-03-22 12:00:00.123 +05:00'") - .addExpectedValues(null, "2018-03-22T07:00:00Z") + .addInsertValues("null", "'2018-03-22 12:00:00.123 +05:00'", "'2018-03-22 12:00:00.123456 +05:00'") + .addExpectedValues(null, "2018-03-22T07:00:00.123000Z", "2018-03-22T07:00:00.123456Z") .build()); // Semi-structured Data Types diff --git a/docs/integrations/sources/cockroachdb.md b/docs/integrations/sources/cockroachdb.md index 0b2e2813e003f..ecec13871e0d4 100644 --- a/docs/integrations/sources/cockroachdb.md +++ b/docs/integrations/sources/cockroachdb.md @@ -95,6 +95,7 @@ Your database user should now be ready for use with Airbyte. | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.8 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | | 0.1.7 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.1.6 | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved discovering tables in case if user does not have permissions to any table | | 0.1.5 | 2021-12-24 | [9004](https://github.com/airbytehq/airbyte/pull/9004) | User can see only permmited tables during discovery | @@ -106,7 +107,8 @@ Your database user should now be ready for use with Airbyte. | Version | Date | Pull Request | Subject | |:--------| :--- | :--- | :--- | -| 0.1.4 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.1.5 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | +| 0.1.4 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.1.3 | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved discovering tables in case if user does not have permissions to any table | | 0.1.2 | 2021-12-24 | [9004](https://github.com/airbytehq/airbyte/pull/9004) | User can see only permmited tables during discovery | | 0.1.1 | 2021-12-24 | [8958](https://github.com/airbytehq/airbyte/pull/8958) | Add support for JdbcType.ARRAY | diff --git a/docs/integrations/sources/db2.md b/docs/integrations/sources/db2.md index bb8429ea3f04b..2f9f6d0a8034c 100644 --- a/docs/integrations/sources/db2.md +++ b/docs/integrations/sources/db2.md @@ -62,6 +62,7 @@ You can also enter your own password for the keystore, but if you don't, the pas | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.8 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | | 0.1.7 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option |**** | 0.1.6 | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved discovering tables in case if user does not have permissions to any table | | 0.1.5 | 2022-02-01 | [9875](https://github.com/airbytehq/airbyte/pull/9875) | Discover only permitted for user tables | diff --git a/docs/integrations/sources/mssql.md b/docs/integrations/sources/mssql.md index 3f8d444dde718..dc88e6bc886aa 100644 --- a/docs/integrations/sources/mssql.md +++ b/docs/integrations/sources/mssql.md @@ -294,6 +294,7 @@ If you do not see a type in this list, assume that it is coerced into a string. | Version | Date | Pull Request | Subject | |:------- | :--------- | :----------------------------------------------------- | :------------------------------------- | +| 0.3.16 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | | 0.3.15 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.3.14 | 2022-01-24 | [9554](https://github.com/airbytehq/airbyte/pull/9554) | Allow handling of java sql date in CDC | | 0.3.13 | 2022-01-07 | [9094](https://github.com/airbytehq/airbyte/pull/9094) | Added support for missed data types | diff --git a/docs/integrations/sources/mysql.md b/docs/integrations/sources/mysql.md index 8c3bea98b77f6..b037d4cb8ae63 100644 --- a/docs/integrations/sources/mysql.md +++ b/docs/integrations/sources/mysql.md @@ -182,6 +182,7 @@ If you do not see a type in this list, assume that it is coerced into a string. | Version | Date | Pull Request | Subject | |:--------|:-------------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------| +| 0.5.5 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | | 0.5.4 | 2022-02-11 | [10251](https://github.com/airbytehq/airbyte/issues/10251) | bug Source MySQL CDC: sync failed when has Zero-date value in mandatory column | | 0.5.2 | 2021-12-14 | [6425](https://github.com/airbytehq/airbyte/issues/6425) | MySQL CDC sync fails because starting binlog position not found in DB | | 0.5.1 | 2021-12-13 | [8582](https://github.com/airbytehq/airbyte/pull/8582) | Update connector fields title/description | diff --git a/docs/integrations/sources/oracle.md b/docs/integrations/sources/oracle.md index 7d575199d9557..d1c19b52edc7c 100644 --- a/docs/integrations/sources/oracle.md +++ b/docs/integrations/sources/oracle.md @@ -132,6 +132,7 @@ Airbite has the ability to connect to the Oracle source with 3 network connectiv | Version | Date | Pull Request | Subject | |:--------| :--- | :--- |:------------------------------------------------| +| 0.3.13 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | | 0.3.12 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.3.11 | 2021-12-24 | [8958](https://github.com/airbytehq/airbyte/pull/8958) | Add support for JdbcType.ARRAY | | 0.3.10 | 2021-12-07 | [8582](https://github.com/airbytehq/airbyte/pull/8582) | Update connector fields title/description | diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index 63a735a4a0ea5..5ce70652610de 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -270,6 +270,7 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp | Version | Date | Pull Request | Subject | |:--------|:-----------|:-------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| 0.4.7 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | | 0.4.6 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.4.5 | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved discovering tables in case if user does not have permissions to any table | | 0.4.4 | 2022-01-26 | [9807](https://github.com/airbytehq/airbyte/pull/9807) | Update connector fields title/description | diff --git a/docs/integrations/sources/snowflake.md b/docs/integrations/sources/snowflake.md index 2e6f9dc57525b..c0ff378eb87c4 100644 --- a/docs/integrations/sources/snowflake.md +++ b/docs/integrations/sources/snowflake.md @@ -76,6 +76,7 @@ Your database user should now be ready for use with Airbyte. | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.8 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | | 0.1.7 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.1.6 | 2022-01-25 | [9623](https://github.com/airbytehq/airbyte/pull/9623) | Add jdbc_url_params support for optional JDBC parameters | | 0.1.5 | 2022-01-19 | [9567](https://github.com/airbytehq/airbyte/pull/9567) | Added parameter for keeping JDBC session alive |