From f6fbee66852ba376c849a9fe838ddcf0285d4d03 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Mon, 7 Nov 2022 14:14:29 +0200 Subject: [PATCH 01/10] Postgres/MySQL Source Strict Encrypt: stop enforce SSL if ssl mode disabled --- .../java/io/airbyte/db/jdbc/JdbcUtils.java | 7 +++- .../io/airbyte/db/jdbc/TestJdbcUtils.java | 36 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java index e359a2331889d..9d300a801a1c6 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java @@ -47,6 +47,7 @@ public class JdbcUtils { // NOTE: this is the plural version of SCHEMA_KEY public static final String SCHEMAS_KEY = "schemas"; public static final String SSL_KEY = "ssl"; + public static final List SSL_MODE_DISABLE = List.of("disable", "disabled"); public static final String SSL_MODE_KEY = "ssl_mode"; public static final String TLS_KEY = "tls"; public static final String USERNAME_KEY = "username"; @@ -114,7 +115,11 @@ public static Map parseJdbcParameters(final String jdbcPropertie * @return true: if ssl has not been set or it has been set with true, false: in all other cases */ public static boolean useSsl(final JsonNode config) { - return !config.has(SSL_KEY) || config.get(SSL_KEY).asBoolean(); + if(!config.has(SSL_KEY)){ + if (config.has(SSL_MODE_KEY) && config.get(SSL_MODE_KEY).has(MODE_KEY)) { + return !SSL_MODE_DISABLE.contains(config.get(SSL_MODE_KEY).get(MODE_KEY).asText()); + } else return true; + } else return config.get(SSL_KEY).asBoolean(); } } diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java index d6a4b139a2bf0..f30ea61fa6eb1 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java @@ -210,6 +210,42 @@ void testUssSslWithSslSetAndValueIntegerTrue() { assertTrue(sslSet); } + @Test + void testUseSslWithEmptySslKeyAndSslModeVerifyFull() { + final JsonNode config = Jsons.jsonNode(ImmutableMap.builder() + .put("host", PSQL_DB.getHost()) + .put("port", PSQL_DB.getFirstMappedPort()) + .put("database", dbName) + .put("username", PSQL_DB.getUsername()) + .put("password", PSQL_DB.getPassword()) + .put("ssl_mode", ImmutableMap.builder() + .put("mode", "verify-full") + .put("ca_certificate", "test_ca_cert") + .put("client_certificate", "test_client_cert") + .put("client_key", "test_client_key") + .put("client_key_password", "test_pass") + .build()) + .build()); + final boolean sslSet = JdbcUtils.useSsl(config); + assertTrue(sslSet); + } + + @Test + void testUseSslWithEmptySslKeyAndSslModeDisable() { + final JsonNode config = Jsons.jsonNode(ImmutableMap.builder() + .put("host", PSQL_DB.getHost()) + .put("port", PSQL_DB.getFirstMappedPort()) + .put("database", dbName) + .put("username", PSQL_DB.getUsername()) + .put("password", PSQL_DB.getPassword()) + .put("ssl_mode", ImmutableMap.builder() + .put("mode", "disable") + .build()) + .build()); + final boolean sslSet = JdbcUtils.useSsl(config); + assertFalse(sslSet); + } + private static void createTableWithAllTypes(final Connection connection) throws SQLException { // jdbctype not included because they are not directly supported in postgres: TINYINT, LONGVARCHAR, // VARBINAR, LONGVARBINARY From 3e2e3c43a7a8891e05078d0fe2741bf08a0bd497 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Mon, 7 Nov 2022 15:08:18 +0200 Subject: [PATCH 02/10] fixed checkstyle --- .../java/io/airbyte/db/jdbc/JdbcUtils.java | 8 ++-- .../io/airbyte/db/jdbc/TestJdbcUtils.java | 44 +++++++++---------- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java index 9d300a801a1c6..4b50ee5791430 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java @@ -115,11 +115,13 @@ public static Map parseJdbcParameters(final String jdbcPropertie * @return true: if ssl has not been set or it has been set with true, false: in all other cases */ public static boolean useSsl(final JsonNode config) { - if(!config.has(SSL_KEY)){ + if (!config.has(SSL_KEY)) { if (config.has(SSL_MODE_KEY) && config.get(SSL_MODE_KEY).has(MODE_KEY)) { return !SSL_MODE_DISABLE.contains(config.get(SSL_MODE_KEY).get(MODE_KEY).asText()); - } else return true; - } else return config.get(SSL_KEY).asBoolean(); + } else + return true; + } else + return config.get(SSL_KEY).asBoolean(); } } diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java index f30ea61fa6eb1..92435db61ce4b 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java @@ -213,19 +213,19 @@ void testUssSslWithSslSetAndValueIntegerTrue() { @Test void testUseSslWithEmptySslKeyAndSslModeVerifyFull() { final JsonNode config = Jsons.jsonNode(ImmutableMap.builder() - .put("host", PSQL_DB.getHost()) - .put("port", PSQL_DB.getFirstMappedPort()) - .put("database", dbName) - .put("username", PSQL_DB.getUsername()) - .put("password", PSQL_DB.getPassword()) - .put("ssl_mode", ImmutableMap.builder() - .put("mode", "verify-full") - .put("ca_certificate", "test_ca_cert") - .put("client_certificate", "test_client_cert") - .put("client_key", "test_client_key") - .put("client_key_password", "test_pass") - .build()) - .build()); + .put("host", PSQL_DB.getHost()) + .put("port", PSQL_DB.getFirstMappedPort()) + .put("database", dbName) + .put("username", PSQL_DB.getUsername()) + .put("password", PSQL_DB.getPassword()) + .put("ssl_mode", ImmutableMap.builder() + .put("mode", "verify-full") + .put("ca_certificate", "test_ca_cert") + .put("client_certificate", "test_client_cert") + .put("client_key", "test_client_key") + .put("client_key_password", "test_pass") + .build()) + .build()); final boolean sslSet = JdbcUtils.useSsl(config); assertTrue(sslSet); } @@ -233,15 +233,15 @@ void testUseSslWithEmptySslKeyAndSslModeVerifyFull() { @Test void testUseSslWithEmptySslKeyAndSslModeDisable() { final JsonNode config = Jsons.jsonNode(ImmutableMap.builder() - .put("host", PSQL_DB.getHost()) - .put("port", PSQL_DB.getFirstMappedPort()) - .put("database", dbName) - .put("username", PSQL_DB.getUsername()) - .put("password", PSQL_DB.getPassword()) - .put("ssl_mode", ImmutableMap.builder() - .put("mode", "disable") - .build()) - .build()); + .put("host", PSQL_DB.getHost()) + .put("port", PSQL_DB.getFirstMappedPort()) + .put("database", dbName) + .put("username", PSQL_DB.getUsername()) + .put("password", PSQL_DB.getPassword()) + .put("ssl_mode", ImmutableMap.builder() + .put("mode", "disable") + .build()) + .build()); final boolean sslSet = JdbcUtils.useSsl(config); assertFalse(sslSet); } From 2bcd621270cd1703c6e35ad3acf020e14962b842 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Mon, 7 Nov 2022 16:16:16 +0200 Subject: [PATCH 03/10] updated changelog --- docs/integrations/sources/mysql.md | 1 + docs/integrations/sources/postgres.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/integrations/sources/mysql.md b/docs/integrations/sources/mysql.md index b2a78ecf2869e..bc03813c24077 100644 --- a/docs/integrations/sources/mysql.md +++ b/docs/integrations/sources/mysql.md @@ -252,6 +252,7 @@ WHERE actor_definition_id ='435bb9a5-7887-4809-aa58-28c27df0d7ad' AND (configura ## Changelog | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.0.12 | 2022-11-07 | [19025](https://github.com/airbytehq/airbyte/pull/19025) | Stop enforce SSL if ssl mode is disabled | | 1.0.11 | 2022-11-03 | [18851](https://github.com/airbytehq/airbyte/pull/18851) | Fix bug with unencrypted CDC connections | | 1.0.10 | 2022-11-02 | [18619](https://github.com/airbytehq/airbyte/pull/18619) | Fix bug with handling Tinyint(1) Unsigned values as boolean | | 1.0.9 | 2022-10-31 | [18538](https://github.com/airbytehq/airbyte/pull/18538) | Encode database name | diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index 4d73c604a67e0..7844f663ea82a 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -400,6 +400,7 @@ The root causes is that the WALs needed for the incremental sync has been remove | Version | Date | Pull Request | Subject | |:--------|:-----------|:-------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.0.23 | 2022-11-07 | [19025](https://github.com/airbytehq/airbyte/pull/19025) | Stop enforce SSL if ssl mode is disabled | | 1.0.22 | 2022-10-31 | [18538](https://github.com/airbytehq/airbyte/pull/18538) | Encode database name | | 1.0.21 | 2022-10-25 | [18256](https://github.com/airbytehq/airbyte/pull/18256) | Disable allow and prefer ssl modes in CDC mode | | 1.0.20 | 2022-10-25 | [18383](https://github.com/airbytehq/airbyte/pull/18383) | Better SSH error handling + messages | From 3d2d0c86acd5ba00798a8a707a3964ca84ed842a Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Tue, 8 Nov 2022 17:30:29 +0200 Subject: [PATCH 04/10] add tests --- .../java/io/airbyte/db/jdbc/JdbcUtils.java | 3 +- .../source/mysql/MySqlSourceTests.java | 29 +++++++++++++++ .../source/postgres/PostgresSourceTest.java | 36 ++++++++++++++++++- 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java index 4b50ee5791430..5529b35668c43 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java @@ -112,7 +112,8 @@ public static Map parseJdbcParameters(final String jdbcPropertie * (e.g. non-zero integers, string true, etc) * * @param config A configuration used to check Jdbc connection - * @return true: if ssl has not been set or it has been set with true, false: in all other cases + * @return true: if ssl has not been set and ssl mode not equals disabled or it has been set with + * true, false: in all other cases */ public static boolean useSsl(final JsonNode config) { if (!config.has(SSL_KEY)) { diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java index 7589550029578..006c723c14dc8 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java @@ -13,14 +13,20 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.base.ssh.SshBastionContainer; +import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.integrations.util.HostPortResolver; import io.airbyte.protocol.models.AirbyteConnectionStatus; import java.sql.Connection; import java.sql.DriverManager; +import java.util.List; +import java.util.Map; import java.util.Properties; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.containers.Network; import org.testcontainers.containers.output.Slf4jLogConsumer; public class MySqlSourceTests { @@ -29,6 +35,8 @@ public class MySqlSourceTests { private static final String TEST_USER = "test"; private static final String TEST_PASSWORD = "test"; + private static final SshBastionContainer bastion = new SshBastionContainer(); + private static final Network network = Network.newNetwork(); @Test public void testSettingTimezones() throws Exception { @@ -86,4 +94,25 @@ private JsonNode buildConfigEscapingNeeded() { JdbcUtils.DATABASE_KEY, "db/foo")); } + @Test + void testCheckWithSSlModeDisabled() throws Exception { + try (final MySQLContainer db = new MySQLContainer<>("mysql:8.0").withNetwork(network)) { + bastion.initAndStartBastion(network); + db.start(); + final JsonNode configWithSSLModeDisabled = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, ImmutableMap.builder() + .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db)) + .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(db)) + .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) + .put(JdbcUtils.SCHEMAS_KEY, List.of("public")) + .put(JdbcUtils.USERNAME_KEY, db.getUsername()) + .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) + .put(JdbcUtils.SSL_MODE_KEY, Map.of(JdbcUtils.MODE_KEY, "disable"))); + + final AirbyteConnectionStatus actual = MySqlSource.sshWrappedSource().check(configWithSSLModeDisabled); + assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, actual.getStatus()); + } finally { + bastion.stopAndClose(); + } + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java index 99712900e88db..120f2dc415ca9 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java @@ -27,7 +27,11 @@ import io.airbyte.db.factory.DSLContextFactory; import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.base.ssh.SshBastionContainer; +import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.integrations.util.HostPortResolver; import io.airbyte.protocol.models.AirbyteCatalog; +import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteStream; import io.airbyte.protocol.models.CatalogHelpers; @@ -52,6 +56,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.testcontainers.containers.Network; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.utility.MountableFile; @@ -116,7 +121,8 @@ class PostgresSourceTest { createRecord(STREAM_NAME_PRIVILEGES_TEST_CASE_VIEW, SCHEMA_NAME, ImmutableMap.of("id", 1, "name", "Zed")), createRecord(STREAM_NAME_PRIVILEGES_TEST_CASE_VIEW, SCHEMA_NAME, ImmutableMap.of("id", 2, "name", "Jack")), createRecord(STREAM_NAME_PRIVILEGES_TEST_CASE_VIEW, SCHEMA_NAME, ImmutableMap.of("id", 3, "name", "Antuan"))); - + private static final SshBastionContainer bastion = new SshBastionContainer(); + private static final Network network = Network.newNetwork(); private static PostgreSQLContainer PSQL_DB; private String dbName; @@ -571,4 +577,32 @@ private JsonNode buildConfigEscapingNeeded() { JdbcUtils.DATABASE_KEY, "db/foo")); } + @Test + void testCheckWithSSlModeDisable() throws Exception { + + try (PostgreSQLContainer db = new PostgreSQLContainer<>("postgres:13-alpine").withNetwork(network)) { + bastion.initAndStartBastion(network); + db.start(); + // stop to enforce ssl for ssl_mode disable + final ImmutableMap.Builder builderWithSSLModeDisable = getDatabaseConfigBuilderWithSSLMode(db, "disable"); + final JsonNode configWithSSLModeDisable = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, builderWithSSLModeDisable); + final AirbyteConnectionStatus connectionStatusForDisabledMode = PostgresSource.sshWrappedSource().check(configWithSSLModeDisable); + assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatusForDisabledMode.getStatus()); + + } finally { + bastion.stopAndClose(); + } + } + + private ImmutableMap.Builder getDatabaseConfigBuilderWithSSLMode(PostgreSQLContainer db, String sslMode) { + return ImmutableMap.builder() + .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db)) + .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(db)) + .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) + .put(JdbcUtils.SCHEMAS_KEY, List.of("public")) + .put(JdbcUtils.USERNAME_KEY, db.getUsername()) + .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) + .put(JdbcUtils.SSL_MODE_KEY, Map.of(JdbcUtils.MODE_KEY, sslMode)); + } + } From 3e198de694575f84622c925c9cb42218a72e6c28 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Tue, 8 Nov 2022 21:31:55 +0200 Subject: [PATCH 05/10] replaced MySQL test to mysql-strict-encrypt module --- ...StrictEncryptJdbcSourceAcceptanceTest.java | 28 ++++++++++++++++++ .../source/mysql/MySqlSourceTests.java | 29 ------------------- 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java index eb58bd9c10c1c..3824234d06e54 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java @@ -22,10 +22,13 @@ import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.Source; +import io.airbyte.integrations.base.ssh.SshBastionContainer; import io.airbyte.integrations.base.ssh.SshHelpers; +import io.airbyte.integrations.base.ssh.SshTunnel; import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.integrations.source.mysql.MySqlSource; import io.airbyte.integrations.source.relationaldb.models.DbStreamState; +import io.airbyte.integrations.util.HostPortResolver; import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; @@ -51,12 +54,15 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.containers.Network; class MySqlStrictEncryptJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { protected static final String TEST_USER = "test"; protected static final String TEST_PASSWORD = "test"; protected static MySQLContainer container; + private static final SshBastionContainer bastion = new SshBastionContainer(); + private static final Network network = Network.newNetwork(); protected Database database; protected DSLContext dslContext; @@ -328,6 +334,28 @@ void testStrictSSLUnsecuredWithTunnel() throws Exception { assertTrue(actual.getMessage().contains("Could not connect with provided SSH configuration.")); } + + @Test + void testCheckWithSSlModeDisabled() throws Exception { + try (final MySQLContainer db = new MySQLContainer<>("mysql:8.0").withNetwork(network)) { + bastion.initAndStartBastion(network); + db.start(); + final JsonNode configWithSSLModeDisabled = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, ImmutableMap.builder() + .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db)) + .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(db)) + .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) + .put(JdbcUtils.SCHEMAS_KEY, List.of("public")) + .put(JdbcUtils.USERNAME_KEY, db.getUsername()) + .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) + .put(JdbcUtils.SSL_MODE_KEY, Map.of(JdbcUtils.MODE_KEY, "disable"))); + + final AirbyteConnectionStatus actual = source.check(configWithSSLModeDisabled); + assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, actual.getStatus()); + } finally { + bastion.stopAndClose(); + } + } + @Override protected boolean supportsPerStream() { return true; diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java index 006c723c14dc8..7589550029578 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java @@ -13,20 +13,14 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.ssh.SshBastionContainer; -import io.airbyte.integrations.base.ssh.SshTunnel; -import io.airbyte.integrations.util.HostPortResolver; import io.airbyte.protocol.models.AirbyteConnectionStatus; import java.sql.Connection; import java.sql.DriverManager; -import java.util.List; -import java.util.Map; import java.util.Properties; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.containers.MySQLContainer; -import org.testcontainers.containers.Network; import org.testcontainers.containers.output.Slf4jLogConsumer; public class MySqlSourceTests { @@ -35,8 +29,6 @@ public class MySqlSourceTests { private static final String TEST_USER = "test"; private static final String TEST_PASSWORD = "test"; - private static final SshBastionContainer bastion = new SshBastionContainer(); - private static final Network network = Network.newNetwork(); @Test public void testSettingTimezones() throws Exception { @@ -94,25 +86,4 @@ private JsonNode buildConfigEscapingNeeded() { JdbcUtils.DATABASE_KEY, "db/foo")); } - @Test - void testCheckWithSSlModeDisabled() throws Exception { - try (final MySQLContainer db = new MySQLContainer<>("mysql:8.0").withNetwork(network)) { - bastion.initAndStartBastion(network); - db.start(); - final JsonNode configWithSSLModeDisabled = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db)) - .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(db)) - .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.SCHEMAS_KEY, List.of("public")) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) - .put(JdbcUtils.SSL_MODE_KEY, Map.of(JdbcUtils.MODE_KEY, "disable"))); - - final AirbyteConnectionStatus actual = MySqlSource.sshWrappedSource().check(configWithSSLModeDisabled); - assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, actual.getStatus()); - } finally { - bastion.stopAndClose(); - } - } - } From cb950f96fe402c9db8decf86473a226b186f0918 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Tue, 8 Nov 2022 22:08:10 +0200 Subject: [PATCH 06/10] fixed Connection Refused for mysql test --- ...ySqlStrictEncryptJdbcSourceAcceptanceTest.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java index 3824234d06e54..c5516cb56e153 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java @@ -42,10 +42,8 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; + import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterAll; @@ -341,8 +339,13 @@ void testCheckWithSSlModeDisabled() throws Exception { bastion.initAndStartBastion(network); db.start(); final JsonNode configWithSSLModeDisabled = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db)) - .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(db)) + .put(JdbcUtils.HOST_KEY, Objects.requireNonNull(db.getContainerInfo() + .getNetworkSettings() + .getNetworks() + .entrySet().stream() + .findFirst() + .get().getValue().getIpAddress())) + .put(JdbcUtils.PORT_KEY, db.getExposedPorts().get(0)) .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) .put(JdbcUtils.SCHEMAS_KEY, List.of("public")) .put(JdbcUtils.USERNAME_KEY, db.getUsername()) From 8d1527c7df336e4fccff2e8829bc95f145336282 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Tue, 8 Nov 2022 23:45:09 +0200 Subject: [PATCH 07/10] replaced Postgres Source strict-encrypt tests into new class --- .../PostgresSourceStrictEncryptTest.java | 77 +++++++++++++++++++ .../source/postgres/PostgresSourceTest.java | 36 +-------- 2 files changed, 78 insertions(+), 35 deletions(-) create mode 100644 airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceStrictEncryptTest.java diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceStrictEncryptTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceStrictEncryptTest.java new file mode 100644 index 0000000000000..b31de10fcc583 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceStrictEncryptTest.java @@ -0,0 +1,77 @@ +package io.airbyte.integrations.source.postgres; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.base.ssh.SshBastionContainer; +import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.protocol.models.AirbyteConnectionStatus; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.PostgreSQLContainer; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class PostgresSourceStrictEncryptTest { + private static final SshBastionContainer bastion = new SshBastionContainer(); + private static final Network network = Network.newNetwork(); + + @Test + void testCheckWithSSlModeDisable() throws Exception { + + try (PostgreSQLContainer db = new PostgreSQLContainer<>("postgres:13-alpine").withNetwork(network)) { + bastion.initAndStartBastion(network); + db.start(); + + // stop to enforce ssl for ssl_mode disable + final ImmutableMap.Builder builderWithSSLModeDisable = getDatabaseConfigBuilderWithSSLMode(db, "disable"); + final JsonNode configWithSSLModeDisable = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, builderWithSSLModeDisable); + final AirbyteConnectionStatus connectionStatusForDisabledMode = new PostgresSourceStrictEncrypt().check(configWithSSLModeDisable); + assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatusForDisabledMode.getStatus()); + + } finally { + bastion.stopAndClose(); + } + } + + @Test + void testCheckWithSSlModePrefer() throws Exception { + + try (PostgreSQLContainer db = new PostgreSQLContainer<>("postgres:13-alpine").withNetwork(network)) { + bastion.initAndStartBastion(network); + db.start(); + //continue to enforce ssl because ssl mode is prefer + final ImmutableMap.Builder builderWithSSLModePrefer = getDatabaseConfigBuilderWithSSLMode(db, "prefer"); + final JsonNode configWithSSLModePrefer = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, builderWithSSLModePrefer); + final AirbyteConnectionStatus connectionStatusForPreferredMode = new PostgresSourceStrictEncrypt().check(configWithSSLModePrefer); + assertEquals(AirbyteConnectionStatus.Status.FAILED, connectionStatusForPreferredMode.getStatus()); + assertEquals("State code: 08004; Message: The server does not support SSL.", connectionStatusForPreferredMode.getMessage()); + + } finally { + bastion.stopAndClose(); + } + } + + private ImmutableMap.Builder getDatabaseConfigBuilderWithSSLMode(PostgreSQLContainer db, String sslMode) { + return ImmutableMap.builder() + .put(JdbcUtils.HOST_KEY, Objects.requireNonNull(db.getContainerInfo() + .getNetworkSettings() + .getNetworks() + .entrySet().stream() + .findFirst() + .get().getValue().getIpAddress())) + .put(JdbcUtils.PORT_KEY, db.getExposedPorts().get(0)) + .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) + .put(JdbcUtils.SCHEMAS_KEY, List.of("public")) + .put(JdbcUtils.USERNAME_KEY, db.getUsername()) + .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) + .put(JdbcUtils.SSL_MODE_KEY, Map.of(JdbcUtils.MODE_KEY, sslMode)); + } + + + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java index 120f2dc415ca9..99712900e88db 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java @@ -27,11 +27,7 @@ import io.airbyte.db.factory.DSLContextFactory; import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.ssh.SshBastionContainer; -import io.airbyte.integrations.base.ssh.SshTunnel; -import io.airbyte.integrations.util.HostPortResolver; import io.airbyte.protocol.models.AirbyteCatalog; -import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteStream; import io.airbyte.protocol.models.CatalogHelpers; @@ -56,7 +52,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.testcontainers.containers.Network; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.utility.MountableFile; @@ -121,8 +116,7 @@ class PostgresSourceTest { createRecord(STREAM_NAME_PRIVILEGES_TEST_CASE_VIEW, SCHEMA_NAME, ImmutableMap.of("id", 1, "name", "Zed")), createRecord(STREAM_NAME_PRIVILEGES_TEST_CASE_VIEW, SCHEMA_NAME, ImmutableMap.of("id", 2, "name", "Jack")), createRecord(STREAM_NAME_PRIVILEGES_TEST_CASE_VIEW, SCHEMA_NAME, ImmutableMap.of("id", 3, "name", "Antuan"))); - private static final SshBastionContainer bastion = new SshBastionContainer(); - private static final Network network = Network.newNetwork(); + private static PostgreSQLContainer PSQL_DB; private String dbName; @@ -577,32 +571,4 @@ private JsonNode buildConfigEscapingNeeded() { JdbcUtils.DATABASE_KEY, "db/foo")); } - @Test - void testCheckWithSSlModeDisable() throws Exception { - - try (PostgreSQLContainer db = new PostgreSQLContainer<>("postgres:13-alpine").withNetwork(network)) { - bastion.initAndStartBastion(network); - db.start(); - // stop to enforce ssl for ssl_mode disable - final ImmutableMap.Builder builderWithSSLModeDisable = getDatabaseConfigBuilderWithSSLMode(db, "disable"); - final JsonNode configWithSSLModeDisable = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, builderWithSSLModeDisable); - final AirbyteConnectionStatus connectionStatusForDisabledMode = PostgresSource.sshWrappedSource().check(configWithSSLModeDisable); - assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatusForDisabledMode.getStatus()); - - } finally { - bastion.stopAndClose(); - } - } - - private ImmutableMap.Builder getDatabaseConfigBuilderWithSSLMode(PostgreSQLContainer db, String sslMode) { - return ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db)) - .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(db)) - .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.SCHEMAS_KEY, List.of("public")) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) - .put(JdbcUtils.SSL_MODE_KEY, Map.of(JdbcUtils.MODE_KEY, sslMode)); - } - } From cca315c6adcd4b988f08ba53831d8221aa0304c4 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Wed, 9 Nov 2022 22:05:54 +0200 Subject: [PATCH 08/10] bump version --- .../connectors/source-mysql-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-mysql/Dockerfile | 2 +- .../connectors/source-postgres-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-postgres/Dockerfile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile index 86a16ad9c7f96..fc05749d57fa9 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile @@ -16,6 +16,6 @@ ENV APPLICATION source-mysql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.11 +LABEL io.airbyte.version=1.0.12 LABEL io.airbyte.name=airbyte/source-mysql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mysql/Dockerfile b/airbyte-integrations/connectors/source-mysql/Dockerfile index b8186ad829520..3889a9a1e60a7 100644 --- a/airbyte-integrations/connectors/source-mysql/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql/Dockerfile @@ -16,6 +16,6 @@ ENV APPLICATION source-mysql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.11 +LABEL io.airbyte.version=1.0.12 LABEL io.airbyte.name=airbyte/source-mysql diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile index 8c43729edb768..fff90a2cc674e 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.22 +LABEL io.airbyte.version=1.0.23 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index 77a80d05f0b80..fc8ad82a1e2a0 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.22 +LABEL io.airbyte.version=1.0.23 LABEL io.airbyte.name=airbyte/source-postgres From 221e76de7dd22c52603ada59c01797f2b0313250 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Wed, 9 Nov 2022 21:18:29 +0000 Subject: [PATCH 09/10] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 60af228a3740c..8880d85e308f5 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -999,7 +999,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 1.0.22 + dockerImageTag: 1.0.23 documentationUrl: https://docs.airbyte.com/integrations/sources/postgres icon: postgresql.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 15a3c8030b740..41619c0ca6dd0 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -9655,7 +9655,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:1.0.22" +- dockerImage: "airbyte/source-postgres:1.0.23" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: From 692ba6b09e9f36d80180bbb963bdf5f34581579e Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Wed, 9 Nov 2022 21:59:14 +0000 Subject: [PATCH 10/10] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 60af228a3740c..3bdd005245a16 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -813,7 +813,7 @@ - name: MySQL sourceDefinitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad dockerRepository: airbyte/source-mysql - dockerImageTag: 1.0.11 + dockerImageTag: 1.0.12 documentationUrl: https://docs.airbyte.com/integrations/sources/mysql icon: mysql.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 15a3c8030b740..5da426ec038d6 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -7739,7 +7739,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mysql:1.0.11" +- dockerImage: "airbyte/source-mysql:1.0.12" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/mysql" connectionSpecification: