Skip to content

Commit c72b75e

Browse files
VitaliiMaltsevoctavia-squidington-iii
VitaliiMaltsev
and
octavia-squidington-iii
authored
Postgres/MySQL Source Strict Encrypt: stop enforce SSL if ssl mode disabled (#19025)
* Postgres/MySQL Source Strict Encrypt: stop enforce SSL if ssl mode disabled * fixed checkstyle * updated changelog * add tests * replaced MySQL test to mysql-strict-encrypt module * fixed Connection Refused for mysql test * replaced Postgres Source strict-encrypt tests into new class * bump version * auto-bump connector version * auto-bump connector version Co-authored-by: Octavia Squidington III <[email protected]>
1 parent f54f4af commit c72b75e

File tree

12 files changed

+168
-14
lines changed

12 files changed

+168
-14
lines changed

‎airbyte-config/init/src/main/resources/seed/source_definitions.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -870,7 +870,7 @@
870870
- name: MySQL
871871
sourceDefinitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad
872872
dockerRepository: airbyte/source-mysql
873-
dockerImageTag: 1.0.11
873+
dockerImageTag: 1.0.12
874874
documentationUrl: https://docs.airbyte.com/integrations/sources/mysql
875875
icon: mysql.svg
876876
sourceType: database
@@ -1070,7 +1070,7 @@
10701070
- name: Postgres
10711071
sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
10721072
dockerRepository: airbyte/source-postgres
1073-
dockerImageTag: 1.0.22
1073+
dockerImageTag: 1.0.23
10741074
documentationUrl: https://docs.airbyte.com/integrations/sources/postgres
10751075
icon: postgresql.svg
10761076
sourceType: database

‎airbyte-config/init/src/main/resources/seed/source_specs.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8167,7 +8167,7 @@
81678167
supportsNormalization: false
81688168
supportsDBT: false
81698169
supported_destination_sync_modes: []
8170-
- dockerImage: "airbyte/source-mysql:1.0.11"
8170+
- dockerImage: "airbyte/source-mysql:1.0.12"
81718171
spec:
81728172
documentationUrl: "https://docs.airbyte.com/integrations/sources/mysql"
81738173
connectionSpecification:
@@ -10152,7 +10152,7 @@
1015210152
supportsNormalization: false
1015310153
supportsDBT: false
1015410154
supported_destination_sync_modes: []
10155-
- dockerImage: "airbyte/source-postgres:1.0.22"
10155+
- dockerImage: "airbyte/source-postgres:1.0.23"
1015610156
spec:
1015710157
documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres"
1015810158
connectionSpecification:

‎airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ public class JdbcUtils {
4747
// NOTE: this is the plural version of SCHEMA_KEY
4848
public static final String SCHEMAS_KEY = "schemas";
4949
public static final String SSL_KEY = "ssl";
50+
public static final List<String> SSL_MODE_DISABLE = List.of("disable", "disabled");
5051
public static final String SSL_MODE_KEY = "ssl_mode";
5152
public static final String TLS_KEY = "tls";
5253
public static final String USERNAME_KEY = "username";
@@ -111,10 +112,17 @@ public static Map<String, String> parseJdbcParameters(final String jdbcPropertie
111112
* (e.g. non-zero integers, string true, etc)
112113
*
113114
* @param config A configuration used to check Jdbc connection
114-
* @return true: if ssl has not been set or it has been set with true, false: in all other cases
115+
* @return true: if ssl has not been set and ssl mode not equals disabled or it has been set with
116+
* true, false: in all other cases
115117
*/
116118
public static boolean useSsl(final JsonNode config) {
117-
return !config.has(SSL_KEY) || config.get(SSL_KEY).asBoolean();
119+
if (!config.has(SSL_KEY)) {
120+
if (config.has(SSL_MODE_KEY) && config.get(SSL_MODE_KEY).has(MODE_KEY)) {
121+
return !SSL_MODE_DISABLE.contains(config.get(SSL_MODE_KEY).get(MODE_KEY).asText());
122+
} else
123+
return true;
124+
} else
125+
return config.get(SSL_KEY).asBoolean();
118126
}
119127

120128
}

‎airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,42 @@ void testUssSslWithSslSetAndValueIntegerTrue() {
210210
assertTrue(sslSet);
211211
}
212212

213+
@Test
214+
void testUseSslWithEmptySslKeyAndSslModeVerifyFull() {
215+
final JsonNode config = Jsons.jsonNode(ImmutableMap.builder()
216+
.put("host", PSQL_DB.getHost())
217+
.put("port", PSQL_DB.getFirstMappedPort())
218+
.put("database", dbName)
219+
.put("username", PSQL_DB.getUsername())
220+
.put("password", PSQL_DB.getPassword())
221+
.put("ssl_mode", ImmutableMap.builder()
222+
.put("mode", "verify-full")
223+
.put("ca_certificate", "test_ca_cert")
224+
.put("client_certificate", "test_client_cert")
225+
.put("client_key", "test_client_key")
226+
.put("client_key_password", "test_pass")
227+
.build())
228+
.build());
229+
final boolean sslSet = JdbcUtils.useSsl(config);
230+
assertTrue(sslSet);
231+
}
232+
233+
@Test
234+
void testUseSslWithEmptySslKeyAndSslModeDisable() {
235+
final JsonNode config = Jsons.jsonNode(ImmutableMap.builder()
236+
.put("host", PSQL_DB.getHost())
237+
.put("port", PSQL_DB.getFirstMappedPort())
238+
.put("database", dbName)
239+
.put("username", PSQL_DB.getUsername())
240+
.put("password", PSQL_DB.getPassword())
241+
.put("ssl_mode", ImmutableMap.builder()
242+
.put("mode", "disable")
243+
.build())
244+
.build());
245+
final boolean sslSet = JdbcUtils.useSsl(config);
246+
assertFalse(sslSet);
247+
}
248+
213249
private static void createTableWithAllTypes(final Connection connection) throws SQLException {
214250
// jdbctype not included because they are not directly supported in postgres: TINYINT, LONGVARCHAR,
215251
// VARBINAR, LONGVARBINARY

‎airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,6 @@ ENV APPLICATION source-mysql-strict-encrypt
1616

1717
COPY --from=build /airbyte /airbyte
1818

19-
LABEL io.airbyte.version=1.0.11
19+
LABEL io.airbyte.version=1.0.12
2020

2121
LABEL io.airbyte.name=airbyte/source-mysql-strict-encrypt

‎airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,13 @@
2222
import io.airbyte.db.factory.DatabaseDriver;
2323
import io.airbyte.db.jdbc.JdbcUtils;
2424
import io.airbyte.integrations.base.Source;
25+
import io.airbyte.integrations.base.ssh.SshBastionContainer;
2526
import io.airbyte.integrations.base.ssh.SshHelpers;
27+
import io.airbyte.integrations.base.ssh.SshTunnel;
2628
import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest;
2729
import io.airbyte.integrations.source.mysql.MySqlSource;
2830
import io.airbyte.integrations.source.relationaldb.models.DbStreamState;
31+
import io.airbyte.integrations.util.HostPortResolver;
2932
import io.airbyte.protocol.models.AirbyteCatalog;
3033
import io.airbyte.protocol.models.AirbyteConnectionStatus;
3134
import io.airbyte.protocol.models.AirbyteConnectionStatus.Status;
@@ -39,10 +42,8 @@
3942
import java.sql.Connection;
4043
import java.sql.DriverManager;
4144
import java.sql.SQLException;
42-
import java.util.ArrayList;
43-
import java.util.Collections;
44-
import java.util.List;
45-
import java.util.Map;
45+
import java.util.*;
46+
4647
import org.jooq.DSLContext;
4748
import org.jooq.SQLDialect;
4849
import org.junit.jupiter.api.AfterAll;
@@ -51,12 +52,15 @@
5152
import org.junit.jupiter.api.BeforeEach;
5253
import org.junit.jupiter.api.Test;
5354
import org.testcontainers.containers.MySQLContainer;
55+
import org.testcontainers.containers.Network;
5456

5557
class MySqlStrictEncryptJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest {
5658

5759
protected static final String TEST_USER = "test";
5860
protected static final String TEST_PASSWORD = "test";
5961
protected static MySQLContainer<?> container;
62+
private static final SshBastionContainer bastion = new SshBastionContainer();
63+
private static final Network network = Network.newNetwork();
6064

6165
protected Database database;
6266
protected DSLContext dslContext;
@@ -328,6 +332,33 @@ void testStrictSSLUnsecuredWithTunnel() throws Exception {
328332
assertTrue(actual.getMessage().contains("Could not connect with provided SSH configuration."));
329333
}
330334

335+
336+
@Test
337+
void testCheckWithSSlModeDisabled() throws Exception {
338+
try (final MySQLContainer<?> db = new MySQLContainer<>("mysql:8.0").withNetwork(network)) {
339+
bastion.initAndStartBastion(network);
340+
db.start();
341+
final JsonNode configWithSSLModeDisabled = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, ImmutableMap.builder()
342+
.put(JdbcUtils.HOST_KEY, Objects.requireNonNull(db.getContainerInfo()
343+
.getNetworkSettings()
344+
.getNetworks()
345+
.entrySet().stream()
346+
.findFirst()
347+
.get().getValue().getIpAddress()))
348+
.put(JdbcUtils.PORT_KEY, db.getExposedPorts().get(0))
349+
.put(JdbcUtils.DATABASE_KEY, db.getDatabaseName())
350+
.put(JdbcUtils.SCHEMAS_KEY, List.of("public"))
351+
.put(JdbcUtils.USERNAME_KEY, db.getUsername())
352+
.put(JdbcUtils.PASSWORD_KEY, db.getPassword())
353+
.put(JdbcUtils.SSL_MODE_KEY, Map.of(JdbcUtils.MODE_KEY, "disable")));
354+
355+
final AirbyteConnectionStatus actual = source.check(configWithSSLModeDisabled);
356+
assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, actual.getStatus());
357+
} finally {
358+
bastion.stopAndClose();
359+
}
360+
}
361+
331362
@Override
332363
protected boolean supportsPerStream() {
333364
return true;

‎airbyte-integrations/connectors/source-mysql/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,6 @@ ENV APPLICATION source-mysql
1616

1717
COPY --from=build /airbyte /airbyte
1818

19-
LABEL io.airbyte.version=1.0.11
19+
LABEL io.airbyte.version=1.0.12
2020

2121
LABEL io.airbyte.name=airbyte/source-mysql

‎airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt
1616

1717
COPY --from=build /airbyte /airbyte
1818

19-
LABEL io.airbyte.version=1.0.22
19+
LABEL io.airbyte.version=1.0.23
2020
LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt

‎airbyte-integrations/connectors/source-postgres/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,5 @@ ENV APPLICATION source-postgres
1616

1717
COPY --from=build /airbyte /airbyte
1818

19-
LABEL io.airbyte.version=1.0.22
19+
LABEL io.airbyte.version=1.0.23
2020
LABEL io.airbyte.name=airbyte/source-postgres

‎airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceStrictEncryptTest.java

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
package io.airbyte.integrations.source.postgres;
2+
3+
import com.fasterxml.jackson.databind.JsonNode;
4+
import com.google.common.collect.ImmutableMap;
5+
import io.airbyte.db.jdbc.JdbcUtils;
6+
import io.airbyte.integrations.base.ssh.SshBastionContainer;
7+
import io.airbyte.integrations.base.ssh.SshTunnel;
8+
import io.airbyte.protocol.models.AirbyteConnectionStatus;
9+
import org.junit.jupiter.api.Test;
10+
import org.testcontainers.containers.Network;
11+
import org.testcontainers.containers.PostgreSQLContainer;
12+
13+
import java.util.List;
14+
import java.util.Map;
15+
import java.util.Objects;
16+
17+
import static org.junit.jupiter.api.Assertions.assertEquals;
18+
19+
public class PostgresSourceStrictEncryptTest {
20+
private static final SshBastionContainer bastion = new SshBastionContainer();
21+
private static final Network network = Network.newNetwork();
22+
23+
@Test
24+
void testCheckWithSSlModeDisable() throws Exception {
25+
26+
try (PostgreSQLContainer<?> db = new PostgreSQLContainer<>("postgres:13-alpine").withNetwork(network)) {
27+
bastion.initAndStartBastion(network);
28+
db.start();
29+
30+
// stop to enforce ssl for ssl_mode disable
31+
final ImmutableMap.Builder<Object, Object> builderWithSSLModeDisable = getDatabaseConfigBuilderWithSSLMode(db, "disable");
32+
final JsonNode configWithSSLModeDisable = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, builderWithSSLModeDisable);
33+
final AirbyteConnectionStatus connectionStatusForDisabledMode = new PostgresSourceStrictEncrypt().check(configWithSSLModeDisable);
34+
assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatusForDisabledMode.getStatus());
35+
36+
} finally {
37+
bastion.stopAndClose();
38+
}
39+
}
40+
41+
@Test
42+
void testCheckWithSSlModePrefer() throws Exception {
43+
44+
try (PostgreSQLContainer<?> db = new PostgreSQLContainer<>("postgres:13-alpine").withNetwork(network)) {
45+
bastion.initAndStartBastion(network);
46+
db.start();
47+
//continue to enforce ssl because ssl mode is prefer
48+
final ImmutableMap.Builder<Object, Object> builderWithSSLModePrefer = getDatabaseConfigBuilderWithSSLMode(db, "prefer");
49+
final JsonNode configWithSSLModePrefer = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, builderWithSSLModePrefer);
50+
final AirbyteConnectionStatus connectionStatusForPreferredMode = new PostgresSourceStrictEncrypt().check(configWithSSLModePrefer);
51+
assertEquals(AirbyteConnectionStatus.Status.FAILED, connectionStatusForPreferredMode.getStatus());
52+
assertEquals("State code: 08004; Message: The server does not support SSL.", connectionStatusForPreferredMode.getMessage());
53+
54+
} finally {
55+
bastion.stopAndClose();
56+
}
57+
}
58+
59+
private ImmutableMap.Builder<Object, Object> getDatabaseConfigBuilderWithSSLMode(PostgreSQLContainer<?> db, String sslMode) {
60+
return ImmutableMap.builder()
61+
.put(JdbcUtils.HOST_KEY, Objects.requireNonNull(db.getContainerInfo()
62+
.getNetworkSettings()
63+
.getNetworks()
64+
.entrySet().stream()
65+
.findFirst()
66+
.get().getValue().getIpAddress()))
67+
.put(JdbcUtils.PORT_KEY, db.getExposedPorts().get(0))
68+
.put(JdbcUtils.DATABASE_KEY, db.getDatabaseName())
69+
.put(JdbcUtils.SCHEMAS_KEY, List.of("public"))
70+
.put(JdbcUtils.USERNAME_KEY, db.getUsername())
71+
.put(JdbcUtils.PASSWORD_KEY, db.getPassword())
72+
.put(JdbcUtils.SSL_MODE_KEY, Map.of(JdbcUtils.MODE_KEY, sslMode));
73+
}
74+
75+
76+
77+
}

‎docs/integrations/sources/mysql.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,7 @@ WHERE actor_definition_id ='435bb9a5-7887-4809-aa58-28c27df0d7ad' AND (configura
252252
## Changelog
253253
| Version | Date | Pull Request | Subject |
254254
|:--------|:-----------|:-----------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------|
255+
| 1.0.12 | 2022-11-07 | [19025](https://github.com/airbytehq/airbyte/pull/19025) | Stop enforce SSL if ssl mode is disabled |
255256
| 1.0.11 | 2022-11-03 | [18851](https://github.com/airbytehq/airbyte/pull/18851) | Fix bug with unencrypted CDC connections |
256257
| 1.0.10 | 2022-11-02 | [18619](https://github.com/airbytehq/airbyte/pull/18619) | Fix bug with handling Tinyint(1) Unsigned values as boolean |
257258
| 1.0.9 | 2022-10-31 | [18538](https://github.com/airbytehq/airbyte/pull/18538) | Encode database name |

‎docs/integrations/sources/postgres.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -400,6 +400,7 @@ The root causes is that the WALs needed for the incremental sync has been remove
400400

401401
| Version | Date | Pull Request | Subject |
402402
|:--------|:-----------|:-------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
403+
| 1.0.23 | 2022-11-07 | [19025](https://github.com/airbytehq/airbyte/pull/19025) | Stop enforce SSL if ssl mode is disabled |
403404
| 1.0.22 | 2022-10-31 | [18538](https://github.com/airbytehq/airbyte/pull/18538) | Encode database name |
404405
| 1.0.21 | 2022-10-25 | [18256](https://github.com/airbytehq/airbyte/pull/18256) | Disable allow and prefer ssl modes in CDC mode |
405406
| 1.0.20 | 2022-10-25 | [18383](https://github.com/airbytehq/airbyte/pull/18383) | Better SSH error handling + messages |

0 commit comments

Comments
 (0)