Skip to content

Commit 637781d

Browse files
VitaliiMaltsevoctavia-squidington-iii
VitaliiMaltsev
and
octavia-squidington-iii
authored
🐛 Postgres Source Strict Encrypt: Allow connections with sslmodes 'allow' and 'prefer' if SSH tunnel established (#19551)
* Postgres Source Strict Encrypt: Allow connections with sslmodes 'allow' and 'prefer' if SSH tunnel established * updated changelog * fixed tests * add test_strictness_level to acceptance-test-config.yml * remove test_strictness_level to due to failed SAT test * bump version * auto-bump connector version Co-authored-by: Octavia Squidington III <[email protected]>
1 parent 2423c7d commit 637781d

File tree

8 files changed

+104
-97
lines changed

8 files changed

+104
-97
lines changed

‎airbyte-config/init/src/main/resources/seed/source_definitions.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1221,7 +1221,7 @@
12211221
- name: Postgres
12221222
sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
12231223
dockerRepository: airbyte/source-postgres
1224-
dockerImageTag: 1.0.25
1224+
dockerImageTag: 1.0.26
12251225
documentationUrl: https://docs.airbyte.com/integrations/sources/postgres
12261226
icon: postgresql.svg
12271227
sourceType: database

‎airbyte-config/init/src/main/resources/seed/source_specs.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -11150,7 +11150,7 @@
1115011150
supportsNormalization: false
1115111151
supportsDBT: false
1115211152
supported_destination_sync_modes: []
11153-
- dockerImage: "airbyte/source-postgres:1.0.25"
11153+
- dockerImage: "airbyte/source-postgres:1.0.26"
1115411154
spec:
1115511155
documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres"
1115611156
connectionSpecification:

‎airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt
1616

1717
COPY --from=build /airbyte /airbyte
1818

19-
LABEL io.airbyte.version=1.0.25
19+
LABEL io.airbyte.version=1.0.26
2020
LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt

‎airbyte-integrations/connectors/source-postgres/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,5 @@ ENV APPLICATION source-postgres
1616

1717
COPY --from=build /airbyte /airbyte
1818

19-
LABEL io.airbyte.version=1.0.25
19+
LABEL io.airbyte.version=1.0.26
2020
LABEL io.airbyte.name=airbyte/source-postgres

‎airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java

+2-9
Original file line numberDiff line numberDiff line change
@@ -91,9 +91,6 @@ public class PostgresSource extends AbstractJdbcSource<JDBCType> implements Sour
9191
public static final String SSL_KEY = "sslkey";
9292
public static final String SSL_PASSWORD = "sslpassword";
9393
public static final String MODE = "mode";
94-
static final Map<String, String> SSL_JDBC_PARAMETERS = ImmutableMap.of(
95-
"ssl", "true",
96-
"sslmode", "require");
9794
private List<String> schemas;
9895
private final FeatureFlags featureFlags;
9996
private static final Set<String> INVALID_CDC_SSL_MODES = ImmutableSet.of("allow", "prefer");
@@ -109,11 +106,7 @@ public static Source sshWrappedSource() {
109106

110107
@Override
111108
protected Map<String, String> getDefaultConnectionProperties(final JsonNode config) {
112-
if (JdbcUtils.useSsl(config)) {
113-
return SSL_JDBC_PARAMETERS;
114-
} else {
115-
return Collections.emptyMap();
116-
}
109+
return Collections.emptyMap();
117110
}
118111

119112
@Override
@@ -174,7 +167,7 @@ public String toJDBCQueryParams(final Map<String, String> sslParams) {
174167
.map((entry) -> {
175168
try {
176169
final String result = switch (entry.getKey()) {
177-
case SSL_MODE -> PARAM_SSLMODE + EQUALS + toSslJdbcParam(SslMode.valueOf(entry.getValue()))
170+
case AbstractJdbcSource.SSL_MODE -> PARAM_SSLMODE + EQUALS + toSslJdbcParam(SslMode.valueOf(entry.getValue()))
178171
+ JdbcUtils.AMPERSAND + PARAM_SSL + EQUALS + (entry.getValue() == DISABLE ? PARAM_SSL_FALSE : PARAM_SSL_TRUE);
179172
case CA_CERTIFICATE_PATH -> SSL_ROOT_CERT + EQUALS + entry.getValue();
180173
case CLIENT_KEY_STORE_URL -> SSL_KEY + EQUALS + Path.of(new URI(entry.getValue()));

‎airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceStrictEncryptTest.java

+95-55
Original file line numberDiff line numberDiff line change
@@ -25,45 +25,112 @@
2525

2626
public class PostgresSourceStrictEncryptTest {
2727

28+
private final PostgresSourceStrictEncrypt source = new PostgresSourceStrictEncrypt();
29+
private final PostgreSQLContainer<?> postgreSQLContainerNoSSL = new PostgreSQLContainer<>("postgres:13-alpine");
30+
private final PostgreSQLContainer<?> postgreSQLContainerWithSSL =
31+
new PostgreSQLContainer<>(DockerImageName.parse("marcosmarxm/postgres-ssl:dev").asCompatibleSubstituteFor("postgres"))
32+
.withCommand("postgres -c ssl=on -c ssl_cert_file=/var/lib/postgresql/server.crt -c ssl_key_file=/var/lib/postgresql/server.key");
33+
private static final List<String> NON_STRICT_SSL_MODES = List.of("disable", "allow", "prefer");
34+
private static final String SSL_MODE_REQUIRE = "require";
35+
2836
private static final SshBastionContainer bastion = new SshBastionContainer();
2937
private static final Network network = Network.newNetwork();
3038

3139
@Test
32-
void testCheckWithSSlModeDisable() throws Exception {
40+
void testSSlModesDisableAllowPreferWithTunnelIfServerDoesNotSupportSSL() throws Exception {
41+
42+
try (PostgreSQLContainer<?> db = postgreSQLContainerNoSSL.withNetwork(network)) {
43+
bastion.initAndStartBastion(network);
44+
db.start();
45+
46+
for (String sslmode : NON_STRICT_SSL_MODES) {
47+
final AirbyteConnectionStatus connectionStatus = checkWithTunnel(db, sslmode);
48+
assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatus.getStatus());
49+
}
50+
51+
} finally {
52+
bastion.stopAndClose();
53+
}
54+
}
55+
56+
@Test
57+
void testSSlModesDisableAllowPreferWithTunnelIfServerSupportSSL() throws Exception {
58+
try (PostgreSQLContainer<?> db = postgreSQLContainerWithSSL.withNetwork(network)) {
59+
60+
bastion.initAndStartBastion(network);
61+
db.start();
62+
for (String sslmode : NON_STRICT_SSL_MODES) {
63+
64+
final AirbyteConnectionStatus connectionStatus = checkWithTunnel(db, sslmode);
65+
assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatus.getStatus());
66+
}
67+
} finally {
68+
bastion.stopAndClose();
69+
}
70+
}
71+
72+
@Test
73+
void testSSlModesDisableAllowPreferWithFailedTunnelIfServerSupportSSL() throws Exception {
74+
try (PostgreSQLContainer<?> db = postgreSQLContainerWithSSL) {
3375

34-
try (PostgreSQLContainer<?> db = new PostgreSQLContainer<>("postgres:13-alpine").withNetwork(network)) {
3576
bastion.initAndStartBastion(network);
3677
db.start();
78+
for (String sslmode : NON_STRICT_SSL_MODES) {
3779

38-
// stop to enforce ssl for ssl_mode disable
39-
final ImmutableMap.Builder<Object, Object> builderWithSSLModeDisable = getDatabaseConfigBuilderWithSSLMode(db, "disable");
40-
final JsonNode configWithSSLModeDisable = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, builderWithSSLModeDisable);
41-
final AirbyteConnectionStatus connectionStatusForDisabledMode = new PostgresSourceStrictEncrypt().check(configWithSSLModeDisable);
42-
assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatusForDisabledMode.getStatus());
80+
final AirbyteConnectionStatus connectionStatus = checkWithTunnel(db, sslmode);
81+
assertEquals(AirbyteConnectionStatus.Status.FAILED, connectionStatus.getStatus());
82+
assertTrue(connectionStatus.getMessage().contains("Connection is not available"));
4383

84+
}
4485
} finally {
4586
bastion.stopAndClose();
4687
}
4788
}
4889

4990
@Test
50-
void testCheckWithSSlModePrefer() throws Exception {
91+
void testSSlRequiredWithTunnelIfServerDoesNotSupportSSL() throws Exception {
5192

52-
try (PostgreSQLContainer<?> db = new PostgreSQLContainer<>("postgres:13-alpine").withNetwork(network)) {
93+
try (PostgreSQLContainer<?> db = postgreSQLContainerNoSSL.withNetwork(network)) {
5394
bastion.initAndStartBastion(network);
5495
db.start();
55-
// continue to enforce ssl because ssl mode is prefer
56-
final ImmutableMap.Builder<Object, Object> builderWithSSLModePrefer = getDatabaseConfigBuilderWithSSLMode(db, "prefer");
57-
final JsonNode configWithSSLModePrefer = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, builderWithSSLModePrefer);
58-
final AirbyteConnectionStatus connectionStatusForPreferredMode = new PostgresSourceStrictEncrypt().check(configWithSSLModePrefer);
59-
assertEquals(AirbyteConnectionStatus.Status.FAILED, connectionStatusForPreferredMode.getStatus());
60-
assertEquals("State code: 08004; Message: The server does not support SSL.", connectionStatusForPreferredMode.getMessage());
96+
final AirbyteConnectionStatus connectionStatus = checkWithTunnel(db, SSL_MODE_REQUIRE);
97+
assertEquals(AirbyteConnectionStatus.Status.FAILED, connectionStatus.getStatus());
98+
assertEquals("State code: 08004; Message: The server does not support SSL.", connectionStatus.getMessage());
6199

62100
} finally {
63101
bastion.stopAndClose();
64102
}
65103
}
66104

105+
@Test
106+
void testSSlRequiredNoTunnelIfServerSupportSSL() throws Exception {
107+
108+
try (PostgreSQLContainer<?> db = postgreSQLContainerWithSSL) {
109+
db.start();
110+
111+
final ImmutableMap<Object, Object> configBuilderWithSSLMode = getDatabaseConfigBuilderWithSSLMode(db, SSL_MODE_REQUIRE).build();
112+
final JsonNode config = Jsons.jsonNode(configBuilderWithSSLMode);
113+
addNoTunnel((ObjectNode) config);
114+
final AirbyteConnectionStatus connectionStatus = source.check(config);
115+
assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatus.getStatus());
116+
}
117+
}
118+
119+
@Test
120+
void testStrictSSLSecuredWithTunnel() throws Exception {
121+
122+
try (PostgreSQLContainer<?> db = postgreSQLContainerWithSSL.withNetwork(network)) {
123+
124+
bastion.initAndStartBastion(network);
125+
db.start();
126+
127+
final AirbyteConnectionStatus connectionStatus = checkWithTunnel(db, SSL_MODE_REQUIRE);
128+
assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatus.getStatus());
129+
} finally {
130+
bastion.stopAndClose();
131+
}
132+
}
133+
67134
private ImmutableMap.Builder<Object, Object> getDatabaseConfigBuilderWithSSLMode(PostgreSQLContainer<?> db, String sslMode) {
68135
return ImmutableMap.builder()
69136
.put(JdbcUtils.HOST_KEY, Objects.requireNonNull(db.getContainerInfo()
@@ -94,53 +161,26 @@ private JsonNode getMockedSSLConfig(String sslMode) {
94161

95162
@Test
96163
void testSslModesUnsecuredNoTunnel() throws Exception {
97-
for (String sslMode : List.of("disable", "allow", "prefer")) {
164+
for (String sslMode : NON_STRICT_SSL_MODES) {
98165
final JsonNode config = getMockedSSLConfig(sslMode);
99-
((ObjectNode) config).putIfAbsent("tunnel_method", Jsons.jsonNode(ImmutableMap.builder()
100-
.put("tunnel_method", "NO_TUNNEL")
101-
.build()));
166+
addNoTunnel((ObjectNode) config);
102167

103-
final AirbyteConnectionStatus actual = new PostgresSourceStrictEncrypt().check(config);
104-
assertEquals(AirbyteConnectionStatus.Status.FAILED, actual.getStatus());
105-
assertTrue(actual.getMessage().contains("Unsecured connection not allowed"));
168+
final AirbyteConnectionStatus connectionStatus = source.check(config);
169+
assertEquals(AirbyteConnectionStatus.Status.FAILED, connectionStatus.getStatus());
170+
assertTrue(connectionStatus.getMessage().contains("Unsecured connection not allowed"));
106171
}
107172
}
108173

109-
@Test
110-
void testSslModeRequiredNoTunnel() throws Exception {
111-
112-
try (PostgreSQLContainer<?> db =
113-
new PostgreSQLContainer<>(DockerImageName.parse("marcosmarxm/postgres-ssl:dev").asCompatibleSubstituteFor("postgres"))
114-
.withCommand("postgres -c ssl=on -c ssl_cert_file=/var/lib/postgresql/server.crt -c ssl_key_file=/var/lib/postgresql/server.key")) {
115-
db.start();
116-
117-
final ImmutableMap<Object, Object> configBuilderWithSslModeRequire = getDatabaseConfigBuilderWithSSLMode(db, "require").build();
118-
final JsonNode config = Jsons.jsonNode(configBuilderWithSslModeRequire);
119-
((ObjectNode) config).putIfAbsent("tunnel_method", Jsons.jsonNode(ImmutableMap.builder()
120-
.put("tunnel_method", "NO_TUNNEL")
121-
.build()));
122-
final AirbyteConnectionStatus connectionStatusForPreferredMode = new PostgresSourceStrictEncrypt().check(config);
123-
assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatusForPreferredMode.getStatus());
124-
}
174+
private AirbyteConnectionStatus checkWithTunnel(PostgreSQLContainer<?> db, String sslmode) throws Exception {
175+
final ImmutableMap.Builder<Object, Object> configBuilderWithSSLMode = getDatabaseConfigBuilderWithSSLMode(db, sslmode);
176+
final JsonNode configWithSSLModeDisable = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, configBuilderWithSSLMode);
177+
return source.check(configWithSSLModeDisable);
125178
}
126179

127-
@Test
128-
void testStrictSSLSecuredWithTunnel() throws Exception {
129-
try (PostgreSQLContainer<?> db =
130-
new PostgreSQLContainer<>(DockerImageName.parse("marcosmarxm/postgres-ssl:dev").asCompatibleSubstituteFor("postgres"))
131-
.withCommand("postgres -c ssl=on -c ssl_cert_file=/var/lib/postgresql/server.crt -c ssl_key_file=/var/lib/postgresql/server.key")
132-
.withNetwork(network)) {
133-
134-
bastion.initAndStartBastion(network);
135-
db.start();
136-
137-
final ImmutableMap.Builder<Object, Object> builderWithSSLModePrefer = getDatabaseConfigBuilderWithSSLMode(db, "require");
138-
final JsonNode configWithSslAndSsh = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, builderWithSSLModePrefer);
139-
final AirbyteConnectionStatus connectionStatusForPreferredMode = new PostgresSourceStrictEncrypt().check(configWithSslAndSsh);
140-
assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatusForPreferredMode.getStatus());
141-
} finally {
142-
bastion.stopAndClose();
143-
}
180+
private static void addNoTunnel(ObjectNode config) {
181+
config.putIfAbsent("tunnel_method", Jsons.jsonNode(ImmutableMap.builder()
182+
.put("tunnel_method", "NO_TUNNEL")
183+
.build()));
144184
}
145185

146186
}

‎airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java

+2-29
Original file line numberDiff line numberDiff line change
@@ -199,18 +199,6 @@ private JsonNode getConfig(final PostgreSQLContainer<?> psqlDb, final String dbN
199199
.build());
200200
}
201201

202-
private JsonNode getConfigWithSsl(final PostgreSQLContainer<?> psqlDb, final String dbName) {
203-
return Jsons.jsonNode(ImmutableMap.builder()
204-
.put("host", psqlDb.getHost())
205-
.put("port", psqlDb.getFirstMappedPort())
206-
.put("database", dbName)
207-
.put("schemas", List.of(SCHEMA_NAME))
208-
.put("username", psqlDb.getUsername())
209-
.put("password", psqlDb.getPassword())
210-
.put("ssl", true)
211-
.build());
212-
}
213-
214202
private JsonNode getConfig(final PostgreSQLContainer<?> psqlDb, final String dbName, final String user, final String password) {
215203
return Jsons.jsonNode(ImmutableMap.builder()
216204
.put(JdbcUtils.HOST_KEY, psqlDb.getHost())
@@ -481,22 +469,6 @@ void testIsCdc() {
481469
assertTrue(PostgresUtils.isCdc(config));
482470
}
483471

484-
@Test
485-
void testGetDefaultConnectionPropertiesWithoutSsl() {
486-
final JsonNode config = getConfig(PSQL_DB, dbName);
487-
final Map<String, String> defaultConnectionProperties = new PostgresSource().getDefaultConnectionProperties(config);
488-
assertEquals(defaultConnectionProperties, Collections.emptyMap());
489-
};
490-
491-
@Test
492-
void testGetDefaultConnectionPropertiesWithSsl() {
493-
final JsonNode config = getConfigWithSsl(PSQL_DB, dbName);
494-
final Map<String, String> defaultConnectionProperties = new PostgresSource().getDefaultConnectionProperties(config);
495-
assertEquals(defaultConnectionProperties, ImmutableMap.of(
496-
"ssl", "true",
497-
"sslmode", "require"));
498-
};
499-
500472
@Test
501473
void testGetUsername() {
502474
final String username = "airbyte-user";
@@ -568,7 +540,8 @@ private JsonNode buildConfigEscapingNeeded() {
568540
JdbcUtils.HOST_KEY, "localhost",
569541
JdbcUtils.PORT_KEY, 1111,
570542
JdbcUtils.USERNAME_KEY, "user",
571-
JdbcUtils.DATABASE_KEY, "db/foo"));
543+
JdbcUtils.DATABASE_KEY, "db/foo",
544+
JdbcUtils.SSL_KEY, "false"));
572545
}
573546

574547
}

‎docs/integrations/sources/postgres.md

+1
Original file line numberDiff line numberDiff line change
@@ -404,6 +404,7 @@ The root causes is that the WALs needed for the incremental sync has been remove
404404

405405
| Version | Date | Pull Request | Subject |
406406
|:--------|:-----------|:-------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
407+
| 1.0.26 | 2022-11-18 | [19551](https://github.com/airbytehq/airbyte/pull/19551) | Fixes bug with ssl modes |
407408
| 1.0.25 | 2022-11-16 | [19004](https://github.com/airbytehq/airbyte/pull/19004) | Use Debezium heartbeats to improve CDC replication of large databases. |
408409
| 1.0.24 | 2022-11-07 | [19291](https://github.com/airbytehq/airbyte/pull/19291) | Default timeout is reduced from 1 min to 10sec |
409410
| 1.0.23 | 2022-11-07 | [19025](https://github.com/airbytehq/airbyte/pull/19025) | Stop enforce SSL if ssl mode is disabled |

0 commit comments

Comments
 (0)