Skip to content

Commit 804af5d

Browse files
ryankfuoctavia-squidington-iii
ryankfu
and
octavia-squidington-iii
authored
Adds TLS check to mongodb destination and migrates util constants (#18892)
* Adds TLS check to mongodb destination and migrates util constants * Migrates MongodbSourceUitls to general purprose Utils file * Updates expected_spec.json to include SSH tunnel * Bumps connector version and removes connector from being hidden in UI * auto-bump connector version Co-authored-by: Octavia Squidington III <[email protected]>
1 parent 5c9e5d9 commit 804af5d

File tree

15 files changed

+249
-111
lines changed

15 files changed

+249
-111
lines changed

‎airbyte-config/init/src/main/resources/seed/destination_definitions.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,7 @@
206206
- name: MongoDB
207207
destinationDefinitionId: 8b746512-8c2e-6ac1-4adc-b59faafd473c
208208
dockerRepository: airbyte/destination-mongodb
209-
dockerImageTag: 0.1.8
209+
dockerImageTag: 0.1.9
210210
documentationUrl: https://docs.airbyte.com/integrations/destinations/mongodb
211211
icon: mongodb.svg
212212
releaseStage: alpha

‎airbyte-config/init/src/main/resources/seed/destination_specs.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3338,7 +3338,7 @@
33383338
supported_destination_sync_modes:
33393339
- "overwrite"
33403340
- "append"
3341-
- dockerImage: "airbyte/destination-mongodb:0.1.8"
3341+
- dockerImage: "airbyte/destination-mongodb:0.1.9"
33423342
spec:
33433343
documentationUrl: "https://docs.airbyte.com/integrations/destinations/mongodb"
33443344
connectionSpecification:

‎airbyte-db/db-lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,15 @@
66

77
import static java.util.Arrays.asList;
88
import static org.bson.BsonType.ARRAY;
9+
import static org.bson.BsonType.DATE_TIME;
10+
import static org.bson.BsonType.DECIMAL128;
911
import static org.bson.BsonType.DOCUMENT;
12+
import static org.bson.BsonType.DOUBLE;
13+
import static org.bson.BsonType.INT32;
14+
import static org.bson.BsonType.INT64;
15+
import static org.bson.BsonType.OBJECT_ID;
16+
import static org.bson.BsonType.STRING;
17+
import static org.bson.BsonType.TIMESTAMP;
1018
import static org.bson.codecs.configuration.CodecRegistries.fromProviders;
1119

1220
import com.fasterxml.jackson.databind.JsonNode;
@@ -20,6 +28,7 @@
2028
import io.airbyte.commons.json.Jsons;
2129
import io.airbyte.commons.util.MoreIterators;
2230
import io.airbyte.db.DataTypeUtils;
31+
import io.airbyte.db.jdbc.JdbcUtils;
2332
import io.airbyte.protocol.models.CommonField;
2433
import io.airbyte.protocol.models.JsonSchemaType;
2534
import io.airbyte.protocol.models.TreeNode;
@@ -28,6 +37,7 @@
2837
import java.util.Collections;
2938
import java.util.HashSet;
3039
import java.util.List;
40+
import java.util.Set;
3141
import org.bson.BsonBinary;
3242
import org.bson.BsonDateTime;
3343
import org.bson.BsonDocument;
@@ -53,6 +63,29 @@ public class MongoUtils {
5363

5464
private static final Logger LOGGER = LoggerFactory.getLogger(MongoUtils.class);
5565

66+
// Shared constants
67+
public static final String MONGODB_SERVER_URL = "mongodb://%s%s:%s/%s?authSource=admin&ssl=%s";
68+
public static final String MONGODB_CLUSTER_URL = "mongodb+srv://%s%s/%s?retryWrites=true&w=majority&tls=true";
69+
public static final String MONGODB_REPLICA_URL = "mongodb://%s%s/%s?authSource=admin&directConnection=false&ssl=true";
70+
public static final String USER = "user";
71+
public static final String INSTANCE_TYPE = "instance_type";
72+
public static final String INSTANCE = "instance";
73+
public static final String CLUSTER_URL = "cluster_url";
74+
public static final String SERVER_ADDRESSES = "server_addresses";
75+
public static final String REPLICA_SET = "replica_set";
76+
77+
// MongodbDestination specific constants
78+
public static final String AUTH_TYPE = "auth_type";
79+
public static final String AUTHORIZATION = "authorization";
80+
public static final String LOGIN_AND_PASSWORD = "login/password";
81+
public static final String AIRBYTE_DATA_HASH = "_airbyte_data_hash";
82+
83+
// MongodbSource specific constants
84+
public static final String AUTH_SOURCE = "auth_source";
85+
public static final String PRIMARY_KEY = "_id";
86+
public static final Set<BsonType> ALLOWED_CURSOR_TYPES = Set.of(DOUBLE, STRING, DOCUMENT, OBJECT_ID, DATE_TIME,
87+
INT32, TIMESTAMP, INT64, DECIMAL128);
88+
5689
private static final String MISSING_TYPE = "missing";
5790
private static final String NULL_TYPE = "null";
5891
public static final String AIRBYTE_SUFFIX = "_aibyte_transform";
@@ -136,6 +169,14 @@ private static ObjectNode readDocument(final BsonReader reader, final ObjectNode
136169
return jsonNodes;
137170
}
138171

172+
/**
173+
* Determines whether TLS/SSL should be enabled for a standalone instance of MongoDB.
174+
*/
175+
public static boolean tlsEnabledForStandaloneInstance(final JsonNode config, final JsonNode instanceConfig) {
176+
return config.has(JdbcUtils.TLS_KEY) ? config.get(JdbcUtils.TLS_KEY).asBoolean()
177+
: (instanceConfig.has(JdbcUtils.TLS_KEY) ? instanceConfig.get(JdbcUtils.TLS_KEY).asBoolean() : true);
178+
}
179+
139180
public static void transformToStringIfMarked(final ObjectNode jsonNodes, final List<String> columnNames, final String fieldName) {
140181
if (columnNames.contains(fieldName + AIRBYTE_SUFFIX)) {
141182
final JsonNode data = jsonNodes.get(fieldName);

‎airbyte-integrations/connectors/destination-mongodb-strict-encrypt/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,5 @@ ENV APPLICATION destination-mongodb-strict-encrypt
1616

1717
COPY --from=build /airbyte /airbyte
1818

19-
LABEL io.airbyte.version=0.1.8
19+
LABEL io.airbyte.version=0.1.9
2020
LABEL io.airbyte.name=airbyte/destination-mongodb-strict-encrypt

‎airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.destination.mongodb/MongodbDestinationStrictEncrypt.java

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,16 @@
44

55
package io.airbyte.integrations.destination.mongodb;
66

7+
import com.fasterxml.jackson.databind.JsonNode;
78
import com.fasterxml.jackson.databind.node.ObjectNode;
9+
import io.airbyte.commons.exceptions.ConfigErrorException;
810
import io.airbyte.commons.json.Jsons;
11+
import io.airbyte.db.mongodb.MongoUtils;
12+
import io.airbyte.db.mongodb.MongoUtils.MongoInstanceType;
913
import io.airbyte.integrations.base.Destination;
1014
import io.airbyte.integrations.base.IntegrationRunner;
1115
import io.airbyte.integrations.base.spec_modification.SpecModifyingDestination;
16+
import io.airbyte.protocol.models.AirbyteConnectionStatus;
1217
import io.airbyte.protocol.models.ConnectorSpecification;
1318
import org.slf4j.Logger;
1419
import org.slf4j.LoggerFactory;
@@ -21,6 +26,17 @@ public MongodbDestinationStrictEncrypt() {
2126
super(MongodbDestination.sshWrappedDestination());
2227
}
2328

29+
@Override
30+
public AirbyteConnectionStatus check(final JsonNode config) throws Exception {
31+
final JsonNode instanceConfig = config.get(MongoUtils.INSTANCE_TYPE);
32+
final MongoInstanceType instance = MongoInstanceType.fromValue(instanceConfig.get(MongoUtils.INSTANCE).asText());
33+
// If the MongoDb destination connector is not set up to use a TLS connection, then check should fail
34+
if (instance.equals(MongoInstanceType.STANDALONE) && !MongoUtils.tlsEnabledForStandaloneInstance(config, instanceConfig)) {
35+
throw new ConfigErrorException("TLS connection must be used to read from MongoDB.");
36+
}
37+
return super.check(config);
38+
}
39+
2440
@Override
2541
public ConnectorSpecification modifySpec(final ConnectorSpecification originalSpec) throws Exception {
2642
final ConnectorSpecification spec = Jsons.clone(originalSpec);

‎airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationStrictEncryptAcceptanceTest.java

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,19 @@
55
package io.airbyte.integrations.destination.mongodb;
66

77
import static com.mongodb.client.model.Projections.excludeId;
8+
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
9+
import static org.assertj.core.api.AssertionsForClassTypes.catchThrowable;
10+
import static org.junit.jupiter.api.Assertions.assertEquals;
811

912
import com.fasterxml.jackson.databind.JsonNode;
13+
import com.fasterxml.jackson.databind.node.ObjectNode;
1014
import com.google.common.collect.ImmutableMap;
1115
import com.mongodb.client.MongoCursor;
16+
import io.airbyte.commons.exceptions.ConfigErrorException;
1217
import io.airbyte.commons.json.Jsons;
1318
import io.airbyte.db.jdbc.JdbcUtils;
1419
import io.airbyte.db.mongodb.MongoDatabase;
20+
import io.airbyte.db.mongodb.MongoUtils;
1521
import io.airbyte.db.mongodb.MongoUtils.MongoInstanceType;
1622
import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest;
1723
import java.io.IOException;
@@ -21,6 +27,7 @@
2127
import java.util.List;
2228
import org.bson.Document;
2329
import org.junit.jupiter.api.BeforeAll;
30+
import org.junit.jupiter.api.Test;
2431

2532
public class MongodbDestinationStrictEncryptAcceptanceTest extends DestinationAcceptanceTest {
2633

@@ -102,9 +109,27 @@ protected List<JsonNode> retrieveRecords(final TestDestinationEnv testEnv,
102109
return result;
103110
}
104111

112+
@Test
113+
void testCheck() throws Exception {
114+
final JsonNode instanceConfig = Jsons.jsonNode(ImmutableMap.builder()
115+
.put("instance", MongoInstanceType.STANDALONE.getType())
116+
.put("tls", false)
117+
.build());
118+
119+
final JsonNode invalidStandaloneConfig = getConfig();
120+
121+
((ObjectNode) invalidStandaloneConfig).put(MongoUtils.INSTANCE_TYPE, instanceConfig);
122+
123+
final Throwable throwable = catchThrowable(() -> new MongodbDestinationStrictEncrypt().check(invalidStandaloneConfig));
124+
assertThat(throwable).isInstanceOf(ConfigErrorException.class);
125+
assertThat(((ConfigErrorException) throwable)
126+
.getDisplayMessage()
127+
.contains("TLS connection must be used to read from MongoDB."));
128+
}
129+
105130
@Override
106131
protected void setup(final TestDestinationEnv testEnv) {
107-
var credentials = String.format("%s:%s@", config.get(AUTH_TYPE).get(JdbcUtils.USERNAME_KEY).asText(),
132+
final var credentials = String.format("%s:%s@", config.get(AUTH_TYPE).get(JdbcUtils.USERNAME_KEY).asText(),
108133
config.get(AUTH_TYPE).get(JdbcUtils.PASSWORD_KEY).asText());
109134
final String connectionString = String.format("mongodb+srv://%s%s/%s?retryWrites=true&w=majority&tls=true",
110135
credentials,

‎airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/test/resources/expected_spec.json

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,120 @@
135135
}
136136
}
137137
]
138+
},
139+
"tunnel_method": {
140+
"type": "object",
141+
"title": "SSH Tunnel Method",
142+
"description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.",
143+
"oneOf": [
144+
{
145+
"title": "No Tunnel",
146+
"required": ["tunnel_method"],
147+
"properties": {
148+
"tunnel_method": {
149+
"description": "No ssh tunnel needed to connect to database",
150+
"type": "string",
151+
"const": "NO_TUNNEL",
152+
"order": 0
153+
}
154+
}
155+
},
156+
{
157+
"title": "SSH Key Authentication",
158+
"required": [
159+
"tunnel_method",
160+
"tunnel_host",
161+
"tunnel_port",
162+
"tunnel_user",
163+
"ssh_key"
164+
],
165+
"properties": {
166+
"tunnel_method": {
167+
"description": "Connect through a jump server tunnel host using username and ssh key",
168+
"type": "string",
169+
"const": "SSH_KEY_AUTH",
170+
"order": 0
171+
},
172+
"tunnel_host": {
173+
"title": "SSH Tunnel Jump Server Host",
174+
"description": "Hostname of the jump server host that allows inbound ssh tunnel.",
175+
"type": "string",
176+
"order": 1
177+
},
178+
"tunnel_port": {
179+
"title": "SSH Connection Port",
180+
"description": "Port on the proxy/jump server that accepts inbound ssh connections.",
181+
"type": "integer",
182+
"minimum": 0,
183+
"maximum": 65536,
184+
"default": 22,
185+
"examples": ["22"],
186+
"order": 2
187+
},
188+
"tunnel_user": {
189+
"title": "SSH Login Username",
190+
"description": "OS-level username for logging into the jump server host.",
191+
"type": "string",
192+
"order": 3
193+
},
194+
"ssh_key": {
195+
"title": "SSH Private Key",
196+
"description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )",
197+
"type": "string",
198+
"airbyte_secret": true,
199+
"multiline": true,
200+
"order": 4
201+
}
202+
}
203+
},
204+
{
205+
"title": "Password Authentication",
206+
"required": [
207+
"tunnel_method",
208+
"tunnel_host",
209+
"tunnel_port",
210+
"tunnel_user",
211+
"tunnel_user_password"
212+
],
213+
"properties": {
214+
"tunnel_method": {
215+
"description": "Connect through a jump server tunnel host using username and password authentication",
216+
"type": "string",
217+
"const": "SSH_PASSWORD_AUTH",
218+
"order": 0
219+
},
220+
"tunnel_host": {
221+
"title": "SSH Tunnel Jump Server Host",
222+
"description": "Hostname of the jump server host that allows inbound ssh tunnel.",
223+
"type": "string",
224+
"order": 1
225+
},
226+
"tunnel_port": {
227+
"title": "SSH Connection Port",
228+
"description": "Port on the proxy/jump server that accepts inbound ssh connections.",
229+
"type": "integer",
230+
"minimum": 0,
231+
"maximum": 65536,
232+
"default": 22,
233+
"examples": ["22"],
234+
"order": 2
235+
},
236+
"tunnel_user": {
237+
"title": "SSH Login Username",
238+
"description": "OS-level username for logging into the jump server host",
239+
"type": "string",
240+
"order": 3
241+
},
242+
"tunnel_user_password": {
243+
"title": "Password",
244+
"description": "OS-level password for logging into the jump server host",
245+
"type": "string",
246+
"airbyte_secret": true,
247+
"order": 4
248+
}
249+
}
250+
}
251+
]
138252
}
139253
}
140254
}

‎airbyte-integrations/connectors/destination-mongodb/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,5 @@ ENV APPLICATION destination-mongodb
1616

1717
COPY --from=build /airbyte /airbyte
1818

19-
LABEL io.airbyte.version=0.1.8
19+
LABEL io.airbyte.version=0.1.9
2020
LABEL io.airbyte.name=airbyte/destination-mongodb

0 commit comments

Comments
 (0)