Skip to content

Commit 265986b

Browse files
irynakrukirynakruk
authored
Create a secure-only MongoDb destination (#6945)
* Added mongodb destination strict encrypt
1 parent 059645f commit 265986b

File tree

16 files changed

+427
-22
lines changed

16 files changed

+427
-22
lines changed

‎airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/8b746512-8c2e-6ac1-4adc-b59faafd473c.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,6 @@
22
"destinationDefinitionId": "8b746512-8c2e-6ac1-4adc-b59faafd473c",
33
"name": "MongoDB",
44
"dockerRepository": "airbyte/destination-mongodb",
5-
"dockerImageTag": "0.1.1",
5+
"dockerImageTag": "0.1.2",
66
"documentationUrl": "https://docs.airbyte.io/integrations/destinations/mongodb"
77
}

‎airbyte-config/init/src/main/resources/seed/destination_definitions.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -93,5 +93,5 @@
9393
- destinationDefinitionId: 8b746512-8c2e-6ac1-4adc-b59faafd473c
9494
name: MongoDB
9595
dockerRepository: airbyte/destination-mongodb
96-
dockerImageTag: 0.1.1
96+
dockerImageTag: 0.1.2
9797
documentationUrl: https://docs.airbyte.io/integrations/destinations/mongodb

‎airbyte-integrations/connectors/destination-mongodb-strict-encrypt/.dockerignore

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
*
2+
!Dockerfile
3+
!build

‎airbyte-integrations/connectors/destination-mongodb-strict-encrypt/Dockerfile

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
FROM airbyte/integration-base-java:dev
2+
3+
WORKDIR /airbyte
4+
ENV APPLICATION destination-mongodb-strict-encrypt
5+
6+
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
7+
8+
RUN tar xf ${APPLICATION}.tar --strip-components=1
9+
10+
LABEL io.airbyte.version=0.1.0
11+
LABEL io.airbyte.name=airbyte/destination-mongodb-strict-encrypt

‎airbyte-integrations/connectors/destination-mongodb-strict-encrypt/README.md

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# MongoDB Test Configuration
2+
3+
In order to test the MongoDB secure only destination, you need a service account key file.
4+
5+
## Community Contributor
6+
7+
As a community contributor, you will need access to a MongoDB to run tests.
8+
9+
1. Create a new account or log into an already created account for mongodb
10+
2. Go to the `Database Access` page and add new database user with read and write permissions
11+
3. Add new database with default collection
12+
4. Add host, port or cluster_url, database name, username and password to `secrets/credentials.json` file
13+
```
14+
{
15+
"database": "database_name",
16+
"user": "user",
17+
"password": "password",
18+
"cluster_url": "cluster_url",
19+
"host": "host",
20+
"port": "port"
21+
}
22+
```
23+
24+
## Airbyte Employee
25+
26+
1. Access the `MONGODB_TEST_CREDS` secret on the LastPass
27+
1. Create a file with the contents at `secrets/credentials.json`

‎airbyte-integrations/connectors/destination-mongodb-strict-encrypt/build.gradle

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
plugins {
2+
id 'application'
3+
id 'airbyte-docker'
4+
id 'airbyte-integration-test-java'
5+
}
6+
7+
application {
8+
mainClass = 'io.airbyte.integrations.destination.mongodb.MongodbDestinationStrictEncrypt'
9+
applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0']
10+
}
11+
12+
dependencies {
13+
implementation project(':airbyte-db:lib')
14+
implementation project(':airbyte-config:models')
15+
implementation project(':airbyte-integrations:bases:base-java')
16+
implementation project(':airbyte-protocol:models')
17+
18+
implementation project(':airbyte-integrations:connectors:destination-mongodb')
19+
implementation 'org.mongodb:mongodb-driver-sync:4.3.0'
20+
21+
testImplementation 'org.testcontainers:mongodb:1.15.3'
22+
23+
integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mongodb-strict-encrypt')
24+
integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test')
25+
26+
implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
27+
}

‎airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.destination.mongodb/MongodbDestinationStrictEncrypt.java

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/*
2+
* Copyright (c) 2021 Airbyte, Inc., all rights reserved.
3+
*/
4+
5+
package io.airbyte.integrations.destination.mongodb;
6+
7+
import com.fasterxml.jackson.databind.node.ObjectNode;
8+
import io.airbyte.commons.json.Jsons;
9+
import io.airbyte.integrations.base.Destination;
10+
import io.airbyte.integrations.base.IntegrationRunner;
11+
import io.airbyte.integrations.base.spec_modification.SpecModifyingDestination;
12+
import io.airbyte.protocol.models.ConnectorSpecification;
13+
import org.slf4j.Logger;
14+
import org.slf4j.LoggerFactory;
15+
16+
public class MongodbDestinationStrictEncrypt extends SpecModifyingDestination implements Destination {
17+
18+
private static final Logger LOGGER = LoggerFactory.getLogger(MongodbDestinationStrictEncrypt.class);
19+
20+
public MongodbDestinationStrictEncrypt() {
21+
super(new MongodbDestination());
22+
}
23+
24+
@Override
25+
public ConnectorSpecification modifySpec(ConnectorSpecification originalSpec) throws Exception {
26+
final ConnectorSpecification spec = Jsons.clone(originalSpec);
27+
// removing tls property for a standalone instance to disable possibility to switch off a tls connection
28+
((ObjectNode) spec.getConnectionSpecification().get("properties").get("instance_type").get("oneOf").get(0).get("properties")).remove("tls");
29+
return spec;
30+
}
31+
32+
public static void main(String[] args) throws Exception {
33+
final Destination destination = new MongodbDestinationStrictEncrypt();
34+
LOGGER.info("starting destination: {}", MongodbDestinationStrictEncrypt.class);
35+
new IntegrationRunner(destination).run(args);
36+
LOGGER.info("completed destination: {}", MongodbDestinationStrictEncrypt.class);
37+
}
38+
39+
}

‎airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationStrictEncryptAcceptanceTest.java

+123
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
/*
2+
* Copyright (c) 2021 Airbyte, Inc., all rights reserved.
3+
*/
4+
5+
package io.airbyte.integrations.destination.mongodb;
6+
7+
import static com.mongodb.client.model.Projections.excludeId;
8+
9+
import com.fasterxml.jackson.databind.JsonNode;
10+
import com.google.common.collect.ImmutableMap;
11+
import com.mongodb.client.MongoCursor;
12+
import io.airbyte.commons.json.Jsons;
13+
import io.airbyte.db.mongodb.MongoDatabase;
14+
import io.airbyte.db.mongodb.MongoUtils.MongoInstanceType;
15+
import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest;
16+
import java.io.IOException;
17+
import java.nio.file.Files;
18+
import java.nio.file.Path;
19+
import java.util.ArrayList;
20+
import java.util.List;
21+
import org.bson.Document;
22+
import org.junit.jupiter.api.BeforeAll;
23+
24+
public class MongodbDestinationStrictEncryptAcceptanceTest extends DestinationAcceptanceTest {
25+
26+
private static final Path CREDENTIALS_PATH = Path.of("secrets/credentials.json");
27+
28+
private static final String DATABASE = "database";
29+
private static final String AUTH_TYPE = "auth_type";
30+
private static final String INSTANCE_TYPE = "instance_type";
31+
private static final String AIRBYTE_DATA = "_airbyte_data";
32+
33+
private static JsonNode config;
34+
private static JsonNode failCheckConfig;
35+
36+
private MongoDatabase mongoDatabase;
37+
private MongodbNameTransformer namingResolver = new MongodbNameTransformer();
38+
39+
@BeforeAll
40+
static void setupConfig() throws IOException {
41+
if (!Files.exists(CREDENTIALS_PATH)) {
42+
throw new IllegalStateException(
43+
"Must provide path to a MongoDB credentials file. By default {module-root}/" + CREDENTIALS_PATH
44+
+ ". Override by setting setting path with the CREDENTIALS_PATH constant.");
45+
}
46+
final String credentialsJsonString = new String(Files.readAllBytes(CREDENTIALS_PATH));
47+
final JsonNode credentialsJson = Jsons.deserialize(credentialsJsonString);
48+
49+
final JsonNode instanceConfig = Jsons.jsonNode(ImmutableMap.builder()
50+
.put("instance", MongoInstanceType.STANDALONE.getType())
51+
.put("host", credentialsJson.get("host").asText())
52+
.put("port", credentialsJson.get("port").asInt())
53+
.build());
54+
55+
final JsonNode authConfig = Jsons.jsonNode(ImmutableMap.builder()
56+
.put("authorization", "login/password")
57+
.put("username", credentialsJson.get("user").asText())
58+
.put("password", credentialsJson.get("password").asText())
59+
.build());
60+
61+
config = Jsons.jsonNode(ImmutableMap.builder()
62+
.put(DATABASE, credentialsJson.get(DATABASE).asText())
63+
.put(AUTH_TYPE, authConfig)
64+
.put(INSTANCE_TYPE, instanceConfig)
65+
.build());
66+
67+
failCheckConfig = Jsons.jsonNode(ImmutableMap.builder()
68+
.put(DATABASE, credentialsJson.get(DATABASE).asText())
69+
.put(AUTH_TYPE, Jsons.jsonNode(ImmutableMap.builder()
70+
.put("authorization", "none")
71+
.build()))
72+
.put(INSTANCE_TYPE, instanceConfig)
73+
.build());
74+
}
75+
76+
@Override
77+
protected String getImageName() {
78+
return "airbyte/destination-mongodb-strict-encrypt:dev";
79+
}
80+
81+
@Override
82+
protected JsonNode getConfig() {
83+
return Jsons.clone(config);
84+
}
85+
86+
@Override
87+
protected JsonNode getFailCheckConfig() {
88+
return Jsons.clone(failCheckConfig);
89+
}
90+
91+
@Override
92+
protected List<JsonNode> retrieveRecords(TestDestinationEnv testEnv, String streamName, String namespace, JsonNode streamSchema) {
93+
var collection = mongoDatabase.getOrCreateNewCollection(namingResolver.getRawTableName(streamName));
94+
List<JsonNode> result = new ArrayList<>();
95+
try (MongoCursor<Document> cursor = collection.find().projection(excludeId()).iterator()) {
96+
while (cursor.hasNext()) {
97+
result.add(Jsons.jsonNode(cursor.next().get(AIRBYTE_DATA)));
98+
}
99+
}
100+
return result;
101+
}
102+
103+
@Override
104+
protected void setup(TestDestinationEnv testEnv) {
105+
String connectionString = String.format("mongodb://%s:%s@%s:%s/%s?authSource=admin&ssl=true",
106+
config.get(AUTH_TYPE).get("username").asText(),
107+
config.get(AUTH_TYPE).get("password").asText(),
108+
config.get(INSTANCE_TYPE).get("host").asText(),
109+
config.get(INSTANCE_TYPE).get("port").asText(),
110+
config.get(DATABASE).asText());
111+
112+
mongoDatabase = new MongoDatabase(connectionString, config.get(DATABASE).asText());
113+
}
114+
115+
@Override
116+
protected void tearDown(TestDestinationEnv testEnv) throws Exception {
117+
for (String collectionName : mongoDatabase.getCollectionNames()) {
118+
mongoDatabase.getDatabase().getCollection(collectionName).drop();
119+
}
120+
mongoDatabase.close();
121+
}
122+
123+
}

‎airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/test/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationStrictEncryptTest.java

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/*
2+
* Copyright (c) 2021 Airbyte, Inc., all rights reserved.
3+
*/
4+
5+
package io.airbyte.integrations.destination.mongodb;
6+
7+
import static org.junit.jupiter.api.Assertions.assertEquals;
8+
9+
import io.airbyte.commons.json.Jsons;
10+
import io.airbyte.commons.resources.MoreResources;
11+
import io.airbyte.protocol.models.ConnectorSpecification;
12+
import org.junit.jupiter.api.Test;
13+
14+
public class MongodbDestinationStrictEncryptTest {
15+
16+
@Test
17+
void testSpec() throws Exception {
18+
final ConnectorSpecification actual = new MongodbDestinationStrictEncrypt().spec();
19+
final ConnectorSpecification expected = Jsons.deserialize(MoreResources.readResource("expected_spec.json"), ConnectorSpecification.class);
20+
21+
assertEquals(expected, actual);
22+
}
23+
24+
}

0 commit comments

Comments
 (0)