Skip to content

🐛 Fix Oracle spec to declare sid instead of database param, Redshift to allow additionalProperties, MSSQL test and spec to declare spec type correctly #4874

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Jul 21, 2021
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"destinationDefinitionId": "3986776d-2319-4de9-8af8-db14c0996e72",
"name": "Oracle (Alpha)",
"dockerRepository": "airbyte/destination-oracle",
"dockerImageTag": "0.1.1",
"dockerImageTag": "0.1.2",
"documentationUrl": "https://docs.airbyte.io/integrations/destinations/oracle"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"destinationDefinitionId": "d4353156-9217-4cad-8dd7-c108fd4f74cf",
"name": "MS SQL Server",
"dockerRepository": "airbyte/destination-mssql",
"dockerImageTag": "0.1.4",
"dockerImageTag": "0.1.5",
"documentationUrl": "https://docs.airbyte.io/integrations/destinations/mssql"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"destinationDefinitionId": "f7a7d195-377f-cf5b-70a5-be6b819019dc",
"name": "Redshift",
"dockerRepository": "airbyte/destination-redshift",
"dockerImageTag": "0.3.10",
"dockerImageTag": "0.3.11",
"documentationUrl": "https://docs.airbyte.io/integrations/destinations/redshift",
"icon": "redshift.svg"
}
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@
- destinationDefinitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc
name: Redshift
dockerRepository: airbyte/destination-redshift
dockerImageTag: 0.3.10
dockerImageTag: 0.3.11
documentationUrl: https://docs.airbyte.io/integrations/destinations/redshift
icon: redshift.svg
- destinationDefinitionId: af7c921e-5892-4ff2-b6c1-4a5ab258fb7e
Expand All @@ -63,10 +63,10 @@
- destinationDefinitionId: d4353156-9217-4cad-8dd7-c108fd4f74cf
name: MS SQL Server
dockerRepository: airbyte/destination-mssql
dockerImageTag: 0.1.4
dockerImageTag: 0.1.5
documentationUrl: https://docs.airbyte.io/integrations/destinations/mssql
- destinationDefinitionId: 3986776d-2319-4de9-8af8-db14c0996e72
name: Oracle (Alpha)
dockerRepository: airbyte/destination-oracle
dockerImageTag: 0.1.1
dockerImageTag: 0.1.2
documentationUrl: https://docs.airbyte.io/integrations/destinations/oracle
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,18 @@
import io.airbyte.commons.io.IOs;
import io.airbyte.commons.json.Jsons;
import io.airbyte.commons.util.AutoCloseableIterator;
import io.airbyte.protocol.models.AirbyteConnectionStatus;
import io.airbyte.protocol.models.AirbyteMessage;
import io.airbyte.protocol.models.AirbyteMessage.Type;
import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
import io.airbyte.validation.json.JsonSchemaValidator;

import java.nio.file.Path;
import java.util.Optional;
import java.util.Scanner;
import java.util.Set;
import java.util.function.Consumer;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -66,11 +69,10 @@ public IntegrationRunner(Source source) {
this(new IntegrationCliParser(), Destination::defaultOutputRecordCollector, null, source);
}

@VisibleForTesting
IntegrationRunner(IntegrationCliParser cliParser,
Consumer<AirbyteMessage> outputRecordCollector,
Destination destination,
Source source) {
@VisibleForTesting IntegrationRunner(IntegrationCliParser cliParser,
Consumer<AirbyteMessage> outputRecordCollector,
Destination destination,
Source source) {
Preconditions.checkState(destination != null ^ source != null, "can only pass in a destination or a source");
this.cliParser = cliParser;
this.outputRecordCollector = outputRecordCollector;
Expand All @@ -81,12 +83,11 @@ public IntegrationRunner(Source source) {
validator = new JsonSchemaValidator();
}

@VisibleForTesting
IntegrationRunner(IntegrationCliParser cliParser,
Consumer<AirbyteMessage> outputRecordCollector,
Destination destination,
Source source,
JsonSchemaValidator jsonSchemaValidator) {
@VisibleForTesting IntegrationRunner(IntegrationCliParser cliParser,
Consumer<AirbyteMessage> outputRecordCollector,
Destination destination,
Source source,
JsonSchemaValidator jsonSchemaValidator) {
this(cliParser, outputRecordCollector, destination, source);
this.validator = jsonSchemaValidator;
}
Expand All @@ -104,7 +105,23 @@ public void run(String[] args) throws Exception {
case SPEC -> outputRecordCollector.accept(new AirbyteMessage().withType(Type.SPEC).withSpec(integration.spec()));
case CHECK -> {
final JsonNode config = parseConfig(parsed.getConfigPath());
validateConfig(integration.spec().getConnectionSpecification(), config, "CHECK");
try {
validateConfig(integration.spec().getConnectionSpecification(), config, "CHECK");
} catch (Exception e) {
// if validation fails don't throw an exception, return a failed connection check message
outputRecordCollector
.accept(
new AirbyteMessage()
.withType(Type.CONNECTION_STATUS)
.withConnectionStatus(
new AirbyteConnectionStatus()
.withStatus(AirbyteConnectionStatus.Status.FAILED)
.withMessage(e.getMessage()
)
)
);
}

outputRecordCollector.accept(new AirbyteMessage().withType(Type.CONNECTION_STATUS).withConnectionStatus(integration.check(config)));
}
// source only
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar

RUN tar xf ${APPLICATION}.tar --strip-components=1

LABEL io.airbyte.version=0.1.4
LABEL io.airbyte.version=0.1.5
LABEL io.airbyte.name=airbyte/destination-mssql
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@
"additionalProperties": false,
"description": "Data transfer will not be encrypted.",
"required": ["ssl_method"],
"type": "object",
"properties": {
"ssl_method": {
"type": "string",
Expand All @@ -78,6 +79,7 @@
"additionalProperties": false,
"description": "Use the cert provided by the server without verification. (For testing purposes only!)",
"required": ["ssl_method"],
"type": "object",
"properties": {
"ssl_method": {
"type": "string",
Expand All @@ -91,6 +93,7 @@
"additionalProperties": false,
"description": "Verify and use the cert provided by the server.",
"required": ["ssl_method", "trustStoreName", "trustStorePassword"],
"type": "object",
"properties": {
"ssl_method": {
"type": "string",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import io.airbyte.commons.json.Jsons;
import io.airbyte.commons.string.Strings;
import io.airbyte.db.Database;
Expand Down Expand Up @@ -65,13 +66,14 @@ protected boolean supportsDBT() {
}

private JsonNode getConfig(MSSQLServerContainer<?> db) {

return Jsons.jsonNode(ImmutableMap.builder()
.put("host", db.getHost())
.put("port", db.getFirstMappedPort())
.put("username", db.getUsername())
.put("password", db.getPassword())
.put("schema", "testSchema")
.put("ssl_method", "encrypted_trust_server_certificate")
.put("ssl_method", Jsons.jsonNode(ImmutableMap.of("method", "encrypted_trust_server_certificate")))
.build());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar

RUN tar xf ${APPLICATION}.tar --strip-components=1

LABEL io.airbyte.version=0.1.1
LABEL io.airbyte.version=0.1.2
LABEL io.airbyte.name=airbyte/destination-oracle
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Oracle Destination Spec",
"type": "object",
"required": ["host", "port", "username", "database"],
"required": ["host", "port", "username", "sid"],
"additionalProperties": false,
"properties": {
"host": {
Expand All @@ -27,9 +27,9 @@
"examples": ["1521"],
"order": 1
},
"database": {
"title": "DB Name",
"description": "Name of the database.",
"sid": {
"title": "SID",
"description": "SID",
"type": "string",
"order": 2
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar

RUN tar xf ${APPLICATION}.tar --strip-components=1

LABEL io.airbyte.version=0.3.10
LABEL io.airbyte.version=0.3.11
LABEL io.airbyte.name=airbyte/destination-redshift
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
"title": "Redshift Destination Spec",
"type": "object",
"required": ["host", "port", "database", "username", "password", "schema"],
"additionalProperties": false,
"additionalProperties": true,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should we create an issue to do this for all specs?

"properties": {
"host": {
"description": "Host Endpoint of the Redshift Cluster (must include the cluster-id, region and end with .redshift.amazonaws.com)",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
"username",
"password"
],

"additionalProperties": false,
"properties": {
"host": {
Expand Down