diff --git a/8cee7c4a-8047-4cb9-a86d-f7fd87b2d1f8.json b/8cee7c4a-8047-4cb9-a86d-f7fd87b2d1f8.json new file mode 100644 index 0000000000000..d4aba424f81b0 --- /dev/null +++ b/8cee7c4a-8047-4cb9-a86d-f7fd87b2d1f8.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "8cee7c4a-8047-4cb9-a86d-f7fd87b2d1f8", + "name": "DataScope", + "dockerRepository": "airbyte/datascope", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://docs.airbyte.io/integrations/sources/datascope" +} diff --git a/airbyte-config/init/src/main/resources/icons/datascope.svg b/airbyte-config/init/src/main/resources/icons/datascope.svg new file mode 100644 index 0000000000000..ff4bc717a9238 --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/datascope.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/writer.py b/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/writer.py index 06847d7273821..2a5c1c7981ac4 100644 --- a/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/writer.py +++ b/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/writer.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # from collections import Mapping @@ -25,7 +25,7 @@ def __init__(self, client: KvDbClient): self.client = client def delete_stream_entries(self, stream_name: str): - """ Deletes all the records belonging to the input stream """ + """Deletes all the records belonging to the input stream""" keys_to_delete = [] for key in self.client.list_keys(prefix=f"{stream_name}__ab__"): keys_to_delete.append(key) diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/acceptance.py index 58c194c5d1376..1302b2f57e10e 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/acceptance.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # @@ -10,7 +10,7 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - """ This fixture is a placeholder for external resources that acceptance test might require.""" + """This fixture is a placeholder for external resources that acceptance test might require.""" # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/acceptance.py index 108075487440f..950b53b59d416 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/acceptance.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # @@ -10,5 +10,5 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - """ This fixture is a placeholder for external resources that acceptance test might require.""" + """This fixture is a placeholder for external resources that acceptance test might require.""" yield diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py index 64b27fe5095f5..135f4d9906d40 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # import hashlib @@ -41,7 +41,7 @@ def __init__(self, service: str, aws_access_key_id: str, aws_secret_access_key: @staticmethod def _sign_msg(key: bytes, msg: str) -> bytes: - """ Sign message using key """ + """Sign message using key""" return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest() def _get_authorization_header(self, prepared_request: requests.PreparedRequest) -> str: diff --git a/airbyte-integrations/connectors/source-amazon-sqs/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-amazon-sqs/integration_tests/acceptance.py index a1c52d74fe81e..34f2f625e15bb 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-amazon-sqs/integration_tests/acceptance.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # @@ -10,7 +10,7 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - """ This fixture is a placeholder for external resources that acceptance test might require.""" + """This fixture is a placeholder for external resources that acceptance test might require.""" # TODO: setup test dependencies yield # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-amplitude/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-amplitude/integration_tests/acceptance.py index 58c194c5d1376..480378d7bf203 100644 --- a/airbyte-integrations/connectors/source-amplitude/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-amplitude/integration_tests/acceptance.py @@ -1,16 +1,7 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -import pytest - -pytest_plugins = ("source_acceptance_test.plugin",) - - -@pytest.fixture(scope="session", autouse=True) -def connector_setup(): - """ This fixture is a placeholder for external resources that acceptance test might require.""" - # TODO: setup test dependencies if needed. otherwise remove the TODO comments - yield - # TODO: clean up test dependencies +def test_dummy_test(): + assert True diff --git a/airbyte-integrations/connectors/source-apify-dataset/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/acceptance.py index 108075487440f..950b53b59d416 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/acceptance.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # @@ -10,5 +10,5 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - """ This fixture is a placeholder for external resources that acceptance test might require.""" + """This fixture is a placeholder for external resources that acceptance test might require.""" yield diff --git a/airbyte-integrations/connectors/source-appsflyer/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-appsflyer/integration_tests/acceptance.py index 58c194c5d1376..1302b2f57e10e 100644 --- a/airbyte-integrations/connectors/source-appsflyer/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-appsflyer/integration_tests/acceptance.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # @@ -10,7 +10,7 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - """ This fixture is a placeholder for external resources that acceptance test might require.""" + """This fixture is a placeholder for external resources that acceptance test might require.""" # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-asana/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-asana/integration_tests/acceptance.py index 58c194c5d1376..950b53b59d416 100644 --- a/airbyte-integrations/connectors/source-asana/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-asana/integration_tests/acceptance.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # @@ -10,7 +10,5 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - """ This fixture is a placeholder for external resources that acceptance test might require.""" - # TODO: setup test dependencies if needed. otherwise remove the TODO comments + """This fixture is a placeholder for external resources that acceptance test might require.""" yield - # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-aws-cloudtrail/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-aws-cloudtrail/integration_tests/acceptance.py index 58c194c5d1376..1302b2f57e10e 100644 --- a/airbyte-integrations/connectors/source-aws-cloudtrail/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-aws-cloudtrail/integration_tests/acceptance.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # @@ -10,7 +10,7 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - """ This fixture is a placeholder for external resources that acceptance test might require.""" + """This fixture is a placeholder for external resources that acceptance test might require.""" # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-bamboo-hr/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-bamboo-hr/integration_tests/acceptance.py index 108075487440f..950b53b59d416 100644 --- a/airbyte-integrations/connectors/source-bamboo-hr/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-bamboo-hr/integration_tests/acceptance.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # @@ -10,5 +10,5 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - """ This fixture is a placeholder for external resources that acceptance test might require.""" + """This fixture is a placeholder for external resources that acceptance test might require.""" yield diff --git a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/acceptance.py index 58c194c5d1376..1302b2f57e10e 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/acceptance.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # @@ -10,7 +10,7 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - """ This fixture is a placeholder for external resources that acceptance test might require.""" + """This fixture is a placeholder for external resources that acceptance test might require.""" # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-bing-ads/integration_tests/acceptance.py index 108075487440f..950b53b59d416 100644 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/acceptance.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # @@ -10,5 +10,5 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - """ This fixture is a placeholder for external resources that acceptance test might require.""" + """This fixture is a placeholder for external resources that acceptance test might require.""" yield diff --git a/airbyte-integrations/connectors/source-cart/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-cart/integration_tests/acceptance.py index 108075487440f..950b53b59d416 100644 --- a/airbyte-integrations/connectors/source-cart/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-cart/integration_tests/acceptance.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # @@ -10,5 +10,5 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - """ This fixture is a placeholder for external resources that acceptance test might require.""" + """This fixture is a placeholder for external resources that acceptance test might require.""" yield diff --git a/airbyte-integrations/connectors/source-chargebee/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-chargebee/integration_tests/acceptance.py index a1c52d74fe81e..34f2f625e15bb 100644 --- a/airbyte-integrations/connectors/source-chargebee/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-chargebee/integration_tests/acceptance.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # @@ -10,7 +10,7 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - """ This fixture is a placeholder for external resources that acceptance test might require.""" + """This fixture is a placeholder for external resources that acceptance test might require.""" # TODO: setup test dependencies yield # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-close-com/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-close-com/integration_tests/acceptance.py index 108075487440f..950b53b59d416 100644 --- a/airbyte-integrations/connectors/source-close-com/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-close-com/integration_tests/acceptance.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # @@ -10,5 +10,5 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - """ This fixture is a placeholder for external resources that acceptance test might require.""" + """This fixture is a placeholder for external resources that acceptance test might require.""" yield diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachJdbcSourceOperations.java b/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachJdbcSourceOperations.java index e24ef00a4caf5..f29eee7df9f1a 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachJdbcSourceOperations.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachJdbcSourceOperations.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.source.cockroachdb; import com.fasterxml.jackson.databind.JsonNode; @@ -14,7 +18,7 @@ public class CockroachJdbcSourceOperations extends JdbcSourceOperations { @Override - protected void putBoolean(ObjectNode node, String columnName, ResultSet resultSet, int index) throws SQLException { + protected void putBoolean(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { if ("bit".equalsIgnoreCase(resultSet.getMetaData().getColumnTypeName(index))) { node.put(columnName, resultSet.getByte(index)); } else { @@ -28,12 +32,12 @@ protected void putDouble(final ObjectNode node, final String columnName, final R } @Override - protected void putNumber(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { + protected void putBigDecimal(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { node.put(columnName, resultSet.getBigDecimal(index)); } @Override - public JsonNode rowToJson(ResultSet queryContext) throws SQLException { + public JsonNode rowToJson(final ResultSet queryContext) throws SQLException { final int columnCount = queryContext.getMetaData().getColumnCount(); final ObjectNode jsonNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); @@ -43,16 +47,16 @@ public JsonNode rowToJson(ResultSet queryContext) throws SQLException { if (!queryContext.wasNull()) { setJsonField(queryContext, i, jsonNode); } - } catch (SQLException e) { + } catch (final SQLException e) { putCockroachSpecialDataType(queryContext, i, jsonNode); } } return jsonNode; } - private void putCockroachSpecialDataType(ResultSet resultSet, int index, ObjectNode node) throws SQLException { - String columnType = resultSet.getMetaData().getColumnTypeName(index); - String columnName = resultSet.getMetaData().getColumnName(index); + private void putCockroachSpecialDataType(final ResultSet resultSet, final int index, final ObjectNode node) throws SQLException { + final String columnType = resultSet.getMetaData().getColumnTypeName(index); + final String columnName = resultSet.getMetaData().getColumnName(index); try { if ("numeric".equalsIgnoreCase(columnType)) { final double value = resultSet.getDouble(index); @@ -64,4 +68,5 @@ private void putCockroachSpecialDataType(ResultSet resultSet, int index, ObjectN node.put(columnName, (Double) null); } } + } diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java index bac0377247848..4688fabe7a564 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. */ package io.airbyte.integrations.source.cockroachdb; @@ -8,12 +8,16 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; import io.airbyte.integrations.standardtest.source.TestDataHolder; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.protocol.models.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaType; import java.sql.SQLException; +import java.util.Objects; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,34 +27,40 @@ public class CockroachDbSourceDatatypeTest extends AbstractSourceDatabaseTypeTes private CockroachContainer container; private JsonNode config; + private DSLContext dslContext; private static final Logger LOGGER = LoggerFactory .getLogger(CockroachDbSourceDatatypeTest.class); @Override protected Database setupDatabase() throws SQLException { - container = new CockroachContainer("cockroachdb/cockroach"); + container = new CockroachContainer("cockroachdb/cockroach:v20.2.18"); container.start(); config = Jsons.jsonNode(ImmutableMap.builder() - .put("host", container.getHost()) - // by some reason it return not a port number as exposed and mentioned in logs - .put("port", container.getFirstMappedPort() - 1) - .put("database", container.getDatabaseName()) - .put("username", container.getUsername()) - .put("password", container.getPassword()) - .put("ssl", false) + .put(JdbcUtils.HOST_KEY, Objects.requireNonNull(container.getContainerInfo() + .getNetworkSettings() + .getNetworks() + .entrySet().stream() + .findFirst() + .get().getValue().getIpAddress())) + .put(JdbcUtils.PORT_KEY, container.getExposedPorts().get(1)) + .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) + .put(JdbcUtils.USERNAME_KEY, container.getUsername()) + .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) + .put(JdbcUtils.SSL_KEY, false) .build()); LOGGER.warn("PPP:config:" + config); - final Database database = Databases.createDatabase( - config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:postgresql://%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "org.postgresql.Driver", + dslContext = DSLContextFactory.create( + config.get(JdbcUtils.USERNAME_KEY).asText(), + config.get(JdbcUtils.PASSWORD_KEY).asText(), + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), + config.get(JdbcUtils.HOST_KEY).asText(), + config.get(JdbcUtils.PORT_KEY).asInt(), + config.get(JdbcUtils.DATABASE_KEY).asText()), SQLDialect.POSTGRES); + final Database database = new Database(dslContext); database.query(ctx -> ctx.fetch("CREATE SCHEMA TEST;")); database.query(ctx -> ctx.fetch("CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy');")); @@ -74,6 +84,7 @@ protected JsonNode getConfig() { @Override protected void tearDown(final TestDestinationEnv testEnv) { + dslContext.close(); container.close(); } @@ -83,16 +94,16 @@ protected void initTests() { TestDataHolder.builder() .sourceType("array") .fullSourceDataType("STRING[]") - .airbyteType(JsonSchemaPrimitive.STRING) + .airbyteType(JsonSchemaType.ARRAY) .addInsertValues("ARRAY['sky', 'road', 'car']", "null") - .addExpectedValues("{sky,road,car}", null) + .addExpectedValues("[\"sky\",\"road\",\"car\"]", null) .build()); addDataTypeTestData( TestDataHolder.builder() .sourceType("bit") .fullSourceDataType("BIT(3)") - .airbyteType(JsonSchemaPrimitive.NUMBER) + .airbyteType(JsonSchemaType.NUMBER) .addInsertValues("B'101'") .addExpectedValues("101") .build()); @@ -100,7 +111,7 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("bigint") - .airbyteType(JsonSchemaPrimitive.NUMBER) + .airbyteType(JsonSchemaType.NUMBER) .addInsertValues("-9223372036854775808", "9223372036854775807", "0", "null") .addExpectedValues("-9223372036854775808", "9223372036854775807", "0", null) .build()); @@ -108,7 +119,7 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("bigserial") - .airbyteType(JsonSchemaPrimitive.NUMBER) + .airbyteType(JsonSchemaType.NUMBER) .addInsertValues("1", "9223372036854775807", "0", "-9223372036854775808") .addExpectedValues("1", "9223372036854775807", "0", "-9223372036854775808") .build()); @@ -116,7 +127,7 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("serial") - .airbyteType(JsonSchemaPrimitive.NUMBER) + .airbyteType(JsonSchemaType.NUMBER) .addInsertValues("1", "2147483647", "0", "-2147483647") .addExpectedValues("1", "2147483647", "0", "-2147483647") .build()); @@ -124,7 +135,7 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("smallserial") - .airbyteType(JsonSchemaPrimitive.NUMBER) + .airbyteType(JsonSchemaType.NUMBER) .addInsertValues("1", "32767", "0", "-32767") .addExpectedValues("1", "32767", "0", "-32767") .build()); @@ -133,7 +144,7 @@ protected void initTests() { TestDataHolder.builder() .sourceType("bit_varying") .fullSourceDataType("BIT VARYING(5)") - .airbyteType(JsonSchemaPrimitive.NUMBER) + .airbyteType(JsonSchemaType.NUMBER) .addInsertValues("B'101'", "null") .addExpectedValues("101", null) .build()); @@ -141,7 +152,7 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("boolean") - .airbyteType(JsonSchemaPrimitive.BOOLEAN) + .airbyteType(JsonSchemaType.BOOLEAN) .addInsertValues("true", "'yes'", "'1'", "false", "'no'", "'0'", "null") .addExpectedValues("true", "true", "true", "false", "false", "false", null) .build()); @@ -150,15 +161,15 @@ protected void initTests() { TestDataHolder.builder() .sourceType("bytea") .fullSourceDataType("bytea[]") - .airbyteType(JsonSchemaPrimitive.OBJECT) + .airbyteType(JsonSchemaType.OBJECT) .addInsertValues("ARRAY['☃'::bytes, 'ї'::bytes]") - .addExpectedValues("{\"\\\\xe29883\",\"\\\\xd197\"}") + .addExpectedValues("[\"\\\\xe29883\",\"\\\\xd197\"]") .build()); addDataTypeTestData( TestDataHolder.builder() .sourceType("blob") - .airbyteType(JsonSchemaPrimitive.OBJECT) + .airbyteType(JsonSchemaType.OBJECT) .addInsertValues("decode('1234', 'hex')") .addExpectedValues("EjQ=") .build()); @@ -166,7 +177,7 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("character") - .airbyteType(JsonSchemaPrimitive.STRING) + .airbyteType(JsonSchemaType.STRING) .addInsertValues("'a'", "'*'", "null") .addExpectedValues("a", "*", null) .build()); @@ -175,7 +186,7 @@ protected void initTests() { TestDataHolder.builder() .sourceType("character") .fullSourceDataType("character(8)") - .airbyteType(JsonSchemaPrimitive.STRING) + .airbyteType(JsonSchemaType.STRING) .addInsertValues("'{asb123}'", "'{asb12}'") .addExpectedValues("{asb123}", "{asb12} ") .build()); @@ -183,7 +194,7 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("varchar") - .airbyteType(JsonSchemaPrimitive.STRING) + .airbyteType(JsonSchemaType.STRING) .addInsertValues("'a'", "'abc'", "'Миші йдуть на південь, не питай чому;'", "'櫻花分店'", "''", "null", "'\\xF0\\x9F\\x9A\\x80'") .addExpectedValues("a", "abc", "Миші йдуть на південь, не питай чому;", "櫻花分店", "", @@ -194,7 +205,7 @@ protected void initTests() { TestDataHolder.builder() .sourceType("varchar") .fullSourceDataType("character(12)") - .airbyteType(JsonSchemaPrimitive.STRING) + .airbyteType(JsonSchemaType.STRING) .addInsertValues("'a'", "'abc'", "'Миші йдуть;'", "'櫻花分店'", "''", "null") .addExpectedValues("a ", "abc ", "Миші йдуть; ", "櫻花分店 ", @@ -204,7 +215,7 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("date") - .airbyteType(JsonSchemaPrimitive.STRING) + .airbyteType(JsonSchemaType.STRING) .addInsertValues("'1999-01-08'", "null") .addExpectedValues("1999-01-08T00:00:00Z", null) .build()); @@ -212,7 +223,7 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("float8") - .airbyteType(JsonSchemaPrimitive.NUMBER) + .airbyteType(JsonSchemaType.NUMBER) .addInsertValues("'123'", "'1234567890.1234567'", "null", "'infinity'", "'+infinity'", "'+inf'", "'inf'", "'-inf'", "'-infinity'", "'nan'") .addExpectedValues("123.0", "1.2345678901234567E9", null, @@ -222,7 +233,7 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("float") - .airbyteType(JsonSchemaPrimitive.NUMBER) + .airbyteType(JsonSchemaType.NUMBER) .addInsertValues("'123'", "'1234567890.1234567'", "null", "'infinity'", "'+infinity'", "'+inf'", "'inf'", "'-inf'", "'-infinity'", "'nan'") .addExpectedValues("123.0", "1.2345678901234567E9", null, "Infinity", @@ -232,7 +243,7 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("inet") - .airbyteType(JsonSchemaPrimitive.STRING) + .airbyteType(JsonSchemaType.STRING) .addInsertValues("'198.24.10.0/24'", "'198.24.10.0'", "'198.10/8'", "null") .addExpectedValues("198.24.10.0/24", "198.24.10.0", "198.10.0.0/8", null) .build()); @@ -240,7 +251,7 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("int") - .airbyteType(JsonSchemaPrimitive.NUMBER) + .airbyteType(JsonSchemaType.NUMBER) .addInsertValues("null", "-2147483648", "2147483647") .addExpectedValues(null, "-2147483648", "2147483647") .build()); @@ -248,7 +259,7 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("interval") - .airbyteType(JsonSchemaPrimitive.STRING) + .airbyteType(JsonSchemaType.STRING) .addInsertValues("null", "'P1Y2M3DT4H5M6S'", "'-178000000'", "'178000000'") .addExpectedValues(null, "1 year 2 mons 3 days 04:05:06", "-49444:26:40", "49444:26:40") .build()); @@ -256,7 +267,7 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("json") - .airbyteType(JsonSchemaPrimitive.STRING) + .airbyteType(JsonSchemaType.STRING) .addInsertValues("null", "'{\"a\": 10, \"b\": 15}'") .addExpectedValues(null, "{\"a\": 10, \"b\": 15}") .build()); @@ -264,7 +275,7 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("jsonb") - .airbyteType(JsonSchemaPrimitive.STRING) + .airbyteType(JsonSchemaType.STRING) .addInsertValues("null", "'[1, 2, 3]'::jsonb") .addExpectedValues(null, "[1, 2, 3]") .build()); @@ -272,7 +283,7 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("numeric") - .airbyteType(JsonSchemaPrimitive.NUMBER) + .airbyteType(JsonSchemaType.NUMBER) .addInsertValues("'99999'", "null") .addExpectedValues("99999", null) .build()); @@ -280,7 +291,7 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("decimal") - .airbyteType(JsonSchemaPrimitive.NUMBER) + .airbyteType(JsonSchemaType.NUMBER) .addInsertValues("'+inf'", "999", "'-inf'", "'+infinity'", "'-infinity'", "'nan'") .addExpectedValues("Infinity", "999", "-Infinity", "Infinity", "-Infinity", "NaN") .build()); @@ -288,7 +299,7 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("smallint") - .airbyteType(JsonSchemaPrimitive.NUMBER) + .airbyteType(JsonSchemaType.NUMBER) .addInsertValues("null", "-32768", "32767") .addExpectedValues(null, "-32768", "32767") .build()); @@ -296,7 +307,7 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("text") - .airbyteType(JsonSchemaPrimitive.STRING) + .airbyteType(JsonSchemaType.STRING) .addInsertValues("'a'", "'abc'", "'Миші йдуть;'", "'櫻花分店'", "''", "null", "'\\xF0\\x9F\\x9A\\x80'") .addExpectedValues("a", "abc", "Миші йдуть;", "櫻花分店", "", null, "\\xF0\\x9F\\x9A\\x80") @@ -307,17 +318,17 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("time") - .airbyteType(JsonSchemaPrimitive.STRING) + .airbyteType(JsonSchemaType.STRING) .addInsertValues("'04:05:06'", null) .addExpectedValues("1970-01-01T04:05:06Z") .addNullExpectedValue() .build()); - // Time (04:05:06) would be represented like "1970-01-01T04:05:06Z" + // Time (04:05:06) would be represented like "1970-01-01T04:05:06Z" addDataTypeTestData( TestDataHolder.builder() .sourceType("timetz") - .airbyteType(JsonSchemaPrimitive.STRING) + .airbyteType(JsonSchemaType.STRING) .addInsertValues("'04:05:06Z'", null) .addExpectedValues("1970-01-01T04:05:06Z") .addNullExpectedValue() @@ -326,15 +337,15 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("timestamp") - .airbyteType(JsonSchemaPrimitive.STRING) - .addInsertValues("TIMESTAMP '2004-10-19 10:23:54'", "null") - .addExpectedValues("2004-10-19T10:23:54Z", null) + .airbyteType(JsonSchemaType.STRING) + .addInsertValues("TIMESTAMP '2004-10-19 10:23:54'", "TIMESTAMP '2004-10-19 10:23:54.123456'", "null") + .addExpectedValues("2004-10-19T10:23:54.000000Z", "2004-10-19T10:23:54.123456Z", null) .build()); addDataTypeTestData( TestDataHolder.builder() .sourceType("uuid") - .airbyteType(JsonSchemaPrimitive.STRING) + .airbyteType(JsonSchemaType.STRING) .addInsertValues("'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'", "null") .addExpectedValues("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11", null) .build()); @@ -343,7 +354,7 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("mood") - .airbyteType(JsonSchemaPrimitive.STRING) + .airbyteType(JsonSchemaType.STRING) .addInsertValues("'happy'", "null") .addExpectedValues("happy", null) .build()); @@ -352,9 +363,18 @@ protected void initTests() { TestDataHolder.builder() .sourceType("text") .fullSourceDataType("text[]") - .airbyteType(JsonSchemaPrimitive.STRING) + .airbyteType(JsonSchemaType.ARRAY) + .addInsertValues("'{10000, 10000, 10000, 10000}'", "null") + .addExpectedValues("[\"10000\",\"10000\",\"10000\",\"10000\"]", null) + .build()); + + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("int") + .fullSourceDataType("int[]") + .airbyteType(JsonSchemaType.ARRAY) .addInsertValues("'{10000, 10000, 10000, 10000}'", "null") - .addExpectedValues("{10000,10000,10000,10000}", null) + .addExpectedValues("[\"10000\",\"10000\",\"10000\",\"10000\"]", null) .build()); } diff --git a/airbyte-integrations/connectors/source-confluence/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-confluence/integration_tests/acceptance.py index 58c194c5d1376..1302b2f57e10e 100644 --- a/airbyte-integrations/connectors/source-confluence/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-confluence/integration_tests/acceptance.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # @@ -10,7 +10,7 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - """ This fixture is a placeholder for external resources that acceptance test might require.""" + """This fixture is a placeholder for external resources that acceptance test might require.""" # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-datascope/.dockerignore b/airbyte-integrations/connectors/source-datascope/.dockerignore new file mode 100644 index 0000000000000..0874a7b966fe8 --- /dev/null +++ b/airbyte-integrations/connectors/source-datascope/.dockerignore @@ -0,0 +1,7 @@ +* +!Dockerfile +!Dockerfile.test +!main.py +!source_datascope +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-datascope/Dockerfile b/airbyte-integrations/connectors/source-datascope/Dockerfile new file mode 100644 index 0000000000000..eca337bae828a --- /dev/null +++ b/airbyte-integrations/connectors/source-datascope/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.7-slim + +# Bash is installed for more convenient debugging. +RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* + +WORKDIR /airbyte/integration_code +COPY source_datascope ./source_datascope +COPY main.py ./ +COPY setup.py ./ +RUN pip install . + +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-datascope +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" diff --git a/airbyte-integrations/connectors/source-datascope/README.md b/airbyte-integrations/connectors/source-datascope/README.md new file mode 100644 index 0000000000000..9574c69653f29 --- /dev/null +++ b/airbyte-integrations/connectors/source-datascope/README.md @@ -0,0 +1,131 @@ +# Datascope Source + +This is the repository for the Datascope source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/datascope). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-datascope:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/datascope) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_datascope/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source datascope test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-datascope:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-datascope:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-datascope:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-datascope:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-datascope:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-datascope:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](source-acceptance-tests.md) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-datascope:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-datascope:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-datascope/acceptance-test-config.yml b/airbyte-integrations/connectors/source-datascope/acceptance-test-config.yml new file mode 100644 index 0000000000000..2341697f2c794 --- /dev/null +++ b/airbyte-integrations/connectors/source-datascope/acceptance-test-config.yml @@ -0,0 +1,15 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/contributing-to-airbyte/building-new-connector/source-acceptance-tests.md) +# for more information about how to configure these tests +connector_image: airbyte/source-datascope:dev +tests: + spec: + - spec_path: "source_datascope/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + validate_output_from_all_streams: yes diff --git a/airbyte-integrations/connectors/source-datascope/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-datascope/acceptance-test-docker.sh new file mode 100644 index 0000000000000..1425ff74f1511 --- /dev/null +++ b/airbyte-integrations/connectors/source-datascope/acceptance-test-docker.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input diff --git a/airbyte-integrations/connectors/source-datascope/build.gradle b/airbyte-integrations/connectors/source-datascope/build.gradle new file mode 100644 index 0000000000000..8fe360c9b363d --- /dev/null +++ b/airbyte-integrations/connectors/source-datascope/build.gradle @@ -0,0 +1,14 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_datascope' +} + +dependencies { + implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) + implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-datascope/integration_tests/__init__.py b/airbyte-integrations/connectors/source-datascope/integration_tests/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/airbyte-integrations/connectors/source-datascope/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-datascope/integration_tests/abnormal_state.json new file mode 100644 index 0000000000000..52b0f2c2118f4 --- /dev/null +++ b/airbyte-integrations/connectors/source-datascope/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-datascope/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-datascope/integration_tests/acceptance.py new file mode 100644 index 0000000000000..056971f954502 --- /dev/null +++ b/airbyte-integrations/connectors/source-datascope/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-datascope/integration_tests/catalog.json b/airbyte-integrations/connectors/source-datascope/integration_tests/catalog.json new file mode 100644 index 0000000000000..6799946a68514 --- /dev/null +++ b/airbyte-integrations/connectors/source-datascope/integration_tests/catalog.json @@ -0,0 +1,39 @@ +{ + "streams": [ + { + "name": "TODO fix this file", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": "column1", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + }, + { + "name": "table1", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-datascope/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-datascope/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..4e809b3494993 --- /dev/null +++ b/airbyte-integrations/connectors/source-datascope/integration_tests/configured_catalog.json @@ -0,0 +1,109 @@ +{ + "streams": [ + { + "stream": { + "name": "forms", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "code": { + "type": ["null", "string"] + }, + "form_name": { + "type": ["null", "string"] + }, + "form_state": { + "type": ["null", "string"] + }, + "user_name": { + "type": ["null", "string"] + }, + "user_identifier": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "form_answer_id": { + "type": ["null", "number"] + }, + "latitude": { + "type": ["null", "number"] + }, + "longitude": { + "type": ["null", "number"] + }, + "assign_id": { + "type": ["null", "number"] + }, + "assign_internal_id": { + "type": ["null", "number"] + }, + "assign_location_name": { + "type": ["null", "string"] + }, + "answers": { + "type": "array", + "items": { + "type": "object", + "properties": { + "form_code": { + "type": ["null", "string"] + }, + "form_id": { + "type": ["null", "number"] + }, + "form_answer_id": { + "type": ["null", "number"] + }, + "question_name": { + "type": ["null", "string"] + }, + "question_value": { + "type": ["null", "string", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "question_type": { + "type": ["null", "string"] + }, + "metadata_type": { + "type": ["null", "string"] + }, + "metadata_id": { + "type": ["null", "number"] + }, + "subform_index": { + "type": ["null", "number"] + }, + "question_id": { + "type": ["null", "number"] + } + } + } + } + } + }, + "supportedSyncModes": ["full_refresh", "incremental"], + "sourceDefinedCursor": true, + "defaultCursorField": ["updated_at"], + "sourceDefinedPrimaryKey": [["form_answer_id"]], + "namespace": null + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append", + "config": { + "sync_mode": "full_refresh", + "cursorField": ["updated_at"], + "destination_sync_mode": "append", + "primaryKey": [["form_answer_id"]], + "aliasName": "forms" + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-datascope/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-datascope/integration_tests/invalid_config.json new file mode 100644 index 0000000000000..f3732995784f2 --- /dev/null +++ b/airbyte-integrations/connectors/source-datascope/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "todo-wrong-field": "this should be an incomplete config file, used in standard tests" +} diff --git a/airbyte-integrations/connectors/source-datascope/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-datascope/integration_tests/sample_config.json new file mode 100644 index 0000000000000..36742c811816d --- /dev/null +++ b/airbyte-integrations/connectors/source-datascope/integration_tests/sample_config.json @@ -0,0 +1,6 @@ +{ + "form_id": "1", + "start_date": "2021-01-01T00:00:00Z", + "api_key": "bd982ndndj9ndshf09hs8dfhs9d", + "schema_type": "fixed" +} diff --git a/airbyte-integrations/connectors/source-datascope/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-datascope/integration_tests/sample_state.json new file mode 100644 index 0000000000000..3587e579822d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-datascope/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-datascope/main.py b/airbyte-integrations/connectors/source-datascope/main.py new file mode 100644 index 0000000000000..8177009ae2434 --- /dev/null +++ b/airbyte-integrations/connectors/source-datascope/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_datascope import SourceDatascope + +if __name__ == "__main__": + source = SourceDatascope() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-datascope/requirements.txt b/airbyte-integrations/connectors/source-datascope/requirements.txt new file mode 100644 index 0000000000000..0411042aa0911 --- /dev/null +++ b/airbyte-integrations/connectors/source-datascope/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-datascope/setup.py b/airbyte-integrations/connectors/source-datascope/setup.py new file mode 100644 index 0000000000000..5d88b56e92123 --- /dev/null +++ b/airbyte-integrations/connectors/source-datascope/setup.py @@ -0,0 +1,28 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "source-acceptance-test", +] + +setup( + name="source_datascope", + description="Source implementation for Datascope.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-datascope/source_datascope/__init__.py b/airbyte-integrations/connectors/source-datascope/source_datascope/__init__.py new file mode 100644 index 0000000000000..55b1497ec1a8f --- /dev/null +++ b/airbyte-integrations/connectors/source-datascope/source_datascope/__init__.py @@ -0,0 +1,27 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from .source import SourceDatascope + +__all__ = ["SourceDatascope"] diff --git a/airbyte-integrations/connectors/source-datascope/source_datascope/schemas/TODO.md b/airbyte-integrations/connectors/source-datascope/source_datascope/schemas/TODO.md new file mode 100644 index 0000000000000..cf1efadb3c9c9 --- /dev/null +++ b/airbyte-integrations/connectors/source-datascope/source_datascope/schemas/TODO.md @@ -0,0 +1,25 @@ +# TODO: Define your stream schemas +Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). + +The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. + +The schema of a stream is the return value of `Stream.get_json_schema`. + +## Static schemas +By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. + +Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. + +## Dynamic schemas +If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). + +## Dynamically modifying static schemas +Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: +``` +def get_json_schema(self): + schema = super().get_json_schema() + schema['dynamically_determined_property'] = "property" + return schema +``` + +Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-datascope/source_datascope/schemas/forms.json b/airbyte-integrations/connectors/source-datascope/source_datascope/schemas/forms.json new file mode 100644 index 0000000000000..9a93f1d2509da --- /dev/null +++ b/airbyte-integrations/connectors/source-datascope/source_datascope/schemas/forms.json @@ -0,0 +1,86 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "code": { + "type": ["null", "string"] + }, + "form_name": { + "type": ["null", "string"] + }, + "form_state": { + "type": ["null", "string"] + }, + "user_name": { + "type": ["null", "string"] + }, + "user_identifier": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "form_answer_id": { + "type": ["null", "number"] + }, + "latitude": { + "type": ["null", "number"] + }, + "longitude": { + "type": ["null", "number"] + }, + "assign_id": { + "type": ["null", "number"] + }, + "assign_internal_id": { + "type": ["null", "number"] + }, + "assign_location_name": { + "type": ["null", "string"] + }, + "answers": { + "type": "array", + "items": { + "type": "object", + "properties": { + "form_code": { + "type": ["null", "string"] + }, + "form_id": { + "type": ["null", "number"] + }, + "form_answer_id": { + "type": ["null", "number"] + }, + "question_name": { + "type": ["null", "string"] + }, + "question_value": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "question_type": { + "type": ["null", "string"] + }, + "metadata_type": { + "type": ["null", "string"] + }, + "metadata_id": { + "type": ["null", "number"] + }, + "subform_index": { + "type": ["null", "number"] + }, + "question_id": { + "type": ["null", "number"] + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-datascope/source_datascope/source.py b/airbyte-integrations/connectors/source-datascope/source_datascope/source.py new file mode 100644 index 0000000000000..31cd22ccca74b --- /dev/null +++ b/airbyte-integrations/connectors/source-datascope/source_datascope/source.py @@ -0,0 +1,226 @@ +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from datetime import datetime, timedelta +import logging +import backoff +import time +from airbyte_cdk.logger import AirbyteLogger + +import requests +from requests.exceptions import ConnectionError +from requests.structures import CaseInsensitiveDict +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from airbyte_cdk.sources.streams.http.auth import NoAuth + + + +# Basic full refresh stream +class DatascopeStream(HttpStream, ABC): + + # TODO: Fill in the url base. Required. + url_base = "https://mydatascope.com/api/external/" + + queries_per_hour = 1000 + + def __init__(self, token: str, form_id: str, start_date: str, schema_type: str, **kwargs): + super().__init__() + self.token = token + self.form_id = form_id + self.start_date = start_date + self.schema_type = schema_type + self.BASE_URL = "https://mydatascope.com" + self._headers = { + "Authorization": f"Bearer {self.token}", + "Content-Type": "application/json", + "Accept": "application/json", + } + + + @backoff.on_exception(backoff.expo, requests.exceptions.ConnectionError, max_tries=7) + def _request(self, url: str, method: str = "GET", data: dict = None) -> List[dict]: + response = requests.request(method, url, headers=self._headers, json=data) + + if response.status_code == 200: + response_data = response.json() + if isinstance(response_data, list): + return response_data + else: + return [response_data] + + @property + def cursor_field(self) -> str: + if self.schema_type == 'dynamic_updates': + return "updated_at_unix" + else: + return "created_at" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + decoded_response = response.json() + if decoded_response: + #offset by created form_answer_id + last_object_date = decoded_response[-1]["form_answer_id"] + if self.schema_type == 'dynamic_updates': + last_object_date = decoded_response[-1]["updated_at_unix"] + if last_object_date: + return {"offset": last_object_date} + else: + return None + else: + return None + + def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + # The api requires that we include the base currency as a query param so we do that in this method + params = { 'form_id': self.form_id, 'token': self.token, 'start': self.start_date} + if next_page_token: + params.update(**next_page_token) + return params + + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: + time.sleep(3600 / self.queries_per_hour) + return response.json() + + +class DatascopeIncrementalStream(DatascopeStream, ABC): + + def filter_by_state(self, stream_state: Mapping[str, Any] = None, record: Mapping[str, Any] = None) -> Iterable: + record_date = record.get(self.cursor_field) + stream_date = self.start_date + if stream_state.get(self.cursor_field): + if self.schema_type == 'dynamic': + stream_date = datetime.strptime(stream_state.get(self.cursor_field).split('+')[0] , '%Y-%m-%dT%H:%M:%S') + elif self.schema_type == 'fixed': + stream_date = datetime.strptime(stream_state.get(self.cursor_field).split('+')[0] , '%Y-%m-%dT%H:%M:%S') + elif self.schema_type == 'dynamic_updates': + stream_date = stream_state.get(self.cursor_field) + stream_state = int(stream_date) + record_date = int(record_date) + AirbyteLogger().log("INFO", f"record date: {record_date}") + AirbyteLogger().log("INFO", f"stream date: {stream_date}") + if not stream_date or record_date > stream_date: + yield record + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: + if self.schema_type == 'dynamic' or self.schema_type == 'fixed': + return get_updated_state_old(self, current_stream_state, latest_record) + current_stream_state = current_stream_state or {} + latest_record_date = self.start_date + start_date = self.start_date + if latest_record: + latest_record_date = latest_record[self.cursor_field] + current_stream_state_date = current_stream_state.get(self.cursor_field, self.start_date) + cursor_state = max(current_stream_state_date, latest_record_date) + return {self.cursor_field: cursor_state } + + def get_updated_state_old(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: + current_stream_state = current_stream_state or {} + latest_record_date = self.start_date + if latest_record: + last_date = latest_record[self.cursor_field].split('.')[0] + '+0000' + if self.schema_type == 'dynamic' or self.schema_type == 'dynamic_updates': + latest_record_date = datetime.strptime(last_date, '%d/%m/%Y %H:%M%z') + else: + latest_record_date = datetime.strptime(last_date, '%Y-%m-%dT%H:%M:%S%z') + current_stream_state_date = current_stream_state.get(self.cursor_field, self.start_date) + if isinstance(latest_record_date,str): + latest_record_date = datetime.strptime(latest_record_date.split('+')[0], '%d/%m/%Y %H:%M:%S') + if isinstance(current_stream_state_date,str): + current_stream_state_date = datetime.strptime(current_stream_state_date.split('+')[0] + 'Z', '%Y-%m-%dT%H:%M:%S%z') + AirbyteLogger().log("INFO", f"current state: {current_stream_state_date}") + AirbyteLogger().log("INFO", f"lest_state: {latest_record_date}") + cursor_state = max(current_stream_state_date, latest_record_date) + AirbyteLogger().log("INFO", f"max date: {cursor_state}") + return {self.cursor_field: cursor_state } + + def request_params(self, next_page_token: Mapping[str, Any] = None, stream_state: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + # The api requires that we include the base currency as a query param so we do that in this method + stream_state = stream_state or {} + start_date = self.start_date + last_date = stream_state.get(self.cursor_field, start_date) + params = { 'form_id': self.form_id, 'token': self.token} + if self.schema_type == 'dynamic_updates': + params['date_modified'] = True + elif self.schema_type != 'dynamic': + params['pagination'] = True + + if self.schema_type != 'dynamic_updates': + if not isinstance(last_date, str): + last_date = last_date.strftime('%m/%d/%Y') + + if next_page_token: + params.update(**next_page_token) + else: + params.update(**{'offset': last_date}) + + AirbyteLogger().log("INFO", f"schema type: {self.schema_type}") + AirbyteLogger().log("INFO", f"params request: {params}") + AirbyteLogger().log("INFO", f"cursor: {self.cursor_field}") + return params + + + +class Forms(DatascopeIncrementalStream): + primary_key = "form_answer_id" + + def path(self, **kwargs) -> str: + if self.schema_type == 'dynamic' or self.schema_type == 'dynamic_updates': + return "v3/answers" + else: + return "v3/answers_static" + + def get_json_schema(self): + schema = super().get_json_schema() + schema = self._request(f"{self.BASE_URL}/api/external/v2/airbyte_schema?token={self.token}&form_id={self.form_id}")[0] + return schema + +class Forms(DatascopeIncrementalStream): + primary_key = "form_answer_id" + + def path(self, **kwargs) -> str: + return "v3/answers" + + def get_json_schema(self): + schema = super().get_json_schema() + if str(self.schema_type) != 'fixed': + schema = self._request(f"{self.BASE_URL}/api/external/v2/airbyte_schema?token={self.token}&form_id={self.form_id}")[0] + return schema + + +# Source +class SourceDatascope(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + return True, None + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + auth = TokenAuthenticator(config['api_key']) + start_date = datetime.strptime(config['start_date'], '%Y-%m-%dT%H:%M:%S%z') + schema_type = config.get('schema_type', 'fixed') + if schema_type == 'dynamic_updates': + start_date = time.mktime(start_date.timetuple()) + AirbyteLogger().log("INFO", f"start date: {config['start_date']}") + return [Forms(authenticator=auth, token=config['api_key'], form_id=config['form_id'], start_date=start_date, schema_type=schema_type)] diff --git a/airbyte-integrations/connectors/source-datascope/source_datascope/spec.json b/airbyte-integrations/connectors/source-datascope/source_datascope/spec.json new file mode 100644 index 0000000000000..ffbbbdf413186 --- /dev/null +++ b/airbyte-integrations/connectors/source-datascope/source_datascope/spec.json @@ -0,0 +1,33 @@ +{ + "documentationUrl": "https://docsurl.com", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Datascope Spec", + "type": "object", + "required": ["api_key", "form_id", "start_date"], + "additionalProperties": false, + "properties": { + "api_key": { + "type": "string", + "description": "Sign in on your DataScope account and copy it from Integrations (https://mydatascope.com/webhooks) on section 'API Key'", + "airbyte_secret": true + }, + "start_date": { + "type": "string", + "description": "Start date", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + "examples": ["2017-01-25T00:00:00Z"] + }, + "form_id": { + "type": "string", + "description": "You can find the form ID on the URL at the moment to edit a form (https://mydatascope.com/task_forms/FORM_ID/edit)." + }, + "schema_type": { + "type": "string", + "description": "Please select type of schema 'fixed' or 'dynamic'. Fixed will use same schema for all forms and dynamic will change schema depending on the questions of the form", + "enum": ["fixed", "dynamic", "dynamic_updates"], + "examples": ["Fixed or Dynamic, Dynamic with updates"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-datascope/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-datascope/unit_tests/unit_test.py new file mode 100644 index 0000000000000..e1814314fc3b0 --- /dev/null +++ b/airbyte-integrations/connectors/source-datascope/unit_tests/unit_test.py @@ -0,0 +1,7 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +def test_example_method(): + assert True diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java index 3731c00630583..1d25d4784f08b 100644 --- a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java @@ -164,8 +164,10 @@ protected void tearDown(TestDestinationEnv testEnv) { /* Helpers */ private String getCertificate() throws IOException, InterruptedException { - // To enable SSL connection on the server, we need to generate self-signed certificates for the server and add them to the configuration. - // Then you need to enable SSL connection and specify on which port it will work. These changes will take effect after restart. + // To enable SSL connection on the server, we need to generate self-signed certificates for the + // server and add them to the configuration. + // Then you need to enable SSL connection and specify on which port it will work. These changes will + // take effect after restart. // The certificate for generating a user certificate has the extension *.arm. db.execInContainer("su", "-", "db2inst1", "-c", "gsk8capicmd_64 -keydb -create -db \"server.kdb\" -pw \"" + TEST_KEY_STORE_PASS + "\" -stash"); db.execInContainer("su", "-", "db2inst1", "-c", "gsk8capicmd_64 -cert -create -db \"server.kdb\" -pw \"" + TEST_KEY_STORE_PASS diff --git a/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java b/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java index 542a02cd1dab9..e8b4dc35f6e3d 100644 --- a/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java +++ b/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java @@ -8,7 +8,6 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.jdbc.Db2JdbcStreamingQueryConfiguration; -import io.airbyte.db.jdbc.JdbcSourceOperations; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; diff --git a/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2SourceOperations.java b/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2SourceOperations.java index 29e55bfc25258..17800463f6833 100644 --- a/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2SourceOperations.java +++ b/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2SourceOperations.java @@ -8,7 +8,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.commons.json.Jsons; import io.airbyte.db.jdbc.JdbcSourceOperations; -import java.sql.JDBCType; import java.sql.ResultSet; import java.sql.SQLException; import java.util.Collections; diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index a7a97d3bbb0be..0a3dcdb33263a 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -1,6 +1,7 @@ # # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # + import logging from datetime import datetime from typing import Any, List, Mapping, MutableMapping, Optional, Tuple, Type diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py index f35d8d7596b3f..6bab56862fa87 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py @@ -1,6 +1,7 @@ # # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # + import time import pendulum