Skip to content

🎉 New Source: DataScope #4636

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 38 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
cf7578f
first modify logo + name
agrass Jun 2, 2021
5f6e56f
logo svg
agrass Jun 2, 2021
c4d069f
add logo2
agrass Jun 3, 2021
f4369e6
Merge branch 'master' of https://github.com/airbytehq/airbyte
agrass Jun 5, 2021
7f67682
Merge branch 'master' of https://github.com/airbytehq/airbyte
Jun 5, 2021
8a6e2f3
Merge branch 'master' of https://github.com/agrass/ds_connect
Jun 8, 2021
a186f57
new connector datascope
Jun 11, 2021
f8b0b02
first version connector
Jun 22, 2021
135437f
fix conflicts
Jul 6, 2021
5b835be
Merge branch 'master' of https://github.com/airbytehq/airbyte
Jul 6, 2021
aa7770e
Merge branch 'master' of https://github.com/airbytehq/airbyte
agrass Jul 6, 2021
11ab6e5
Merge branch 'master' of https://github.com/agrass/airbyte
agrass Jul 8, 2021
13bf723
undo test changes
agrass Jul 8, 2021
96954b1
revert not integration changes
agrass Jul 8, 2021
f1fd584
Merge branch 'master' of https://github.com/airbytehq/airbyte
agrass Jul 8, 2021
5254d88
configure example specs
agrass Jul 8, 2021
f4917f4
first version of tests
agrass Jul 8, 2021
bfe4aa8
add definitions
agrass Jul 8, 2021
e513e4e
Delete old2_answers.json
agrass Jul 9, 2021
7f12b7e
Delete employees.json
agrass Jul 9, 2021
b692433
Delete customers.json
agrass Jul 9, 2021
3e906ae
clean unused schemas
agrass Jul 9, 2021
9b4b08b
add airbyte_secret: true:
agrass Jul 9, 2021
3d61ad9
add api key description
agrass Jul 9, 2021
d33f0f3
Update 8cee7c4a-8047-4cb9-a86d-f7fd87b2d1f8.json
agrass Jul 9, 2021
c643bb8
Update airbyte-integrations/connectors/source-datascope/source_datasc…
agrass Jul 9, 2021
5d59a74
improve description schema type
agrass Jul 9, 2021
9749ec0
add form id description
agrass Jul 9, 2021
d13b463
Update airbyte-integrations/connectors/source-datascope/source_datasc…
agrass Jul 9, 2021
e8f42a2
Merge branch 'master' into master
agrass Jul 22, 2021
6d2d195
fix authenticator, pagination, duplicate data, standard endpoints
agrass Nov 12, 2021
87a9ac4
Merge branch 'master' of https://github.com/agrass/airbyte
agrass Nov 12, 2021
3149744
fix conflict
agrass Nov 12, 2021
6b48904
fix api pagination static schemas
agrass Dec 22, 2021
ea883df
run format
agrass Dec 22, 2021
094f732
update version and to last connector version
Aug 25, 2022
897f334
fix file changed
Aug 25, 2022
d7939a6
undo format
Aug 25, 2022
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions 8cee7c4a-8047-4cb9-a86d-f7fd87b2d1f8.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"sourceDefinitionId": "8cee7c4a-8047-4cb9-a86d-f7fd87b2d1f8",
"name": "DataScope",
"dockerRepository": "airbyte/datascope",
"dockerImageTag": "0.1.0",
"documentationUrl": "https://docs.airbyte.io/integrations/sources/datascope"
}
1 change: 1 addition & 0 deletions airbyte-config/init/src/main/resources/icons/datascope.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#

from collections import Mapping
Expand All @@ -25,7 +25,7 @@ def __init__(self, client: KvDbClient):
self.client = client

def delete_stream_entries(self, stream_name: str):
""" Deletes all the records belonging to the input stream """
"""Deletes all the records belonging to the input stream"""
keys_to_delete = []
for key in self.client.list_keys(prefix=f"{stream_name}__ab__"):
keys_to_delete.append(key)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#


Expand All @@ -10,7 +10,7 @@

@pytest.fixture(scope="session", autouse=True)
def connector_setup():
""" This fixture is a placeholder for external resources that acceptance test might require."""
"""This fixture is a placeholder for external resources that acceptance test might require."""
# TODO: setup test dependencies if needed. otherwise remove the TODO comments
yield
# TODO: clean up test dependencies
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#


Expand All @@ -10,5 +10,5 @@

@pytest.fixture(scope="session", autouse=True)
def connector_setup():
""" This fixture is a placeholder for external resources that acceptance test might require."""
"""This fixture is a placeholder for external resources that acceptance test might require."""
yield
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#

import hashlib
Expand Down Expand Up @@ -41,7 +41,7 @@ def __init__(self, service: str, aws_access_key_id: str, aws_secret_access_key:

@staticmethod
def _sign_msg(key: bytes, msg: str) -> bytes:
""" Sign message using key """
"""Sign message using key"""
return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest()

def _get_authorization_header(self, prepared_request: requests.PreparedRequest) -> str:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#


Expand All @@ -10,7 +10,7 @@

@pytest.fixture(scope="session", autouse=True)
def connector_setup():
""" This fixture is a placeholder for external resources that acceptance test might require."""
"""This fixture is a placeholder for external resources that acceptance test might require."""
# TODO: setup test dependencies
yield
# TODO: clean up test dependencies
Original file line number Diff line number Diff line change
@@ -1,16 +1,7 @@
#
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#


import pytest

pytest_plugins = ("source_acceptance_test.plugin",)


@pytest.fixture(scope="session", autouse=True)
def connector_setup():
""" This fixture is a placeholder for external resources that acceptance test might require."""
# TODO: setup test dependencies if needed. otherwise remove the TODO comments
yield
# TODO: clean up test dependencies
def test_dummy_test():
assert True
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#


Expand All @@ -10,5 +10,5 @@

@pytest.fixture(scope="session", autouse=True)
def connector_setup():
""" This fixture is a placeholder for external resources that acceptance test might require."""
"""This fixture is a placeholder for external resources that acceptance test might require."""
yield
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#


Expand All @@ -10,7 +10,7 @@

@pytest.fixture(scope="session", autouse=True)
def connector_setup():
""" This fixture is a placeholder for external resources that acceptance test might require."""
"""This fixture is a placeholder for external resources that acceptance test might require."""
# TODO: setup test dependencies if needed. otherwise remove the TODO comments
yield
# TODO: clean up test dependencies
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#


Expand All @@ -10,7 +10,5 @@

@pytest.fixture(scope="session", autouse=True)
def connector_setup():
""" This fixture is a placeholder for external resources that acceptance test might require."""
# TODO: setup test dependencies if needed. otherwise remove the TODO comments
"""This fixture is a placeholder for external resources that acceptance test might require."""
yield
# TODO: clean up test dependencies
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#


Expand All @@ -10,7 +10,7 @@

@pytest.fixture(scope="session", autouse=True)
def connector_setup():
""" This fixture is a placeholder for external resources that acceptance test might require."""
"""This fixture is a placeholder for external resources that acceptance test might require."""
# TODO: setup test dependencies if needed. otherwise remove the TODO comments
yield
# TODO: clean up test dependencies
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#


Expand All @@ -10,5 +10,5 @@

@pytest.fixture(scope="session", autouse=True)
def connector_setup():
""" This fixture is a placeholder for external resources that acceptance test might require."""
"""This fixture is a placeholder for external resources that acceptance test might require."""
yield
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#


Expand All @@ -10,7 +10,7 @@

@pytest.fixture(scope="session", autouse=True)
def connector_setup():
""" This fixture is a placeholder for external resources that acceptance test might require."""
"""This fixture is a placeholder for external resources that acceptance test might require."""
# TODO: setup test dependencies if needed. otherwise remove the TODO comments
yield
# TODO: clean up test dependencies
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#


Expand All @@ -10,5 +10,5 @@

@pytest.fixture(scope="session", autouse=True)
def connector_setup():
""" This fixture is a placeholder for external resources that acceptance test might require."""
"""This fixture is a placeholder for external resources that acceptance test might require."""
yield
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#


Expand All @@ -10,5 +10,5 @@

@pytest.fixture(scope="session", autouse=True)
def connector_setup():
""" This fixture is a placeholder for external resources that acceptance test might require."""
"""This fixture is a placeholder for external resources that acceptance test might require."""
yield
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#


Expand All @@ -10,7 +10,7 @@

@pytest.fixture(scope="session", autouse=True)
def connector_setup():
""" This fixture is a placeholder for external resources that acceptance test might require."""
"""This fixture is a placeholder for external resources that acceptance test might require."""
# TODO: setup test dependencies
yield
# TODO: clean up test dependencies
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#


Expand All @@ -10,5 +10,5 @@

@pytest.fixture(scope="session", autouse=True)
def connector_setup():
""" This fixture is a placeholder for external resources that acceptance test might require."""
"""This fixture is a placeholder for external resources that acceptance test might require."""
yield
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
/*
* Copyright (c) 2022 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.source.cockroachdb;

import com.fasterxml.jackson.databind.JsonNode;
Expand All @@ -14,7 +18,7 @@
public class CockroachJdbcSourceOperations extends JdbcSourceOperations {

@Override
protected void putBoolean(ObjectNode node, String columnName, ResultSet resultSet, int index) throws SQLException {
protected void putBoolean(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException {
if ("bit".equalsIgnoreCase(resultSet.getMetaData().getColumnTypeName(index))) {
node.put(columnName, resultSet.getByte(index));
} else {
Expand All @@ -28,12 +32,12 @@ protected void putDouble(final ObjectNode node, final String columnName, final R
}

@Override
protected void putNumber(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException {
protected void putBigDecimal(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException {
node.put(columnName, resultSet.getBigDecimal(index));
}

@Override
public JsonNode rowToJson(ResultSet queryContext) throws SQLException {
public JsonNode rowToJson(final ResultSet queryContext) throws SQLException {
final int columnCount = queryContext.getMetaData().getColumnCount();
final ObjectNode jsonNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap());

Expand All @@ -43,16 +47,16 @@ public JsonNode rowToJson(ResultSet queryContext) throws SQLException {
if (!queryContext.wasNull()) {
setJsonField(queryContext, i, jsonNode);
}
} catch (SQLException e) {
} catch (final SQLException e) {
putCockroachSpecialDataType(queryContext, i, jsonNode);
}
}
return jsonNode;
}

private void putCockroachSpecialDataType(ResultSet resultSet, int index, ObjectNode node) throws SQLException {
String columnType = resultSet.getMetaData().getColumnTypeName(index);
String columnName = resultSet.getMetaData().getColumnName(index);
private void putCockroachSpecialDataType(final ResultSet resultSet, final int index, final ObjectNode node) throws SQLException {
final String columnType = resultSet.getMetaData().getColumnTypeName(index);
final String columnName = resultSet.getMetaData().getColumnName(index);
try {
if ("numeric".equalsIgnoreCase(columnType)) {
final double value = resultSet.getDouble(index);
Expand All @@ -64,4 +68,5 @@ private void putCockroachSpecialDataType(ResultSet resultSet, int index, ObjectN
node.put(columnName, (Double) null);
}
}

}
Loading