From 56e764fb4b5818bba429a1e16fd0823d51fcc6da Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 7 Dec 2021 14:58:46 +0200 Subject: [PATCH 1/7] Added custom Exception which allow to report error details. Enhanced check connection method --- .../source_paypal_transaction/source.py | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 766be4255c8eb..8800454dadc50 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -4,6 +4,8 @@ import logging import time +import json +from json.decoder import JSONDecodeError from abc import ABC from datetime import datetime, timedelta from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union @@ -16,6 +18,30 @@ from dateutil.parser import isoparse +class PaypalHttpException(Exception): + """Just for formatting the exception as Square""" + + def __init__(self, error: requests.exceptions.HTTPError): + self.error = error + + def __str__(self): + message = repr(self.error) + + if self.error.response.content: + content = self.error.response.content.decode() + try: + details = json.loads(content) + except json.decoder.JSONDecodeError as e: + details = content + + message = f"{message} Details: {details}" + + return message + + def __repr__(self): + return self.__str__() + + def get_endpoint(is_sandbox: bool = False) -> str: if is_sandbox: endpoint = "https://api-m.sandbox.paypal.com" @@ -227,6 +253,12 @@ def stream_slices( return slices + def _send_request(self, request: requests.PreparedRequest, request_kwargs: Mapping[str, Any]) -> requests.Response: + try: + return super()._send_request(request, request_kwargs) + except requests.exceptions.HTTPError as http_error: + raise PaypalHttpException(http_error) + class Transactions(PaypalTransactionStream): """List Paypal Transactions on a specific date range @@ -351,9 +383,24 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: # Try to initiate a stream and validate input date params try: + # validate input date ranges Transactions(authenticator=authenticator, **config).validate_input_dates() + + # validate if Paypal is able to extract data for given start_data + start_date = isoparse(config['start_date']) + end_date = start_date + timedelta(days=1) + stream_slice = { + "start_date": start_date.isoformat(), + "end_date": end_date.isoformat() + } + records = Transactions(authenticator=authenticator, **config).read_records(sync_mode='full', stream_slice=stream_slice) + list(records) + except Exception as e: - return False, e + if "Data for the given start date is not available" in repr(e): + return False, f"Data for the given start date ({config['start_date']}) is not available, please use more recent start date" + else: + return False, e return True, None From 51fedeb0dd1bc60dd786dac2850f59950493c21a Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 7 Dec 2021 17:51:01 +0200 Subject: [PATCH 2/7] fixed formatting issues --- .../source_paypal_transaction/source.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 8800454dadc50..ee87140232221 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -2,10 +2,9 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # +import json import logging import time -import json -from json.decoder import JSONDecodeError from abc import ABC from datetime import datetime, timedelta from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union @@ -31,7 +30,7 @@ def __str__(self): content = self.error.response.content.decode() try: details = json.loads(content) - except json.decoder.JSONDecodeError as e: + except json.decoder.JSONDecodeError: details = content message = f"{message} Details: {details}" @@ -387,13 +386,13 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: Transactions(authenticator=authenticator, **config).validate_input_dates() # validate if Paypal is able to extract data for given start_data - start_date = isoparse(config['start_date']) + start_date = isoparse(config["start_date"]) end_date = start_date + timedelta(days=1) stream_slice = { "start_date": start_date.isoformat(), - "end_date": end_date.isoformat() + "end_date": end_date.isoformat(), } - records = Transactions(authenticator=authenticator, **config).read_records(sync_mode='full', stream_slice=stream_slice) + records = Transactions(authenticator=authenticator, **config).read_records(sync_mode="full", stream_slice=stream_slice) list(records) except Exception as e: From 07bbdef5f5385599d8313502b11d04137930c9af Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Fri, 10 Dec 2021 08:32:08 +0200 Subject: [PATCH 3/7] minor changes --- .../source_paypal_transaction/source.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index ee87140232221..2d51f5fad82c8 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -10,6 +10,7 @@ from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union import requests +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream @@ -18,7 +19,7 @@ class PaypalHttpException(Exception): - """Just for formatting the exception as Square""" + """HTTPError Exception with detailed info""" def __init__(self, error: requests.exceptions.HTTPError): self.error = error @@ -385,15 +386,19 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: # validate input date ranges Transactions(authenticator=authenticator, **config).validate_input_dates() - # validate if Paypal is able to extract data for given start_data + # validate if Paypal API is able to extract data for given start_data start_date = isoparse(config["start_date"]) end_date = start_date + timedelta(days=1) stream_slice = { "start_date": start_date.isoformat(), "end_date": end_date.isoformat(), } - records = Transactions(authenticator=authenticator, **config).read_records(sync_mode="full", stream_slice=stream_slice) - list(records) + records = Transactions(authenticator=authenticator, **config).read_records( + sync_mode=SyncMode.full_refresh, + stream_slice=stream_slice + ) + # Try to read one value from records iterator + next(records, None) except Exception as e: if "Data for the given start date is not available" in repr(e): From bde034b8fac0aa09a7c9f07e86a1241a280d4482 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 16 Dec 2021 13:06:19 +0200 Subject: [PATCH 4/7] ... --- .../source_paypal_transaction/source.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 2d51f5fad82c8..e05f656f75b45 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -10,7 +10,6 @@ from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union import requests -from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream @@ -394,20 +393,18 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: "end_date": end_date.isoformat(), } records = Transactions(authenticator=authenticator, **config).read_records( - sync_mode=SyncMode.full_refresh, + sync_mode=None, stream_slice=stream_slice ) # Try to read one value from records iterator next(records, None) - + return True, None except Exception as e: if "Data for the given start date is not available" in repr(e): return False, f"Data for the given start date ({config['start_date']}) is not available, please use more recent start date" else: return False, e - return True, None - def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ :param config: A Mapping of the user input configuration as defined in the connector spec. From e56a1746b5a27c0de96c97504acfa65466b964cc Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 16 Dec 2021 14:37:03 +0200 Subject: [PATCH 5/7] edited README --- .../connectors/source-paypal-transaction/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/README.md b/airbyte-integrations/connectors/source-paypal-transaction/README.md index 98e55ff6675af..d4fa2d383d783 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/README.md +++ b/airbyte-integrations/connectors/source-paypal-transaction/README.md @@ -99,7 +99,8 @@ Customize `acceptance-test-config.yml` file to configure tests. See [Source Acce If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. To run your integration tests with acceptance tests, from the connector root, run ``` -python -m pytest integration_tests -p integration_tests.acceptance +docker build . --no-cache -t airbyte/source-paypal-transaction:dev \ +&& python -m pytest -p source_acceptance_test.plugin ``` To run your integration tests with docker From 3e5a92ccc9949fe3e9a0f9cd3820d3a5186b9b66 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 16 Dec 2021 17:05:52 +0200 Subject: [PATCH 6/7] bumped the version, added changelog --- .../d913b0f2-cc51-4e55-a44c-8ba1697b9239.json | 2 +- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- .../connectors/source-paypal-transaction/Dockerfile | 2 +- docs/integrations/sources/paypal-transaction.md | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d913b0f2-cc51-4e55-a44c-8ba1697b9239.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d913b0f2-cc51-4e55-a44c-8ba1697b9239.json index 558223e7b925f..84b99570642b4 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d913b0f2-cc51-4e55-a44c-8ba1697b9239.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d913b0f2-cc51-4e55-a44c-8ba1697b9239.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "d913b0f2-cc51-4e55-a44c-8ba1697b9239", "name": "Paypal Transaction", "dockerRepository": "airbyte/source-paypal-transaction", - "dockerImageTag": "0.1.2", + "dockerImageTag": "0.1.3", "documentationUrl": "https://docs.airbyte.io/integrations/sources/paypal-transaction", "icon": "paypal.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 31750399f70f2..9235f813eaa08 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -469,7 +469,7 @@ - name: Paypal Transaction sourceDefinitionId: d913b0f2-cc51-4e55-a44c-8ba1697b9239 dockerRepository: airbyte/source-paypal-transaction - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.io/integrations/sources/paypal-transaction icon: paypal.svg sourceType: api diff --git a/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile b/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile index ef14562d18733..d0f4b175fa160 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile +++ b/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/source-paypal-transaction diff --git a/docs/integrations/sources/paypal-transaction.md b/docs/integrations/sources/paypal-transaction.md index f9023b31105b5..c21d6cdaaaa46 100644 --- a/docs/integrations/sources/paypal-transaction.md +++ b/docs/integrations/sources/paypal-transaction.md @@ -57,6 +57,7 @@ Transactions sync is performed with default `stream_slice_period` = 1 day, it me | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.3 | 2021-12-16 | [8580](https://github.com/airbytehq/airbyte/pull/8580) | Added more logs during `check connection` stage | | 0.1.2 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | | 0.1.1 | 2021-08-03 | [5155](https://github.com/airbytehq/airbyte/pull/5155) | fix start\_date\_min limit | | 0.1.0 | 2021-06-10 | [4240](https://github.com/airbytehq/airbyte/pull/4240) | PayPal Transaction Search API | From 87ad2b010904d4df5578a26b34795e5ef04243eb Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 16 Dec 2021 17:19:13 +0200 Subject: [PATCH 7/7] generated new specs --- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 8d81c1adf9c4f..149093a4ace98 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -3457,7 +3457,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-snowflake:0.3.20" +- dockerImage: "airbyte/destination-snowflake:0.3.21" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/snowflake" connectionSpecification: diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 62eb4f80348fe..56e1fef01dca2 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -4804,7 +4804,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-paypal-transaction:0.1.2" +- dockerImage: "airbyte/source-paypal-transaction:0.1.3" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/paypal-transactions" connectionSpecification: