From 36f842b2912157863d047421d45f78f4fb623462 Mon Sep 17 00:00:00 2001 From: TymoshokDmytro Date: Fri, 9 Jul 2021 09:55:13 +0300 Subject: [PATCH 1/7] Modify _send_request in square-source.py --- .../connectors/source-square/source_square/source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-square/source_square/source.py b/airbyte-integrations/connectors/source-square/source_square/source.py index 53ec57864b4f7..2f98217f2caee 100644 --- a/airbyte-integrations/connectors/source-square/source_square/source.py +++ b/airbyte-integrations/connectors/source-square/source_square/source.py @@ -87,9 +87,9 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp records = json_response.get(self.data_field, []) if self.data_field is not None else json_response yield from records - def _send_request(self, request: requests.PreparedRequest) -> requests.Response: + def _send_request(self, request: requests.PreparedRequest, request_kwargs: Mapping[str, Any]) -> requests.Response: try: - return super()._send_request(request) + return super()._send_request(request, request_kwargs) except requests.exceptions.HTTPError as e: square_exception = parse_square_error_response(e) if square_exception: From 8510f8494caee4af128b3d0e99a0c68bfc65d81d Mon Sep 17 00:00:00 2001 From: TymoshokDmytro Date: Fri, 9 Jul 2021 15:26:27 +0300 Subject: [PATCH 2/7] Bump new version 0.1.1 with updates --- .../connectors/source-square/Dockerfile | 2 +- .../connectors/source-square/source_square/source.py | 12 +++++++----- docs/integrations/sources/square.md | 3 ++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-square/Dockerfile b/airbyte-integrations/connectors/source-square/Dockerfile index eb8116a457895..3da622a68275b 100644 --- a/airbyte-integrations/connectors/source-square/Dockerfile +++ b/airbyte-integrations/connectors/source-square/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-square diff --git a/airbyte-integrations/connectors/source-square/source_square/source.py b/airbyte-integrations/connectors/source-square/source_square/source.py index 2f98217f2caee..e0215fbed04b2 100644 --- a/airbyte-integrations/connectors/source-square/source_square/source.py +++ b/airbyte-integrations/connectors/source-square/source_square/source.py @@ -94,10 +94,6 @@ def _send_request(self, request: requests.PreparedRequest, request_kwargs: Mappi square_exception = parse_square_error_response(e) if square_exception: self.logger.error(str(square_exception)) - # Exiting is made for not to have a huge traceback in the airbyte log. - # The explicit square error message already been out with the command above. - exit(1) - raise e @@ -367,12 +363,18 @@ def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: "No locations found. Orders cannot be extracted without locations. " "Check https://developer.squareup.com/explorer/square/locations-api/list-locations" ) - exit(1) + return [None] separated_locations = separate_items_by_count(location_ids, self.locations_per_requets) for location in separated_locations: yield {"location_ids": location} + def read_records(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + if not stream_slice: + yield from [] + + yield from super().read_records(stream_slice=stream_slice, **kwargs) + class SourceSquare(AbstractSource): api_version = "2021-06-16" # Latest Stable Release diff --git a/docs/integrations/sources/square.md b/docs/integrations/sources/square.md index ae51d12d19284..43410a66250cb 100644 --- a/docs/integrations/sources/square.md +++ b/docs/integrations/sources/square.md @@ -79,4 +79,5 @@ Some Square API endpoints has different page size limitation | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | -| 0.1.0 | 2021-06-30 | [4439](https://github.com/airbytehq/airbyte/pull/4439) | Initial release supporting the Square API | \ No newline at end of file +| 0.1.0 | 2021-06-30 | [4439](https://github.com/airbytehq/airbyte/pull/4439) | Initial release supporting the Square API | +| 0.1.1 | 2021-07-09 | [4645](https://github.com/airbytehq/airbyte/pull/4645) | Update _send_request method due to Airbyte CDK changes | From 67aa7adb4ce0eb10fbfd78dcd2399322d4ec4006 Mon Sep 17 00:00:00 2001 From: TymoshokDmytro Date: Fri, 9 Jul 2021 15:58:37 +0300 Subject: [PATCH 3/7] Bump version of square connector to 0.1.1 in airbyte-config --- .../77225a51-cd15-4a13-af02-65816bd0ecf4.json | 2 +- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/77225a51-cd15-4a13-af02-65816bd0ecf4.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/77225a51-cd15-4a13-af02-65816bd0ecf4.json index d10a442f9c048..494d089c3e157 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/77225a51-cd15-4a13-af02-65816bd0ecf4.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/77225a51-cd15-4a13-af02-65816bd0ecf4.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "77225a51-cd15-4a13-af02-65816bd0ecf4", "name": "Square", "dockerRepository": "airbyte/source-square", - "dockerImageTag": "0.1.0", + "dockerImageTag": "0.1.1", "documentationUrl": "https://docs.airbyte.io/integrations/sources/square" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 0ab6f14de2e0b..ba62225a1f76e 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -356,7 +356,7 @@ - sourceDefinitionId: 77225a51-cd15-4a13-af02-65816bd0ecf4 name: Square dockerRepository: airbyte/source-square - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/square - sourceDefinitionId: 325e0640-e7b3-4e24-b823-3361008f603f name: Zendesk Sunshine From 9448927d348746507e5037fe8a16957619df82f9 Mon Sep 17 00:00:00 2001 From: TymoshokDmytro Date: Mon, 12 Jul 2021 13:13:15 +0300 Subject: [PATCH 4/7] Resolve code review qustions --- .../connectors/source-square/source_square/source.py | 8 +------- docs/integrations/sources/square.md | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/source-square/source_square/source.py b/airbyte-integrations/connectors/source-square/source_square/source.py index e0215fbed04b2..2810735fd3d08 100644 --- a/airbyte-integrations/connectors/source-square/source_square/source.py +++ b/airbyte-integrations/connectors/source-square/source_square/source.py @@ -363,18 +363,12 @@ def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: "No locations found. Orders cannot be extracted without locations. " "Check https://developer.squareup.com/explorer/square/locations-api/list-locations" ) - return [None] + yield from [] separated_locations = separate_items_by_count(location_ids, self.locations_per_requets) for location in separated_locations: yield {"location_ids": location} - def read_records(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: - if not stream_slice: - yield from [] - - yield from super().read_records(stream_slice=stream_slice, **kwargs) - class SourceSquare(AbstractSource): api_version = "2021-06-16" # Latest Stable Release diff --git a/docs/integrations/sources/square.md b/docs/integrations/sources/square.md index 43410a66250cb..6bef3160a5f76 100644 --- a/docs/integrations/sources/square.md +++ b/docs/integrations/sources/square.md @@ -79,5 +79,5 @@ Some Square API endpoints has different page size limitation | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | -| 0.1.0 | 2021-06-30 | [4439](https://github.com/airbytehq/airbyte/pull/4439) | Initial release supporting the Square API | | 0.1.1 | 2021-07-09 | [4645](https://github.com/airbytehq/airbyte/pull/4645) | Update _send_request method due to Airbyte CDK changes | +| 0.1.0 | 2021-06-30 | [4439](https://github.com/airbytehq/airbyte/pull/4439) | Initial release supporting the Square API | From 48cbef8dc7bf8289b78cc982d8f8cb5dc42a86de Mon Sep 17 00:00:00 2001 From: TymoshokDmytro Date: Mon, 12 Jul 2021 15:17:12 +0300 Subject: [PATCH 5/7] Add retry strategy for TeamMemberWages --- .../connectors/source-square/source_square/source.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-square/source_square/source.py b/airbyte-integrations/connectors/source-square/source_square/source.py index 2810735fd3d08..c931cb8c2e64d 100644 --- a/airbyte-integrations/connectors/source-square/source_square/source.py +++ b/airbyte-integrations/connectors/source-square/source_square/source.py @@ -23,6 +23,7 @@ # import json +import sys from abc import ABC, abstractmethod from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple @@ -94,7 +95,7 @@ def _send_request(self, request: requests.PreparedRequest, request_kwargs: Mappi square_exception = parse_square_error_response(e) if square_exception: self.logger.error(str(square_exception)) - raise e + raise type(e)(f"{square_exception}, {e}").with_traceback(sys.exc_info()[2]) # Some streams require next_page_token in request query parameters (TeamMemberWages, Customers) @@ -306,6 +307,14 @@ def request_params(self, **kwargs) -> MutableMapping[str, Any]: params_payload["limit"] = self.items_per_page_limit return params_payload + # This stream is tricky because once in a while it returns 404 error 'Not Found for url'. + # Thus the retry strategy was implemented. + def should_retry(self, response: requests.Response) -> bool: + return response.status_code == 404 or super().should_retry(response) + + def backoff_time(self, response: requests.Response) -> Optional[float]: + return 3 + class Customers(SquareStreamPageParam): """ Docs: https://developer.squareup.com/reference/square_2021-06-16/customers-api/list-customers """ From 5a3733d20ccb2ff63b99559499092d720e82d591 Mon Sep 17 00:00:00 2001 From: TymoshokDmytro Date: Mon, 12 Jul 2021 22:49:49 +0300 Subject: [PATCH 6/7] Remove complex exception handling --- .../connectors/source-square/source_square/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-square/source_square/source.py b/airbyte-integrations/connectors/source-square/source_square/source.py index c931cb8c2e64d..8d4e34655f024 100644 --- a/airbyte-integrations/connectors/source-square/source_square/source.py +++ b/airbyte-integrations/connectors/source-square/source_square/source.py @@ -95,7 +95,7 @@ def _send_request(self, request: requests.PreparedRequest, request_kwargs: Mappi square_exception = parse_square_error_response(e) if square_exception: self.logger.error(str(square_exception)) - raise type(e)(f"{square_exception}, {e}").with_traceback(sys.exc_info()[2]) + raise e # Some streams require next_page_token in request query parameters (TeamMemberWages, Customers) From 4e46f54b2d58fa9b871eb0e9efa938c952538d86 Mon Sep 17 00:00:00 2001 From: TymoshokDmytro Date: Mon, 12 Jul 2021 23:20:19 +0300 Subject: [PATCH 7/7] Remove unused import --- .../connectors/source-square/source_square/source.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-square/source_square/source.py b/airbyte-integrations/connectors/source-square/source_square/source.py index 8d4e34655f024..912d13d564c22 100644 --- a/airbyte-integrations/connectors/source-square/source_square/source.py +++ b/airbyte-integrations/connectors/source-square/source_square/source.py @@ -23,7 +23,6 @@ # import json -import sys from abc import ABC, abstractmethod from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple