From 86db8ff850ab615793494b20649445a2e67fe983 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 2 May 2022 10:02:00 -0700 Subject: [PATCH 01/12] read workflows --- .../connectors/source-hubspot/source_hubspot/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index 4a321404950af..0a9da5766fee1 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -52,7 +52,7 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> error_msg = None common_params = self.get_common_params(config=config) try: - contacts = Contacts(**common_params) + contacts = Workflows(**common_params) _ = contacts.properties except HTTPError as error: alive = False From bab930de63fb4bfbbc7ce85a098c0c5fbc615d9a Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 2 May 2022 10:13:49 -0700 Subject: [PATCH 02/12] try except --- .../source-hubspot/source_hubspot/streams.py | 61 +++++++++++-------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py index 57221ba6d0dfc..78a36731cdc8e 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py @@ -331,32 +331,41 @@ def read_records( pagination_complete = False next_page_token = None - with AirbyteSentry.start_transaction("read_records", self.name), AirbyteSentry.start_transaction_span("read_records"): - while not pagination_complete: - - properties_list = list(self.properties.keys()) - if properties_list: - stream_records, response = self._read_stream_records( - properties_list=properties_list, - stream_slice=stream_slice, - stream_state=stream_state, - next_page_token=next_page_token, - ) - records = [value for key, value in stream_records.items()] - else: - response = self.handle_request(stream_slice=stream_slice, stream_state=stream_state, next_page_token=next_page_token) - records = self._transform(self.parse_response(response, stream_state=stream_state, stream_slice=stream_slice)) - - if self.filter_old_records: - records = self._filter_old_records(records) - yield from records - - next_page_token = self.next_page_token(response) - if not next_page_token: - pagination_complete = True - - # Always return an empty generator just in case no records were ever yielded - yield from [] + try: + with AirbyteSentry.start_transaction("read_records", self.name), AirbyteSentry.start_transaction_span("read_records"): + while not pagination_complete: + + properties_list = list(self.properties.keys()) + if properties_list: + stream_records, response = self._read_stream_records( + properties_list=properties_list, + stream_slice=stream_slice, + stream_state=stream_state, + next_page_token=next_page_token, + ) + records = [value for key, value in stream_records.items()] + else: + response = self.handle_request( + stream_slice=stream_slice, stream_state=stream_state, next_page_token=next_page_token + ) + records = self._transform(self.parse_response(response, stream_state=stream_state, stream_slice=stream_slice)) + + if self.filter_old_records: + records = self._filter_old_records(records) + yield from records + + next_page_token = self.next_page_token(response) + if not next_page_token: + pagination_complete = True + + # Always return an empty generator just in case no records were ever yielded + yield from [] + except requests.exceptions.HTTPError as e: + status_code = e.response.status_code + if status_code == 403: + raise RuntimeError("Invalid permissions. Please ensure the all scopes are authorized for.") + else: + raise e @staticmethod def _convert_datetime_to_string(dt: pendulum.datetime, declared_format: str = None) -> str: From 251f6615cf4c7a0e20ca7ace244b592c039f18c5 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 2 May 2022 10:14:35 -0700 Subject: [PATCH 03/12] Revert "read workflows" This reverts commit 86db8ff850ab615793494b20649445a2e67fe983. --- .../connectors/source-hubspot/source_hubspot/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index 0a9da5766fee1..4a321404950af 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -52,7 +52,7 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> error_msg = None common_params = self.get_common_params(config=config) try: - contacts = Workflows(**common_params) + contacts = Contacts(**common_params) _ = contacts.properties except HTTPError as error: alive = False From 01b8510aa47b3d1e3507f351d022c91c68d292fa Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 2 May 2022 10:29:46 -0700 Subject: [PATCH 04/12] Check campaigns --- .../connectors/source-hubspot/source_hubspot/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index 4a321404950af..5dfcaccc79c82 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -52,7 +52,7 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> error_msg = None common_params = self.get_common_params(config=config) try: - contacts = Contacts(**common_params) + contacts = Campaigns(**common_params) _ = contacts.properties except HTTPError as error: alive = False From 5d556b7d0e47ad2ca18115d190a7ff0330c55fa7 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 2 May 2022 11:10:53 -0700 Subject: [PATCH 05/12] Check by reading data --- .../connectors/source-hubspot/source_hubspot/source.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index 5dfcaccc79c82..8ffab802b78b2 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -6,7 +6,7 @@ import logging from typing import Any, Iterator, List, Mapping, MutableMapping, Optional, Tuple -from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog +from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.deprecated.base_source import ConfiguredAirbyteStream from airbyte_cdk.sources.streams import Stream @@ -48,17 +48,17 @@ class SourceHubspot(AbstractSource): def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: """Check connection""" - alive = True - error_msg = None + common_params = self.get_common_params(config=config) try: contacts = Campaigns(**common_params) - _ = contacts.properties + next(contacts.read_records(sync_mode=SyncMode.full_refresh)) + return True, None except HTTPError as error: alive = False error_msg = repr(error) - return alive, error_msg + return alive, error_msg @staticmethod def get_api(config: Mapping[str, Any]) -> API: From ddd3c27b402742e68cb1a4c91a4ca131fc7ab806 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 2 May 2022 11:21:31 -0700 Subject: [PATCH 06/12] Check all streams --- .../source-hubspot/source_hubspot/source.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index 8ffab802b78b2..e450728e69697 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -48,17 +48,14 @@ class SourceHubspot(AbstractSource): def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: """Check connection""" - - common_params = self.get_common_params(config=config) try: - contacts = Campaigns(**common_params) - next(contacts.read_records(sync_mode=SyncMode.full_refresh)) - return True, None + for stream in self.streams(config=config): + next(stream.read_records(sync_mode=SyncMode.full_refresh)) except HTTPError as error: - alive = False error_msg = repr(error) - return alive, error_msg + return False, error_msg + return True, None @staticmethod def get_api(config: Mapping[str, Any]) -> API: From 6aa75b48b36ee5ef4eb313c8b7b68702dec2cea2 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 2 May 2022 11:26:26 -0700 Subject: [PATCH 07/12] requests_mock.ANY --- .../connectors/source-hubspot/unit_tests/test_source.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py index 54b1722420f33..5000512df5e0d 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py @@ -119,7 +119,7 @@ def test_check_connection_backoff_on_limit_reached(requests_mock, config): {"json": [], "status_code": 200}, ] - requests_mock.register_uri("GET", "/properties/v2/contact/properties", responses) + requests_mock.register_uri("GET", requests_mock.ANY, responses) source = SourceHubspot() alive, error = source.check_connection(logger=logger, config=config) @@ -133,7 +133,7 @@ def test_check_connection_backoff_on_server_error(requests_mock, config): {"json": {"error": "something bad"}, "status_code": 500}, {"json": [], "status_code": 200}, ] - requests_mock.register_uri("GET", "/properties/v2/contact/properties", responses) + requests_mock.register_uri("GET", requests_mock.ANY, responses) source = SourceHubspot() alive, error = source.check_connection(logger=logger, config=config) @@ -360,7 +360,7 @@ def test_search_based_stream_should_not_attempt_to_get_more_than_10k_records(req "results": [{"id": f"{y}", "updatedAt": "2022-02-25T16:43:11Z"} for y in range(100)], "paging": { "next": { - "after": f"{x*100}", + "after": f"{x * 100}", } }, }, @@ -376,7 +376,7 @@ def test_search_based_stream_should_not_attempt_to_get_more_than_10k_records(req "results": [{"id": f"{y}", "updatedAt": "2022-03-01T00:00:00Z"} for y in range(100)], "paging": { "next": { - "after": f"{x*100}", + "after": f"{x * 100}", } }, }, From 2d24b8842ef608f7903d99deae784f7f67494ee8 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 2 May 2022 13:31:27 -0700 Subject: [PATCH 08/12] mock all http methods --- .../connectors/source-hubspot/unit_tests/test_source.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py index 5000512df5e0d..852f11e84a9e1 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py @@ -40,7 +40,7 @@ def test_check_connection_ok(requests_mock, config): {"json": [], "status_code": 200}, ] - requests_mock.register_uri("GET", "/properties/v2/contact/properties", responses) + requests_mock.register_uri(test_check_connection_ok, test_check_connection_ok, responses) ok, error_msg = SourceHubspot().check_connection(logger, config=config) assert ok @@ -119,7 +119,7 @@ def test_check_connection_backoff_on_limit_reached(requests_mock, config): {"json": [], "status_code": 200}, ] - requests_mock.register_uri("GET", requests_mock.ANY, responses) + requests_mock.register_uri(requests_mock.ANY, requests_mock.ANY, responses) source = SourceHubspot() alive, error = source.check_connection(logger=logger, config=config) @@ -133,7 +133,7 @@ def test_check_connection_backoff_on_server_error(requests_mock, config): {"json": {"error": "something bad"}, "status_code": 500}, {"json": [], "status_code": 200}, ] - requests_mock.register_uri("GET", requests_mock.ANY, responses) + requests_mock.register_uri(requests_mock.ANY, requests_mock.ANY, responses) source = SourceHubspot() alive, error = source.check_connection(logger=logger, config=config) From 7a4fcbf6deb4d3606b8da8973b0a88e33daf4149 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 2 May 2022 15:29:59 -0700 Subject: [PATCH 09/12] Try with get and post --- .../connectors/source-hubspot/unit_tests/test_source.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py index 852f11e84a9e1..b2d55f2a98651 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py @@ -133,7 +133,8 @@ def test_check_connection_backoff_on_server_error(requests_mock, config): {"json": {"error": "something bad"}, "status_code": 500}, {"json": [], "status_code": 200}, ] - requests_mock.register_uri(requests_mock.ANY, requests_mock.ANY, responses) + requests_mock.register_uri("GET", requests_mock.ANY, responses) + requests_mock.register_uri("POST", requests_mock.ANY, responses) source = SourceHubspot() alive, error = source.check_connection(logger=logger, config=config) From fe97450e4d6c76e8b48ec80579e4ffb3c179a427 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 2 May 2022 17:08:01 -0700 Subject: [PATCH 10/12] reset to master --- .../source-hubspot/source_hubspot/source.py | 13 ++++++++----- .../source-hubspot/unit_tests/test_source.py | 11 +++++------ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index e450728e69697..4a321404950af 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -6,7 +6,7 @@ import logging from typing import Any, Iterator, List, Mapping, MutableMapping, Optional, Tuple -from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, SyncMode +from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.deprecated.base_source import ConfiguredAirbyteStream from airbyte_cdk.sources.streams import Stream @@ -48,14 +48,17 @@ class SourceHubspot(AbstractSource): def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: """Check connection""" + alive = True + error_msg = None + common_params = self.get_common_params(config=config) try: - for stream in self.streams(config=config): - next(stream.read_records(sync_mode=SyncMode.full_refresh)) + contacts = Contacts(**common_params) + _ = contacts.properties except HTTPError as error: + alive = False error_msg = repr(error) - return False, error_msg - return True, None + return alive, error_msg @staticmethod def get_api(config: Mapping[str, Any]) -> API: diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py index b2d55f2a98651..54b1722420f33 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py @@ -40,7 +40,7 @@ def test_check_connection_ok(requests_mock, config): {"json": [], "status_code": 200}, ] - requests_mock.register_uri(test_check_connection_ok, test_check_connection_ok, responses) + requests_mock.register_uri("GET", "/properties/v2/contact/properties", responses) ok, error_msg = SourceHubspot().check_connection(logger, config=config) assert ok @@ -119,7 +119,7 @@ def test_check_connection_backoff_on_limit_reached(requests_mock, config): {"json": [], "status_code": 200}, ] - requests_mock.register_uri(requests_mock.ANY, requests_mock.ANY, responses) + requests_mock.register_uri("GET", "/properties/v2/contact/properties", responses) source = SourceHubspot() alive, error = source.check_connection(logger=logger, config=config) @@ -133,8 +133,7 @@ def test_check_connection_backoff_on_server_error(requests_mock, config): {"json": {"error": "something bad"}, "status_code": 500}, {"json": [], "status_code": 200}, ] - requests_mock.register_uri("GET", requests_mock.ANY, responses) - requests_mock.register_uri("POST", requests_mock.ANY, responses) + requests_mock.register_uri("GET", "/properties/v2/contact/properties", responses) source = SourceHubspot() alive, error = source.check_connection(logger=logger, config=config) @@ -361,7 +360,7 @@ def test_search_based_stream_should_not_attempt_to_get_more_than_10k_records(req "results": [{"id": f"{y}", "updatedAt": "2022-02-25T16:43:11Z"} for y in range(100)], "paging": { "next": { - "after": f"{x * 100}", + "after": f"{x*100}", } }, }, @@ -377,7 +376,7 @@ def test_search_based_stream_should_not_attempt_to_get_more_than_10k_records(req "results": [{"id": f"{y}", "updatedAt": "2022-03-01T00:00:00Z"} for y in range(100)], "paging": { "next": { - "after": f"{x * 100}", + "after": f"{x*100}", } }, }, From 2ebcfb4f19b20d34dc5539eb01927c841fe74d01 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 2 May 2022 17:10:42 -0700 Subject: [PATCH 11/12] log name --- .../connectors/source-hubspot/source_hubspot/streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py index 78a36731cdc8e..764618dd22987 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py @@ -363,7 +363,7 @@ def read_records( except requests.exceptions.HTTPError as e: status_code = e.response.status_code if status_code == 403: - raise RuntimeError("Invalid permissions. Please ensure the all scopes are authorized for.") + raise RuntimeError(f"Invalid permissions for {self.name}. Please ensure the all scopes are authorized for.") else: raise e From fbd9d92ad866b40f28e1bfaf34d4de97bc5aacc4 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 2 May 2022 17:17:34 -0700 Subject: [PATCH 12/12] bump version --- .../connectors/source-hubspot/Dockerfile | 2 +- docs/integrations/sources/hubspot.md | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile index 504b0f5d2164d..c00e724d30857 100644 --- a/airbyte-integrations/connectors/source-hubspot/Dockerfile +++ b/airbyte-integrations/connectors/source-hubspot/Dockerfile @@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.55 +LABEL io.airbyte.version=0.1.56 LABEL io.airbyte.name=airbyte/source-hubspot diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md index 353218e67d614..b4e38c7966a65 100644 --- a/docs/integrations/sources/hubspot.md +++ b/docs/integrations/sources/hubspot.md @@ -147,12 +147,13 @@ If you are using OAuth, most of the streams require the appropriate [scopes](htt | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------| -| 0.1.55 | 2022-04-28 | [12424](https://github.com/airbytehq/airbyte/pull/12424) | Correct schema for ticket_pipeline stream | -| 0.1.54 | 2022-04-28 | [12335](https://github.com/airbytehq/airbyte/pull/12335) | Mock time slep in unit test s | -| 0.1.53 | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230) | chaneg spec json to yaml format | +| 0.1.56 | 2022-05-02 | [12515](https://github.com/airbytehq/airbyte/pull/12515) | Extra logs for troubleshooting 403 errors | +| 0.1.55 | 2022-04-28 | [12424](https://github.com/airbytehq/airbyte/pull/12424) | Correct schema for ticket_pipeline stream | +| 0.1.54 | 2022-04-28 | [12335](https://github.com/airbytehq/airbyte/pull/12335) | Mock time slep in unit test s | +| 0.1.53 | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230) | chaneg spec json to yaml format | | 0.1.52 | 2022-03-25 | [11423](https://github.com/airbytehq/airbyte/pull/11423) | Add tickets associations to engagements streams | -| 0.1.51 | 2022-03-24 | [11321](https://github.com/airbytehq/airbyte/pull/11321) | Fix updated at field non exists issue | -| 0.1.50 | 2022-03-22 | [11266](https://github.com/airbytehq/airbyte/pull/11266) | Fix Engagements Stream Pagination | +| 0.1.51 | 2022-03-24 | [11321](https://github.com/airbytehq/airbyte/pull/11321) | Fix updated at field non exists issue | +| 0.1.50 | 2022-03-22 | [11266](https://github.com/airbytehq/airbyte/pull/11266) | Fix Engagements Stream Pagination | | 0.1.49 | 2022-03-17 | [11218](https://github.com/airbytehq/airbyte/pull/11218) | Anchor hyperlink in input configuration | | 0.1.48 | 2022-03-16 | [11105](https://github.com/airbytehq/airbyte/pull/11105) | Fix float numbers, upd docs | | 0.1.47 | 2022-03-15 | [11121](https://github.com/airbytehq/airbyte/pull/11121) | Add partition keys where appropriate |