From 781381b9915e3025846ea64d53aae773c354d052 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Mon, 11 Mar 2024 07:31:27 +0200 Subject: [PATCH 01/12] sourcte hubspot: add integration tests --- .../connectors/source-hubspot/poetry.lock | 30 +- .../connectors/source-hubspot/pyproject.toml | 2 + .../source-hubspot/source_hubspot/source.py | 5 +- .../source-hubspot/source_hubspot/streams.py | 2 +- .../source-hubspot/unit_tests/conftest.py | 7 +- .../unit_tests/integrations/__init__.py | 0 .../unit_tests/integrations/config_builder.py | 19 + .../integrations/request_builders/__init__.py | 7 + .../integrations/request_builders/other.py | 63 ++ .../request_builders/web_analytics.py | 93 +++ .../integrations/response_builder/__init__.py | 9 + .../integrations/response_builder/helpers.py | 28 + .../integrations/response_builder/other.py | 14 + .../response_builder/pagination.py | 18 + .../response_builder/web_analytics.py | 28 + .../integrations/test_engagements_calls.py | 9 + .../integrations/test_owners_archived.py | 9 + .../test_web_analytics_streams.py | 578 ++++++++++++++++++ .../unit_tests/resource/__init__.py | 0 .../unit_tests/resource/http/__init__.py | 0 .../resource/http/response/__init__.py | 0 .../resource/http/response/companies.json | 15 + .../response/companies_web_analytics.json | 57 ++ .../resource/http/response/contacts.json | 15 + .../http/response/contacts_web_analytics.json | 57 ++ .../resource/http/response/deals.json | 17 + .../http/response/deals_web_analytics.json | 57 ++ .../http/response/engagements_calls.json | 18 + .../engagements_calls_web_analytics.json | 57 ++ .../http/response/engagements_emails.json | 18 + .../engagements_emails_web_analytics.json | 57 ++ .../http/response/engagements_meetings.json | 18 + .../engagements_meetings_web_analytics.json | 57 ++ .../http/response/engagements_notes.json | 18 + .../engagements_notes_web_analytics.json | 57 ++ .../http/response/engagements_tasks.json | 18 + .../engagements_tasks_web_analytics.json | 57 ++ .../resource/http/response/goals.json | 14 + .../http/response/goals_web_analytics.json | 57 ++ .../resource/http/response/line_items.json | 14 + .../response/line_items_web_analytics.json | 57 ++ .../resource/http/response/products.json | 14 + .../http/response/products_web_analytics.json | 57 ++ .../resource/http/response/properties.json | 34 ++ .../resource/http/response/tickets.json | 14 + .../http/response/tickets_web_analytics.json | 57 ++ 46 files changed, 1826 insertions(+), 6 deletions(-) create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/__init__.py create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/config_builder.py create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/__init__.py create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/other.py create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/web_analytics.py create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/__init__.py create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/helpers.py create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/other.py create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/web_analytics.py create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/__init__.py create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/__init__.py create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/__init__.py create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/companies.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/companies_web_analytics.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts_web_analytics.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/deals.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/deals_web_analytics.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_calls.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_calls_web_analytics.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_emails.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_emails_web_analytics.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_meetings.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_meetings_web_analytics.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_notes.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_notes_web_analytics.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_tasks.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_tasks_web_analytics.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/goals.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/goals_web_analytics.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/line_items.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/line_items_web_analytics.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/products.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/products_web_analytics.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/properties.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/tickets.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/tickets_web_analytics.json diff --git a/airbyte-integrations/connectors/source-hubspot/poetry.lock b/airbyte-integrations/connectors/source-hubspot/poetry.lock index 10e4cc8d33a89..5f9e1cdf49c1f 100644 --- a/airbyte-integrations/connectors/source-hubspot/poetry.lock +++ b/airbyte-integrations/connectors/source-hubspot/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "airbyte-cdk" @@ -301,6 +301,21 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "freezegun" +version = "0.3.4" +description = "Let your Python tests travel through time" +optional = false +python-versions = "*" +files = [ + {file = "freezegun-0.3.4-py2.py3-none-any.whl", hash = "sha256:d15d5daa22260891955d436899f94c8b80525daa895aec74c0afa5a25ac0230e"}, + {file = "freezegun-0.3.4.tar.gz", hash = "sha256:8d5eb5656c324125cce80e2e9ae572af6da997b7065b3bb6599c20f1b28dcf46"}, +] + +[package.dependencies] +python-dateutil = ">=1.0,<2.0 || >2.0" +six = "*" + [[package]] name = "genson" version = "1.2.2" @@ -730,6 +745,17 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "pytz" +version = "2024.1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, +] + [[package]] name = "pytzdata" version = "2020.1" @@ -1047,4 +1073,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9,<3.12" -content-hash = "c2ede1134d353ed454678bde83d4114935f614a63f4e086bb3df790798d9fb4e" +content-hash = "3bcf37ae2e3aa7ce4696200427d493f1d804b19b68d0d2045d59662411ce0083" diff --git a/airbyte-integrations/connectors/source-hubspot/pyproject.toml b/airbyte-integrations/connectors/source-hubspot/pyproject.toml index 6d37d0ae1b2c6..82b0817a81c3a 100644 --- a/airbyte-integrations/connectors/source-hubspot/pyproject.toml +++ b/airbyte-integrations/connectors/source-hubspot/pyproject.toml @@ -27,3 +27,5 @@ requests-mock = "^1.9.3" mock = "^5.1.0" pytest-mock = "^3.6" pytest = "^6.2" +pytz = "2024.1" +freezegun = "0.3.4" diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index baa562164e875..5be5e2395b465 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -195,11 +195,12 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: self.logger.info("No scopes to grant when authenticating with API key.") available_streams = streams - available_streams.extend(self.get_custom_object_streams(api=api, common_params=common_params)) + custom_object_streams = list(self.get_custom_object_streams(api=api, common_params=common_params)) + available_streams.extend(custom_object_streams) if enable_experimental_streams: custom_objects_web_analytics_streams = self.get_web_analytics_custom_objects_stream( - custom_object_stream_instances=self.get_custom_object_streams(api=api, common_params=common_params), + custom_object_stream_instances=custom_object_streams, common_params=common_params, ) available_streams.extend(custom_objects_web_analytics_streams) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py index 7445af401cd19..8778e9f57f90b 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py @@ -780,7 +780,7 @@ def _get_field_props(field_type: str) -> Mapping[str, List[str]]: @property @lru_cache() def properties(self) -> Mapping[str, Any]: - """Some entities has dynamic set of properties, so we trying to resolve those at runtime""" + """Some entities have dynamic set of properties, so we're trying to resolve those at runtime""" props = {} if not self.entity: return props diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py index ea8070af9d879..af7f3f76bca08 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py @@ -7,7 +7,7 @@ from source_hubspot.streams import API NUMBER_OF_PROPERTIES = 2000 - +from airbyte_cdk.test.mock_http import HttpMocker @pytest.fixture(name="oauth_config") def oauth_config_fixture(): @@ -85,3 +85,8 @@ def fake_properties_list(): @pytest.fixture(name="api") def api(some_credentials): return API(some_credentials) + + +@pytest.fixture +def http_mocker(): + return None diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/__init__.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/config_builder.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/config_builder.py new file mode 100644 index 0000000000000..6bc3093014f22 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/config_builder.py @@ -0,0 +1,19 @@ +from typing import Any, Mapping + + +class ConfigBuilder: + def __init__(self): + self._config = { + "enable_experimental_streams": True + } + + def with_start_date(self, start_date: str): + self._config["start_date"] = start_date + return self + + def with_auth(self, credentials: Mapping[str, str]): + self._config["credentials"] = credentials + return self + + def build(self) -> Mapping[str, Any]: + return self._config diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/__init__.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/__init__.py new file mode 100644 index 0000000000000..1c24a3d65bf37 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/__init__.py @@ -0,0 +1,7 @@ +import abc + + +class AbstractRequestBuilder: + @abc.abstractmethod + def build(self): + pass diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/other.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/other.py new file mode 100644 index 0000000000000..9b906b82976af --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/other.py @@ -0,0 +1,63 @@ +from airbyte_cdk.test.mock_http import HttpRequest +from . import AbstractRequestBuilder + + +class OAuthRequestBuilder(AbstractRequestBuilder): + URL = "https://api.hubapi.com/oauth/v1/token" + + def __init__(self): + self._params = {} + + def with_client_id(self, client_id: str): + self._params["client_id"] = client_id + return self + + def with_client_secret(self, client_secret: str): + self._params["client_secret"] = client_secret + return self + + def with_refresh_token(self, refresh_token: str): + self._params["refresh_token"] = refresh_token + return self + + def build(self) -> HttpRequest: + client_id, client_secret, refresh_token = self._params["client_id"], self._params["client_secret"], self._params["refresh_token"] + return HttpRequest( + url=self.URL, + body=f"grant_type=refresh_token&client_id={client_id}&client_secret={client_secret}&refresh_token={refresh_token}" + ) + + +class ScopesRequestBuilder(AbstractRequestBuilder): + URL = "https://api.hubapi.com/oauth/v1/access-tokens/{token}" + + def __init__(self): + self._token = None + + def with_access_token(self, token: str): + self._token = token + return self + + def build(self) -> HttpRequest: + return HttpRequest(url=self.URL.format(token=self._token)) + + +class CustomObjectsRequestBuilder(AbstractRequestBuilder): + URL = "https://api.hubapi.com/crm/v3/schemas" + + def build(self) -> HttpRequest: + return HttpRequest(url=self.URL) + + +class PropertiesRequestBuilder(AbstractRequestBuilder): + URL = "https://api.hubapi.com/properties/v2/{resource}/properties" + + def __init__(self): + self._resource = None + + def for_entity(self, entity): + self._resource = entity + return self + + def build(self) -> HttpRequest: + return HttpRequest(url=self.URL.format(resource=self._resource)) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/web_analytics.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/web_analytics.py new file mode 100644 index 0000000000000..08b1de8c6fc54 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/web_analytics.py @@ -0,0 +1,93 @@ +from typing import Any, Dict, Iterable, Tuple +from airbyte_cdk.test.mock_http import HttpRequest +from . import AbstractRequestBuilder + + +class WebAnalyticsRequestBuilder(AbstractRequestBuilder): + URL = "https://api.hubapi.com/events/v3/events" + + def __init__(self): + self._token = None + self._query_params = {} + self._request_body = None + self._headers = {} + + def with_token(self, token: str): + self._token = token + return self + + @property + def headers(self) -> Dict[str, Any]: + return {"Authorization": f"Bearer {self._token}"} + + def with_query(self, qp): + self._query_params = qp + return self + + def build(self) -> HttpRequest: + return HttpRequest( + url=self.URL, + query_params=self._query_params, + headers=self.headers, + body=self._request_body + ) + + +class CRMStreamRequestBuilder(AbstractRequestBuilder): + URL = "https://api.hubapi.com/crm/v3/objects/{resource}" + + def __init__(self): + self._resource = None + self._associations = "" + self._dt_range = "" + self._properties = "" + + def for_entity(self, entity): + self._resource = entity + return self + + def with_dt_range(self, start_date: Tuple, end_date: Tuple): + self._dt_range = "&".join(["{}={}".format(*start_date), "{}={}".format(*end_date)]) + return self + + def with_associations(self, associations: Iterable[str]): + self._associations = "&".join([f"associations={a}" for a in associations]) + return self + + def with_properties(self, properties: Iterable[str]): + self._properties = "properties=" + ",".join(properties) + return self + + @property + def _limit(self): + return "limit=100" + + @property + def _archived(self): + return "archived=false" + + @property + def _query_params(self): + return [ + self._archived, + self._associations, + self._limit, + self._dt_range, + self._properties + ] + + def build(self): + q = "&".join(filter(None, self._query_params)) + return HttpRequest(url=self.URL.format(resource=self._resource), query_params=q) + + +class IncrementalCRMStreamRequestBuilder(CRMStreamRequestBuilder): + @property + def _query_params(self): + return [ + self._limit, + self._dt_range, + self._archived, + self._associations, + self._properties + ] diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/__init__.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/__init__.py new file mode 100644 index 0000000000000..2a4c6acdd8272 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/__init__.py @@ -0,0 +1,9 @@ +import abc +from airbyte_cdk.test.mock_http import HttpResponse + + +class AbstractResponseBuilder: + @abc.abstractmethod + def build(self) -> HttpResponse: + pass + diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/helpers.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/helpers.py new file mode 100644 index 0000000000000..d43bf5849d06e --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/helpers.py @@ -0,0 +1,28 @@ +import json +from typing import Any, List, Optional, Union +from airbyte_cdk.test.mock_http import HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + PaginationStrategy +) + + +class RootHttpResponseBuilder(HttpResponseBuilder): + def __init__( + self, + template: List[Any], + records_path: Optional[Union[FieldPath, NestedPath]] = None, + pagination_strategy: Optional[PaginationStrategy] = None + ): + self._response = template + self._records: List[RecordBuilder] = [] + self._records_path = records_path + self._pagination_strategy = pagination_strategy + self._status_code = 200 + + def build(self) -> HttpResponse: + self._response.extend([record.build() for record in self._records]) + return HttpResponse(json.dumps(self._response), self._status_code) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/other.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/other.py new file mode 100644 index 0000000000000..8ca1e0bb4598e --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/other.py @@ -0,0 +1,14 @@ +import json +from typing import List + +from airbyte_cdk.test.mock_http import HttpResponse +from . import AbstractResponseBuilder + + +class ScopesAbstractResponseBuilder(AbstractResponseBuilder): + def __init__(self, scopes: List[str]): + self._scopes = scopes + + def build(self): + body = json.dumps({"scopes": self._scopes}) + return HttpResponse(body=body, status_code=200) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py new file mode 100644 index 0000000000000..8ceaa4ee77a8e --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py @@ -0,0 +1,18 @@ +from typing import Any, Dict +from airbyte_cdk.test.mock_http.response_builder import PaginationStrategy + + +class WebAnalyticsPaginationStrategy(PaginationStrategy): + NEXT_PAGE_TOKEN = {"after": "this_page_last_record_id"} + + def update(self, response: Dict[str, Any]) -> None: + response["paging"] = { + "next": { + "link": "link_to_the_next_page", + **self.NEXT_PAGE_TOKEN + }, + "prev": { + "before": None, + "link": None + } + } diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/web_analytics.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/web_analytics.py new file mode 100644 index 0000000000000..9f3871bcc96e7 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/web_analytics.py @@ -0,0 +1,28 @@ +import json +from airbyte_cdk.test.mock_http.response_builder import HttpResponseBuilder, find_template, FieldPath +from airbyte_cdk.test.mock_http import HttpResponse +from . import AbstractResponseBuilder +from .pagination import WebAnalyticsPaginationStrategy + + +class WebAnalyticsResponseBuilder(HttpResponseBuilder): + @property + def pagination_strategy(self): + return self._pagination_strategy + + @classmethod + def for_stream(cls, stream: str): + return cls(find_template(stream, __file__), FieldPath("results"), WebAnalyticsPaginationStrategy()) + + +class GenericAbstractResponseBuilder(AbstractResponseBuilder): + def __init__(self): + self._body = {} + + def with_value(self, key: str, value: str): + self._body[key] = value + return self + + def build(self): + body = json.dumps(self._body) + return HttpResponse(body, status_code=200) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py new file mode 100644 index 0000000000000..1e83df361dc0f --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py @@ -0,0 +1,9 @@ +from unittest import TestCase + + +class TestEngagementCallsStreamFullRefresh(TestCase): + pass + + +class TestEngagementCallsStreamIncremental(TestCase): + pass diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py new file mode 100644 index 0000000000000..2a67733e8dd82 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py @@ -0,0 +1,9 @@ +from unittest import TestCase + + +class TestOwnersArchivedStreamFullRefresh(TestCase): + pass + + +class TestOwnersArchivedStreamIncremental(TestCase): + pass diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py new file mode 100644 index 0000000000000..5dad2e53ac4aa --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py @@ -0,0 +1,578 @@ +import copy +import http +from typing import Any, List, Dict, Optional, Tuple + +import mock +import pytz +import pytest +import freezegun +from datetime import datetime, timedelta +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, + PaginationStrategy +) +from .config_builder import ConfigBuilder +from .response_builder.helpers import RootHttpResponseBuilder +from .response_builder.other import ScopesAbstractResponseBuilder +from .response_builder.web_analytics import WebAnalyticsResponseBuilder, GenericAbstractResponseBuilder +from .request_builders.web_analytics import WebAnalyticsRequestBuilder, IncrementalCRMStreamRequestBuilder, CRMStreamRequestBuilder +from .request_builders.other import OAuthRequestBuilder, CustomObjectsRequestBuilder, ScopesRequestBuilder, PropertiesRequestBuilder +from airbyte_protocol.models import AirbyteStateMessage, FailureType, SyncMode +from source_hubspot import SourceHubspot + + +CRM_STREAMS = ( + ("tickets_web_analytics", "tickets", "ticket", ["contacts", "deals", "companies"]), + ("deals_web_analytics", "deals", "deal", ["contacts", "companies", "line_items"]), + ("companies_web_analytics", "companies", "company", ["contacts"]), + ("contacts_web_analytics", "contacts", "contact", ["contacts", "companies"]), + ("engagements_calls_web_analytics", "engagements_calls", "calls", ["contacts", "deal", "company", "tickets"]), + ("engagements_emails_web_analytics", "engagements_emails", "emails", ["contacts", "deal", "company", "tickets"]), + ("engagements_meetings_web_analytics", "engagements_meetings", "meetings", ["contacts", "deal", "company", "tickets"]), + ("engagements_notes_web_analytics", "engagements_notes", "notes", ["contacts", "deal", "company", "tickets"]), + ("engagements_tasks_web_analytics", "engagements_tasks", "tasks", ["contacts", "deal", "company", "tickets"]), +) + +CRM_INCREMENTAL_STREAMS = ( + ("goals_web_analytics", "goals", "goal_targets", []), + ("line_items_web_analytics", "line_items", "line_item", []), + ("products_web_analytics", "products", "product", []), +) + + +@freezegun.freeze_time("2024-03-03T14:42:00Z") +class WebAnalytics: + DT_FORMAT = '%Y-%m-%dT%H:%M:%SZ' + + @classmethod + def now(cls): + return datetime.now(pytz.utc) + + @classmethod + def start_date(cls): + return cls.now() - timedelta(days=30) + + @classmethod + def updated_at(cls): + return cls.now() - timedelta(days=1) + + @classmethod + def dt_str(cls, dt: datetime.date) -> str: + return dt.strftime(cls.DT_FORMAT) + + @classmethod + def oauth_config(cls, start_date: Optional[str] = None) -> Dict[str, Any]: + start_date = start_date or cls.dt_str(cls.start_date()) + return ConfigBuilder().with_start_date(start_date).with_auth( + { + "credentials_title": "OAuth Credentials", + "redirect_uri": "https://airbyte.io", + "client_id": "client_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token", + } + ).build() + + @classmethod + def private_token_config(cls, token: str, start_date: Optional[str] = None) -> Dict[str, Any]: + start_date = start_date or cls.dt_str(cls.start_date()) + return ConfigBuilder().with_start_date(start_date).with_auth( + { + "credentials_title": "Private App Credentials", + "access_token": token, + } + ).build() + + @classmethod + def mock_oauth(cls, http_mocker: HttpMocker, token: str): + creds = cls.oauth_config()["credentials"] + req = OAuthRequestBuilder().with_client_id( + creds["client_id"] + ).with_client_secret( + creds["client_secret"] + ).with_refresh_token( + creds["refresh_token"] + ).build() + response = GenericAbstractResponseBuilder().with_value("access_token", token).with_value("expires_in", 7200).build() + http_mocker.post(req, response) + + @classmethod + def mock_scopes(cls, http_mocker: HttpMocker, token: str, scopes: List[str]): + http_mocker.get(ScopesRequestBuilder().with_access_token(token).build(), ScopesAbstractResponseBuilder(scopes).build()) + + @classmethod + def mock_custom_objects(cls, http_mocker: HttpMocker): + http_mocker.get( + CustomObjectsRequestBuilder().build(), + HttpResponseBuilder({}, records_path=FieldPath("results"), pagination_strategy=None).build() + ) + + @classmethod + def mock_properties(cls, http_mocker: HttpMocker, object_type: str, properties: Dict[str, str]): + templates = find_template(f"properties", __file__) + record_builder = lambda: RecordBuilder(copy.deepcopy(templates[0]), id_path=None, cursor_path=None) + + response_builder = RootHttpResponseBuilder(templates) + for name, type in properties.items(): + record = record_builder().with_field(FieldPath("name"), name).with_field(FieldPath("type"), type) + response_builder = response_builder.with_record(record) + + http_mocker.get( + PropertiesRequestBuilder().for_entity(object_type).build(), + response_builder.build() + ) + + @classmethod + def mock_parent_object( + cls, + http_mocker: HttpMocker, + object_ids: List[str], + object_type: str, + stream_name: str, + associations: List[str], + properties: List[str], + date_range: Optional[Tuple[str,...]] = None, + ): + response_builder = WebAnalyticsResponseBuilder.for_stream(stream_name) + for object_id in object_ids: + record = cls.record_builder(stream_name, FieldPath("updatedAt")).with_field( + FieldPath("updatedAt"), cls.dt_str(cls.updated_at()) + ).with_field( + FieldPath("id"), object_id + ) + response_builder = response_builder.with_record(record) + + http_mocker.get( + CRMStreamRequestBuilder().for_entity(object_type).with_associations(associations).with_properties(properties).build(), + response_builder.build() + ) + + @classmethod + def mock_response(cls, http_mocker: HttpMocker, request, responses): + if not isinstance(responses, (list, tuple)): + responses = [responses] + http_mocker.get(request, responses) + + @classmethod + def web_analytics_request( + cls, + stream: str, + token: str, + object_id: str, + object_type: str, + start_date: Optional[str] = None, + end_date: Optional[str] = None, + first_page: bool = True + ): + start_date = start_date or cls.dt_str(cls.start_date()) + end_date = end_date or cls.dt_str(cls.now()) + query = { + "limit": 100, + "occurredAfter": start_date, + "occurredBefore": end_date, + "objectId": object_id, + "objectType": object_type + } + + if not first_page: + response_builder = WebAnalyticsResponseBuilder.for_stream(stream) + query.update(response_builder.pagination_strategy.NEXT_PAGE_TOKEN) + return WebAnalyticsRequestBuilder().with_token(token).with_query(query).build() + + @classmethod + def web_analytics_response(cls, stream: str, with_pagination: bool = False, updated_on: Optional[str] = None) -> HttpResponse: + updated_on = updated_on or cls.dt_str(cls.updated_at()) + record = cls.record_builder(stream, FieldPath("occurredAt")).with_field(FieldPath("updatedAt"), updated_on) + response_builder = WebAnalyticsResponseBuilder.for_stream(stream) + response = response_builder.with_record(record) + if with_pagination: + response = response.with_pagination() + return response.build() + + @classmethod + def record_builder(cls, stream: str, record_cursor_path): + return create_record_builder( + find_template(stream, __file__), records_path=FieldPath("results"), record_id_path=None, record_cursor_path=record_cursor_path + ) + + @classmethod + def catalog(cls, stream: str, sync_mode: SyncMode): + return CatalogBuilder().with_stream(stream, sync_mode).build() + + @classmethod + def read_from_stream( + cls, cfg, stream: str, sync_mode: SyncMode, state: Optional[List[AirbyteStateMessage]] = None, expecting_exception: bool = False + ) -> EntrypointOutput: + return read(SourceHubspot(), cfg, cls.catalog(stream, sync_mode), state, expecting_exception) + + +@freezegun.freeze_time("2024-03-03T14:42:00Z") +class TestCRMWebAnalyticsStreamFullRefresh(WebAnalytics): + SCOPES = ["tickets", "crm.objects.contacts.read", "crm.objects.companies.read", "contacts", "crm.objects.deals.read", "oauth"] + OBJECT_ID = "testID" + ACCESS_TOKEN = "new_access_token" + PROPERTIES = { + "closed_date": "datetime", + "createdate": "datetime", + } + + @classmethod + def extended_dt_ranges(cls) -> Tuple[Tuple[str, ...], ...]: + return ( + (cls.dt_str(cls.now() - timedelta(days=60)), cls.dt_str(cls.now() - timedelta(days=30))), + (cls.dt_str(cls.now() - timedelta(days=30)), cls.dt_str(cls.now())), + ) + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) + @HttpMocker() + def test_given_one_page_when_read_stream_oauth_then_return_records( + self, stream_name, parent_stream_name, object_type, parent_stream_associations, http_mocker: HttpMocker + ): + self.mock_oauth(http_mocker, self.ACCESS_TOKEN) + self.mock_scopes(http_mocker, self.ACCESS_TOKEN, self.SCOPES) + self.mock_custom_objects(http_mocker) + self.mock_properties(http_mocker, object_type, self.PROPERTIES) + self.mock_parent_object( + http_mocker, [self.OBJECT_ID], object_type, parent_stream_name, parent_stream_associations, list(self.PROPERTIES.keys()) + ) + self.mock_response( + http_mocker, + self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), + self.web_analytics_response(stream_name) + ) + output = self.read_from_stream(self.oauth_config(), stream_name, SyncMode.full_refresh) + assert len(output.records) == 1 + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) + @HttpMocker() + def test_given_one_page_when_read_stream_private_token_then_return_records( + self, stream_name, parent_stream_name, object_type, parent_stream_associations, http_mocker: HttpMocker + ): + self.mock_custom_objects(http_mocker) + self.mock_properties(http_mocker, object_type, self.PROPERTIES) + self.mock_parent_object( + http_mocker, [self.OBJECT_ID], object_type, parent_stream_name, parent_stream_associations, list(self.PROPERTIES.keys()) + ) + self.mock_response( + http_mocker, + self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), + self.web_analytics_response(stream_name) + ) + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) + assert len(output.records) == 1 + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) + @HttpMocker() + def test_given_two_pages_when_read_then_return_records( + self, stream_name, parent_stream_name, object_type, parent_stream_associations, http_mocker: HttpMocker + ): + self.mock_custom_objects(http_mocker) + self.mock_properties(http_mocker, object_type, self.PROPERTIES) + self.mock_parent_object( + http_mocker, [self.OBJECT_ID], object_type, parent_stream_name, parent_stream_associations, list(self.PROPERTIES.keys()) + ) + self.mock_response( + http_mocker, + self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), + self.web_analytics_response(stream_name, with_pagination=True) + ) + self.mock_response( + http_mocker, + self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type, first_page=False), + self.web_analytics_response(stream_name) + ) + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) + assert len(output.records) == 2 + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) + @HttpMocker() + def test_given_wide_date_range_and_multiple_parent_records_when_read_then_return_records( + self, stream_name, parent_stream_name, object_type, parent_stream_associations, http_mocker: HttpMocker + ): + date_ranges = self.extended_dt_ranges() + self.mock_custom_objects(http_mocker) + self.mock_properties(http_mocker, object_type, self.PROPERTIES) + start_to_end = (date_ranges[0][0], date_ranges[-1][-1]) + self.mock_parent_object( + http_mocker, + [self.OBJECT_ID, "another_object_id"], + object_type, + parent_stream_name, + parent_stream_associations, + list(self.PROPERTIES.keys()), + start_to_end + ) + for dt_range in date_ranges: + for _id in (self.OBJECT_ID, "another_object_id"): + start, end = dt_range + web_analytics_response = self.web_analytics_response(stream_name) + self.mock_response( + http_mocker, + self.web_analytics_request(stream_name, self.ACCESS_TOKEN, _id, object_type, start, end), + web_analytics_response + ) + config_start_dt = date_ranges[0][0] + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN, config_start_dt), stream_name, SyncMode.full_refresh) + assert len(output.records) == 4 + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) + @HttpMocker() + def test_given_error_response_when_read_analytics_then_get_trace_message( + self, stream_name, parent_stream_name, object_type, parent_stream_associations, http_mocker: HttpMocker + ): + self.mock_custom_objects(http_mocker) + self.mock_properties(http_mocker, object_type, self.PROPERTIES) + self.mock_parent_object( + http_mocker, [self.OBJECT_ID], object_type, parent_stream_name, parent_stream_associations, list(self.PROPERTIES.keys()) + ) + self.mock_response( + http_mocker, + self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), + HttpResponse(status_code=500, body="{}") + ) + with mock.patch("time.sleep"): + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) + assert len(output.records) == 0 + assert len(output.trace_messages) > 0 + assert len(output.errors) > 0 + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) + @HttpMocker() + def test_given_500_then_200_when_read_then_return_records( + self, stream_name, parent_stream_name, object_type, parent_stream_associations, http_mocker: HttpMocker + ): + self.mock_custom_objects(http_mocker) + self.mock_properties(http_mocker, object_type, self.PROPERTIES) + self.mock_parent_object( + http_mocker, [self.OBJECT_ID], object_type, parent_stream_name, parent_stream_associations, list(self.PROPERTIES.keys()) + ) + self.mock_response( + http_mocker, + self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), + [ + HttpResponse(status_code=500, body="{}"), + self.web_analytics_response(stream_name) + ] + ) + with mock.patch("time.sleep"): + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) + assert len(output.records) == 1 + assert len(output.trace_messages) > 0 + assert len(output.errors) == 0 + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) + @HttpMocker() + def test_given_missing_scopes_error_when_read_then_hault( + self, + stream_name, + parent_stream_name, + object_type, + parent_stream_associations, + http_mocker: HttpMocker + ): + self.mock_oauth(http_mocker, self.ACCESS_TOKEN) + self.mock_scopes(http_mocker, self.ACCESS_TOKEN, []) + self.read_from_stream(self.oauth_config(), stream_name, SyncMode.full_refresh, expecting_exception=True) + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) + @HttpMocker() + def test_given_unauthorized_error_when_read_then_hault( + self, + stream_name, + parent_stream_name, + object_type, + parent_stream_associations, + http_mocker: HttpMocker + ): + self.mock_custom_objects(http_mocker) + self.mock_properties(http_mocker, object_type, self.PROPERTIES) + self.mock_parent_object( + http_mocker, [self.OBJECT_ID], object_type, parent_stream_name, parent_stream_associations, list(self.PROPERTIES.keys()) + ) + self.mock_response( + http_mocker, + self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), + HttpResponse(status_code=http.HTTPStatus.UNAUTHORIZED, body="{}") + ) + with mock.patch("time.sleep"): + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) + assert len(output.records) == 0 + assert len(output.trace_messages) > 0 + assert len(output.errors) > 0 + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) + @HttpMocker() + def test_given_one_page_when_read_then_get_transformed_records( + self, + stream_name, + parent_stream_name, + object_type, + parent_stream_associations, + http_mocker: HttpMocker + ): + self.mock_custom_objects(http_mocker) + self.mock_properties(http_mocker, object_type, self.PROPERTIES) + self.mock_parent_object( + http_mocker, [self.OBJECT_ID], object_type, parent_stream_name, parent_stream_associations, list(self.PROPERTIES.keys()) + ) + self.mock_response( + http_mocker, + self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), + self.web_analytics_response(stream_name) + ) + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) + record = output.records[0].record.data + assert "properties" not in record + prop_fields = len([f for f in record if f.startswith("properties_")]) + assert prop_fields > 0 + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) + @HttpMocker() + def test_given_one_page_when_read_then_get_no_records_filtered( + self, + stream_name, + parent_stream_name, + object_type, + parent_stream_associations, + http_mocker: HttpMocker + ): + self.mock_custom_objects(http_mocker) + self.mock_properties(http_mocker, object_type, self.PROPERTIES) + self.mock_parent_object( + http_mocker, [self.OBJECT_ID], object_type, parent_stream_name, parent_stream_associations, list(self.PROPERTIES.keys()) + ) + self.mock_response( + http_mocker, + self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), + self.web_analytics_response(stream_name, updated_on=self.dt_str(self.now() - timedelta(days=365))) + ) + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) + assert len(output.records) == 1 + + +@freezegun.freeze_time("2024-03-03T14:42:00Z") +class TestIncrementalCRMWebAnalyticsStreamFullRefresh(TestCRMWebAnalyticsStreamFullRefresh): + SCOPES = ["e-commerce", "oauth", "crm.objects.feedback_submissions.read", "crm.objects.goals.read"] + + @classmethod + def dt_conversion(cls, dt: str) -> str: + return str(int(datetime.strptime(dt, cls.DT_FORMAT).replace(tzinfo=pytz.utc).timestamp()) * 1000) + + @classmethod + def mock_parent_object( + cls, + http_mocker: HttpMocker, + object_ids: List[str], + object_type: str, + stream_name: str, + associations: List[str], + properties: List[str], + date_range: Optional[Tuple[str]] = None, + ): + date_range = date_range or (cls.dt_str(cls.start_date()), cls.dt_str(cls.now())) + response_builder = WebAnalyticsResponseBuilder.for_stream(stream_name) + for object_id in object_ids: + record = cls.record_builder(stream_name, FieldPath("updatedAt")).with_field( + FieldPath("updatedAt"), cls.dt_str(cls.updated_at()) + ).with_field( + FieldPath("id"), object_id + ) + response_builder = response_builder.with_record(record) + + start, end = date_range + http_mocker.get( + IncrementalCRMStreamRequestBuilder().for_entity(object_type).with_associations(associations).with_dt_range( + ("startTimestamp", cls.dt_conversion(start)), + ("endTimestamp", cls.dt_conversion(end)) + ).with_properties(properties).build(), + response_builder.build() + ) + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) + def test_given_one_page_when_read_stream_oauth_then_return_records( + self, stream_name, parent_stream_name, object_type, parent_stream_associations + ): + super().test_given_one_page_when_read_stream_oauth_then_return_records( + stream_name, parent_stream_name, object_type, parent_stream_associations + ) + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) + def test_given_one_page_when_read_stream_private_token_then_return_records( + self, stream_name, parent_stream_name, object_type, parent_stream_associations + ): + super().test_given_one_page_when_read_stream_private_token_then_return_records( + stream_name, parent_stream_name, object_type, parent_stream_associations + ) + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) + def test_given_two_pages_when_read_then_return_records( + self, stream_name, parent_stream_name, object_type, parent_stream_associations + ): + super().test_given_two_pages_when_read_then_return_records( + stream_name, parent_stream_name, object_type, parent_stream_associations + ) + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) + def test_given_wide_date_range_and_multiple_parent_records_when_read_then_return_records( + self, stream_name, parent_stream_name, object_type, parent_stream_associations + ): + super().test_given_wide_date_range_and_multiple_parent_records_when_read_then_return_records( + stream_name, parent_stream_name, object_type, parent_stream_associations + ) + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) + def test_given_error_response_when_read_analytics_then_get_trace_message( + self, stream_name, parent_stream_name, object_type, parent_stream_associations + ): + super().test_given_error_response_when_read_analytics_then_get_trace_message( + stream_name, parent_stream_name, object_type, parent_stream_associations + ) + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) + def test_given_500_then_200_when_read_then_return_records( + self, stream_name, parent_stream_name, object_type, parent_stream_associations + ): + super().test_given_500_then_200_when_read_then_return_records( + stream_name, parent_stream_name, object_type, parent_stream_associations + ) + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) + def test_given_missing_scopes_error_when_read_then_hault( + self, stream_name, parent_stream_name, object_type, parent_stream_associations + ): + super().test_given_missing_scopes_error_when_read_then_hault( + stream_name, parent_stream_name, object_type, parent_stream_associations + ) + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) + def test_given_unauthorized_error_when_read_then_hault( + self, stream_name, parent_stream_name, object_type, parent_stream_associations + ): + super().test_given_unauthorized_error_when_read_then_hault( + stream_name, parent_stream_name, object_type, parent_stream_associations + ) + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) + def test_given_one_page_when_read_then_get_transformed_records( + self, stream_name, parent_stream_name, object_type, parent_stream_associations + ): + super().test_given_one_page_when_read_then_get_transformed_records( + stream_name, parent_stream_name, object_type, parent_stream_associations + ) + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) + def test_given_one_page_when_read_then_get_no_records_filtered( + self, stream_name, parent_stream_name, object_type, parent_stream_associations + ): + super().test_given_one_page_when_read_then_get_no_records_filtered( + stream_name, parent_stream_name, object_type, parent_stream_associations + ) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/__init__.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/__init__.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/__init__.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/companies.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/companies.json new file mode 100644 index 0000000000000..f6d3182ab2dee --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/companies.json @@ -0,0 +1,15 @@ +{ + "results": [ + { + "id": "312929580", + "createdAt": "2021-02-23T20:08:49.603Z", + "updatedAt": "2021-02-23T20:08:53.371Z", + "archived": false, + "contacts": ["contact A"], + "properties": { + "closed_date": "2021-02-23T20:08:49.603000+00:00", + "createdate": "2021-02-23T20:08:49.603000+00:00" + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/companies_web_analytics.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/companies_web_analytics.json new file mode 100644 index 0000000000000..6e06874030fc4 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/companies_web_analytics.json @@ -0,0 +1,57 @@ +{ + "results": [ + { + "objectType": "COMPANY", + "objectId": "153", + "eventType": "pe8727216_airbyte_company_custom_event", + "occurredAt": "2023-12-01T21:50:11.799Z", + "id": "b850d903-254c-4df6-b159-9263b2b7eed21", + "properties": { + "hs_asset_description": "dolor architecto", + "hs_asset_type": "Donec scelerisque est sed libero tempor venenatis", + "hs_browser": "Donec lobortis nibh sit amet dictum cursus", + "hs_campaign_id": "illum quas dolor modi exercitationem", + "hs_city": "reiciendis placeat dolor placeat architecto dolor", + "hs_country": "Mauris sagittis sem eu neque pellentesque gravida", + "hs_hs_device_name": "elit. molestias, elit. amet", + "hs_device_type": "accusantium dolor sit elit. veniam reprehenderit", + "hs_title": "In egestas nisl commodo pellentesque facilisis", + "hs_form_correlation_id": "Pellentesque tincidunt purus id rhoncus blandit", + "hs_element_class": "Nulla luctus tortor at fermentum aliquet", + "hs_element_id": "culpa! sit ipsum sit Lorem consectetur quas odit", + "hs_element_text": "Hic molestias, Lorem ipsum, possimus adipisicing", + "hs_language": "officiis adipisicing", + "hs_document_id": "Proin sed lacus eget sapien viverra vestibulum vel sit amet massa", + "hs_presentation_id": "Mauris et nisl a metus porttitor molestie id viverra ex", + "hs_user_id": "Quisque consequat nunc at eleifend tempus", + "hs_link_href": "officiis exercitationem adipisicing odit dolor", + "hs_operating_system": "magnam, molestias,", + "hs_operating_version": "Lorem ipsum culpa! illum elit. esse esse officiis", + "hs_page_content_type": "elit. libero Lorem", + "hs_page_id": "magnam, magnam,", + "hs_page_title": "Praesent auctor sem et purus facilisis, at volutpat lorem tristique", + "hs_page_url": "accusantium quas architecto ipsum ipsum possimus", + "hs_parent_module_id": "Aliquam eleifend ex in ligula gravida mollis", + "hs_referrer": "Sed nec eros quis sem euismod tempor", + "hs_region": "consectetur ipsum, architecto ipsum Lorem nobis", + "hs_url": "Aenean feugiat quam in urna iaculis, vitae ultrices metus scelerisque", + "hs_screen_height": "Donec laoreet est ut lorem viverra tempus", + "hs_screen_width": "Curabitur sit amet augue luctus, congue erat congue, vestibulum lectus", + "hs_touchpoint_source": "dolor", + "hs_tracking_name": "quas ipsum amet illum molestias,", + "hs_user_agent": "Aliquam eget libero consectetur, consectetur sem ac, vehicula orci", + "hs_utm_campaign": "magnam, Lorem modi culpa!", + "hs_utm_content": "Proin pulvinar nulla sed lacus venenatis blandit", + "hs_utm_medium": "placeat dolor dolor consectetur elit.", + "hs_utm_source": "Duis non erat at justo euismod lobortis", + "hs_utm_term": "Proin sit amet leo tincidunt, ultrices dolor ac, hendrerit mi", + "hs_base_url": "Maecenas ut risus eget ligula pharetra pellentesque sed et mi", + "hs_form_id": "Quisque cursus sem sit amet libero feugiat rhoncus", + "hs_form_type": "Pellentesque tristique velit at velit lacinia vulputate", + "hs_url_domain": "Quisque rhoncus diam at ex eleifend aliquet", + "hs_url_path": "Suspendisse blandit sem in consectetur imperdiet", + "hs_visitor_type": "Nunc porttitor metus a mauris commodo, id dictum lectus feugiat" + } + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts.json new file mode 100644 index 0000000000000..c844ac35e2bba --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts.json @@ -0,0 +1,15 @@ +{ + "results": [ + { + "id": "312929580", + "createdAt": "2021-02-23T20:08:49.603Z", + "updatedAt": "2021-02-23T20:08:53.371Z", + "archived": false, + "companies": ["5000526215", "5000526215"], + "properties": { + "closed_date": "2021-02-23T20:08:49.603000+00:00", + "createdate": "2021-02-23T20:08:49.603000+00:00" + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts_web_analytics.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts_web_analytics.json new file mode 100644 index 0000000000000..147c3bd5002e4 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts_web_analytics.json @@ -0,0 +1,57 @@ +{ + "results": [ + { + "objectType": "CONTACT", + "objectId": "153", + "eventType": "pe8727216_airbyte_contact_custom_event", + "occurredAt": "2023-12-01T21:50:11.801Z", + "id": "b850d903-254c-4df6-b159-9263b2b7eed8", + "properties": { + "hs_asset_description": "dolor architecto", + "hs_asset_type": "Donec scelerisque est sed libero tempor venenatis", + "hs_browser": "Donec lobortis nibh sit amet dictum cursus", + "hs_campaign_id": "illum quas dolor modi exercitationem", + "hs_city": "reiciendis placeat dolor placeat architecto dolor", + "hs_country": "Mauris sagittis sem eu neque pellentesque gravida", + "hs_hs_device_name": "elit. molestias, elit. amet", + "hs_device_type": "accusantium dolor sit elit. veniam reprehenderit", + "hs_title": "In egestas nisl commodo pellentesque facilisis", + "hs_form_correlation_id": "Pellentesque tincidunt purus id rhoncus blandit", + "hs_element_class": "Nulla luctus tortor at fermentum aliquet", + "hs_element_id": "culpa! sit ipsum sit Lorem consectetur quas odit", + "hs_element_text": "Hic molestias, Lorem ipsum, possimus adipisicing", + "hs_language": "officiis adipisicing", + "hs_document_id": "Proin sed lacus eget sapien viverra vestibulum vel sit amet massa", + "hs_presentation_id": "Mauris et nisl a metus porttitor molestie id viverra ex", + "hs_user_id": "Quisque consequat nunc at eleifend tempus", + "hs_link_href": "officiis exercitationem adipisicing odit dolor", + "hs_operating_system": "magnam, molestias,", + "hs_operating_version": "Lorem ipsum culpa! illum elit. esse esse officiis", + "hs_page_content_type": "elit. libero Lorem", + "hs_page_id": "magnam, magnam,", + "hs_page_title": "Praesent auctor sem et purus facilisis, at volutpat lorem tristique", + "hs_page_url": "accusantium quas architecto ipsum ipsum possimus", + "hs_parent_module_id": "Aliquam eleifend ex in ligula gravida mollis", + "hs_referrer": "Sed nec eros quis sem euismod tempor", + "hs_region": "consectetur ipsum, architecto ipsum Lorem nobis", + "hs_url": "Aenean feugiat quam in urna iaculis, vitae ultrices metus scelerisque", + "hs_screen_height": "Donec laoreet est ut lorem viverra tempus", + "hs_screen_width": "Curabitur sit amet augue luctus, congue erat congue, vestibulum lectus", + "hs_touchpoint_source": "dolor", + "hs_tracking_name": "quas ipsum amet illum molestias,", + "hs_user_agent": "Aliquam eget libero consectetur, consectetur sem ac, vehicula orci", + "hs_utm_campaign": "magnam, Lorem modi culpa!", + "hs_utm_content": "Proin pulvinar nulla sed lacus venenatis blandit", + "hs_utm_medium": "placeat dolor dolor consectetur elit.", + "hs_utm_source": "Duis non erat at justo euismod lobortis", + "hs_utm_term": "Proin sit amet leo tincidunt, ultrices dolor ac, hendrerit mi", + "hs_base_url": "Maecenas ut risus eget ligula pharetra pellentesque sed et mi", + "hs_form_id": "Quisque cursus sem sit amet libero feugiat rhoncus", + "hs_form_type": "Pellentesque tristique velit at velit lacinia vulputate", + "hs_url_domain": "Quisque rhoncus diam at ex eleifend aliquet", + "hs_url_path": "Suspendisse blandit sem in consectetur imperdiet", + "hs_visitor_type": "Nunc porttitor metus a mauris commodo, id dictum lectus feugiat" + } + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/deals.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/deals.json new file mode 100644 index 0000000000000..5cc06608a709c --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/deals.json @@ -0,0 +1,17 @@ +{ + "results": [ + { + "id": "312929580", + "createdAt": "2021-02-23T20:08:49.603Z", + "updatedAt": "2021-02-23T20:08:53.371Z", + "archived": false, + "companies": ["company A"], + "contacts": ["contact A"], + "line_items": ["line item A"], + "properties": { + "closed_date": "2021-02-23T20:08:49.603000+00:00", + "createdate": "2021-02-23T20:08:49.603000+00:00" + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/deals_web_analytics.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/deals_web_analytics.json new file mode 100644 index 0000000000000..2249ba47c4b84 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/deals_web_analytics.json @@ -0,0 +1,57 @@ +{ + "results": [ + { + "objectType": "DEAL", + "objectId": "152", + "eventType": "pe8727216_airbyte_deal_custom_event", + "occurredAt": "2023-12-01T21:50:11.798Z", + "id": "b850d903-254c-4df6-b159-9263b2b7eed1", + "properties": { + "hs_asset_description": "dolor architecto", + "hs_asset_type": "Donec scelerisque est sed libero tempor venenatis", + "hs_browser": "Donec lobortis nibh sit amet dictum cursus", + "hs_campaign_id": "illum quas dolor modi exercitationem", + "hs_city": "reiciendis placeat dolor placeat architecto dolor", + "hs_country": "Mauris sagittis sem eu neque pellentesque gravida", + "hs_hs_device_name": "elit. molestias, elit. amet", + "hs_device_type": "accusantium dolor sit elit. veniam reprehenderit", + "hs_title": "In egestas nisl commodo pellentesque facilisis", + "hs_form_correlation_id": "Pellentesque tincidunt purus id rhoncus blandit", + "hs_element_class": "Nulla luctus tortor at fermentum aliquet", + "hs_element_id": "culpa! sit ipsum sit Lorem consectetur quas odit", + "hs_element_text": "Hic molestias, Lorem ipsum, possimus adipisicing", + "hs_language": "officiis adipisicing", + "hs_document_id": "Proin sed lacus eget sapien viverra vestibulum vel sit amet massa", + "hs_presentation_id": "Mauris et nisl a metus porttitor molestie id viverra ex", + "hs_user_id": "Quisque consequat nunc at eleifend tempus", + "hs_link_href": "officiis exercitationem adipisicing odit dolor", + "hs_operating_system": "magnam, molestias,", + "hs_operating_version": "Lorem ipsum culpa! illum elit. esse esse officiis", + "hs_page_content_type": "elit. libero Lorem", + "hs_page_id": "magnam, magnam,", + "hs_page_title": "Praesent auctor sem et purus facilisis, at volutpat lorem tristique", + "hs_page_url": "accusantium quas architecto ipsum ipsum possimus", + "hs_parent_module_id": "Aliquam eleifend ex in ligula gravida mollis", + "hs_referrer": "Sed nec eros quis sem euismod tempor", + "hs_region": "consectetur ipsum, architecto ipsum Lorem nobis", + "hs_url": "Aenean feugiat quam in urna iaculis, vitae ultrices metus scelerisque", + "hs_screen_height": "Donec laoreet est ut lorem viverra tempus", + "hs_screen_width": "Curabitur sit amet augue luctus, congue erat congue, vestibulum lectus", + "hs_touchpoint_source": "dolor", + "hs_tracking_name": "quas ipsum amet illum molestias,", + "hs_user_agent": "Aliquam eget libero consectetur, consectetur sem ac, vehicula orci", + "hs_utm_campaign": "magnam, Lorem modi culpa!", + "hs_utm_content": "Proin pulvinar nulla sed lacus venenatis blandit", + "hs_utm_medium": "placeat dolor dolor consectetur elit.", + "hs_utm_source": "Duis non erat at justo euismod lobortis", + "hs_utm_term": "Proin sit amet leo tincidunt, ultrices dolor ac, hendrerit mi", + "hs_base_url": "Maecenas ut risus eget ligula pharetra pellentesque sed et mi", + "hs_form_id": "Quisque cursus sem sit amet libero feugiat rhoncus", + "hs_form_type": "Pellentesque tristique velit at velit lacinia vulputate", + "hs_url_domain": "Quisque rhoncus diam at ex eleifend aliquet", + "hs_url_path": "Suspendisse blandit sem in consectetur imperdiet", + "hs_visitor_type": "Nunc porttitor metus a mauris commodo, id dictum lectus feugiat" + } + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_calls.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_calls.json new file mode 100644 index 0000000000000..65a6634ab5eb0 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_calls.json @@ -0,0 +1,18 @@ +{ + "results": [ + { + "id": "312929680", + "createdAt": "2021-02-23T20:08:49.603Z", + "updatedAt": "2021-02-23T20:08:53.371Z", + "archived": false, + "companies": ["company A"], + "contacts": ["contact A"], + "deals": ["deal A"], + "tickets": ["ticket A"], + "properties": { + "closed_date": "2021-02-23T20:08:49.603000+00:00", + "createdate": "2021-02-23T20:08:49.603000+00:00" + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_calls_web_analytics.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_calls_web_analytics.json new file mode 100644 index 0000000000000..d5839a708aa43 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_calls_web_analytics.json @@ -0,0 +1,57 @@ +{ + "results": [ + { + "objectType": "ENGAGEMENT_CALL", + "objectId": "133", + "eventType": "pe8727216_airbyte_engagement_call_custom_event", + "occurredAt": "2023-12-01T21:50:11.001Z", + "id": "b850d903-254c-4df6-b159-9263b2b6eed8", + "properties": { + "hs_asset_description": "dolor architecto", + "hs_asset_type": "Donec scelerisque est sed libero tempor venenatis", + "hs_browser": "Donec lobortis nibh sit amet dictum cursus", + "hs_campaign_id": "illum quas dolor modi exercitationem", + "hs_city": "reiciendis placeat dolor placeat architecto dolor", + "hs_country": "Mauris sagittis sem eu neque pellentesque gravida", + "hs_hs_device_name": "elit. molestias, elit. amet", + "hs_device_type": "accusantium dolor sit elit. veniam reprehenderit", + "hs_title": "In egestas nisl commodo pellentesque facilisis", + "hs_form_correlation_id": "Pellentesque tincidunt purus id rhoncus blandit", + "hs_element_class": "Nulla luctus tortor at fermentum aliquet", + "hs_element_id": "culpa! sit ipsum sit Lorem consectetur quas odit", + "hs_element_text": "Hic molestias, Lorem ipsum, possimus adipisicing", + "hs_language": "officiis adipisicing", + "hs_document_id": "Proin sed lacus eget sapien viverra vestibulum vel sit amet massa", + "hs_presentation_id": "Mauris et nisl a metus porttitor molestie id viverra ex", + "hs_user_id": "Quisque consequat nunc at eleifend tempus", + "hs_link_href": "officiis exercitationem adipisicing odit dolor", + "hs_operating_system": "magnam, molestias,", + "hs_operating_version": "Lorem ipsum culpa! illum elit. esse esse officiis", + "hs_page_content_type": "elit. libero Lorem", + "hs_page_id": "magnam, magnam,", + "hs_page_title": "Praesent auctor sem et purus facilisis, at volutpat lorem tristique", + "hs_page_url": "accusantium quas architecto ipsum ipsum possimus", + "hs_parent_module_id": "Aliquam eleifend ex in ligula gravida mollis", + "hs_referrer": "Sed nec eros quis sem euismod tempor", + "hs_region": "consectetur ipsum, architecto ipsum Lorem nobis", + "hs_url": "Aenean feugiat quam in urna iaculis, vitae ultrices metus scelerisque", + "hs_screen_height": "Donec laoreet est ut lorem viverra tempus", + "hs_screen_width": "Curabitur sit amet augue luctus, congue erat congue, vestibulum lectus", + "hs_touchpoint_source": "dolor", + "hs_tracking_name": "quas ipsum amet illum molestias,", + "hs_user_agent": "Aliquam eget libero consectetur, consectetur sem ac, vehicula orci", + "hs_utm_campaign": "magnam, Lorem modi culpa!", + "hs_utm_content": "Proin pulvinar nulla sed lacus venenatis blandit", + "hs_utm_medium": "placeat dolor dolor consectetur elit.", + "hs_utm_source": "Duis non erat at justo euismod lobortis", + "hs_utm_term": "Proin sit amet leo tincidunt, ultrices dolor ac, hendrerit mi", + "hs_base_url": "Maecenas ut risus eget ligula pharetra pellentesque sed et mi", + "hs_form_id": "Quisque cursus sem sit amet libero feugiat rhoncus", + "hs_form_type": "Pellentesque tristique velit at velit lacinia vulputate", + "hs_url_domain": "Quisque rhoncus diam at ex eleifend aliquet", + "hs_url_path": "Suspendisse blandit sem in consectetur imperdiet", + "hs_visitor_type": "Nunc porttitor metus a mauris commodo, id dictum lectus feugiat" + } + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_emails.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_emails.json new file mode 100644 index 0000000000000..805867d39e403 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_emails.json @@ -0,0 +1,18 @@ +{ + "results": [ + { + "id": "312929681", + "createdAt": "2021-02-23T20:08:49.603Z", + "updatedAt": "2021-02-23T20:08:53.371Z", + "archived": false, + "companies": ["company A"], + "contacts": ["contact A"], + "deals": ["deal A"], + "tickets": ["ticket A"], + "properties": { + "closed_date": "2021-02-23T20:08:49.603000+00:00", + "createdate": "2021-02-23T20:08:49.603000+00:00" + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_emails_web_analytics.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_emails_web_analytics.json new file mode 100644 index 0000000000000..2926d84896e09 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_emails_web_analytics.json @@ -0,0 +1,57 @@ +{ + "results": [ + { + "objectType": "ENGAGEMENT_EMAIL", + "objectId": "134", + "eventType": "pe8727216_airbyte_engagement_email_custom_event", + "occurredAt": "2023-12-01T21:50:11.002Z", + "id": "b850d903-254c-4df6-b159-9263b2b6eed9", + "properties": { + "hs_asset_description": "dolor architecto", + "hs_asset_type": "Donec scelerisque est sed libero tempor venenatis", + "hs_browser": "Donec lobortis nibh sit amet dictum cursus", + "hs_campaign_id": "illum quas dolor modi exercitationem", + "hs_city": "reiciendis placeat dolor placeat architecto dolor", + "hs_country": "Mauris sagittis sem eu neque pellentesque gravida", + "hs_hs_device_name": "elit. molestias, elit. amet", + "hs_device_type": "accusantium dolor sit elit. veniam reprehenderit", + "hs_title": "In egestas nisl commodo pellentesque facilisis", + "hs_form_correlation_id": "Pellentesque tincidunt purus id rhoncus blandit", + "hs_element_class": "Nulla luctus tortor at fermentum aliquet", + "hs_element_id": "culpa! sit ipsum sit Lorem consectetur quas odit", + "hs_element_text": "Hic molestias, Lorem ipsum, possimus adipisicing", + "hs_language": "officiis adipisicing", + "hs_document_id": "Proin sed lacus eget sapien viverra vestibulum vel sit amet massa", + "hs_presentation_id": "Mauris et nisl a metus porttitor molestie id viverra ex", + "hs_user_id": "Quisque consequat nunc at eleifend tempus", + "hs_link_href": "officiis exercitationem adipisicing odit dolor", + "hs_operating_system": "magnam, molestias,", + "hs_operating_version": "Lorem ipsum culpa! illum elit. esse esse officiis", + "hs_page_content_type": "elit. libero Lorem", + "hs_page_id": "magnam, magnam,", + "hs_page_title": "Praesent auctor sem et purus facilisis, at volutpat lorem tristique", + "hs_page_url": "accusantium quas architecto ipsum ipsum possimus", + "hs_parent_module_id": "Aliquam eleifend ex in ligula gravida mollis", + "hs_referrer": "Sed nec eros quis sem euismod tempor", + "hs_region": "consectetur ipsum, architecto ipsum Lorem nobis", + "hs_url": "Aenean feugiat quam in urna iaculis, vitae ultrices metus scelerisque", + "hs_screen_height": "Donec laoreet est ut lorem viverra tempus", + "hs_screen_width": "Curabitur sit amet augue luctus, congue erat congue, vestibulum lectus", + "hs_touchpoint_source": "dolor", + "hs_tracking_name": "quas ipsum amet illum molestias,", + "hs_user_agent": "Aliquam eget libero consectetur, consectetur sem ac, vehicula orci", + "hs_utm_campaign": "magnam, Lorem modi culpa!", + "hs_utm_content": "Proin pulvinar nulla sed lacus venenatis blandit", + "hs_utm_medium": "placeat dolor dolor consectetur elit.", + "hs_utm_source": "Duis non erat at justo euismod lobortis", + "hs_utm_term": "Proin sit amet leo tincidunt, ultrices dolor ac, hendrerit mi", + "hs_base_url": "Maecenas ut risus eget ligula pharetra pellentesque sed et mi", + "hs_form_id": "Quisque cursus sem sit amet libero feugiat rhoncus", + "hs_form_type": "Pellentesque tristique velit at velit lacinia vulputate", + "hs_url_domain": "Quisque rhoncus diam at ex eleifend aliquet", + "hs_url_path": "Suspendisse blandit sem in consectetur imperdiet", + "hs_visitor_type": "Nunc porttitor metus a mauris commodo, id dictum lectus feugiat" + } + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_meetings.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_meetings.json new file mode 100644 index 0000000000000..2d34bfa5b7b80 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_meetings.json @@ -0,0 +1,18 @@ +{ + "results": [ + { + "id": "312929682", + "createdAt": "2021-02-23T20:08:49.603Z", + "updatedAt": "2021-02-23T20:08:53.371Z", + "archived": false, + "companies": ["company A"], + "contacts": ["contact A"], + "deals": ["deal A"], + "tickets": ["ticket A"], + "properties": { + "closed_date": "2021-02-23T20:08:49.603000+00:00", + "createdate": "2021-02-23T20:08:49.603000+00:00" + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_meetings_web_analytics.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_meetings_web_analytics.json new file mode 100644 index 0000000000000..d7540506411b6 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_meetings_web_analytics.json @@ -0,0 +1,57 @@ +{ + "results": [ + { + "objectType": "ENGAGEMENT_MEETING", + "objectId": "135", + "eventType": "pe8727216_airbyte_engagement_meeting_custom_event", + "occurredAt": "2023-12-01T21:50:12.001Z", + "id": "b850d903-254c-4df6-b159-9263c2b6eed8", + "properties": { + "hs_asset_description": "dolor architecto", + "hs_asset_type": "Donec scelerisque est sed libero tempor venenatis", + "hs_browser": "Donec lobortis nibh sit amet dictum cursus", + "hs_campaign_id": "illum quas dolor modi exercitationem", + "hs_city": "reiciendis placeat dolor placeat architecto dolor", + "hs_country": "Mauris sagittis sem eu neque pellentesque gravida", + "hs_hs_device_name": "elit. molestias, elit. amet", + "hs_device_type": "accusantium dolor sit elit. veniam reprehenderit", + "hs_title": "In egestas nisl commodo pellentesque facilisis", + "hs_form_correlation_id": "Pellentesque tincidunt purus id rhoncus blandit", + "hs_element_class": "Nulla luctus tortor at fermentum aliquet", + "hs_element_id": "culpa! sit ipsum sit Lorem consectetur quas odit", + "hs_element_text": "Hic molestias, Lorem ipsum, possimus adipisicing", + "hs_language": "officiis adipisicing", + "hs_document_id": "Proin sed lacus eget sapien viverra vestibulum vel sit amet massa", + "hs_presentation_id": "Mauris et nisl a metus porttitor molestie id viverra ex", + "hs_user_id": "Quisque consequat nunc at eleifend tempus", + "hs_link_href": "officiis exercitationem adipisicing odit dolor", + "hs_operating_system": "magnam, molestias,", + "hs_operating_version": "Lorem ipsum culpa! illum elit. esse esse officiis", + "hs_page_content_type": "elit. libero Lorem", + "hs_page_id": "magnam, magnam,", + "hs_page_title": "Praesent auctor sem et purus facilisis, at volutpat lorem tristique", + "hs_page_url": "accusantium quas architecto ipsum ipsum possimus", + "hs_parent_module_id": "Aliquam eleifend ex in ligula gravida mollis", + "hs_referrer": "Sed nec eros quis sem euismod tempor", + "hs_region": "consectetur ipsum, architecto ipsum Lorem nobis", + "hs_url": "Aenean feugiat quam in urna iaculis, vitae ultrices metus scelerisque", + "hs_screen_height": "Donec laoreet est ut lorem viverra tempus", + "hs_screen_width": "Curabitur sit amet augue luctus, congue erat congue, vestibulum lectus", + "hs_touchpoint_source": "dolor", + "hs_tracking_name": "quas ipsum amet illum molestias,", + "hs_user_agent": "Aliquam eget libero consectetur, consectetur sem ac, vehicula orci", + "hs_utm_campaign": "magnam, Lorem modi culpa!", + "hs_utm_content": "Proin pulvinar nulla sed lacus venenatis blandit", + "hs_utm_medium": "placeat dolor dolor consectetur elit.", + "hs_utm_source": "Duis non erat at justo euismod lobortis", + "hs_utm_term": "Proin sit amet leo tincidunt, ultrices dolor ac, hendrerit mi", + "hs_base_url": "Maecenas ut risus eget ligula pharetra pellentesque sed et mi", + "hs_form_id": "Quisque cursus sem sit amet libero feugiat rhoncus", + "hs_form_type": "Pellentesque tristique velit at velit lacinia vulputate", + "hs_url_domain": "Quisque rhoncus diam at ex eleifend aliquet", + "hs_url_path": "Suspendisse blandit sem in consectetur imperdiet", + "hs_visitor_type": "Nunc porttitor metus a mauris commodo, id dictum lectus feugiat" + } + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_notes.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_notes.json new file mode 100644 index 0000000000000..c812f807dc723 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_notes.json @@ -0,0 +1,18 @@ +{ + "results": [ + { + "id": "312929690", + "createdAt": "2021-02-23T20:08:49.603Z", + "updatedAt": "2021-02-23T20:08:53.371Z", + "archived": false, + "companies": ["company A"], + "contacts": ["contact A"], + "deals": ["deal A"], + "tickets": ["ticket A"], + "properties": { + "closed_date": "2021-02-23T20:08:49.603000+00:00", + "createdate": "2021-02-23T20:08:49.603000+00:00" + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_notes_web_analytics.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_notes_web_analytics.json new file mode 100644 index 0000000000000..859c04fa8e0fa --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_notes_web_analytics.json @@ -0,0 +1,57 @@ +{ + "results": [ + { + "objectType": "ENGAGEMENT_NOTE", + "objectId": "146", + "eventType": "pe8727216_airbyte_engagement_note_custom_event", + "occurredAt": "2023-12-01T21:50:14.001Z", + "id": "b850d903-254c-4df6-b159-9233c2b6eed8", + "properties": { + "hs_asset_description": "dolor architecto", + "hs_asset_type": "Donec scelerisque est sed libero tempor venenatis", + "hs_browser": "Donec lobortis nibh sit amet dictum cursus", + "hs_campaign_id": "illum quas dolor modi exercitationem", + "hs_city": "reiciendis placeat dolor placeat architecto dolor", + "hs_country": "Mauris sagittis sem eu neque pellentesque gravida", + "hs_hs_device_name": "elit. molestias, elit. amet", + "hs_device_type": "accusantium dolor sit elit. veniam reprehenderit", + "hs_title": "In egestas nisl commodo pellentesque facilisis", + "hs_form_correlation_id": "Pellentesque tincidunt purus id rhoncus blandit", + "hs_element_class": "Nulla luctus tortor at fermentum aliquet", + "hs_element_id": "culpa! sit ipsum sit Lorem consectetur quas odit", + "hs_element_text": "Hic molestias, Lorem ipsum, possimus adipisicing", + "hs_language": "officiis adipisicing", + "hs_document_id": "Proin sed lacus eget sapien viverra vestibulum vel sit amet massa", + "hs_presentation_id": "Mauris et nisl a metus porttitor molestie id viverra ex", + "hs_user_id": "Quisque consequat nunc at eleifend tempus", + "hs_link_href": "officiis exercitationem adipisicing odit dolor", + "hs_operating_system": "magnam, molestias,", + "hs_operating_version": "Lorem ipsum culpa! illum elit. esse esse officiis", + "hs_page_content_type": "elit. libero Lorem", + "hs_page_id": "magnam, magnam,", + "hs_page_title": "Praesent auctor sem et purus facilisis, at volutpat lorem tristique", + "hs_page_url": "accusantium quas architecto ipsum ipsum possimus", + "hs_parent_module_id": "Aliquam eleifend ex in ligula gravida mollis", + "hs_referrer": "Sed nec eros quis sem euismod tempor", + "hs_region": "consectetur ipsum, architecto ipsum Lorem nobis", + "hs_url": "Aenean feugiat quam in urna iaculis, vitae ultrices metus scelerisque", + "hs_screen_height": "Donec laoreet est ut lorem viverra tempus", + "hs_screen_width": "Curabitur sit amet augue luctus, congue erat congue, vestibulum lectus", + "hs_touchpoint_source": "dolor", + "hs_tracking_name": "quas ipsum amet illum molestias,", + "hs_user_agent": "Aliquam eget libero consectetur, consectetur sem ac, vehicula orci", + "hs_utm_campaign": "magnam, Lorem modi culpa!", + "hs_utm_content": "Proin pulvinar nulla sed lacus venenatis blandit", + "hs_utm_medium": "placeat dolor dolor consectetur elit.", + "hs_utm_source": "Duis non erat at justo euismod lobortis", + "hs_utm_term": "Proin sit amet leo tincidunt, ultrices dolor ac, hendrerit mi", + "hs_base_url": "Maecenas ut risus eget ligula pharetra pellentesque sed et mi", + "hs_form_id": "Quisque cursus sem sit amet libero feugiat rhoncus", + "hs_form_type": "Pellentesque tristique velit at velit lacinia vulputate", + "hs_url_domain": "Quisque rhoncus diam at ex eleifend aliquet", + "hs_url_path": "Suspendisse blandit sem in consectetur imperdiet", + "hs_visitor_type": "Nunc porttitor metus a mauris commodo, id dictum lectus feugiat" + } + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_tasks.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_tasks.json new file mode 100644 index 0000000000000..d7aa760f0e769 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_tasks.json @@ -0,0 +1,18 @@ +{ + "results": [ + { + "id": "312929661", + "createdAt": "2021-02-23T20:08:49.603Z", + "updatedAt": "2021-02-23T20:08:53.371Z", + "archived": false, + "companies": ["company A"], + "contacts": ["contact A"], + "deals": ["deal A"], + "tickets": ["ticket A"], + "properties": { + "closed_date": "2021-02-23T20:08:49.603000+00:00", + "createdate": "2021-02-23T20:08:49.603000+00:00" + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_tasks_web_analytics.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_tasks_web_analytics.json new file mode 100644 index 0000000000000..2ece55edc1c4e --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_tasks_web_analytics.json @@ -0,0 +1,57 @@ +{ + "results": [ + { + "objectType": "ENGAGEMENT_TASK", + "objectId": "196", + "eventType": "pe8727216_airbyte_engagement_task_custom_event", + "occurredAt": "2023-12-01T21:50:14.301Z", + "id": "b850d903-254c-4df6-b159-9233d2b6eed8", + "properties": { + "hs_asset_description": "dolor architecto", + "hs_asset_type": "Donec scelerisque est sed libero tempor venenatis", + "hs_browser": "Donec lobortis nibh sit amet dictum cursus", + "hs_campaign_id": "illum quas dolor modi exercitationem", + "hs_city": "reiciendis placeat dolor placeat architecto dolor", + "hs_country": "Mauris sagittis sem eu neque pellentesque gravida", + "hs_hs_device_name": "elit. molestias, elit. amet", + "hs_device_type": "accusantium dolor sit elit. veniam reprehenderit", + "hs_title": "In egestas nisl commodo pellentesque facilisis", + "hs_form_correlation_id": "Pellentesque tincidunt purus id rhoncus blandit", + "hs_element_class": "Nulla luctus tortor at fermentum aliquet", + "hs_element_id": "culpa! sit ipsum sit Lorem consectetur quas odit", + "hs_element_text": "Hic molestias, Lorem ipsum, possimus adipisicing", + "hs_language": "officiis adipisicing", + "hs_document_id": "Proin sed lacus eget sapien viverra vestibulum vel sit amet massa", + "hs_presentation_id": "Mauris et nisl a metus porttitor molestie id viverra ex", + "hs_user_id": "Quisque consequat nunc at eleifend tempus", + "hs_link_href": "officiis exercitationem adipisicing odit dolor", + "hs_operating_system": "magnam, molestias,", + "hs_operating_version": "Lorem ipsum culpa! illum elit. esse esse officiis", + "hs_page_content_type": "elit. libero Lorem", + "hs_page_id": "magnam, magnam,", + "hs_page_title": "Praesent auctor sem et purus facilisis, at volutpat lorem tristique", + "hs_page_url": "accusantium quas architecto ipsum ipsum possimus", + "hs_parent_module_id": "Aliquam eleifend ex in ligula gravida mollis", + "hs_referrer": "Sed nec eros quis sem euismod tempor", + "hs_region": "consectetur ipsum, architecto ipsum Lorem nobis", + "hs_url": "Aenean feugiat quam in urna iaculis, vitae ultrices metus scelerisque", + "hs_screen_height": "Donec laoreet est ut lorem viverra tempus", + "hs_screen_width": "Curabitur sit amet augue luctus, congue erat congue, vestibulum lectus", + "hs_touchpoint_source": "dolor", + "hs_tracking_name": "quas ipsum amet illum molestias,", + "hs_user_agent": "Aliquam eget libero consectetur, consectetur sem ac, vehicula orci", + "hs_utm_campaign": "magnam, Lorem modi culpa!", + "hs_utm_content": "Proin pulvinar nulla sed lacus venenatis blandit", + "hs_utm_medium": "placeat dolor dolor consectetur elit.", + "hs_utm_source": "Duis non erat at justo euismod lobortis", + "hs_utm_term": "Proin sit amet leo tincidunt, ultrices dolor ac, hendrerit mi", + "hs_base_url": "Maecenas ut risus eget ligula pharetra pellentesque sed et mi", + "hs_form_id": "Quisque cursus sem sit amet libero feugiat rhoncus", + "hs_form_type": "Pellentesque tristique velit at velit lacinia vulputate", + "hs_url_domain": "Quisque rhoncus diam at ex eleifend aliquet", + "hs_url_path": "Suspendisse blandit sem in consectetur imperdiet", + "hs_visitor_type": "Nunc porttitor metus a mauris commodo, id dictum lectus feugiat" + } + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/goals.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/goals.json new file mode 100644 index 0000000000000..b36f645b719c2 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/goals.json @@ -0,0 +1,14 @@ +{ + "results": [ + { + "id": "312929590", + "createdAt": "2021-02-23T20:08:49.603Z", + "updatedAt": "2021-02-23T20:08:53.371Z", + "archived": false, + "properties": { + "closed_date": "2021-02-23T20:08:49.603000+00:00", + "createdate": "2021-02-23T20:08:49.603000+00:00" + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/goals_web_analytics.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/goals_web_analytics.json new file mode 100644 index 0000000000000..6fa00c8eb1fd3 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/goals_web_analytics.json @@ -0,0 +1,57 @@ +{ + "results": [ + { + "objectType": "GOAL", + "objectId": "155", + "eventType": "pe8727216_airbyte_goal_custom_event", + "occurredAt": "2023-12-01T21:50:11.809Z", + "id": "b850d903-254c-4df6-b159-9263b2b7eee2", + "properties": { + "hs_asset_description": "dolor architecto", + "hs_asset_type": "Donec scelerisque est sed libero tempor venenatis", + "hs_browser": "Donec lobortis nibh sit amet dictum cursus", + "hs_campaign_id": "illum quas dolor modi exercitationem", + "hs_city": "reiciendis placeat dolor placeat architecto dolor", + "hs_country": "Mauris sagittis sem eu neque pellentesque gravida", + "hs_hs_device_name": "elit. molestias, elit. amet", + "hs_device_type": "accusantium dolor sit elit. veniam reprehenderit", + "hs_title": "In egestas nisl commodo pellentesque facilisis", + "hs_form_correlation_id": "Pellentesque tincidunt purus id rhoncus blandit", + "hs_element_class": "Nulla luctus tortor at fermentum aliquet", + "hs_element_id": "culpa! sit ipsum sit Lorem consectetur quas odit", + "hs_element_text": "Hic molestias, Lorem ipsum, possimus adipisicing", + "hs_language": "officiis adipisicing", + "hs_document_id": "Proin sed lacus eget sapien viverra vestibulum vel sit amet massa", + "hs_presentation_id": "Mauris et nisl a metus porttitor molestie id viverra ex", + "hs_user_id": "Quisque consequat nunc at eleifend tempus", + "hs_link_href": "officiis exercitationem adipisicing odit dolor", + "hs_operating_system": "magnam, molestias,", + "hs_operating_version": "Lorem ipsum culpa! illum elit. esse esse officiis", + "hs_page_content_type": "elit. libero Lorem", + "hs_page_id": "magnam, magnam,", + "hs_page_title": "Praesent auctor sem et purus facilisis, at volutpat lorem tristique", + "hs_page_url": "accusantium quas architecto ipsum ipsum possimus", + "hs_parent_module_id": "Aliquam eleifend ex in ligula gravida mollis", + "hs_referrer": "Sed nec eros quis sem euismod tempor", + "hs_region": "consectetur ipsum, architecto ipsum Lorem nobis", + "hs_url": "Aenean feugiat quam in urna iaculis, vitae ultrices metus scelerisque", + "hs_screen_height": "Donec laoreet est ut lorem viverra tempus", + "hs_screen_width": "Curabitur sit amet augue luctus, congue erat congue, vestibulum lectus", + "hs_touchpoint_source": "dolor", + "hs_tracking_name": "quas ipsum amet illum molestias,", + "hs_user_agent": "Aliquam eget libero consectetur, consectetur sem ac, vehicula orci", + "hs_utm_campaign": "magnam, Lorem modi culpa!", + "hs_utm_content": "Proin pulvinar nulla sed lacus venenatis blandit", + "hs_utm_medium": "placeat dolor dolor consectetur elit.", + "hs_utm_source": "Duis non erat at justo euismod lobortis", + "hs_utm_term": "Proin sit amet leo tincidunt, ultrices dolor ac, hendrerit mi", + "hs_base_url": "Maecenas ut risus eget ligula pharetra pellentesque sed et mi", + "hs_form_id": "Quisque cursus sem sit amet libero feugiat rhoncus", + "hs_form_type": "Pellentesque tristique velit at velit lacinia vulputate", + "hs_url_domain": "Quisque rhoncus diam at ex eleifend aliquet", + "hs_url_path": "Suspendisse blandit sem in consectetur imperdiet", + "hs_visitor_type": "Nunc porttitor metus a mauris commodo, id dictum lectus feugiat" + } + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/line_items.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/line_items.json new file mode 100644 index 0000000000000..0719200682319 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/line_items.json @@ -0,0 +1,14 @@ +{ + "results": [ + { + "id": "312929581", + "createdAt": "2021-02-23T20:08:49.603Z", + "updatedAt": "2021-02-23T20:08:53.371Z", + "archived": false, + "properties": { + "closed_date": "2021-02-23T20:08:49.603000+00:00", + "createdate": "2021-02-23T20:08:49.603000+00:00" + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/line_items_web_analytics.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/line_items_web_analytics.json new file mode 100644 index 0000000000000..2c1c667f540a6 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/line_items_web_analytics.json @@ -0,0 +1,57 @@ +{ + "results": [ + { + "objectType": "LINE_ITEM", + "objectId": "154", + "eventType": "pe8727216_airbyte_line_item_custom_event", + "occurredAt": "2023-12-01T21:50:11.808Z", + "id": "b850d903-254c-4df6-b159-9263b2b7eee1", + "properties": { + "hs_asset_description": "dolor architecto", + "hs_asset_type": "Donec scelerisque est sed libero tempor venenatis", + "hs_browser": "Donec lobortis nibh sit amet dictum cursus", + "hs_campaign_id": "illum quas dolor modi exercitationem", + "hs_city": "reiciendis placeat dolor placeat architecto dolor", + "hs_country": "Mauris sagittis sem eu neque pellentesque gravida", + "hs_hs_device_name": "elit. molestias, elit. amet", + "hs_device_type": "accusantium dolor sit elit. veniam reprehenderit", + "hs_title": "In egestas nisl commodo pellentesque facilisis", + "hs_form_correlation_id": "Pellentesque tincidunt purus id rhoncus blandit", + "hs_element_class": "Nulla luctus tortor at fermentum aliquet", + "hs_element_id": "culpa! sit ipsum sit Lorem consectetur quas odit", + "hs_element_text": "Hic molestias, Lorem ipsum, possimus adipisicing", + "hs_language": "officiis adipisicing", + "hs_document_id": "Proin sed lacus eget sapien viverra vestibulum vel sit amet massa", + "hs_presentation_id": "Mauris et nisl a metus porttitor molestie id viverra ex", + "hs_user_id": "Quisque consequat nunc at eleifend tempus", + "hs_link_href": "officiis exercitationem adipisicing odit dolor", + "hs_operating_system": "magnam, molestias,", + "hs_operating_version": "Lorem ipsum culpa! illum elit. esse esse officiis", + "hs_page_content_type": "elit. libero Lorem", + "hs_page_id": "magnam, magnam,", + "hs_page_title": "Praesent auctor sem et purus facilisis, at volutpat lorem tristique", + "hs_page_url": "accusantium quas architecto ipsum ipsum possimus", + "hs_parent_module_id": "Aliquam eleifend ex in ligula gravida mollis", + "hs_referrer": "Sed nec eros quis sem euismod tempor", + "hs_region": "consectetur ipsum, architecto ipsum Lorem nobis", + "hs_url": "Aenean feugiat quam in urna iaculis, vitae ultrices metus scelerisque", + "hs_screen_height": "Donec laoreet est ut lorem viverra tempus", + "hs_screen_width": "Curabitur sit amet augue luctus, congue erat congue, vestibulum lectus", + "hs_touchpoint_source": "dolor", + "hs_tracking_name": "quas ipsum amet illum molestias,", + "hs_user_agent": "Aliquam eget libero consectetur, consectetur sem ac, vehicula orci", + "hs_utm_campaign": "magnam, Lorem modi culpa!", + "hs_utm_content": "Proin pulvinar nulla sed lacus venenatis blandit", + "hs_utm_medium": "placeat dolor dolor consectetur elit.", + "hs_utm_source": "Duis non erat at justo euismod lobortis", + "hs_utm_term": "Proin sit amet leo tincidunt, ultrices dolor ac, hendrerit mi", + "hs_base_url": "Maecenas ut risus eget ligula pharetra pellentesque sed et mi", + "hs_form_id": "Quisque cursus sem sit amet libero feugiat rhoncus", + "hs_form_type": "Pellentesque tristique velit at velit lacinia vulputate", + "hs_url_domain": "Quisque rhoncus diam at ex eleifend aliquet", + "hs_url_path": "Suspendisse blandit sem in consectetur imperdiet", + "hs_visitor_type": "Nunc porttitor metus a mauris commodo, id dictum lectus feugiat" + } + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/products.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/products.json new file mode 100644 index 0000000000000..c3f5a5213d12a --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/products.json @@ -0,0 +1,14 @@ +{ + "results": [ + { + "id": "312929580", + "createdAt": "2021-02-23T20:08:49.603Z", + "updatedAt": "2021-02-23T20:08:53.371Z", + "archived": false, + "properties": { + "closed_date": "2021-02-23T20:08:49.603000+00:00", + "createdate": "2021-02-23T20:08:49.603000+00:00" + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/products_web_analytics.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/products_web_analytics.json new file mode 100644 index 0000000000000..e734ce91b1ad0 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/products_web_analytics.json @@ -0,0 +1,57 @@ +{ + "results": [ + { + "objectType": "PRODUCT", + "objectId": "153", + "eventType": "pe8727216_airbyte_product_custom_event", + "occurredAt": "2023-12-01T21:50:11.799Z", + "id": "b850d903-254c-4df6-b159-9263b2b7eed3", + "properties": { + "hs_asset_description": "dolor architecto", + "hs_asset_type": "Donec scelerisque est sed libero tempor venenatis", + "hs_browser": "Donec lobortis nibh sit amet dictum cursus", + "hs_campaign_id": "illum quas dolor modi exercitationem", + "hs_city": "reiciendis placeat dolor placeat architecto dolor", + "hs_country": "Mauris sagittis sem eu neque pellentesque gravida", + "hs_hs_device_name": "elit. molestias, elit. amet", + "hs_device_type": "accusantium dolor sit elit. veniam reprehenderit", + "hs_title": "In egestas nisl commodo pellentesque facilisis", + "hs_form_correlation_id": "Pellentesque tincidunt purus id rhoncus blandit", + "hs_element_class": "Nulla luctus tortor at fermentum aliquet", + "hs_element_id": "culpa! sit ipsum sit Lorem consectetur quas odit", + "hs_element_text": "Hic molestias, Lorem ipsum, possimus adipisicing", + "hs_language": "officiis adipisicing", + "hs_document_id": "Proin sed lacus eget sapien viverra vestibulum vel sit amet massa", + "hs_presentation_id": "Mauris et nisl a metus porttitor molestie id viverra ex", + "hs_user_id": "Quisque consequat nunc at eleifend tempus", + "hs_link_href": "officiis exercitationem adipisicing odit dolor", + "hs_operating_system": "magnam, molestias,", + "hs_operating_version": "Lorem ipsum culpa! illum elit. esse esse officiis", + "hs_page_content_type": "elit. libero Lorem", + "hs_page_id": "magnam, magnam,", + "hs_page_title": "Praesent auctor sem et purus facilisis, at volutpat lorem tristique", + "hs_page_url": "accusantium quas architecto ipsum ipsum possimus", + "hs_parent_module_id": "Aliquam eleifend ex in ligula gravida mollis", + "hs_referrer": "Sed nec eros quis sem euismod tempor", + "hs_region": "consectetur ipsum, architecto ipsum Lorem nobis", + "hs_url": "Aenean feugiat quam in urna iaculis, vitae ultrices metus scelerisque", + "hs_screen_height": "Donec laoreet est ut lorem viverra tempus", + "hs_screen_width": "Curabitur sit amet augue luctus, congue erat congue, vestibulum lectus", + "hs_touchpoint_source": "dolor", + "hs_tracking_name": "quas ipsum amet illum molestias,", + "hs_user_agent": "Aliquam eget libero consectetur, consectetur sem ac, vehicula orci", + "hs_utm_campaign": "magnam, Lorem modi culpa!", + "hs_utm_content": "Proin pulvinar nulla sed lacus venenatis blandit", + "hs_utm_medium": "placeat dolor dolor consectetur elit.", + "hs_utm_source": "Duis non erat at justo euismod lobortis", + "hs_utm_term": "Proin sit amet leo tincidunt, ultrices dolor ac, hendrerit mi", + "hs_base_url": "Maecenas ut risus eget ligula pharetra pellentesque sed et mi", + "hs_form_id": "Quisque cursus sem sit amet libero feugiat rhoncus", + "hs_form_type": "Pellentesque tristique velit at velit lacinia vulputate", + "hs_url_domain": "Quisque rhoncus diam at ex eleifend aliquet", + "hs_url_path": "Suspendisse blandit sem in consectetur imperdiet", + "hs_visitor_type": "Nunc porttitor metus a mauris commodo, id dictum lectus feugiat" + } + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/properties.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/properties.json new file mode 100644 index 0000000000000..7b8bdcc783ef1 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/properties.json @@ -0,0 +1,34 @@ +[ + { + "name": "closed_date", + "label": "Close date", + "description": "The date the ticket was closed", + "groupName": "ticketinformation", + "type": "datetime", + "fieldType": "text", + "hidden": false, + "options": [], + "calculated": false, + "externalOptions": false, + "isCustomizedDefault": false, + "deleted": null, + "createdAt": null, + "updatedAt": null, + "displayOrder": -1, + "readOnlyValue": false, + "readOnlyDefinition": true, + "mutableDefinitionNotDeletable": false, + "favorited": false, + "favoritedOrder": -1, + "displayMode": "current_value", + "showCurrencySymbol": null, + "createdUserId": null, + "textDisplayHint": null, + "numberDisplayHint": null, + "optionsAreMutable": null, + "referencedObjectType": null, + "formField": false, + "hubspotDefined": true, + "updatedUserId": null + } +] \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/tickets.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/tickets.json new file mode 100644 index 0000000000000..92e415de83f06 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/tickets.json @@ -0,0 +1,14 @@ +{ + "results": [ + { + "id": "312929579", + "createdAt": "2021-02-23T20:08:49.603Z", + "updatedAt": "2021-02-23T20:08:53.371Z", + "archived": false, + "properties": { + "closed_date": "2021-02-23T20:08:49.603000+00:00", + "createdate": "2021-02-23T20:08:49.603000+00:00" + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/tickets_web_analytics.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/tickets_web_analytics.json new file mode 100644 index 0000000000000..d27231eab6402 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/tickets_web_analytics.json @@ -0,0 +1,57 @@ +{ + "results": [ + { + "objectType": "TICKET", + "objectId": "151", + "eventType": "pe8727216_airbyte_ticket_custom_event", + "occurredAt": "2023-12-01T21:50:11.797Z", + "id": "b850d903-254c-4df6-b159-9263b2b7eed0", + "properties": { + "hs_asset_description": "dolor architecto", + "hs_asset_type": "Donec scelerisque est sed libero tempor venenatis", + "hs_browser": "Donec lobortis nibh sit amet dictum cursus", + "hs_campaign_id": "illum quas dolor modi exercitationem", + "hs_city": "reiciendis placeat dolor placeat architecto dolor", + "hs_country": "Mauris sagittis sem eu neque pellentesque gravida", + "hs_hs_device_name": "elit. molestias, elit. amet", + "hs_device_type": "accusantium dolor sit elit. veniam reprehenderit", + "hs_title": "In egestas nisl commodo pellentesque facilisis", + "hs_form_correlation_id": "Pellentesque tincidunt purus id rhoncus blandit", + "hs_element_class": "Nulla luctus tortor at fermentum aliquet", + "hs_element_id": "culpa! sit ipsum sit Lorem consectetur quas odit", + "hs_element_text": "Hic molestias, Lorem ipsum, possimus adipisicing", + "hs_language": "officiis adipisicing", + "hs_document_id": "Proin sed lacus eget sapien viverra vestibulum vel sit amet massa", + "hs_presentation_id": "Mauris et nisl a metus porttitor molestie id viverra ex", + "hs_user_id": "Quisque consequat nunc at eleifend tempus", + "hs_link_href": "officiis exercitationem adipisicing odit dolor", + "hs_operating_system": "magnam, molestias,", + "hs_operating_version": "Lorem ipsum culpa! illum elit. esse esse officiis", + "hs_page_content_type": "elit. libero Lorem", + "hs_page_id": "magnam, magnam,", + "hs_page_title": "Praesent auctor sem et purus facilisis, at volutpat lorem tristique", + "hs_page_url": "accusantium quas architecto ipsum ipsum possimus", + "hs_parent_module_id": "Aliquam eleifend ex in ligula gravida mollis", + "hs_referrer": "Sed nec eros quis sem euismod tempor", + "hs_region": "consectetur ipsum, architecto ipsum Lorem nobis", + "hs_url": "Aenean feugiat quam in urna iaculis, vitae ultrices metus scelerisque", + "hs_screen_height": "Donec laoreet est ut lorem viverra tempus", + "hs_screen_width": "Curabitur sit amet augue luctus, congue erat congue, vestibulum lectus", + "hs_touchpoint_source": "dolor", + "hs_tracking_name": "quas ipsum amet illum molestias,", + "hs_user_agent": "Aliquam eget libero consectetur, consectetur sem ac, vehicula orci", + "hs_utm_campaign": "magnam, Lorem modi culpa!", + "hs_utm_content": "Proin pulvinar nulla sed lacus venenatis blandit", + "hs_utm_medium": "placeat dolor dolor consectetur elit.", + "hs_utm_source": "Duis non erat at justo euismod lobortis", + "hs_utm_term": "Proin sit amet leo tincidunt, ultrices dolor ac, hendrerit mi", + "hs_base_url": "Maecenas ut risus eget ligula pharetra pellentesque sed et mi", + "hs_form_id": "Quisque cursus sem sit amet libero feugiat rhoncus", + "hs_form_type": "Pellentesque tristique velit at velit lacinia vulputate", + "hs_url_domain": "Quisque rhoncus diam at ex eleifend aliquet", + "hs_url_path": "Suspendisse blandit sem in consectetur imperdiet", + "hs_visitor_type": "Nunc porttitor metus a mauris commodo, id dictum lectus feugiat" + } + } + ] +} \ No newline at end of file From a0f2c2cc0ae57b545d1761832e0cd13e1ae05393 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Mon, 11 Mar 2024 07:33:50 +0200 Subject: [PATCH 02/12] source hubspot: bump version --- airbyte-integrations/connectors/source-hubspot/metadata.yaml | 2 +- docs/integrations/sources/hubspot.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-hubspot/metadata.yaml b/airbyte-integrations/connectors/source-hubspot/metadata.yaml index a42e4b8190f48..ccc87663347e9 100644 --- a/airbyte-integrations/connectors/source-hubspot/metadata.yaml +++ b/airbyte-integrations/connectors/source-hubspot/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c - dockerImageTag: 4.0.0 + dockerImageTag: 4.0.1 dockerRepository: airbyte/source-hubspot documentationUrl: https://docs.airbyte.com/integrations/sources/hubspot githubIssueLabel: source-hubspot diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md index c23b528377530..a49590ab37a40 100644 --- a/docs/integrations/sources/hubspot.md +++ b/docs/integrations/sources/hubspot.md @@ -322,6 +322,7 @@ The connector is restricted by normal HubSpot [rate limitations](https://legacyd | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 4.0.1 | 2024-03-11 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Add integration tests | | 4.0.0 | 2024-03-10 | [35662](https://github.com/airbytehq/airbyte/pull/35662) | Update `Deals Property History` and `Companies Property History` schemas | | 3.3.0 | 2024-02-16 | [34597](https://github.com/airbytehq/airbyte/pull/34597) | Make start date not required, sync all data from default value if it's not provided | | 3.2.0 | 2024-02-15 | [35328](https://github.com/airbytehq/airbyte/pull/35328) | Add mailingIlsListsIncluded and mailingIlsListsExcluded fields to Marketing emails stream schema | From cdfdce505e7f40587f9da25213107c1e00fd8a27 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Sat, 16 Mar 2024 11:49:51 +0200 Subject: [PATCH 03/12] source hubspot: add incremental tests --- .../test_web_analytics_streams.py | 121 +++++++++++++++++- 1 file changed, 116 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py index 5dad2e53ac4aa..847981afce3ee 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py @@ -26,7 +26,7 @@ from .response_builder.web_analytics import WebAnalyticsResponseBuilder, GenericAbstractResponseBuilder from .request_builders.web_analytics import WebAnalyticsRequestBuilder, IncrementalCRMStreamRequestBuilder, CRMStreamRequestBuilder from .request_builders.other import OAuthRequestBuilder, CustomObjectsRequestBuilder, ScopesRequestBuilder, PropertiesRequestBuilder -from airbyte_protocol.models import AirbyteStateMessage, FailureType, SyncMode +from airbyte_protocol.models import AirbyteStateBlob, AirbyteStateType, AirbyteStateMessage, AirbyteStreamState, FailureType, StreamDescriptor, SyncMode from source_hubspot import SourceHubspot @@ -189,9 +189,13 @@ def web_analytics_request( return WebAnalyticsRequestBuilder().with_token(token).with_query(query).build() @classmethod - def web_analytics_response(cls, stream: str, with_pagination: bool = False, updated_on: Optional[str] = None) -> HttpResponse: + def web_analytics_response( + cls, stream: str, with_pagination: bool = False, updated_on: Optional[str] = None, id: Optional[str] = None + ) -> HttpResponse: updated_on = updated_on or cls.dt_str(cls.updated_at()) - record = cls.record_builder(stream, FieldPath("occurredAt")).with_field(FieldPath("updatedAt"), updated_on) + record = cls.record_builder(stream, FieldPath("occurredAt")).with_field(FieldPath("occurredAt"), updated_on) + if id: + record = record.with_field(FieldPath("objectId"), id) response_builder = WebAnalyticsResponseBuilder.for_stream(stream) response = response_builder.with_record(record) if with_pagination: @@ -216,10 +220,11 @@ def read_from_stream( @freezegun.freeze_time("2024-03-03T14:42:00Z") -class TestCRMWebAnalyticsStreamFullRefresh(WebAnalytics): +class TestCRMWebAnalyticsStream(WebAnalytics): SCOPES = ["tickets", "crm.objects.contacts.read", "crm.objects.companies.read", "contacts", "crm.objects.deals.read", "oauth"] OBJECT_ID = "testID" ACCESS_TOKEN = "new_access_token" + CURSOR_FIELD = "occurredAt" PROPERTIES = { "closed_date": "datetime", "createdate": "datetime", @@ -458,9 +463,91 @@ def test_given_one_page_when_read_then_get_no_records_filtered( output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) assert len(output.records) == 1 + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) + @HttpMocker() + def test_given_incremental_sync_when_read_then_state_message_produced_and_state_match_latest_record( + self, stream_name, parent_stream_name, object_type, parent_stream_associations, http_mocker: HttpMocker + ): + self.mock_custom_objects(http_mocker) + self.mock_properties(http_mocker, object_type, self.PROPERTIES) + self.mock_parent_object( + http_mocker, [self.OBJECT_ID], object_type, parent_stream_name, parent_stream_associations, list(self.PROPERTIES.keys()) + ) + self.mock_response( + http_mocker, + self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), + self.web_analytics_response(stream_name, id=self.OBJECT_ID) + ) + output = self.read_from_stream( + self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.incremental + ) + assert len(output.state_messages) == 1 + + cursor_value_from_state_message = output.most_recent_state.get(stream_name, {}).get(self.OBJECT_ID, {}).get(self.CURSOR_FIELD) + cursor_value_from_latest_record = output.records[-1].record.data.get(self.CURSOR_FIELD) + assert cursor_value_from_state_message == cursor_value_from_latest_record + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) + @HttpMocker() + def test_given_state_with_no_current_slice_when_read_then_current_slice_in_state( + self, stream_name, parent_stream_name, object_type, parent_stream_associations, http_mocker: HttpMocker + ): + self.mock_custom_objects(http_mocker) + self.mock_properties(http_mocker, object_type, self.PROPERTIES) + self.mock_parent_object( + http_mocker, [self.OBJECT_ID], object_type, parent_stream_name, parent_stream_associations, list(self.PROPERTIES.keys()) + ) + self.mock_response( + http_mocker, + self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), + self.web_analytics_response(stream_name, id=self.OBJECT_ID) + ) + another_object_id = "another_object_id" + current_state = AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name=stream_name), + stream_state=AirbyteStateBlob(**{another_object_id: {self.CURSOR_FIELD: self.dt_str(self.now())}}) + ) + ) + output = self.read_from_stream( + self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.incremental, state=[current_state] + ) + assert len(output.state_messages) == 1 + assert output.most_recent_state.get(stream_name, {}).get(self.OBJECT_ID, {}).get(self.CURSOR_FIELD) + assert output.most_recent_state.get(stream_name, {}).get(another_object_id, {}).get(self.CURSOR_FIELD) + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) + @HttpMocker() + def test_given_state_with_current_slice_when_read_then_state_is_updated( + self, stream_name, parent_stream_name, object_type, parent_stream_associations, http_mocker: HttpMocker + ): + self.mock_custom_objects(http_mocker) + self.mock_properties(http_mocker, object_type, self.PROPERTIES) + self.mock_parent_object( + http_mocker, [self.OBJECT_ID], object_type, parent_stream_name, parent_stream_associations, list(self.PROPERTIES.keys()) + ) + self.mock_response( + http_mocker, + self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), + self.web_analytics_response(stream_name, id=self.OBJECT_ID) + ) + current_state = AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name=stream_name), + stream_state=AirbyteStateBlob(**{self.OBJECT_ID: {self.CURSOR_FIELD: self.dt_str(self.start_date() - timedelta(days=30))}}) + ) + ) + output = self.read_from_stream( + self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.incremental, state=[current_state] + ) + assert len(output.state_messages) == 1 + assert output.most_recent_state.get(stream_name, {}).get(self.OBJECT_ID, {}).get(self.CURSOR_FIELD) == self.dt_str(self.updated_at()) + @freezegun.freeze_time("2024-03-03T14:42:00Z") -class TestIncrementalCRMWebAnalyticsStreamFullRefresh(TestCRMWebAnalyticsStreamFullRefresh): +class TestIncrementalCRMWebAnalyticsStreamFullRefresh(TestCRMWebAnalyticsStream): SCOPES = ["e-commerce", "oauth", "crm.objects.feedback_submissions.read", "crm.objects.goals.read"] @classmethod @@ -576,3 +663,27 @@ def test_given_one_page_when_read_then_get_no_records_filtered( super().test_given_one_page_when_read_then_get_no_records_filtered( stream_name, parent_stream_name, object_type, parent_stream_associations ) + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) + def test_given_incremental_sync_when_read_then_state_message_produced_and_state_match_latest_record( + self, stream_name, parent_stream_name, object_type, parent_stream_associations + ): + super().test_given_incremental_sync_when_read_then_state_message_produced_and_state_match_latest_record( + stream_name, parent_stream_name, object_type, parent_stream_associations + ) + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) + def test_given_state_with_no_current_slice_when_read_then_current_slice_in_state( + self, stream_name, parent_stream_name, object_type, parent_stream_associations + ): + super().test_given_state_with_no_current_slice_when_read_then_current_slice_in_state( + stream_name, parent_stream_name, object_type, parent_stream_associations + ) + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) + def test_given_state_with_current_slice_when_read_then_state_is_updated( + self, stream_name, parent_stream_name, object_type, parent_stream_associations + ): + super().test_given_state_with_current_slice_when_read_then_state_is_updated( + stream_name, parent_stream_name, object_type, parent_stream_associations + ) From 04dd04f4c3ac7ade9128dd2cf19d4e7352701cad Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Sat, 16 Mar 2024 12:54:18 +0200 Subject: [PATCH 04/12] source hubspot: fix formatting andversion --- .../connectors/source-hubspot/pyproject.toml | 2 +- .../source-hubspot/unit_tests/conftest.py | 2 +- .../integrations/request_builders/other.py | 1 + .../request_builders/web_analytics.py | 2 ++ .../integrations/response_builder/__init__.py | 2 +- .../integrations/response_builder/helpers.py | 9 ++--- .../integrations/response_builder/other.py | 1 + .../response_builder/pagination.py | 1 + .../response_builder/web_analytics.py | 4 ++- .../test_web_analytics_streams.py | 35 +++++++------------ 10 files changed, 26 insertions(+), 33 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/pyproject.toml b/airbyte-integrations/connectors/source-hubspot/pyproject.toml index 82b0817a81c3a..74e5c0083185e 100644 --- a/airbyte-integrations/connectors/source-hubspot/pyproject.toml +++ b/airbyte-integrations/connectors/source-hubspot/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "4.0.0" +version = "4.0.1" name = "source-hubspot" description = "Source implementation for HubSpot." authors = [ "Airbyte ",] diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py index af7f3f76bca08..c64d689bde9ef 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py @@ -7,7 +7,7 @@ from source_hubspot.streams import API NUMBER_OF_PROPERTIES = 2000 -from airbyte_cdk.test.mock_http import HttpMocker + @pytest.fixture(name="oauth_config") def oauth_config_fixture(): diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/other.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/other.py index 9b906b82976af..83ba20fd293b1 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/other.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/other.py @@ -1,4 +1,5 @@ from airbyte_cdk.test.mock_http import HttpRequest + from . import AbstractRequestBuilder diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/web_analytics.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/web_analytics.py index 08b1de8c6fc54..31cb326c94e58 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/web_analytics.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/web_analytics.py @@ -1,5 +1,7 @@ from typing import Any, Dict, Iterable, Tuple + from airbyte_cdk.test.mock_http import HttpRequest + from . import AbstractRequestBuilder diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/__init__.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/__init__.py index 2a4c6acdd8272..ee78eb8ecd45d 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/__init__.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/__init__.py @@ -1,4 +1,5 @@ import abc + from airbyte_cdk.test.mock_http import HttpResponse @@ -6,4 +7,3 @@ class AbstractResponseBuilder: @abc.abstractmethod def build(self) -> HttpResponse: pass - diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/helpers.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/helpers.py index d43bf5849d06e..5b5d8b8a70ad1 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/helpers.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/helpers.py @@ -1,13 +1,8 @@ import json from typing import Any, List, Optional, Union + from airbyte_cdk.test.mock_http import HttpResponse -from airbyte_cdk.test.mock_http.response_builder import ( - FieldPath, - HttpResponseBuilder, - NestedPath, - RecordBuilder, - PaginationStrategy -) +from airbyte_cdk.test.mock_http.response_builder import FieldPath, HttpResponseBuilder, NestedPath, PaginationStrategy, RecordBuilder class RootHttpResponseBuilder(HttpResponseBuilder): diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/other.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/other.py index 8ca1e0bb4598e..75d2099054366 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/other.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/other.py @@ -2,6 +2,7 @@ from typing import List from airbyte_cdk.test.mock_http import HttpResponse + from . import AbstractResponseBuilder diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py index 8ceaa4ee77a8e..9f1af53ee962f 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py @@ -1,4 +1,5 @@ from typing import Any, Dict + from airbyte_cdk.test.mock_http.response_builder import PaginationStrategy diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/web_analytics.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/web_analytics.py index 9f3871bcc96e7..c5fe169ed4dcc 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/web_analytics.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/web_analytics.py @@ -1,6 +1,8 @@ import json -from airbyte_cdk.test.mock_http.response_builder import HttpResponseBuilder, find_template, FieldPath + from airbyte_cdk.test.mock_http import HttpResponse +from airbyte_cdk.test.mock_http.response_builder import FieldPath, HttpResponseBuilder, find_template + from . import AbstractResponseBuilder from .pagination import WebAnalyticsPaginationStrategy diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py index 847981afce3ee..913ebe209eea5 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py @@ -1,34 +1,25 @@ import copy import http -from typing import Any, List, Dict, Optional, Tuple +from datetime import datetime, timedelta +from typing import Any, Dict, List, Optional, Tuple +import freezegun import mock -import pytz import pytest -import freezegun -from datetime import datetime, timedelta +import pytz from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse -from airbyte_cdk.test.mock_http.response_builder import ( - FieldPath, - HttpResponseBuilder, - NestedPath, - RecordBuilder, - create_record_builder, - create_response_builder, - find_template, - PaginationStrategy -) +from airbyte_cdk.test.mock_http.response_builder import FieldPath, HttpResponseBuilder, RecordBuilder, create_record_builder, find_template +from airbyte_protocol.models import AirbyteStateBlob, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, StreamDescriptor, SyncMode +from source_hubspot import SourceHubspot + from .config_builder import ConfigBuilder +from .request_builders.other import CustomObjectsRequestBuilder, OAuthRequestBuilder, PropertiesRequestBuilder, ScopesRequestBuilder +from .request_builders.web_analytics import CRMStreamRequestBuilder, IncrementalCRMStreamRequestBuilder, WebAnalyticsRequestBuilder from .response_builder.helpers import RootHttpResponseBuilder from .response_builder.other import ScopesAbstractResponseBuilder -from .response_builder.web_analytics import WebAnalyticsResponseBuilder, GenericAbstractResponseBuilder -from .request_builders.web_analytics import WebAnalyticsRequestBuilder, IncrementalCRMStreamRequestBuilder, CRMStreamRequestBuilder -from .request_builders.other import OAuthRequestBuilder, CustomObjectsRequestBuilder, ScopesRequestBuilder, PropertiesRequestBuilder -from airbyte_protocol.models import AirbyteStateBlob, AirbyteStateType, AirbyteStateMessage, AirbyteStreamState, FailureType, StreamDescriptor, SyncMode -from source_hubspot import SourceHubspot - +from .response_builder.web_analytics import GenericAbstractResponseBuilder, WebAnalyticsResponseBuilder CRM_STREAMS = ( ("tickets_web_analytics", "tickets", "ticket", ["contacts", "deals", "companies"]), @@ -118,7 +109,7 @@ def mock_custom_objects(cls, http_mocker: HttpMocker): @classmethod def mock_properties(cls, http_mocker: HttpMocker, object_type: str, properties: Dict[str, str]): - templates = find_template(f"properties", __file__) + templates = find_template("properties", __file__) record_builder = lambda: RecordBuilder(copy.deepcopy(templates[0]), id_path=None, cursor_path=None) response_builder = RootHttpResponseBuilder(templates) @@ -140,7 +131,7 @@ def mock_parent_object( stream_name: str, associations: List[str], properties: List[str], - date_range: Optional[Tuple[str,...]] = None, + date_range: Optional[Tuple[str, ...]] = None, ): response_builder = WebAnalyticsResponseBuilder.for_stream(stream_name) for object_id in object_ids: From c72df7cb6e9f47438003e563a510d462210e6a64 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Sat, 16 Mar 2024 13:54:37 +0200 Subject: [PATCH 05/12] source hubspot: add license + format jsons --- .../unit_tests/integrations/__init__.py | 1 + .../unit_tests/integrations/config_builder.py | 2 ++ .../integrations/request_builders/__init__.py | 2 ++ .../integrations/request_builders/other.py | 2 ++ .../request_builders/web_analytics.py | 2 ++ .../integrations/response_builder/__init__.py | 2 ++ .../integrations/response_builder/helpers.py | 2 ++ .../integrations/response_builder/other.py | 2 ++ .../response_builder/pagination.py | 2 ++ .../response_builder/web_analytics.py | 2 ++ .../integrations/test_engagements_calls.py | 2 ++ .../integrations/test_owners_archived.py | 2 ++ .../test_web_analytics_streams.py | 2 ++ .../unit_tests/resource/__init__.py | 1 + .../unit_tests/resource/http/__init__.py | 1 + .../resource/http/response/__init__.py | 1 + .../resource/http/response/companies.json | 10 +++++---- .../resource/http/response/contacts.json | 11 ++++++---- .../resource/http/response/deals.json | 18 ++++++++++----- .../http/response/engagements_calls.json | 22 +++++++++++++------ .../http/response/engagements_emails.json | 22 +++++++++++++------ .../http/response/engagements_meetings.json | 22 +++++++++++++------ .../http/response/engagements_notes.json | 22 +++++++++++++------ .../http/response/engagements_tasks.json | 22 +++++++++++++------ .../resource/http/response/goals.json | 6 ++--- .../resource/http/response/line_items.json | 6 ++--- .../resource/http/response/products.json | 6 ++--- .../resource/http/response/tickets.json | 6 ++--- 28 files changed, 140 insertions(+), 61 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/__init__.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/__init__.py index e69de29bb2d1d..f70ecfc3a89e7 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/__init__.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/__init__.py @@ -0,0 +1 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/config_builder.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/config_builder.py index 6bc3093014f22..048142759ca2a 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/config_builder.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/config_builder.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + from typing import Any, Mapping diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/__init__.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/__init__.py index 1c24a3d65bf37..5fd9458a7a05a 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/__init__.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/__init__.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import abc diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/other.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/other.py index 83ba20fd293b1..17ba71bebf3c1 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/other.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/other.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + from airbyte_cdk.test.mock_http import HttpRequest from . import AbstractRequestBuilder diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/web_analytics.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/web_analytics.py index 31cb326c94e58..c3a59a871982a 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/web_analytics.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/web_analytics.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + from typing import Any, Dict, Iterable, Tuple from airbyte_cdk.test.mock_http import HttpRequest diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/__init__.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/__init__.py index ee78eb8ecd45d..e20b564abdc51 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/__init__.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/__init__.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import abc from airbyte_cdk.test.mock_http import HttpResponse diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/helpers.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/helpers.py index 5b5d8b8a70ad1..595a02232d439 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/helpers.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/helpers.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import json from typing import Any, List, Optional, Union diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/other.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/other.py index 75d2099054366..b907c3f6b3a33 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/other.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/other.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import json from typing import List diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py index 9f1af53ee962f..93f7a4bdf294b 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + from typing import Any, Dict from airbyte_cdk.test.mock_http.response_builder import PaginationStrategy diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/web_analytics.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/web_analytics.py index c5fe169ed4dcc..1870ca311199b 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/web_analytics.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/web_analytics.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import json from airbyte_cdk.test.mock_http import HttpResponse diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py index 1e83df361dc0f..9f73b73ddf163 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + from unittest import TestCase diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py index 2a67733e8dd82..965923f277eba 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + from unittest import TestCase diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py index 913ebe209eea5..a840b479ccff2 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import copy import http from datetime import datetime, timedelta diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/__init__.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/__init__.py index e69de29bb2d1d..f70ecfc3a89e7 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/__init__.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/__init__.py @@ -0,0 +1 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/__init__.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/__init__.py index e69de29bb2d1d..f70ecfc3a89e7 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/__init__.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/__init__.py @@ -0,0 +1 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/__init__.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/__init__.py index e69de29bb2d1d..f70ecfc3a89e7 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/__init__.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/__init__.py @@ -0,0 +1 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/companies.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/companies.json index f6d3182ab2dee..8bc20f24b2153 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/companies.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/companies.json @@ -5,11 +5,13 @@ "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, - "contacts": ["contact A"], + "contacts": [ + "contact A" + ], "properties": { - "closed_date": "2021-02-23T20:08:49.603000+00:00", - "createdate": "2021-02-23T20:08:49.603000+00:00" + "closed_date": "2021-02-23T20:08:49.603000+00:00", + "createdate": "2021-02-23T20:08:49.603000+00:00" } } ] -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts.json index c844ac35e2bba..a70f33015ce7a 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts.json @@ -5,11 +5,14 @@ "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, - "companies": ["5000526215", "5000526215"], + "companies": [ + "5000526215", + "5000526215" + ], "properties": { - "closed_date": "2021-02-23T20:08:49.603000+00:00", - "createdate": "2021-02-23T20:08:49.603000+00:00" + "closed_date": "2021-02-23T20:08:49.603000+00:00", + "createdate": "2021-02-23T20:08:49.603000+00:00" } } ] -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/deals.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/deals.json index 5cc06608a709c..bef7a4a8420b7 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/deals.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/deals.json @@ -5,13 +5,19 @@ "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, - "companies": ["company A"], - "contacts": ["contact A"], - "line_items": ["line item A"], + "companies": [ + "company A" + ], + "contacts": [ + "contact A" + ], + "line_items": [ + "line item A" + ], "properties": { - "closed_date": "2021-02-23T20:08:49.603000+00:00", - "createdate": "2021-02-23T20:08:49.603000+00:00" + "closed_date": "2021-02-23T20:08:49.603000+00:00", + "createdate": "2021-02-23T20:08:49.603000+00:00" } } ] -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_calls.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_calls.json index 65a6634ab5eb0..27cc9fd10bdfa 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_calls.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_calls.json @@ -5,14 +5,22 @@ "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, - "companies": ["company A"], - "contacts": ["contact A"], - "deals": ["deal A"], - "tickets": ["ticket A"], + "companies": [ + "company A" + ], + "contacts": [ + "contact A" + ], + "deals": [ + "deal A" + ], + "tickets": [ + "ticket A" + ], "properties": { - "closed_date": "2021-02-23T20:08:49.603000+00:00", - "createdate": "2021-02-23T20:08:49.603000+00:00" + "closed_date": "2021-02-23T20:08:49.603000+00:00", + "createdate": "2021-02-23T20:08:49.603000+00:00" } } ] -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_emails.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_emails.json index 805867d39e403..5f3d2a7e0ecc8 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_emails.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_emails.json @@ -5,14 +5,22 @@ "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, - "companies": ["company A"], - "contacts": ["contact A"], - "deals": ["deal A"], - "tickets": ["ticket A"], + "companies": [ + "company A" + ], + "contacts": [ + "contact A" + ], + "deals": [ + "deal A" + ], + "tickets": [ + "ticket A" + ], "properties": { - "closed_date": "2021-02-23T20:08:49.603000+00:00", - "createdate": "2021-02-23T20:08:49.603000+00:00" + "closed_date": "2021-02-23T20:08:49.603000+00:00", + "createdate": "2021-02-23T20:08:49.603000+00:00" } } ] -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_meetings.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_meetings.json index 2d34bfa5b7b80..b83afd721d659 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_meetings.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_meetings.json @@ -5,14 +5,22 @@ "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, - "companies": ["company A"], - "contacts": ["contact A"], - "deals": ["deal A"], - "tickets": ["ticket A"], + "companies": [ + "company A" + ], + "contacts": [ + "contact A" + ], + "deals": [ + "deal A" + ], + "tickets": [ + "ticket A" + ], "properties": { - "closed_date": "2021-02-23T20:08:49.603000+00:00", - "createdate": "2021-02-23T20:08:49.603000+00:00" + "closed_date": "2021-02-23T20:08:49.603000+00:00", + "createdate": "2021-02-23T20:08:49.603000+00:00" } } ] -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_notes.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_notes.json index c812f807dc723..52d95cac006d7 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_notes.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_notes.json @@ -5,14 +5,22 @@ "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, - "companies": ["company A"], - "contacts": ["contact A"], - "deals": ["deal A"], - "tickets": ["ticket A"], + "companies": [ + "company A" + ], + "contacts": [ + "contact A" + ], + "deals": [ + "deal A" + ], + "tickets": [ + "ticket A" + ], "properties": { - "closed_date": "2021-02-23T20:08:49.603000+00:00", - "createdate": "2021-02-23T20:08:49.603000+00:00" + "closed_date": "2021-02-23T20:08:49.603000+00:00", + "createdate": "2021-02-23T20:08:49.603000+00:00" } } ] -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_tasks.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_tasks.json index d7aa760f0e769..967c57d780cdf 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_tasks.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_tasks.json @@ -5,14 +5,22 @@ "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, - "companies": ["company A"], - "contacts": ["contact A"], - "deals": ["deal A"], - "tickets": ["ticket A"], + "companies": [ + "company A" + ], + "contacts": [ + "contact A" + ], + "deals": [ + "deal A" + ], + "tickets": [ + "ticket A" + ], "properties": { - "closed_date": "2021-02-23T20:08:49.603000+00:00", - "createdate": "2021-02-23T20:08:49.603000+00:00" + "closed_date": "2021-02-23T20:08:49.603000+00:00", + "createdate": "2021-02-23T20:08:49.603000+00:00" } } ] -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/goals.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/goals.json index b36f645b719c2..56cd0591c9725 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/goals.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/goals.json @@ -6,9 +6,9 @@ "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, "properties": { - "closed_date": "2021-02-23T20:08:49.603000+00:00", - "createdate": "2021-02-23T20:08:49.603000+00:00" + "closed_date": "2021-02-23T20:08:49.603000+00:00", + "createdate": "2021-02-23T20:08:49.603000+00:00" } } ] -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/line_items.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/line_items.json index 0719200682319..2e24921cb6aea 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/line_items.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/line_items.json @@ -6,9 +6,9 @@ "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, "properties": { - "closed_date": "2021-02-23T20:08:49.603000+00:00", - "createdate": "2021-02-23T20:08:49.603000+00:00" + "closed_date": "2021-02-23T20:08:49.603000+00:00", + "createdate": "2021-02-23T20:08:49.603000+00:00" } } ] -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/products.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/products.json index c3f5a5213d12a..37bc7290088c1 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/products.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/products.json @@ -6,9 +6,9 @@ "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, "properties": { - "closed_date": "2021-02-23T20:08:49.603000+00:00", - "createdate": "2021-02-23T20:08:49.603000+00:00" + "closed_date": "2021-02-23T20:08:49.603000+00:00", + "createdate": "2021-02-23T20:08:49.603000+00:00" } } ] -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/tickets.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/tickets.json index 92e415de83f06..fbf02351807ff 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/tickets.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/tickets.json @@ -6,9 +6,9 @@ "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, "properties": { - "closed_date": "2021-02-23T20:08:49.603000+00:00", - "createdate": "2021-02-23T20:08:49.603000+00:00" + "closed_date": "2021-02-23T20:08:49.603000+00:00", + "createdate": "2021-02-23T20:08:49.603000+00:00" } } ] -} +} \ No newline at end of file From d3382a1d6e632086ae294079c5a6af0ac7108cd6 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Sat, 16 Mar 2024 14:03:39 +0200 Subject: [PATCH 06/12] source hubspot: fix .json formatting --- .../resource/http/response/companies.json | 6 ++---- .../http/response/companies_web_analytics.json | 2 +- .../resource/http/response/contacts.json | 7 ++----- .../http/response/contacts_web_analytics.json | 2 +- .../resource/http/response/deals.json | 14 ++++---------- .../http/response/deals_web_analytics.json | 2 +- .../http/response/engagements_calls.json | 18 +++++------------- .../engagements_calls_web_analytics.json | 2 +- .../http/response/engagements_emails.json | 18 +++++------------- .../engagements_emails_web_analytics.json | 2 +- .../http/response/engagements_meetings.json | 18 +++++------------- .../engagements_meetings_web_analytics.json | 2 +- .../http/response/engagements_notes.json | 18 +++++------------- .../engagements_notes_web_analytics.json | 2 +- .../http/response/engagements_tasks.json | 18 +++++------------- .../engagements_tasks_web_analytics.json | 2 +- .../resource/http/response/goals.json | 2 +- .../http/response/goals_web_analytics.json | 2 +- .../resource/http/response/line_items.json | 2 +- .../response/line_items_web_analytics.json | 2 +- .../resource/http/response/products.json | 2 +- .../http/response/products_web_analytics.json | 2 +- .../resource/http/response/properties.json | 2 +- .../resource/http/response/tickets.json | 2 +- .../http/response/tickets_web_analytics.json | 2 +- 25 files changed, 50 insertions(+), 101 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/companies.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/companies.json index 8bc20f24b2153..c27f8edf74dc5 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/companies.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/companies.json @@ -5,13 +5,11 @@ "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, - "contacts": [ - "contact A" - ], + "contacts": ["contact A"], "properties": { "closed_date": "2021-02-23T20:08:49.603000+00:00", "createdate": "2021-02-23T20:08:49.603000+00:00" } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/companies_web_analytics.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/companies_web_analytics.json index 6e06874030fc4..9f66d27f168d1 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/companies_web_analytics.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/companies_web_analytics.json @@ -54,4 +54,4 @@ } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts.json index a70f33015ce7a..bb273173fc34d 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts.json @@ -5,14 +5,11 @@ "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, - "companies": [ - "5000526215", - "5000526215" - ], + "companies": ["5000526215", "5000526215"], "properties": { "closed_date": "2021-02-23T20:08:49.603000+00:00", "createdate": "2021-02-23T20:08:49.603000+00:00" } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts_web_analytics.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts_web_analytics.json index 147c3bd5002e4..4d77dd8f20995 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts_web_analytics.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts_web_analytics.json @@ -54,4 +54,4 @@ } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/deals.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/deals.json index bef7a4a8420b7..fd6283097b536 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/deals.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/deals.json @@ -5,19 +5,13 @@ "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, - "companies": [ - "company A" - ], - "contacts": [ - "contact A" - ], - "line_items": [ - "line item A" - ], + "companies": ["company A"], + "contacts": ["contact A"], + "line_items": ["line item A"], "properties": { "closed_date": "2021-02-23T20:08:49.603000+00:00", "createdate": "2021-02-23T20:08:49.603000+00:00" } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/deals_web_analytics.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/deals_web_analytics.json index 2249ba47c4b84..d1d871ea34af9 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/deals_web_analytics.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/deals_web_analytics.json @@ -54,4 +54,4 @@ } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_calls.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_calls.json index 27cc9fd10bdfa..3824c7e06ef15 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_calls.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_calls.json @@ -5,22 +5,14 @@ "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, - "companies": [ - "company A" - ], - "contacts": [ - "contact A" - ], - "deals": [ - "deal A" - ], - "tickets": [ - "ticket A" - ], + "companies": ["company A"], + "contacts": ["contact A"], + "deals": ["deal A"], + "tickets": ["ticket A"], "properties": { "closed_date": "2021-02-23T20:08:49.603000+00:00", "createdate": "2021-02-23T20:08:49.603000+00:00" } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_calls_web_analytics.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_calls_web_analytics.json index d5839a708aa43..57ca2f6c1f1f0 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_calls_web_analytics.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_calls_web_analytics.json @@ -54,4 +54,4 @@ } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_emails.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_emails.json index 5f3d2a7e0ecc8..f62dd4f3ea303 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_emails.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_emails.json @@ -5,22 +5,14 @@ "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, - "companies": [ - "company A" - ], - "contacts": [ - "contact A" - ], - "deals": [ - "deal A" - ], - "tickets": [ - "ticket A" - ], + "companies": ["company A"], + "contacts": ["contact A"], + "deals": ["deal A"], + "tickets": ["ticket A"], "properties": { "closed_date": "2021-02-23T20:08:49.603000+00:00", "createdate": "2021-02-23T20:08:49.603000+00:00" } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_emails_web_analytics.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_emails_web_analytics.json index 2926d84896e09..ce1fa57616f4c 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_emails_web_analytics.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_emails_web_analytics.json @@ -54,4 +54,4 @@ } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_meetings.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_meetings.json index b83afd721d659..94f86f1759efd 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_meetings.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_meetings.json @@ -5,22 +5,14 @@ "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, - "companies": [ - "company A" - ], - "contacts": [ - "contact A" - ], - "deals": [ - "deal A" - ], - "tickets": [ - "ticket A" - ], + "companies": ["company A"], + "contacts": ["contact A"], + "deals": ["deal A"], + "tickets": ["ticket A"], "properties": { "closed_date": "2021-02-23T20:08:49.603000+00:00", "createdate": "2021-02-23T20:08:49.603000+00:00" } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_meetings_web_analytics.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_meetings_web_analytics.json index d7540506411b6..299d706c6b284 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_meetings_web_analytics.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_meetings_web_analytics.json @@ -54,4 +54,4 @@ } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_notes.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_notes.json index 52d95cac006d7..e18a492f26104 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_notes.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_notes.json @@ -5,22 +5,14 @@ "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, - "companies": [ - "company A" - ], - "contacts": [ - "contact A" - ], - "deals": [ - "deal A" - ], - "tickets": [ - "ticket A" - ], + "companies": ["company A"], + "contacts": ["contact A"], + "deals": ["deal A"], + "tickets": ["ticket A"], "properties": { "closed_date": "2021-02-23T20:08:49.603000+00:00", "createdate": "2021-02-23T20:08:49.603000+00:00" } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_notes_web_analytics.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_notes_web_analytics.json index 859c04fa8e0fa..7bcf0d63137ac 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_notes_web_analytics.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_notes_web_analytics.json @@ -54,4 +54,4 @@ } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_tasks.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_tasks.json index 967c57d780cdf..8730e598fc4bd 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_tasks.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_tasks.json @@ -5,22 +5,14 @@ "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, - "companies": [ - "company A" - ], - "contacts": [ - "contact A" - ], - "deals": [ - "deal A" - ], - "tickets": [ - "ticket A" - ], + "companies": ["company A"], + "contacts": ["contact A"], + "deals": ["deal A"], + "tickets": ["ticket A"], "properties": { "closed_date": "2021-02-23T20:08:49.603000+00:00", "createdate": "2021-02-23T20:08:49.603000+00:00" } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_tasks_web_analytics.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_tasks_web_analytics.json index 2ece55edc1c4e..462586dae566b 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_tasks_web_analytics.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/engagements_tasks_web_analytics.json @@ -54,4 +54,4 @@ } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/goals.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/goals.json index 56cd0591c9725..19a5235e314e8 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/goals.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/goals.json @@ -11,4 +11,4 @@ } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/goals_web_analytics.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/goals_web_analytics.json index 6fa00c8eb1fd3..c7097e531d7a8 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/goals_web_analytics.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/goals_web_analytics.json @@ -54,4 +54,4 @@ } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/line_items.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/line_items.json index 2e24921cb6aea..e5090b5a28b14 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/line_items.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/line_items.json @@ -11,4 +11,4 @@ } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/line_items_web_analytics.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/line_items_web_analytics.json index 2c1c667f540a6..751b909f5a5bd 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/line_items_web_analytics.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/line_items_web_analytics.json @@ -54,4 +54,4 @@ } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/products.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/products.json index 37bc7290088c1..70144a3d6c36d 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/products.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/products.json @@ -11,4 +11,4 @@ } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/products_web_analytics.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/products_web_analytics.json index e734ce91b1ad0..c958901eea56e 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/products_web_analytics.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/products_web_analytics.json @@ -54,4 +54,4 @@ } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/properties.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/properties.json index 7b8bdcc783ef1..fb9f0a76f63e3 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/properties.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/properties.json @@ -31,4 +31,4 @@ "hubspotDefined": true, "updatedUserId": null } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/tickets.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/tickets.json index fbf02351807ff..db24185aa477e 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/tickets.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/tickets.json @@ -11,4 +11,4 @@ } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/tickets_web_analytics.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/tickets_web_analytics.json index d27231eab6402..655fb7107647f 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/tickets_web_analytics.json +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/tickets_web_analytics.json @@ -54,4 +54,4 @@ } } ] -} \ No newline at end of file +} From 78f284ccab1441c395a709b91126c9126f59bb67 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Sat, 16 Mar 2024 16:42:29 +0200 Subject: [PATCH 07/12] source hubspot: add tests for engagements calls --- .../source-hubspot/acceptance-test-config.yml | 30 +-- .../unit_tests/integrations/__init__.py | 132 +++++++++++++ .../request_builders/{other.py => api.py} | 0 .../{web_analytics.py => streams.py} | 10 +- .../response_builder/{other.py => api.py} | 0 .../response_builder/pagination.py | 4 +- .../{web_analytics.py => streams.py} | 6 +- .../integrations/test_engagements_calls.py | 141 ++++++++++++- .../test_web_analytics_streams.py | 187 ++++-------------- 9 files changed, 333 insertions(+), 177 deletions(-) rename airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/{other.py => api.py} (100%) rename airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/{web_analytics.py => streams.py} (87%) rename airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/{other.py => api.py} (100%) rename airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/{web_analytics.py => streams.py} (83%) diff --git a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml index ec50dfc82a725..9d720ff1de39d 100644 --- a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml @@ -29,35 +29,35 @@ acceptance_tests: timeout_seconds: 3600 empty_streams: - name: engagements_calls - bypass_reason: Unable to populate cost $20/month + bypass_reason: Unable to populate (cost $20/month) - covered by integration tests - name: owners_archived - bypass_reason: unable to populate + bypass_reason: Unable to populate - covered by integration tests - name: tickets_web_analytics - bypass_reason: Unable to populate + bypass_reason: Unable to populate - covered by integration tests - name: deals_web_analytics - bypass_reason: Unable to populate + bypass_reason: Unable to populate - covered by integration tests - name: companies_web_analytics - bypass_reason: Unable to populate + bypass_reason: Unable to populate - covered by integration tests - name: engagements_calls_web_analytics - bypass_reason: Unable to populate + bypass_reason: Unable to populate - covered by integration tests - name: engagements_emails_web_analytics - bypass_reason: Unable to populate + bypass_reason: Unable to populate - covered by integration tests - name: engagements_meetings_web_analytics - bypass_reason: Unable to populate + bypass_reason: Unable to populate - covered by integration tests - name: engagements_notes_web_analytics - bypass_reason: Unable to populate + bypass_reason: Unable to populate - covered by integration tests - name: engagements_tasks_web_analytics - bypass_reason: Unable to populate + bypass_reason: Unable to populate - covered by integration tests - name: goals_web_analytics - bypass_reason: Unable to populate + bypass_reason: Unable to populate - covered by integration tests - name: line_items_web_analytics - bypass_reason: Unable to populate + bypass_reason: Unable to populate - covered by integration tests - name: products_web_analytics - bypass_reason: Unable to populate + bypass_reason: Unable to populate - covered by integration tests - name: pets_web_analytics - bypass_reason: Unable to populate + bypass_reason: Unable to populate - covered by integration tests - name: cars_web_analytics - bypass_reason: Unable to populate + bypass_reason: Unable to populate - covered by integration tests full_refresh: tests: - config_path: secrets/config.json diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/__init__.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/__init__.py index f70ecfc3a89e7..775071a0ad2ad 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/__init__.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/__init__.py @@ -1 +1,133 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import copy +from datetime import datetime, timedelta +from typing import Any, Dict, List, Optional + +import freezegun +import pytz +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker +from airbyte_cdk.test.mock_http.response_builder import FieldPath, HttpResponseBuilder, RecordBuilder, create_record_builder, find_template +from airbyte_protocol.models import AirbyteStateMessage, SyncMode +from source_hubspot import SourceHubspot + +from .config_builder import ConfigBuilder +from .request_builders.api import CustomObjectsRequestBuilder, OAuthRequestBuilder, PropertiesRequestBuilder, ScopesRequestBuilder +from .request_builders.streams import CRMStreamRequestBuilder, IncrementalCRMStreamRequestBuilder, WebAnalyticsRequestBuilder +from .response_builder.helpers import RootHttpResponseBuilder +from .response_builder.api import ScopesAbstractResponseBuilder +from .response_builder.streams import GenericAbstractResponseBuilder, HubspotStreamResponseBuilder + + +@freezegun.freeze_time("2024-03-03T14:42:00Z") +class HubspotTestCase: + DT_FORMAT = '%Y-%m-%dT%H:%M:%SZ' + OBJECT_ID = "testID" + ACCESS_TOKEN = "new_access_token" + CURSOR_FIELD = "occurredAt" + PROPERTIES = { + "closed_date": "datetime", + "createdate": "datetime", + } + + @classmethod + def now(cls): + return datetime.now(pytz.utc) + + @classmethod + def start_date(cls): + return cls.now() - timedelta(days=30) + + @classmethod + def updated_at(cls): + return cls.now() - timedelta(days=1) + + @classmethod + def dt_str(cls, dt: datetime.date) -> str: + return dt.strftime(cls.DT_FORMAT) + + @classmethod + def oauth_config(cls, start_date: Optional[str] = None) -> Dict[str, Any]: + start_date = start_date or cls.dt_str(cls.start_date()) + return ConfigBuilder().with_start_date(start_date).with_auth( + { + "credentials_title": "OAuth Credentials", + "redirect_uri": "https://airbyte.io", + "client_id": "client_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token", + } + ).build() + + @classmethod + def private_token_config(cls, token: str, start_date: Optional[str] = None) -> Dict[str, Any]: + start_date = start_date or cls.dt_str(cls.start_date()) + return ConfigBuilder().with_start_date(start_date).with_auth( + { + "credentials_title": "Private App Credentials", + "access_token": token, + } + ).build() + + @classmethod + def mock_oauth(cls, http_mocker: HttpMocker, token: str): + creds = cls.oauth_config()["credentials"] + req = OAuthRequestBuilder().with_client_id( + creds["client_id"] + ).with_client_secret( + creds["client_secret"] + ).with_refresh_token( + creds["refresh_token"] + ).build() + response = GenericAbstractResponseBuilder().with_value("access_token", token).with_value("expires_in", 7200).build() + http_mocker.post(req, response) + + @classmethod + def mock_scopes(cls, http_mocker: HttpMocker, token: str, scopes: List[str]): + http_mocker.get(ScopesRequestBuilder().with_access_token(token).build(), ScopesAbstractResponseBuilder(scopes).build()) + + @classmethod + def mock_custom_objects(cls, http_mocker: HttpMocker): + http_mocker.get( + CustomObjectsRequestBuilder().build(), + HttpResponseBuilder({}, records_path=FieldPath("results"), pagination_strategy=None).build() + ) + + @classmethod + def mock_properties(cls, http_mocker: HttpMocker, object_type: str, properties: Dict[str, str]): + templates = find_template("properties", __file__) + record_builder = lambda: RecordBuilder(copy.deepcopy(templates[0]), id_path=None, cursor_path=None) + + response_builder = RootHttpResponseBuilder(templates) + for name, type in properties.items(): + record = record_builder().with_field(FieldPath("name"), name).with_field(FieldPath("type"), type) + response_builder = response_builder.with_record(record) + + http_mocker.get( + PropertiesRequestBuilder().for_entity(object_type).build(), + response_builder.build() + ) + + @classmethod + def mock_response(cls, http_mocker: HttpMocker, request, responses, method: str = "get"): + if not isinstance(responses, (list, tuple)): + responses = [responses] + getattr(http_mocker, method)(request, responses) + + @classmethod + def record_builder(cls, stream: str, record_cursor_path): + return create_record_builder( + find_template(stream, __file__), records_path=FieldPath("results"), record_id_path=None, record_cursor_path=record_cursor_path + ) + + @classmethod + def catalog(cls, stream: str, sync_mode: SyncMode): + return CatalogBuilder().with_stream(stream, sync_mode).build() + + @classmethod + def read_from_stream( + cls, cfg, stream: str, sync_mode: SyncMode, state: Optional[List[AirbyteStateMessage]] = None, expecting_exception: bool = False + ) -> EntrypointOutput: + return read(SourceHubspot(), cfg, cls.catalog(stream, sync_mode), state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/other.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/api.py similarity index 100% rename from airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/other.py rename to airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/api.py diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/web_analytics.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py similarity index 87% rename from airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/web_analytics.py rename to airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py index c3a59a871982a..e1659010259d5 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/web_analytics.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py @@ -45,6 +45,8 @@ def __init__(self): self._associations = "" self._dt_range = "" self._properties = "" + self._after = None + self._search = False def for_entity(self, entity): self._resource = entity @@ -54,6 +56,10 @@ def with_dt_range(self, start_date: Tuple, end_date: Tuple): self._dt_range = "&".join(["{}={}".format(*start_date), "{}={}".format(*end_date)]) return self + def with_page_token(self, next_page_token: Dict): + self._after = "&".join([f"{str(key)}={str(val)}" for key, val in next_page_token.items()]) + return self + def with_associations(self, associations: Iterable[str]): self._associations = "&".join([f"associations={a}" for a in associations]) return self @@ -76,13 +82,15 @@ def _query_params(self): self._archived, self._associations, self._limit, + self._after, self._dt_range, self._properties ] def build(self): q = "&".join(filter(None, self._query_params)) - return HttpRequest(url=self.URL.format(resource=self._resource), query_params=q) + url = self.URL.format(resource=self._resource) + return HttpRequest(url, query_params=q) class IncrementalCRMStreamRequestBuilder(CRMStreamRequestBuilder): diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/other.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/api.py similarity index 100% rename from airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/other.py rename to airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/api.py diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py index 93f7a4bdf294b..ded25153204f4 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py @@ -5,8 +5,8 @@ from airbyte_cdk.test.mock_http.response_builder import PaginationStrategy -class WebAnalyticsPaginationStrategy(PaginationStrategy): - NEXT_PAGE_TOKEN = {"after": "this_page_last_record_id"} +class HubspotPaginationStrategy(PaginationStrategy): + NEXT_PAGE_TOKEN = {"after": "256"} def update(self, response: Dict[str, Any]) -> None: response["paging"] = { diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/web_analytics.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/streams.py similarity index 83% rename from airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/web_analytics.py rename to airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/streams.py index 1870ca311199b..fd191bfa9e252 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/web_analytics.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/streams.py @@ -6,17 +6,17 @@ from airbyte_cdk.test.mock_http.response_builder import FieldPath, HttpResponseBuilder, find_template from . import AbstractResponseBuilder -from .pagination import WebAnalyticsPaginationStrategy +from .pagination import HubspotPaginationStrategy -class WebAnalyticsResponseBuilder(HttpResponseBuilder): +class HubspotStreamResponseBuilder(HttpResponseBuilder): @property def pagination_strategy(self): return self._pagination_strategy @classmethod def for_stream(cls, stream: str): - return cls(find_template(stream, __file__), FieldPath("results"), WebAnalyticsPaginationStrategy()) + return cls(find_template(stream, __file__), FieldPath("results"), HubspotPaginationStrategy()) class GenericAbstractResponseBuilder(AbstractResponseBuilder): diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py index 9f73b73ddf163..aff602a2b22f4 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py @@ -1,11 +1,142 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. -from unittest import TestCase +import http +import freezegun +import mock +from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import FieldPath +from airbyte_protocol.models import SyncMode -class TestEngagementCallsStreamFullRefresh(TestCase): - pass +from . import HubspotTestCase +from .request_builders.streams import CRMStreamRequestBuilder +from .response_builder.streams import HubspotStreamResponseBuilder -class TestEngagementCallsStreamIncremental(TestCase): - pass +@freezegun.freeze_time("2024-03-03T14:42:00Z") +class TestEngagementCallsStream(HubspotTestCase): + SCOPES = ["crm.objects.contacts.read"] + CURSOR_FIELD = "updatedAt" + STREAM_NAME = "engagements_calls" + OBJECT_TYPE = "calls" + ASSOCIATIONS = ["contacts", "deal", "company", "tickets"] + + def request(self, first_page: bool = True): + request = CRMStreamRequestBuilder().for_entity( + self.OBJECT_TYPE + ).with_associations( + self.ASSOCIATIONS + ).with_properties( + list(self.PROPERTIES.keys()) + ) + if not first_page: + response_builder = HubspotStreamResponseBuilder.for_stream(self.STREAM_NAME) + request = request.with_page_token(response_builder.pagination_strategy.NEXT_PAGE_TOKEN) + return request.build() + + def response(self, with_pagination: bool = False): + response_builder = HubspotStreamResponseBuilder.for_stream(self.STREAM_NAME) + record = self.record_builder(self.STREAM_NAME, FieldPath(self.CURSOR_FIELD)).with_field( + FieldPath(self.CURSOR_FIELD), self.dt_str(self.updated_at()) + ).with_field( + FieldPath("id"), self.OBJECT_TYPE + ) + response = response_builder.with_record(record) + if with_pagination: + response = response.with_pagination() + return response.build() + + @HttpMocker() + def test_given_one_page_when_read_stream_oauth_then_return_records(self, http_mocker: HttpMocker): + self.mock_oauth(http_mocker, self.ACCESS_TOKEN) + self.mock_scopes(http_mocker, self.ACCESS_TOKEN, self.SCOPES) + self.mock_custom_objects(http_mocker) + self.mock_properties(http_mocker, self.OBJECT_TYPE, self.PROPERTIES) + self.mock_response(http_mocker, self.request(), self.response()) + output = self.read_from_stream(self.oauth_config(), self.STREAM_NAME, SyncMode.full_refresh) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_one_page_when_read_stream_private_token_then_return_records(self, http_mocker: HttpMocker): + self.mock_custom_objects(http_mocker) + self.mock_properties(http_mocker, self.OBJECT_TYPE, self.PROPERTIES) + self.mock_response(http_mocker, self.request(), self.response()) + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_two_pages_when_read_then_return_records(self, http_mocker: HttpMocker): + self.mock_custom_objects(http_mocker) + self.mock_properties(http_mocker, self.OBJECT_TYPE, self.PROPERTIES) + self.mock_response(http_mocker, self.request(), self.response(with_pagination=True)) + self.mock_response(http_mocker, self.request(first_page=False), self.response()) + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) + assert len(output.records) == 2 + + @HttpMocker() + def test_given_error_response_when_read_analytics_then_get_trace_message(self, http_mocker: HttpMocker): + self.mock_custom_objects(http_mocker) + self.mock_properties(http_mocker, self.OBJECT_TYPE, self.PROPERTIES) + self.mock_response(http_mocker, self.request(), HttpResponse(status_code=500, body="{}")) + with mock.patch("time.sleep"): + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) + assert len(output.records) == 0 + assert len(output.trace_messages) > 0 + assert len(output.errors) > 0 + + @HttpMocker() + def test_given_500_then_200_when_read_then_return_records(self, http_mocker: HttpMocker): + self.mock_custom_objects(http_mocker) + self.mock_properties(http_mocker, self.OBJECT_TYPE, self.PROPERTIES) + self.mock_response( + http_mocker, + self.request(), + [ + HttpResponse(status_code=500, body="{}"), + self.response() + ] + ) + with mock.patch("time.sleep"): + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) + assert len(output.records) == 1 + assert len(output.trace_messages) > 0 + assert len(output.errors) == 0 + + @HttpMocker() + def test_given_missing_scopes_error_when_read_then_hault(self, http_mocker: HttpMocker): + self.mock_oauth(http_mocker, self.ACCESS_TOKEN) + self.mock_scopes(http_mocker, self.ACCESS_TOKEN, []) + self.read_from_stream(self.oauth_config(), self.STREAM_NAME, SyncMode.full_refresh, expecting_exception=True) + + @HttpMocker() + def test_given_unauthorized_error_when_read_then_hault(self, http_mocker: HttpMocker): + self.mock_custom_objects(http_mocker) + self.mock_properties(http_mocker, self.OBJECT_TYPE, self.PROPERTIES) + self.mock_response(http_mocker, self.request(), HttpResponse(status_code=http.HTTPStatus.UNAUTHORIZED, body="{}")) + with mock.patch("time.sleep"): + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) + assert len(output.records) == 0 + assert len(output.trace_messages) > 0 + assert len(output.errors) > 0 + + @HttpMocker() + def test_given_one_page_when_read_then_get_transformed_records(self, http_mocker: HttpMocker): + self.mock_custom_objects(http_mocker) + self.mock_properties(http_mocker, self.OBJECT_TYPE, self.PROPERTIES) + self.mock_response(http_mocker, self.request(), self.response()) + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) + record = output.records[0].record.data + assert "properties" in record # legacy struct remains to not introduce breaking changes + prop_fields = len([f for f in record if f.startswith("properties_")]) + assert prop_fields > 0 + + @HttpMocker() + def test_given_incremental_sync_when_read_then_state_message_produced_and_state_match_latest_record(self, http_mocker: HttpMocker): + self.mock_custom_objects(http_mocker) + self.mock_properties(http_mocker, self.OBJECT_TYPE, self.PROPERTIES) + self.mock_response(http_mocker, self.request(), self.response()) + output = self.read_from_stream( + self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.incremental + ) + assert len(output.state_messages) == 1 + diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py index a840b479ccff2..ce5fd6a7237df 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py @@ -1,27 +1,20 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. -import copy import http from datetime import datetime, timedelta -from typing import Any, Dict, List, Optional, Tuple +from typing import List, Optional, Tuple import freezegun import mock import pytest import pytz -from airbyte_cdk.test.catalog_builder import CatalogBuilder -from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse -from airbyte_cdk.test.mock_http.response_builder import FieldPath, HttpResponseBuilder, RecordBuilder, create_record_builder, find_template +from airbyte_cdk.test.mock_http.response_builder import FieldPath from airbyte_protocol.models import AirbyteStateBlob, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, StreamDescriptor, SyncMode -from source_hubspot import SourceHubspot -from .config_builder import ConfigBuilder -from .request_builders.other import CustomObjectsRequestBuilder, OAuthRequestBuilder, PropertiesRequestBuilder, ScopesRequestBuilder -from .request_builders.web_analytics import CRMStreamRequestBuilder, IncrementalCRMStreamRequestBuilder, WebAnalyticsRequestBuilder -from .response_builder.helpers import RootHttpResponseBuilder -from .response_builder.other import ScopesAbstractResponseBuilder -from .response_builder.web_analytics import GenericAbstractResponseBuilder, WebAnalyticsResponseBuilder +from . import HubspotTestCase +from .request_builders.streams import CRMStreamRequestBuilder, IncrementalCRMStreamRequestBuilder, WebAnalyticsRequestBuilder +from .response_builder.streams import HubspotStreamResponseBuilder CRM_STREAMS = ( ("tickets_web_analytics", "tickets", "ticket", ["contacts", "deals", "companies"]), @@ -42,118 +35,8 @@ ) -@freezegun.freeze_time("2024-03-03T14:42:00Z") -class WebAnalytics: - DT_FORMAT = '%Y-%m-%dT%H:%M:%SZ' - - @classmethod - def now(cls): - return datetime.now(pytz.utc) - - @classmethod - def start_date(cls): - return cls.now() - timedelta(days=30) - - @classmethod - def updated_at(cls): - return cls.now() - timedelta(days=1) - - @classmethod - def dt_str(cls, dt: datetime.date) -> str: - return dt.strftime(cls.DT_FORMAT) - - @classmethod - def oauth_config(cls, start_date: Optional[str] = None) -> Dict[str, Any]: - start_date = start_date or cls.dt_str(cls.start_date()) - return ConfigBuilder().with_start_date(start_date).with_auth( - { - "credentials_title": "OAuth Credentials", - "redirect_uri": "https://airbyte.io", - "client_id": "client_id", - "client_secret": "client_secret", - "refresh_token": "refresh_token", - } - ).build() - - @classmethod - def private_token_config(cls, token: str, start_date: Optional[str] = None) -> Dict[str, Any]: - start_date = start_date or cls.dt_str(cls.start_date()) - return ConfigBuilder().with_start_date(start_date).with_auth( - { - "credentials_title": "Private App Credentials", - "access_token": token, - } - ).build() - - @classmethod - def mock_oauth(cls, http_mocker: HttpMocker, token: str): - creds = cls.oauth_config()["credentials"] - req = OAuthRequestBuilder().with_client_id( - creds["client_id"] - ).with_client_secret( - creds["client_secret"] - ).with_refresh_token( - creds["refresh_token"] - ).build() - response = GenericAbstractResponseBuilder().with_value("access_token", token).with_value("expires_in", 7200).build() - http_mocker.post(req, response) - - @classmethod - def mock_scopes(cls, http_mocker: HttpMocker, token: str, scopes: List[str]): - http_mocker.get(ScopesRequestBuilder().with_access_token(token).build(), ScopesAbstractResponseBuilder(scopes).build()) - - @classmethod - def mock_custom_objects(cls, http_mocker: HttpMocker): - http_mocker.get( - CustomObjectsRequestBuilder().build(), - HttpResponseBuilder({}, records_path=FieldPath("results"), pagination_strategy=None).build() - ) - - @classmethod - def mock_properties(cls, http_mocker: HttpMocker, object_type: str, properties: Dict[str, str]): - templates = find_template("properties", __file__) - record_builder = lambda: RecordBuilder(copy.deepcopy(templates[0]), id_path=None, cursor_path=None) - - response_builder = RootHttpResponseBuilder(templates) - for name, type in properties.items(): - record = record_builder().with_field(FieldPath("name"), name).with_field(FieldPath("type"), type) - response_builder = response_builder.with_record(record) - - http_mocker.get( - PropertiesRequestBuilder().for_entity(object_type).build(), - response_builder.build() - ) - - @classmethod - def mock_parent_object( - cls, - http_mocker: HttpMocker, - object_ids: List[str], - object_type: str, - stream_name: str, - associations: List[str], - properties: List[str], - date_range: Optional[Tuple[str, ...]] = None, - ): - response_builder = WebAnalyticsResponseBuilder.for_stream(stream_name) - for object_id in object_ids: - record = cls.record_builder(stream_name, FieldPath("updatedAt")).with_field( - FieldPath("updatedAt"), cls.dt_str(cls.updated_at()) - ).with_field( - FieldPath("id"), object_id - ) - response_builder = response_builder.with_record(record) - - http_mocker.get( - CRMStreamRequestBuilder().for_entity(object_type).with_associations(associations).with_properties(properties).build(), - response_builder.build() - ) - - @classmethod - def mock_response(cls, http_mocker: HttpMocker, request, responses): - if not isinstance(responses, (list, tuple)): - responses = [responses] - http_mocker.get(request, responses) +class WebAnalyticsTestCase(HubspotTestCase): + PARENT_CURSOR_FIELD = "updatedAt" @classmethod def web_analytics_request( @@ -177,7 +60,7 @@ def web_analytics_request( } if not first_page: - response_builder = WebAnalyticsResponseBuilder.for_stream(stream) + response_builder = HubspotStreamResponseBuilder.for_stream(stream) query.update(response_builder.pagination_strategy.NEXT_PAGE_TOKEN) return WebAnalyticsRequestBuilder().with_token(token).with_query(query).build() @@ -186,42 +69,44 @@ def web_analytics_response( cls, stream: str, with_pagination: bool = False, updated_on: Optional[str] = None, id: Optional[str] = None ) -> HttpResponse: updated_on = updated_on or cls.dt_str(cls.updated_at()) - record = cls.record_builder(stream, FieldPath("occurredAt")).with_field(FieldPath("occurredAt"), updated_on) + record = cls.record_builder(stream, FieldPath(cls.CURSOR_FIELD)).with_field(FieldPath(cls.CURSOR_FIELD), updated_on) if id: record = record.with_field(FieldPath("objectId"), id) - response_builder = WebAnalyticsResponseBuilder.for_stream(stream) + response_builder = HubspotStreamResponseBuilder.for_stream(stream) response = response_builder.with_record(record) if with_pagination: response = response.with_pagination() return response.build() @classmethod - def record_builder(cls, stream: str, record_cursor_path): - return create_record_builder( - find_template(stream, __file__), records_path=FieldPath("results"), record_id_path=None, record_cursor_path=record_cursor_path - ) - - @classmethod - def catalog(cls, stream: str, sync_mode: SyncMode): - return CatalogBuilder().with_stream(stream, sync_mode).build() + def mock_parent_object( + cls, + http_mocker: HttpMocker, + object_ids: List[str], + object_type: str, + stream_name: str, + associations: List[str], + properties: List[str], + date_range: Optional[Tuple[str, ...]] = None, + ): + response_builder = HubspotStreamResponseBuilder.for_stream(stream_name) + for object_id in object_ids: + record = cls.record_builder(stream_name, FieldPath(cls.PARENT_CURSOR_FIELD)).with_field( + FieldPath(cls.PARENT_CURSOR_FIELD), cls.dt_str(cls.updated_at()) + ).with_field( + FieldPath("id"), object_id + ) + response_builder = response_builder.with_record(record) - @classmethod - def read_from_stream( - cls, cfg, stream: str, sync_mode: SyncMode, state: Optional[List[AirbyteStateMessage]] = None, expecting_exception: bool = False - ) -> EntrypointOutput: - return read(SourceHubspot(), cfg, cls.catalog(stream, sync_mode), state, expecting_exception) + http_mocker.get( + CRMStreamRequestBuilder().for_entity(object_type).with_associations(associations).with_properties(properties).build(), + response_builder.build() + ) @freezegun.freeze_time("2024-03-03T14:42:00Z") -class TestCRMWebAnalyticsStream(WebAnalytics): +class TestCRMWebAnalyticsStream(WebAnalyticsTestCase): SCOPES = ["tickets", "crm.objects.contacts.read", "crm.objects.companies.read", "contacts", "crm.objects.deals.read", "oauth"] - OBJECT_ID = "testID" - ACCESS_TOKEN = "new_access_token" - CURSOR_FIELD = "occurredAt" - PROPERTIES = { - "closed_date": "datetime", - "createdate": "datetime", - } @classmethod def extended_dt_ranges(cls) -> Tuple[Tuple[str, ...], ...]: @@ -559,10 +444,10 @@ def mock_parent_object( date_range: Optional[Tuple[str]] = None, ): date_range = date_range or (cls.dt_str(cls.start_date()), cls.dt_str(cls.now())) - response_builder = WebAnalyticsResponseBuilder.for_stream(stream_name) + response_builder = HubspotStreamResponseBuilder.for_stream(stream_name) for object_id in object_ids: - record = cls.record_builder(stream_name, FieldPath("updatedAt")).with_field( - FieldPath("updatedAt"), cls.dt_str(cls.updated_at()) + record = cls.record_builder(stream_name, FieldPath(cls.PARENT_CURSOR_FIELD)).with_field( + FieldPath(cls.PARENT_CURSOR_FIELD), cls.dt_str(cls.updated_at()) ).with_field( FieldPath("id"), object_id ) From 2d5dbfdea78a673dedd1eda3f1417d74a7eca174 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Sat, 16 Mar 2024 17:37:49 +0200 Subject: [PATCH 08/12] source hubspot: added test for owners archived stream --- .../unit_tests/integrations/__init__.py | 8 ++-- .../integrations/request_builders/streams.py | 23 ++++++++++ .../integrations/response_builder/api.py | 2 +- .../integrations/response_builder/streams.py | 2 +- .../integrations/test_engagements_calls.py | 2 +- .../integrations/test_owners_archived.py | 44 ++++++++++++++++--- .../http/response/owners_archived.json | 14 ++++++ 7 files changed, 83 insertions(+), 12 deletions(-) create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/owners_archived.json diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/__init__.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/__init__.py index 775071a0ad2ad..f8ebf27a08c25 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/__init__.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/__init__.py @@ -17,8 +17,8 @@ from .request_builders.api import CustomObjectsRequestBuilder, OAuthRequestBuilder, PropertiesRequestBuilder, ScopesRequestBuilder from .request_builders.streams import CRMStreamRequestBuilder, IncrementalCRMStreamRequestBuilder, WebAnalyticsRequestBuilder from .response_builder.helpers import RootHttpResponseBuilder -from .response_builder.api import ScopesAbstractResponseBuilder -from .response_builder.streams import GenericAbstractResponseBuilder, HubspotStreamResponseBuilder +from .response_builder.api import ScopesResponseBuilder +from .response_builder.streams import GenericResponseBuilder, HubspotStreamResponseBuilder @freezegun.freeze_time("2024-03-03T14:42:00Z") @@ -81,12 +81,12 @@ def mock_oauth(cls, http_mocker: HttpMocker, token: str): ).with_refresh_token( creds["refresh_token"] ).build() - response = GenericAbstractResponseBuilder().with_value("access_token", token).with_value("expires_in", 7200).build() + response = GenericResponseBuilder().with_value("access_token", token).with_value("expires_in", 7200).build() http_mocker.post(req, response) @classmethod def mock_scopes(cls, http_mocker: HttpMocker, token: str, scopes: List[str]): - http_mocker.get(ScopesRequestBuilder().with_access_token(token).build(), ScopesAbstractResponseBuilder(scopes).build()) + http_mocker.get(ScopesRequestBuilder().with_access_token(token).build(), ScopesResponseBuilder(scopes).build()) @classmethod def mock_custom_objects(cls, http_mocker: HttpMocker): diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py index e1659010259d5..e3b3583b1b7e4 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py @@ -103,3 +103,26 @@ def _query_params(self): self._associations, self._properties ] + + +class OwnersArchivedStreamRequestBuilder(AbstractRequestBuilder): + URL = "https://api.hubapi.com/crm/v3/owners" + + @property + def _limit(self): + return "limit=100" + + @property + def _archived(self): + return "archived=true" + + @property + def _query_params(self): + return [ + self._limit, + self._archived, + ] + + def build(self): + q = "&".join(filter(None, self._query_params)) + return HttpRequest(self.URL, query_params=q) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/api.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/api.py index b907c3f6b3a33..77ee027d612e8 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/api.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/api.py @@ -8,7 +8,7 @@ from . import AbstractResponseBuilder -class ScopesAbstractResponseBuilder(AbstractResponseBuilder): +class ScopesResponseBuilder(AbstractResponseBuilder): def __init__(self, scopes: List[str]): self._scopes = scopes diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/streams.py index fd191bfa9e252..6b02d952e14c0 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/streams.py @@ -19,7 +19,7 @@ def for_stream(cls, stream: str): return cls(find_template(stream, __file__), FieldPath("results"), HubspotPaginationStrategy()) -class GenericAbstractResponseBuilder(AbstractResponseBuilder): +class GenericResponseBuilder(AbstractResponseBuilder): def __init__(self): self._body = {} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py index aff602a2b22f4..ab6c6974e7d2c 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py @@ -39,7 +39,7 @@ def response(self, with_pagination: bool = False): record = self.record_builder(self.STREAM_NAME, FieldPath(self.CURSOR_FIELD)).with_field( FieldPath(self.CURSOR_FIELD), self.dt_str(self.updated_at()) ).with_field( - FieldPath("id"), self.OBJECT_TYPE + FieldPath("id"), self.OBJECT_ID ) response = response_builder.with_record(record) if with_pagination: diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py index 965923f277eba..eef1554548a07 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py @@ -1,11 +1,45 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. -from unittest import TestCase +import freezegun +from airbyte_cdk.test.mock_http import HttpMocker +from airbyte_cdk.test.mock_http.response_builder import FieldPath +from airbyte_protocol.models import SyncMode +from . import HubspotTestCase +from .request_builders.streams import OwnersArchivedStreamRequestBuilder +from .response_builder.streams import HubspotStreamResponseBuilder -class TestOwnersArchivedStreamFullRefresh(TestCase): - pass +@freezegun.freeze_time("2024-03-03T14:42:00Z") +class TestOwnersArchivedStream(HubspotTestCase): + """ + The test case contains a single test - this is just a sanity check, as the tested + stream is identical to the `Owners` stream (which is covered by acceptance tests), except for a single url param. + """ + SCOPES = ["crm.objects.owners.read"] + CURSOR_FIELD = "updatedAt" + STREAM_NAME = "owners_archived" -class TestOwnersArchivedStreamIncremental(TestCase): - pass + def request(self): + return OwnersArchivedStreamRequestBuilder().build() + + def response(self, with_pagination: bool = False): + response_builder = HubspotStreamResponseBuilder.for_stream(self.STREAM_NAME) + record = self.record_builder(self.STREAM_NAME, FieldPath(self.CURSOR_FIELD)).with_field( + FieldPath(self.CURSOR_FIELD), self.dt_str(self.updated_at()) + ).with_field( + FieldPath("id"), self.OBJECT_ID + ) + response = response_builder.with_record(record) + if with_pagination: + response = response.with_pagination() + return response.build() + + @HttpMocker() + def test_given_one_page_when_read_stream_oauth_then_return_records(self, http_mocker: HttpMocker): + self.mock_oauth(http_mocker, self.ACCESS_TOKEN) + self.mock_scopes(http_mocker, self.ACCESS_TOKEN, self.SCOPES) + self.mock_custom_objects(http_mocker) + self.mock_response(http_mocker, self.request(), self.response()) + output = self.read_from_stream(self.oauth_config(), self.STREAM_NAME, SyncMode.full_refresh) + assert len(output.records) == 1 diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/owners_archived.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/owners_archived.json new file mode 100644 index 0000000000000..28851eafbb2d6 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/owners_archived.json @@ -0,0 +1,14 @@ +{ + "results": [ + { + "id": "52550153", + "email": "integration-test@airbyte.io", + "firstName": "Team-1", + "lastName": "Airbyte", + "userId": 12282590, + "createdAt": "2020-10-28T21:17:56.082Z", + "updatedAt": "2023-01-31T00:25:34.448Z", + "archived": true + } + ] +} From e1ea1d53f9bd5378793e24f93b3ebed7a62b6147 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Mon, 25 Mar 2024 21:24:42 +0200 Subject: [PATCH 09/12] source-hubspot: review fixes --- .../integrations/request_builders/streams.py | 13 ++- .../integrations/test_engagements_calls.py | 68 +++++++----- .../integrations/test_owners_archived.py | 25 ++++- .../test_web_analytics_streams.py | 100 ++++++++++++++---- 4 files changed, 152 insertions(+), 54 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py index e3b3583b1b7e4..10c2519080f7c 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py @@ -98,6 +98,7 @@ class IncrementalCRMStreamRequestBuilder(CRMStreamRequestBuilder): def _query_params(self): return [ self._limit, + self._after, self._dt_range, self._archived, self._associations, @@ -108,6 +109,9 @@ def _query_params(self): class OwnersArchivedStreamRequestBuilder(AbstractRequestBuilder): URL = "https://api.hubapi.com/crm/v3/owners" + def __init__(self): + self._after = None + @property def _limit(self): return "limit=100" @@ -118,10 +122,15 @@ def _archived(self): @property def _query_params(self): - return [ + return filter(None, [ self._limit, + self._after, self._archived, - ] + ]) + + def with_page_token(self, next_page_token: Dict): + self._after = "&".join([f"{str(key)}={str(val)}" for key, val in next_page_token.items()]) + return self def build(self): q = "&".join(filter(None, self._query_params)) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py index ab6c6974e7d2c..a162c48328306 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py @@ -8,6 +8,7 @@ from airbyte_cdk.test.mock_http.response_builder import FieldPath from airbyte_protocol.models import SyncMode +from typing import Dict, Optional from . import HubspotTestCase from .request_builders.streams import CRMStreamRequestBuilder from .response_builder.streams import HubspotStreamResponseBuilder @@ -21,62 +22,77 @@ class TestEngagementCallsStream(HubspotTestCase): OBJECT_TYPE = "calls" ASSOCIATIONS = ["contacts", "deal", "company", "tickets"] - def request(self, first_page: bool = True): - request = CRMStreamRequestBuilder().for_entity( + @property + def response_builder(self): + return HubspotStreamResponseBuilder.for_stream(self.STREAM_NAME) + + def request(self, page_token: Optional[Dict[str, str]] = None): + request_builder = CRMStreamRequestBuilder().for_entity( self.OBJECT_TYPE ).with_associations( self.ASSOCIATIONS ).with_properties( list(self.PROPERTIES.keys()) ) - if not first_page: - response_builder = HubspotStreamResponseBuilder.for_stream(self.STREAM_NAME) - request = request.with_page_token(response_builder.pagination_strategy.NEXT_PAGE_TOKEN) - return request.build() + if page_token: + request_builder = request_builder.with_page_token(page_token) + return request_builder.build() def response(self, with_pagination: bool = False): - response_builder = HubspotStreamResponseBuilder.for_stream(self.STREAM_NAME) record = self.record_builder(self.STREAM_NAME, FieldPath(self.CURSOR_FIELD)).with_field( FieldPath(self.CURSOR_FIELD), self.dt_str(self.updated_at()) ).with_field( FieldPath("id"), self.OBJECT_ID ) - response = response_builder.with_record(record) + response = self.response_builder.with_record(record) if with_pagination: response = response.with_pagination() return response.build() - @HttpMocker() - def test_given_one_page_when_read_stream_oauth_then_return_records(self, http_mocker: HttpMocker): + def _set_up_oauth(self, http_mocker: HttpMocker): self.mock_oauth(http_mocker, self.ACCESS_TOKEN) self.mock_scopes(http_mocker, self.ACCESS_TOKEN, self.SCOPES) + + def _set_up_requests(self, http_mocker: HttpMocker, with_oauth: bool = False): + if with_oauth: + self._set_up_oauth(http_mocker) self.mock_custom_objects(http_mocker) self.mock_properties(http_mocker, self.OBJECT_TYPE, self.PROPERTIES) + + @HttpMocker() + def test_given_oauth_authentication_when_read_then_perform_authenticated_queries(self, http_mocker: HttpMocker): + self._set_up_requests(http_mocker, with_oauth=True) + self.read_from_stream(self.oauth_config(), self.STREAM_NAME, SyncMode.full_refresh) + + @HttpMocker() + def test_given_records_when_read_extract_desired_records(self, http_mocker: HttpMocker): + self._set_up_requests(http_mocker, with_oauth=True) self.mock_response(http_mocker, self.request(), self.response()) output = self.read_from_stream(self.oauth_config(), self.STREAM_NAME, SyncMode.full_refresh) assert len(output.records) == 1 @HttpMocker() def test_given_one_page_when_read_stream_private_token_then_return_records(self, http_mocker: HttpMocker): - self.mock_custom_objects(http_mocker) - self.mock_properties(http_mocker, self.OBJECT_TYPE, self.PROPERTIES) + self._set_up_requests(http_mocker) self.mock_response(http_mocker, self.request(), self.response()) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) assert len(output.records) == 1 @HttpMocker() def test_given_two_pages_when_read_then_return_records(self, http_mocker: HttpMocker): - self.mock_custom_objects(http_mocker) - self.mock_properties(http_mocker, self.OBJECT_TYPE, self.PROPERTIES) + self._set_up_requests(http_mocker) self.mock_response(http_mocker, self.request(), self.response(with_pagination=True)) - self.mock_response(http_mocker, self.request(first_page=False), self.response()) + self.mock_response( + http_mocker, + self.request(page_token=self.response_builder.pagination_strategy.NEXT_PAGE_TOKEN), + self.response() + ) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) assert len(output.records) == 2 @HttpMocker() def test_given_error_response_when_read_analytics_then_get_trace_message(self, http_mocker: HttpMocker): - self.mock_custom_objects(http_mocker) - self.mock_properties(http_mocker, self.OBJECT_TYPE, self.PROPERTIES) + self._set_up_requests(http_mocker) self.mock_response(http_mocker, self.request(), HttpResponse(status_code=500, body="{}")) with mock.patch("time.sleep"): output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) @@ -86,8 +102,7 @@ def test_given_error_response_when_read_analytics_then_get_trace_message(self, h @HttpMocker() def test_given_500_then_200_when_read_then_return_records(self, http_mocker: HttpMocker): - self.mock_custom_objects(http_mocker) - self.mock_properties(http_mocker, self.OBJECT_TYPE, self.PROPERTIES) + self._set_up_requests(http_mocker) self.mock_response( http_mocker, self.request(), @@ -103,15 +118,14 @@ def test_given_500_then_200_when_read_then_return_records(self, http_mocker: Htt assert len(output.errors) == 0 @HttpMocker() - def test_given_missing_scopes_error_when_read_then_hault(self, http_mocker: HttpMocker): + def test_given_missing_scopes_error_when_read_then_stop_sync(self, http_mocker: HttpMocker): self.mock_oauth(http_mocker, self.ACCESS_TOKEN) self.mock_scopes(http_mocker, self.ACCESS_TOKEN, []) self.read_from_stream(self.oauth_config(), self.STREAM_NAME, SyncMode.full_refresh, expecting_exception=True) @HttpMocker() - def test_given_unauthorized_error_when_read_then_hault(self, http_mocker: HttpMocker): - self.mock_custom_objects(http_mocker) - self.mock_properties(http_mocker, self.OBJECT_TYPE, self.PROPERTIES) + def test_given_unauthorized_error_when_read_then_stop_sync(self, http_mocker: HttpMocker): + self._set_up_requests(http_mocker) self.mock_response(http_mocker, self.request(), HttpResponse(status_code=http.HTTPStatus.UNAUTHORIZED, body="{}")) with mock.patch("time.sleep"): output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) @@ -120,9 +134,8 @@ def test_given_unauthorized_error_when_read_then_hault(self, http_mocker: HttpMo assert len(output.errors) > 0 @HttpMocker() - def test_given_one_page_when_read_then_get_transformed_records(self, http_mocker: HttpMocker): - self.mock_custom_objects(http_mocker) - self.mock_properties(http_mocker, self.OBJECT_TYPE, self.PROPERTIES) + def test_given_one_page_when_read_then_get_records_with_flattened_properties(self, http_mocker: HttpMocker): + self._set_up_requests(http_mocker) self.mock_response(http_mocker, self.request(), self.response()) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) record = output.records[0].record.data @@ -132,8 +145,7 @@ def test_given_one_page_when_read_then_get_transformed_records(self, http_mocker @HttpMocker() def test_given_incremental_sync_when_read_then_state_message_produced_and_state_match_latest_record(self, http_mocker: HttpMocker): - self.mock_custom_objects(http_mocker) - self.mock_properties(http_mocker, self.OBJECT_TYPE, self.PROPERTIES) + self._set_up_requests(http_mocker) self.mock_response(http_mocker, self.request(), self.response()) output = self.read_from_stream( self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.incremental diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py index eef1554548a07..d54a94ca6a9d8 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py @@ -21,25 +21,40 @@ class TestOwnersArchivedStream(HubspotTestCase): STREAM_NAME = "owners_archived" def request(self): - return OwnersArchivedStreamRequestBuilder().build() + return OwnersArchivedStreamRequestBuilder() + + @property + def response_builder(self): + return HubspotStreamResponseBuilder.for_stream(self.STREAM_NAME) def response(self, with_pagination: bool = False): - response_builder = HubspotStreamResponseBuilder.for_stream(self.STREAM_NAME) record = self.record_builder(self.STREAM_NAME, FieldPath(self.CURSOR_FIELD)).with_field( FieldPath(self.CURSOR_FIELD), self.dt_str(self.updated_at()) ).with_field( FieldPath("id"), self.OBJECT_ID ) - response = response_builder.with_record(record) + response = self.response_builder.with_record(record) if with_pagination: response = response.with_pagination() - return response.build() + return response @HttpMocker() def test_given_one_page_when_read_stream_oauth_then_return_records(self, http_mocker: HttpMocker): self.mock_oauth(http_mocker, self.ACCESS_TOKEN) self.mock_scopes(http_mocker, self.ACCESS_TOKEN, self.SCOPES) self.mock_custom_objects(http_mocker) - self.mock_response(http_mocker, self.request(), self.response()) + self.mock_response(http_mocker, self.request().build(), self.response().build()) output = self.read_from_stream(self.oauth_config(), self.STREAM_NAME, SyncMode.full_refresh) assert len(output.records) == 1 + + @HttpMocker() + def test_given_two_pages_when_read_stream_private_token_then_return_records(self, http_mocker: HttpMocker): + self.mock_custom_objects(http_mocker) + self.mock_response(http_mocker, self.request().build(), self.response(with_pagination=True).build()) + self.mock_response( + http_mocker, + self.request().with_page_token(self.response_builder.pagination_strategy.NEXT_PAGE_TOKEN).build(), + self.response().build() + ) + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) + assert len(output.records) == 2 diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py index ce5fd6a7237df..60e05b4234f74 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py @@ -38,6 +38,10 @@ class WebAnalyticsTestCase(HubspotTestCase): PARENT_CURSOR_FIELD = "updatedAt" + @classmethod + def response_builder(cls, stream): + return HubspotStreamResponseBuilder.for_stream(stream) + @classmethod def web_analytics_request( cls, @@ -60,8 +64,7 @@ def web_analytics_request( } if not first_page: - response_builder = HubspotStreamResponseBuilder.for_stream(stream) - query.update(response_builder.pagination_strategy.NEXT_PAGE_TOKEN) + query.update(cls.response_builder(stream).pagination_strategy.NEXT_PAGE_TOKEN) return WebAnalyticsRequestBuilder().with_token(token).with_query(query).build() @classmethod @@ -72,8 +75,7 @@ def web_analytics_response( record = cls.record_builder(stream, FieldPath(cls.CURSOR_FIELD)).with_field(FieldPath(cls.CURSOR_FIELD), updated_on) if id: record = record.with_field(FieldPath("objectId"), id) - response_builder = HubspotStreamResponseBuilder.for_stream(stream) - response = response_builder.with_record(record) + response = cls.response_builder(stream).with_record(record) if with_pagination: response = response.with_pagination() return response.build() @@ -87,9 +89,11 @@ def mock_parent_object( stream_name: str, associations: List[str], properties: List[str], + first_page: bool = True, + with_pagination: bool = False, date_range: Optional[Tuple[str, ...]] = None, ): - response_builder = HubspotStreamResponseBuilder.for_stream(stream_name) + response_builder = cls.response_builder(stream_name) for object_id in object_ids: record = cls.record_builder(stream_name, FieldPath(cls.PARENT_CURSOR_FIELD)).with_field( FieldPath(cls.PARENT_CURSOR_FIELD), cls.dt_str(cls.updated_at()) @@ -97,11 +101,13 @@ def mock_parent_object( FieldPath("id"), object_id ) response_builder = response_builder.with_record(record) + if with_pagination: + response_builder = response_builder.with_pagination() - http_mocker.get( - CRMStreamRequestBuilder().for_entity(object_type).with_associations(associations).with_properties(properties).build(), - response_builder.build() - ) + request_builder = CRMStreamRequestBuilder().for_entity(object_type).with_associations(associations).with_properties(properties) + if not first_page: + request_builder = request_builder.with_page_token(response_builder.pagination_strategy.NEXT_PAGE_TOKEN) + http_mocker.get(request_builder.build(), response_builder.build()) @freezegun.freeze_time("2024-03-03T14:42:00Z") @@ -176,6 +182,44 @@ def test_given_two_pages_when_read_then_return_records( output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) assert len(output.records) == 2 + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) + @HttpMocker() + def test_given_two_parent_pages_when_read_then_return_records( + self, stream_name, parent_stream_name, object_type, parent_stream_associations, http_mocker: HttpMocker + ): + self.mock_custom_objects(http_mocker) + self.mock_properties(http_mocker, object_type, self.PROPERTIES) + self.mock_parent_object( + http_mocker, + [self.OBJECT_ID], + object_type, + parent_stream_name, + parent_stream_associations, + with_pagination=True, + properties=list(self.PROPERTIES.keys()) + ) + self.mock_parent_object( + http_mocker, + ["another_object_id"], + object_type, + parent_stream_name, + parent_stream_associations, + first_page=False, + properties=list(self.PROPERTIES.keys()) + ) + self.mock_response( + http_mocker, + self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), + self.web_analytics_response(stream_name) + ) + self.mock_response( + http_mocker, + self.web_analytics_request(stream_name, self.ACCESS_TOKEN, "another_object_id", object_type), + self.web_analytics_response(stream_name) + ) + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) + assert len(output.records) == 2 + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) @HttpMocker() def test_given_wide_date_range_and_multiple_parent_records_when_read_then_return_records( @@ -192,7 +236,7 @@ def test_given_wide_date_range_and_multiple_parent_records_when_read_then_return parent_stream_name, parent_stream_associations, list(self.PROPERTIES.keys()), - start_to_end + date_range=start_to_end ) for dt_range in date_ranges: for _id in (self.OBJECT_ID, "another_object_id"): @@ -205,7 +249,7 @@ def test_given_wide_date_range_and_multiple_parent_records_when_read_then_return ) config_start_dt = date_ranges[0][0] output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN, config_start_dt), stream_name, SyncMode.full_refresh) - assert len(output.records) == 4 + assert len(output.records) == 4 # 2 parent objects * 2 datetime slices @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) @HttpMocker() @@ -328,6 +372,7 @@ def test_given_one_page_when_read_then_get_no_records_filtered( parent_stream_associations, http_mocker: HttpMocker ): + # validate that no filter is applied on the record set received from the API response self.mock_custom_objects(http_mocker) self.mock_properties(http_mocker, object_type, self.PROPERTIES) self.mock_parent_object( @@ -441,10 +486,12 @@ def mock_parent_object( stream_name: str, associations: List[str], properties: List[str], + first_page: bool = True, + with_pagination: bool = False, date_range: Optional[Tuple[str]] = None, ): date_range = date_range or (cls.dt_str(cls.start_date()), cls.dt_str(cls.now())) - response_builder = HubspotStreamResponseBuilder.for_stream(stream_name) + response_builder = cls.response_builder(stream_name) for object_id in object_ids: record = cls.record_builder(stream_name, FieldPath(cls.PARENT_CURSOR_FIELD)).with_field( FieldPath(cls.PARENT_CURSOR_FIELD), cls.dt_str(cls.updated_at()) @@ -452,15 +499,22 @@ def mock_parent_object( FieldPath("id"), object_id ) response_builder = response_builder.with_record(record) + if with_pagination: + response_builder = response_builder.with_pagination() start, end = date_range - http_mocker.get( - IncrementalCRMStreamRequestBuilder().for_entity(object_type).with_associations(associations).with_dt_range( - ("startTimestamp", cls.dt_conversion(start)), - ("endTimestamp", cls.dt_conversion(end)) - ).with_properties(properties).build(), - response_builder.build() - ) + request_builder = IncrementalCRMStreamRequestBuilder().for_entity( + object_type + ).with_associations( + associations + ).with_dt_range( + ("startTimestamp", cls.dt_conversion(start)), + ("endTimestamp", cls.dt_conversion(end)) + ).with_properties(properties) + if not first_page: + request_builder = request_builder.with_page_token(response_builder.pagination_strategy.NEXT_PAGE_TOKEN) + + http_mocker.get(request_builder.build(), response_builder.build()) @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) def test_given_one_page_when_read_stream_oauth_then_return_records( @@ -565,3 +619,11 @@ def test_given_state_with_current_slice_when_read_then_state_is_updated( super().test_given_state_with_current_slice_when_read_then_state_is_updated( stream_name, parent_stream_name, object_type, parent_stream_associations ) + + @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) + def test_given_two_parent_pages_when_read_then_return_records( + self, stream_name, parent_stream_name, object_type, parent_stream_associations + ): + super().test_given_two_parent_pages_when_read_then_return_records( + stream_name, parent_stream_name, object_type, parent_stream_associations + ) From 2956396e442bb3aaaf512c8f39bb0a07ae5c8751 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Mon, 25 Mar 2024 21:40:43 +0200 Subject: [PATCH 10/12] source hubspot: fix formatting errors --- .../unit_tests/integrations/test_engagements_calls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py index a162c48328306..395be501e4882 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py @@ -1,6 +1,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. import http +from typing import Dict, Optional import freezegun import mock @@ -8,7 +9,6 @@ from airbyte_cdk.test.mock_http.response_builder import FieldPath from airbyte_protocol.models import SyncMode -from typing import Dict, Optional from . import HubspotTestCase from .request_builders.streams import CRMStreamRequestBuilder from .response_builder.streams import HubspotStreamResponseBuilder From a34c5522faf40195e60c1a6ff0db5f8f84efeda1 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Tue, 9 Apr 2024 22:36:17 +0300 Subject: [PATCH 11/12] use stream state as a model --- .../unit_tests/integrations/test_web_analytics_streams.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py index 60e05b4234f74..7c7ed1dd4f4ee 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py @@ -406,7 +406,7 @@ def test_given_incremental_sync_when_read_then_state_message_produced_and_state_ ) assert len(output.state_messages) == 1 - cursor_value_from_state_message = output.most_recent_state.get(stream_name, {}).get(self.OBJECT_ID, {}).get(self.CURSOR_FIELD) + cursor_value_from_state_message = output.most_recent_state.stream_state.dict().get(stream_name, {}).get(self.OBJECT_ID, {}).get(self.CURSOR_FIELD) cursor_value_from_latest_record = output.records[-1].record.data.get(self.CURSOR_FIELD) assert cursor_value_from_state_message == cursor_value_from_latest_record @@ -437,8 +437,8 @@ def test_given_state_with_no_current_slice_when_read_then_current_slice_in_state self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.incremental, state=[current_state] ) assert len(output.state_messages) == 1 - assert output.most_recent_state.get(stream_name, {}).get(self.OBJECT_ID, {}).get(self.CURSOR_FIELD) - assert output.most_recent_state.get(stream_name, {}).get(another_object_id, {}).get(self.CURSOR_FIELD) + assert output.most_recent_state.stream_state.dict().get(stream_name, {}).get(self.OBJECT_ID, {}).get(self.CURSOR_FIELD) + assert output.most_recent_state.stream_state.dict().get(stream_name, {}).get(another_object_id, {}).get(self.CURSOR_FIELD) @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) @HttpMocker() @@ -466,7 +466,7 @@ def test_given_state_with_current_slice_when_read_then_state_is_updated( self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.incremental, state=[current_state] ) assert len(output.state_messages) == 1 - assert output.most_recent_state.get(stream_name, {}).get(self.OBJECT_ID, {}).get(self.CURSOR_FIELD) == self.dt_str(self.updated_at()) + assert output.most_recent_state.stream_state.dict().get(stream_name, {}).get(self.OBJECT_ID, {}).get(self.CURSOR_FIELD) == self.dt_str(self.updated_at()) @freezegun.freeze_time("2024-03-03T14:42:00Z") From 36aac48ac8b23e98421409f01bf74e9be90ff4e9 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Wed, 10 Apr 2024 20:09:12 +0300 Subject: [PATCH 12/12] fix for integration tests --- .../unit_tests/integrations/test_web_analytics_streams.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py index 7c7ed1dd4f4ee..3f1ceff5b3988 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py @@ -406,7 +406,7 @@ def test_given_incremental_sync_when_read_then_state_message_produced_and_state_ ) assert len(output.state_messages) == 1 - cursor_value_from_state_message = output.most_recent_state.stream_state.dict().get(stream_name, {}).get(self.OBJECT_ID, {}).get(self.CURSOR_FIELD) + cursor_value_from_state_message = output.most_recent_state.stream_state.dict().get(self.OBJECT_ID, {}).get(self.CURSOR_FIELD) cursor_value_from_latest_record = output.records[-1].record.data.get(self.CURSOR_FIELD) assert cursor_value_from_state_message == cursor_value_from_latest_record @@ -437,8 +437,8 @@ def test_given_state_with_no_current_slice_when_read_then_current_slice_in_state self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.incremental, state=[current_state] ) assert len(output.state_messages) == 1 - assert output.most_recent_state.stream_state.dict().get(stream_name, {}).get(self.OBJECT_ID, {}).get(self.CURSOR_FIELD) - assert output.most_recent_state.stream_state.dict().get(stream_name, {}).get(another_object_id, {}).get(self.CURSOR_FIELD) + assert output.most_recent_state.stream_state.dict().get(self.OBJECT_ID, {}).get(self.CURSOR_FIELD) + assert output.most_recent_state.stream_state.dict().get(another_object_id, {}).get(self.CURSOR_FIELD) @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) @HttpMocker() @@ -466,7 +466,7 @@ def test_given_state_with_current_slice_when_read_then_state_is_updated( self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.incremental, state=[current_state] ) assert len(output.state_messages) == 1 - assert output.most_recent_state.stream_state.dict().get(stream_name, {}).get(self.OBJECT_ID, {}).get(self.CURSOR_FIELD) == self.dt_str(self.updated_at()) + assert output.most_recent_state.stream_state.dict().get(self.OBJECT_ID, {}).get(self.CURSOR_FIELD) == self.dt_str(self.updated_at()) @freezegun.freeze_time("2024-03-03T14:42:00Z")