Source Hubspot: fix for property_history stream (#13596)

* #259 on call: Source Hubspot - fix for the property_history stream which did not emit any records

* #259 Source Hubspot: upd changelog

* #259 oncall: hubspot review fixes

* auto-bump connector version

Co-authored-by: Octavia Squidington III <[email protected]>

Author: davydov-d

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -390,7 +390,7 @@
 - name: HubSpot
   sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c
   dockerRepository: airbyte/source-hubspot
-  dockerImageTag: 0.1.67
+  dockerImageTag: 0.1.68
   documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot
   icon: hubspot.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -3633,7 +3633,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-hubspot:0.1.67"
+- dockerImage: "airbyte/source-hubspot:0.1.68"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot"
     connectionSpecification:

airbyte-integrations/connectors/source-hubspot/Dockerfile

@@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.67
+LABEL io.airbyte.version=0.1.68
 LABEL io.airbyte.name=airbyte/source-hubspot

airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml

@@ -18,88 +18,32 @@ tests:
   basic_read:
     - config_path: "secrets/config.json"
       timeout_seconds: 600
-      configured_catalog_path: "sample_files/full_refresh_catalog.json"
-      empty_streams: ["workflows", "form_submissions", "ticket_pipelines", "property_history"]
+      configured_catalog_path: "sample_files/basic_read_catalog.json"
+      empty_streams: ["workflows", "form_submissions", "ticket_pipelines"]
+      expect_records:
+        path: "integration_tests/expected_records.txt"
     - config_path: "secrets/config_oauth.json"
       timeout_seconds: 600
-      configured_catalog_path: "sample_files/configured_catalog_for_oauth_config.json"
-      # The `campaigns` stream is empty in this case, because we use a catalog with
-      # incremental streams: subscription_changes and email_events (it takes a long time to read)
-      # and therefore the start date is set at 2021-10-10 for `config_oauth.json`,
-      # but the campaign was created on 2021-01-11
-      empty_streams: ["campaigns", "workflows", "contacts_list_memberships", "form_submissions", "ticket_pipelines", "property_history"]
+      configured_catalog_path: "sample_files/basic_read_oauth_catalog.json"
+      empty_streams: ["workflows", "contacts_list_memberships", "form_submissions", "ticket_pipelines"]
+      expect_records:
+        path: "integration_tests/expected_records.txt"
   incremental:
     - config_path: "secrets/config.json"
-      configured_catalog_path: "sample_files/configured_catalog.json"
+      configured_catalog_path: "sample_files/incremental_catalog.json"
       future_state_path: "integration_tests/abnormal_state.json"
   full_refresh:
     - config_path: "secrets/config.json"
+      # This config is the same as for basic read except for missing `property_history` stream.
+      # This stream emits records with dynamic timestamps under some conditions
+      # (properties like `hs_time_in_lead`, `hs_time_in_subscriber` etc.) which makes it impossible to compare records.
+      # Ignoring these fields makes testing this stream senseless because those valuable values of other property types
+      # will also be ignored. Instead, we test this stream against expected records in basic read test.
       configured_catalog_path: "sample_files/full_refresh_catalog.json"
-      ignored_fields:
-        "companies": ["properties", "hs_time_in_customer"]
-        "companies": ["properties", "hs_time_in_evangelist"]
-        "companies": ["properties", "hs_time_in_lead"]
-        "companies": ["properties", "hs_time_in_marketingqualifiedlead"]
-        "companies": ["properties", "hs_time_in_opportunity"]
-        "companies": ["properties", "hs_time_in_other"]
-        "companies": ["properties", "hs_time_in_salesqualifiedlead"]
-        "companies": ["properties", "hs_time_in_subscriber"]
-        "contacts": ["properties", "hs_time_in_customer"]
-        "contacts": ["properties", "hs_time_in_evangelist"]
-        "contacts": ["properties", "hs_time_in_lead"]
-        "contacts": ["properties", "hs_time_in_marketingqualifiedlead"]
-        "contacts": ["properties", "hs_time_in_opportunity"]
-        "contacts": ["properties", "hs_time_in_other"]
-        "contacts": ["properties", "hs_time_in_salesqualifiedlead"]
-        "contacts": ["properties", "hs_time_in_subscriber"]
-        "deals": ["properties", "hs_time_in_9567448"]
-        "deals": ["properties", "hs_time_in_9567449"]
-        "deals": ["properties", "hs_time_in_appointmentscheduled"]
-        "deals": ["properties", "hs_time_in_closedlost"]
-        "deals": ["properties", "hs_time_in_closedwon"]
-        "deals": ["properties", "hs_time_in_contractsent"]
-        "deals": ["properties", "hs_time_in_customclosedwonstage"]
-        "deals": ["properties", "hs_time_in_decisionmakerboughtin"]
-        "deals": ["properties", "hs_time_in_presentationscheduled"]
-        "deals": ["properties", "hs_time_in_qualifiedtobuy"]
-        "tickets": ["properties", "hs_time_in_1"]
-        "tickets": ["properties", "hs_time_in_2"]
-        "tickets": ["properties", "hs_time_in_3"]
-        "tickets": ["properties", "hs_time_in_4"]
-        "property_history": ["property", "hs_time_in_lead"]
-        "property_history": ["property", "hs_time_in_subscriber"]
     - config_path: "secrets/config_oauth.json"
-      configured_catalog_path: "sample_files/configured_catalog_for_oauth_config.json"
-      ignored_fields:
-        "companies": ["properties", "hs_time_in_customer"]
-        "companies": ["properties", "hs_time_in_evangelist"]
-        "companies": ["properties", "hs_time_in_lead"]
-        "companies": ["properties", "hs_time_in_marketingqualifiedlead"]
-        "companies": ["properties", "hs_time_in_opportunity"]
-        "companies": ["properties", "hs_time_in_other"]
-        "companies": ["properties", "hs_time_in_salesqualifiedlead"]
-        "companies": ["properties", "hs_time_in_subscriber"]
-        "contacts": ["properties", "hs_time_in_customer"]
-        "contacts": ["properties", "hs_time_in_evangelist"]
-        "contacts": ["properties", "hs_time_in_lead"]
-        "contacts": ["properties", "hs_time_in_marketingqualifiedlead"]
-        "contacts": ["properties", "hs_time_in_opportunity"]
-        "contacts": ["properties", "hs_time_in_other"]
-        "contacts": ["properties", "hs_time_in_salesqualifiedlead"]
-        "contacts": ["properties", "hs_time_in_subscriber"]
-        "deals": ["properties", "hs_time_in_9567448"]
-        "deals": ["properties", "hs_time_in_9567449"]
-        "deals": ["properties", "hs_time_in_appointmentscheduled"]
-        "deals": ["properties", "hs_time_in_closedlost"]
-        "deals": ["properties", "hs_time_in_closedwon"]
-        "deals": ["properties", "hs_time_in_contractsent"]
-        "deals": ["properties", "hs_time_in_customclosedwonstage"]
-        "deals": ["properties", "hs_time_in_decisionmakerboughtin"]
-        "deals": ["properties", "hs_time_in_presentationscheduled"]
-        "deals": ["properties", "hs_time_in_qualifiedtobuy"]
-        "tickets": ["properties", "hs_time_in_1"]
-        "tickets": ["properties", "hs_time_in_2"]
-        "tickets": ["properties", "hs_time_in_3"]
-        "tickets": ["properties", "hs_time_in_4"]
-        "property_history": ["property", "hs_time_in_lead"]
-        "property_history": ["property", "hs_time_in_subscriber"]
+      # This config is the same as for basic read except for missing `property_history` stream.
+      # This stream emits records with dynamic timestamps under some conditions
+      # (properties like `hs_time_in_lead`, `hs_time_in_subscriber` etc.) which makes it impossible to compare records.
+      # Ignoring these fields makes testing this stream senseless because those valuable values of other property types
+      # will also be ignored. Instead, we test this stream against expected records in basic read test.
+      configured_catalog_path: "sample_files/full_refresh_oauth_catalog.json"

airbyte-integrations/connectors/source-hubspot/integration_tests/abnormal_state.json

@@ -49,5 +49,8 @@
   },
   "tickets": {
     "updatedAt": "2221-10-12T13:37:56.412000+00:00"
+  },
+  "property_history": {
+    "timestamp": 7966012325000
   }
 }