Source Hubspot: enable SAT high strictness (#21775)

* Source Hubspot: enable SAT high strictness

* Ignore stream companies for basic read because one of the fields are constantly changing

* Ignore streams that change often

* Remove contacts stream from empty_streams

* Updated expected records

* add not empty streams to expected records

* Updated expacted records

* Added allowedHosts

* Skiped property_history stream for basic read

* Resolve packages dependecy conflict

---------

Co-authored-by: Serhii Lazebnyi <[email protected]>
Co-authored-by: Serhii Lazebnyi <[email protected]>

Author: arsenlosenko

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -801,6 +801,9 @@
   icon: hubspot.svg
   sourceType: api
   releaseStage: generally_available
+  allowedHosts:
+    hosts:
+      - api.hubapi.com
 - name: IP2Whois
   sourceDefinitionId: f23b7b7c-d705-49a3-9042-09add3b104a5
   dockerRepository: airbyte/source-ip2whois

airbyte-integrations/connectors/source-hubspot/README.md

@@ -91,7 +91,7 @@ and place them into `secrets/config.json`.
 python main.py spec
 python main.py check --config secrets/config.json
 python main.py discover --config secrets/config.json
-python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json
+python main.py read --config secrets/config.json --catalog sample_files/basic_read_catalog.json
 ```
 
 ## Testing

airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml

@@ -1,91 +1,69 @@
 connector_image: airbyte/source-hubspot:dev
+test_strictness_level: high
 acceptance_tests:
   spec:
     tests:
-      - spec_path: "source_hubspot/spec.yaml"
-        backward_compatibility_tests_config:
-          # API Key authentication is deleted from authorization choices so it is not backward compatible with previous spec
-          disable_for_version: "0.2.3"
+      - spec_path: source_hubspot/spec.yaml
   connection:
     tests:
-      - config_path: "secrets/config.json"
-        status: "succeed"
-      - config_path: "integration_tests/invalid_config.json"
-        status: "failed"
-      - config_path: "integration_tests/invalid_config_oauth.json"
-        status: "failed"
-      - config_path: "integration_tests/invalid_config_wrong_title.json"
-        status: "failed"
+      - config_path: secrets/config_oauth.json
+        status: succeed
+      - config_path: integration_tests/invalid_config.json
+        status: failed
+      - config_path: integration_tests/invalid_config_oauth.json
+        status: failed
+      - config_path: integration_tests/invalid_config_wrong_title.json
+        status: failed
   discovery:
     tests:
-      - config_path: "secrets/config.json"
-        backward_compatibility_tests_config:
-          # Quote stream is deleted so it is not backward compatible with previous discovery
-          disable_for_version: "0.2.3"
+      - config_path: secrets/config_oauth.json
   basic_read:
     tests:
-      - config_path: "secrets/config.json"
-        timeout_seconds: 600
-        configured_catalog_path: "sample_files/basic_read_oauth_catalog.json"
+      - config_path: secrets/config_oauth.json
         empty_streams:
-          - name: "form_submissions"
-            bypass_reason: "unable to populate"
-          - name: "form_submissions"
-            bypass_reason: "unable to populate"
-          - name: "ticket_pipelines"
-            bypass_reason: "unable to populate"
-          - name: "engagements_meetings"
-            bypass_reason: "unable to populate"
-          - name: "engagements_emails"
-            bypass_reason: "unable to populate"
-          - name: "engagements"
-            bypass_reason: "unable to populate"
-          - name: "engagements_calls"
-            bypass_reason: "unable to populate"
-          - name: "quotes"
-            bypass_reason: "unable to populate"
-        # This test commented out, since it produces errors during active testing
-        # expect_records:
-        #   path: "integration_tests/expected_records.jsonl"
-  incremental:
-    tests:
-      - config_path: "secrets/config.json"
-        configured_catalog_path: "sample_files/incremental_catalog.json"
-        future_state:
-          future_state_path: "integration_tests/abnormal_state.json"
+        - name: property_history
+          bypass_reason: has randomly calculated records
+        - name: form_submissions
+          bypass_reason: unable to populate
+        - name: ticket_pipelines
+          bypass_reason: unable to populate
+        - name: engagements_meetings
+          bypass_reason: unable to populate
+        - name: engagements_emails
+          bypass_reason: unable to populate
+        - name: engagements_calls
+          bypass_reason: unable to populate
+        - name: quotes
+          bypass_reason: unable to populate
+        timeout_seconds: 3600
+        expect_records:
+          path: integration_tests/expected_records.jsonl
+          exact_order: yes
+          extra_fields: yes
+          extra_records: no
   full_refresh:
     tests:
-      - config_path: "secrets/config.json"
-        configured_catalog_path: "sample_files/full_refresh_oauth_catalog.json"
+      - config_path: secrets/config_oauth.json
+        configured_catalog_path: sample_files/full_refresh_oauth_catalog.json
         ignored_fields:
-          "contact_lists": [ "properties", "ilsFilterBranch" ]
-          "companies": [ "properties", "hs_time_in_customer" ]
-          "companies": [ "properties", "hs_time_in_evangelist" ]
-          "companies": [ "properties", "hs_time_in_lead" ]
-          "companies": [ "properties", "hs_time_in_marketingqualifiedlead" ]
-          "companies": [ "properties", "hs_time_in_opportunity" ]
-          "companies": [ "properties", "hs_time_in_other" ]
-          "companies": [ "properties", "hs_time_in_salesqualifiedlead" ]
-          "companies": [ "properties", "hs_time_in_subscriber" ]
-          "contacts": [ "properties", "hs_time_in_customer" ]
-          "contacts": [ "properties", "hs_time_in_evangelist" ]
-          "contacts": [ "properties", "hs_time_in_lead" ]
-          "contacts": [ "properties", "hs_time_in_marketingqualifiedlead" ]
-          "contacts": [ "properties", "hs_time_in_opportunity" ]
-          "contacts": [ "properties", "hs_time_in_other" ]
-          "contacts": [ "properties", "hs_time_in_salesqualifiedlead" ]
-          "contacts": [ "properties", "hs_time_in_subscriber" ]
-          "deals": [ "properties", "hs_time_in_9567448" ]
-          "deals": [ "properties", "hs_time_in_9567449" ]
-          "deals": [ "properties", "hs_time_in_appointmentscheduled" ]
-          "deals": [ "properties", "hs_time_in_closedlost" ]
-          "deals": [ "properties", "hs_time_in_closedwon" ]
-          "deals": [ "properties", "hs_time_in_contractsent" ]
-          "deals": [ "properties", "hs_time_in_customclosedwonstage" ]
-          "deals": [ "properties", "hs_time_in_decisionmakerboughtin" ]
-          "deals": [ "properties", "hs_time_in_presentationscheduled" ]
-          "deals": [ "properties", "hs_time_in_qualifiedtobuy" ]
-          "tickets": [ "properties", "hs_time_in_1" ]
-          "tickets": [ "properties", "hs_time_in_2" ]
-          "tickets": [ "properties", "hs_time_in_3" ]
-          "tickets": [ "properties", "hs_time_in_4" ]
+          companies:
+          - properties
+          - hs_time_in_subscriber
+          contact_lists:
+          - properties
+          - ilsFilterBranch
+          contacts:
+          - properties
+          - hs_time_in_subscriber
+          deals:
+          - properties
+          - hs_time_in_qualifiedtobuy
+          tickets:
+          - properties
+          - hs_time_in_4
+  incremental:
+    tests:
+      - config_path: secrets/config_oauth.json
+        configured_catalog_path: sample_files/incremental_catalog.json
+        future_state:
+          future_state_path: integration_tests/abnormal_state.json

airbyte-integrations/connectors/source-hubspot/integration_tests/conftest.py

@@ -9,7 +9,7 @@
 
 @pytest.fixture(scope="session", name="config")
 def config_fixture():
-    with open("secrets/config.json", "r") as config_file:
+    with open("secrets/config_oauth.json", "r") as config_file:
         return json.load(config_file)