🎉 Source Mixpanel: increase unit_test coverage up to 90% (airbytehq#13085)

Refactor tests, improve coverage up to 90%

Author: roman-romanov-o

airbyte-integrations/connectors/source-mixpanel/unit_tests/conftest.py

@@ -2,18 +2,25 @@
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
 
+from datetime import timedelta
+
 import pendulum
 import pytest
 
 
 @pytest.fixture
-def config():
+def start_date():
+    return pendulum.parse("2017-01-25T00:00:00Z").date()
+
+
+@pytest.fixture
+def config(start_date):
     return {
         "api_secret": "unexisting-secret",
         "attribution_window": 5,
         "project_timezone": "UTC",
         "select_properties_by_default": True,
-        "start_date": pendulum.parse("2017-01-25T00:00:00Z").date(),
-        "end_date": pendulum.parse("2017-02-25T00:00:00Z").date(),
+        "start_date": start_date,
+        "end_date": start_date + timedelta(days=31),
         "region": "US",
     }

airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py

@@ -2,6 +2,8 @@
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
 
+import copy
+
 import pytest
 from airbyte_cdk import AirbyteLogger
 from source_mixpanel.source import FunnelsList, SourceMixpanel, TokenAuthenticatorBase64
@@ -18,11 +20,21 @@ def check_connection_url(config):
     return get_url_to_mock(funnel_list)
 
 
-@pytest.mark.parametrize("response_code,expect_success", [(200, True), (400, False)])
-def test_check_connection(requests_mock, check_connection_url, config, response_code, expect_success):
-    requests_mock.register_uri("GET", check_connection_url, setup_response(response_code, {}))
+@pytest.mark.parametrize(
+    "response_code,expect_success,response_json",
+    [
+        (200, True, {}),
+        (400, False, {"error": "Request error"}),
+        (500, False, {"error": "Server error"}),
+    ],
+)
+def test_check_connection(requests_mock, check_connection_url, config, response_code, expect_success, response_json):
+    requests_mock.register_uri("GET", check_connection_url, setup_response(response_code, response_json))
     ok, error = SourceMixpanel().check_connection(logger, config)
     assert ok == expect_success and error != expect_success
+    expected_error = response_json.get("error")
+    if expected_error:
+        assert error == expected_error
 
 
 def test_check_connection_bad_config():
@@ -40,3 +52,11 @@ def test_check_connection_incomplete(config):
 def test_streams(config):
     streams = SourceMixpanel().streams(config)
     assert len(streams) == 7
+
+
+def test_streams_string_date(config):
+    config = copy.deepcopy(config)
+    config["start_date"] = "2020-01-01"
+    config["end_date"] = "2020-01-02"
+    streams = SourceMixpanel().streams(config)
+    assert len(streams) == 7

airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py

@@ -2,8 +2,10 @@
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
 
+from datetime import timedelta
 from unittest.mock import MagicMock
 
+import pendulum
 import pytest
 from airbyte_cdk import AirbyteLogger
 from airbyte_cdk.models import SyncMode
@@ -157,24 +159,27 @@ def test_engage_stream_incremental(requests_mock, engage_response):
 
     stream = Engage(authenticator=MagicMock())
 
-    records = stream.read_records(sync_mode=SyncMode.incremental, cursor_field=["created"], stream_state={"created": "2008-12-12T11:20:47"})
+    stream_state = {"created": "2008-12-12T11:20:47"}
+    records = stream.read_records(sync_mode=SyncMode.incremental, cursor_field=["created"], stream_state=stream_state)
 
-    records_length = sum(1 for _ in records)
-    assert records_length == 1
+    records = [item for item in records]
+    assert len(records) == 1
+    assert stream.get_updated_state(current_stream_state=stream_state, latest_record=records[-1]) == {"created": "2008-12-12T11:20:47"}
 
 
 def test_cohort_members_stream_incremental(requests_mock, engage_response, cohorts_response):
     requests_mock.register_uri("POST", MIXPANEL_BASE_URL + "engage?page_size=1000", engage_response)
     requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "cohorts/list", cohorts_response)
 
     stream = CohortMembers(authenticator=MagicMock())
-
+    stream_state = {"created": "2008-12-12T11:20:47"}
     records = stream.read_records(
-        sync_mode=SyncMode.incremental, cursor_field=["created"], stream_state={"created": "2008-12-12T11:20:47"}, stream_slice={"id": 1000}
+        sync_mode=SyncMode.incremental, cursor_field=["created"], stream_state=stream_state, stream_slice={"id": 1000}
     )
 
-    records_length = sum(1 for _ in records)
-    assert records_length == 1
+    records = [item for item in records]
+    assert len(records) == 1
+    assert stream.get_updated_state(current_stream_state=stream_state, latest_record=records[-1]) == {"created": "2008-12-12T11:20:47"}
 
 
 @pytest.fixture
@@ -199,13 +204,15 @@ def funnels_list_url(config):
 
 
 @pytest.fixture
-def funnels_response():
+def funnels_response(start_date):
+    first_date = start_date + timedelta(days=1)
+    second_date = start_date + timedelta(days=10)
     return setup_response(
         200,
         {
-            "meta": {"dates": ["2016-09-12" "2016-09-19" "2016-09-26"]},
+            "meta": {"dates": [str(first_date), str(second_date)]},
             "data": {
-                "2016-09-12": {
+                str(first_date): {
                     "steps": [],
                     "analysis": {
                         "completion": 20524,
@@ -214,7 +221,7 @@ def funnels_response():
                         "worst": 1,
                     },
                 },
-                "2016-09-19": {
+                str(second_date): {
                     "steps": [],
                     "analysis": {
                         "completion": 20500,
@@ -242,6 +249,25 @@ def test_funnels_stream(requests_mock, config, funnels_response, funnels_list_re
             records_arr.append(record)
 
     assert len(records_arr) == 4
+    last_record = records_arr[-1]
+    # Test without current state date
+    new_state = stream.get_updated_state(current_stream_state={}, latest_record=records_arr[-1])
+    assert new_state == {str(last_record["funnel_id"]): {"date": last_record["date"]}}
+
+    # Test with current state, that lesser than last record date
+    last_record_date = pendulum.parse(last_record["date"]).date()
+    new_state = stream.get_updated_state(
+        current_stream_state={str(last_record["funnel_id"]): {"date": str(last_record_date - timedelta(days=1))}},
+        latest_record=records_arr[-1],
+    )
+    assert new_state == {str(last_record["funnel_id"]): {"date": last_record["date"]}}
+
+    # Test with current state, that is greater, than last record date
+    new_state = stream.get_updated_state(
+        current_stream_state={str(last_record["funnel_id"]): {"date": str(last_record_date + timedelta(days=1))}},
+        latest_record=records_arr[-1],
+    )
+    assert new_state == {str(last_record["funnel_id"]): {"date": str(last_record_date + timedelta(days=1))}}
 
 
 @pytest.fixture
@@ -269,6 +295,25 @@ def test_engage_schema(requests_mock, engage_schema_response):
     assert records_length == 3
 
 
+def test_update_engage_schema(requests_mock):
+    stream = EngageSchema(authenticator=MagicMock())
+    requests_mock.register_uri(
+        "GET",
+        get_url_to_mock(stream),
+        setup_response(
+            200,
+            {
+                "results": {
+                    "$someNewSchemaField": {"count": 124, "type": "string"},
+                }
+            },
+        ),
+    )
+    engage_stream = Engage(authenticator=MagicMock())
+    engage_schema = engage_stream.get_json_schema()
+    assert "someNewSchemaField" in engage_schema["properties"]
+
+
 @pytest.fixture
 def annotations_response():
     return setup_response(