source-postgres: enable SAT high test strictness level (#20549)

Author: rodireich

airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.2.25
+Enable bypass reason for future state test config.[#20549](https://github.com/airbytehq/airbyte/pull/20549)
+
 ## 0.2.24
 Check for nullity of docker runner in `previous_discovered_catalog_fixture`.[#20899](https://github.com/airbytehq/airbyte/pull/20899)
 

airbyte-integrations/bases/source-acceptance-test/Dockerfile

@@ -33,7 +33,7 @@ COPY pytest.ini setup.py ./
 COPY source_acceptance_test ./source_acceptance_test
 RUN pip install .
 
-LABEL io.airbyte.version=0.2.24
+LABEL io.airbyte.version=0.2.25
 LABEL io.airbyte.name=airbyte/source-acceptance-test
 
 ENTRYPOINT ["python", "-m", "pytest", "-p", "source_acceptance_test.plugin", "-r", "fEsx"]

airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py

@@ -144,6 +144,7 @@ class FullRefreshConfig(BaseConfig):
 class FutureStateConfig(BaseConfig):
     future_state_path: Optional[str] = Field(description="Path to a state file with values in far future")
     missing_streams: List[EmptyStreamConfiguration] = Field(default=[], description="List of missings streams with valid bypass reasons.")
+    bypass_reason: Optional[str]
 
 
 class IncrementalConfig(BaseConfig):

airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_incremental.py

@@ -18,10 +18,14 @@
 @pytest.fixture(name="future_state_configuration")
 def future_state_configuration_fixture(inputs, base_path, test_strictness_level) -> Tuple[Path, List[EmptyStreamConfiguration]]:
     """Fixture with connector's future state path (relative to base_path)"""
-    if inputs.future_state and inputs.future_state.future_state_path:
+    if inputs.future_state and inputs.future_state.bypass_reason is not None:
+        pytest.skip("`future_state` has a bypass reason, skipping.")
+    elif inputs.future_state and inputs.future_state.future_state_path:
         return Path(base_path) / inputs.future_state.future_state_path, inputs.future_state.missing_streams
     elif test_strictness_level is Config.TestStrictnessLevel.high:
-        pytest.fail("High test strictness level error: a future state configuration must be provided in high test strictness level.")
+        pytest.fail(
+            "High test strictness level error: a future state configuration must be provided in high test strictness level or a bypass reason should be filled."
+        )
     else:
         pytest.skip("`future_state` not specified, skipping.")
 

airbyte-integrations/bases/source-acceptance-test/unit_tests/test_incremental.py

@@ -695,32 +695,39 @@ def test_state_with_abnormally_large_values(mocker, read_output, expectation):
     [
         pytest.param(
             Config.TestStrictnessLevel.high,
-            MagicMock(future_state=MagicMock(future_state_path="my_future_state_path", missing_streams=["foo", "bar"])),
+            MagicMock(future_state=MagicMock(future_state_path="my_future_state_path", missing_streams=["foo", "bar"], bypass_reason=None)),
             False,
             False,
             id="high test strictness level, future_state_path and missing streams are defined: run the test.",
         ),
         pytest.param(
             Config.TestStrictnessLevel.low,
-            MagicMock(future_state=MagicMock(future_state_path="my_future_state_path", missing_streams=["foo", "bar"])),
+            MagicMock(future_state=MagicMock(future_state_path="my_future_state_path", missing_streams=["foo", "bar"], bypass_reason=None)),
             False,
             False,
             id="low test strictness level, future_state_path and missing_streams are defined: run the test.",
         ),
         pytest.param(
             Config.TestStrictnessLevel.high,
-            MagicMock(future_state=MagicMock(future_state_path=None)),
+            MagicMock(future_state=MagicMock(future_state_path=None, bypass_reason=None)),
             True,
             False,
             id="high test strictness level, future_state_path and missing streams are defined: fail the test.",
         ),
         pytest.param(
             Config.TestStrictnessLevel.low,
-            MagicMock(future_state=MagicMock(future_state_path=None)),
+            MagicMock(future_state=MagicMock(future_state_path=None, bypass_reason=None)),
             False,
             True,
             id="low test strictness level, future_state_path not defined: skip the test.",
         ),
+        pytest.param(
+            Config.TestStrictnessLevel.high,
+            MagicMock(future_state=MagicMock(bypass_reason="valid bypass reason")),
+            False,
+            True,
+            id="high test strictness level, bypass_reason: skip test.",
+        ),
     ],
 )
 def test_future_state_configuration_fixture(mocker, test_strictness_level, inputs, expect_fail, expect_skip):

airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml

@@ -1,7 +1,43 @@
 # See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference)
 # for more information about how to configure these tests
 connector_image: airbyte/source-postgres:dev
-tests:
+test_strictness_level: high
+acceptance_tests:
   spec:
-    - spec_path: "src/test-integration/resources/expected_spec.json"
-      config_path: "src/test-integration/resources/dummy_config.json"
+    tests:
+      - spec_path: "src/test-integration/resources/expected_spec.json"
+        config_path: "secrets/config.json"
+      - spec_path: "src/test-integration/resources/expected_spec.json"
+        config_path: "secrets/config_cdc.json"
+  connection:
+    tests:
+      - config_path: "secrets/config.json"
+        status: "succeed"
+      - config_path: "secrets/config_cdc.json"
+        status: "succeed"
+  discovery:
+    tests:
+      - config_path: "secrets/config.json"
+      - config_path: "secrets/config_cdc.json"
+  basic_read:
+    tests:
+      - config_path: "secrets/config.json"
+        expect_records:
+          path: "integration_tests/expected_records.txt"
+      - config_path: "secrets/config_cdc.json"
+        expect_records:
+          path: "integration_tests/expected_records.txt"
+  full_refresh:
+    tests:
+      - config_path: "secrets/config.json"
+      - config_path: "secrets/config_cdc.json"
+  incremental:
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/incremental_configured_catalog.json"
+        future_state:
+          bypass_reason: "A java.lang.NullPointerException is thrown when a state with an invalid cursor value is passed"
+      - config_path: "secrets/config_cdc.json"
+        configured_catalog_path: "integration_tests/incremental_configured_catalog.json"
+        future_state:
+          bypass_reason: "A java.lang.NullPointerException is thrown when a state with an invalid cursor value is passed"

airbyte-integrations/connectors/source-postgres/acceptance-test-docker.sh

@@ -0,0 +1,5 @@
+This directory contains files used to run Source Acceptance Tests.
+* `abnormal_state.json` describes a connector state with a non-existing cursor value.
+* `expected_records.txt` lists all the records expected as the output of the basic read operation.
+* `incremental_configured_catalog.json` is a configured catalog used as an input of the `incremental` test.
+* `seed.sql` is the query we manually ran on a test postgres instance to seed it with test data and enable CDC.

airbyte-integrations/connectors/source-postgres/integration_tests/README.md

@@ -0,0 +1,13 @@
+[
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "id": 4
+      },
+      "stream_descriptor": {
+        "name": "id_and_name"
+      }
+    }
+  }
+]

airbyte-integrations/connectors/source-postgres/integration_tests/abnormal_state.json

@@ -0,0 +1,3 @@
+{"stream": "id_and_name", "data": {"id": 1, "name": "picard"}, "emitted_at": 999999}
+{"stream": "id_and_name", "data": {"id": 2, "name": "crusher"}, "emitted_at": 999999}
+{"stream": "id_and_name", "data": {"id": 3, "name": "vash"}, "emitted_at": 999999}

airbyte-integrations/connectors/source-postgres/integration_tests/expected_records.txt

@@ -0,0 +1,32 @@
+{
+    "streams": [
+        {
+            "stream": {
+                "name": "id_and_name",
+                "json_schema": {
+                    "type": "object",
+                    "properties": {
+                        "name": {
+                            "type": "string"
+                        },
+                        "id": {
+                            "type": "number",
+                            "airbyte_type": "integer"
+                        }
+                    }
+                },
+                "supported_sync_modes": [
+                    "full_refresh",
+                    "incremental"
+                ],
+                "default_cursor_field": [],
+                "source_defined_primary_key": [],
+                "namespace": "public"
+            },
+            "sync_mode": "incremental",
+            "destination_sync_mode": "append",
+            "cursor_field": ["id"],
+            "user_defined_primary_key": ["id"]
+        }
+    ]
+}

airbyte-integrations/connectors/source-postgres/integration_tests/incremental_configured_catalog.json

@@ -0,0 +1,36 @@
+ALTER ROLE postgres WITH REPLICATION;
+
+CREATE
+    TABLE
+        id_and_name(
+            id INTEGER,
+            name VARCHAR(200)
+        );
+
+INSERT
+    INTO
+        id_and_name(
+            id,
+            name
+        )
+    VALUES(
+        1,
+        'picard'
+    ),
+    (
+        2,
+        'crusher'
+    ),
+    (
+        3,
+        'vash'
+    );
+
+SELECT
+    pg_create_logical_replication_slot(
+        'debezium_slot',
+        'pgoutput'
+    );
+
+CREATE
+    PUBLICATION publication FOR ALL TABLES;

airbyte-integrations/connectors/source-postgres/integration_tests/seed.sql

@@ -236,6 +236,7 @@ This test verifies that sync produces no records when run with the STATE with ab
 | `configured_catalog_path` | string | `integration_tests/configured_catalog.json` | Path to configured catalog                                         |     |
 | `future_state_path`       | string | None                                        | Path to the state file with abnormally large cursor values         |     |
 | `timeout_seconds`         | int    | 20\*60                                      | Test execution timeout in seconds                                  |     |
+| `bypass_reason`           | string | None                                        | Explain why this test is bypassed                                 |     |
 
 ## Strictness level