🎉 🎉 Source FB Marketing: performance and reliability fixes

airbyte-integrations/connectors/source-facebook-marketing/README.md

@@ -39,7 +39,7 @@ From the Airbyte repository root, run:
 **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/facebook-marketing)
 to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_facebook_marketing/spec.json` file.
 Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
-See `sample_files/sample_config.json` for a sample config file.
+See `integration_tests/sample_config.json` for a sample config file.
 
 **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source facebook-marketing test creds`
 and place them into `secrets/config.json`.
@@ -73,7 +73,7 @@ Then run any of the connector commands as follows:
 docker run --rm airbyte/source-facebook-marketing:dev spec
 docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-facebook-marketing:dev check --config /secrets/config.json
 docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-facebook-marketing:dev discover --config /secrets/config.json
-docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-facebook-marketing:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
+docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-facebook-marketing:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
 ```
 ## Testing
    Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named.

airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml

@@ -13,23 +13,10 @@ tests:
     - config_path: "secrets/config.json"
   basic_read:
     - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
-      timeout_seconds: 600
+      empty_streams: ["videos"]
   incremental:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog_without_insights.json"
       future_state_path: "integration_tests/future_state.json"
   full_refresh:
     - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
-      # Ad Insights API has estimated metrics in response, which is calculated based on another metrics.
-      # Sometimes API doesn't return estimated metrics. E.g, cost_per_estimated_ad_recallers is calculated
-      # as total amount spent divided by estimated ad recall lift rate. When second metric is equal to zero
-      # API may or may not return value. Such behavior causes sequential reads test failing.
-      # Because one read response contains this metric, and other doesn't.
-      # Therefore, it's needed to ignore fields like this in API responses.
-      ignored_fields:
-        "ads_insights_age_and_gender": ["cost_per_estimated_ad_recallers"]
-        "ad_creatives": ["thumbnail_url"]
-        "ad_account": ["age"]
-        "images": ["permalink_url"]

airbyte-integrations/connectors/source-facebook-marketing/integration_tests/configured_catalog.json

@@ -64,61 +64,29 @@
       "stream": {
         "name": "ads_insights",
         "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
         "default_cursor_field": ["date_start"],
-        "source_defined_primary_key": null,
-        "namespace": null
-      },
-      "sync_mode": "incremental",
-      "cursor_field": ["date_start"],
-      "destination_sync_mode": "append",
-      "primary_key": null
-    },
-    {
-      "stream": {
-        "name": "ads_insights_age_and_gender",
-        "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["date_start"],
-        "source_defined_primary_key": null,
-        "namespace": null
+        "source_defined_primary_key": []
       },
       "sync_mode": "incremental",
+      "primary_key": [],
       "cursor_field": ["date_start"],
-      "destination_sync_mode": "append",
-      "primary_key": null
+      "destination_sync_mode": "append"
     },
     {
       "stream": {
-        "name": "images",
+        "name": "ads_insights_platform_and_device",
         "json_schema": {},
+        "default_cursor_field": ["date_start"],
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["updated_time"],
-        "source_defined_primary_key": [["id"]],
-        "namespace": null
+        "source_defined_primary_key": []
       },
       "sync_mode": "incremental",
-      "cursor_field": null,
-      "destination_sync_mode": "append",
-      "primary_key": null
-    },
-    {
-      "stream": {
-        "name": "ad_account",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"],
-        "source_defined_cursor": null,
-        "default_cursor_field": null,
-        "source_defined_primary_key": [["id"]],
-        "namespace": null
-      },
-      "sync_mode": "full_refresh",
-      "cursor_field": null,
-      "destination_sync_mode": "append",
-      "primary_key": null
+      "primary_key": [],
+      "cursor_field": ["date_start"],
+      "destination_sync_mode": "append"
     }
   ]
 }

airbyte-integrations/connectors/source-facebook-marketing/integration_tests/configured_catalog_without_insights.json

@@ -7,8 +7,7 @@
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
         "default_cursor_field": ["updated_time"],
-        "source_defined_primary_key": [["id"]],
-        "namespace": null
+        "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "incremental",
       "cursor_field": null,
@@ -22,13 +21,11 @@
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
         "default_cursor_field": ["updated_time"],
-        "source_defined_primary_key": [["id"]],
-        "namespace": null
+        "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "incremental",
       "cursor_field": null,
-      "destination_sync_mode": "append",
-      "primary_key": null
+      "destination_sync_mode": "append"
     },
     {
       "stream": {
@@ -37,8 +34,7 @@
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
         "default_cursor_field": ["updated_time"],
-        "source_defined_primary_key": [["id"]],
-        "namespace": null
+        "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "incremental",
       "cursor_field": null,

airbyte-integrations/connectors/source-facebook-marketing/integration_tests/conftest.py

@@ -26,4 +26,7 @@ def config_with_wrong_account_fixture(config):
 
 @pytest.fixture(scope="session", name="config_with_include_deleted")
 def config_with_include_deleted_fixture(config):
-    return {**config, "include_deleted": True}
+    new_config = {**config, "include_deleted": True}
+    new_config.pop("_limit", None)
+    new_config.pop("end_date", None)
+    return new_config