🐛 Source Facebook Marketing: fix url parsing and add report that exposes conversions. (#5826)

Author: manavkohli

airbyte-integrations/connectors/source-facebook-marketing/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.14
+LABEL io.airbyte.version=0.2.15
 LABEL io.airbyte.name=airbyte/source-facebook-marketing

airbyte-integrations/connectors/source-facebook-marketing/integration_tests/abnormal_state.json

@@ -38,5 +38,9 @@
   "ads_insights_region": {
     "date_start": "2021-07-25T13:34:26Z",
     "include_deleted": true
+  },
+  "ads_insights_action_types": {
+    "date_start": "2021-07-25T13:34:26Z",
+    "include_deleted": true
   }
 }

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py

@@ -35,6 +35,7 @@
     Ads,
     AdSets,
     AdsInsights,
+    AdsInsightsActionType,
     AdsInsightsAgeAndGender,
     AdsInsightsCountry,
     AdsInsightsDma,
@@ -124,6 +125,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]:
             AdsInsightsRegion(**insights_args),
             AdsInsightsDma(**insights_args),
             AdsInsightsPlatformAndDevice(**insights_args),
+            AdsInsightsActionType(**insights_args),
         ]
 
     def spec(self, *args, **kwargs) -> ConnectorSpecification:

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams.py

@@ -46,16 +46,19 @@
 backoff_policy = retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5)
 
 
-def remove_params_from_url(url, params):
-    parsed_url = urlparse.urlparse(url)
-    res_query = []
-    for q in parsed_url.query.split("&"):
-        key, value = q.split("=")
-        if key not in params:
-            res_query.append(f"{key}={value}")
-
-    parse_result = parsed_url._replace(query="&".join(res_query))
-    return urlparse.urlunparse(parse_result)
+def remove_params_from_url(url: str, params: [str]) -> str:
+    """
+    Parses a URL and removes the query parameters specified in params
+    :param url: URL
+    :param params: list of query parameters
+    :return: URL with params removed
+    """
+    parsed = urlparse.urlparse(url)
+    query = urlparse.parse_qs(parsed.query, keep_blank_values=True)
+    filtered = dict((k, v) for k, v in query.items() if k not in params)
+    return urlparse.urlunparse(
+        [parsed.scheme, parsed.netloc, parsed.path, parsed.params, urlparse.urlencode(filtered, doseq=True), parsed.fragment]
+    )
 
 
 class FBMarketingStream(Stream, ABC):
@@ -479,3 +482,8 @@ class AdsInsightsDma(AdsInsights):
 class AdsInsightsPlatformAndDevice(AdsInsights):
     breakdowns = ["publisher_platform", "platform_position", "impression_device"]
     action_breakdowns = ["action_type"]  # FB Async Job fails for unknown reason if we set other breakdowns
+
+
+class AdsInsightsActionType(AdsInsights):
+    breakdowns = []
+    action_breakdowns = ["action_type"]

airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_streams.py

@@ -0,0 +1,66 @@
+#
+# MIT License
+#
+# Copyright (c) 2020 Airbyte
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+from source_facebook_marketing.streams import remove_params_from_url
+
+
+class TestUrlParsing:
+    def test_empty_url(self):
+        url = ""
+        parsed_url = remove_params_from_url(url=url, params=[])
+        assert parsed_url == url
+
+    def test_does_not_raise_exception_for_invalid_url(self):
+        url = "abcd"
+        parsed_url = remove_params_from_url(url=url, params=["test"])
+        assert parsed_url == url
+
+    def test_escaped_characters(self):
+        url = "https://google.com?test=123%23%24%25%2A&test2=456"
+        parsed_url = remove_params_from_url(url=url, params=["test3"])
+        assert parsed_url == url
+
+    def test_no_params_url(self):
+        url = "https://google.com"
+        parsed_url = remove_params_from_url(url=url, params=["test"])
+        assert parsed_url == url
+
+    def test_no_params_arg(self):
+        url = "https://google.com?"
+        parsed_url = remove_params_from_url(url=url, params=["test"])
+        assert parsed_url == "https://google.com"
+
+    def test_partially_empty_params(self):
+        url = "https://google.com?test=122&&"
+        parsed_url = remove_params_from_url(url=url, params=[])
+        assert parsed_url == "https://google.com?test=122"
+
+    def test_no_matching_params(self):
+        url = "https://google.com?test=123"
+        parsed_url = remove_params_from_url(url=url, params=["test2"])
+        assert parsed_url == url
+
+    def test_removes_params(self):
+        url = "https://google.com?test=123&test2=456"
+        parsed_url = remove_params_from_url(url=url, params=["test2"])
+        assert parsed_url == "https://google.com?test=123"