🐛 Sendgrid source: Gracefully handle malformed responses from sendgrid API (#4839)

Author: sherifnada

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87.json

@@ -2,7 +2,7 @@
   "sourceDefinitionId": "fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87",
   "name": "Sendgrid",
   "dockerRepository": "airbyte/source-sendgrid",
-  "dockerImageTag": "0.2.5",
+  "dockerImageTag": "0.2.6",
   "documentationUrl": "https://hub.docker.com/r/airbyte/source-sendgrid",
   "icon": "sendgrid.svg"
 }

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -73,7 +73,7 @@
 - sourceDefinitionId: fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87
   name: Sendgrid
   dockerRepository: airbyte/source-sendgrid
-  dockerImageTag: 0.2.5
+  dockerImageTag: 0.2.6
   documentationUrl: https://hub.docker.com/r/airbyte/source-sendgrid
   icon: sendgrid.svg
 - sourceDefinitionId: 9e0556f4-69df-4522-a3fb-03264d36b348

airbyte-integrations/connectors/source-sendgrid/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.5
+LABEL io.airbyte.version=0.2.6
 LABEL io.airbyte.name=airbyte/source-sendgrid

airbyte-integrations/connectors/source-sendgrid/README.md

@@ -98,3 +98,6 @@ You've checked out the repo, implemented a million dollar feature, and you're re
 1. Create a Pull Request
 1. Pat yourself on the back for being an awesome contributor
 1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master
+
+### Changelog
+See the [docs](https://docs.airbyte.io/integrations/sources/sendgrid#changelog) for the changelog.

airbyte-integrations/connectors/source-sendgrid/integration_tests/integration_test.py

@@ -0,0 +1,27 @@
+#
+# MIT License
+#
+# Copyright (c) 2020 Airbyte
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+
+
+def test_example():
+    assert True

airbyte-integrations/connectors/source-sendgrid/setup.py

@@ -31,6 +31,6 @@
     author="Airbyte",
     author_email="[email protected]",
     packages=find_packages(),
-    install_requires=["airbyte-cdk~=0.1", "backoff", "requests", "pytest==6.1.2"],
+    install_requires=["airbyte-cdk~=0.1", "backoff", "requests", "pytest==6.1.2", "pytest-mock"],
     package_data={"": ["*.json", "schemas/*.json"]},
 )

airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py

@@ -48,8 +48,23 @@ def parse_response(
     ) -> Iterable[Mapping]:
         json_response = response.json()
         records = json_response.get(self.data_field, []) if self.data_field is not None else json_response
-        for record in records:
-            yield record
+
+        if records is not None:
+            for record in records:
+                yield record
+        else:
+            # TODO sendgrid's API is sending null responses at times. This seems like a bug on the API side, so we're adding
+            #  log statements to help reproduce and prevent the connector from failing.
+            err_msg = (
+                f"Response contained no valid JSON data. Response body: {response.text}\n"
+                f"Response status: {response.status_code}\n"
+                f"Response body: {response.text}\n"
+                f"Response headers: {response.headers}\n"
+                f"Request URL: {response.request.url}\n"
+                f"Request body: {response.request.body}\n"
+            )
+            # do NOT print request headers as it contains auth token
+            self.logger.info(err_msg)
 
 
 class SendgridStreamOffsetPagination(SendgridStream):
@@ -72,7 +87,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str,
         return {"offset": self.offset}
 
 
-class SendgridStreamIncrementalMixin(HttpStream):
+class SendgridStreamIncrementalMixin(HttpStream, ABC):
     cursor_field = "created"
 
     def __init__(self, start_time: int, **kwargs):

airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py

@@ -22,9 +22,28 @@
 # SOFTWARE.
 #
 
+from unittest.mock import MagicMock
 
+import pytest
+import requests
 from airbyte_cdk.logger import AirbyteLogger
 from source_sendgrid.source import SourceSendgrid
+from source_sendgrid.streams import SendgridStream
+
+
+@pytest.fixture(name="sendgrid_stream")
+def sendgrid_stream_fixture(mocker) -> SendgridStream:
+    # Wipe the internal list of abstract methods to allow instantiating the abstract class without implementing its abstract methods
+    mocker.patch("source_sendgrid.streams.SendgridStream.__abstractmethods__", set())
+    # Mypy yells at us because we're init'ing an abstract class
+    return SendgridStream()  # type: ignore
+
+
+def test_parse_response_gracefully_handles_nulls(mocker, sendgrid_stream: SendgridStream):
+    response = requests.Response()
+    mocker.patch.object(response, "json", return_value=None)
+    mocker.patch.object(response, "request", return_value=MagicMock())
+    assert [] == list(sendgrid_stream.parse_response(response))
 
 
 def test_source_wrong_credentials():

docs/integrations/sources/sendgrid.md

@@ -41,3 +41,6 @@ Generate a API key using the [Sendgrid documentation](https://sendgrid.com/docs/
 
 We recommend creating a key specifically for Airbyte access. This will allow you to control which resources Airbyte should be able to access. The API key should be read-only on all resources except Marketing, where it needs Full Access.
 
+| Version | Date       | Pull Request | Subject |
+| :------ | :--------  | :-----       | :------ |
+| 0.2.6   | 2021-07-19 | [4839](https://github.com/airbytehq/airbyte/pull/4839) | Gracefully handle malformed responses from the API |