Small spec fixes to make sure they work with connector form UI (#21587)

Author: Joe Reuter

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -1510,7 +1510,7 @@
 - name: S3
   sourceDefinitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2
   dockerRepository: airbyte/source-s3
-  dockerImageTag: 0.1.29
+  dockerImageTag: 0.1.30
   documentationUrl: https://docs.airbyte.com/integrations/sources/s3
   icon: s3.svg
   sourceType: file
@@ -1558,7 +1558,7 @@
 - name: Sendgrid
   sourceDefinitionId: fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87
   dockerRepository: airbyte/source-sendgrid
-  dockerImageTag: 0.2.16
+  dockerImageTag: 0.3.0
   documentationUrl: https://docs.airbyte.com/integrations/sources/sendgrid
   icon: sendgrid.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -12660,7 +12660,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-s3:0.1.29"
+- dockerImage: "airbyte/source-s3:0.1.30"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/s3"
     changelogUrl: "https://docs.airbyte.com/integrations/sources/s3"
@@ -12869,14 +12869,10 @@
                 - "infer"
                 - "error"
                 order: 1
-                allOf:
-                - title: "UnexpectedFieldBehaviorEnum"
-                  description: "An enumeration."
-                  enum:
-                  - "ignore"
-                  - "infer"
-                  - "error"
-                  type: "string"
+                enum:
+                - "ignore"
+                - "infer"
+                - "error"
               block_size:
                 title: "Block Size"
                 description: "The chunk size in bytes to process at a time in memory\
@@ -13236,7 +13232,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-sendgrid:0.2.16"
+- dockerImage: "airbyte/source-sendgrid:0.3.0"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/sendgrid"
     connectionSpecification:
@@ -13256,9 +13252,7 @@
           order: 0
         start_time:
           title: "Start time"
-          type:
-          - "integer"
-          - "string"
+          type: "string"
           description: "Start time in ISO8601 format. Any data before this time point\
             \ will not be replicated."
           examples:

airbyte-integrations/connectors/source-s3/Dockerfile

@@ -17,5 +17,5 @@ COPY source_s3 ./source_s3
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.29
+LABEL io.airbyte.version=0.1.30
 LABEL io.airbyte.name=airbyte/source-s3

airbyte-integrations/connectors/source-s3/acceptance-test-config.yml

@@ -4,7 +4,8 @@ acceptance_tests:
   spec:
     tests:
     - backward_compatibility_tests_config:
-        disable_for_version: 0.1.26
+        # skip because allOf gets converted into enum
+        disable_for_version: 0.1.29
       spec_path: integration_tests/spec.json
 
   connection:

airbyte-integrations/connectors/source-s3/acceptance-test-docker.sh

@@ -186,14 +186,7 @@
                 "default": "infer",
                 "examples": ["ignore", "infer", "error"],
                 "order": 1,
-                "allOf": [
-                  {
-                    "title": "UnexpectedFieldBehaviorEnum",
-                    "description": "An enumeration.",
-                    "enum": ["ignore", "infer", "error"],
-                    "type": "string"
-                  }
-                ]
+                "enum": ["ignore", "infer", "error"]
               },
               "block_size": {
                 "title": "Block Size",

airbyte-integrations/connectors/source-s3/integration_tests/spec.json

@@ -87,6 +87,21 @@ def change_format_to_oneOf(schema: dict) -> dict:
             schema["properties"][prop]["oneOf"] = schema["properties"][prop].pop("anyOf")
         return schema
 
+    @staticmethod
+    def remove_enum_allOf(schema: dict) -> dict:
+        """
+        allOfs are not supported by the UI, but pydantic is automatically writing them for enums.
+        Unpack them into the root
+        """
+        objects_to_check = schema["properties"]["format"]["oneOf"]
+        for object in objects_to_check:
+            for key in object["properties"]:
+                property = object["properties"][key]
+                if "allOf" in property and "enum" in property["allOf"][0]:
+                    property["enum"] = property["allOf"][0]["enum"]
+                    property.pop("allOf")
+        return schema
+
     @staticmethod
     def check_provider_added(schema: dict) -> None:
         if "provider" not in schema["properties"]:
@@ -110,4 +125,5 @@ def schema(cls, *args: Any, **kwargs: Any) -> Dict[str, Any]:
         cls.check_provider_added(schema)
         schema = cls.change_format_to_oneOf(schema)
         schema = cls.resolve_refs(schema)
+        schema = cls.remove_enum_allOf(schema)
         return schema

airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/spec.py

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.16
+LABEL io.airbyte.version=0.3.0
 LABEL io.airbyte.name=airbyte/source-sendgrid

airbyte-integrations/connectors/source-sendgrid/Dockerfile

@@ -2,6 +2,8 @@ connector_image: airbyte/source-sendgrid:dev
 tests:
   spec:
     - spec_path: "source_sendgrid/spec.json"
+      backward_compatibility_tests_config:
+        disable_for_version: "0.2.16"
   connection:
     - config_path: "secrets/config.json"
       status: "succeed"

airbyte-integrations/connectors/source-sendgrid/acceptance-test-config.yml

@@ -16,7 +16,7 @@
       },
       "start_time": {
         "title": "Start time",
-        "type": ["integer", "string"],
+        "type": "string",
         "description": "Start time in ISO8601 format. Any data before this time point will not be replicated.",
         "examples": [
           "2021-12-12",

airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json

@@ -5,7 +5,7 @@
 import datetime
 import urllib
 from abc import ABC, abstractmethod
-from typing import Any, Iterable, Mapping, MutableMapping, Optional, Union
+from typing import Any, Iterable, Mapping, MutableMapping, Optional
 
 import pendulum
 import requests
@@ -91,7 +91,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str,
 class SendgridStreamIncrementalMixin(HttpStream, ABC):
     cursor_field = "created"
 
-    def __init__(self, start_time: Optional[Union[int, str]], **kwargs):
+    def __init__(self, start_time: Optional[str], **kwargs):
         super().__init__(**kwargs)
         self._start_time = start_time or 0
         if isinstance(self._start_time, str):

airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py

@@ -28,6 +28,7 @@
 )
 
 FAKE_NOW = pendulum.DateTime(2022, 1, 1, tzinfo=pendulum.timezone("utc"))
+FAKE_NOW_ISO_STRING = FAKE_NOW.to_iso8601_string()
 
 
 @pytest.fixture(name="sendgrid_stream")
@@ -59,7 +60,7 @@ def test_source_wrong_credentials():
 
 
 def test_messages_stream_request_params(mock_pendulum_now):
-    start_time = 1558359830
+    start_time = "2019-05-20T13:43:50.000Z"
     stream = Messages(start_time)
     state = {"last_event_time": 1558359000}
     request_params = stream.request_params(state)
@@ -70,7 +71,7 @@ def test_messages_stream_request_params(mock_pendulum_now):
 
 
 def test_streams():
-    streams = SourceSendgrid().streams(config={"apikey": "wrong.api.key123", "start_time": FAKE_NOW})
+    streams = SourceSendgrid().streams(config={"apikey": "wrong.api.key123", "start_time": FAKE_NOW_ISO_STRING})
 
     assert len(streams) == 15
 
@@ -89,10 +90,10 @@ def test_pagination(mocker):
 
 @patch.multiple(SendgridStreamIncrementalMixin, __abstractmethods__=set())
 def test_stream_state():
-    stream = SendgridStreamIncrementalMixin(start_time=FAKE_NOW)
+    stream = SendgridStreamIncrementalMixin(start_time=FAKE_NOW_ISO_STRING)
     state = {}
     request_params = stream.request_params(stream_state=state)
-    assert request_params == {"end_time": pendulum.now().int_timestamp, "start_time": FAKE_NOW}
+    assert request_params == {"end_time": pendulum.now().int_timestamp, "start_time": int(FAKE_NOW.timestamp())}
 
 
 @pytest.mark.parametrize(

airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py

@@ -209,6 +209,7 @@ The Jsonl parser uses pyarrow hence,only the line-delimited JSON format is suppo
 
 | Version | Date       | Pull Request                                                                                                    | Subject                                                                                 |
 |:--------|:-----------|:----------------------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------------------|
+| 0.1.30  | 2023-01-25 | [21587](https://github.com/airbytehq/airbyte/pull/21587) | Make sure spec works as expected in UI                                                                          |
 | 0.1.29  | 2023-01-19 | [21604](https://github.com/airbytehq/airbyte/pull/21604)                                                        | Handle OSError: skip unreachable keys and keep working on accessible ones. Warn a customer |
 | 0.1.28  | 2023-01-10 | [21210](https://github.com/airbytehq/airbyte/pull/21210)                                                        | Update block size for json file format                                                  |
 | 0.1.27  | 2022-12-08 | [20262](https://github.com/airbytehq/airbyte/pull/20262)                                                        | Check config settings for CSV file format                                               |

docs/integrations/sources/s3.md

@@ -78,6 +78,7 @@ The connector is restricted by normal Sendgrid [requests limitation](https://sen
 
 | Version | Date       | Pull Request                                             | Subject                                               |
 |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------|
+| 0.3.0  | 2023-01-25 | [21587](https://github.com/airbytehq/airbyte/pull/21587) | Make sure spec works as expected in UI - make starttime parameter an ISO string instead of a timestamp                                                                          |
 | 0.2.16  | 2022-11-02 | [18847](https://github.com/airbytehq/airbyte/pull/18847) | Skip the stream on `400, 401 - authorization required` with log message |
 | 0.2.15  | 2022-10-19 | [18182](https://github.com/airbytehq/airbyte/pull/18182) | Mark the sendgrid api key secret in the spec          |
 | 0.2.14  | 2022-09-07 | [16400](https://github.com/airbytehq/airbyte/pull/16400) | Change Start Time config parameter to datetime string |