testing changes to streams

Author: chebalski

airbyte-integrations/connectors/source-medrio/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.7.11-alpine3.14 as base
+FROM python:3.9.11-alpine3.15 as base
 
 # build and load all requirements
 FROM base as builder
@@ -40,5 +40,5 @@ COPY source_medrio ./source_medrio
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.version=0.1.1
 LABEL io.airbyte.name=airbyte/source-medrio

airbyte-integrations/connectors/source-medrio/Makefile

@@ -8,21 +8,23 @@ build : login
 	@echo image=${IMG}
 	@docker buildx build . \
     --platform "linux/amd64,linux/arm64" \
-    --tag ${LATEST} \
+	--tag ${IMG} \
     --push
 	@docker buildx build . \
     --platform "linux/arm64" \
-    --tag ${LATEST} \
+	--tag ${IMG} \
     --load
 
 dev : login
 	@echo image=${IMG}
 	@docker buildx build . \
     --platform "linux/amd64,linux/arm64" \
+	--tag ${IMG} \
     --tag ${DEV} \
     --push
 	@docker buildx build . \
     --platform "linux/arm64" \
+	--tag ${IMG} \
     --tag ${DEV} \
     --load
 

airbyte-integrations/connectors/source-medrio/README.md

@@ -40,7 +40,7 @@ To build using Gradle, from the Airbyte repository root, run:
 
 #### Create credentials
 **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/medrio-dataviews)
-to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_medrio_dataviews/spec.json` file.
+to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_medrio/spec.json` file.
 Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
 See `integration_tests/sample_config.json` for a sample config file.
 
@@ -65,16 +65,6 @@ docker buildx build . \
     --tag 878324840923.dkr.ecr.us-east-1.amazonaws.com/airbyte/source-medrio:dev \
     --push &
 
-docker buildx build . \
-    --platform "linux/amd64" \
-    --tag 878324840923.dkr.ecr.us-east-1.amazonaws.com/airbyte/source-medrio:dev-amd \
-    --push &
-
-docker buildx build . \
-    --platform "linux/arm64" \
-    --tag 878324840923.dkr.ecr.us-east-1.amazonaws.com/airbyte/source-medrio:dev-arm \
-    --push &
-
 docker buildx build . \
     --platform "linux/arm64" \
     --tag 878324840923.dkr.ecr.us-east-1.amazonaws.com/airbyte/source-medrio:dev \

airbyte-integrations/connectors/source-medrio/source_medrio/source.py

@@ -9,8 +9,8 @@
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator
 from requests.api import request
-from .medrio_odm import MedrioOdmApi, MedrioOdmXml
-from .streams import ApprovalEvent, FormStatus, Studies, Queries, MedrioOdm
+from .medrio_odm import MedrioOdmApi
+from .streams import ApprovalEvent, FormStatus, Studies, Queries
 
 logger = AirbyteLogger()
 
@@ -44,9 +44,7 @@ def get_token(self, config, api_version: str) -> TokenAuthenticator:
         if api_version in auth_urls:
             request_url, grant_type = auth_urls.get(api_version)
         else:
-            raise KeyError(
-                f"api_version {api_version} not one of {list(auth_urls.keys())}"
-            )
+            raise KeyError(f"api_version {api_version} not one of {list(auth_urls.keys())}")
         headers = {"Content-Type": "application/x-www-form-urlencoded"}
         data = {
             "username": config["api_user_name"],
@@ -61,9 +59,7 @@ def get_token(self, config, api_version: str) -> TokenAuthenticator:
         else:
             raise RuntimeError(response.json().get("message"))
 
-    def discover(
-        self, logger: AirbyteLogger, config: Mapping[str, Any]
-    ) -> AirbyteCatalog:
+    def discover(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteCatalog:
         return super().discover(logger, config)
 
     def read(
@@ -81,10 +77,10 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         odm_api = MedrioOdmApi(api_key=config["enterprise_api_key"])
         studies = odm_api.get_studies()
         streams = []
-        for study_name in config["medrio_study_name_array"]:
-            stream = MedrioOdm(api=copy.deepcopy(odm_api), study_id=studies[study_name])
-            stream.update_schema(extra_schema={}, stream_name=study_name)
-            streams.append(stream)
+        # for study_name in config["medrio_study_name_array"]:
+        #     stream = MedrioOdm(api=copy.deepcopy(odm_api), study_id=studies[study_name])
+        #     stream.update_schema(extra_schema={}, stream_name=study_name)
+        #     streams.append(stream)
         return streams + [
             Studies(authenticator=auth_v1),
             Queries(authenticator=auth_v2),

airbyte-integrations/connectors/source-medrio/source_medrio/spec.json

@@ -7,8 +7,7 @@
     "required": [
       "api_user_name",
       "api_user_password",
-      "enterprise_api_key",
-      "medrio_study_name_array"
+      "enterprise_api_key"
     ],
     "additionalProperties": false,
     "properties": {
@@ -28,7 +27,8 @@
       },
       "medrio_study_name_array": {
         "type": "array",
-        "description": "Select Medrio studies to replicate by name."
+        "description": "Select Medrio studies to replicate by name.",
+        "default": []
       }
     }
   }

airbyte-integrations/connectors/source-medrio/source_medrio/streams.py

@@ -63,16 +63,6 @@ def request_params(
         return params
 
 
-class MedrioV1Stream(MedrioHttpStream):
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    def get_json_schema(self):
-        schema = super().get_json_schema()
-        schema["dynamically_determined_property"] = "property"
-        return schema
-
-
 class MedrioV1Stream(MedrioHttpStream):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
@@ -202,61 +192,61 @@ def path(self, **kwargs) -> str:
         return "QueryReports"
 
 
-class ClinicalData(MedrioV2StreamIncremental):
-    primary_key = "GlobalDatumId"
-
-    def __init__(self, api: MedrioOdmApi, study_id: str, **kwargs):
-        self.api = api
-        self.study_id = study_id
-        super().__init__(**kwargs)
-
-    def get_json_schema(self):
-        self.name = "odm_clinical"  # change just to get shared json_schema
-        schema = super().get_json_schema()
-        schema["properties"].update(self.extra_schema)
-        self.name = self.stream_name  # change name back now that we've got schema
-        return schema
-
-
-class DataAudit(MedrioV2StreamIncremental):
-    primary_key = "GlobalDatumId"
-    cursor_field = "DataEntryUtcDateVal"
-
-    def path(self, **kwargs) -> str:
-        return "DataAuditReports"
-
-
-class MedrioOdm(Stream):
-    primary_key = ["FormOID", "SubjectKey", "ItemGroupRepeatKey"]
-    cursor_field = "DateTimeStamp"
-    stream_name = None
-    name = "odm_clinical"
-
-    def __init__(self, api: MedrioOdmApi, study_id: str, **kwargs):
-        self.api = api
-        self.study_id = study_id
-        super().__init__(**kwargs)
-
-    def get_json_schema(self):
-        self.name = "odm_clinical"  # change just to get shared json_schema
-        schema = super().get_json_schema()
-        schema["properties"].update(self.extra_schema)
-        self.name = self.stream_name  # change name back now that we've got schema
-        return schema
-
-    def update_schema(self, extra_schema: dict, stream_name: str):
-        self.extra_schema = extra_schema
-        self.name = stream_name  # set the name of the schema
-        self.stream_name = stream_name  # capture extra param to keep name
-
-    def read_records(
-        self,
-        sync_mode: SyncMode,
-        cursor_field: List[str] = None,
-        stream_slice: Mapping[str, Any] = None,
-        stream_state: Mapping[str, Any] = None,
-    ) -> Iterable[Mapping[str, Any]]:
-        xml_string = self.api.main(content_type="AllData", study_id=self.study_id)
-        odm_xml = MedrioOdmXml(xml_string)
-        records = odm_xml.parse_clinical()
-        return records
+# class ClinicalData(MedrioV2StreamIncremental):
+#     primary_key = "GlobalDatumId"
+
+#     def __init__(self, api: MedrioOdmApi, study_id: str, **kwargs):
+#         self.api = api
+#         self.study_id = study_id
+#         super().__init__(**kwargs)
+
+#     def get_json_schema(self):
+#         self.name = "odm_clinical"  # change just to get shared json_schema
+#         schema = super().get_json_schema()
+#         schema["properties"].update(self.extra_schema)
+#         self.name = self.stream_name  # change name back now that we've got schema
+#         return schema
+
+
+# class DataAudit(MedrioV2StreamIncremental):
+#     primary_key = "GlobalDatumId"
+#     cursor_field = "DataEntryUtcDateVal"
+
+#     def path(self, **kwargs) -> str:
+#         return "DataAuditReports"
+
+
+# class MedrioOdm(Stream):
+#     primary_key = ["FormOID", "SubjectKey", "ItemGroupRepeatKey"]
+#     cursor_field = "DateTimeStamp"
+#     stream_name = None
+#     name = "odm_clinical"
+
+#     def __init__(self, api: MedrioOdmApi, study_id: str, **kwargs):
+#         self.api = api
+#         self.study_id = study_id
+#         super().__init__(**kwargs)
+
+#     def get_json_schema(self):
+#         self.name = "odm_clinical"  # change just to get shared json_schema
+#         schema = super().get_json_schema()
+#         schema["properties"].update(self.extra_schema)
+#         self.name = self.stream_name  # change name back now that we've got schema
+#         return schema
+
+#     def update_schema(self, extra_schema: dict, stream_name: str):
+#         self.extra_schema = extra_schema
+#         self.name = stream_name  # set the name of the schema
+#         self.stream_name = stream_name  # capture extra param to keep name
+
+#     def read_records(
+#         self,
+#         sync_mode: SyncMode,
+#         cursor_field: List[str] = None,
+#         stream_slice: Mapping[str, Any] = None,
+#         stream_state: Mapping[str, Any] = None,
+#     ) -> Iterable[Mapping[str, Any]]:
+#         xml_string = self.api.main(content_type="AllData", study_id=self.study_id)
+#         odm_xml = MedrioOdmXml(xml_string)
+#         records = odm_xml.parse_clinical()
+#         return records

airbyte-integrations/connectors/source-medrio/unit_tests/test_source.py

@@ -4,17 +4,17 @@
 
 from unittest.mock import MagicMock
 
-from source_medrio.source import SourceMedrioDataviews
+from source_medrio.source import SourceMedrio
 
 
 def test_check_connection(mocker):
-    source = SourceMedrioDataviews()
+    source = SourceMedrio()
     logger_mock, config_mock = MagicMock(), MagicMock()
     assert source.check_connection(logger_mock, config_mock) == (True, None)
 
 
 def test_streams(mocker):
-    source = SourceMedrioDataviews()
+    source = SourceMedrio()
     config_mock = MagicMock()
     streams = source.streams(config_mock)
     # TODO: replace this with your streams number