✨ Source auth0: add new streams Organizations, OrganizationMembers, OrganizationMemberRoles

airbyte-integrations/connectors/source-auth0/Dockerfile

@@ -34,5 +34,5 @@ COPY source_auth0 ./source_auth0
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.0
+LABEL io.airbyte.version=0.3.0
 LABEL io.airbyte.name=airbyte/source-auth0

airbyte-integrations/connectors/source-auth0/acceptance-test-config.yml

@@ -1,22 +1,30 @@
 connector_image: airbyte/source-auth0:dev
-tests:
+acceptance_tests:
   spec:
-    - spec_path: "source_auth0/spec.yaml"
+    tests:
+      - spec_path: "source_auth0/spec.yaml"
   connection:
-    - config_path: "secrets/config.json"
-      status: "succeed"
-    - config_path: "integration_tests/invalid_config.json"
-      status: "failed"
+    tests:
+      - config_path: "secrets/config.json"
+        status: "succeed"
+      - config_path: "integration_tests/invalid_config.json"
+        status: "failed"
   discovery:
-    - config_path: "secrets/config.json"
+    tests:
+      - config_path: "secrets/config.json"
   basic_read:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
-      empty_streams: []
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
+        empty_streams: []
+        fail_on_extra_columns: false
   incremental:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
-      future_state_path: "integration_tests/abnormal_state.json"
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
+        future_state:
+          future_state_path: "integration_tests/abnormal_state.json"
   full_refresh:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"

airbyte-integrations/connectors/source-auth0/integration_tests/configured_catalog.json

@@ -20,6 +20,36 @@
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite",
       "primary_key": [["client_id"]]
+    },
+    {
+      "stream": {
+        "name": "organizations",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite",
+      "primary_key": [["id"]]
+    },
+    {
+      "stream": {
+        "name": "organization_members",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite",
+      "primary_key": [["id"]]
+    },
+    {
+      "stream": {
+        "name": "organization_member_roles",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite",
+      "primary_key": [["id"]]
     }
   ]
 }

airbyte-integrations/connectors/source-auth0/metadata.yaml

@@ -2,7 +2,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 6c504e48-14aa-4221-9a72-19cf5ff1ae78
-  dockerImageTag: 0.2.0
+  dockerImageTag: 0.3.0
   dockerRepository: airbyte/source-auth0
   githubIssueLabel: source-auth0
   icon: auth0.svg

airbyte-integrations/connectors/source-auth0/source_auth0/schemas/clients.json

@@ -121,7 +121,7 @@
       }
     },
     "signing_keys": {
-      "type": ["array", "null"],
+      "type": ["null", "array"],
       "items": {
         "type": ["object", "null"],
         "additionalProperties": true

airbyte-integrations/connectors/source-auth0/source_auth0/schemas/organization_member_roles.json

@@ -0,0 +1,22 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": ["object", "null"],
+  "additionalProperties": true,
+  "properties": {
+    "id": {
+      "type": ["string", "null"]
+    },
+    "org_id": {
+      "type": ["string", "null"]
+    },
+    "user_id": {
+      "type": ["string", "null"]
+    },
+    "name": {
+      "type": ["string", "null"]
+    },
+    "description": {
+      "type": ["string", "null"]
+    }
+  }
+}

airbyte-integrations/connectors/source-auth0/source_auth0/schemas/organization_members.json

@@ -0,0 +1,25 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "type": ["object", "null"],
+    "additionalProperties": true,
+    "properties":{
+        "id": {
+          "type": ["string", "null"]
+        },
+        "org_id": {
+          "type": ["string", "null"]
+        },
+        "user_id": {
+          "type": ["string", "null"]
+        },
+        "name": {
+          "type": ["string", "null"]
+        },
+        "email": {
+          "type": ["string", "null"]
+        },
+        "picture": {
+          "type": ["string", "null"]
+        }
+    }
+}

airbyte-integrations/connectors/source-auth0/source_auth0/schemas/organizations.json

@@ -0,0 +1,24 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": ["object", "null"],
+  "additionalProperties": true,
+  "properties": {
+    "id": {
+      "type": ["string", "null"]
+    },
+    "name": {
+      "type": ["string", "null"]
+    },
+    "display_name": {
+      "type": ["string", "null"]
+    },
+    "branding": {
+      "type": ["object", "null"],
+      "additionalProperties": true
+    },
+    "metadata": {
+      "type": ["object", "null"],
+      "additionalProperties": true
+    }
+  }
+}

airbyte-integrations/connectors/source-auth0/source_auth0/schemas/users.json

@@ -56,8 +56,11 @@
       "type": ["string", "null"]
     },
     "multifactor": {
-      "type": ["object", "null"],
-      "additionalProperties": true
+      "type": ["null", "array"],
+      "additionalProperties": true,
+      "items": {
+        "type": ["string", "null"]
+      }
     },
     "last_ip": {
       "type": ["string", "null"]

airbyte-integrations/connectors/source-auth0/source_auth0/source.py

@@ -10,12 +10,21 @@
 
 import pendulum
 import requests
+from airbyte_cdk.models import SyncMode
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import IncrementalMixin, Stream
 from airbyte_cdk.sources.streams.http import HttpStream
 from source_auth0.utils import get_api_endpoint, initialize_authenticator
 
 
+def read_full_refresh(stream_instance: Stream):
+    slices = stream_instance.stream_slices(sync_mode=SyncMode.full_refresh)
+    for _slice in slices:
+        records = stream_instance.read_records(stream_slice=_slice, sync_mode=SyncMode.full_refresh)
+        for record in records:
+            yield record
+
+
 # Basic full refresh stream
 class Auth0Stream(HttpStream, ABC):
     api_version = "v2"
@@ -123,13 +132,69 @@ class Clients(Auth0Stream):
     primary_key = "client_id"
     resource_name = "clients"
 
+
 class Users(IncrementalAuth0Stream):
     min_id = "1900-01-01T00:00:00.000Z"
     primary_key = "user_id"
     resource_name = "users"
     cursor_field = "updated_at"
 
 
+class Organizations(Auth0Stream):
+    primary_key = "id"
+    resource_name = "organizations"
+
+
+class OrganizationMembers(Auth0Stream):
+    primary_key = "id"
+    resource_name = "members"
+
+    def __init__(self, url_base: str, *args, **kwargs):
+        super().__init__(url_base=url_base, *args, **kwargs)
+        self.organizations = Organizations(url_base=url_base, *args, **kwargs)
+
+    def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]:
+        for org in read_full_refresh(self.organizations):
+            for member in super().read_records(stream_slice={"organization_id": org["id"]}, **kwargs):
+                yield member
+
+    def path(self, stream_slice: Mapping[str, Any], **kwargs) -> str:
+        return f"organizations/{stream_slice['organization_id']}/members"
+
+    def parse_response(self, response: requests.Response, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping]:
+        record = response.json().get(self.resource_name)
+        for r in record:
+            r["org_id"] = stream_slice["organization_id"]
+            r["id"] = stream_slice["organization_id"] + "_" + r["user_id"]
+            yield r
+
+
+class OrganizationMemberRoles(Auth0Stream):
+    primary_key = "id"
+    resource_name = "roles"
+
+    def __init__(self, url_base: str, *args, **kwargs):
+        super().__init__(url_base=url_base, *args, **kwargs)
+        self.organization_members = OrganizationMembers(url_base=url_base, *args, **kwargs)
+
+    def path(self, stream_slice: Mapping[str, Any], **kwargs) -> str:
+        return f"organizations/{stream_slice['organization_id']}/members/{stream_slice['user_id']}/roles"
+
+    def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]:
+        for org_member in read_full_refresh(self.organization_members):
+            for role in super().read_records(
+                stream_slice={"organization_id": org_member["org_id"], "user_id": org_member["user_id"]}, **kwargs
+            ):
+                yield role
+
+    def parse_response(self, response: requests.Response, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping]:
+        record = response.json().get(self.resource_name)
+        for r in record:
+            r["org_id"] = stream_slice["organization_id"]
+            r["user_id"] = stream_slice["user_id"]
+            yield r
+
+
 # Source
 class SourceAuth0(AbstractSource):
     def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, any]:
@@ -152,4 +217,10 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) ->
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         initialization_params = {"authenticator": initialize_authenticator(config), "url_base": config.get("base_url")}
-        return [Clients(**initialization_params), Users(**initialization_params)]
+        return [
+            Clients(**initialization_params),
+            Organizations(**initialization_params),
+            OrganizationMembers(**initialization_params),
+            OrganizationMemberRoles(**initialization_params),
+            Users(**initialization_params),
+        ]

airbyte-integrations/connectors/source-auth0/unit_tests/conftest.py

@@ -240,7 +240,50 @@ def clients_instance():
         },
         "organization_usage": "deny",
         "organization_require_behavior": "no_prompt",
-        "client_authentication_methods": {"private_key_jwt": {"credentials": ["object"]}}
+        "client_authentication_methods": {"private_key_jwt": {"credentials": ["object"]}},
+    }
+
+
+@pytest.fixture()
+def organization_instance():
+    """
+    Clients instance object response
+    """
+    return {
+        "id": "my_org_id",
+        "name": "My application",
+        "display_name": "My display_name",
+        "branding": "brand",
+        "metadata": "metadata_example",
+    }
+
+
+@pytest.fixture()
+def organization_member_instance():
+    """
+    Clients instance object response
+    """
+    return {
+        "id": "my_org_id_my_user_id",
+        "org_id": "my_org_id",
+        "user_id": "my_user_id",
+        "name": "my_name",
+        "email": "my_email",
+        "picture": "my_picture",
+    }
+
+
+@pytest.fixture()
+def organization_member_roles_instance():
+    """
+    Clients instance object response
+    """
+    return {
+        "id": "something",
+        "org_id": "my_org_id",
+        "user_id": "my_user_id",
+        "name": "my_name",
+        "description": "desc",
     }
 
 

airbyte-integrations/connectors/source-auth0/unit_tests/test_source.py

@@ -6,7 +6,15 @@
 
 from airbyte_cdk.sources.streams.http.requests_native_auth.token import TokenAuthenticator
 from source_auth0.authenticator import Auth0Oauth2Authenticator
-from source_auth0.source import SourceAuth0, Clients, Users, initialize_authenticator
+from source_auth0.source import (
+    Clients,
+    OrganizationMemberRoles,
+    OrganizationMembers,
+    Organizations,
+    SourceAuth0,
+    Users,
+    initialize_authenticator,
+)
 
 
 class TestAuthentication:
@@ -71,14 +79,23 @@ def test_check_streams(self, requests_mock, oauth_config, api_url):
         source_auth0 = SourceAuth0()
         requests_mock.get(f"{api_url}/api/v2/users?per_page=1", json={"connect": "ok"})
         requests_mock.get(f"{api_url}/api/v2/clients?per_page=1", json={"connect": "ok"})
+        requests_mock.get(f"{api_url}/api/v2/organizations?per_page=1", json={"connect": "ok"})
+        requests_mock.get(f"{api_url}/api/v2/organizations/test_org_id/members?per_page=1", json={"connect": "ok"})
+        requests_mock.get(f"{api_url}/api/v2/organizations/test_org_id/members/test_user_id/roles?per_page=1", json={"connect": "ok"})
         requests_mock.post(f"{api_url}/oauth/token", json={"access_token": "test_token", "expires_in": 948})
         streams = source_auth0.streams(config=oauth_config)
 
-        streams_supported = [Clients, Users]
+        streams_supported = [
+            Clients,
+            Organizations,
+            OrganizationMembers,
+            OrganizationMemberRoles,
+            Users,
+        ]
 
         # check the number of streams supported
         assert len(streams) == len(streams_supported)
 
         # and each stream to be specific stream
-        assert isinstance(streams[0], streams_supported[0])
-        assert isinstance(streams[1], streams_supported[1])
+        for s in range(len(streams)):
+            assert isinstance(streams[s], streams_supported[s])