🎉 Source Okta: return deprovisioned users (#15001)

* add statuses to user filtering

* bump version

* upd

* add comment

* auto-bump connector version [ci skip]

* format

Co-authored-by: Octavia Squidington III <[email protected]>

Author: annalvova05

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -628,7 +628,7 @@
 - name: Okta
   sourceDefinitionId: 1d4fdb25-64fc-4569-92da-fcdca79a8372
   dockerRepository: airbyte/source-okta
-  dockerImageTag: 0.1.8
+  dockerImageTag: 0.1.9
   documentationUrl: https://docs.airbyte.io/integrations/sources/okta
   icon: okta.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -6027,7 +6027,7 @@
         - - "client_secret"
         oauthFlowOutputParameters:
         - - "access_token"
-- dockerImage: "airbyte/source-okta:0.1.8"
+- dockerImage: "airbyte/source-okta:0.1.9"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/okta"
     connectionSpecification:

airbyte-integrations/connectors/source-okta/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.8
+LABEL io.airbyte.version=0.1.9
 LABEL io.airbyte.name=airbyte/source-okta

airbyte-integrations/connectors/source-okta/acceptance-test-config.yml

@@ -16,6 +16,8 @@ tests:
   basic_read:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
+      expect_records:
+        path: "integration_tests/expected_records.txt"
     - config_path: "secrets/config_api_token.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
   full_refresh:

airbyte-integrations/connectors/source-okta/integration_tests/expected_records.txt

@@ -0,0 +1 @@
+{"stream": "users", "data": {"id": "00u5vhynsalJZ2eMg5d7", "status": "DEPROVISIONED", "created":"2022-07-21T20:46:27.000Z", "activated":"2022-07-21T20:47:13.000Z", "statusChanged": "2022-07-21T20:49:13.000Z", "lastLogin": null, "lastUpdated": "2022-07-21T20:49:13.000Z", "passwordChanged": "2022-07-21T20:47:35.000Z", "type": {"id": "otymj7cw2AJFAHvOO5d6"}, "profile": {"firstName": "Test", "lastName": "DEPROVISIONED", "mobilePhone": null, "secondEmail": null, "login": "[email protected]", "email": "[email protected]"}, "credentials": {"emails": [{"value": "[email protected]", "status": "VERIFIED", "type": "PRIMARY"}], "provider": {"type": "OKTA", "name": "OKTA"}}, "_links": {"self": {"href": "https://dev-01177082.okta.com/api/v1/users/00u5vhynsalJZ2eMg5d7"}}}, "emitted_at": 1655800476224}

airbyte-integrations/connectors/source-okta/source_okta/schemas/groups.json

@@ -1,9 +1,6 @@
 {
   "properties": {
     "_links": {
-      "additionalProperties": {
-        "type": ["array", "object", "null"]
-      },
       "type": ["object", "null"]
     },
     "created": {

airbyte-integrations/connectors/source-okta/source_okta/schemas/shared/shared-user.json

@@ -2,9 +2,6 @@
   "$id": "shared-users.json",
   "properties": {
     "_links": {
-      "additionalProperties": {
-        "type": ["object", "null"]
-      },
       "type": ["object", "null"]
     },
     "activated": {

airbyte-integrations/connectors/source-okta/source_okta/source.py

@@ -199,10 +199,28 @@ def request_params(
 class Users(IncrementalOktaStream):
     cursor_field = "lastUpdated"
     primary_key = "id"
+    # Should add all statuses to filter. Considering Okta documentation https://developer.okta.com/docs/reference/api/users/#list-all-users,
+    # users with "DEPROVISIONED" status are not returned by default.
+    statuses = ["ACTIVE", "DEPROVISIONED", "LOCKED_OUT", "PASSWORD_EXPIRED", "PROVISIONED", "RECOVERY", "STAGED", "SUSPENDED"]
 
     def path(self, **kwargs) -> str:
         return "users"
 
+    def request_params(
+        self,
+        stream_state: Mapping[str, Any],
+        stream_slice: Mapping[str, any] = None,
+        next_page_token: Mapping[str, Any] = None,
+    ) -> MutableMapping[str, Any]:
+        params = super().request_params(stream_state, stream_slice, next_page_token)
+        status_filters = " or ".join([f'status eq "{status}"' for status in self.statuses])
+        if "filter" in params:
+            # add status_filters to existing filters
+            params["filter"] = f'{params["filter"]} and ({status_filters})'
+        else:
+            params["filter"] = status_filters
+        return params
+
 
 class CustomRoles(OktaStream):
     primary_key = "id"

airbyte-integrations/connectors/source-okta/unit_tests/conftest.py

@@ -86,6 +86,12 @@ def auth_token_config():
     return {"credentials": {"auth_type": "api_token", "api_token": "test_token"}}
 
 
+@pytest.fixture()
+def user_status_filter():
+    statuses = ["ACTIVE", "DEPROVISIONED", "LOCKED_OUT", "PASSWORD_EXPIRED", "PROVISIONED", "RECOVERY", "STAGED", "SUSPENDED"]
+    return " or ".join([f'status eq "{status}"' for status in statuses])
+
+
 @pytest.fixture()
 def users_instance(api_url):
     """

airbyte-integrations/connectors/source-okta/unit_tests/test_source.py

@@ -19,7 +19,6 @@
 
 
 class TestAuthentication:
-
     def test_init_token_authentication_init(self, token_config, auth_token_config):
         source_okta = SourceOkta()
         token_authenticator_instance = source_okta.initialize_authenticator(config=token_config)