🎉 Source Okta: added parameter 'start_date'

airbyte-integrations/connectors/source-okta/source_okta/source.py

@@ -82,6 +82,10 @@ class IncrementalOktaStream(OktaStream, ABC):
     def cursor_field(self) -> str:
         pass
 
+    def __init__(self, url_base, default_start_date: pendulum.Pendulum = None, **kwargs):
+        self.default_start_date = default_start_date
+        super().__init__(url_base=url_base, **kwargs)
+
     def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]:
         lowest_date = str(pendulum.datetime.min)
         return {
@@ -134,7 +138,7 @@ def request_params(
         stream_slice: Mapping[str, any] = None,
         next_page_token: Mapping[str, Any] = None,
     ) -> MutableMapping[str, Any]:
-        params = OktaStream.request_params(self, stream_state, stream_slice, next_page_token)
+        params = super().request_params(stream_state, stream_slice, next_page_token)
         latest_entry = stream_state.get(self.cursor_field)
         if latest_entry:
             params["after"] = latest_entry
@@ -180,9 +184,11 @@ def request_params(
         # The log stream use a different params to get data
         # https://developer.okta.com/docs/reference/api/system-log/#datetime-filter
         stream_state = stream_state or {}
-        params = OktaStream.request_params(self, stream_state, stream_slice, next_page_token)
+        params = super().request_params(stream_state, stream_slice, next_page_token)
         latest_entry = stream_state.get(self.cursor_field)
-        if latest_entry:
+        if self.default_start_date and not latest_entry:
+            params["since"] = self.default_start_date
+        elif latest_entry:
             params["since"] = latest_entry
             # [Test-driven Development] Set until When the cursor value from the stream state
             #   is abnormally large, otherwise the server side that sets now to until
@@ -329,14 +335,20 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         auth = self.initialize_authenticator(config)
         api_endpoint = self.get_api_endpoint(config)
 
+        default_start_date = pendulum.parse(config["start_date"]) if "start_date" in config else None
         initialization_params = {
+            "authenticator": auth,
+            "url_base": api_endpoint
+        }
+        initialization_params_with_start_date = {
             "authenticator": auth,
             "url_base": api_endpoint,
+            "default_start_date": default_start_date
         }
 
         return [
             Groups(**initialization_params),
-            Logs(**initialization_params),
+            Logs(**initialization_params_with_start_date),
             Users(**initialization_params),
             GroupMembers(**initialization_params),
             CustomRoles(**initialization_params),

airbyte-integrations/connectors/source-okta/source_okta/spec.json

@@ -13,6 +13,13 @@
         "description": "The Okta domain. See the <a href=\"https://docs.airbyte.io/integrations/sources/okta\">docs</a> for instructions on how to find it.",
         "airbyte_secret": false
       },
+      "start_date": {
+        "type": "string",
+        "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$",
+        "description": "UTC date and time in the format YYYY-MM-DDTHH:MM:SSZ. Any data before this date will not be replicated.",
+        "examples": ["2022-07-22T00:00:00Z"],
+        "title": "Start Date"
+      },
       "credentials": {
         "title": "Authorization Method *",
         "type": "object",
@@ -81,7 +88,7 @@
     "oauth_config_specification": {
       "complete_oauth_output_specification": {
         "type": "object",
-        "additionalProperties": false,
+        "additionalProperties": true,
         "properties": {
           "refresh_token": {
             "type": "string",
@@ -91,7 +98,7 @@
       },
       "complete_oauth_server_input_specification": {
         "type": "object",
-        "additionalProperties": false,
+        "additionalProperties": true,
         "properties": {
           "client_id": {
             "type": "string"
@@ -103,7 +110,7 @@
       },
       "complete_oauth_server_output_specification": {
         "type": "object",
-        "additionalProperties": false,
+        "additionalProperties": true,
         "properties": {
           "client_id": {
             "type": "string",
@@ -117,7 +124,7 @@
       },
       "oauth_user_input_from_connector_config_specification": {
         "type": "object",
-        "additionalProperties": false,
+        "additionalProperties": true,
         "properties": {
           "domain": {
             "type": "string",

airbyte-integrations/connectors/source-okta/unit_tests/test_streams.py

@@ -245,7 +245,12 @@ def test_group_members_request_params_with_latest_entry(self, patch_base_class,
             "stream_state": {"id": "some_test_id"},
             "next_page_token": {"next_cursor": "123"},
         }
-        assert stream.request_params(**inputs) == {"limit": 200, "next_cursor": "123", "after": "some_test_id"}
+        assert stream.request_params(**inputs) == {
+            "limit": 200,
+            "next_cursor": "123",
+            "filter": 'id gt "some_test_id"',
+            "after": "some_test_id",
+        }
 
     def test_group_members_slice_stream(self, requests_mock, patch_base_class, group_members_instance, groups_instance, url_base, api_url):
         stream = GroupMembers(url_base=url_base)
@@ -299,13 +304,22 @@ def test_logs_parse_response(self, requests_mock, patch_base_class, logs_instanc
     def test_logs_request_params_for_since(self, patch_base_class, logs_instance, url_base):
         stream = Logs(url_base=url_base)
         inputs = {"stream_state": {"published": "2022-07-19T15:54:11.545Z"}, "stream_slice": None}
-        assert stream.request_params(**inputs) == {"limit": 200, "since": "2022-07-19T15:54:11.545Z"}
+        assert stream.request_params(**inputs) == {
+            "limit": 200,
+            "filter": 'published gt "2022-07-19T15:54:11.545Z"',
+            "since": "2022-07-19T15:54:11.545Z",
+        }
 
     def test_logs_request_params_for_until(self, patch_base_class, logs_instance, url_base):
         stream = Logs(url_base=url_base)
         testing_date = datetime.datetime.utcnow() + datetime.timedelta(days=10)
         inputs = {"stream_state": {"published": testing_date.isoformat()}, "stream_slice": None}
-        assert stream.request_params(**inputs) == {"limit": 200, "since": testing_date.isoformat(), "until": testing_date.isoformat()}
+        assert stream.request_params(**inputs) == {
+            "limit": 200,
+            "filter": f'published gt "{testing_date.isoformat()}"',
+            "since": testing_date.isoformat(),
+            "until": testing_date.isoformat(),
+        }
 
 
 class TestStreamUserRoleAssignment:

docs/integrations/sources/okta.md

@@ -1,12 +1,65 @@
 # Okta
 
-## Sync overview
-
-This source can sync data for the [Okta API](https://developer.okta.com/docs/reference/). It supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run.
-
-### Output schema
-
-This Source is capable of syncing the following core Streams:
+Okta is the complete identity solution for all your apps and people that’s universal, reliable, and easy
+
+## Prerequisites
+* Created Okta account with added application on [Add Application Page](https://okta-domain.okta.com/enduser/catalog) page. (change okta-domain to you'r domain received after complete registration)
+
+## Airbyte OSS
+* Name
+* Okta-Domain
+* Start Date
+* Personal Api Token (look [here](https://developer.okta.com/docs/guides/find-your-domain/-/main/) to find it)
+
+## Airbyte Cloud
+* Name
+* Start Date
+* Client ID (received when application was added).
+* Client Secret (received when application was added).
+* Refresh Token (received when application was added)
+
+## Setup guide
+### Step 1: Set up Okta
+
+1. Create account on Okta by following link [signup](https://www.okta.com/free-trial/)
+2. Confirm your Email
+3. Choose authorization method (Application or SMS)
+4. Add application in your [Dashboard](https://okta-domain.okta.com/app/UserHome)
+
+### For Airbyte Cloud:
+
+1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account.
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**.
+3. On the source setup page, select **Okta** from the Source type dropdown and enter a name for this connector.
+4. Add **Name**
+5. Add **Okta-Domain**
+6. Add **Start date**
+7. Choose the method of authentication
+8. If you select Token authentication - fill the field  **Personal Api Token** 
+9. If you select OAuth2.0 authorization - fill the fields **Client ID**, **Client Secret**, **Refresh Token**
+10. Click `Set up source`.
+
+### For Airbyte OSS:
+
+1. Go to local Airbyte page.
+2. Use API token from requirements and Okta [domain](https://developer.okta.com/docs/guides/find-your-domain/-/main/). 
+3. Go to local Airbyte page.
+4. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**. 
+5. On the Set up the source page select **Okta** from the Source type dropdown. 
+6. Add **Name**
+7. Add **Okta-Domain**
+8. Add **Start date**
+9. Paste all data to required fields fill the fields **Client ID**, **Client Secret**, **Refresh Token**
+10. Click `Set up source`.
+
+
+## Supported sync modes
+
+The Okta source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
+ - Full Refresh
+ - Incremental
+
+## Supported Streams
 
 - [Users](https://developer.okta.com/docs/reference/api/users/#list-users)
 - [User Role Assignments](https://developer.okta.com/docs/reference/api/roles/#list-roles-assigned-to-a-user)
@@ -16,51 +69,15 @@ This Source is capable of syncing the following core Streams:
 - [System Log](https://developer.okta.com/docs/reference/api/system-log/#get-started)
 - [Custom Roles](https://developer.okta.com/docs/reference/api/roles/#list-roles)
 
-### Data type mapping
-
-| Integration Type | Airbyte Type | Notes |
-| :--------------- | :----------- | :---- |
-| `string`         | `string`     |       |
-| `number`         | `number`     |       |
-| `array`          | `array`      |       |
-| `object`         | `object`     |       |
-
-### Features
-
-| Feature           | Supported?\(Yes/No\) | Notes |
-| :---------------- | :------------------- | :---- |
-| Full Refresh Sync | Yes                  |       |
-| Incremental Sync  | Yes                  |       |
-| Namespaces        | No                   |       |
-
-### Performance considerations
+## Performance considerations
 
 The connector is restricted by normal Okta [requests limitation](https://developer.okta.com/docs/reference/rate-limits/).
 
-## Getting started
-
-### Requirements
-
-You can use [OAuth2.0](https://developer.okta.com/docs/guides/implement-grant-type/authcodepkce/main/) 
-or an [API token](https://developer.okta.com/docs/guides/create-an-api-token/overview/) to authenticate your Okta account. 
-If you choose to authenticate with OAuth2.0, [register](https://dev-01177082-admin.okta.com/admin/apps/active) your Okta application.
-
-### Setup guide
-
-1. Use API token from requirements and Okta [domain](https://developer.okta.com/docs/guides/find-your-domain/-/main/). 
-2. Go to local Airbyte page.
-3. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**. 
-4. On the Set up the source page select **Okta** from the Source type dropdown. 
-5. Paste all data to required fields.
-6. Click `Set up source`.
-
-**Note:**
-Different Okta APIs require different admin privilege levels. API tokens inherit the privilege level of the admin account used to create them
-
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                                        |
 |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------|
+| 0.1.10  | 2022-07-29 | [15050](https://github.com/airbytehq/airbyte/pull/15050) | Added parameter 'start_date' for Logs stream                                   |
 | 0.1.9   | 2022-07-25 | [15001](https://github.com/airbytehq/airbyte/pull/15001) | Return deprovisioned users                                                     |
 | 0.1.8   | 2022-07-19 | [14710](https://github.com/airbytehq/airbyte/pull/14710) | Implement OAuth2.0 authorization method                                        |
 | 0.1.7   | 2022-07-13 | [14556](https://github.com/airbytehq/airbyte/pull/14556) | add User_Role_Assignments and Group_Role_Assignments streams (full fetch only) |