From 71ab084a70488b5a48e23572a5dccd0851af6dce Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 29 Dec 2022 17:26:56 +0100 Subject: [PATCH 1/9] update config with access tokens --- .../streams/http/requests_native_auth/oauth.py | 16 +++++++++++++--- .../test_requests_native_auth.py | 4 +++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py index ca4398ddd08fc..f584d2888d4e7 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py @@ -113,6 +113,7 @@ def __init__( grant_type: str = "refresh_token", client_id_config_path: Sequence[str] = ("credentials", "client_id"), client_secret_config_path: Sequence[str] = ("credentials", "client_secret"), + access_token_config_path: Sequence[str] = ("credentials", "access_token"), refresh_token_config_path: Sequence[str] = ("credentials", "refresh_token"), ): """ @@ -129,10 +130,12 @@ def __init__( grant_type (str, optional): OAuth grant type. Defaults to "refresh_token". client_id_config_path (Sequence[str]): Dpath to the client_id field in the connector configuration. Defaults to ("credentials", "client_id"). client_secret_config_path (Sequence[str]): Dpath to the client_secret field in the connector configuration. Defaults to ("credentials", "client_secret"). + access_token_config_path (Sequence[str]): Dpath to the access_token field in the connector configuration. Defaults to ("credentials", "access_token"). refresh_token_config_path (Sequence[str]): Dpath to the refresh_token field in the connector configuration. Defaults to ("credentials", "refresh_token"). """ self._client_id_config_path = client_id_config_path self._client_secret_config_path = client_secret_config_path + self._access_token_config_path = access_token_config_path self._refresh_token_config_path = refresh_token_config_path self._refresh_token_name = refresh_token_name self._connector_config = observe_connector_config(connector_config) @@ -181,14 +184,21 @@ def get_client_secret(self) -> str: def get_refresh_token(self) -> str: return dpath.util.get(self._connector_config, self._refresh_token_config_path) - def set_refresh_token(self, new_refresh_token: str): - """Set the new refresh token value. The mutation of the connector_config object will emit an Airbyte control message. + + def _update_config_with_access_and_refresh_tokens(self, new_access_token: str, new_refresh_token: str): + """Update the connector configuration with new access and refresh token values. + The mutation of the connector_config object will emit Airbyte control messages. Args: + new_access_token (str): The new access token value. new_refresh_token (str): The new refresh token value. """ + # TODO alafanechere this will sequentially emit two control messages. + # We should rework the observer/config mutation logic if we want to have atomic config updates in a single control message. + dpath.util.set(self._connector_config, self._access_token_config_path, new_access_token) dpath.util.set(self._connector_config, self._refresh_token_config_path, new_refresh_token) + def get_access_token(self) -> str: """Retrieve new access and refresh token if the access token has expired. The new refresh token is persisted with the set_refresh_token function @@ -200,7 +210,7 @@ def get_access_token(self) -> str: new_access_token, access_token_expires_in, new_refresh_token = self.refresh_access_token() self.access_token = new_access_token self.set_token_expiry_date(t0, access_token_expires_in) - self.set_refresh_token(new_refresh_token) + self._update_config_with_access_and_refresh_tokens(new_access_token, new_refresh_token) return self.access_token def refresh_access_token(self) -> Tuple[str, int, str]: diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py index 368669bf12238..093dd205a3cc2 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py @@ -218,9 +218,11 @@ def test_get_access_token(self, capsys, mocker, connector_config): authenticator.token_has_expired = mocker.Mock(return_value=True) access_token = authenticator.get_access_token() captured = capsys.readouterr() - airbyte_message = json.loads(captured.out) + airbyte_message = json.loads(captured.out.split("\n")[-2]) expected_new_config = connector_config.copy() + expected_new_config["credentials"]["access_token"] = "new_access_token" expected_new_config["credentials"]["refresh_token"] = "new_refresh_token" + assert airbyte_message["control"]["connectorConfig"]["config"] == expected_new_config assert authenticator.access_token == access_token == "new_access_token" assert authenticator.get_refresh_token() == "new_refresh_token" From 833d978fe1b2cff2779397bc747eeba1f8c6954f Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 29 Dec 2022 18:54:44 +0100 Subject: [PATCH 2/9] read access token expiration date from config and update it --- .../http/requests_native_auth/oauth.py | 34 +++++++++++-------- .../test_requests_native_auth.py | 5 ++- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py index f584d2888d4e7..2cd987c166b4c 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py @@ -104,8 +104,6 @@ def __init__( connector_config: Mapping[str, Any], token_refresh_endpoint: str, scopes: List[str] = None, - token_expiry_date: pendulum.DateTime = None, - token_expiry_date_format: str = None, access_token_name: str = "access_token", expires_in_name: str = "expires_in", refresh_token_name: str = "refresh_token", @@ -115,6 +113,7 @@ def __init__( client_secret_config_path: Sequence[str] = ("credentials", "client_secret"), access_token_config_path: Sequence[str] = ("credentials", "access_token"), refresh_token_config_path: Sequence[str] = ("credentials", "refresh_token"), + access_token_expiration_datetime_config_path: Sequence[str] = ("credentials", "access_token_expiration_datetime"), ): """ @@ -122,7 +121,6 @@ def __init__( connector_config (Mapping[str, Any]): The full connector configuration token_refresh_endpoint (str): Full URL to the token refresh endpoint scopes (List[str], optional): List of OAuth scopes to pass in the refresh token request body. Defaults to None. - token_expiry_date (pendulum.DateTime, optional): Datetime at which the current token will expire. Defaults to None. access_token_name (str, optional): Name of the access token field, used to parse the refresh token response. Defaults to "access_token". expires_in_name (str, optional): Name of the name of the field that characterizes when the current access token will expire, used to parse the refresh token response. Defaults to "expires_in". refresh_token_name (str, optional): Name of the name of the refresh token field, used to parse the refresh token response. Defaults to "refresh_token". @@ -132,11 +130,13 @@ def __init__( client_secret_config_path (Sequence[str]): Dpath to the client_secret field in the connector configuration. Defaults to ("credentials", "client_secret"). access_token_config_path (Sequence[str]): Dpath to the access_token field in the connector configuration. Defaults to ("credentials", "access_token"). refresh_token_config_path (Sequence[str]): Dpath to the refresh_token field in the connector configuration. Defaults to ("credentials", "refresh_token"). + access_token_expiration_datetime_config_path (Sequence[str]): Dpath to the access_token_expiration_datetime field in the connector configuration. Defaults to ("credentials", "access_token_expiration_datetime"). """ self._client_id_config_path = client_id_config_path self._client_secret_config_path = client_secret_config_path self._access_token_config_path = access_token_config_path self._refresh_token_config_path = refresh_token_config_path + self._access_token_expiration_datetime_config_path = access_token_expiration_datetime_config_path self._refresh_token_name = refresh_token_name self._connector_config = observe_connector_config(connector_config) self._validate_connector_config() @@ -145,13 +145,12 @@ def __init__( self.get_client_id(), self.get_client_secret(), self.get_refresh_token(), - scopes, - token_expiry_date, - token_expiry_date_format, - access_token_name, - expires_in_name, - refresh_request_body, - grant_type, + scopes=scopes, + token_expiry_date=self.get_access_token_expiration_datetime(), + access_token_name=access_token_name, + expires_in_name=expires_in_name, + refresh_request_body=refresh_request_body, + grant_type=grant_type ) def _validate_connector_config(self): @@ -164,6 +163,7 @@ def _validate_connector_config(self): (self._client_id_config_path, self.get_client_id, "client_id_config_path"), (self._client_secret_config_path, self.get_client_secret, "client_secret_config_path"), (self._refresh_token_config_path, self.get_refresh_token, "refresh_token_config_path"), + (self._access_token_expiration_datetime_config_path, self.get_access_token_expiration_datetime, "access_token_expiration_datetime_config_path"), ]: try: assert getter() @@ -184,19 +184,24 @@ def get_client_secret(self) -> str: def get_refresh_token(self) -> str: return dpath.util.get(self._connector_config, self._refresh_token_config_path) + def get_access_token_expiration_datetime(self) -> pendulum.DateTime: + return pendulum.parse(dpath.util.get(self._connector_config, self._access_token_expiration_datetime_config_path)) + - def _update_config_with_access_and_refresh_tokens(self, new_access_token: str, new_refresh_token: str): + def _update_config_with_access_and_refresh_tokens(self, new_access_token: str, new_refresh_token: str, new_access_token_expiration_datetime: pendulum.DateTime): """Update the connector configuration with new access and refresh token values. The mutation of the connector_config object will emit Airbyte control messages. Args: new_access_token (str): The new access token value. new_refresh_token (str): The new refresh token value. + new_access_token_expiration_datetime (pendulum.DateTime): The new access token expiration date. """ - # TODO alafanechere this will sequentially emit two control messages. + # TODO alafanechere this will sequentially emit three control messages. # We should rework the observer/config mutation logic if we want to have atomic config updates in a single control message. dpath.util.set(self._connector_config, self._access_token_config_path, new_access_token) dpath.util.set(self._connector_config, self._refresh_token_config_path, new_refresh_token) + dpath.util.set(self._connector_config, self._access_token_expiration_datetime_config_path, new_access_token_expiration_datetime) def get_access_token(self) -> str: @@ -206,11 +211,10 @@ def get_access_token(self) -> str: str: The current access_token, updated if it was previously expired. """ if self.token_has_expired(): - t0 = pendulum.now() new_access_token, access_token_expires_in, new_refresh_token = self.refresh_access_token() self.access_token = new_access_token - self.set_token_expiry_date(t0, access_token_expires_in) - self._update_config_with_access_and_refresh_tokens(new_access_token, new_refresh_token) + self.set_token_expiry_date(pendulum.now("UTC"), access_token_expires_in) + self._update_config_with_access_and_refresh_tokens(new_access_token, new_refresh_token, self.get_token_expiry_date()) return self.access_token def refresh_access_token(self) -> Tuple[str, int, str]: diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py index 093dd205a3cc2..fa898b452ad8d 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py @@ -5,6 +5,7 @@ import json import logging +import freezegun import pendulum import pytest import requests @@ -188,6 +189,7 @@ def connector_config(self): "refresh_token": "my_refresh_token", "client_id": "my_client_id", "client_secret": "my_client_secret", + "access_token_expiration_datetime": "2022-12-31T00:00:00+00:00" } } @@ -209,6 +211,7 @@ def test_init_with_invalid_config(self, invalid_connector_config): token_refresh_endpoint="foobar", ) + @freezegun.freeze_time("2022-12-31") def test_get_access_token(self, capsys, mocker, connector_config): authenticator = SingleUseRefreshTokenOauth2Authenticator( connector_config, @@ -222,7 +225,7 @@ def test_get_access_token(self, capsys, mocker, connector_config): expected_new_config = connector_config.copy() expected_new_config["credentials"]["access_token"] = "new_access_token" expected_new_config["credentials"]["refresh_token"] = "new_refresh_token" - + expected_new_config["credentials"]["access_token_expiration_datetime"] = "2022-12-31T00:00:42+00:00" assert airbyte_message["control"]["connectorConfig"]["config"] == expected_new_config assert authenticator.access_token == access_token == "new_access_token" assert authenticator.get_refresh_token() == "new_refresh_token" From 3c864f7a24bbc320deaef77b217c8195232d06c6 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 29 Dec 2022 19:04:57 +0100 Subject: [PATCH 3/9] bump version --- airbyte-cdk/python/CHANGELOG.md | 3 +++ airbyte-cdk/python/setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 739ca1d8c5f68..16a321727a0b0 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.16.3 +Do not eagerly refresh access token in `SingleUseRefreshTokenOauth2Authenticator` [#20923](https://github.com/airbytehq/airbyte/pull/20923) + ## 0.16.2 Fix the naming of OAuthAuthenticator diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index d3ac09c8c966c..043e1456f8753 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.16.2", + version="0.16.3", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", From bd6920a25410111bf44b54972d7029584d4aa701 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 30 Dec 2022 11:20:12 +0100 Subject: [PATCH 4/9] discard usage of observed config in the authenticator --- .../python/airbyte_cdk/config_observation.py | 21 ++++++++++--------- .../http/requests_native_auth/oauth.py | 9 ++++---- .../test_requests_native_auth.py | 5 ++--- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/config_observation.py b/airbyte-cdk/python/airbyte_cdk/config_observation.py index 8d886e44bed9f..c477f7646f0eb 100644 --- a/airbyte-cdk/python/airbyte_cdk/config_observation.py +++ b/airbyte-cdk/python/airbyte_cdk/config_observation.py @@ -55,16 +55,7 @@ def set_config(self, config: ObservedDict) -> None: self.config = config def update(self) -> None: - self._emit_airbyte_control_message() - - def _emit_airbyte_control_message(self) -> None: - control_message = AirbyteControlMessage( - type=OrchestratorType.CONNECTOR_CONFIG, - emitted_at=time.time() * 1000, - connectorConfig=AirbyteControlConnectorConfigMessage(config=self.config), - ) - airbyte_message = AirbyteMessage(type=Type.CONTROL, control=control_message) - print(airbyte_message.json(exclude_unset=True)) + emit_configuration_as_airbyte_control_message(self.config) def observe_connector_config(non_observed_connector_config: MutableMapping[str, Any]): @@ -74,3 +65,13 @@ def observe_connector_config(non_observed_connector_config: MutableMapping[str, observed_connector_config = ObservedDict(non_observed_connector_config, connector_config_observer) connector_config_observer.set_config(observed_connector_config) return observed_connector_config + + +def emit_configuration_as_airbyte_control_message(config: MutableMapping): + control_message = AirbyteControlMessage( + type=OrchestratorType.CONNECTOR_CONFIG, + emitted_at=time.time() * 1000, + connectorConfig=AirbyteControlConnectorConfigMessage(config=config), + ) + airbyte_message = AirbyteMessage(type=Type.CONTROL, control=control_message) + print(airbyte_message.json(exclude_unset=True)) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py index 2cd987c166b4c..18a7b5a32f375 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py @@ -6,7 +6,7 @@ import dpath import pendulum -from airbyte_cdk.config_observation import observe_connector_config +from airbyte_cdk.config_observation import emit_configuration_as_airbyte_control_message from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_oauth import AbstractOauth2Authenticator @@ -138,7 +138,7 @@ def __init__( self._refresh_token_config_path = refresh_token_config_path self._access_token_expiration_datetime_config_path = access_token_expiration_datetime_config_path self._refresh_token_name = refresh_token_name - self._connector_config = observe_connector_config(connector_config) + self._connector_config = connector_config self._validate_connector_config() super().__init__( token_refresh_endpoint, @@ -197,12 +197,10 @@ def _update_config_with_access_and_refresh_tokens(self, new_access_token: str, n new_refresh_token (str): The new refresh token value. new_access_token_expiration_datetime (pendulum.DateTime): The new access token expiration date. """ - # TODO alafanechere this will sequentially emit three control messages. - # We should rework the observer/config mutation logic if we want to have atomic config updates in a single control message. dpath.util.set(self._connector_config, self._access_token_config_path, new_access_token) dpath.util.set(self._connector_config, self._refresh_token_config_path, new_refresh_token) dpath.util.set(self._connector_config, self._access_token_expiration_datetime_config_path, new_access_token_expiration_datetime) - + emit_configuration_as_airbyte_control_message(self._connector_config) def get_access_token(self) -> str: """Retrieve new access and refresh token if the access token has expired. @@ -227,3 +225,4 @@ def refresh_access_token(self) -> Tuple[str, int, str]: ) except Exception as e: raise Exception(f"Error while refreshing access token and refresh token: {e}") from e + diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py index fa898b452ad8d..663e0db8826bf 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py @@ -198,11 +198,10 @@ def invalid_connector_config(self): return {"no_credentials_key": "foo"} def test_init(self, connector_config): - authenticator = SingleUseRefreshTokenOauth2Authenticator( + SingleUseRefreshTokenOauth2Authenticator( connector_config, token_refresh_endpoint="foobar", ) - assert isinstance(authenticator._connector_config, ObservedDict) def test_init_with_invalid_config(self, invalid_connector_config): with pytest.raises(ValueError): @@ -221,7 +220,7 @@ def test_get_access_token(self, capsys, mocker, connector_config): authenticator.token_has_expired = mocker.Mock(return_value=True) access_token = authenticator.get_access_token() captured = capsys.readouterr() - airbyte_message = json.loads(captured.out.split("\n")[-2]) + airbyte_message = json.loads(captured.out) expected_new_config = connector_config.copy() expected_new_config["credentials"]["access_token"] = "new_access_token" expected_new_config["credentials"]["refresh_token"] = "new_refresh_token" From ebffd3666403786be271e3dbb6c2328b612ce112 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 30 Dec 2022 12:21:30 +0100 Subject: [PATCH 5/9] clean --- .../http/requests_native_auth/oauth.py | 58 ++++++++++++------- .../test_requests_native_auth.py | 9 ++- 2 files changed, 43 insertions(+), 24 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py index 18a7b5a32f375..702c25a953a6a 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py @@ -113,7 +113,7 @@ def __init__( client_secret_config_path: Sequence[str] = ("credentials", "client_secret"), access_token_config_path: Sequence[str] = ("credentials", "access_token"), refresh_token_config_path: Sequence[str] = ("credentials", "refresh_token"), - access_token_expiration_datetime_config_path: Sequence[str] = ("credentials", "access_token_expiration_datetime"), + token_expiry_date_config_path: Sequence[str] = ("credentials", "token_expiry_date"), ): """ @@ -130,13 +130,13 @@ def __init__( client_secret_config_path (Sequence[str]): Dpath to the client_secret field in the connector configuration. Defaults to ("credentials", "client_secret"). access_token_config_path (Sequence[str]): Dpath to the access_token field in the connector configuration. Defaults to ("credentials", "access_token"). refresh_token_config_path (Sequence[str]): Dpath to the refresh_token field in the connector configuration. Defaults to ("credentials", "refresh_token"). - access_token_expiration_datetime_config_path (Sequence[str]): Dpath to the access_token_expiration_datetime field in the connector configuration. Defaults to ("credentials", "access_token_expiration_datetime"). + token_expiry_date_config_path (Sequence[str]): Dpath to the token_expiry_date field in the connector configuration. Defaults to ("credentials", "token_expiry_date"). """ self._client_id_config_path = client_id_config_path self._client_secret_config_path = client_secret_config_path self._access_token_config_path = access_token_config_path self._refresh_token_config_path = refresh_token_config_path - self._access_token_expiration_datetime_config_path = access_token_expiration_datetime_config_path + self._token_expiry_date_config_path = token_expiry_date_config_path self._refresh_token_name = refresh_token_name self._connector_config = connector_config self._validate_connector_config() @@ -146,7 +146,7 @@ def __init__( self.get_client_secret(), self.get_refresh_token(), scopes=scopes, - token_expiry_date=self.get_access_token_expiration_datetime(), + token_expiry_date=self.get_token_expiry_date(), access_token_name=access_token_name, expires_in_name=expires_in_name, refresh_request_body=refresh_request_body, @@ -159,11 +159,17 @@ def _validate_connector_config(self): Raises: ValueError: Raised if the defined getters are not returning a value. """ + try: + assert self.access_token + except KeyError: + raise ValueError( + f"This authenticator expects a value under the {self._access_token_config_path} field path. Please check your configuration structure or change the access_token_config_path value at initialization of this authenticator." + ) for field_path, getter, parameter_name in [ (self._client_id_config_path, self.get_client_id, "client_id_config_path"), (self._client_secret_config_path, self.get_client_secret, "client_secret_config_path"), (self._refresh_token_config_path, self.get_refresh_token, "refresh_token_config_path"), - (self._access_token_expiration_datetime_config_path, self.get_access_token_expiration_datetime, "access_token_expiration_datetime_config_path"), + (self._token_expiry_date_config_path, self.get_token_expiry_date, "token_expiry_date_config_path"), ]: try: assert getter() @@ -181,26 +187,34 @@ def get_client_id(self) -> str: def get_client_secret(self) -> str: return dpath.util.get(self._connector_config, self._client_secret_config_path) + @property + def access_token(self) -> str: + return dpath.util.get(self._connector_config, self._access_token_config_path) + + @access_token.setter + def access_token(self, new_access_token: str): + dpath.util.set(self._connector_config, self._access_token_config_path, new_access_token) + def get_refresh_token(self) -> str: return dpath.util.get(self._connector_config, self._refresh_token_config_path) - def get_access_token_expiration_datetime(self) -> pendulum.DateTime: - return pendulum.parse(dpath.util.get(self._connector_config, self._access_token_expiration_datetime_config_path)) + def set_refresh_token(self, new_refresh_token: str): + dpath.util.set(self._connector_config, self._refresh_token_config_path, new_refresh_token) + def get_token_expiry_date(self) -> pendulum.DateTime: + return pendulum.parse(dpath.util.get(self._connector_config, self._token_expiry_date_config_path)) - def _update_config_with_access_and_refresh_tokens(self, new_access_token: str, new_refresh_token: str, new_access_token_expiration_datetime: pendulum.DateTime): - """Update the connector configuration with new access and refresh token values. - The mutation of the connector_config object will emit Airbyte control messages. + def set_token_expiry_date(self, new_token_expiry_date): + dpath.util.set(self._connector_config, self._token_expiry_date_config_path, new_token_expiry_date) - Args: - new_access_token (str): The new access token value. - new_refresh_token (str): The new refresh token value. - new_access_token_expiration_datetime (pendulum.DateTime): The new access token expiration date. - """ - dpath.util.set(self._connector_config, self._access_token_config_path, new_access_token) - dpath.util.set(self._connector_config, self._refresh_token_config_path, new_refresh_token) - dpath.util.set(self._connector_config, self._access_token_expiration_datetime_config_path, new_access_token_expiration_datetime) - emit_configuration_as_airbyte_control_message(self._connector_config) + + def token_has_expired(self) -> bool: + """Returns True if the token is expired""" + return pendulum.now("UTC") > self.token_expiry_date + + @staticmethod + def get_new_token_expiry_date(access_token_expires_in: int): + return pendulum.now("UTC").add(seconds=access_token_expires_in) def get_access_token(self) -> str: """Retrieve new access and refresh token if the access token has expired. @@ -210,9 +224,11 @@ def get_access_token(self) -> str: """ if self.token_has_expired(): new_access_token, access_token_expires_in, new_refresh_token = self.refresh_access_token() + new_token_expiry_date = self.get_new_token_expiry_date(access_token_expires_in) self.access_token = new_access_token - self.set_token_expiry_date(pendulum.now("UTC"), access_token_expires_in) - self._update_config_with_access_and_refresh_tokens(new_access_token, new_refresh_token, self.get_token_expiry_date()) + self.set_refresh_token(new_refresh_token) + self.set_token_expiry_date(new_token_expiry_date) + emit_configuration_as_airbyte_control_message(self._connector_config) return self.access_token def refresh_access_token(self) -> Tuple[str, int, str]: diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py index 663e0db8826bf..310d41fa43bf3 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py @@ -189,7 +189,7 @@ def connector_config(self): "refresh_token": "my_refresh_token", "client_id": "my_client_id", "client_secret": "my_client_secret", - "access_token_expiration_datetime": "2022-12-31T00:00:00+00:00" + "token_expiry_date": "2022-12-31T00:00:00+00:00" } } @@ -198,10 +198,13 @@ def invalid_connector_config(self): return {"no_credentials_key": "foo"} def test_init(self, connector_config): - SingleUseRefreshTokenOauth2Authenticator( + authenticator = SingleUseRefreshTokenOauth2Authenticator( connector_config, token_refresh_endpoint="foobar", ) + assert authenticator.access_token == connector_config["credentials"]["access_token"] + assert authenticator.get_refresh_token() == connector_config["credentials"]["refresh_token"] + assert authenticator.get_token_expiry_date() == pendulum.parse(connector_config["credentials"]["token_expiry_date"]) def test_init_with_invalid_config(self, invalid_connector_config): with pytest.raises(ValueError): @@ -224,7 +227,7 @@ def test_get_access_token(self, capsys, mocker, connector_config): expected_new_config = connector_config.copy() expected_new_config["credentials"]["access_token"] = "new_access_token" expected_new_config["credentials"]["refresh_token"] = "new_refresh_token" - expected_new_config["credentials"]["access_token_expiration_datetime"] = "2022-12-31T00:00:42+00:00" + expected_new_config["credentials"]["token_expiry_date"] = "2022-12-31T00:00:42+00:00" assert airbyte_message["control"]["connectorConfig"]["config"] == expected_new_config assert authenticator.access_token == access_token == "new_access_token" assert authenticator.get_refresh_token() == "new_refresh_token" From a0ca5cd24dd9f0f832957eeaf0a986ae86e89e07 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 30 Dec 2022 12:51:21 +0100 Subject: [PATCH 6/9] clean --- .../http/requests_native_auth/test_requests_native_auth.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py index 310d41fa43bf3..720a62a555860 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py @@ -9,7 +9,6 @@ import pendulum import pytest import requests -from airbyte_cdk.config_observation import ObservedDict from airbyte_cdk.sources.streams.http.requests_native_auth import ( BasicHttpAuthenticator, MultipleTokenAuthenticator, From 4bb78064d4df6a6b0e06b238c7959a0110ff052e Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 30 Dec 2022 12:54:13 +0100 Subject: [PATCH 7/9] clean --- .../sources/streams/http/requests_native_auth/oauth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py index 702c25a953a6a..3a1d24f5a3fac 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py @@ -210,7 +210,7 @@ def set_token_expiry_date(self, new_token_expiry_date): def token_has_expired(self) -> bool: """Returns True if the token is expired""" - return pendulum.now("UTC") > self.token_expiry_date + return pendulum.now("UTC") > self.get_token_expiry_date() @staticmethod def get_new_token_expiry_date(access_token_expires_in: int): From 27357a962358093771f7093064510d4de04cca62 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 30 Dec 2022 16:08:24 +0100 Subject: [PATCH 8/9] format --- .../sources/streams/http/requests_native_auth/oauth.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py index 3a1d24f5a3fac..2815f52309130 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py @@ -150,7 +150,7 @@ def __init__( access_token_name=access_token_name, expires_in_name=expires_in_name, refresh_request_body=refresh_request_body, - grant_type=grant_type + grant_type=grant_type, ) def _validate_connector_config(self): @@ -194,7 +194,7 @@ def access_token(self) -> str: @access_token.setter def access_token(self, new_access_token: str): dpath.util.set(self._connector_config, self._access_token_config_path, new_access_token) - + def get_refresh_token(self) -> str: return dpath.util.get(self._connector_config, self._refresh_token_config_path) @@ -204,14 +204,13 @@ def set_refresh_token(self, new_refresh_token: str): def get_token_expiry_date(self) -> pendulum.DateTime: return pendulum.parse(dpath.util.get(self._connector_config, self._token_expiry_date_config_path)) - def set_token_expiry_date(self, new_token_expiry_date): + def set_token_expiry_date(self, new_token_expiry_date): dpath.util.set(self._connector_config, self._token_expiry_date_config_path, new_token_expiry_date) - def token_has_expired(self) -> bool: """Returns True if the token is expired""" return pendulum.now("UTC") > self.get_token_expiry_date() - + @staticmethod def get_new_token_expiry_date(access_token_expires_in: int): return pendulum.now("UTC").add(seconds=access_token_expires_in) @@ -241,4 +240,3 @@ def refresh_access_token(self) -> Tuple[str, int, str]: ) except Exception as e: raise Exception(f"Error while refreshing access token and refresh token: {e}") from e - From 018cdc3123a270c944585cc0c0b8ddaa5cacd3f2 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 30 Dec 2022 20:16:01 +0100 Subject: [PATCH 9/9] fix setter --- .../sources/streams/http/requests_native_auth/oauth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py index 2815f52309130..59749ccb9744a 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py @@ -205,7 +205,7 @@ def get_token_expiry_date(self) -> pendulum.DateTime: return pendulum.parse(dpath.util.get(self._connector_config, self._token_expiry_date_config_path)) def set_token_expiry_date(self, new_token_expiry_date): - dpath.util.set(self._connector_config, self._token_expiry_date_config_path, new_token_expiry_date) + dpath.util.set(self._connector_config, self._token_expiry_date_config_path, str(new_token_expiry_date)) def token_has_expired(self) -> bool: """Returns True if the token is expired"""