From 2754a2a8661f710f8f6153295ed801240aaf1cc5 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Mon, 4 Oct 2021 19:20:37 +0300 Subject: [PATCH 01/20] fixed test which check incorrect cred config --- .../source-square/integration_tests/invalid_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-square/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-square/integration_tests/invalid_config.json index b679b36c8db8f..26722cda63f8a 100644 --- a/airbyte-integrations/connectors/source-square/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-square/integration_tests/invalid_config.json @@ -1,6 +1,6 @@ { "api_key": "API_KEY", "is_sandbox": true, - "start_date": "START_DATE", + "start_date": "2021-06-01", "include_deleted_objects": false } From 4b29f136a2c17515442199398b093adc1f1af0b2 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Wed, 6 Oct 2021 19:23:28 +0300 Subject: [PATCH 02/20] Added oauth2 authentication --- .../source-square/acceptance-test-config.yml | 6 ++ .../integration_tests/configured_catalog.json | 12 --- .../configured_catalog_oauth.json | 40 ++++++++++ .../integration_tests/invalid_config.json | 8 +- .../source-square/source_square/source.py | 66 +++++++++++++++-- .../source-square/source_square/spec.json | 74 +++++++++++++++++-- .../unit_tests/connection_test.py | 9 ++- 7 files changed, 185 insertions(+), 30 deletions(-) create mode 100644 airbyte-integrations/connectors/source-square/integration_tests/configured_catalog_oauth.json diff --git a/airbyte-integrations/connectors/source-square/acceptance-test-config.yml b/airbyte-integrations/connectors/source-square/acceptance-test-config.yml index 68414755e1ea8..c395e448d4051 100644 --- a/airbyte-integrations/connectors/source-square/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-square/acceptance-test-config.yml @@ -5,6 +5,8 @@ tests: connection: - config_path: "secrets/config.json" status: "succeed" + - config_path: "secrets/config_oauth.json" + status: "succeed" - config_path: "integration_tests/invalid_config.json" status: "failed" discovery: @@ -12,6 +14,8 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config_oauth.json" + configured_catalog_path: "integration_tests/configured_catalog_oauth.json" incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" @@ -19,3 +23,5 @@ tests: full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config_oauth.json" + configured_catalog_path: "integration_tests/configured_catalog_oauth.json" diff --git a/airbyte-integrations/connectors/source-square/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-square/integration_tests/configured_catalog.json index f34fd4fc8ea05..b47d38d39da0d 100644 --- a/airbyte-integrations/connectors/source-square/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-square/integration_tests/configured_catalog.json @@ -143,18 +143,6 @@ "sync_mode": "full_refresh", "cursor_field": ["id"], "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "orders", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": true, - "default_cursor_field": ["id"] - }, - "sync_mode": "full_refresh", - "cursor_field": ["id"], - "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-square/integration_tests/configured_catalog_oauth.json b/airbyte-integrations/connectors/source-square/integration_tests/configured_catalog_oauth.json new file mode 100644 index 0000000000000..6c989ca3afff4 --- /dev/null +++ b/airbyte-integrations/connectors/source-square/integration_tests/configured_catalog_oauth.json @@ -0,0 +1,40 @@ +{ + "streams": [ + { + "stream": { + "name": "locations", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "full_refresh", + "cursor_field": ["id"], + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "team_members", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "full_refresh", + "cursor_field": ["id"], + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "team_member_wages", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "full_refresh", + "cursor_field": ["id"], + "destination_sync_mode": "overwrite" + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-square/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-square/integration_tests/invalid_config.json index 26722cda63f8a..aad93991d42fd 100644 --- a/airbyte-integrations/connectors/source-square/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-square/integration_tests/invalid_config.json @@ -1,6 +1,6 @@ { - "api_key": "API_KEY", - "is_sandbox": true, - "start_date": "2021-06-01", - "include_deleted_objects": false + "authorization": {"auth_type": "Apikey", "api_key": "bla"}, + "is_sandbox": true, + "start_date": "2021-06-01", + "include_deleted_objects": false } diff --git a/airbyte-integrations/connectors/source-square/source_square/source.py b/airbyte-integrations/connectors/source-square/source_square/source.py index 86f1b48dbb199..62082dbf24592 100644 --- a/airbyte-integrations/connectors/source-square/source_square/source.py +++ b/airbyte-integrations/connectors/source-square/source_square/source.py @@ -12,7 +12,7 @@ from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator, TokenAuthenticator from source_square.utils import separate_items_by_count @@ -358,16 +358,73 @@ def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: yield {"location_ids": location} +class Oauth2AuthenticatorSquare(Oauth2Authenticator): + + def refresh_access_token(self): + """Handle differences in expiration attr: + from API: "expires_at": "2021-11-05T14:26:57Z" + expected: "expires_in": number of seconds + """ + token, expires_at = super().refresh_access_token() + expires_in = pendulum.parse(expires_at) - pendulum.now() + return token, expires_in.seconds + + class SourceSquare(AbstractSource): - api_version = "2021-06-16" # Latest Stable Release + api_version = "2021-09-15" # Latest Stable Release + + @staticmethod + def get_auth(config): + + authorization = config.get("authorization", {}) + auth_type = authorization.get("auth_type") + if auth_type == "Oauth": + scopes = [ + "CUSTOMERS_READ", + "EMPLOYEES_READ", + "ITEMS_READ", + "MERCHANT_PROFILE_READ", + "ORDERS_READ", + "PAYMENTS_READ", + "TIMECARDS_READ", + # OAuth Permissions: + # https://developer.squareup.com/docs/oauth-api/square-permissions + # https://developer.squareup.com/reference/square/enums/OAuthPermission + # "DISPUTES_READ", + # "GIFTCARDS_READ", + # "INVENTORY_READ", + # "INVOICES_READ", + # "TIMECARDS_SETTINGS_READ", + # "LOYALTY_READ", + # "ONLINE_STORE_SITE_READ", + # "ONLINE_STORE_SNIPPETS_READ", + # "SUBSCRIPTIONS_READ", + ] + + auth = Oauth2AuthenticatorSquare( + token_refresh_endpoint="https://connect.squareup.com/oauth2/token", + client_secret=authorization.get("client_secret"), + client_id=authorization.get("client_id"), + refresh_token=authorization.get("refresh_token"), + scopes=scopes, + expires_in_name="expires_at", + ) + elif auth_type == "Apikey": + auth = TokenAuthenticator(token=authorization.get("api_key")) + else: + raise Exception(f"Invalid auth type: {auth_type}") + + return auth def check_connection(self, logger, config) -> Tuple[bool, any]: headers = { "Square-Version": self.api_version, - "Authorization": "Bearer {}".format(config["api_key"]), "Content-Type": "application/json", } + auth = self.get_auth(config) + headers.update(auth.get_auth_header()) + url = "https://connect.squareup{}.com/v2/catalog/info".format("sandbox" if config["is_sandbox"] else "") try: @@ -383,9 +440,8 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: def streams(self, config: Mapping[str, Any]) -> List[Stream]: - auth = TokenAuthenticator(token=config["api_key"]) args = { - "authenticator": auth, + "authenticator": self.get_auth(config), "is_sandbox": config["is_sandbox"], "api_version": self.api_version, "start_date": config["start_date"], diff --git a/airbyte-integrations/connectors/source-square/source_square/spec.json b/airbyte-integrations/connectors/source-square/source_square/spec.json index 46dd0b4f681f1..f9f95748a2657 100644 --- a/airbyte-integrations/connectors/source-square/source_square/spec.json +++ b/airbyte-integrations/connectors/source-square/source_square/spec.json @@ -4,25 +4,20 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Square Source CDK Specifications", "type": "object", - "required": ["api_key", "is_sandbox"], + "required": ["authorization", "is_sandbox"], "additionalProperties": false, "properties": { - "api_key": { - "type": "string", - "description": "The API key for a Square application", - "airbyte_secret": true - }, "is_sandbox": { "type": "boolean", "description": "Determines the sandbox (true) or production (false) API version", "examples": [true, false], - "default": true + "default": false }, "start_date": { "type": "string", "description": "The start date to sync data. Leave blank for full sync. Format: YYYY-MM-DD.", "examples": ["2021-01-01"], - "default": "1970-01-01", + "default": "2021-01-01", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" }, "include_deleted_objects": { @@ -30,6 +25,69 @@ "description": "In some streams there is and option to include deleted objects (Items, Categories, Discounts, Taxes)", "examples": [true, false], "default": false + }, + "authorization": { + "type": "object", + "title": "Authentication Type", + "oneOf": [ + { + "title": "Oauth authentication", + "type": "object", + "required": [ + "auth_type", + "client_id", + "client_secret", + "refresh_token" + ], + "properties": { + "auth_type": { + "type": "string", + "const": "Oauth", + "enum": ["Oauth"], + "default": "Oauth", + "order": 0 + }, + "client_id": { + "title": "Client ID", + "type": "string", + "description": "The Square-issued ID of your application", + "airbyte_secret": true + }, + "client_secret": { + "title": "Client Secret", + "type": "string", + "description": "The Square-issued application secret for your application", + "airbyte_secret": true + }, + "refresh_token": { + "title": "Refresh Token", + "type": "string", + "description": "A refresh token generated using the above client ID and secret", + "airbyte_secret": true + } + } + }, + { + "type": "object", + "title": "API Key", + "required": ["auth_type", "api_key"], + "properties": { + "auth_type": { + "type": "string", + "const": "Apikey", + "enum": ["Apikey"], + "default": "Apikey", + "order": 1 + }, + "api_key": { + "title": "API key token", + "type": "string", + "description": "The API key for a Square application", + "airbyte_secret": true + } + } + } + ] } } } diff --git a/airbyte-integrations/connectors/source-square/unit_tests/connection_test.py b/airbyte-integrations/connectors/source-square/unit_tests/connection_test.py index f47cb7ca956b9..9cea6292ec827 100644 --- a/airbyte-integrations/connectors/source-square/unit_tests/connection_test.py +++ b/airbyte-integrations/connectors/source-square/unit_tests/connection_test.py @@ -8,5 +8,12 @@ def test_source_wrong_credentials(): source = SourceSquare() - status, error = source.check_connection(logger=AirbyteLogger(), config={"api_key": "wrong.api.key", "is_sandbox": True}) + config = { + "authorization": {"auth_type": "Apikey", "api_key": "bla"}, + "is_sandbox": True, + "start_date": "2021-06-01", + "include_deleted_objects": False, + } + status, error = source.check_connection(logger=AirbyteLogger(), config=config) assert not status + From 633b967cb0903df098eb63b95f747bd17d440e60 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Wed, 6 Oct 2021 19:36:09 +0300 Subject: [PATCH 03/20] Added oauth creds --- .github/workflows/publish-command.yml | 1 + .github/workflows/test-command.yml | 1 + tools/bin/ci_credentials.sh | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 6f79ae9d1f461..65c6c67d2e801 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -151,6 +151,7 @@ jobs: SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS: ${{ secrets.SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS }} SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS: ${{ secrets.SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS }} SOURCE_SQUARE_CREDS: ${{ secrets.SOURCE_SQUARE_CREDS }} + SOURCE_SQUARE_CREDS_OAUTH: ${{ secrets.SOURCE_SQUARE_CREDS_OAUTH }} SOURCE_MARKETO_TEST_CREDS: ${{ secrets.SOURCE_MARKETO_TEST_CREDS }} SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG: ${{ secrets.SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG }} SOURCE_RECURLY_INTEGRATION_TEST_CREDS: ${{ secrets.SOURCE_RECURLY_INTEGRATION_TEST_CREDS }} diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 0275fe11d14f7..1c4cd0cf2e409 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -146,6 +146,7 @@ jobs: SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS: ${{ secrets.SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS }} SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS: ${{ secrets.SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS }} SOURCE_SQUARE_CREDS: ${{ secrets.SOURCE_SQUARE_CREDS }} + SOURCE_SQUARE_CREDS_OAUTH: ${{ secrets.SOURCE_SQUARE_CREDS_OAUTH }} SOURCE_MARKETO_TEST_CREDS: ${{ secrets.SOURCE_MARKETO_TEST_CREDS }} SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG: ${{ secrets.SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG }} SOURCE_RECURLY_INTEGRATION_TEST_CREDS: ${{ secrets.SOURCE_RECURLY_INTEGRATION_TEST_CREDS }} diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index add334226dcd9..d9aa9de13cee4 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -122,6 +122,7 @@ write_standard_creds source-smartsheets "$SMARTSHEETS_TEST_CREDS" write_standard_creds source-snapchat-marketing "$SOURCE_SNAPCHAT_MARKETING_CREDS" write_standard_creds source-snowflake "$SNOWFLAKE_INTEGRATION_TEST_CREDS" "config.json" write_standard_creds source-square "$SOURCE_SQUARE_CREDS" +write_standard_creds source-square "$SOURCE_SQUARE_CREDS_OAUTH" "config_oauth.json" write_standard_creds source-stripe "$SOURCE_STRIPE_CREDS" write_standard_creds source-stripe "$STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS" "connected_account_config.json" write_standard_creds source-surveymonkey "$SURVEYMONKEY_TEST_CREDS" From 9e11b0d11e264a2283e6583b4ff1e3fe64a3846e Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Thu, 7 Oct 2021 22:42:07 +0300 Subject: [PATCH 04/20] fixed formatting --- .../connectors/source-square/source_square/source.py | 5 ++--- .../connectors/source-square/unit_tests/connection_test.py | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-square/source_square/source.py b/airbyte-integrations/connectors/source-square/source_square/source.py index 62082dbf24592..a735383cd9aa4 100644 --- a/airbyte-integrations/connectors/source-square/source_square/source.py +++ b/airbyte-integrations/connectors/source-square/source_square/source.py @@ -359,11 +359,10 @@ def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: class Oauth2AuthenticatorSquare(Oauth2Authenticator): - def refresh_access_token(self): """Handle differences in expiration attr: - from API: "expires_at": "2021-11-05T14:26:57Z" - expected: "expires_in": number of seconds + from API: "expires_at": "2021-11-05T14:26:57Z" + expected: "expires_in": number of seconds """ token, expires_at = super().refresh_access_token() expires_in = pendulum.parse(expires_at) - pendulum.now() diff --git a/airbyte-integrations/connectors/source-square/unit_tests/connection_test.py b/airbyte-integrations/connectors/source-square/unit_tests/connection_test.py index 9cea6292ec827..9442d723d3714 100644 --- a/airbyte-integrations/connectors/source-square/unit_tests/connection_test.py +++ b/airbyte-integrations/connectors/source-square/unit_tests/connection_test.py @@ -16,4 +16,3 @@ def test_source_wrong_credentials(): } status, error = source.check_connection(logger=AirbyteLogger(), config=config) assert not status - From 2993e77a97bf60ff67ab14d0705e84167e271813 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Fri, 15 Oct 2021 15:56:07 +0300 Subject: [PATCH 05/20] added oauth2 spec section, added missing type hints --- .../connectors/source-square/acceptance-test-config.yml | 4 ++-- .../integration_tests/configured_catalog_oauth.json | 2 +- .../source-square/integration_tests/invalid_config.json | 8 ++++---- .../connectors/source-square/source_square/source.py | 9 ++++++--- .../connectors/source-square/source_square/spec.json | 8 ++++++++ 5 files changed, 21 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-square/acceptance-test-config.yml b/airbyte-integrations/connectors/source-square/acceptance-test-config.yml index c395e448d4051..e3cfdf9227838 100644 --- a/airbyte-integrations/connectors/source-square/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-square/acceptance-test-config.yml @@ -11,6 +11,7 @@ tests: status: "failed" discovery: - config_path: "secrets/config.json" + - config_path: "secrets/config_oauth.json" basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" @@ -23,5 +24,4 @@ tests: full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - - config_path: "secrets/config_oauth.json" - configured_catalog_path: "integration_tests/configured_catalog_oauth.json" + diff --git a/airbyte-integrations/connectors/source-square/integration_tests/configured_catalog_oauth.json b/airbyte-integrations/connectors/source-square/integration_tests/configured_catalog_oauth.json index 6c989ca3afff4..316537a5a67fe 100644 --- a/airbyte-integrations/connectors/source-square/integration_tests/configured_catalog_oauth.json +++ b/airbyte-integrations/connectors/source-square/integration_tests/configured_catalog_oauth.json @@ -37,4 +37,4 @@ "destination_sync_mode": "overwrite" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-square/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-square/integration_tests/invalid_config.json index aad93991d42fd..14f3e3793c1ab 100644 --- a/airbyte-integrations/connectors/source-square/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-square/integration_tests/invalid_config.json @@ -1,6 +1,6 @@ { - "authorization": {"auth_type": "Apikey", "api_key": "bla"}, - "is_sandbox": true, - "start_date": "2021-06-01", - "include_deleted_objects": false + "authorization": { "auth_type": "Apikey", "api_key": "some_api_key" }, + "is_sandbox": true, + "start_date": "2021-06-01", + "include_deleted_objects": false } diff --git a/airbyte-integrations/connectors/source-square/source_square/source.py b/airbyte-integrations/connectors/source-square/source_square/source.py index a735383cd9aa4..e0e1b1441f4ff 100644 --- a/airbyte-integrations/connectors/source-square/source_square/source.py +++ b/airbyte-integrations/connectors/source-square/source_square/source.py @@ -8,11 +8,13 @@ import pendulum import requests +from airbyte_cdk.logger import AirbyteLogger from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator, TokenAuthenticator +from requests.auth import AuthBase from source_square.utils import separate_items_by_count @@ -359,7 +361,7 @@ def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: class Oauth2AuthenticatorSquare(Oauth2Authenticator): - def refresh_access_token(self): + def refresh_access_token(self) -> Tuple[str, int]: """Handle differences in expiration attr: from API: "expires_at": "2021-11-05T14:26:57Z" expected: "expires_in": number of seconds @@ -373,11 +375,12 @@ class SourceSquare(AbstractSource): api_version = "2021-09-15" # Latest Stable Release @staticmethod - def get_auth(config): + def get_auth(config: Mapping[str, Any]) -> AuthBase: authorization = config.get("authorization", {}) auth_type = authorization.get("auth_type") if auth_type == "Oauth": + # scopes needed for all currently supported streams: scopes = [ "CUSTOMERS_READ", "EMPLOYEES_READ", @@ -415,7 +418,7 @@ def get_auth(config): return auth - def check_connection(self, logger, config) -> Tuple[bool, any]: + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: headers = { "Square-Version": self.api_version, diff --git a/airbyte-integrations/connectors/source-square/source_square/spec.json b/airbyte-integrations/connectors/source-square/source_square/spec.json index f9f95748a2657..5c856b567f824 100644 --- a/airbyte-integrations/connectors/source-square/source_square/spec.json +++ b/airbyte-integrations/connectors/source-square/source_square/spec.json @@ -90,5 +90,13 @@ ] } } + }, + "authSpecification": { + "auth_type": "oauth2.0", + "oauth2Specification": { + "rootObject": ["authorization", 0], + "oauthFlowInitParameters": [["client_id"], ["client_secret"]], + "oauthFlowOutputParameters": [["refresh_token"]] + } } } From fd940c788fc08e9045cf2b042142e9147c6c2e63 Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Mon, 1 Nov 2021 12:36:54 +0200 Subject: [PATCH 06/20] Added java part of Square OAuth --- .../oauth/OAuthImplementationFactory.java | 2 + .../airbyte/oauth/flows/SquareOAuthFlow.java | 114 ++++++++++++++++++ .../SquareOAuthFlowIntegrationTest.java | 88 ++++++++++++++ .../oauth/flows/SquareOAuthFlowTest.java | 87 +++++++++++++ 4 files changed, 291 insertions(+) create mode 100644 airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java create mode 100644 airbyte-oauth/src/test-integration/java/io.airbyte.oauth.flows/SquareOAuthFlowIntegrationTest.java create mode 100644 airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SquareOAuthFlowTest.java diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java index 2c195d0c8d259..ff4dd9e24c38e 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java @@ -10,6 +10,7 @@ import io.airbyte.oauth.flows.FacebookMarketingOAuthFlow; import io.airbyte.oauth.flows.GithubOAuthFlow; import io.airbyte.oauth.flows.SalesforceOAuthFlow; +import io.airbyte.oauth.flows.SquareOAuthFlow; import io.airbyte.oauth.flows.TrelloOAuthFlow; import io.airbyte.oauth.flows.google.GoogleAdsOAuthFlow; import io.airbyte.oauth.flows.google.GoogleAnalyticsOAuthFlow; @@ -33,6 +34,7 @@ public OAuthImplementationFactory(final ConfigRepository configRepository) { .put("airbyte/source-google-sheets", new GoogleSheetsOAuthFlow(configRepository)) .put("airbyte/source-salesforce", new SalesforceOAuthFlow(configRepository)) .put("airbyte/source-trello", new TrelloOAuthFlow(configRepository)) + .put("airbyte/source-square", new SquareOAuthFlow(configRepository)) .build(); } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java new file mode 100644 index 0000000000000..a145ddbc4ec5d --- /dev/null +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.oauth.BaseOAuthFlow; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URLDecoder; +import java.net.http.HttpClient; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.UUID; +import java.util.function.Supplier; +import org.apache.http.client.utils.URIBuilder; + +public class SquareOAuthFlow extends BaseOAuthFlow { + + private static final String SCOPE_VALUE = + "ITEMS_READ+CUSTOMERS_WRITE+MERCHANT_PROFILE_READ+EMPLOYEES_READ+PAYMENTS_READ+CUSTOMERS_READ+TIMECARDS_READ+ORDERS_READ"; + private static final String AUTHORIZE_URL = "https://connect.squareup.com/oauth2/authorize"; + private static final String ACCESS_TOKEN_URL = "https://connect.squareup.com/oauth2/token"; + + public SquareOAuthFlow(ConfigRepository configRepository) { + super(configRepository); + } + + @VisibleForTesting + public SquareOAuthFlow(ConfigRepository configRepository, + HttpClient httpClient, + Supplier stateSupplier) { + super(configRepository, httpClient, stateSupplier); + } + + @Override + protected String formatConsentUrl(UUID definitionId, String clientId, String redirectUrl) + throws IOException { + try { + // Need to have decoded format, otherwice square fails saying that scope is incorrect + return URLDecoder.decode(new URIBuilder(AUTHORIZE_URL) + .addParameter("client_id", clientId) + // .addParameter("redirect_uri", redirectUrl) + .addParameter("scope", SCOPE_VALUE) + .addParameter("session", "False") + .addParameter("state", getState()) + .build().toString(), StandardCharsets.UTF_8); + } catch (URISyntaxException e) { + throw new IOException("Failed to format Consent URL for OAuth flow", e); + } + } + + @Override + protected String getAccessTokenUrl() { + return ACCESS_TOKEN_URL; + } + + @Override + protected Map extractRefreshToken(final JsonNode data, String accessTokenUrl) + throws IOException { + System.out.println(data); + if (data.has("refresh_token")) { + return Map.of("authorization", Map.of("refresh_token", data.get("refresh_token").asText())); + } else { + throw new IOException( + String.format("Missing 'refresh_token' in query params from %s", ACCESS_TOKEN_URL)); + } + } + + @Override + protected Map getAccessTokenQueryParameters(String clientId, + String clientSecret, + String authCode, + String redirectUrl) { + return ImmutableMap.builder() + // required + .put("client_id", clientId) + // .put("redirect_uri", redirectUrl) + .put("client_secret", clientSecret) + .put("code", authCode) + .put("grant_type", "authorization_code") + .put("scopes", "[\n" + + " \"ITEMS_READ\",\n" + + " \"MERCHANT_PROFILE_READ\",\n" + + " \"EMPLOYEES_READ\",\n" + + " \"PAYMENTS_READ\",\n" + + " \"CUSTOMERS_READ\",\n" + + " \"TIMECARDS_READ\",\n" + + " \"ORDERS_READ\"\n" + + " ]") + .build(); + + } + + @Override + protected String getClientIdUnsafe(final JsonNode config) { + // the config object containing client ID and secret is nested inside the "authorization" object + Preconditions.checkArgument(config.hasNonNull("authorization")); + return super.getClientIdUnsafe(config.get("authorization")); + } + + @Override + protected String getClientSecretUnsafe(final JsonNode config) { + // the config object containing client ID and secret is nested inside the "authorization" object + Preconditions.checkArgument(config.hasNonNull("authorization")); + return super.getClientSecretUnsafe(config.get("authorization")); + } + +} diff --git a/airbyte-oauth/src/test-integration/java/io.airbyte.oauth.flows/SquareOAuthFlowIntegrationTest.java b/airbyte-oauth/src/test-integration/java/io.airbyte.oauth.flows/SquareOAuthFlowIntegrationTest.java new file mode 100644 index 0000000000000..d48e3166951a9 --- /dev/null +++ b/airbyte-oauth/src/test-integration/java/io.airbyte.oauth.flows/SquareOAuthFlowIntegrationTest.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.SourceOAuthParameter; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.oauth.OAuthFlowImplementation; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class SquareOAuthFlowIntegrationTest extends OAuthFlowIntegrationTest { + + protected static final Path CREDENTIALS_PATH = Path.of("secrets/square.json"); + protected static final String REDIRECT_URL = "http://localhost:8000/code"; + protected static final int SERVER_LISTENING_PORT = 8000; + + @Override + protected Path get_credentials_path() { + return CREDENTIALS_PATH; + } + + @Override + protected OAuthFlowImplementation getFlowObject(ConfigRepository configRepository) { + return new SquareOAuthFlow(configRepository); + } + + @Override + protected int getServerListeningPort() { + return SERVER_LISTENING_PORT; + } + + @BeforeEach + public void setup() throws IOException { + super.setup(); + } + + @Test + public void testFullSquareOAuthFlow() throws InterruptedException, ConfigNotFoundException, IOException, JsonValidationException { + int limit = 20; + final UUID workspaceId = UUID.randomUUID(); + final UUID definitionId = UUID.randomUUID(); + final String fullConfigAsString = new String(Files.readAllBytes(CREDENTIALS_PATH)); + final JsonNode credentialsJson = Jsons.deserialize(fullConfigAsString); + when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() + .withOauthParameterId(UUID.randomUUID()) + .withSourceDefinitionId(definitionId) + .withWorkspaceId(workspaceId) + .withConfiguration(Jsons.jsonNode( + Map.of("authorization", + ImmutableMap.builder() + .put("client_id", credentialsJson.get("client_id").asText()) + .put("client_secret", credentialsJson.get("client_secret").asText()) + .build()))))); + + final String url = flow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); + LOGGER.info("Waiting for user consent at: {}", url); + + // TODO: To automate, start a selenium job to navigate to the Consent URL and click on allowing + // access... + while (!serverHandler.isSucceeded() && limit > 0) { + Thread.sleep(1000); + limit -= 1; + } + assertTrue(serverHandler.isSucceeded(), "Failed to get User consent on time"); + final Map params = flow.completeSourceOAuth(workspaceId, definitionId, + Map.of("code", serverHandler.getParamValue()), REDIRECT_URL); + LOGGER.info("Response from completing OAuth Flow is: {}", params.toString()); + assertTrue(params.containsKey("access_token")); + assertTrue(params.get("access_token").toString().length() > 0); + } + +} diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SquareOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SquareOAuthFlowTest.java new file mode 100644 index 0000000000000..3f7682f53b8f3 --- /dev/null +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SquareOAuthFlowTest.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.SourceOAuthParameter; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.net.http.HttpClient; +import java.net.http.HttpResponse; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class SquareOAuthFlowTest { + + private UUID workspaceId; + private UUID definitionId; + private SquareOAuthFlow squareAuthFlow; + private HttpClient httpClient; + + private static final String REDIRECT_URL = "https://airbyte.io"; + + private static String getConstantState() { + return "state"; + } + + @BeforeEach + public void setup() throws IOException, JsonValidationException { + workspaceId = UUID.randomUUID(); + definitionId = UUID.randomUUID(); + ConfigRepository configRepository = mock(ConfigRepository.class); + httpClient = mock(HttpClient.class); + when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() + .withOauthParameterId(UUID.randomUUID()) + .withSourceDefinitionId(definitionId) + .withWorkspaceId(workspaceId) + .withConfiguration(Jsons.jsonNode( + Map.of("authorization", + ImmutableMap.builder() + .put("client_id", "test_client_id") + .put("client_secret", "test_client_secret") + .build()))))); + squareAuthFlow = new SquareOAuthFlow(configRepository, httpClient, + SquareOAuthFlowTest::getConstantState); + + } + + // @Test + // public void testGetSourceConcentUrl() throws IOException, ConfigNotFoundException { + // final String concentUrl = + // squareAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); + // assertEquals( + // // + // "https://connect.squareup.com/oauth2/authorize?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&scope=CUSTOMERS_WRITE%2BMERCHANT_PROFILE_READ%2BEMPLOYEES_READ%2BPAYMENTS_READ%2BCUSTOMERS_READ%2BTIMECARDS_READ%2BORDERS_READ&session=False&state=state", + // "https://connect.squareup.com/oauth2/authorize?client_id=test_client_id&scope=ITEMS_READ+CUSTOMERS_READ&session=False&state=state", + // concentUrl); + // } + + @Test + public void testCompleteSourceOAuth() throws IOException, InterruptedException, + ConfigNotFoundException { + + Map returnedCredentials = Map.of("refresh_token", "refresh_token_response"); + final HttpResponse response = mock(HttpResponse.class); + when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); + when(httpClient.send(any(), any())).thenReturn(response); + final Map queryParams = Map.of("code", "test_code"); + final Map actualQueryParams = + squareAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); + assertEquals(Jsons.serialize(Map.of("authorization", returnedCredentials)), + Jsons.serialize(actualQueryParams)); + } + +} From 1e93f2e7ca36dab6ca7e160dcb81db2112f56642 Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Mon, 1 Nov 2021 15:28:34 +0200 Subject: [PATCH 07/20] fixed checkstyle --- .../airbyte/oauth/flows/SquareOAuthFlow.java | 1 - .../oauth/flows/SquareOAuthFlowTest.java | 20 +++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java index a145ddbc4ec5d..99eeeb3c43bd8 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java @@ -63,7 +63,6 @@ protected String getAccessTokenUrl() { @Override protected Map extractRefreshToken(final JsonNode data, String accessTokenUrl) throws IOException { - System.out.println(data); if (data.has("refresh_token")) { return Map.of("authorization", Map.of("refresh_token", data.get("refresh_token").asText())); } else { diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SquareOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SquareOAuthFlowTest.java index 3f7682f53b8f3..184ef3205d56b 100644 --- a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SquareOAuthFlowTest.java +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SquareOAuthFlowTest.java @@ -58,16 +58,16 @@ public void setup() throws IOException, JsonValidationException { } - // @Test - // public void testGetSourceConcentUrl() throws IOException, ConfigNotFoundException { - // final String concentUrl = - // squareAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); - // assertEquals( - // // - // "https://connect.squareup.com/oauth2/authorize?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&scope=CUSTOMERS_WRITE%2BMERCHANT_PROFILE_READ%2BEMPLOYEES_READ%2BPAYMENTS_READ%2BCUSTOMERS_READ%2BTIMECARDS_READ%2BORDERS_READ&session=False&state=state", - // "https://connect.squareup.com/oauth2/authorize?client_id=test_client_id&scope=ITEMS_READ+CUSTOMERS_READ&session=False&state=state", - // concentUrl); - // } + @Test + public void testGetSourceConcentUrl() throws IOException, ConfigNotFoundException { + final String concentUrl = + squareAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); + assertEquals( + "https://connect.squareup.com/oauth2/authorize?client_id=test_client_id&scope=ITEMS_READ" + + "+CUSTOMERS_WRITE+MERCHANT_PROFILE_READ+EMPLOYEES_READ+PAYMENTS_READ+CUSTOMERS_READ" + + "+TIMECARDS_READ+ORDERS_READ&session=False&state=state", + concentUrl); + } @Test public void testCompleteSourceOAuth() throws IOException, InterruptedException, From b7a1fee52f17c7979f4ce3b5f83a5ae635add97c Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Mon, 1 Nov 2021 20:02:05 +0200 Subject: [PATCH 08/20] removed commented code --- .../src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java index 99eeeb3c43bd8..c79c3651292e9 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java @@ -45,7 +45,6 @@ protected String formatConsentUrl(UUID definitionId, String clientId, String red // Need to have decoded format, otherwice square fails saying that scope is incorrect return URLDecoder.decode(new URIBuilder(AUTHORIZE_URL) .addParameter("client_id", clientId) - // .addParameter("redirect_uri", redirectUrl) .addParameter("scope", SCOPE_VALUE) .addParameter("session", "False") .addParameter("state", getState()) @@ -79,7 +78,6 @@ protected Map getAccessTokenQueryParameters(String clientId, return ImmutableMap.builder() // required .put("client_id", clientId) - // .put("redirect_uri", redirectUrl) .put("client_secret", clientSecret) .put("code", authCode) .put("grant_type", "authorization_code") From 21a0117db0c44b4674d401547e51f016e098505b Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 2 Nov 2021 15:01:10 +0200 Subject: [PATCH 09/20] added support for old format of spec.json files, updated change logs docs --- .../connectors/source-square/source_square/source.py | 2 ++ .../connectors/source-square/source_square/spec.json | 2 +- docs/integrations/sources/square.md | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-square/source_square/source.py b/airbyte-integrations/connectors/source-square/source_square/source.py index e0e1b1441f4ff..b0729e92106d1 100644 --- a/airbyte-integrations/connectors/source-square/source_square/source.py +++ b/airbyte-integrations/connectors/source-square/source_square/source.py @@ -413,6 +413,8 @@ def get_auth(config: Mapping[str, Any]) -> AuthBase: ) elif auth_type == "Apikey": auth = TokenAuthenticator(token=authorization.get("api_key")) + elif not auth_type and config.get("api_key"): + auth = TokenAuthenticator(token=config.get("api_key")) else: raise Exception(f"Invalid auth type: {auth_type}") diff --git a/airbyte-integrations/connectors/source-square/source_square/spec.json b/airbyte-integrations/connectors/source-square/source_square/spec.json index 5c856b567f824..56c66eb853699 100644 --- a/airbyte-integrations/connectors/source-square/source_square/spec.json +++ b/airbyte-integrations/connectors/source-square/source_square/spec.json @@ -5,7 +5,7 @@ "title": "Square Source CDK Specifications", "type": "object", "required": ["authorization", "is_sandbox"], - "additionalProperties": false, + "additionalProperties": true, "properties": { "is_sandbox": { "type": "boolean", diff --git a/docs/integrations/sources/square.md b/docs/integrations/sources/square.md index 4ef083eeb2398..2713cdd2490fa 100644 --- a/docs/integrations/sources/square.md +++ b/docs/integrations/sources/square.md @@ -77,6 +77,7 @@ Some Square API endpoints has different page size limitation | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.2 | 2021-11-01 | [6842](https://github.com/airbytehq/airbyte/pull/6842) | Added oauth2 support | | 0.1.1 | 2021-07-09 | [4645](https://github.com/airbytehq/airbyte/pull/4645) | Update \_send\_request method due to Airbyte CDK changes | | 0.1.0 | 2021-06-30 | [4439](https://github.com/airbytehq/airbyte/pull/4439) | Initial release supporting the Square API | From 1367280581f2fdb456e4f4a7394768b23eda5244 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Wed, 1 Dec 2021 19:45:45 +0200 Subject: [PATCH 10/20] renamed spec property 'authentication' to default 'credentials'. fixed changes in java part --- .../source-square/source_square/source.py | 12 +- .../source-square/source_square/spec.json | 8 +- .../unit_tests/connection_test.py | 2 +- .../airbyte/oauth/flows/SquareOAuthFlow.java | 111 ------------------ .../SquareOAuthFlowIntegrationTest.java | 10 +- .../oauth/flows/SquareOAuthFlowTest.java | 87 -------------- 6 files changed, 17 insertions(+), 213 deletions(-) diff --git a/airbyte-integrations/connectors/source-square/source_square/source.py b/airbyte-integrations/connectors/source-square/source_square/source.py index 1bd120664bb8c..ae465043a8ca6 100644 --- a/airbyte-integrations/connectors/source-square/source_square/source.py +++ b/airbyte-integrations/connectors/source-square/source_square/source.py @@ -377,8 +377,8 @@ class SourceSquare(AbstractSource): @staticmethod def get_auth(config: Mapping[str, Any]) -> AuthBase: - authorization = config.get("authorization", {}) - auth_type = authorization.get("auth_type") + credential = config.get("credentials", {}) + auth_type = credential.get("auth_type") if auth_type == "Oauth": # scopes needed for all currently supported streams: scopes = [ @@ -405,14 +405,14 @@ def get_auth(config: Mapping[str, Any]) -> AuthBase: auth = Oauth2AuthenticatorSquare( token_refresh_endpoint="https://connect.squareup.com/oauth2/token", - client_secret=authorization.get("client_secret"), - client_id=authorization.get("client_id"), - refresh_token=authorization.get("refresh_token"), + client_secret=credential.get("client_secret"), + client_id=credential.get("client_id"), + refresh_token=credential.get("refresh_token"), scopes=scopes, expires_in_name="expires_at", ) elif auth_type == "Apikey": - auth = TokenAuthenticator(token=authorization.get("api_key")) + auth = TokenAuthenticator(token=credential.get("api_key")) elif not auth_type and config.get("api_key"): auth = TokenAuthenticator(token=config.get("api_key")) else: diff --git a/airbyte-integrations/connectors/source-square/source_square/spec.json b/airbyte-integrations/connectors/source-square/source_square/spec.json index 56c66eb853699..485674a3a8101 100644 --- a/airbyte-integrations/connectors/source-square/source_square/spec.json +++ b/airbyte-integrations/connectors/source-square/source_square/spec.json @@ -4,7 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Square Source CDK Specifications", "type": "object", - "required": ["authorization", "is_sandbox"], + "required": ["is_sandbox"], "additionalProperties": true, "properties": { "is_sandbox": { @@ -26,9 +26,9 @@ "examples": [true, false], "default": false }, - "authorization": { + "credentials": { "type": "object", - "title": "Authentication Type", + "title": "Credential Type", "oneOf": [ { "title": "Oauth authentication", @@ -94,7 +94,7 @@ "authSpecification": { "auth_type": "oauth2.0", "oauth2Specification": { - "rootObject": ["authorization", 0], + "rootObject": ["credentials", 0], "oauthFlowInitParameters": [["client_id"], ["client_secret"]], "oauthFlowOutputParameters": [["refresh_token"]] } diff --git a/airbyte-integrations/connectors/source-square/unit_tests/connection_test.py b/airbyte-integrations/connectors/source-square/unit_tests/connection_test.py index 9442d723d3714..5e028777a7cf8 100644 --- a/airbyte-integrations/connectors/source-square/unit_tests/connection_test.py +++ b/airbyte-integrations/connectors/source-square/unit_tests/connection_test.py @@ -9,7 +9,7 @@ def test_source_wrong_credentials(): source = SourceSquare() config = { - "authorization": {"auth_type": "Apikey", "api_key": "bla"}, + "credentials": {"auth_type": "Apikey", "api_key": "bla"}, "is_sandbox": True, "start_date": "2021-06-01", "include_deleted_objects": False, diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java index c79c3651292e9..e69de29bb2d1d 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java @@ -1,111 +0,0 @@ -/* - * Copyright (c) 2021 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.oauth.flows; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.oauth.BaseOAuthFlow; -import java.io.IOException; -import java.net.URISyntaxException; -import java.net.URLDecoder; -import java.net.http.HttpClient; -import java.nio.charset.StandardCharsets; -import java.util.Map; -import java.util.UUID; -import java.util.function.Supplier; -import org.apache.http.client.utils.URIBuilder; - -public class SquareOAuthFlow extends BaseOAuthFlow { - - private static final String SCOPE_VALUE = - "ITEMS_READ+CUSTOMERS_WRITE+MERCHANT_PROFILE_READ+EMPLOYEES_READ+PAYMENTS_READ+CUSTOMERS_READ+TIMECARDS_READ+ORDERS_READ"; - private static final String AUTHORIZE_URL = "https://connect.squareup.com/oauth2/authorize"; - private static final String ACCESS_TOKEN_URL = "https://connect.squareup.com/oauth2/token"; - - public SquareOAuthFlow(ConfigRepository configRepository) { - super(configRepository); - } - - @VisibleForTesting - public SquareOAuthFlow(ConfigRepository configRepository, - HttpClient httpClient, - Supplier stateSupplier) { - super(configRepository, httpClient, stateSupplier); - } - - @Override - protected String formatConsentUrl(UUID definitionId, String clientId, String redirectUrl) - throws IOException { - try { - // Need to have decoded format, otherwice square fails saying that scope is incorrect - return URLDecoder.decode(new URIBuilder(AUTHORIZE_URL) - .addParameter("client_id", clientId) - .addParameter("scope", SCOPE_VALUE) - .addParameter("session", "False") - .addParameter("state", getState()) - .build().toString(), StandardCharsets.UTF_8); - } catch (URISyntaxException e) { - throw new IOException("Failed to format Consent URL for OAuth flow", e); - } - } - - @Override - protected String getAccessTokenUrl() { - return ACCESS_TOKEN_URL; - } - - @Override - protected Map extractRefreshToken(final JsonNode data, String accessTokenUrl) - throws IOException { - if (data.has("refresh_token")) { - return Map.of("authorization", Map.of("refresh_token", data.get("refresh_token").asText())); - } else { - throw new IOException( - String.format("Missing 'refresh_token' in query params from %s", ACCESS_TOKEN_URL)); - } - } - - @Override - protected Map getAccessTokenQueryParameters(String clientId, - String clientSecret, - String authCode, - String redirectUrl) { - return ImmutableMap.builder() - // required - .put("client_id", clientId) - .put("client_secret", clientSecret) - .put("code", authCode) - .put("grant_type", "authorization_code") - .put("scopes", "[\n" - + " \"ITEMS_READ\",\n" - + " \"MERCHANT_PROFILE_READ\",\n" - + " \"EMPLOYEES_READ\",\n" - + " \"PAYMENTS_READ\",\n" - + " \"CUSTOMERS_READ\",\n" - + " \"TIMECARDS_READ\",\n" - + " \"ORDERS_READ\"\n" - + " ]") - .build(); - - } - - @Override - protected String getClientIdUnsafe(final JsonNode config) { - // the config object containing client ID and secret is nested inside the "authorization" object - Preconditions.checkArgument(config.hasNonNull("authorization")); - return super.getClientIdUnsafe(config.get("authorization")); - } - - @Override - protected String getClientSecretUnsafe(final JsonNode config) { - // the config object containing client ID and secret is nested inside the "authorization" object - Preconditions.checkArgument(config.hasNonNull("authorization")); - return super.getClientSecretUnsafe(config.get("authorization")); - } - -} diff --git a/airbyte-oauth/src/test-integration/java/io.airbyte.oauth.flows/SquareOAuthFlowIntegrationTest.java b/airbyte-oauth/src/test-integration/java/io.airbyte.oauth.flows/SquareOAuthFlowIntegrationTest.java index d48e3166951a9..03da842f2471b 100644 --- a/airbyte-oauth/src/test-integration/java/io.airbyte.oauth.flows/SquareOAuthFlowIntegrationTest.java +++ b/airbyte-oauth/src/test-integration/java/io.airbyte.oauth.flows/SquareOAuthFlowIntegrationTest.java @@ -16,6 +16,7 @@ import io.airbyte.oauth.OAuthFlowImplementation; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; +import java.net.http.HttpClient; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; @@ -31,13 +32,13 @@ public class SquareOAuthFlowIntegrationTest extends OAuthFlowIntegrationTest { protected static final int SERVER_LISTENING_PORT = 8000; @Override - protected Path get_credentials_path() { + protected Path getCredentialsPath() { return CREDENTIALS_PATH; } @Override - protected OAuthFlowImplementation getFlowObject(ConfigRepository configRepository) { - return new SquareOAuthFlow(configRepository); + protected OAuthFlowImplementation getFlowImplementation(final ConfigRepository configRepository, final HttpClient httpClient) { + return new SquareOAuthFlow(configRepository, httpClient); } @Override @@ -45,6 +46,7 @@ protected int getServerListeningPort() { return SERVER_LISTENING_PORT; } + @Override @BeforeEach public void setup() throws IOException { super.setup(); @@ -68,7 +70,7 @@ public void testFullSquareOAuthFlow() throws InterruptedException, ConfigNotFoun .put("client_secret", credentialsJson.get("client_secret").asText()) .build()))))); - final String url = flow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); + final String url = flow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL, Jsons.emptyObject(), null); LOGGER.info("Waiting for user consent at: {}", url); // TODO: To automate, start a selenium job to navigate to the Consent URL and click on allowing diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SquareOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SquareOAuthFlowTest.java index 184ef3205d56b..e69de29bb2d1d 100644 --- a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SquareOAuthFlowTest.java +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SquareOAuthFlowTest.java @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2021 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.oauth.flows; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.config.SourceOAuthParameter; -import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.validation.json.JsonValidationException; -import java.io.IOException; -import java.net.http.HttpClient; -import java.net.http.HttpResponse; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class SquareOAuthFlowTest { - - private UUID workspaceId; - private UUID definitionId; - private SquareOAuthFlow squareAuthFlow; - private HttpClient httpClient; - - private static final String REDIRECT_URL = "https://airbyte.io"; - - private static String getConstantState() { - return "state"; - } - - @BeforeEach - public void setup() throws IOException, JsonValidationException { - workspaceId = UUID.randomUUID(); - definitionId = UUID.randomUUID(); - ConfigRepository configRepository = mock(ConfigRepository.class); - httpClient = mock(HttpClient.class); - when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withSourceDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode( - Map.of("authorization", - ImmutableMap.builder() - .put("client_id", "test_client_id") - .put("client_secret", "test_client_secret") - .build()))))); - squareAuthFlow = new SquareOAuthFlow(configRepository, httpClient, - SquareOAuthFlowTest::getConstantState); - - } - - @Test - public void testGetSourceConcentUrl() throws IOException, ConfigNotFoundException { - final String concentUrl = - squareAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); - assertEquals( - "https://connect.squareup.com/oauth2/authorize?client_id=test_client_id&scope=ITEMS_READ" - + "+CUSTOMERS_WRITE+MERCHANT_PROFILE_READ+EMPLOYEES_READ+PAYMENTS_READ+CUSTOMERS_READ" - + "+TIMECARDS_READ+ORDERS_READ&session=False&state=state", - concentUrl); - } - - @Test - public void testCompleteSourceOAuth() throws IOException, InterruptedException, - ConfigNotFoundException { - - Map returnedCredentials = Map.of("refresh_token", "refresh_token_response"); - final HttpResponse response = mock(HttpResponse.class); - when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); - when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = - squareAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); - assertEquals(Jsons.serialize(Map.of("authorization", returnedCredentials)), - Jsons.serialize(actualQueryParams)); - } - -} From ce9103691d5d14b75205918fd2e0521599c9d885 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Wed, 1 Dec 2021 20:21:29 +0200 Subject: [PATCH 11/20] recovered empty files --- .../airbyte/oauth/flows/SquareOAuthFlow.java | 89 +++++++++++++++++++ .../oauth/flows/SquareOAuthFlowTest.java | 27 ++++++ 2 files changed, 116 insertions(+) diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java index e69de29bb2d1d..5c3f863a2b399 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.oauth.BaseOAuth2Flow; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URLDecoder; +import java.net.http.HttpClient; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.List; +import java.util.UUID; +import java.util.function.Supplier; +import org.apache.http.client.utils.URIBuilder; + +public class SquareOAuthFlow extends BaseOAuth2Flow { + + private static final String SCOPE_VALUE = + "ITEMS_READ+CUSTOMERS_WRITE+MERCHANT_PROFILE_READ+EMPLOYEES_READ+PAYMENTS_READ+CUSTOMERS_READ+TIMECARDS_READ+ORDERS_READ"; + private static final String AUTHORIZE_URL = "https://connect.squareup.com/oauth2/authorize"; + private static final String ACCESS_TOKEN_URL = "https://connect.squareup.com/oauth2/token"; + + public SquareOAuthFlow(ConfigRepository configRepository, final HttpClient httpClient) { + super(configRepository, httpClient); + } + + @VisibleForTesting + public SquareOAuthFlow(ConfigRepository configRepository, + HttpClient httpClient, + Supplier stateSupplier) { + super(configRepository, httpClient, stateSupplier); + } + + @Override + protected String formatConsentUrl(final UUID definitionId, + final String clientId, + final String redirectUrl, + final JsonNode inputOAuthConfiguration) + throws IOException { + try { + // Need to have decoded format, otherwice square fails saying that scope is incorrect + return URLDecoder.decode(new URIBuilder(AUTHORIZE_URL) + .addParameter("client_id", clientId) + .addParameter("scope", SCOPE_VALUE) + .addParameter("session", "False") + .addParameter("state", getState()) + .build().toString(), StandardCharsets.UTF_8); + } catch (URISyntaxException e) { + throw new IOException("Failed to format Consent URL for OAuth flow", e); + } + } + + @Override + protected String getAccessTokenUrl() { + return ACCESS_TOKEN_URL; + } + + @Override + protected Map getAccessTokenQueryParameters(String clientId, + String clientSecret, + String authCode, + String redirectUrl) { + return ImmutableMap.builder() + // required + .put("client_id", clientId) + .put("client_secret", clientSecret) + .put("code", authCode) + .put("grant_type", "authorization_code") + .put("scopes", "[\n" + + " \"ITEMS_READ\",\n" + + " \"MERCHANT_PROFILE_READ\",\n" + + " \"EMPLOYEES_READ\",\n" + + " \"PAYMENTS_READ\",\n" + + " \"CUSTOMERS_READ\",\n" + + " \"TIMECARDS_READ\",\n" + + " \"ORDERS_READ\"\n" + + " ]") + .build(); + } + +} diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SquareOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SquareOAuthFlowTest.java index e69de29bb2d1d..377babf3ec95b 100644 --- a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SquareOAuthFlowTest.java +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SquareOAuthFlowTest.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.oauth.BaseOAuthFlow; +import io.airbyte.oauth.MoreOAuthParameters; +import java.util.List; +import java.util.Map; + +public class SquareOAuthFlowTest extends BaseOAuthFlowTest { + + @Override + protected BaseOAuthFlow getOAuthFlow() { + return new SquareOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState); + } + + @Override + protected String getExpectedConsentUrl() { + return "https://connect.squareup.com/oauth2/authorize?client_id=test_client_id&scope=ITEMS_READ" + + "+CUSTOMERS_WRITE+MERCHANT_PROFILE_READ+EMPLOYEES_READ+PAYMENTS_READ+CUSTOMERS_READ" + + "+TIMECARDS_READ+ORDERS_READ&session=False&state=state"; + } + +} \ No newline at end of file From a8d6c70026b67692ecab882ac7e6e2a6b1c8c1b2 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Thu, 2 Dec 2021 13:17:19 +0200 Subject: [PATCH 12/20] updated OAuthImplementationFactory.java --- .../io/airbyte/oauth/OAuthImplementationFactory.java | 1 + .../java/io/airbyte/oauth/flows/SquareOAuthFlow.java | 2 -- .../io/airbyte/oauth/flows/SquareOAuthFlowTest.java | 10 +++------- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java index 32deb1ddf235e..50f53f5db9413 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java @@ -21,6 +21,7 @@ import io.airbyte.oauth.flows.SlackOAuthFlow; import io.airbyte.oauth.flows.SnapchatMarketingOAuthFlow; import io.airbyte.oauth.flows.SurveymonkeyOAuthFlow; +import io.airbyte.oauth.flows.SquareOAuthFlow; import io.airbyte.oauth.flows.TrelloOAuthFlow; import io.airbyte.oauth.flows.facebook.FacebookMarketingOAuthFlow; import io.airbyte.oauth.flows.facebook.FacebookPagesOAuthFlow; diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java index 5c3f863a2b399..84f7ccec647cc 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.oauth.BaseOAuth2Flow; @@ -16,7 +15,6 @@ import java.net.http.HttpClient; import java.nio.charset.StandardCharsets; import java.util.Map; -import java.util.List; import java.util.UUID; import java.util.function.Supplier; import org.apache.http.client.utils.URIBuilder; diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SquareOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SquareOAuthFlowTest.java index 377babf3ec95b..00c27aad9d82f 100644 --- a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SquareOAuthFlowTest.java +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SquareOAuthFlowTest.java @@ -4,11 +4,7 @@ package io.airbyte.oauth.flows; -import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.oauth.BaseOAuthFlow; -import io.airbyte.oauth.MoreOAuthParameters; -import java.util.List; -import java.util.Map; public class SquareOAuthFlowTest extends BaseOAuthFlowTest { @@ -20,8 +16,8 @@ protected BaseOAuthFlow getOAuthFlow() { @Override protected String getExpectedConsentUrl() { return "https://connect.squareup.com/oauth2/authorize?client_id=test_client_id&scope=ITEMS_READ" - + "+CUSTOMERS_WRITE+MERCHANT_PROFILE_READ+EMPLOYEES_READ+PAYMENTS_READ+CUSTOMERS_READ" - + "+TIMECARDS_READ+ORDERS_READ&session=False&state=state"; + + "+CUSTOMERS_WRITE+MERCHANT_PROFILE_READ+EMPLOYEES_READ+PAYMENTS_READ+CUSTOMERS_READ" + + "+TIMECARDS_READ+ORDERS_READ&session=False&state=state"; } -} \ No newline at end of file +} From ce2477e697069ffec7d5d00e2eee432b25bb24c9 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Thu, 2 Dec 2021 14:52:04 +0200 Subject: [PATCH 13/20] fixed issue with autheticator for sub streams, added config catalog with all streams, updated docs --- .../integration_tests/configured_catalog.json | 12 ++++++++++++ .../connectors/source-square/source_square/source.py | 8 +++++--- .../io/airbyte/oauth/OAuthImplementationFactory.java | 3 ++- docs/integrations/sources/square.md | 11 ++++++----- 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/airbyte-integrations/connectors/source-square/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-square/integration_tests/configured_catalog.json index b47d38d39da0d..f34fd4fc8ea05 100644 --- a/airbyte-integrations/connectors/source-square/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-square/integration_tests/configured_catalog.json @@ -143,6 +143,18 @@ "sync_mode": "full_refresh", "cursor_field": ["id"], "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "orders", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "full_refresh", + "cursor_field": ["id"], + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-square/source_square/source.py b/airbyte-integrations/connectors/source-square/source_square/source.py index ae465043a8ca6..576dcde028493 100644 --- a/airbyte-integrations/connectors/source-square/source_square/source.py +++ b/airbyte-integrations/connectors/source-square/source_square/source.py @@ -4,7 +4,7 @@ import json from abc import ABC, abstractmethod -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union import pendulum import requests @@ -13,6 +13,7 @@ from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth.core import HttpAuthenticator from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator, TokenAuthenticator from requests.auth import AuthBase from source_square.utils import separate_items_by_count @@ -37,8 +38,9 @@ def parse_square_error_response(error: requests.exceptions.HTTPError) -> SquareE class SquareStream(HttpStream, ABC): - def __init__(self, is_sandbox: bool, api_version: str, start_date: str, include_deleted_objects: bool, **kwargs): - super().__init__(**kwargs) + def __init__(self, is_sandbox: bool, api_version: str, start_date: str, include_deleted_objects: bool, authenticator: Union[AuthBase, HttpAuthenticator]): + super().__init__(authenticator) + self._authenticator = authenticator self.is_sandbox = is_sandbox self.api_version = api_version # Converting users ISO 8601 format (YYYY-MM-DD) to RFC 3339 (2021-06-14T13:47:56.799Z) diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java index 50f53f5db9413..d56ba7e8e4004 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java @@ -20,8 +20,8 @@ import io.airbyte.oauth.flows.SalesforceOAuthFlow; import io.airbyte.oauth.flows.SlackOAuthFlow; import io.airbyte.oauth.flows.SnapchatMarketingOAuthFlow; -import io.airbyte.oauth.flows.SurveymonkeyOAuthFlow; import io.airbyte.oauth.flows.SquareOAuthFlow; +import io.airbyte.oauth.flows.SurveymonkeyOAuthFlow; import io.airbyte.oauth.flows.TrelloOAuthFlow; import io.airbyte.oauth.flows.facebook.FacebookMarketingOAuthFlow; import io.airbyte.oauth.flows.facebook.FacebookPagesOAuthFlow; @@ -58,6 +58,7 @@ public OAuthImplementationFactory(final ConfigRepository configRepository, final .put("airbyte/source-salesforce", new SalesforceOAuthFlow(configRepository, httpClient)) .put("airbyte/source-slack", new SlackOAuthFlow(configRepository, httpClient)) .put("airbyte/source-snapchat-marketing", new SnapchatMarketingOAuthFlow(configRepository, httpClient)) + .put("airbyte/source-square", new SquareOAuthFlow(configRepository, httpClient)) .put("airbyte/source-surveymonkey", new SurveymonkeyOAuthFlow(configRepository, httpClient)) .put("airbyte/source-trello", new TrelloOAuthFlow(configRepository, httpClient)) .put("airbyte/source-youtube-analytics", new YouTubeAnalyticsOAuthFlow(configRepository, httpClient)) diff --git a/docs/integrations/sources/square.md b/docs/integrations/sources/square.md index 8f24ae72af497..354d7c32896fb 100644 --- a/docs/integrations/sources/square.md +++ b/docs/integrations/sources/square.md @@ -75,9 +75,10 @@ Some Square API endpoints has different page size limitation ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.2 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | -| 0.1.1 | 2021-07-09 | [4645](https://github.com/airbytehq/airbyte/pull/4645) | Update \_send\_request method due to Airbyte CDK changes | -| 0.1.0 | 2021-06-30 | [4439](https://github.com/airbytehq/airbyte/pull/4439) | Initial release supporting the Square API | +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :--- |:---------------------------------------------------------| +| 0.1.3 | 2021-12-02 | [6842](https://github.com/airbytehq/airbyte/pull/6842) | Added oauth support | +| 0.1.2 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | +| 0.1.1 | 2021-07-09 | [4645](https://github.com/airbytehq/airbyte/pull/4645) | Update \_send\_request method due to Airbyte CDK changes | +| 0.1.0 | 2021-06-30 | [4439](https://github.com/airbytehq/airbyte/pull/4439) | Initial release supporting the Square API | From 9d7c3a36141eb14a290551736fa83ddfa773bada Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Mon, 27 Dec 2021 15:51:42 +0200 Subject: [PATCH 14/20] use advanced_auth --- .../source-square/source_square/spec.json | 44 +++++++++++++++++++ .../airbyte/oauth/flows/SquareOAuthFlow.java | 2 +- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-square/source_square/spec.json b/airbyte-integrations/connectors/source-square/source_square/spec.json index 190c473b79e2e..23c77b753f8fe 100644 --- a/airbyte-integrations/connectors/source-square/source_square/spec.json +++ b/airbyte-integrations/connectors/source-square/source_square/spec.json @@ -101,5 +101,49 @@ "oauthFlowInitParameters": [["client_id"], ["client_secret"]], "oauthFlowOutputParameters": [["refresh_token"]] } + }, + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "Oauth", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "refresh_token": { + "type": "string", + "path_in_connector_config": ["credentials", "refresh_token"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["credentials", "client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["credentials", "client_secret"] + } + } + } + } } + } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java index 84f7ccec647cc..b30c27ecbf962 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java @@ -44,7 +44,7 @@ protected String formatConsentUrl(final UUID definitionId, final JsonNode inputOAuthConfiguration) throws IOException { try { - // Need to have decoded format, otherwice square fails saying that scope is incorrect + // Need to have decoded format, otherwise square fails saying that scope is incorrect return URLDecoder.decode(new URIBuilder(AUTHORIZE_URL) .addParameter("client_id", clientId) .addParameter("scope", SCOPE_VALUE) From 27020e7896e075ba809a671eb263f006734abdda Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 11 Jan 2022 16:50:37 +0200 Subject: [PATCH 15/20] added advanced_auth --- .../connectors/source-square/source_square/source.py | 9 ++++++++- .../java/io/airbyte/oauth/flows/SquareOAuthFlow.java | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-square/source_square/source.py b/airbyte-integrations/connectors/source-square/source_square/source.py index 576dcde028493..6d0bd64c638f3 100644 --- a/airbyte-integrations/connectors/source-square/source_square/source.py +++ b/airbyte-integrations/connectors/source-square/source_square/source.py @@ -38,7 +38,14 @@ def parse_square_error_response(error: requests.exceptions.HTTPError) -> SquareE class SquareStream(HttpStream, ABC): - def __init__(self, is_sandbox: bool, api_version: str, start_date: str, include_deleted_objects: bool, authenticator: Union[AuthBase, HttpAuthenticator]): + def __init__( + self, + is_sandbox: bool, + api_version: str, + start_date: str, + include_deleted_objects: bool, + authenticator: Union[AuthBase, HttpAuthenticator], + ): super().__init__(authenticator) self._authenticator = authenticator self.is_sandbox = is_sandbox diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java index b30c27ecbf962..9ac9b788b8a67 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java @@ -57,7 +57,7 @@ protected String formatConsentUrl(final UUID definitionId, } @Override - protected String getAccessTokenUrl() { + protected String getAccessTokenUrl(final JsonNode inputOAuthConfiguration) { return ACCESS_TOKEN_URL; } From b7ba86dc4eaa293f930dfaf5f53f4104da630d4f Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 11 Jan 2022 20:55:54 +0200 Subject: [PATCH 16/20] moved scopes to private property --- .../airbyte/oauth/flows/SquareOAuthFlow.java | 43 +++++++++++++------ .../oauth/flows/SquareOAuthFlowTest.java | 6 +-- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java index 9ac9b788b8a67..1e9d821a67a1b 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SquareOAuthFlow.java @@ -14,15 +14,37 @@ import java.net.URLDecoder; import java.net.http.HttpClient; import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.UUID; import java.util.function.Supplier; +import java.util.stream.Collectors; import org.apache.http.client.utils.URIBuilder; public class SquareOAuthFlow extends BaseOAuth2Flow { - private static final String SCOPE_VALUE = - "ITEMS_READ+CUSTOMERS_WRITE+MERCHANT_PROFILE_READ+EMPLOYEES_READ+PAYMENTS_READ+CUSTOMERS_READ+TIMECARDS_READ+ORDERS_READ"; + private static final List SCOPES = Arrays.asList( + "CUSTOMERS_READ", + "EMPLOYEES_READ", + "ITEMS_READ", + "MERCHANT_PROFILE_READ", + "ORDERS_READ", + "PAYMENTS_READ", + "TIMECARDS_READ" + // OAuth Permissions: + // https://developer.squareup.com/docs/oauth-api/square-permissions + // https://developer.squareup.com/reference/square/enums/OAuthPermission + // "DISPUTES_READ", + // "GIFTCARDS_READ", + // "INVENTORY_READ", + // "INVOICES_READ", + // "TIMECARDS_SETTINGS_READ", + // "LOYALTY_READ", + // "ONLINE_STORE_SITE_READ", + // "ONLINE_STORE_SNIPPETS_READ", + // "SUBSCRIPTIONS_READ", + ); private static final String AUTHORIZE_URL = "https://connect.squareup.com/oauth2/authorize"; private static final String ACCESS_TOKEN_URL = "https://connect.squareup.com/oauth2/token"; @@ -47,7 +69,7 @@ protected String formatConsentUrl(final UUID definitionId, // Need to have decoded format, otherwise square fails saying that scope is incorrect return URLDecoder.decode(new URIBuilder(AUTHORIZE_URL) .addParameter("client_id", clientId) - .addParameter("scope", SCOPE_VALUE) + .addParameter("scope", String.join("+", SCOPES)) .addParameter("session", "False") .addParameter("state", getState()) .build().toString(), StandardCharsets.UTF_8); @@ -66,21 +88,18 @@ protected Map getAccessTokenQueryParameters(String clientId, String clientSecret, String authCode, String redirectUrl) { + String scopes = SCOPES.stream() + .map(name -> ('"' + name + '"')) + .collect(Collectors.joining(",")); + scopes = '[' + scopes + ']'; + return ImmutableMap.builder() // required .put("client_id", clientId) .put("client_secret", clientSecret) .put("code", authCode) .put("grant_type", "authorization_code") - .put("scopes", "[\n" - + " \"ITEMS_READ\",\n" - + " \"MERCHANT_PROFILE_READ\",\n" - + " \"EMPLOYEES_READ\",\n" - + " \"PAYMENTS_READ\",\n" - + " \"CUSTOMERS_READ\",\n" - + " \"TIMECARDS_READ\",\n" - + " \"ORDERS_READ\"\n" - + " ]") + .put("scopes", scopes) .build(); } diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SquareOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SquareOAuthFlowTest.java index 00c27aad9d82f..5c54891086a59 100644 --- a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SquareOAuthFlowTest.java +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SquareOAuthFlowTest.java @@ -15,9 +15,9 @@ protected BaseOAuthFlow getOAuthFlow() { @Override protected String getExpectedConsentUrl() { - return "https://connect.squareup.com/oauth2/authorize?client_id=test_client_id&scope=ITEMS_READ" - + "+CUSTOMERS_WRITE+MERCHANT_PROFILE_READ+EMPLOYEES_READ+PAYMENTS_READ+CUSTOMERS_READ" - + "+TIMECARDS_READ+ORDERS_READ&session=False&state=state"; + return "https://connect.squareup.com/oauth2/authorize?client_id=test_client_id" + + "&scope=CUSTOMERS_READ+EMPLOYEES_READ+ITEMS_READ+MERCHANT_PROFILE_READ+ORDERS_READ+PAYMENTS_READ+TIMECARDS_READ" + + "&session=False&state=state"; } } From ce3d06165c4bbbe1592e22203d6b6c545deec9a9 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Mon, 17 Jan 2022 21:24:25 +0200 Subject: [PATCH 17/20] updated source version --- .../77225a51-cd15-4a13-af02-65816bd0ecf4.json | 2 +- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-integrations/connectors/source-square/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/77225a51-cd15-4a13-af02-65816bd0ecf4.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/77225a51-cd15-4a13-af02-65816bd0ecf4.json index 7daaf44dedab3..e5fb4122ccebc 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/77225a51-cd15-4a13-af02-65816bd0ecf4.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/77225a51-cd15-4a13-af02-65816bd0ecf4.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "77225a51-cd15-4a13-af02-65816bd0ecf4", "name": "Square", "dockerRepository": "airbyte/source-square", - "dockerImageTag": "0.1.3", + "dockerImageTag": "0.1.4", "documentationUrl": "https://docs.airbyte.io/integrations/sources/square", "icon": "square.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 374869a1e9196..24995c1425bfb 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -670,7 +670,7 @@ - name: Square sourceDefinitionId: 77225a51-cd15-4a13-af02-65816bd0ecf4 dockerRepository: airbyte/source-square - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/sources/square icon: square.svg sourceType: api diff --git a/airbyte-integrations/connectors/source-square/Dockerfile b/airbyte-integrations/connectors/source-square/Dockerfile index ec5e91b1622fd..ef116771ded83 100644 --- a/airbyte-integrations/connectors/source-square/Dockerfile +++ b/airbyte-integrations/connectors/source-square/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/source-square From 64e2835873fd7589c6e870df7f54a0e4d96902f0 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Mon, 17 Jan 2022 21:48:51 +0200 Subject: [PATCH 18/20] Revert "updated source version" This reverts commit ce3d06165c4bbbe1592e22203d6b6c545deec9a9. --- .../77225a51-cd15-4a13-af02-65816bd0ecf4.json | 2 +- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-integrations/connectors/source-square/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/77225a51-cd15-4a13-af02-65816bd0ecf4.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/77225a51-cd15-4a13-af02-65816bd0ecf4.json index e5fb4122ccebc..7daaf44dedab3 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/77225a51-cd15-4a13-af02-65816bd0ecf4.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/77225a51-cd15-4a13-af02-65816bd0ecf4.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "77225a51-cd15-4a13-af02-65816bd0ecf4", "name": "Square", "dockerRepository": "airbyte/source-square", - "dockerImageTag": "0.1.4", + "dockerImageTag": "0.1.3", "documentationUrl": "https://docs.airbyte.io/integrations/sources/square", "icon": "square.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 372fbe1101018..7160cb6ecc2d3 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -676,7 +676,7 @@ - name: Square sourceDefinitionId: 77225a51-cd15-4a13-af02-65816bd0ecf4 dockerRepository: airbyte/source-square - dockerImageTag: 0.1.4 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.io/integrations/sources/square icon: square.svg sourceType: api diff --git a/airbyte-integrations/connectors/source-square/Dockerfile b/airbyte-integrations/connectors/source-square/Dockerfile index ef116771ded83..ec5e91b1622fd 100644 --- a/airbyte-integrations/connectors/source-square/Dockerfile +++ b/airbyte-integrations/connectors/source-square/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/source-square From 0f67641a3d51e601cb06f773eb5a3d4fa621789d Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 18 Jan 2022 08:46:52 +0200 Subject: [PATCH 19/20] updated source version --- airbyte-integrations/connectors/source-square/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-square/Dockerfile b/airbyte-integrations/connectors/source-square/Dockerfile index ec5e91b1622fd..ef116771ded83 100644 --- a/airbyte-integrations/connectors/source-square/Dockerfile +++ b/airbyte-integrations/connectors/source-square/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/source-square From d9f0ca71b22159c3c11106bdf46691e2ae98b09b Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 18 Jan 2022 09:10:06 +0200 Subject: [PATCH 20/20] added new version for airbyte index --- .../77225a51-cd15-4a13-af02-65816bd0ecf4.json | 2 +- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 118 ++++++++++++++++-- 3 files changed, 109 insertions(+), 13 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/77225a51-cd15-4a13-af02-65816bd0ecf4.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/77225a51-cd15-4a13-af02-65816bd0ecf4.json index 7daaf44dedab3..e5fb4122ccebc 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/77225a51-cd15-4a13-af02-65816bd0ecf4.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/77225a51-cd15-4a13-af02-65816bd0ecf4.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "77225a51-cd15-4a13-af02-65816bd0ecf4", "name": "Square", "dockerRepository": "airbyte/source-square", - "dockerImageTag": "0.1.3", + "dockerImageTag": "0.1.4", "documentationUrl": "https://docs.airbyte.io/integrations/sources/square", "icon": "square.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 7160cb6ecc2d3..372fbe1101018 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -676,7 +676,7 @@ - name: Square sourceDefinitionId: 77225a51-cd15-4a13-af02-65816bd0ecf4 dockerRepository: airbyte/source-square - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/sources/square icon: square.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index e8cafd8576377..1a6fc9b152605 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -7049,7 +7049,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-square:0.1.3" +- dockerImage: "airbyte/source-square:0.1.4" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/square" connectionSpecification: @@ -7057,15 +7057,9 @@ title: "Square Source CDK Specifications" type: "object" required: - - "api_key" - "is_sandbox" - additionalProperties: false + additionalProperties: true properties: - api_key: - type: "string" - description: "The API key for a Square application." - title: "API Key" - airbyte_secret: true is_sandbox: type: "boolean" description: "Determines whether to use the sandbox or production environment." @@ -7073,7 +7067,7 @@ examples: - true - false - default: true + default: false start_date: type: "string" description: "UTC date in the format YYYY-MM-DD. Any data before this date\ @@ -7081,20 +7075,122 @@ title: "Start Date" examples: - "2021-01-01" - default: "1970-01-01" + default: "2021-01-01" pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" include_deleted_objects: type: "boolean" description: "In some streams there is an option to include deleted objects\ \ (Items, Categories, Discounts, Taxes)" - title: "Include Deleded Objects" + title: "Include Deleted Objects" examples: - true - false default: false + credentials: + type: "object" + title: "Credential Type" + oneOf: + - title: "Oauth authentication" + type: "object" + required: + - "auth_type" + - "client_id" + - "client_secret" + - "refresh_token" + properties: + auth_type: + type: "string" + const: "Oauth" + enum: + - "Oauth" + default: "Oauth" + order: 0 + client_id: + title: "Client ID" + type: "string" + description: "The Square-issued ID of your application" + airbyte_secret: true + client_secret: + title: "Client Secret" + type: "string" + description: "The Square-issued application secret for your application" + airbyte_secret: true + refresh_token: + title: "Refresh Token" + type: "string" + description: "A refresh token generated using the above client ID\ + \ and secret" + airbyte_secret: true + - type: "object" + title: "API Key" + required: + - "auth_type" + - "api_key" + properties: + auth_type: + type: "string" + const: "Apikey" + enum: + - "Apikey" + default: "Apikey" + order: 1 + api_key: + title: "API key token" + type: "string" + description: "The API key for a Square application" + airbyte_secret: true supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] + authSpecification: + auth_type: "oauth2.0" + oauth2Specification: + rootObject: + - "credentials" + - "0" + oauthFlowInitParameters: + - - "client_id" + - - "client_secret" + oauthFlowOutputParameters: + - - "refresh_token" + advanced_auth: + auth_flow_type: "oauth2.0" + predicate_key: + - "credentials" + - "auth_type" + predicate_value: "Oauth" + oauth_config_specification: + complete_oauth_output_specification: + type: "object" + additionalProperties: false + properties: + refresh_token: + type: "string" + path_in_connector_config: + - "credentials" + - "refresh_token" + complete_oauth_server_input_specification: + type: "object" + additionalProperties: false + properties: + client_id: + type: "string" + client_secret: + type: "string" + complete_oauth_server_output_specification: + type: "object" + additionalProperties: false + properties: + client_id: + type: "string" + path_in_connector_config: + - "credentials" + - "client_id" + client_secret: + type: "string" + path_in_connector_config: + - "credentials" + - "client_secret" - dockerImage: "airbyte/source-strava:0.1.2" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/strava"