Skip to content

Commit e902607

Browse files
artem1205artem1205
authored
Source Azure Blob Storage: support OAuth (#36825)
Signed-off-by: Artem Inzhyyants <[email protected]>
1 parent dbf882c commit e902607

21 files changed

+479
-36
lines changed

‎airbyte-integrations/connectors/source-azure-blob-storage/README.md

+20
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,26 @@ For information about how to use this connector within Airbyte, see [the documen
1111
* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation)
1212

1313

14+
### Generate new oauth token
15+
16+
Tenant id should be provided by user, reason:
17+
https://learn.microsoft.com/en-us/answers/questions/1531138/which-tenant-id-do-i-have-to-use-to-get-tokens-and
18+
19+
1. GET https://login.microsoftonline.com/<tenant_id>/oauth2/v2.0/authorize
20+
?response_type=code
21+
&client_id=<client_id>
22+
&scope=offline_access https://storage.azure.com/.default
23+
&redirect_uri=http://localhost:8000/auth_flow
24+
&response_mode=query
25+
&state=1234
26+
27+
2. POST https://login.microsoftonline.com/<tenant_id>/oauth2/v2.0/token
28+
client_id:<client_id>
29+
code:<code obtained from previous request>
30+
redirect_uri:http://localhost:8000/auth_flow
31+
grant_type:authorization_code
32+
client_secret:<client_secret>
33+
1434
### Installing the connector
1535
From this connector directory, run:
1636
```bash

‎airbyte-integrations/connectors/source-azure-blob-storage/acceptance-test-config.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ acceptance_tests:
99
expect_records:
1010
path: integration_tests/expected_records/csv_custom_encoding.jsonl
1111
exact_order: true
12-
- config_path: secrets/csv_custom_format_config.json
12+
- config_path: secrets/csv_custom_format_encoding_config.json
1313
expect_records:
1414
path: integration_tests/expected_records/csv_custom_format.jsonl
1515
exact_order: true
@@ -64,7 +64,7 @@ acceptance_tests:
6464
status: succeed
6565
- config_path: secrets/csv_custom_encoding_config.json
6666
status: succeed
67-
- config_path: secrets/csv_custom_format_config.json
67+
- config_path: secrets/csv_custom_format_encoding_config.json
6868
status: succeed
6969
- config_path: secrets/csv_user_schema_config.json
7070
status: succeed
@@ -92,7 +92,7 @@ acceptance_tests:
9292
tests:
9393
- config_path: secrets/config.json
9494
- config_path: secrets/csv_custom_encoding_config.json
95-
- config_path: secrets/csv_custom_format_config.json
95+
- config_path: secrets/csv_custom_format_encoding_config.json
9696
- config_path: secrets/csv_user_schema_config.json
9797
- config_path: secrets/csv_no_header_config.json
9898
- config_path: secrets/csv_skip_rows_config.json

‎airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configs/config_integration_avro.json

+4-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
{
22
"azure_blob_storage_endpoint": "http://localhost:10000/account1",
33
"azure_blob_storage_account_name": "account1",
4-
"azure_blob_storage_account_key": "key1",
4+
"credentials": {
5+
"auth_type": "storage_account_key",
6+
"azure_blob_storage_account_key": "key1"
7+
},
58
"azure_blob_storage_container_name": "testcontainer",
69
"streams": [
710
{

‎airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configs/config_integration_csv.json

+4-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
{
22
"azure_blob_storage_endpoint": "http://localhost:10000/account1",
33
"azure_blob_storage_account_name": "account1",
4-
"azure_blob_storage_account_key": "key1",
4+
"credentials": {
5+
"auth_type": "storage_account_key",
6+
"azure_blob_storage_account_key": "key1"
7+
},
58
"azure_blob_storage_container_name": "testcontainer",
69
"streams": [
710
{

‎airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configs/config_integration_jsonl.json

+4-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
{
22
"azure_blob_storage_endpoint": "http://localhost:10000/account1",
33
"azure_blob_storage_account_name": "account1",
4-
"azure_blob_storage_account_key": "key1",
4+
"credentials": {
5+
"auth_type": "storage_account_key",
6+
"azure_blob_storage_account_key": "key1"
7+
},
58
"azure_blob_storage_container_name": "testcontainer",
69
"streams": [
710
{

‎airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configs/config_integration_parquet.json

+4-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
{
22
"azure_blob_storage_endpoint": "http://localhost:10000/account1",
33
"azure_blob_storage_account_name": "account1",
4-
"azure_blob_storage_account_key": "key1",
4+
"credentials": {
5+
"auth_type": "storage_account_key",
6+
"azure_blob_storage_account_key": "key1"
7+
},
58
"azure_blob_storage_container_name": "testcontainer",
69
"streams": [
710
{

‎airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/spec.json

+128-9
Original file line numberDiff line numberDiff line change
@@ -358,15 +358,81 @@
358358
"order": 2,
359359
"type": "string"
360360
},
361-
"azure_blob_storage_account_key": {
362-
"title": "Azure Blob Storage account key",
363-
"description": "The Azure blob storage account key.",
364-
"airbyte_secret": true,
365-
"examples": [
366-
"Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd=="
367-
],
361+
"credentials": {
362+
"title": "Authentication",
363+
"description": "Credentials for connecting to the Azure Blob Storage",
364+
"type": "object",
368365
"order": 3,
369-
"type": "string"
366+
"oneOf": [
367+
{
368+
"title": "Authenticate via Oauth2",
369+
"type": "object",
370+
"properties": {
371+
"auth_type": {
372+
"title": "Auth Type",
373+
"default": "oauth2",
374+
"const": "oauth2",
375+
"enum": ["oauth2"],
376+
"type": "string"
377+
},
378+
"tenant_id": {
379+
"title": "Tenant ID",
380+
"description": "Tenant ID of the Microsoft Azure Application user",
381+
"airbyte_secret": true,
382+
"type": "string"
383+
},
384+
"client_id": {
385+
"title": "Client ID",
386+
"description": "Client ID of your Microsoft developer application",
387+
"airbyte_secret": true,
388+
"type": "string"
389+
},
390+
"client_secret": {
391+
"title": "Client Secret",
392+
"description": "Client Secret of your Microsoft developer application",
393+
"airbyte_secret": true,
394+
"type": "string"
395+
},
396+
"refresh_token": {
397+
"title": "Refresh Token",
398+
"description": "Refresh Token of your Microsoft developer application",
399+
"airbyte_secret": true,
400+
"type": "string"
401+
}
402+
},
403+
"required": [
404+
"tenant_id",
405+
"client_id",
406+
"client_secret",
407+
"refresh_token",
408+
"auth_type"
409+
]
410+
},
411+
{
412+
"title": "Authenticate via Storage Account Key",
413+
"type": "object",
414+
"properties": {
415+
"auth_type": {
416+
"title": "Auth Type",
417+
"default": "storage_account_key",
418+
"const": "storage_account_key",
419+
"enum": ["storage_account_key"],
420+
"type": "string"
421+
},
422+
"azure_blob_storage_account_key": {
423+
"title": "Azure Blob Storage account key",
424+
"description": "The Azure blob storage account key.",
425+
"airbyte_secret": true,
426+
"examples": [
427+
"Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd=="
428+
],
429+
"order": 3,
430+
"type": "string"
431+
}
432+
},
433+
"required": ["azure_blob_storage_account_key", "auth_type"]
434+
}
435+
]
370436
},
371437
"azure_blob_storage_container_name": {
372438
"title": "Azure blob storage container (Bucket) Name",
@@ -386,8 +452,61 @@
386452
"required": [
387453
"streams",
388454
"azure_blob_storage_account_name",
389-
"azure_blob_storage_account_key",
455+
"credentials",
390456
"azure_blob_storage_container_name"
391457
]
458+
},
459+
"advanced_auth": {
460+
"auth_flow_type": "oauth2.0",
461+
"predicate_key": ["credentials", "auth_type"],
462+
"predicate_value": "oauth2",
463+
"oauth_config_specification": {
464+
"oauth_user_input_from_connector_config_specification": {
465+
"type": "object",
466+
"additionalProperties": false,
467+
"properties": {
468+
"tenant_id": {
469+
"type": "string",
470+
"path_in_connector_config": ["credentials", "tenant_id"]
471+
}
472+
}
473+
},
474+
"complete_oauth_output_specification": {
475+
"type": "object",
476+
"additionalProperties": false,
477+
"properties": {
478+
"refresh_token": {
479+
"type": "string",
480+
"path_in_connector_config": ["credentials", "refresh_token"]
481+
}
482+
}
483+
},
484+
"complete_oauth_server_input_specification": {
485+
"type": "object",
486+
"additionalProperties": false,
487+
"properties": {
488+
"client_id": {
489+
"type": "string"
490+
},
491+
"client_secret": {
492+
"type": "string"
493+
}
494+
}
495+
},
496+
"complete_oauth_server_output_specification": {
497+
"type": "object",
498+
"additionalProperties": false,
499+
"properties": {
500+
"client_id": {
501+
"type": "string",
502+
"path_in_connector_config": ["credentials", "client_id"]
503+
},
504+
"client_secret": {
505+
"type": "string",
506+
"path_in_connector_config": ["credentials", "client_secret"]
507+
}
508+
}
509+
}
510+
}
392511
}
393512
}

‎airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ data:
77
connectorSubtype: file
88
connectorType: source
99
definitionId: fdaaba68-4875-4ed9-8fcd-4ae1e0a25093
10-
dockerImageTag: 0.3.6
10+
dockerImageTag: 0.4.0
1111
dockerRepository: airbyte/source-azure-blob-storage
1212
documentationUrl: https://docs.airbyte.com/integrations/sources/azure-blob-storage
1313
githubIssueLabel: source-azure-blob-storage

‎airbyte-integrations/connectors/source-azure-blob-storage/pyproject.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
33
build-backend = "poetry.core.masonry.api"
44

55
[tool.poetry]
6-
version = "0.3.6"
6+
version = "0.4.0"
77
name = "source-azure-blob-storage"
88
description = "Source implementation for Azure Blob Storage."
99
authors = [ "Airbyte <[email protected]>",]

‎airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/config.py

+47-7
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,51 @@
22
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
33
#
44

5-
from typing import Any, Dict, Optional
5+
from typing import Any, Dict, Literal, Optional, Union
66

77
import dpath.util
88
from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec
9-
from pydantic import AnyUrl, Field
9+
from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
10+
from pydantic import AnyUrl, BaseModel, Field
11+
12+
13+
class Oauth2(BaseModel):
14+
class Config(OneOfOptionConfig):
15+
title = "Authenticate via Oauth2"
16+
discriminator = "auth_type"
17+
18+
auth_type: Literal["oauth2"] = Field("oauth2", const=True)
19+
tenant_id: str = Field(title="Tenant ID", description="Tenant ID of the Microsoft Azure Application user", airbyte_secret=True)
20+
client_id: str = Field(
21+
title="Client ID",
22+
description="Client ID of your Microsoft developer application",
23+
airbyte_secret=True,
24+
)
25+
client_secret: str = Field(
26+
title="Client Secret",
27+
description="Client Secret of your Microsoft developer application",
28+
airbyte_secret=True,
29+
)
30+
refresh_token: str = Field(
31+
title="Refresh Token",
32+
description="Refresh Token of your Microsoft developer application",
33+
airbyte_secret=True,
34+
)
35+
36+
37+
class StorageAccountKey(BaseModel):
38+
class Config(OneOfOptionConfig):
39+
title = "Authenticate via Storage Account Key"
40+
discriminator = "auth_type"
41+
42+
auth_type: Literal["storage_account_key"] = Field("storage_account_key", const=True)
43+
azure_blob_storage_account_key: str = Field(
44+
title="Azure Blob Storage account key",
45+
description="The Azure blob storage account key.",
46+
airbyte_secret=True,
47+
examples=["Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd=="],
48+
order=3,
49+
)
1050

1151

1252
class Config(AbstractFileBasedSpec):
@@ -25,11 +65,11 @@ def documentation_url(cls) -> AnyUrl:
2565
examples=["airbyte5storage"],
2666
order=2,
2767
)
28-
azure_blob_storage_account_key: str = Field(
29-
title="Azure Blob Storage account key",
30-
description="The Azure blob storage account key.",
31-
airbyte_secret=True,
32-
examples=["Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd=="],
68+
credentials: Union[Oauth2, StorageAccountKey] = Field(
69+
title="Authentication",
70+
description="Credentials for connecting to the Azure Blob Storage",
71+
discriminator="auth_type",
72+
type="object",
3373
order=3,
3474
)
3575
azure_blob_storage_container_name: str = Field(

0 commit comments

Comments
 (0)