From 24d806ff612e6e89442d66a1c61a83825b6bd038 Mon Sep 17 00:00:00 2001 From: Xingyuan Chen Date: Sat, 5 Nov 2022 17:25:05 +0800 Subject: [PATCH 1/3] add virtual-hosted-style option for S3 source --- .../connectors/source-s3/source_s3/s3_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-s3/source_s3/s3_utils.py b/airbyte-integrations/connectors/source-s3/source_s3/s3_utils.py index 607495951fa9a..522604b8fca76 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/s3_utils.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/s3_utils.py @@ -48,7 +48,8 @@ def _get_s3_client_args(provider: dict, config: Config) -> dict: client_kv_args["endpoint_url"] = endpoint client_kv_args["use_ssl"] = provider.get("use_ssl", True) client_kv_args["verify"] = provider.get("verify_ssl_cert", True) - + client_kv_args["config"] = Config(s3={"addressing_style": provider.get("addressing_style", "auto")}) + return client_kv_args From 9400983f3d4a26cb9883d1b30ea7a6dcc7e63c75 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 8 Nov 2022 21:59:52 +1100 Subject: [PATCH 2/3] update s3 version --- airbyte-integrations/connectors/source-s3/Dockerfile | 2 +- docs/integrations/sources/s3.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-s3/Dockerfile b/airbyte-integrations/connectors/source-s3/Dockerfile index 2413c0c799b71..c38be4e387ef8 100644 --- a/airbyte-integrations/connectors/source-s3/Dockerfile +++ b/airbyte-integrations/connectors/source-s3/Dockerfile @@ -17,5 +17,5 @@ COPY source_s3 ./source_s3 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.25 +LABEL io.airbyte.version=0.1.26 LABEL io.airbyte.name=airbyte/source-s3 diff --git a/docs/integrations/sources/s3.md b/docs/integrations/sources/s3.md index a4de3ac969df9..3da49e0b32bf6 100644 --- a/docs/integrations/sources/s3.md +++ b/docs/integrations/sources/s3.md @@ -205,6 +205,7 @@ The Jsonl parser uses pyarrow hence,only the line-delimited JSON format is suppo | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------------------| +| 0.1.26 | 2022-11-08 | [19006](https://github.com/airbytehq/airbyte/pull/19006) | Add virtual-hosted-style option | | 0.1.24 | 2022-10-28 | [18602](https://github.com/airbytehq/airbyte/pull/18602) | Wrap errors into AirbyteTracedException pointing to a problem file | | 0.1.23 | 2022-10-10 | [17991](https://github.com/airbytehq/airbyte/pull/17991) | Fix pyarrow to JSON schema type conversion for arrays | | 0.1.23 | 2022-10-10 | [17800](https://github.com/airbytehq/airbyte/pull/17800) | Deleted `use_ssl` and `verify_ssl_cert` flags and hardcoded to `True` | From 461a2310e2b2f5db5c214a994fdfc0cd7a599fa5 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Tue, 8 Nov 2022 11:31:29 +0000 Subject: [PATCH 3/3] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-s3/integration_tests/config_minio.json | 2 +- airbyte-integrations/connectors/source-s3/source_s3/s3_utils.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index a06be8346b897..6d6530814b387 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1120,7 +1120,7 @@ - name: S3 sourceDefinitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2 dockerRepository: airbyte/source-s3 - dockerImageTag: 0.1.25 + dockerImageTag: 0.1.26 documentationUrl: https://docs.airbyte.com/integrations/sources/s3 icon: s3.svg sourceType: file diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 8aae238c3e4d6..a2d63ac591304 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -10739,7 +10739,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-s3:0.1.25" +- dockerImage: "airbyte/source-s3:0.1.26" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/s3" changelogUrl: "https://docs.airbyte.com/integrations/sources/s3" diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/config_minio.json b/airbyte-integrations/connectors/source-s3/integration_tests/config_minio.json index 6bf9fc09c36db..1a0863be9c8b6 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/config_minio.json +++ b/airbyte-integrations/connectors/source-s3/integration_tests/config_minio.json @@ -6,7 +6,7 @@ "aws_access_key_id": "123456", "aws_secret_access_key": "123456key", "path_prefix": "", - "endpoint": "http://10.0.56.135:9000" + "endpoint": "http://10.0.229.255:9000" }, "format": { "filetype": "csv" diff --git a/airbyte-integrations/connectors/source-s3/source_s3/s3_utils.py b/airbyte-integrations/connectors/source-s3/source_s3/s3_utils.py index 522604b8fca76..eff5f76e63bb4 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/s3_utils.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/s3_utils.py @@ -49,7 +49,7 @@ def _get_s3_client_args(provider: dict, config: Config) -> dict: client_kv_args["use_ssl"] = provider.get("use_ssl", True) client_kv_args["verify"] = provider.get("verify_ssl_cert", True) client_kv_args["config"] = Config(s3={"addressing_style": provider.get("addressing_style", "auto")}) - + return client_kv_args