From 2e3586c64ab665b7bc2830e9e85ae88ec7bcfcdd Mon Sep 17 00:00:00 2001 From: Alvaro Queiroz Date: Mon, 10 Jan 2022 18:11:27 -0300 Subject: [PATCH 01/12] use instanceprofile to auth if id is not provided --- .../destination/s3/S3DestinationConfig.java | 12 ++++++------ .../destination-s3/src/main/resources/spec.json | 4 +--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java index 3aea3ceceed3a..b1584b04d1577 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.destination.s3; +import com.amazonaws.auth.InstanceProfileCredentialsProvider; import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSStaticCredentialsProvider; @@ -87,8 +88,8 @@ public static S3DestinationConfig getS3DestinationConfig(final JsonNode config) config.get("s3_bucket_name").asText(), bucketPath, config.get("s3_bucket_region").asText(), - config.get("access_key_id").asText(), - config.get("secret_access_key").asText(), + config.get("access_key_id") == null ? "" : config.get("access_key_id").asText(), + config.get("secret_access_key") == null ? "" : config.get("secret_access_key").asText(), partSize, format); } @@ -128,11 +129,10 @@ public S3FormatConfig getFormatConfig() { public AmazonS3 getS3Client() { final AWSCredentials awsCreds = new BasicAWSCredentials(accessKeyId, secretAccessKey); - if (endpoint == null || endpoint.isEmpty()) { + if (accessKeyId == null && secretAccessKey == null || accessKeyId.isEmpty() && secretAccessKey.isEmpty()) { return AmazonS3ClientBuilder.standard() - .withCredentials(new AWSStaticCredentialsProvider(awsCreds)) - .withRegion(bucketRegion) - .build(); + .withCredentials(new InstanceProfileCredentialsProvider(false)) + .build(); } final ClientConfiguration clientConfiguration = new ClientConfiguration(); diff --git a/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json index ceb0d8998cf2e..14b2ede1793a6 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json @@ -12,8 +12,6 @@ "s3_bucket_name", "s3_bucket_path", "s3_bucket_region", - "access_key_id", - "secret_access_key", "format" ], "additionalProperties": false, @@ -72,7 +70,7 @@ }, "access_key_id": { "type": "string", - "description": "The access key id to access the S3 bucket. Airbyte requires Read and Write permissions to the given bucket.", + "description": "The access key id to access the S3 bucket. Airbyte requires Read and Write permissions to the given bucket, if not set, Airbyte will rely on Instance Profile", "title": "S3 Key Id", "airbyte_secret": true, "examples": ["A012345678910EXAMPLE"] From 0278279ff6107d6eacbc25e64dc32471db24e39c Mon Sep 17 00:00:00 2001 From: Alvaro Queiroz Date: Mon, 10 Jan 2022 18:22:00 -0300 Subject: [PATCH 02/12] restore support for using endpoint --- .../integrations/destination/s3/S3DestinationConfig.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java index b1584b04d1577..e379bf5bb5207 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java @@ -133,6 +133,13 @@ public AmazonS3 getS3Client() { return AmazonS3ClientBuilder.standard() .withCredentials(new InstanceProfileCredentialsProvider(false)) .build(); + } + + else if (endpoint == null || endpoint.isEmpty()) { + return AmazonS3ClientBuilder.standard() + .withCredentials(new AWSStaticCredentialsProvider(awsCreds)) + .withRegion(bucketRegion) + .build(); } final ClientConfiguration clientConfiguration = new ClientConfiguration(); From 61413d8e1f00b1ff48cfcef4d5281b5fc0ae289c Mon Sep 17 00:00:00 2001 From: Alvaro Queiroz Date: Mon, 10 Jan 2022 18:51:43 -0300 Subject: [PATCH 03/12] update readme --- airbyte-integrations/connectors/destination-s3/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/destination-s3/README.md b/airbyte-integrations/connectors/destination-s3/README.md index e163606e68a71..a5b7f9bb7c4cc 100644 --- a/airbyte-integrations/connectors/destination-s3/README.md +++ b/airbyte-integrations/connectors/destination-s3/README.md @@ -8,6 +8,7 @@ As a community contributor, you will need access to AWS to run the integration t - Create an S3 bucket for testing. - Get your `access_key_id` and `secret_access_key` that can read and write to the above bucket. +- if you leave `access_key_id` and `secret_access_key` in blank, the authentication will rely on the instance profile authentication - Paste the bucket and key information into the config files under [`./sample_secrets`](./sample_secrets). - Rename the directory from `sample_secrets` to `secrets`. - Feel free to modify the config files with different settings in the acceptance test file (e.g. `S3CsvDestinationAcceptanceTest.java`, method `getFormatConfig`), as long as they follow the schema defined in [spec.json](src/main/resources/spec.json). From 7678abe67115897ad5e172ee9140d91995120a9c Mon Sep 17 00:00:00 2001 From: Alvaro Queiroz Date: Mon, 10 Jan 2022 18:58:05 -0300 Subject: [PATCH 04/12] update changelog --- docs/integrations/destinations/s3.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/integrations/destinations/s3.md b/docs/integrations/destinations/s3.md index cd887d24c8c5c..7c1e09fe820db 100644 --- a/docs/integrations/destinations/s3.md +++ b/docs/integrations/destinations/s3.md @@ -223,6 +223,7 @@ Under the hood, an Airbyte data stream in Json schema is first converted to an A | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.2.3 | 2022-01-10 | [\#9399](https://github.com/airbytehq/airbyte/pull/9399) | Use instance profile authentication if credentials are not provided | | 0.2.2 | 2021-12-21 | [\#8574](https://github.com/airbytehq/airbyte/pull/8574) | Added namespace to Avro and Parquet record types | | 0.2.1 | 2021-12-20 | [\#8974](https://github.com/airbytehq/airbyte/pull/8974) | Release a new version to ensure there is no excessive logging. | | 0.2.0 | 2021-12-15 | [\#8607](https://github.com/airbytehq/airbyte/pull/8607) | Change the output filename for CSV files - it's now `bucketPath/namespace/streamName/timestamp_epochMillis_randomUuid.csv` | From 03b71d17b72156dbbc377909a48b272b7985f1ce Mon Sep 17 00:00:00 2001 From: Alvaro Queiroz Date: Mon, 10 Jan 2022 20:33:22 -0300 Subject: [PATCH 05/12] update documentation, add setup guide --- docs/integrations/destinations/s3.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/integrations/destinations/s3.md b/docs/integrations/destinations/s3.md index 7c1e09fe820db..b3e6564406c2b 100644 --- a/docs/integrations/destinations/s3.md +++ b/docs/integrations/destinations/s3.md @@ -199,7 +199,7 @@ Under the hood, an Airbyte data stream in Json schema is first converted to an A #### Requirements 1. Allow connections from Airbyte server to your AWS S3/ Minio S3 cluster \(if they exist in separate VPCs\). -2. An S3 bucket with credentials. +2. An S3 bucket with credentials or a instanceprofile with read/write permissions configured for the host(ec2, eks). #### Setup Guide @@ -211,12 +211,15 @@ Under the hood, an Airbyte data stream in Json schema is first converted to an A * **S3 Bucket Region** * **Access Key Id** * See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key. + * See [this](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html) on how to create a instanceprofile. * We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) to objects in the staging bucket. + * If the Access Key and Secret Access Key are not provided, the authentication will rely on the instanceprofile. * **Secret Access Key** * Corresponding key to the above key id. * Make sure your S3 bucket is accessible from the machine running Airbyte. * This depends on your networking setup. * You can check AWS S3 documentation with a tutorial on how to properly configure your S3's access [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-control-overview.html). + * If you will use instance profile authentication, make sure the role has permission to read/write on the bucket. * The easiest way to verify if Airbyte is able to connect to your S3 bucket is via the check connection tool in the UI. ## CHANGELOG From 4167e2a25efc35d3adbfc185126ebc0e7e9c6654 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Queiroz?= Date: Tue, 11 Jan 2022 16:28:23 -0300 Subject: [PATCH 06/12] Update docs/integrations/destinations/s3.md Co-authored-by: Edward Gao --- docs/integrations/destinations/s3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/destinations/s3.md b/docs/integrations/destinations/s3.md index b3e6564406c2b..f8d508537b6a2 100644 --- a/docs/integrations/destinations/s3.md +++ b/docs/integrations/destinations/s3.md @@ -199,7 +199,7 @@ Under the hood, an Airbyte data stream in Json schema is first converted to an A #### Requirements 1. Allow connections from Airbyte server to your AWS S3/ Minio S3 cluster \(if they exist in separate VPCs\). -2. An S3 bucket with credentials or a instanceprofile with read/write permissions configured for the host(ec2, eks). +2. An S3 bucket with credentials or an instanceprofile with read/write permissions configured for the host (ec2, eks). #### Setup Guide From 0f8adb28f8521df8dace7cf2e43fe762435d8d5b Mon Sep 17 00:00:00 2001 From: Alvaro Queiroz Date: Tue, 11 Jan 2022 17:24:31 -0300 Subject: [PATCH 07/12] minor fixes --- .../integrations/destination/s3/S3DestinationConfig.java | 7 ++++++- .../connectors/destination-s3/src/main/resources/spec.json | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java index e379bf5bb5207..8d798fb85cbad 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java @@ -129,7 +129,12 @@ public S3FormatConfig getFormatConfig() { public AmazonS3 getS3Client() { final AWSCredentials awsCreds = new BasicAWSCredentials(accessKeyId, secretAccessKey); - if (accessKeyId == null && secretAccessKey == null || accessKeyId.isEmpty() && secretAccessKey.isEmpty()) { + if (accessKeyId.isEmpty() && !secretAccessKey.isEmpty() + || !accessKeyId.isEmpty() && secretAccessKey.isEmpty()) { + throw new Exception(); + } + + if (accessKeyId.isEmpty() && secretAccessKey.isEmpty()) { return AmazonS3ClientBuilder.standard() .withCredentials(new InstanceProfileCredentialsProvider(false)) .build(); diff --git a/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json index 14b2ede1793a6..1251f6a400d99 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json @@ -70,14 +70,14 @@ }, "access_key_id": { "type": "string", - "description": "The access key id to access the S3 bucket. Airbyte requires Read and Write permissions to the given bucket, if not set, Airbyte will rely on Instance Profile", + "description": "The access key id to access the S3 bucket. Airbyte requires Read and Write permissions to the given bucket, if not set, Airbyte will rely on Instance Profile.", "title": "S3 Key Id", "airbyte_secret": true, "examples": ["A012345678910EXAMPLE"] }, "secret_access_key": { "type": "string", - "description": "The corresponding secret to the access key id.", + "description": "The corresponding secret to the access key id, if S3 Key Id is set, then S3 Access Key must also be provided", "title": "S3 Access Key", "airbyte_secret": true, "examples": ["a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"] From 0faf5d3513b1b718e45d0de988ecc2563674139b Mon Sep 17 00:00:00 2001 From: Alvaro Queiroz Date: Tue, 11 Jan 2022 17:31:48 -0300 Subject: [PATCH 08/12] add error message --- .../integrations/destination/s3/S3DestinationConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java index 8d798fb85cbad..db4add8a7d08f 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java @@ -131,7 +131,7 @@ public AmazonS3 getS3Client() { if (accessKeyId.isEmpty() && !secretAccessKey.isEmpty() || !accessKeyId.isEmpty() && secretAccessKey.isEmpty()) { - throw new Exception(); + throw new Exception("Either both accessKeyId and secretAccessKey are provided, or none"); } if (accessKeyId.isEmpty() && secretAccessKey.isEmpty()) { From 4ec2af3bc3d28c9ee1b69b35c28cfc4844c87bf8 Mon Sep 17 00:00:00 2001 From: Alvaro Queiroz Date: Tue, 11 Jan 2022 17:36:46 -0300 Subject: [PATCH 09/12] now using RuntimeException --- .../integrations/destination/s3/S3DestinationConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java index db4add8a7d08f..4d2d4aee52683 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java @@ -131,7 +131,7 @@ public AmazonS3 getS3Client() { if (accessKeyId.isEmpty() && !secretAccessKey.isEmpty() || !accessKeyId.isEmpty() && secretAccessKey.isEmpty()) { - throw new Exception("Either both accessKeyId and secretAccessKey are provided, or none"); + throw new RuntimeException("Either both accessKeyId and secretAccessKey are provided, or none"); } if (accessKeyId.isEmpty() && secretAccessKey.isEmpty()) { From 9c565397fa4471089dd92cb36efa3aa48eafa37b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Queiroz?= Date: Tue, 11 Jan 2022 19:27:06 -0300 Subject: [PATCH 10/12] Update airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java Co-authored-by: Edward Gao --- .../integrations/destination/s3/S3DestinationConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java index 4d2d4aee52683..79ea47bee1853 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java @@ -131,7 +131,7 @@ public AmazonS3 getS3Client() { if (accessKeyId.isEmpty() && !secretAccessKey.isEmpty() || !accessKeyId.isEmpty() && secretAccessKey.isEmpty()) { - throw new RuntimeException("Either both accessKeyId and secretAccessKey are provided, or none"); + throw new RuntimeException("Either both accessKeyId and secretAccessKey should be provided, or neither"); } if (accessKeyId.isEmpty() && secretAccessKey.isEmpty()) { From 7bc31f9b7183c1094b61918c9d9d2ab7f2e8b9dd Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Wed, 12 Jan 2022 20:14:40 -0300 Subject: [PATCH 11/12] bump connector version --- .../4816b78f-1489-44c1-9060-4b19d5fa9362.json | 2 +- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- airbyte-integrations/connectors/destination-s3/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/4816b78f-1489-44c1-9060-4b19d5fa9362.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/4816b78f-1489-44c1-9060-4b19d5fa9362.json index a7e817b4dba75..07e795e2e3889 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/4816b78f-1489-44c1-9060-4b19d5fa9362.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/4816b78f-1489-44c1-9060-4b19d5fa9362.json @@ -2,7 +2,7 @@ "destinationDefinitionId": "4816b78f-1489-44c1-9060-4b19d5fa9362", "name": "S3", "dockerRepository": "airbyte/destination-s3", - "dockerImageTag": "0.2.2", + "dockerImageTag": "0.2.4", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/s3", "icon": "s3.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 38afff07e973b..2283678f5b86c 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -167,7 +167,7 @@ - name: S3 destinationDefinitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362 dockerRepository: airbyte/destination-s3 - dockerImageTag: 0.2.3 + dockerImageTag: 0.2.4 documentationUrl: https://docs.airbyte.io/integrations/destinations/s3 icon: s3.svg - name: SFTP-JSON diff --git a/airbyte-integrations/connectors/destination-s3/Dockerfile b/airbyte-integrations/connectors/destination-s3/Dockerfile index 74be9eb7dc0e5..7334a212258e6 100644 --- a/airbyte-integrations/connectors/destination-s3/Dockerfile +++ b/airbyte-integrations/connectors/destination-s3/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-s3 COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.3 +LABEL io.airbyte.version=0.2.4 LABEL io.airbyte.name=airbyte/destination-s3 From fdbbaf672c0d1df3f61da39cfa4f0ece122e203c Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Thu, 13 Jan 2022 21:10:56 -0300 Subject: [PATCH 12/12] update seed file --- .../init/src/main/resources/seed/destination_specs.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 52d0b112569d0..9276dac6c07b6 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -3419,8 +3419,6 @@ - "s3_bucket_name" - "s3_bucket_path" - "s3_bucket_region" - - "access_key_id" - - "secret_access_key" - "format" additionalProperties: false properties: @@ -3478,14 +3476,16 @@ access_key_id: type: "string" description: "The access key id to access the S3 bucket. Airbyte requires\ - \ Read and Write permissions to the given bucket." + \ Read and Write permissions to the given bucket, if not set, Airbyte\ + \ will rely on Instance Profile." title: "S3 Key Id" airbyte_secret: true examples: - "A012345678910EXAMPLE" secret_access_key: type: "string" - description: "The corresponding secret to the access key id." + description: "The corresponding secret to the access key id, if S3 Key Id\ + \ is set, then S3 Access Key must also be provided" title: "S3 Access Key" airbyte_secret: true examples: