Skip to content

Commit b2cc34f

Browse files
skotambkarskotambkar
authored
adds support for s3 and s3control private link (#3770)
cr https://code.amazon.com/reviews/CR-42972833
1 parent 4a3fa39 commit b2cc34f

File tree

11 files changed

+528
-115
lines changed

11 files changed

+528
-115
lines changed

‎CHANGELOG_PENDING.md

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
### SDK Features
22

33
### SDK Enhancements
4+
* `service/s3`: Amazon S3 now supports AWS PrivateLink, providing direct access to S3 via a private endpoint within your virtual private network.
45

56
### SDK Bugs
67
* `aws/session`: Fixed a bug that prevented credentials from being sourced from the environment if the loaded shared config profile contained partial SSO configuration. ([#3769](https://github.com/aws/aws-sdk-go/pull/3769))

‎example/service/s3/usingPrivateLink/README.md

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# Example
2+
3+
This example demonstrates how you can use the AWS SDK for Go's Amazon S3 client
4+
to use AWS PrivateLink for Amazon S3.
5+
6+
# Usage
7+
8+
To access S3 bucket data using the s3 interface endpoints, prefix the vpc
9+
endpoint with `bucket`. For eg, use endpoint url as `https://bucket.vpce-0xxxxxxx-xxx8xxg.s3.us-west-2.vpce.amazonaws.com`
10+
to access S3 bucket data via the associated vpc endpoint. The SDK may mutate
11+
this endpoint as per the input provided to work with ARNs.
12+
13+
To access S3 access point data using the s3 interface endpoints, prefix the vpc
14+
endpoint with `accesspoint`. For eg, use endpoint url as `https://accesspoint.vpce-0xxxxxxx-xxxx8xxg.s3.us-west-2.vpce.amazonaws.com`
15+
to access S3 access point data via the associated vpc endpoint. The SDK may
16+
mutate this endpoint as per the input provided to work with ARNs.
17+
18+
To work with S3 control using the s3 interface endpoints, prefix the vpc endpoint
19+
with `control`. For eg, use endpoint url as `https://control.vpce-0xxxxxxx-xxx8xxg.s3.us-west-2.vpce.amazonaws.com`
20+
to use S3 Control operations with the associated vpc endpoint. The SDK may mutate
21+
this endpoint as per the input provided to work with ARNs.
22+
23+
The example will create s3 client's that use appropriate vpc endpoint url. The example
24+
will then create a bucket of the name provided in code. Replace the value of
25+
the `accountID` const with the account ID for your AWS account. The
26+
`vpcBucketEndpointUrl`, `vpcAccesspointEndpoint`, `vpcControlEndpoint`, `bucket`,
27+
`keyName`, and `accessPoint` const variables need to be updated to match the name
28+
of the appropriate vpc endpoint, Bucket, Object Key, and Access Point that will be
29+
created by the example.
30+
31+
```sh
32+
AWS_REGION=<region> go run -tags example usingPrivateLink.go
33+
```

‎example/service/s3/usingPrivateLink/usingPrivateLink.go

+105
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
// +build example
2+
3+
package main
4+
5+
import (
6+
"fmt"
7+
"io/ioutil"
8+
9+
"github.com/aws/aws-sdk-go/aws"
10+
"github.com/aws/aws-sdk-go/aws/arn"
11+
"github.com/aws/aws-sdk-go/aws/session"
12+
"github.com/aws/aws-sdk-go/service/s3"
13+
"github.com/aws/aws-sdk-go/service/s3control"
14+
)
15+
16+
const (
17+
bucketName = "myBucketName"
18+
keyName = "myKeyName"
19+
accountID = "123456789012"
20+
accessPoint = "accesspointname"
21+
22+
// vpcBucketEndpoint will be used by the SDK to resolve an endpoint, when making a call to
23+
// access `bucket` data using s3 interface endpoint. This endpoint may be mutated by the SDK,
24+
// as per the input provided to work with ARNs.
25+
vpcBucketEndpoint = "https://bucket.vpce-0xxxxxxx-xxx8xxg.s3.us-west-2.vpce.amazonaws.com"
26+
27+
// vpcAccesspointEndpoint will be used by the SDK to resolve an endpoint, when making a call to
28+
// access `access-point` data using s3 interface endpoint. This endpoint may be mutated by the SDK,
29+
// as per the input provided to work with ARNs.
30+
vpcAccesspointEndpoint = "https://accesspoint.vpce-0xxxxxxx-xxx8xxg.s3.us-west-2.vpce.amazonaws.com"
31+
32+
// vpcControlEndpoint will be used by the SDK to resolve an endpoint, when making a call to
33+
// access `control` data using s3 interface endpoint. This endpoint may be mutated by the SDK,
34+
// as per the input provided to work with ARNs.
35+
vpcControlEndpoint = "https://control.vpce-0xxxxxxx-xxx8xxg.s3.us-west-2.vpce.amazonaws.com"
36+
)
37+
38+
func main() {
39+
sess := session.Must(session.NewSession())
40+
41+
s3BucketSvc := s3.New(sess, &aws.Config{
42+
Endpoint: aws.String(vpcBucketEndpoint),
43+
})
44+
45+
s3AccesspointSvc := s3.New(sess, &aws.Config{
46+
Endpoint: aws.String(vpcAccesspointEndpoint),
47+
})
48+
49+
s3ControlSvc := s3control.New(sess, &aws.Config{
50+
Endpoint: aws.String(vpcControlEndpoint),
51+
})
52+
53+
// Create an S3 Bucket
54+
fmt.Println("create s3 bucket")
55+
_, err := s3BucketSvc.CreateBucket(&s3.CreateBucketInput{
56+
Bucket: aws.String(bucketName),
57+
})
58+
if err != nil {
59+
panic(fmt.Errorf("failed to create bucket: %v", err))
60+
}
61+
62+
// Wait for S3 Bucket to Exist
63+
fmt.Println("wait for s3 bucket to exist")
64+
err = s3BucketSvc.WaitUntilBucketExists(&s3.HeadBucketInput{
65+
Bucket: aws.String(bucketName),
66+
})
67+
if err != nil {
68+
panic(fmt.Sprintf("bucket failed to materialize: %v", err))
69+
}
70+
71+
// Create an Access Point referring to the bucket
72+
fmt.Println("create an access point")
73+
_, err = s3ControlSvc.CreateAccessPoint(&s3control.CreateAccessPointInput{
74+
AccountId: aws.String(accountID),
75+
Bucket: aws.String(bucketName),
76+
Name: aws.String(accessPoint),
77+
})
78+
if err != nil {
79+
panic(fmt.Sprintf("failed to create access point: %v", err))
80+
}
81+
82+
// Use the SDK's ARN builder to create an ARN for the Access Point.
83+
apARN := arn.ARN{
84+
Partition: "aws",
85+
Service: "s3",
86+
Region: aws.StringValue(sess.Config.Region),
87+
AccountID: accountID,
88+
Resource: "accesspoint/" + accessPoint,
89+
}
90+
91+
// And Use Access Point ARN where bucket parameters are accepted
92+
fmt.Println("get object using access point")
93+
getObjectOutput, err := s3AccesspointSvc.GetObject(&s3.GetObjectInput{
94+
Bucket: aws.String(apARN.String()),
95+
Key: aws.String("somekey"),
96+
})
97+
if err != nil {
98+
panic(fmt.Sprintf("failed get object request: %v", err))
99+
}
100+
101+
_, err = ioutil.ReadAll(getObjectOutput.Body)
102+
if err != nil {
103+
panic(fmt.Sprintf("failed to read object body: %v", err))
104+
}
105+
}

‎private/model/api/customization_passes.go

+4
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,10 @@ func supressSmokeTest(a *API) error {
9494

9595
// Customizes the API generation to replace values specific to S3.
9696
func s3Customizations(a *API) error {
97+
98+
// back-fill signing name as 's3'
99+
a.Metadata.SigningName = "s3"
100+
97101
var strExpires *Shape
98102

99103
var keepContentMD5Ref = map[string]struct{}{

‎service/s3/endpoint.go

+3-10
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ func endpointHandler(req *request.Request) {
9898
Request: req,
9999
}
100100

101-
if resReq.IsCrossPartition() {
101+
if len(resReq.Request.ClientInfo.PartitionID) != 0 && resReq.IsCrossPartition() {
102102
req.Error = s3shared.NewClientPartitionMismatchError(resource,
103103
req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
104104
return
@@ -110,11 +110,6 @@ func endpointHandler(req *request.Request) {
110110
return
111111
}
112112

113-
if resReq.HasCustomEndpoint() {
114-
req.Error = s3shared.NewInvalidARNWithCustomEndpointError(resource, nil)
115-
return
116-
}
117-
118113
switch tv := resource.(type) {
119114
case arn.AccessPointARN:
120115
err = updateRequestAccessPointEndpoint(req, tv)
@@ -155,8 +150,7 @@ func updateRequestAccessPointEndpoint(req *request.Request, accessPoint arn.Acce
155150
req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
156151
}
157152

158-
// Ignore the disable host prefix for access points since custom endpoints
159-
// are not supported.
153+
// Ignore the disable host prefix for access points
160154
req.Config.DisableEndpointHostPrefix = aws.Bool(false)
161155

162156
if err := accessPointEndpointBuilder(accessPoint).build(req); err != nil {
@@ -181,8 +175,7 @@ func updateRequestOutpostAccessPointEndpoint(req *request.Request, accessPoint a
181175
req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
182176
}
183177

184-
// Ignore the disable host prefix for access points since custom endpoints
185-
// are not supported.
178+
// Ignore the disable host prefix for access points
186179
req.Config.DisableEndpointHostPrefix = aws.Bool(false)
187180

188181
if err := outpostAccessPointEndpointBuilder(accessPoint).build(req); err != nil {

‎service/s3/endpoint_builder.go

+26-16
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,11 @@ const (
2222
outpostAccessPointPrefixTemplate = accessPointPrefixTemplate + "{" + outpostPrefixLabel + "}."
2323
)
2424

25+
// hasCustomEndpoint returns true if endpoint is a custom endpoint
26+
func hasCustomEndpoint(r *request.Request) bool {
27+
return len(aws.StringValue(r.Config.Endpoint)) > 0
28+
}
29+
2530
// accessPointEndpointBuilder represents the endpoint builder for access point arn
2631
type accessPointEndpointBuilder arn.AccessPointARN
2732

@@ -55,16 +60,19 @@ func (a accessPointEndpointBuilder) build(req *request.Request) error {
5560
req.ClientInfo.PartitionID, cfgRegion, err)
5661
}
5762

58-
if err = updateRequestEndpoint(req, endpoint.URL); err != nil {
59-
return err
60-
}
63+
endpoint.URL = endpoints.AddScheme(endpoint.URL, aws.BoolValue(req.Config.DisableSSL))
6164

62-
const serviceEndpointLabel = "s3-accesspoint"
65+
if !hasCustomEndpoint(req) {
66+
if err = updateRequestEndpoint(req, endpoint.URL); err != nil {
67+
return err
68+
}
69+
const serviceEndpointLabel = "s3-accesspoint"
6370

64-
// dual stack provided by endpoint resolver
65-
cfgHost := req.HTTPRequest.URL.Host
66-
if strings.HasPrefix(cfgHost, "s3") {
67-
req.HTTPRequest.URL.Host = serviceEndpointLabel + cfgHost[2:]
71+
// dual stack provided by endpoint resolver
72+
cfgHost := req.HTTPRequest.URL.Host
73+
if strings.HasPrefix(cfgHost, "s3") {
74+
req.HTTPRequest.URL.Host = serviceEndpointLabel + cfgHost[2:]
75+
}
6876
}
6977

7078
protocol.HostPrefixBuilder{
@@ -116,14 +124,17 @@ func (o outpostAccessPointEndpointBuilder) build(req *request.Request) error {
116124
req.ClientInfo.PartitionID, resolveRegion, err)
117125
}
118126

119-
if err = updateRequestEndpoint(req, endpoint.URL); err != nil {
120-
return err
121-
}
127+
endpoint.URL = endpoints.AddScheme(endpoint.URL, aws.BoolValue(req.Config.DisableSSL))
122128

123-
// add url host as s3-outposts
124-
cfgHost := req.HTTPRequest.URL.Host
125-
if strings.HasPrefix(cfgHost, endpointsID) {
126-
req.HTTPRequest.URL.Host = resolveService + cfgHost[len(endpointsID):]
129+
if !hasCustomEndpoint(req) {
130+
if err = updateRequestEndpoint(req, endpoint.URL); err != nil {
131+
return err
132+
}
133+
// add url host as s3-outposts
134+
cfgHost := req.HTTPRequest.URL.Host
135+
if strings.HasPrefix(cfgHost, endpointsID) {
136+
req.HTTPRequest.URL.Host = resolveService + cfgHost[len(endpointsID):]
137+
}
127138
}
128139

129140
protocol.HostPrefixBuilder{
@@ -159,7 +170,6 @@ func resolveRegionalEndpoint(r *request.Request, region string, endpointsID stri
159170
}
160171

161172
func updateRequestEndpoint(r *request.Request, endpoint string) (err error) {
162-
endpoint = endpoints.AddScheme(endpoint, aws.BoolValue(r.Config.DisableSSL))
163173

164174
r.HTTPRequest.URL, err = url.Parse(endpoint + r.Operation.HTTPPath)
165175
if err != nil {

0 commit comments

Comments
 (0)