add support for automatic region extraction for S3 Express (#2104)

also add ListDirectoryBuckets API to test this PR

```
mc alias set s3express https://s3express-control.us-west-2.amazonaws.com xxx xxxx --api s3v4
mc ls s3express/
[2025-05-09 15:55:55 PDT]     0B harsha-test--usw2-az1--x-s3/
```

Author: harshavardhana

api-datatypes.go

@@ -32,6 +32,8 @@ type BucketInfo struct {
 	Name string `json:"name"`
 	// Date the bucket was created.
 	CreationDate time.Time `json:"creationDate"`
+	// BucketRegion region where the bucket is present
+	BucketRegion string `json:"bucketRegion"`
 }
 
 // StringMap represents map with custom UnmarshalXML

api-list.go

@@ -58,6 +58,64 @@ func (c *Client) ListBuckets(ctx context.Context) ([]BucketInfo, error) {
 	return listAllMyBucketsResult.Buckets.Bucket, nil
 }
 
+// ListDirectoryBuckets list all buckets owned by this authenticated user.
+//
+// This call requires explicit authentication, no anonymous requests are
+// allowed for listing buckets.
+//
+// api := client.New(....)
+// dirBuckets, err := api.ListDirectoryBuckets(context.Background())
+func (c *Client) ListDirectoryBuckets(ctx context.Context) (iter.Seq2[BucketInfo, error], error) {
+	fetchBuckets := func(continuationToken string) ([]BucketInfo, string, error) {
+		metadata := requestMetadata{contentSHA256Hex: emptySHA256Hex}
+		metadata.queryValues = url.Values{}
+		metadata.queryValues.Set("max-directory-buckets", "1000")
+		if continuationToken != "" {
+			metadata.queryValues.Set("continuation-token", continuationToken)
+		}
+
+		// Execute GET on service.
+		resp, err := c.executeMethod(ctx, http.MethodGet, metadata)
+		defer closeResponse(resp)
+		if err != nil {
+			return nil, "", err
+		}
+		if resp != nil {
+			if resp.StatusCode != http.StatusOK {
+				return nil, "", httpRespToErrorResponse(resp, "", "")
+			}
+		}
+
+		results := listAllMyDirectoryBucketsResult{}
+		if err = xmlDecoder(resp.Body, &results); err != nil {
+			return nil, "", err
+		}
+
+		return results.Buckets.Bucket, results.ContinuationToken, nil
+	}
+
+	return func(yield func(BucketInfo, error) bool) {
+		var continuationToken string
+		for {
+			buckets, token, err := fetchBuckets(continuationToken)
+			if err != nil {
+				yield(BucketInfo{}, err)
+				return
+			}
+			for _, bucket := range buckets {
+				if !yield(bucket, nil) {
+					return
+				}
+			}
+			if token == "" {
+				// nothing to continue
+				return
+			}
+			continuationToken = token
+		}
+	}, nil
+}
+
 // Bucket List Operations.
 func (c *Client) listObjectsV2(ctx context.Context, bucketName string, opts ListObjectsOptions) iter.Seq[ObjectInfo] {
 	// Default listing is delimited at "/"

api-s3-datatypes.go

@@ -35,6 +35,14 @@ type listAllMyBucketsResult struct {
 	Owner owner
 }
 
+// listAllMyDirectoryBucketsResult container for listDirectoryBuckets response.
+type listAllMyDirectoryBucketsResult struct {
+	Buckets struct {
+		Bucket []BucketInfo
+	}
+	ContinuationToken string
+}
+
 // owner container for bucket owner information.
 type owner struct {
 	DisplayName string

api.go

@@ -828,7 +828,7 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
 	// make sure to de-dup calls to credential services, this reduces
 	// the overall load to the endpoint generating credential service.
 	value, err, _ := c.credsGroup.Do(metadata.bucketName, func() (credentials.Value, error) {
-		if s3utils.IsS3ExpressBucket(metadata.bucketName) {
+		if s3utils.IsS3ExpressBucket(metadata.bucketName) && s3utils.IsAmazonEndpoint(*c.endpointURL) {
 			return c.CreateSession(ctx, metadata.bucketName, SessionReadWrite)
 		}
 		// Get credentials from the configured credentials provider.
@@ -851,7 +851,7 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
 		sessionToken    = value.SessionToken
 	)
 
-	if s3utils.IsS3ExpressBucket(metadata.bucketName) {
+	if s3utils.IsS3ExpressBucket(metadata.bucketName) && sessionToken != "" {
 		req.Header.Set("x-amz-s3session-token", sessionToken)
 	}
 
@@ -940,8 +940,13 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
 		// Streaming signature is used by default for a PUT object request.
 		// Additionally, we also look if the initialized client is secure,
 		// if yes then we don't need to perform streaming signature.
-		req = signer.StreamingSignV4(req, accessKeyID,
-			secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher())
+		if s3utils.IsAmazonExpressRegionalEndpoint(*c.endpointURL) {
+			req = signer.StreamingSignV4Express(req, accessKeyID,
+				secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher())
+		} else {
+			req = signer.StreamingSignV4(req, accessKeyID,
+				secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher())
+		}
 	default:
 		// Set sha256 sum for signature calculation only with signature version '4'.
 		shaHeader := unsignedPayload
@@ -956,8 +961,12 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
 		}
 		req.Header.Set("X-Amz-Content-Sha256", shaHeader)
 
-		// Add signature version '4' authorization header.
-		req = signer.SignV4Trailer(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.trailer)
+		if s3utils.IsAmazonExpressRegionalEndpoint(*c.endpointURL) {
+			req = signer.SignV4TrailerExpress(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.trailer)
+		} else {
+			// Add signature version '4' authorization header.
+			req = signer.SignV4Trailer(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.trailer)
+		}
 	}
 
 	// Return request.
@@ -989,9 +998,18 @@ func (c *Client) makeTargetURL(bucketName, objectName, bucketLocation string, is
 			host = c.s3AccelerateEndpoint
 		} else {
 			// Do not change the host if the endpoint URL is a FIPS S3 endpoint or a S3 PrivateLink interface endpoint
-			if !s3utils.IsAmazonFIPSEndpoint(*c.endpointURL) && !s3utils.IsAmazonPrivateLinkEndpoint(*c.endpointURL) && !s3utils.IsS3ExpressBucket(bucketName) {
-				// Fetch new host based on the bucket location.
-				host = getS3Endpoint(bucketLocation, c.s3DualstackEnabled)
+			if !s3utils.IsAmazonFIPSEndpoint(*c.endpointURL) && !s3utils.IsAmazonPrivateLinkEndpoint(*c.endpointURL) {
+				if s3utils.IsAmazonExpressRegionalEndpoint(*c.endpointURL) {
+					if bucketName == "" {
+						host = getS3ExpressEndpoint(bucketLocation, false)
+					} else {
+						// Fetch new host based on the bucket location.
+						host = getS3ExpressEndpoint(bucketLocation, s3utils.IsS3ExpressBucket(bucketName))
+					}
+				} else {
+					// Fetch new host based on the bucket location.
+					host = getS3Endpoint(bucketLocation, c.s3DualstackEnabled)
+				}
 			}
 		}
 	}

create-session.go

@@ -78,6 +78,10 @@ func (c *Client) CreateSession(ctx context.Context, bucketName string, sessionMo
 		return credentials.Value{}, err
 	}
 
+	if resp.StatusCode != http.StatusOK {
+		return credentials.Value{}, httpRespToErrorResponse(resp, bucketName, "")
+	}
+
 	credSession := &createSessionResult{}
 	dec := xml.NewDecoder(resp.Body)
 	if err = dec.Decode(credSession); err != nil {
@@ -103,12 +107,15 @@ func (c *Client) createSessionRequest(ctx context.Context, bucketName string, se
 	// Set get bucket location always as path style.
 	targetURL := *c.endpointURL
 
+	// Fetch new host based on the bucket location.
+	host := getS3ExpressEndpoint(c.region, s3utils.IsS3ExpressBucket(bucketName))
+
 	// as it works in makeTargetURL method from api.go file
-	if h, p, err := net.SplitHostPort(targetURL.Host); err == nil {
+	if h, p, err := net.SplitHostPort(host); err == nil {
 		if targetURL.Scheme == "http" && p == "80" || targetURL.Scheme == "https" && p == "443" {
-			targetURL.Host = h
+			host = h
 			if ip := net.ParseIP(h); ip != nil && ip.To16() != nil {
-				targetURL.Host = "[" + h + "]"
+				host = "[" + h + "]"
 			}
 		}
 	}
@@ -118,7 +125,7 @@ func (c *Client) createSessionRequest(ctx context.Context, bucketName string, se
 	var urlStr string
 
 	if isVirtualStyle {
-		urlStr = c.endpointURL.Scheme + "://" + bucketName + "." + targetURL.Host + "/?session"
+		urlStr = c.endpointURL.Scheme + "://" + bucketName + "." + host + "/?session"
 	} else {
 		targetURL.Path = path.Join(bucketName, "") + "/"
 		targetURL.RawQuery = urlValues.Encode()
@@ -170,6 +177,6 @@ func (c *Client) createSessionRequest(ctx context.Context, bucketName string, se
 
 	req.Header.Set("X-Amz-Content-Sha256", contentSha256)
 	req.Header.Set("x-amz-create-session-mode", string(sessionMode))
-	req = signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, c.region)
+	req = signer.SignV4Express(*req, accessKeyID, secretAccessKey, sessionToken, c.region)
 	return req, nil
 }

s3-endpoints.go renamed to endpoints.go

@@ -22,6 +22,66 @@ type awsS3Endpoint struct {
 	dualstackEndpoint string
 }
 
+type awsS3ExpressEndpoint struct {
+	regionalEndpoint string
+	zonalEndpoints   []string
+}
+
+var awsS3ExpressEndpointMap = map[string]awsS3ExpressEndpoint{
+	"us-east-1": {
+		"s3express-control.us-east-1.amazonaws.com",
+		[]string{
+			"s3express-use1-az4.us-east-1.amazonaws.com",
+			"s3express-use1-az5.us-east-1.amazonaws.com",
+			"3express-use1-az6.us-east-1.amazonaws.com",
+		},
+	},
+	"us-east-2": {
+		"s3express-control.us-east-2.amazonaws.com",
+		[]string{
+			"s3express-use2-az1.us-east-2.amazonaws.com",
+			"s3express-use2-az2.us-east-2.amazonaws.com",
+		},
+	},
+	"us-west-2": {
+		"s3express-control.us-west-2.amazonaws.com",
+		[]string{
+			"s3express-usw2-az1.us-west-2.amazonaws.com",
+			"s3express-usw2-az3.us-west-2.amazonaws.com",
+			"s3express-usw2-az4.us-west-2.amazonaws.com",
+		},
+	},
+	"ap-south-1": {
+		"s3express-control.ap-south-1.amazonaws.com",
+		[]string{
+			"s3express-aps1-az1.ap-south-1.amazonaws.com",
+			"s3express-aps1-az3.ap-south-1.amazonaws.com",
+		},
+	},
+	"ap-northeast-1": {
+		"s3express-control.ap-northeast-1.amazonaws.com",
+		[]string{
+			"s3express-apne1-az1.ap-northeast-1.amazonaws.com",
+			"s3express-apne1-az4.ap-northeast-1.amazonaws.com",
+		},
+	},
+	"eu-west-1": {
+		"s3express-control.eu-west-1.amazonaws.com",
+		[]string{
+			"s3express-euw1-az1.eu-west-1.amazonaws.com",
+			"s3express-euw1-az3.eu-west-1.amazonaws.com",
+		},
+	},
+	"eu-north-1": {
+		"s3express-control.eu-north-1.amazonaws.com",
+		[]string{
+			"s3express-eun1-az1.eu-north-1.amazonaws.com",
+			"s3express-eun1-az2.eu-north-1.amazonaws.com",
+			"s3express-eun1-az3.eu-north-1.amazonaws.com",
+		},
+	},
+}
+
 // awsS3EndpointMap Amazon S3 endpoint map.
 var awsS3EndpointMap = map[string]awsS3Endpoint{
 	"us-east-1": {
@@ -182,6 +242,19 @@ var awsS3EndpointMap = map[string]awsS3Endpoint{
 	},
 }
 
+// getS3ExpressEndpoint get Amazon S3 Express endpoing based on the region
+// optionally if zonal is set returns first zonal endpoint.
+func getS3ExpressEndpoint(region string, zonal bool) (endpoint string) {
+	s3ExpEndpoint, ok := awsS3ExpressEndpointMap[region]
+	if !ok {
+		return ""
+	}
+	if zonal {
+		return s3ExpEndpoint.zonalEndpoints[0]
+	}
+	return s3ExpEndpoint.regionalEndpoint
+}
+
 // getS3Endpoint get Amazon S3 endpoint based on the bucket location.
 func getS3Endpoint(bucketLocation string, useDualstack bool) (endpoint string) {
 	s3Endpoint, ok := awsS3EndpointMap[bucketLocation]

examples/s3/list-directory-buckets.go

@@ -0,0 +1,55 @@
+//go:build example
+// +build example
+
+/*
+ * MinIO Go Library for Amazon S3 Compatible Cloud Storage
+ * Copyright 2015-2025 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"context"
+	"log"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+func main() {
+	// Note: YOUR-ACCESSKEYID and YOUR-SECRETACCESSKEY are
+	// dummy values, please replace them with original values.
+
+	// Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access.
+	// This boolean value is the last argument for New().
+
+	// New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically
+	// determined based on the Endpoint value.
+	s3Client, err := minio.New("s3express-control.us-east-1.amazonaws.com", &minio.Options{
+		Creds:  credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""),
+		Secure: true,
+	})
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	dirBuckets, err := s3Client.ListDirectoryBuckets(context.Background())
+	if err != nil {
+		log.Fatalln(err)
+	}
+	for bucket, err := range dirBuckets {
+		log.Println(bucket, err)
+	}
+}