Add S3 prefix support (#498)

* Start working on S3 prefix
* Make amends with linter.
* Simplify buildKey and add more info on prefix setting in example template
* Make buildKey an S3 method.

Author: dmitry-mukhin

config.toml.example

@@ -26,6 +26,9 @@ path = "test"
   endpoint_url = "https://s3.us-west-2.amazonaws.com"
   region = "us-west-2"
   bucket = "example-bucket-name"
+  # If you use prefix, you may need to add a path to `server.hostname` setting
+  # e.g. https://example-bucket-name.s3.us-west-2.amazonaws.com/example/prefix/
+  prefix = "example/prefix"
 
 # API keys to be used to access Youtube and Vimeo.
 # These can be either specified as string parameter or array of string (so those will be rotated).

pkg/fs/s3.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"path"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
@@ -24,13 +25,16 @@ type S3Config struct {
 	Region string `toml:"region"`
 	// EndpointURL is an HTTP endpoint of the S3 API
 	EndpointURL string `toml:"endpoint_url"`
+	// Prefix is a prefix (subfolder) to use to build key names
+	Prefix string `toml:"prefix"`
 }
 
 // S3 implements file storage for S3-compatible providers.
 type S3 struct {
 	api      s3iface.S3API
 	uploader *s3manager.Uploader
 	bucket   string
+	prefix   string
 }
 
 func NewS3(c S3Config) (*S3, error) {
@@ -47,6 +51,7 @@ func NewS3(c S3Config) (*S3, error) {
 		api:      s3.New(sess),
 		uploader: s3manager.NewUploader(sess),
 		bucket:   c.Bucket,
+		prefix:   c.Prefix,
 	}, nil
 }
 
@@ -55,21 +60,23 @@ func (s *S3) Open(_name string) (http.File, error) {
 }
 
 func (s *S3) Delete(ctx context.Context, name string) error {
+	key := s.buildKey(name)
 	_, err := s.api.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{
 		Bucket: &s.bucket,
-		Key:    &name,
+		Key:    &key,
 	})
 	return err
 }
 
 func (s *S3) Create(ctx context.Context, name string, reader io.Reader) (int64, error) {
-	logger := log.WithField("name", name)
+	key := s.buildKey(name)
+	logger := log.WithField("key", key)
 
 	logger.Infof("uploading file to %s", s.bucket)
 	r := &readerWithN{Reader: reader}
 	_, err := s.uploader.UploadWithContext(ctx, &s3manager.UploadInput{
 		Bucket: &s.bucket,
-		Key:    &name,
+		Key:    &key,
 		Body:   r,
 	})
 	if err != nil {
@@ -81,12 +88,13 @@ func (s *S3) Create(ctx context.Context, name string, reader io.Reader) (int64,
 }
 
 func (s *S3) Size(ctx context.Context, name string) (int64, error) {
-	logger := log.WithField("name", name)
+	key := s.buildKey(name)
+	logger := log.WithField("key", key)
 
 	logger.Debugf("getting file size from %s", s.bucket)
 	resp, err := s.api.HeadObjectWithContext(ctx, &s3.HeadObjectInput{
 		Bucket: &s.bucket,
-		Key:    &name,
+		Key:    &key,
 	})
 	if err != nil {
 		if awsErr, ok := err.(awserr.Error); ok {
@@ -100,6 +108,10 @@ func (s *S3) Size(ctx context.Context, name string) (int64, error) {
 	return *resp.ContentLength, nil
 }
 
+func (s *S3) buildKey(name string) string {
+	return path.Join(s.prefix, name)
+}
+
 type readerWithN struct {
 	io.Reader
 	n int

pkg/fs/s3_test.go

@@ -18,7 +18,7 @@ import (
 
 func TestS3_Create(t *testing.T) {
 	files := make(map[string][]byte)
-	stor, err := newMockS3(files)
+	stor, err := newMockS3(files, "")
 	assert.NoError(t, err)
 
 	written, err := stor.Create(testCtx, "1/test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3}))
@@ -32,7 +32,7 @@ func TestS3_Create(t *testing.T) {
 
 func TestS3_Size(t *testing.T) {
 	files := make(map[string][]byte)
-	stor, err := newMockS3(files)
+	stor, err := newMockS3(files, "")
 	assert.NoError(t, err)
 
 	_, err = stor.Create(testCtx, "1/test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3}))
@@ -45,7 +45,7 @@ func TestS3_Size(t *testing.T) {
 
 func TestS3_NoSize(t *testing.T) {
 	files := make(map[string][]byte)
-	stor, err := newMockS3(files)
+	stor, err := newMockS3(files, "")
 	assert.NoError(t, err)
 
 	_, err = stor.Size(testCtx, "1/test")
@@ -54,7 +54,7 @@ func TestS3_NoSize(t *testing.T) {
 
 func TestS3_Delete(t *testing.T) {
 	files := make(map[string][]byte)
-	stor, err := newMockS3(files)
+	stor, err := newMockS3(files, "")
 	assert.NoError(t, err)
 
 	_, err = stor.Create(testCtx, "1/test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3}))
@@ -70,17 +70,30 @@ func TestS3_Delete(t *testing.T) {
 	assert.False(t, ok)
 }
 
+func TestS3_BuildKey(t *testing.T) {
+	files := make(map[string][]byte)
+
+	stor, _ := newMockS3(files, "")
+	key := stor.buildKey("test-fn")
+	assert.EqualValues(t, "test-fn", key)
+
+	stor, _ = newMockS3(files, "mock-prefix")
+	key = stor.buildKey("test-fn")
+	assert.EqualValues(t, "mock-prefix/test-fn", key)
+}
+
 type mockS3API struct {
 	s3iface.S3API
 	files map[string][]byte
 }
 
-func newMockS3(files map[string][]byte) (*S3, error) {
+func newMockS3(files map[string][]byte, prefix string) (*S3, error) {
 	api := &mockS3API{files: files}
 	return &S3{
 		api:      api,
 		uploader: s3manager.NewUploaderWithClient(api),
 		bucket:   "mock-bucket",
+		prefix:   prefix,
 	}, nil
 }