Merge pull request #317 from Contextualist/feat-s3

Support S3-compatible provider as storage backend

Author: mxpv

README.md

@@ -67,7 +67,10 @@ Minimal configuration would look like this:
 ```toml
 [server]
 port = 8080
-data_dir = "/data/podsync/"
+
+[storage]
+  [storage.local]
+  data_dir = "/data/podsync/"
 
 [tokens]
 youtube = "PASTE YOUR API KEY HERE"

cloud_formation.yml

@@ -169,8 +169,11 @@ Resources:
           tee /home/ec2-user/podsync/config.toml <<EOF
           [server]
           port = ${PodsyncPort}
-          data_dir = "/home/ec2-user/podsync/data"
           hostname = "http://$publichost:${PodsyncPort}"
+          
+          [storage]
+            [storage.local]
+            data_dir = "/home/ec2-user/podsync/data"
 
           [tokens]
           youtube = "${YouTubeApiKey}"

cmd/podsync/config.go

@@ -9,9 +9,11 @@ import (
 	"github.com/hashicorp/go-multierror"
 	"github.com/pelletier/go-toml"
 	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
 
 	"github.com/mxpv/podsync/pkg/db"
 	"github.com/mxpv/podsync/pkg/feed"
+	"github.com/mxpv/podsync/pkg/fs"
 	"github.com/mxpv/podsync/pkg/model"
 	"github.com/mxpv/podsync/pkg/ytdl"
 	"github.com/mxpv/podsync/services/web"
@@ -20,6 +22,8 @@ import (
 type Config struct {
 	// Server is the web server configuration
 	Server web.Config `toml:"server"`
+	// S3 is the optional configuration for S3-compatible storage provider
+	Storage fs.Config `toml:"storage"`
 	// Log is the optional logging configuration
 	Log Log `toml:"log"`
 	// Database configuration
@@ -74,8 +78,17 @@ func LoadConfig(path string) (*Config, error) {
 func (c *Config) validate() error {
 	var result *multierror.Error
 
-	if c.Server.DataDir == "" {
-		result = multierror.Append(result, errors.New("data directory is required"))
+	if c.Server.DataDir != "" {
+		log.Warnf(`server.data_dir is deprecated, and will be removed in a future release. Use the following config instead:
+
+[storage]
+  [storage.local]
+  data_dir = "%s"
+
+`, c.Server.DataDir)
+		if c.Storage.Local.DataDir == "" {
+			c.Storage.Local.DataDir = c.Server.DataDir
+		}
 	}
 
 	if c.Server.Path != "" {
@@ -85,6 +98,19 @@ func (c *Config) validate() error {
 		}
 	}
 
+	switch c.Storage.Type {
+	case "local":
+		if c.Storage.Local.DataDir == "" {
+			result = multierror.Append(result, errors.New("data directory is required for local storage"))
+		}
+	case "s3":
+		if c.Storage.S3.EndpointURL == "" || c.Storage.S3.Region == "" || c.Storage.S3.Bucket == "" {
+			result = multierror.Append(result, errors.New("S3 storage requires endpoint_url, region and bucket to be set"))
+		}
+	default:
+		result = multierror.Append(result, errors.Errorf("unknown storage type: %s", c.Storage.Type))
+	}
+
 	if len(c.Feeds) == 0 {
 		result = multierror.Append(result, errors.New("at least one feed must be specified"))
 	}
@@ -107,6 +133,10 @@ func (c *Config) applyDefaults(configPath string) {
 		}
 	}
 
+	if c.Storage.Type == "" {
+		c.Storage.Type = "local"
+	}
+
 	if c.Log.Filename != "" {
 		if c.Log.MaxSize == 0 {
 			c.Log.MaxSize = model.DefaultLogMaxSize

cmd/podsync/main.go

@@ -26,6 +26,7 @@ import (
 
 type Opts struct {
 	ConfigPath string `long:"config" short:"c" default:"config.toml" env:"PODSYNC_CONFIG_PATH"`
+	Headless   bool   `long:"headless"`
 	Debug      bool   `long:"debug"`
 	NoBanner   bool   `long:"no-banner"`
 }
@@ -59,8 +60,6 @@ func main() {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	group, ctx := errgroup.WithContext(ctx)
-
 	// Parse args
 	opts := Opts{}
 	_, err := flags.Parse(&opts)
@@ -110,8 +109,21 @@ func main() {
 	if err != nil {
 		log.WithError(err).Fatal("failed to open database")
 	}
-
-	storage, err := fs.NewLocal(cfg.Server.DataDir)
+	defer func() {
+		if err := database.Close(); err != nil {
+			log.WithError(err).Error("failed to close database")
+		}
+	}()
+
+	var storage fs.Storage
+	switch cfg.Storage.Type {
+	case "local":
+		storage, err = fs.NewLocal(cfg.Storage.Local.DataDir)
+	case "s3":
+		storage, err = fs.NewS3(cfg.Storage.S3)
+	default:
+		log.Fatalf("unknown storage type: %s", cfg.Storage.Type)
+	}
 	if err != nil {
 		log.WithError(err).Fatal("failed to open storage")
 	}
@@ -133,6 +145,24 @@ func main() {
 		log.WithError(err).Fatal("failed to create updater")
 	}
 
+	// In Headless mode, do one round of feed updates and quit
+	if opts.Headless {
+		for _, feed := range cfg.Feeds {
+			if err := manager.Update(ctx, feed); err != nil {
+				log.WithError(err).Errorf("failed to update feed: %s", feed.URL)
+			}
+		}
+		return
+	}
+
+	group, ctx := errgroup.WithContext(ctx)
+	defer func() {
+		if err := group.Wait(); err != nil && (err != context.Canceled && err != http.ErrServerClosed) {
+			log.WithError(err).Error("wait error")
+		}
+		log.Info("gracefully stopped")
+	}()
+
 	// Queue of feeds to update
 	updates := make(chan *feed.Config, 16)
 	defer close(updates)
@@ -191,6 +221,10 @@ func main() {
 		}
 	})
 
+	if cfg.Storage.Type == "s3" {
+		return // S3 content is hosted externally
+	}
+
 	// Run web server
 	srv := web.New(cfg.Server, storage)
 
@@ -222,14 +256,4 @@ func main() {
 			}
 		}
 	})
-
-	if err := group.Wait(); err != nil && (err != context.Canceled && err != http.ErrServerClosed) {
-		log.WithError(err).Error("wait error")
-	}
-
-	if err := database.Close(); err != nil {
-		log.WithError(err).Error("failed to close database")
-	}
-
-	log.Info("gracefully stopped")
 }

config.toml.example

@@ -11,7 +11,21 @@ hostname = "https://my.test.host:4443"
 bind_address = "172.20.10.2"
 # Specify path for reverse proxy and only [A-Za-z0-9]
 path = "test"
-data_dir = "/app/data" # Don't change if you run podsync via docker
+
+# Configure where to store the episode data
+[storage]
+  # Could be "local" (default) for the local file system, or "s3" for a S3-compatible storage provider (e.g. AWS S3)
+  type = "local"
+
+  [storage.local]
+  data_dir = "/app/data" # Don't change if you run podsync via docker
+
+  # To configure for a S3 provider, set key and secret in environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`, respectively;
+  # then fillout the API endpoint, region, and bucket below.
+  [storage.s3]
+  endpoint = "https://s3.us-west-2.amazonaws.com"
+  region = "us-west-2"
+  bucket = "example-bucket-name"
 
 # API keys to be used to access Youtube and Vimeo.
 # These can be either specified as string parameter or array of string (so those will be rotated).

go.mod

@@ -2,6 +2,7 @@ module github.com/mxpv/podsync
 
 require (
 	github.com/BrianHicks/finch v0.0.0-20140409222414-419bd73c29ec
+	github.com/aws/aws-sdk-go v1.44.30
 	github.com/dgraph-io/badger v1.6.2
 	github.com/eduncan911/podcast v1.4.2
 	github.com/gilliek/go-opml v1.0.0
@@ -20,7 +21,6 @@ require (
 	google.golang.org/api v0.0.0-20180718221112-efcb5f25ac56
 	google.golang.org/appengine v1.1.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
-	gopkg.in/yaml.v2 v2.2.8 // indirect
 )
 
 go 1.13

go.sum

@@ -7,6 +7,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/aws/aws-sdk-go v1.44.30 h1:w7sTp6jFWRaZCDg08fUx8X4IOU4sgbCR+e+1qSmf+bc=
+github.com/aws/aws-sdk-go v1.44.30/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -43,6 +45,10 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
 github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -91,8 +97,9 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/oauth2 v0.0.0-20180620175406-ef147856a6dd h1:QQhib242ErYDSMitlBm8V7wYCm/1a25hV8qMadIKLPA=
 golang.org/x/oauth2 v0.0.0-20180620175406-ef147856a6dd/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -106,11 +113,16 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=

pkg/fs/local.go

@@ -11,6 +11,11 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
+// LocalConfig is the storage configuration for local file system
+type LocalConfig struct {
+	DataDir string `yaml:"data_dir"`
+}
+
 // Local implements local file storage
 type Local struct {
 	rootDir string
@@ -65,3 +70,18 @@ func (l *Local) copyFile(source io.Reader, destinationPath string) (int64, error
 
 	return written, nil
 }
+
+func (l *Local) Size(_ctx context.Context, name string) (int64, error) {
+	file, err := l.Open(name)
+	if err != nil {
+		return 0, err
+	}
+	defer file.Close()
+
+	stat, err := file.Stat()
+	if err != nil {
+		return 0, err
+	}
+
+	return stat.Size(), nil
+}

pkg/fs/local_test.go

@@ -52,7 +52,7 @@ func TestLocal_Size(t *testing.T) {
 	_, err = stor.Create(testCtx, "1/test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3}))
 	assert.NoError(t, err)
 
-	sz, err := Size(stor, "1/test")
+	sz, err := stor.Size(testCtx, "1/test")
 	assert.NoError(t, err)
 	assert.EqualValues(t, 5, sz)
 }
@@ -61,7 +61,7 @@ func TestLocal_NoSize(t *testing.T) {
 	stor, err := NewLocal("")
 	assert.NoError(t, err)
 
-	_, err = Size(stor, "1/test")
+	_, err = stor.Size(testCtx, "1/test")
 	assert.True(t, os.IsNotExist(err))
 }
 
@@ -80,7 +80,7 @@ func TestLocal_Delete(t *testing.T) {
 	err = stor.Delete(testCtx, "1/test")
 	assert.NoError(t, err)
 
-	_, err = Size(stor, "1/test")
+	_, err = stor.Size(testCtx, "1/test")
 	assert.True(t, os.IsNotExist(err))
 
 	_, err = os.Stat(filepath.Join(tmpDir, "1", "test"))