Validate alertmanager storage configuration when sharding enabled. (#4162)

* Validate alertmanager storage configuration when sharding enabled.

Storing the "fullstate" objects is not implemented for the "local" and
"configdb" storage backend types, or for the legacy storage
configuration. Check that these are not given when sharding is enabled.

Signed-off-by: Steve Simpson <[email protected]>

* Review comments.

Signed-off-by: Steve Simpson <[email protected]>

Author: stevesg

integration/alertmanager_test.go

@@ -235,13 +235,10 @@ func TestAlertmanagerClustering(t *testing.T) {
 
 func TestAlertmanagerSharding(t *testing.T) {
 	tests := map[string]struct {
-		legacyAlertStore  bool
 		replicationFactor int
 	}{
-		"legacy alertstore, RF = 2": {legacyAlertStore: true, replicationFactor: 2},
-		"bucket alertstore, RF = 2": {legacyAlertStore: false, replicationFactor: 2},
-		"legacy alertstore, RF = 3": {legacyAlertStore: true, replicationFactor: 3},
-		"bucket alertstore, RF = 3": {legacyAlertStore: false, replicationFactor: 3},
+		"RF = 2": {replicationFactor: 2},
+		"RF = 3": {replicationFactor: 3},
 	}
 
 	for testName, testCfg := range tests {
@@ -250,7 +247,7 @@ func TestAlertmanagerSharding(t *testing.T) {
 			require.NoError(t, err)
 			defer s.Close()
 
-			flags := mergeFlags(AlertmanagerFlags(), AlertmanagerS3Flags(testCfg.legacyAlertStore))
+			flags := mergeFlags(AlertmanagerFlags(), AlertmanagerS3Flags(false))
 
 			// Start dependencies.
 			consul := e2edb.NewConsul()

pkg/alertmanager/alertstore/config.go

@@ -70,3 +70,13 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	cfg.Local.RegisterFlagsWithPrefix(prefix, f)
 	cfg.RegisterFlagsWithPrefix(prefix, f)
 }
+
+// IsFullStateSupported returns if the given configuration supports access to FullState objects.
+func (cfg *Config) IsFullStateSupported() bool {
+	for _, backend := range bucket.SupportedBackends {
+		if cfg.Backend == backend {
+			return true
+		}
+	}
+	return false
+}

pkg/alertmanager/multitenant.go

@@ -86,7 +86,9 @@ const (
 var (
 	statusTemplate *template.Template
 
-	errInvalidExternalURL = errors.New("the configured external URL is invalid: should not end with /")
+	errInvalidExternalURL         = errors.New("the configured external URL is invalid: should not end with /")
+	errShardingLegacyStorage      = errors.New("deprecated -alertmanager.storage.* not supported with -alertmanager.sharding-enabled, use -alertmanager-storage.*")
+	errShardingUnsupportedStorage = errors.New("the configured alertmanager storage backend is not supported when sharding is enabled")
 )
 
 func init() {
@@ -175,7 +177,7 @@ func (cfg *ClusterConfig) RegisterFlags(f *flag.FlagSet) {
 }
 
 // Validate config and returns error on failure
-func (cfg *MultitenantAlertmanagerConfig) Validate() error {
+func (cfg *MultitenantAlertmanagerConfig) Validate(storageCfg alertstore.Config) error {
 	if cfg.ExternalURL.URL != nil && strings.HasSuffix(cfg.ExternalURL.Path, "/") {
 		return errInvalidExternalURL
 	}
@@ -188,6 +190,15 @@ func (cfg *MultitenantAlertmanagerConfig) Validate() error {
 		return err
 	}
 
+	if cfg.ShardingEnabled {
+		if !cfg.Store.IsDefaults() {
+			return errShardingLegacyStorage
+		}
+		if !storageCfg.IsFullStateSupported() {
+			return errShardingUnsupportedStorage
+		}
+	}
+
 	return nil
 }
 

pkg/alertmanager/multitenant_test.go

@@ -94,45 +94,75 @@ func mockAlertmanagerConfig(t *testing.T) *MultitenantAlertmanagerConfig {
 
 func TestMultitenantAlertmanagerConfig_Validate(t *testing.T) {
 	tests := map[string]struct {
-		setup    func(t *testing.T, cfg *MultitenantAlertmanagerConfig)
+		setup    func(t *testing.T, cfg *MultitenantAlertmanagerConfig, storageCfg *alertstore.Config)
 		expected error
 	}{
 		"should pass with default config": {
-			setup:    func(t *testing.T, cfg *MultitenantAlertmanagerConfig) {},
+			setup:    func(t *testing.T, cfg *MultitenantAlertmanagerConfig, storageCfg *alertstore.Config) {},
 			expected: nil,
 		},
 		"should fail if persistent interval is 0": {
-			setup: func(t *testing.T, cfg *MultitenantAlertmanagerConfig) {
+			setup: func(t *testing.T, cfg *MultitenantAlertmanagerConfig, storageCfg *alertstore.Config) {
 				cfg.Persister.Interval = 0
 			},
 			expected: errInvalidPersistInterval,
 		},
 		"should fail if persistent interval is negative": {
-			setup: func(t *testing.T, cfg *MultitenantAlertmanagerConfig) {
+			setup: func(t *testing.T, cfg *MultitenantAlertmanagerConfig, storageCfg *alertstore.Config) {
 				cfg.Persister.Interval = -1
 			},
 			expected: errInvalidPersistInterval,
 		},
 		"should fail if external URL ends with /": {
-			setup: func(t *testing.T, cfg *MultitenantAlertmanagerConfig) {
+			setup: func(t *testing.T, cfg *MultitenantAlertmanagerConfig, storageCfg *alertstore.Config) {
 				require.NoError(t, cfg.ExternalURL.Set("http://localhost/prefix/"))
 			},
 			expected: errInvalidExternalURL,
 		},
 		"should succeed if external URL does not end with /": {
-			setup: func(t *testing.T, cfg *MultitenantAlertmanagerConfig) {
+			setup: func(t *testing.T, cfg *MultitenantAlertmanagerConfig, storageCfg *alertstore.Config) {
 				require.NoError(t, cfg.ExternalURL.Set("http://localhost/prefix"))
 			},
 			expected: nil,
 		},
+		"should succeed if sharding enabled and new storage configuration given with bucket client": {
+			setup: func(t *testing.T, cfg *MultitenantAlertmanagerConfig, storageCfg *alertstore.Config) {
+				cfg.ShardingEnabled = true
+				storageCfg.Backend = "s3"
+			},
+			expected: nil,
+		},
+		"should fail if sharding enabled and new storage store configuration given with local type": {
+			setup: func(t *testing.T, cfg *MultitenantAlertmanagerConfig, storageCfg *alertstore.Config) {
+				cfg.ShardingEnabled = true
+				storageCfg.Backend = "local"
+			},
+			expected: errShardingUnsupportedStorage,
+		},
+		"should fail if sharding enabled and new storage store configuration given with configdb type": {
+			setup: func(t *testing.T, cfg *MultitenantAlertmanagerConfig, storageCfg *alertstore.Config) {
+				cfg.ShardingEnabled = true
+				storageCfg.Backend = "configdb"
+			},
+			expected: errShardingUnsupportedStorage,
+		},
+		"should fail if sharding enabled and legacy store configuration given": {
+			setup: func(t *testing.T, cfg *MultitenantAlertmanagerConfig, storageCfg *alertstore.Config) {
+				cfg.ShardingEnabled = true
+				cfg.Store.Type = "s3"
+			},
+			expected: errShardingLegacyStorage,
+		},
 	}
 
 	for testName, testData := range tests {
 		t.Run(testName, func(t *testing.T) {
 			cfg := &MultitenantAlertmanagerConfig{}
+			storageCfg := alertstore.Config{}
 			flagext.DefaultValues(cfg)
-			testData.setup(t, cfg)
-			assert.Equal(t, testData.expected, cfg.Validate())
+			flagext.DefaultValues(&storageCfg)
+			testData.setup(t, cfg, &storageCfg)
+			assert.Equal(t, testData.expected, cfg.Validate(storageCfg))
 		})
 	}
 }

pkg/cortex/cortex.go

@@ -236,12 +236,12 @@ func (c *Config) Validate(log log.Logger) error {
 	if err := c.Compactor.Validate(); err != nil {
 		return errors.Wrap(err, "invalid compactor config")
 	}
-	if err := c.Alertmanager.Validate(); err != nil {
-		return errors.Wrap(err, "invalid alertmanager config")
-	}
 	if err := c.AlertmanagerStorage.Validate(); err != nil {
 		return errors.Wrap(err, "invalid alertmanager storage config")
 	}
+	if err := c.Alertmanager.Validate(c.AlertmanagerStorage); err != nil {
+		return errors.Wrap(err, "invalid alertmanager config")
+	}
 
 	if c.Storage.Engine == storage.StorageEngineBlocks && c.Querier.SecondStoreEngine != storage.StorageEngineChunks && len(c.Schema.Configs) > 0 {
 		level.Warn(log).Log("schema configuration is not used by the blocks storage engine, and will have no effect")

pkg/storage/bucket/client.go

@@ -37,7 +37,7 @@ const (
 )
 
 var (
-	supportedBackends = []string{S3, GCS, Azure, Swift, Filesystem}
+	SupportedBackends = []string{S3, GCS, Azure, Swift, Filesystem}
 
 	ErrUnsupportedStorageBackend = errors.New("unsupported storage backend")
 )
@@ -63,7 +63,7 @@ type Config struct {
 
 // Returns the supportedBackends for the package and any custom backends injected into the config.
 func (cfg *Config) supportedBackends() []string {
-	return append(supportedBackends, cfg.ExtraBackends...)
+	return append(SupportedBackends, cfg.ExtraBackends...)
 }
 
 // RegisterFlags registers the backend storage config.