rule: Added ability to specify multiple remote write targets. (#4927)

This is mainly for non surprising remote write YAML format, but also for ability
to add more targets, similar to agent supported model.

Signed-off-by: Bartlomiej Plotka <[email protected]>

Author: bwplotka

cmd/thanos/rule.go

@@ -126,7 +126,7 @@ func registerRule(app *extkingpin.App) {
 	cmd.Flag("eval-interval", "The default evaluation interval to use.").
 		Default("30s").DurationVar(&conf.evalInterval)
 
-	conf.rwConfig = extflag.RegisterPathOrContent(cmd, "remote-write.config", "YAML config for the remote-write server where samples should be sent to (see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). This automatically enables stateless mode for ruler and no series will be stored in the ruler's TSDB. If an empty config (or file) is provided, the flag is ignored and ruler is run with its own TSDB.", extflag.WithEnvSubstitution())
+	conf.rwConfig = extflag.RegisterPathOrContent(cmd, "remote-write.config", "YAML config for the remote-write configurations, that specify servers where samples should be sent to (see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). This automatically enables stateless mode for ruler and no series will be stored in the ruler's TSDB. If an empty config (or file) is provided, the flag is ignored and ruler is run with its own TSDB.", extflag.WithEnvSubstitution())
 
 	reqLogDecision := cmd.Flag("log.request.decision", "Deprecation Warning - This flag would be soon deprecated, and replaced with `request.logging-config`. Request Logging for logging the start and end of requests. By default this flag is disabled. LogFinishCall: Logs the finish call of the requests. LogStartAndFinishCall: Logs the start and finish call of the requests. NoLogCall: Disable request logging.").Default("").Enum("NoLogCall", "LogFinishCall", "LogStartAndFinishCall", "")
 
@@ -350,18 +350,21 @@ func runRule(
 	}
 
 	if len(rwCfgYAML) > 0 {
-		var rwCfg config.RemoteWriteConfig
+		var rwCfg struct {
+			RemoteWriteConfigs []*config.RemoteWriteConfig `yaml:"remote_write,omitempty"`
+		}
 		if err := yaml.Unmarshal(rwCfgYAML, &rwCfg); err != nil {
-			return err
+			return errors.Wrapf(err, "failed to parse remote write config %v", string(rwCfgYAML))
 		}
-		walDir := filepath.Join(conf.dataDir, rwCfg.Name)
+
+		walDir := filepath.Join(conf.dataDir, "wal")
 		// flushDeadline is set to 1m, but it is for metadata watcher only so not used here.
 		remoteStore := remote.NewStorage(logger, reg, func() (int64, error) {
 			return 0, nil
 		}, walDir, 1*time.Minute, nil)
 		if err := remoteStore.ApplyConfig(&config.Config{
 			GlobalConfig:       config.DefaultGlobalConfig,
-			RemoteWriteConfigs: []*config.RemoteWriteConfig{&rwCfg},
+			RemoteWriteConfigs: rwCfg.RemoteWriteConfigs,
 		}); err != nil {
 			return errors.Wrap(err, "applying config to remote storage")
 		}

docs/components/rule.md

@@ -210,7 +210,7 @@ Stateless ruler enables nearly indefinite horizontal scalability. Ruler doesn't
 
 The WAL only storage reuses the upstream [Prometheus agent](https://prometheus.io/blog/2021/11/16/agent/) and it is compatible with the old TSDB data. For more design purpose of this mode, please refer to the [proposal](https://thanos.io/tip/proposals-done/202005-scalable-rule-storage.md/).
 
-Stateless mode can be enabled by providing `--remote-write.config` or `--remote-write.config-file` flag. For example:
+Stateless mode can be enabled by providing [Prometheus remote write config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write) in file via `--remote-write.config` or inlined `--remote-write.config-file` flag. For example:
 
 ```bash
 thanos rule \
@@ -227,7 +227,27 @@ thanos rule \
     --remote-write.config-file  'rw-config.yaml'
 ```
 
-The remote write config file is exactly the same as the [Prometheus remote write config format](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write).
+Where `rw-config.yaml` could look as follows:
+
+```yaml
+remote_write:
+- url: http://e2e_test_rule_remote_write-receive-1:8081/api/v1/receive
+  name: thanos-receiver
+  follow_redirects: false
+- url: https://e2e_test_rule_remote_write-receive-2:443/api/v1/receive
+  remote_timeout: 30s
+  follow_redirects: true
+  queue_config:
+    capacity: 120000
+    max_shards: 50
+    min_shards: 1
+    max_samples_per_send: 40000
+    batch_send_deadline: 5s
+    min_backoff: 5s
+    max_backoff: 5m
+```
+
+You can pass this in file using `--remote-write.config-file=` or inline it using `--remote-write.config=`.
 
 **NOTE:**
 1. `metadata_config` is not supported in this mode and will be ignored if provided in the remote write configuration.
@@ -378,17 +398,19 @@ Flags:
       --remote-write.config=<content>  
                                  Alternative to 'remote-write.config-file' flag
                                  (mutually exclusive). Content of YAML config
-                                 for the remote-write server where samples
-                                 should be sent to (see
+                                 for the remote-write configurations, that
+                                 specify servers where samples should be sent to
+                                 (see
                                  https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write).
                                  This automatically enables stateless mode for
                                  ruler and no series will be stored in the
                                  ruler's TSDB. If an empty config (or file) is
                                  provided, the flag is ignored and ruler is run
                                  with its own TSDB.
       --remote-write.config-file=<file-path>  
-                                 Path to YAML config for the remote-write server
-                                 where samples should be sent to (see
+                                 Path to YAML config for the remote-write
+                                 configurations, that specify servers where
+                                 samples should be sent to (see
                                  https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write).
                                  This automatically enables stateless mode for
                                  ruler and no series will be stored in the

test/e2e/e2ethanos/services.go

@@ -551,11 +551,11 @@ func NewTSDBRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.Aler
 	return newRuler(e, name, ruleSubDir, amCfg, queryCfg, nil)
 }
 
-func NewStatelessRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.AlertmanagerConfig, queryCfg []httpconfig.Config, remoteWriteCfg *config.RemoteWriteConfig) (*e2e.InstrumentedRunnable, error) {
+func NewStatelessRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.AlertmanagerConfig, queryCfg []httpconfig.Config, remoteWriteCfg []*config.RemoteWriteConfig) (*e2e.InstrumentedRunnable, error) {
 	return newRuler(e, name, ruleSubDir, amCfg, queryCfg, remoteWriteCfg)
 }
 
-func newRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.AlertmanagerConfig, queryCfg []httpconfig.Config, remoteWriteCfg *config.RemoteWriteConfig) (*e2e.InstrumentedRunnable, error) {
+func newRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.AlertmanagerConfig, queryCfg []httpconfig.Config, remoteWriteCfg []*config.RemoteWriteConfig) (*e2e.InstrumentedRunnable, error) {
 	dir := filepath.Join(e.SharedDir(), "data", "rule", name)
 	container := filepath.Join(ContainerSharedDir, "data", "rule", name)
 
@@ -592,7 +592,9 @@ func newRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.Alertman
 		"--resend-delay":                  "5s",
 	}
 	if remoteWriteCfg != nil {
-		rwCfgBytes, err := yaml.Marshal(remoteWriteCfg)
+		rwCfgBytes, err := yaml.Marshal(struct {
+			RemoteWriteConfigs []*config.RemoteWriteConfig `yaml:"remote_write,omitempty"`
+		}{remoteWriteCfg})
 		if err != nil {
 			return nil, errors.Wrapf(err, "generate remote write config: %v", remoteWriteCfg)
 		}

test/e2e/rule_test.go

@@ -492,7 +492,12 @@ func TestRule_CanRemoteWriteData(t *testing.T) {
 	testutil.Ok(t, e2e.StartAndWaitReady(receiver))
 	rwURL := mustURLParse(t, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write")))
 
-	q, err := e2ethanos.NewQuerierBuilder(e, "1", receiver.InternalEndpoint("grpc")).Build()
+	receiver2, err := e2ethanos.NewIngestingReceiver(e, "2")
+	testutil.Ok(t, err)
+	testutil.Ok(t, e2e.StartAndWaitReady(receiver2))
+	rwURL2 := mustURLParse(t, e2ethanos.RemoteWriteEndpoint(receiver2.InternalEndpoint("remote-write")))
+
+	q, err := e2ethanos.NewQuerierBuilder(e, "1", receiver.InternalEndpoint("grpc"), receiver2.InternalEndpoint("grpc")).Build()
 	testutil.Ok(t, err)
 	testutil.Ok(t, e2e.StartAndWaitReady(q))
 	r, err := e2ethanos.NewStatelessRuler(e, "1", rulesSubDir, []alert.AlertmanagerConfig{
@@ -515,9 +520,9 @@ func TestRule_CanRemoteWriteData(t *testing.T) {
 				Scheme: "http",
 			},
 		},
-	}, &config.RemoteWriteConfig{
-		URL:  &common_cfg.URL{URL: rwURL},
-		Name: "thanos-receiver",
+	}, []*config.RemoteWriteConfig{
+		{URL: &common_cfg.URL{URL: rwURL}, Name: "thanos-receiver"},
+		{URL: &common_cfg.URL{URL: rwURL2}, Name: "thanos-receiver2"},
 	})
 	testutil.Ok(t, err)
 	testutil.Ok(t, e2e.StartAndWaitReady(r))
@@ -536,6 +541,12 @@ func TestRule_CanRemoteWriteData(t *testing.T) {
 				"receive":   "1",
 				"tenant_id": "default-tenant",
 			},
+			{
+				"__name__":  "test_absent_metric",
+				"job":       "thanos-receive",
+				"receive":   "2",
+				"tenant_id": "default-tenant",
+			},
 		})
 	})
 }