Adding the Ruler to the Single Binary

CHANGELOG.md

@@ -37,6 +37,7 @@
 * [ENHANCEMENT] Experimental TSDB: Added support to enforce max query time range length via `-store.max-query-length`. #2826
 * [ENHANCEMENT] Ingester: Added new metric `cortex_ingester_flush_series_in_progress` that reports number of ongoing flush-series operations. Useful when calling `/flush` handler: if `cortex_ingester_flush_queue_length + cortex_ingester_flush_series_in_progress` is 0, all flushes are finished. #2778
 * [ENHANCEMENT] Memberlist members can join cluster via SRV records. #2788
+* [ENHANCEMENT] Added ruler to the single binary.  #2854
 * [BUGFIX] Fixed a bug in the index intersect code causing storage to return more chunks/series than required. #2796
 * [BUGFIX] Fixed the number of reported keys in the background cache queue. #2764
 * [BUGFIX] Fix race in processing of headers in sharded queries. #2762

development/tsdb-blocks-storage-s3-single-binary/.data-minio/.gitignore

@@ -1,3 +1,4 @@
 *
+!cortex-rules
 !cortex-tsdb
 !.gitignore

development/tsdb-blocks-storage-s3-single-binary/.data-minio/cortex-rules/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

development/tsdb-blocks-storage-s3-single-binary/config/cortex.yaml

@@ -56,3 +56,19 @@ tsdb:
 
 storage:
   engine: tsdb
+
+ruler:
+  enable_api: true
+  enable_sharding: true
+  poll_interval: 2s
+  storage:
+    type: s3
+    s3:
+      bucketnames: cortex-rules
+      s3forcepathstyle: true
+      s3: http://cortex:supersecret@minio.:9000
+  ring:
+    kvstore:
+      store: consul
+      consul:
+        host: consul:8500

docs/configuration/config-file-reference.md

@@ -816,7 +816,8 @@ ruler_client:
 [poll_interval: <duration> | default = 1m]
 
 storage:
-  # Method to use for backend rule storage (configdb, azure, gcs, s3)
+  # Method to use for backend rule storage (configdb, azure, gcs, s3, swift,
+  # local)
   # CLI flag: -ruler.storage.type
   [type: <string> | default = "configdb"]
 
@@ -958,6 +959,11 @@ storage:
     # CLI flag: -ruler.storage.swift.container-name
     [container_name: <string> | default = "cortex"]
 
+  local:
+    # Directory to scan for rules
+    # CLI flag: -ruler.storage.local.directory
+    [directory: <string> | default = ""]
+
 # file path to store temporary rule files for the prometheus rule managers
 # CLI flag: -ruler.rule-path
 [rule_path: <string> | default = "/rules"]

docs/configuration/single-process-config-blocks-gossip-1.yaml

@@ -80,4 +80,16 @@ tsdb:
     dir: /tmp/cortex/storage
 
 frontend_worker:
-  match_max_concurrent: true
+  match_max_concurrent: true
+
+ruler:
+  enable_api: true
+  enable_sharding: true
+  poll_interval: 2s
+  storage:
+    type: local
+    local:
+      directory: /tmp/cortex/rules
+  ring:
+    kvstore:
+      store: memberlist

docs/configuration/single-process-config-blocks-gossip-2.yaml

@@ -79,4 +79,16 @@ tsdb:
     dir: /tmp/cortex/storage
 
 frontend_worker:
-  match_max_concurrent: true
+  match_max_concurrent: true
+
+ruler:
+  enable_api: true
+  enable_sharding: true
+  poll_interval: 2s
+  storage:
+    type: local
+    local:
+      directory: /tmp/cortex/rules
+  ring:
+    kvstore:
+      store: memberlist

docs/configuration/single-process-config-blocks-tls.yaml

@@ -98,3 +98,12 @@ frontend_worker:
     tls_cert_path: "client.crt"
     tls_key_path: "client.key"
     tls_ca_path: "root.crt"
+
+ruler:
+  enable_api: true
+  enable_sharding: false
+  poll_interval: 2s
+  storage:
+    type: local
+    local:
+      directory: /tmp/cortex/rules

docs/configuration/single-process-config-blocks.yaml

@@ -84,4 +84,13 @@ compactor:
       store: inmemory
 
 frontend_worker:
-  match_max_concurrent: true
+  match_max_concurrent: true
+
+ruler:
+  enable_api: true
+  enable_sharding: false
+  poll_interval: 2s
+  storage:
+    type: local
+    local:
+      directory: /tmp/cortex/rules

docs/configuration/single-process-config.yaml

@@ -82,4 +82,13 @@ purger:
   object_store_type: filesystem
 
 frontend_worker:
-  match_max_concurrent: true
+  match_max_concurrent: true
+
+ruler:
+  enable_api: true
+  enable_sharding: false
+  poll_interval: 2s
+  storage:
+    type: local
+    local:
+      directory: /tmp/cortex/rules

integration/api_ruler_test.go

@@ -3,6 +3,8 @@
 package main
 
 import (
+	"path"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -94,3 +96,38 @@ func TestRulerAPI(t *testing.T) {
 	// Ensure no service-specific metrics prefix is used by the wrong service.
 	assertServiceMetricsPrefixes(t, Ruler, ruler)
 }
+
+func TestRulerAPISingleBinary(t *testing.T) {
+	s, err := e2e.NewScenario(networkName)
+	require.NoError(t, err)
+	defer s.Close()
+
+	namespace := "ns"
+	user := "fake"
+
+	configOverrides := map[string]string{
+		"-ruler.storage.local.directory": filepath.Join(e2e.ContainerSharedDir, "ruler_configs"),
+	}
+
+	// Start Cortex components.
+	require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config.yaml", cortexConfigFile))
+	require.NoError(t, writeFileToSharedDir(s, path.Join("ruler_configs", user, namespace), []byte(cortexRulerUserConfigYaml)))
+	cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex", cortexConfigFile, configOverrides, "", 9009, 9095)
+	require.NoError(t, s.StartAndWaitReady(cortex))
+
+	// Create a client with the ruler address configured
+	c, err := e2ecortex.NewClient("", "", "", cortex.HTTPEndpoint(), "")
+	require.NoError(t, err)
+
+	// Wait until the user manager is created
+	require.NoError(t, cortex.WaitSumMetrics(e2e.Equals(1), "cortex_ruler_managers_total"))
+
+	// Check to ensure the rules running in the cortex match what was set
+	rgs, err := c.GetRuleGroups()
+	require.NoError(t, err)
+
+	retrievedNamespace, exists := rgs[namespace]
+	require.True(t, exists)
+	require.Len(t, retrievedNamespace, 1)
+	require.Equal(t, retrievedNamespace[0].Name, "rule")
+}

integration/configs.go

@@ -45,6 +45,18 @@ const (
 receivers:
   - name: "example_receiver"
 `
+
+	cortexRulerUserConfigYaml = `groups:
+- name: rule
+  interval: 100s
+  rules:
+  - record: test_rule
+    alert: ""
+    expr: up
+    for: 0s
+    labels: {}
+    annotations: {}	
+`
 )
 
 var (

integration/e2ecortex/services.go

@@ -237,6 +237,9 @@ func NewSingleBinary(name string, flags map[string]string, image string, otherPo
 			"-ingester.concurrent-flushes":   "10",
 			"-ingester.max-transfer-retries": "10",
 			"-ingester.num-tokens":           "512",
+			// Ruler
+			"-ruler.storage.type":            "local",
+			"-ruler.storage.local.directory": "/tmp/cortex/rules",
 		}, flags))...),
 		e2e.NewHTTPReadinessProbe(httpPort, "/ready", 200, 299),
 		httpPort,

pkg/cortex/cortex.go

@@ -20,6 +20,9 @@ import (
 	"google.golang.org/grpc/health/grpc_health_v1"
 	"gopkg.in/yaml.v2"
 
+	"github.com/prometheus/prometheus/promql"
+	prom_storage "github.com/prometheus/prometheus/storage"
+
 	"github.com/cortexproject/cortex/pkg/alertmanager"
 	"github.com/cortexproject/cortex/pkg/api"
 	"github.com/cortexproject/cortex/pkg/chunk"
@@ -219,9 +222,8 @@ type Cortex struct {
 	StoreGateway *storegateway.StoreGateway
 	MemberlistKV *memberlist.KVInitService
 
-	// Queryables that the querier should use to query the long
-	// term storage. It depends on the storage engine used.
-	StoreQueryables []querier.QueryableWithFilter
+	Queryable prom_storage.SampleAndChunkQueryable
+	Engine    *promql.Engine
 }
 
 // New makes a new Cortex.

pkg/cortex/cortex_test.go

@@ -1,20 +1,27 @@
 package cortex
 
 import (
+	"net/url"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/cortexproject/cortex/pkg/chunk/aws"
 	"github.com/cortexproject/cortex/pkg/chunk/storage"
 	"github.com/cortexproject/cortex/pkg/ingester"
 	"github.com/cortexproject/cortex/pkg/ring"
 	"github.com/cortexproject/cortex/pkg/ring/kv"
+	"github.com/cortexproject/cortex/pkg/ruler"
 	"github.com/cortexproject/cortex/pkg/storage/backend/s3"
 	"github.com/cortexproject/cortex/pkg/storage/tsdb"
+	"github.com/cortexproject/cortex/pkg/util/flagext"
 	"github.com/cortexproject/cortex/pkg/util/services"
 )
 
 func TestCortex(t *testing.T) {
+	rulerURL, err := url.Parse("inmemory:///rules")
+	require.NoError(t, err)
+
 	cfg := Config{
 		Storage: storage.Config{
 			Engine: storage.StorageEngineTSDB, // makes config easier
@@ -47,6 +54,16 @@ func TestCortex(t *testing.T) {
 				},
 			},
 		},
+		Ruler: ruler.Config{
+			StoreConfig: ruler.RuleStoreConfig{
+				Type: "s3",
+				S3: aws.S3Config{
+					S3: flagext.URLValue{
+						URL: rulerURL,
+					},
+				},
+			},
+		},
 		Target: All,
 	}