/ready now returns 200, not 204

CHANGELOG.md

@@ -40,6 +40,7 @@
 * [CHANGE] Utilize separate protos for rule state and storage. Experimental ruler API will not be functional until the rollout is complete. #2226
 * [CHANGE] Frontend worker in querier now starts after all Querier module dependencies are started. This fixes issue where frontend worker started to send queries to querier before it was ready to serve them (mostly visible when using experimental blocks storage). #2246
 * [CHANGE] Lifecycler component now enters Failed state on errors, and doesn't exit the process. (Important if you're vendoring Cortex and use Lifecycler) #2251
+* [CHANGE] /ready handler now returns 200 instead of 204.
 * [FEATURE] Added experimental storage API to the ruler service that is enabled when the `-experimental.ruler.enable-api` is set to true #2269
   * `-ruler.storage.type` flag now allows `s3`,`gcs`, and `azure` values
   * `-ruler.storage.(s3|gcs|azure)` flags exist to allow the configuration of object clients set for rule storage

integration/backward_compatibility_test.go

@@ -16,24 +16,34 @@ import (
 	"github.com/cortexproject/cortex/integration/e2ecortex"
 )
 
+type serviceSetupFn func(name string, s *e2ecortex.CortexService)
+
 var (
 	// If you change the image tag, remember to update it in the preloading done
 	// by CircleCI too (see .circleci/config.yml).
-	previousVersionImages = []string{
-		"quay.io/cortexproject/cortex:v0.6.0",
-		"quay.io/cortexproject/cortex:v0.7.0",
+	previousVersionImages = map[string]serviceSetupFn{
+		"quay.io/cortexproject/cortex:v0.6.0": func(name string, s *e2ecortex.CortexService) {
+			// 0.6.0 used 204 status code for querier and ingester
+			// distributor didn't have /ready page, and we used check on the /ring page instead
+			s.SetReadinessProbe(e2e.NewHTTPReadinessProbe(s.HTTPPort(), "/ready", 204))
+		},
+
+		"quay.io/cortexproject/cortex:v0.7.0": func(name string, s *e2ecortex.CortexService) {
+			// 0.7.0 used 204 status code for all components
+			s.SetReadinessProbe(e2e.NewHTTPReadinessProbe(s.HTTPPort(), "/ready", 204))
+		},
 	}
 )
 
 func TestBackwardCompatibilityWithChunksStorage(t *testing.T) {
-	for _, previousImage := range previousVersionImages {
+	for previousImage, setupFn := range previousVersionImages {
 		t.Run(fmt.Sprintf("Backward compatibility upgrading from %s", previousImage), func(t *testing.T) {
-			runBackwardCompatibilityTestWithChunksStorage(t, previousImage)
+			runBackwardCompatibilityTestWithChunksStorage(t, previousImage, setupFn)
 		})
 	}
 }
 
-func runBackwardCompatibilityTestWithChunksStorage(t *testing.T, previousImage string) {
+func runBackwardCompatibilityTestWithChunksStorage(t *testing.T, previousImage string, setupFn serviceSetupFn) {
 	s, err := e2e.NewScenario(networkName)
 	require.NoError(t, err)
 	defer s.Close()
@@ -61,6 +71,7 @@ func runBackwardCompatibilityTestWithChunksStorage(t *testing.T, previousImage s
 
 	// Start other Cortex components (ingester running on previous version).
 	ingester1 := e2ecortex.NewIngester("ingester-1", consul.NetworkHTTPEndpoint(), flagsForOldImage, previousImage)
+	setupFn("ingester", ingester1)
 	distributor := e2ecortex.NewDistributor("distributor", consul.NetworkHTTPEndpoint(), flagsForOldImage, "")
 	require.NoError(t, s.StartAndWaitReady(distributor, ingester1))
 
@@ -90,11 +101,15 @@ func runBackwardCompatibilityTestWithChunksStorage(t *testing.T, previousImage s
 
 	// Query the new ingester both with the old and the new querier.
 	for _, image := range []string{previousImage, ""} {
-		flags := ChunksStorageFlags
+		var querier *e2ecortex.CortexService
+
 		if image == previousImage {
-			flags = flagsForOldImage
+			querier = e2ecortex.NewQuerier("querier", consul.NetworkHTTPEndpoint(), flagsForOldImage, image)
+			setupFn("querier", querier)
+		} else {
+			querier = e2ecortex.NewQuerier("querier", consul.NetworkHTTPEndpoint(), ChunksStorageFlags, image)
 		}
-		querier := e2ecortex.NewQuerier("querier", consul.NetworkHTTPEndpoint(), flags, image)
+
 		require.NoError(t, s.StartAndWaitReady(querier))
 
 		// Wait until the querier has updated the ring.

integration/e2e/service.go

@@ -393,7 +393,7 @@ func (p *HTTPReadinessProbe) Ready(service *ConcreteService) (err error) {
 		return nil
 	}
 
-	return fmt.Errorf("got no expected status code: %v, expected: %v", res.StatusCode, p.expectedStatus)
+	return fmt.Errorf("got status code: %v, expected: %v", res.StatusCode, p.expectedStatus)
 }
 
 // TCPReadinessProbe checks readiness by ensure a TCP connection can be established.

integration/e2ecortex/services.go

@@ -48,7 +48,7 @@ func NewDistributorWithConfigFile(name, consulAddress, configFile string, flags
 			"-ring.store":      "consul",
 			"-consul.hostname": consulAddress,
 		}, flags))...),
-		e2e.NewHTTPReadinessProbe(httpPort, "/ready", 204),
+		e2e.NewHTTPReadinessProbe(httpPort, "/ready", 200),
 		httpPort,
 		grpcPort,
 	)
@@ -83,7 +83,7 @@ func NewQuerierWithConfigFile(name, consulAddress, configFile string, flags map[
 			"-querier.frontend-client.backoff-retries":    "1",
 			"-querier.worker-parallelism":                 "1",
 		}, flags))...),
-		e2e.NewHTTPReadinessProbe(httpPort, "/ready", 204),
+		e2e.NewHTTPReadinessProbe(httpPort, "/ready", 200),
 		httpPort,
 		grpcPort,
 	)
@@ -117,7 +117,7 @@ func NewIngesterWithConfigFile(name, consulAddress, configFile string, flags map
 			"-ring.store":      "consul",
 			"-consul.hostname": consulAddress,
 		}, flags))...),
-		e2e.NewHTTPReadinessProbe(httpPort, "/ready", 204),
+		e2e.NewHTTPReadinessProbe(httpPort, "/ready", 200),
 		httpPort,
 		grpcPort,
 	)
@@ -143,7 +143,7 @@ func NewTableManagerWithConfigFile(name, configFile string, flags map[string]str
 			"-target":    "table-manager",
 			"-log.level": "warn",
 		}, flags))...),
-		e2e.NewHTTPReadinessProbe(httpPort, "/ready", 204),
+		e2e.NewHTTPReadinessProbe(httpPort, "/ready", 200),
 		httpPort,
 		grpcPort,
 	)
@@ -169,7 +169,7 @@ func NewQueryFrontendWithConfigFile(name, configFile string, flags map[string]st
 			"-target":    "query-frontend",
 			"-log.level": "warn",
 		}, flags))...),
-		e2e.NewHTTPReadinessProbe(httpPort, "/ready", 204),
+		e2e.NewHTTPReadinessProbe(httpPort, "/ready", 200),
 		httpPort,
 		grpcPort,
 	)
@@ -186,7 +186,7 @@ func NewSingleBinary(name string, flags map[string]string, image string, httpPor
 		e2e.NewCommandWithoutEntrypoint("cortex", e2e.BuildArgs(e2e.MergeFlags(map[string]string{
 			"-log.level": "warn",
 		}, flags))...),
-		e2e.NewHTTPReadinessProbe(httpPort, "/ready", 204),
+		e2e.NewHTTPReadinessProbe(httpPort, "/ready", 200),
 		httpPort,
 		grpcPort,
 		otherPorts...,
@@ -205,7 +205,7 @@ func NewAlertmanager(name string, flags map[string]string, image string) *Cortex
 			"-target":    "alertmanager",
 			"-log.level": "warn",
 		}, flags))...),
-		e2e.NewHTTPReadinessProbe(httpPort, "/ready", 204),
+		e2e.NewHTTPReadinessProbe(httpPort, "/ready", 200),
 		httpPort,
 		grpcPort,
 	)

pkg/cortex/cortex.go

@@ -401,9 +401,11 @@ func (t *Cortex) readyHandler(sm *services.Manager) http.HandlerFunc {
 		if t.ingester != nil {
 			if err := t.ingester.CheckReady(r.Context()); err != nil {
 				http.Error(w, "Ingester not ready: "+err.Error(), http.StatusServiceUnavailable)
+				return
 			}
 		}
-		w.WriteHeader(http.StatusNoContent)
+
+		http.Error(w, "ready", http.StatusOK)
 	}
 }