diff --git a/CHANGELOG.md b/CHANGELOG.md index a24fa290185..49178f1ad22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ * [CHANGE] Utilize separate protos for rule state and storage. Experimental ruler API will not be functional until the rollout is complete. #2226 * [CHANGE] Frontend worker in querier now starts after all Querier module dependencies are started. This fixes issue where frontend worker started to send queries to querier before it was ready to serve them (mostly visible when using experimental blocks storage). #2246 * [CHANGE] Lifecycler component now enters Failed state on errors, and doesn't exit the process. (Important if you're vendoring Cortex and use Lifecycler) #2251 +* [CHANGE] `/ready` handler now returns 200 instead of 204. #2330 * [FEATURE] Added experimental storage API to the ruler service that is enabled when the `-experimental.ruler.enable-api` is set to true #2269 * `-ruler.storage.type` flag now allows `s3`,`gcs`, and `azure` values * `-ruler.storage.(s3|gcs|azure)` flags exist to allow the configuration of object clients set for rule storage diff --git a/integration/backward_compatibility_test.go b/integration/backward_compatibility_test.go index 619ff293f7d..58d5f7781b9 100644 --- a/integration/backward_compatibility_test.go +++ b/integration/backward_compatibility_test.go @@ -20,7 +20,11 @@ var ( // If you change the image tag, remember to update it in the preloading done // by CircleCI too (see .circleci/config.yml). previousVersionImages = []string{ + // 0.6.0 used 204 status code for querier and ingester + // distributor didn't have /ready page, and we used check on the /ring page instead "quay.io/cortexproject/cortex:v0.6.0", + + // 0.7.0 used 204 status code for all components "quay.io/cortexproject/cortex:v0.7.0", } ) @@ -90,11 +94,14 @@ func runBackwardCompatibilityTestWithChunksStorage(t *testing.T, previousImage s // Query the new ingester both with the old and the new querier. for _, image := range []string{previousImage, ""} { - flags := ChunksStorageFlags + var querier *e2ecortex.CortexService + if image == previousImage { - flags = flagsForOldImage + querier = e2ecortex.NewQuerier("querier", consul.NetworkHTTPEndpoint(), flagsForOldImage, image) + } else { + querier = e2ecortex.NewQuerier("querier", consul.NetworkHTTPEndpoint(), ChunksStorageFlags, image) } - querier := e2ecortex.NewQuerier("querier", consul.NetworkHTTPEndpoint(), flags, image) + require.NoError(t, s.StartAndWaitReady(querier)) // Wait until the querier has updated the ring. diff --git a/integration/e2e/db/db.go b/integration/e2e/db/db.go index efc40610faa..de758cddf70 100644 --- a/integration/e2e/db/db.go +++ b/integration/e2e/db/db.go @@ -25,7 +25,7 @@ func NewMinio(port int, bktName string) *e2e.HTTPService { "minio/minio:RELEASE.2019-12-30T05-45-39Z", // Create the "cortex" bucket before starting minio e2e.NewCommandWithoutEntrypoint("sh", "-c", fmt.Sprintf("mkdir -p /data/%s && minio server --address :%v --quiet /data", bktName, port)), - e2e.NewHTTPReadinessProbe(port, "/minio/health/ready", 200), + e2e.NewHTTPReadinessProbe(port, "/minio/health/ready", 200, 200), port, ) m.SetEnvVars(map[string]string{ @@ -78,7 +78,7 @@ func NewDynamoDB() *e2e.HTTPService { "amazon/dynamodb-local:1.11.477", e2e.NewCommand("-jar", "DynamoDBLocal.jar", "-inMemory", "-sharedDb"), // DynamoDB doesn't have a readiness probe, so we check if the / works even if returns 400 - e2e.NewHTTPReadinessProbe(8000, "/", 400), + e2e.NewHTTPReadinessProbe(8000, "/", 400, 400), 8000, ) } diff --git a/integration/e2e/service.go b/integration/e2e/service.go index 8bccf508ba2..27db1ee4f53 100644 --- a/integration/e2e/service.go +++ b/integration/e2e/service.go @@ -361,16 +361,18 @@ type ReadinessProbe interface { // HTTPReadinessProbe checks readiness by making HTTP call and checking for expected HTTP status code type HTTPReadinessProbe struct { - port int - path string - expectedStatus int + port int + path string + expectedStatusRangeStart int + expectedStatusRangeEnd int } -func NewHTTPReadinessProbe(port int, path string, expectedStatus int) *HTTPReadinessProbe { +func NewHTTPReadinessProbe(port int, path string, expectedStatusRangeStart, expectedStatusRangeEnd int) *HTTPReadinessProbe { return &HTTPReadinessProbe{ - port: port, - path: path, - expectedStatus: expectedStatus, + port: port, + path: path, + expectedStatusRangeStart: expectedStatusRangeStart, + expectedStatusRangeEnd: expectedStatusRangeEnd, } } @@ -389,11 +391,11 @@ func (p *HTTPReadinessProbe) Ready(service *ConcreteService) (err error) { defer runutil.ExhaustCloseWithErrCapture(&err, res.Body, "response readiness") - if res.StatusCode == p.expectedStatus { + if p.expectedStatusRangeStart <= res.StatusCode && res.StatusCode <= p.expectedStatusRangeEnd { return nil } - return fmt.Errorf("got no expected status code: %v, expected: %v", res.StatusCode, p.expectedStatus) + return fmt.Errorf("got status code: %v, expected code in range: [%v, %v]", res.StatusCode, p.expectedStatusRangeStart, p.expectedStatusRangeEnd) } // TCPReadinessProbe checks readiness by ensure a TCP connection can be established. diff --git a/integration/e2ecortex/services.go b/integration/e2ecortex/services.go index 6ab7c866800..1f57d30dfcb 100644 --- a/integration/e2ecortex/services.go +++ b/integration/e2ecortex/services.go @@ -48,7 +48,7 @@ func NewDistributorWithConfigFile(name, consulAddress, configFile string, flags "-ring.store": "consul", "-consul.hostname": consulAddress, }, flags))...), - e2e.NewHTTPReadinessProbe(httpPort, "/ready", 204), + e2e.NewHTTPReadinessProbe(httpPort, "/ready", 200, 299), httpPort, grpcPort, ) @@ -83,7 +83,7 @@ func NewQuerierWithConfigFile(name, consulAddress, configFile string, flags map[ "-querier.frontend-client.backoff-retries": "1", "-querier.worker-parallelism": "1", }, flags))...), - e2e.NewHTTPReadinessProbe(httpPort, "/ready", 204), + e2e.NewHTTPReadinessProbe(httpPort, "/ready", 200, 299), httpPort, grpcPort, ) @@ -117,7 +117,7 @@ func NewIngesterWithConfigFile(name, consulAddress, configFile string, flags map "-ring.store": "consul", "-consul.hostname": consulAddress, }, flags))...), - e2e.NewHTTPReadinessProbe(httpPort, "/ready", 204), + e2e.NewHTTPReadinessProbe(httpPort, "/ready", 200, 299), httpPort, grpcPort, ) @@ -143,7 +143,7 @@ func NewTableManagerWithConfigFile(name, configFile string, flags map[string]str "-target": "table-manager", "-log.level": "warn", }, flags))...), - e2e.NewHTTPReadinessProbe(httpPort, "/ready", 204), + e2e.NewHTTPReadinessProbe(httpPort, "/ready", 200, 299), httpPort, grpcPort, ) @@ -169,7 +169,7 @@ func NewQueryFrontendWithConfigFile(name, configFile string, flags map[string]st "-target": "query-frontend", "-log.level": "warn", }, flags))...), - e2e.NewHTTPReadinessProbe(httpPort, "/ready", 204), + e2e.NewHTTPReadinessProbe(httpPort, "/ready", 200, 299), httpPort, grpcPort, ) @@ -186,7 +186,7 @@ func NewSingleBinary(name string, flags map[string]string, image string, httpPor e2e.NewCommandWithoutEntrypoint("cortex", e2e.BuildArgs(e2e.MergeFlags(map[string]string{ "-log.level": "warn", }, flags))...), - e2e.NewHTTPReadinessProbe(httpPort, "/ready", 204), + e2e.NewHTTPReadinessProbe(httpPort, "/ready", 200, 299), httpPort, grpcPort, otherPorts..., @@ -205,7 +205,7 @@ func NewAlertmanager(name string, flags map[string]string, image string) *Cortex "-target": "alertmanager", "-log.level": "warn", }, flags))...), - e2e.NewHTTPReadinessProbe(httpPort, "/ready", 204), + e2e.NewHTTPReadinessProbe(httpPort, "/ready", 200, 299), httpPort, grpcPort, ) @@ -223,8 +223,7 @@ func NewRuler(name string, flags map[string]string, image string) *CortexService "-target": "ruler", "-log.level": "warn", }, flags))...), - // The alertmanager doesn't expose a readiness probe, so we just check if the / returns 200 - e2e.NewHTTPReadinessProbe(httpPort, "/", 200), + e2e.NewHTTPReadinessProbe(httpPort, "/ready", 200, 299), httpPort, grpcPort, ) diff --git a/pkg/cortex/cortex.go b/pkg/cortex/cortex.go index c2372fac852..3eb3762c0c2 100644 --- a/pkg/cortex/cortex.go +++ b/pkg/cortex/cortex.go @@ -401,9 +401,11 @@ func (t *Cortex) readyHandler(sm *services.Manager) http.HandlerFunc { if t.ingester != nil { if err := t.ingester.CheckReady(r.Context()); err != nil { http.Error(w, "Ingester not ready: "+err.Error(), http.StatusServiceUnavailable) + return } } - w.WriteHeader(http.StatusNoContent) + + http.Error(w, "ready", http.StatusOK) } }