Add cortex_query_samples_total metric

CHANGELOG.md

@@ -40,6 +40,7 @@
 * [ENHANCEMENT] Ruler: Add support for filtering by `state` and `health` field on Rules API. #6040
 * [ENHANCEMENT] Compactor: Split cleaner cycle for active and deleted tenants. #6112
 * [ENHANCEMENT] Compactor: Introduce cleaner visit marker. #6113
+* [ENHANCEMENT] Query Frontend: Add cortex_query_samples_total metric. #6142
 * [BUGFIX] Configsdb: Fix endline issue in db password. #5920
 * [BUGFIX] Ingester: Fix `user` and `type` labels for the `cortex_ingester_tsdb_head_samples_appended_total` TSDB metric. #5952
 * [BUGFIX] Querier: Enforce max query length check for `/api/v1/series` API even though `ignoreMaxQueryLength` is set to true. #6018

pkg/frontend/transport/handler.go

@@ -91,6 +91,7 @@ type Handler struct {
 	// Metrics.
 	querySeconds    *prometheus.CounterVec
 	querySeries     *prometheus.CounterVec
+	querySamples    *prometheus.CounterVec
 	queryChunkBytes *prometheus.CounterVec
 	queryDataBytes  *prometheus.CounterVec
 	rejectedQueries *prometheus.CounterVec
@@ -116,6 +117,11 @@ func NewHandler(cfg HandlerConfig, roundTripper http.RoundTripper, log log.Logge
 			Help: "Number of series fetched to execute a query.",
 		}, []string{"user"})
 
+		h.querySamples = promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
+			Name: "cortex_query_samples_total",
+			Help: "Number of samples fetched to execute a query.",
+		}, []string{"user"})
+
 		h.queryChunkBytes = promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
 			Name: "cortex_query_fetched_chunks_bytes_total",
 			Help: "Size of all chunks fetched to execute a query in bytes.",
@@ -137,6 +143,7 @@ func NewHandler(cfg HandlerConfig, roundTripper http.RoundTripper, log log.Logge
 		h.activeUsers = util.NewActiveUsersCleanupWithDefaultValues(func(user string) {
 			h.querySeconds.DeleteLabelValues(user)
 			h.querySeries.DeleteLabelValues(user)
+			h.querySamples.DeleteLabelValues(user)
 			h.queryChunkBytes.DeleteLabelValues(user)
 			h.queryDataBytes.DeleteLabelValues(user)
 			if err := util.DeleteMatchingLabels(h.rejectedQueries, map[string]string{"user": user}); err != nil {
@@ -305,6 +312,7 @@ func (f *Handler) reportQueryStats(r *http.Request, userID string, queryString u
 	// Track stats.
 	f.querySeconds.WithLabelValues(userID).Add(wallTime.Seconds())
 	f.querySeries.WithLabelValues(userID).Add(float64(numSeries))
+	f.querySamples.WithLabelValues(userID).Add(float64(numSamples))
 	f.queryChunkBytes.WithLabelValues(userID).Add(float64(numChunkBytes))
 	f.queryDataBytes.WithLabelValues(userID).Add(float64(numDataBytes))
 	f.activeUsers.UpdateUserTimestamp(userID, time.Now())

pkg/frontend/transport/handler_test.go

@@ -188,7 +188,7 @@ func TestHandler_ServeHTTP(t *testing.T) {
 		{
 			name:               "test handler with stats enabled",
 			cfg:                HandlerConfig{QueryStatsEnabled: true},
-			expectedMetrics:    3,
+			expectedMetrics:    4,
 			roundTripperFunc:   roundTripper,
 			expectedStatusCode: http.StatusOK,
 		},
@@ -202,7 +202,7 @@ func TestHandler_ServeHTTP(t *testing.T) {
 		{
 			name:            "test handler with reasonResponseTooLarge",
 			cfg:             HandlerConfig{QueryStatsEnabled: true},
-			expectedMetrics: 3,
+			expectedMetrics: 4,
 			roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
 				return &http.Response{
 					StatusCode: http.StatusRequestEntityTooLarge,
@@ -218,7 +218,7 @@ func TestHandler_ServeHTTP(t *testing.T) {
 		{
 			name:            "test handler with reasonTooManyRequests",
 			cfg:             HandlerConfig{QueryStatsEnabled: true},
-			expectedMetrics: 3,
+			expectedMetrics: 4,
 			roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
 				return &http.Response{
 					StatusCode: http.StatusTooManyRequests,
@@ -234,7 +234,7 @@ func TestHandler_ServeHTTP(t *testing.T) {
 		{
 			name:            "test handler with reasonTooManySamples",
 			cfg:             HandlerConfig{QueryStatsEnabled: true},
-			expectedMetrics: 3,
+			expectedMetrics: 4,
 			roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
 				return &http.Response{
 					StatusCode: http.StatusUnprocessableEntity,
@@ -250,7 +250,7 @@ func TestHandler_ServeHTTP(t *testing.T) {
 		{
 			name:            "test handler with reasonTooLongRange",
 			cfg:             HandlerConfig{QueryStatsEnabled: true},
-			expectedMetrics: 3,
+			expectedMetrics: 4,
 			roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
 				return &http.Response{
 					StatusCode: http.StatusUnprocessableEntity,
@@ -266,7 +266,7 @@ func TestHandler_ServeHTTP(t *testing.T) {
 		{
 			name:            "test handler with reasonSeriesFetched",
 			cfg:             HandlerConfig{QueryStatsEnabled: true},
-			expectedMetrics: 3,
+			expectedMetrics: 4,
 			roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
 				return &http.Response{
 					StatusCode: http.StatusUnprocessableEntity,
@@ -282,7 +282,7 @@ func TestHandler_ServeHTTP(t *testing.T) {
 		{
 			name:            "test handler with reasonChunksFetched",
 			cfg:             HandlerConfig{QueryStatsEnabled: true},
-			expectedMetrics: 3,
+			expectedMetrics: 4,
 			roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
 				return &http.Response{
 					StatusCode: http.StatusUnprocessableEntity,
@@ -298,7 +298,7 @@ func TestHandler_ServeHTTP(t *testing.T) {
 		{
 			name:            "test handler with reasonChunkBytesFetched",
 			cfg:             HandlerConfig{QueryStatsEnabled: true},
-			expectedMetrics: 3,
+			expectedMetrics: 4,
 			roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
 				return &http.Response{
 					StatusCode: http.StatusUnprocessableEntity,
@@ -314,7 +314,7 @@ func TestHandler_ServeHTTP(t *testing.T) {
 		{
 			name:            "test handler with reasonDataBytesFetched",
 			cfg:             HandlerConfig{QueryStatsEnabled: true},
-			expectedMetrics: 3,
+			expectedMetrics: 4,
 			roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
 				return &http.Response{
 					StatusCode: http.StatusUnprocessableEntity,
@@ -330,7 +330,7 @@ func TestHandler_ServeHTTP(t *testing.T) {
 		{
 			name:            "test handler with reasonSeriesLimitStoreGateway",
 			cfg:             HandlerConfig{QueryStatsEnabled: true},
-			expectedMetrics: 3,
+			expectedMetrics: 4,
 			roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
 				return &http.Response{
 					StatusCode: http.StatusUnprocessableEntity,
@@ -346,7 +346,7 @@ func TestHandler_ServeHTTP(t *testing.T) {
 		{
 			name:            "test handler with reasonChunksLimitStoreGateway",
 			cfg:             HandlerConfig{QueryStatsEnabled: true},
-			expectedMetrics: 3,
+			expectedMetrics: 4,
 			roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
 				return &http.Response{
 					StatusCode: http.StatusUnprocessableEntity,
@@ -362,7 +362,7 @@ func TestHandler_ServeHTTP(t *testing.T) {
 		{
 			name:            "test handler with reasonBytesLimitStoreGateway",
 			cfg:             HandlerConfig{QueryStatsEnabled: true},
-			expectedMetrics: 3,
+			expectedMetrics: 4,
 			roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
 				return &http.Response{
 					StatusCode: http.StatusUnprocessableEntity,
@@ -393,6 +393,7 @@ func TestHandler_ServeHTTP(t *testing.T) {
 				reg,
 				"cortex_query_seconds_total",
 				"cortex_query_fetched_series_total",
+				"cortex_query_samples_total",
 				"cortex_query_fetched_chunks_bytes_total",
 			)