Fixes a panic in the query-tee. (#4465)

* Fixes a panic in the query-tee.

This was happening because of a race.
I've change the test suite to cover the code and the race detector shows it up.

The panic:

```
panic: runtime error: index out of range [1] with length 1

goroutine 26118 [running]:
github.com/cortexproject/cortex/tools/querytee.(*ProxyEndpoint).executeBackendRequests(0xc0003160c0, 0xc0002c2600, 0xc0004c44e0)
	/__w/cortex/cortex/tools/querytee/proxy_endpoint.go:122 +0x5a5
created by github.com/cortexproject/cortex/tools/querytee.(*ProxyEndpoint).ServeHTTP
	/__w/cortex/cortex/tools/querytee/proxy_endpoint.go:57 +0x25b
```

Signed-off-by: Cyril Tovena <[email protected]>

* update changelog.

Signed-off-by: Cyril Tovena <[email protected]>

Author: cyriltovena

CHANGELOG.md

@@ -39,6 +39,7 @@
 * [ENHANCEMENT] Updated Prometheus to include changes from prometheus/prometheus#9083. Now whenever `/labels` API calls include matchers, blocks store is queried for `LabelNames` with matchers instead of `Series` calls which was inefficient. #4380
 * [ENHANCEMENT] Exemplars are now emitted for all gRPC calls and many operations tracked by histograms. #4462
 * [ENHANCEMENT] New options `-server.http-listen-network` and `-server.grpc-listen-network` allow binding as 'tcp4' or 'tcp6'. #4462
+* [BUGFIX] Fixes a panic in the query-tee when comparing result. #4465
 * [BUGFIX] Frontend: Fixes @ modifier functions (start/end) when splitting queries by time. #4464
 * [BUGFIX] Compactor: compactor will no longer try to compact blocks that are already marked for deletion. Previously compactor would consider blocks marked for deletion within `-compactor.deletion-delay / 2` period as eligible for compaction. #4328
 * [BUGFIX] HA Tracker: when cleaning up obsolete elected replicas from KV store, tracker didn't update number of cluster per user correctly. #4336

tools/querytee/proxy_endpoint.go

@@ -74,7 +74,10 @@ func (p *ProxyEndpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 func (p *ProxyEndpoint) executeBackendRequests(r *http.Request, resCh chan *backendResponse) {
 	responses := make([]*backendResponse, 0, len(p.backends))
 
-	wg := sync.WaitGroup{}
+	var (
+		wg  = sync.WaitGroup{}
+		mtx = sync.Mutex{}
+	)
 	wg.Add(len(p.backends))
 
 	for _, b := range p.backends {
@@ -105,7 +108,9 @@ func (p *ProxyEndpoint) executeBackendRequests(r *http.Request, resCh chan *back
 
 			// Keep track of the response if required.
 			if p.comparator != nil {
+				mtx.Lock()
 				responses = append(responses, res)
+				mtx.Unlock()
 			}
 
 			resCh <- res

tools/querytee/proxy_test.go

@@ -17,9 +17,13 @@ import (
 )
 
 var testRoutes = []Route{
-	{Path: "/api/v1/query", RouteName: "api_v1_query", Methods: []string{"GET"}, ResponseComparator: nil},
+	{Path: "/api/v1/query", RouteName: "api_v1_query", Methods: []string{"GET"}, ResponseComparator: &testComparator{}},
 }
 
+type testComparator struct{}
+
+func (testComparator) Compare(expected, actual []byte) error { return nil }
+
 func Test_NewProxy(t *testing.T) {
 	cfg := ProxyConfig{}
 
@@ -156,6 +160,10 @@ func Test_Proxy_RequestsForwarding(t *testing.T) {
 				BackendReadTimeout: time.Second,
 			}
 
+			if len(backendURLs) == 2 {
+				cfg.CompareResponses = true
+			}
+
 			p, err := NewProxy(cfg, log.NewNopLogger(), testRoutes, nil)
 			require.NoError(t, err)
 			require.NotNil(t, p)