[store] Add response hints to label APIs (thanos-io#3437)

* Add hints support to labels API responses

Signed-off-by: Goutham Veeramachaneni <[email protected]>

* Move to dedicated hints types and add tests

Signed-off-by: Goutham Veeramachaneni <[email protected]>

* Add external labels to the values returned in API

Signed-off-by: Goutham Veeramachaneni <[email protected]>

* Add changelog entries

Fixes thanos-io#2984

Signed-off-by: Goutham Veeramachaneni <[email protected]>
Signed-off-by: Oghenebrume50 <[email protected]>

Author: gouthamve

CHANGELOG.md

@@ -31,13 +31,15 @@ We use _breaking :warning:_ to mark changes that are not backward compatible (re
 - [#3423](https://github.com/thanos-io/thanos/pull/3423) UI: Thanos Configurations file page: Display all available configuration for each component on the UI.
 - [#3431](https://github.com/thanos-io/thanos/pull/3431) Store: Added experimental support to lazy load index-headers at query time. When enabled via `--store.enable-index-header-lazy-reader` flag, the store-gateway will load into memory an index-header only once it's required at query time. Index-header will be automatically released after `--store.index-header-lazy-reader-idle-timeout` of inactivity.
   - This, generally, reduces baseline memory usage of store when inactive, as well as a total number of mapped files (which is limited to 64k in some systems.
+- [#3437](https://github.com/thanos-io/thanos/pull/3437) StoreAPI: Added `hints` field to `LabelNamesResponse` and `LabelValuesResponse`. Hints in an opaque data structure that can be used to carry additional information from the store and its content is implementation specific.
 
 ### Fixed
 
 - [#3257](https://github.com/thanos-io/thanos/pull/3257) Ruler: Prevent Ruler from crashing when using default DNS to lookup hosts that results in "No such hosts" errors.
 - [#3331](https://github.com/thanos-io/thanos/pull/3331) Disable Azure blob exception logging
 - [#3341](https://github.com/thanos-io/thanos/pull/3341) Disable Azure blob syslog exception logging
 - [#3414](https://github.com/thanos-io/thanos/pull/3414) Set CORS for Query Frontend
+- [#3437](https://github.com/thanos-io/thanos/pull/3437) Add external labels to Labels APIs.
 
 ### Changed
 

pkg/store/bucket.go

@@ -1066,6 +1066,8 @@ func chunksSize(chks []storepb.AggrChunk) (size int) {
 
 // LabelNames implements the storepb.StoreServer interface.
 func (s *BucketStore) LabelNames(ctx context.Context, req *storepb.LabelNamesRequest) (*storepb.LabelNamesResponse, error) {
+	resHints := &hintspb.LabelNamesResponseHints{}
+
 	g, gctx := errgroup.WithContext(ctx)
 
 	s.mtx.RLock()
@@ -1077,7 +1079,12 @@ func (s *BucketStore) LabelNames(ctx context.Context, req *storepb.LabelNamesReq
 		if !b.overlapsClosedInterval(req.Start, req.End) {
 			continue
 		}
+
+		resHints.AddQueriedBlock(b.meta.ULID)
+
 		indexr := b.indexReader(gctx)
+		extLabels := b.meta.Thanos.Labels
+
 		g.Go(func() error {
 			defer runutil.CloseWithLogOnErr(s.logger, indexr, "label names")
 
@@ -1087,7 +1094,17 @@ func (s *BucketStore) LabelNames(ctx context.Context, req *storepb.LabelNamesReq
 				return errors.Wrap(err, "label names")
 			}
 
+			// Add  a set for the external labels as well.
+			// We're not adding them directly to res because there could be duplicates.
+			extRes := make([]string, 0, len(extLabels))
+			for lName := range extLabels {
+				extRes = append(extRes, lName)
+			}
+
 			sort.Strings(res)
+			sort.Strings(extRes)
+
+			res = strutil.MergeSlices(res, extRes)
 
 			mtx.Lock()
 			sets = append(sets, res)
@@ -1102,13 +1119,22 @@ func (s *BucketStore) LabelNames(ctx context.Context, req *storepb.LabelNamesReq
 	if err := g.Wait(); err != nil {
 		return nil, status.Error(codes.Internal, err.Error())
 	}
+
+	anyHints, err := types.MarshalAny(resHints)
+	if err != nil {
+		return nil, status.Error(codes.Unknown, errors.Wrap(err, "marshal label names response hints").Error())
+	}
+
 	return &storepb.LabelNamesResponse{
 		Names: strutil.MergeSlices(sets...),
+		Hints: anyHints,
 	}, nil
 }
 
 // LabelValues implements the storepb.StoreServer interface.
 func (s *BucketStore) LabelValues(ctx context.Context, req *storepb.LabelValuesRequest) (*storepb.LabelValuesResponse, error) {
+	resHints := &hintspb.LabelValuesResponseHints{}
+
 	g, gctx := errgroup.WithContext(ctx)
 
 	s.mtx.RLock()
@@ -1120,7 +1146,12 @@ func (s *BucketStore) LabelValues(ctx context.Context, req *storepb.LabelValuesR
 		if !b.overlapsClosedInterval(req.Start, req.End) {
 			continue
 		}
+
+		resHints.AddQueriedBlock(b.meta.ULID)
+
 		indexr := b.indexReader(gctx)
+		extLabels := b.meta.Thanos.Labels
+
 		g.Go(func() error {
 			defer runutil.CloseWithLogOnErr(s.logger, indexr, "label values")
 
@@ -1130,6 +1161,11 @@ func (s *BucketStore) LabelValues(ctx context.Context, req *storepb.LabelValuesR
 				return errors.Wrap(err, "index header label values")
 			}
 
+			// Add the external label value as well.
+			if extLabelValue, ok := extLabels[req.Label]; ok {
+				res = strutil.MergeSlices(res, []string{extLabelValue})
+			}
+
 			mtx.Lock()
 			sets = append(sets, res)
 			mtx.Unlock()
@@ -1143,8 +1179,15 @@ func (s *BucketStore) LabelValues(ctx context.Context, req *storepb.LabelValuesR
 	if err := g.Wait(); err != nil {
 		return nil, status.Error(codes.Aborted, err.Error())
 	}
+
+	anyHints, err := types.MarshalAny(resHints)
+	if err != nil {
+		return nil, status.Error(codes.Unknown, errors.Wrap(err, "marshal label values response hints").Error())
+	}
+
 	return &storepb.LabelValuesResponse{
 		Values: strutil.MergeSlices(sets...),
+		Hints:  anyHints,
 	}, nil
 }
 

pkg/store/bucket_e2e_test.go

@@ -629,7 +629,8 @@ func TestBucketStore_LabelNames_e2e(t *testing.T) {
 			End:   timestamp.FromTime(maxTime),
 		})
 		testutil.Ok(t, err)
-		testutil.Equals(t, []string{"a", "b", "c"}, vals.Names)
+		// ext2 is added by the prepareStoreWithTestBlocks function.
+		testutil.Equals(t, []string{"a", "b", "c", "ext1", "ext2"}, vals.Names)
 
 		// Outside the time range.
 		vals, err = s.store.LabelNames(ctx, &storepb.LabelNamesRequest{