Ensure that compaction continues for block with incomplete time-range

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Changelog
 
 ## master / unreleased
+[BUGFIX] Compactor: Fixes #4770 - an edge case in compactor where compaction stops when a tenant stops ingesting samples. #4771
 
 ## 1.13.0 in progress
 * [CHANGE] Changed default for `-ingester.min-ready-duration` from 1 minute to 15 seconds. #4539

pkg/compactor/shuffle_sharding_grouper.go

@@ -361,7 +361,6 @@ func groupBlocksByCompactableRanges(blocks []*metadata.Meta, ranges []int64) []b
 	// the same size still fits in the range. To do it, we consider valid a group only
 	// if it's before the most recent block or if it fully covers the range.
 	highestMinTime := blocks[len(blocks)-1].MinTime
-
 	for idx := 0; idx < len(groups); {
 		group := groups[idx]
 
@@ -377,6 +376,13 @@ func groupBlocksByCompactableRanges(blocks []*metadata.Meta, ranges []int64) []b
 			continue
 		}
 
+		// If the group's maxTime is after 1 block range, we can compact assuming that
+		// all the required blocks have already been uploaded.
+		if int64(ulid.Now()) > group.maxTime()+group.rangeLength() {
+			idx++
+			continue
+		}
+
 		// We hit into a group which would compact recent blocks prematurely,
 		// so we need to filter it out.
 

pkg/compactor/shuffle_sharding_grouper_test.go

@@ -18,82 +18,92 @@ import (
 )
 
 func TestShuffleShardingGrouper_Groups(t *testing.T) {
-	block1ulid := ulid.MustNew(1, nil)
-	block2ulid := ulid.MustNew(2, nil)
-	block3ulid := ulid.MustNew(3, nil)
-	block4ulid := ulid.MustNew(4, nil)
-	block5ulid := ulid.MustNew(5, nil)
-	block6ulid := ulid.MustNew(6, nil)
-	block7ulid := ulid.MustNew(7, nil)
-	block8ulid := ulid.MustNew(8, nil)
-	block9ulid := ulid.MustNew(9, nil)
-	block10ulid := ulid.MustNew(10, nil)
-	block11ulid := ulid.MustNew(11, nil)
-	block12ulid := ulid.MustNew(12, nil)
-	block13ulid := ulid.MustNew(13, nil)
-	block14ulid := ulid.MustNew(14, nil)
-	block15ulid := ulid.MustNew(15, nil)
+	block1hto2hExt1Ulid := ulid.MustNew(1, nil)
+	block3hto4hExt1Ulid := ulid.MustNew(2, nil)
+	block0hto1hExt1Ulid := ulid.MustNew(3, nil)
+	block2hto3hExt1Ulid := ulid.MustNew(4, nil)
+	block1hto2hExt2Ulid := ulid.MustNew(5, nil)
+	block0hto1hExt2Ulid := ulid.MustNew(6, nil)
+	block0to1hExt3Ulid := ulid.MustNew(7, nil)
+	block4hto6hExt2Ulid := ulid.MustNew(8, nil)
+	block6hto8hExt2Ulid := ulid.MustNew(9, nil)
+	block1hto2hExt1UlidCopy := ulid.MustNew(10, nil)
+	block0hto20hExt1Ulid := ulid.MustNew(11, nil)
+	block21hto40hExt1Ulid := ulid.MustNew(12, nil)
+	block21hto40hExt1UlidCopy := ulid.MustNew(13, nil)
+	block0hto45mExt1Ulid := ulid.MustNew(14, nil)
+	block0hto1h30mExt1Ulid := ulid.MustNew(15, nil)
+	blocklast1hExt1Ulid := ulid.MustNew(16, nil)
+	blocklast1hExt1UlidCopy := ulid.MustNew(17, nil)
 
 	blocks :=
 		map[ulid.ULID]*metadata.Meta{
-			block1ulid: {
-				BlockMeta: tsdb.BlockMeta{ULID: block1ulid, MinTime: 1 * time.Hour.Milliseconds(), MaxTime: 2 * time.Hour.Milliseconds()},
+			block1hto2hExt1Ulid: {
+				BlockMeta: tsdb.BlockMeta{ULID: block1hto2hExt1Ulid, MinTime: 1 * time.Hour.Milliseconds(), MaxTime: 2 * time.Hour.Milliseconds()},
 				Thanos:    metadata.Thanos{Labels: map[string]string{"external": "1"}},
 			},
-			block2ulid: {
-				BlockMeta: tsdb.BlockMeta{ULID: block2ulid, MinTime: 3 * time.Hour.Milliseconds(), MaxTime: 4 * time.Hour.Milliseconds()},
+			block1hto2hExt1UlidCopy: {
+				BlockMeta: tsdb.BlockMeta{ULID: block1hto2hExt1UlidCopy, MinTime: 1 * time.Hour.Milliseconds(), MaxTime: 2 * time.Hour.Milliseconds()},
 				Thanos:    metadata.Thanos{Labels: map[string]string{"external": "1"}},
 			},
-			block3ulid: {
-				BlockMeta: tsdb.BlockMeta{ULID: block3ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()},
+			block3hto4hExt1Ulid: {
+				BlockMeta: tsdb.BlockMeta{ULID: block3hto4hExt1Ulid, MinTime: 3 * time.Hour.Milliseconds(), MaxTime: 4 * time.Hour.Milliseconds()},
 				Thanos:    metadata.Thanos{Labels: map[string]string{"external": "1"}},
 			},
-			block4ulid: {
-				BlockMeta: tsdb.BlockMeta{ULID: block4ulid, MinTime: 2 * time.Hour.Milliseconds(), MaxTime: 3 * time.Hour.Milliseconds()},
+			block0hto1hExt1Ulid: {
+				BlockMeta: tsdb.BlockMeta{ULID: block0hto1hExt1Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()},
 				Thanos:    metadata.Thanos{Labels: map[string]string{"external": "1"}},
 			},
-			block5ulid: {
-				BlockMeta: tsdb.BlockMeta{ULID: block5ulid, MinTime: 1 * time.Hour.Milliseconds(), MaxTime: 2 * time.Hour.Milliseconds()},
-				Thanos:    metadata.Thanos{Labels: map[string]string{"external": "2"}},
+			block2hto3hExt1Ulid: {
+				BlockMeta: tsdb.BlockMeta{ULID: block2hto3hExt1Ulid, MinTime: 2 * time.Hour.Milliseconds(), MaxTime: 3 * time.Hour.Milliseconds()},
+				Thanos:    metadata.Thanos{Labels: map[string]string{"external": "1"}},
 			},
-			block6ulid: {
-				BlockMeta: tsdb.BlockMeta{ULID: block6ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()},
+			block1hto2hExt2Ulid: {
+				BlockMeta: tsdb.BlockMeta{ULID: block1hto2hExt2Ulid, MinTime: 1 * time.Hour.Milliseconds(), MaxTime: 2 * time.Hour.Milliseconds()},
 				Thanos:    metadata.Thanos{Labels: map[string]string{"external": "2"}},
 			},
-			block7ulid: {
-				BlockMeta: tsdb.BlockMeta{ULID: block7ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()},
-				Thanos:    metadata.Thanos{Labels: map[string]string{"external": "1"}},
-			},
-			block8ulid: {
-				BlockMeta: tsdb.BlockMeta{ULID: block8ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()},
+			block0hto1hExt2Ulid: {
+				BlockMeta: tsdb.BlockMeta{ULID: block0hto1hExt2Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()},
 				Thanos:    metadata.Thanos{Labels: map[string]string{"external": "2"}},
 			},
-			block9ulid: {
-				BlockMeta: tsdb.BlockMeta{ULID: block9ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()},
+			block0to1hExt3Ulid: {
+				BlockMeta: tsdb.BlockMeta{ULID: block0to1hExt3Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()},
 				Thanos:    metadata.Thanos{Labels: map[string]string{"external": "3"}},
 			},
-			block10ulid: {
-				BlockMeta: tsdb.BlockMeta{ULID: block10ulid, MinTime: 4 * time.Hour.Milliseconds(), MaxTime: 6 * time.Hour.Milliseconds()},
+			block4hto6hExt2Ulid: {
+				BlockMeta: tsdb.BlockMeta{ULID: block4hto6hExt2Ulid, MinTime: 4 * time.Hour.Milliseconds(), MaxTime: 6 * time.Hour.Milliseconds()},
 				Thanos:    metadata.Thanos{Labels: map[string]string{"external": "2"}},
 			},
-			block11ulid: {
-				BlockMeta: tsdb.BlockMeta{ULID: block11ulid, MinTime: 6 * time.Hour.Milliseconds(), MaxTime: 8 * time.Hour.Milliseconds()},
+			block6hto8hExt2Ulid: {
+				BlockMeta: tsdb.BlockMeta{ULID: block6hto8hExt2Ulid, MinTime: 6 * time.Hour.Milliseconds(), MaxTime: 8 * time.Hour.Milliseconds()},
 				Thanos:    metadata.Thanos{Labels: map[string]string{"external": "2"}},
 			},
-			block12ulid: {
-				BlockMeta: tsdb.BlockMeta{ULID: block12ulid, MinTime: 1 * time.Hour.Milliseconds(), MaxTime: 2 * time.Hour.Milliseconds()},
+			block0hto20hExt1Ulid: {
+				BlockMeta: tsdb.BlockMeta{ULID: block0hto20hExt1Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 20 * time.Hour.Milliseconds()},
+				Thanos:    metadata.Thanos{Labels: map[string]string{"external": "1"}},
+			},
+			block21hto40hExt1Ulid: {
+				BlockMeta: tsdb.BlockMeta{ULID: block21hto40hExt1Ulid, MinTime: 21 * time.Hour.Milliseconds(), MaxTime: 40 * time.Hour.Milliseconds()},
 				Thanos:    metadata.Thanos{Labels: map[string]string{"external": "1"}},
 			},
-			block13ulid: {
-				BlockMeta: tsdb.BlockMeta{ULID: block13ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 20 * time.Hour.Milliseconds()},
+			block21hto40hExt1UlidCopy: {
+				BlockMeta: tsdb.BlockMeta{ULID: block21hto40hExt1UlidCopy, MinTime: 21 * time.Hour.Milliseconds(), MaxTime: 40 * time.Hour.Milliseconds()},
 				Thanos:    metadata.Thanos{Labels: map[string]string{"external": "1"}},
 			},
-			block14ulid: {
-				BlockMeta: tsdb.BlockMeta{ULID: block14ulid, MinTime: 21 * time.Hour.Milliseconds(), MaxTime: 40 * time.Hour.Milliseconds()},
+			block0hto45mExt1Ulid: {
+				BlockMeta: tsdb.BlockMeta{ULID: block0hto45mExt1Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 45 * time.Minute.Milliseconds()},
 				Thanos:    metadata.Thanos{Labels: map[string]string{"external": "1"}},
 			},
-			block15ulid: {
-				BlockMeta: tsdb.BlockMeta{ULID: block15ulid, MinTime: 21 * time.Hour.Milliseconds(), MaxTime: 40 * time.Hour.Milliseconds()},
+			block0hto1h30mExt1Ulid: {
+				BlockMeta: tsdb.BlockMeta{ULID: block0hto1h30mExt1Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1*time.Hour.Milliseconds() + 30*time.Minute.Milliseconds()},
+				Thanos:    metadata.Thanos{Labels: map[string]string{"external": "1"}},
+			},
+			blocklast1hExt1Ulid: {
+				BlockMeta: tsdb.BlockMeta{ULID: blocklast1hExt1Ulid, MinTime: int64(ulid.Now()) - 1*time.Hour.Milliseconds(), MaxTime: int64(ulid.Now())},
+				Thanos:    metadata.Thanos{Labels: map[string]string{"external": "1"}},
+			},
+			blocklast1hExt1UlidCopy: {
+				BlockMeta: tsdb.BlockMeta{ULID: blocklast1hExt1UlidCopy, MinTime: int64(ulid.Now()) - 1*time.Hour.Milliseconds(), MaxTime: int64(ulid.Now())},
 				Thanos:    metadata.Thanos{Labels: map[string]string{"external": "1"}},
 			},
 		}
@@ -105,38 +115,52 @@ func TestShuffleShardingGrouper_Groups(t *testing.T) {
 	}{
 		"test basic grouping": {
 			ranges: []time.Duration{2 * time.Hour, 4 * time.Hour},
-			blocks: map[ulid.ULID]*metadata.Meta{block1ulid: blocks[block1ulid], block2ulid: blocks[block2ulid], block3ulid: blocks[block3ulid], block4ulid: blocks[block4ulid], block5ulid: blocks[block5ulid], block6ulid: blocks[block6ulid]},
+			blocks: map[ulid.ULID]*metadata.Meta{block1hto2hExt1Ulid: blocks[block1hto2hExt1Ulid], block3hto4hExt1Ulid: blocks[block3hto4hExt1Ulid], block0hto1hExt1Ulid: blocks[block0hto1hExt1Ulid], block2hto3hExt1Ulid: blocks[block2hto3hExt1Ulid], block1hto2hExt2Ulid: blocks[block1hto2hExt2Ulid], block0hto1hExt2Ulid: blocks[block0hto1hExt2Ulid]},
 			expected: [][]ulid.ULID{
-				{block5ulid, block6ulid},
-				{block1ulid, block3ulid},
-				{block2ulid, block4ulid},
+				{block1hto2hExt2Ulid, block0hto1hExt2Ulid},
+				{block1hto2hExt1Ulid, block0hto1hExt1Ulid},
+				{block3hto4hExt1Ulid, block2hto3hExt1Ulid},
 			},
 		},
 		"test no compaction": {
 			ranges:   []time.Duration{2 * time.Hour, 4 * time.Hour},
-			blocks:   map[ulid.ULID]*metadata.Meta{block7ulid: blocks[block7ulid], block8ulid: blocks[block8ulid], block9ulid: blocks[block9ulid]},
+			blocks:   map[ulid.ULID]*metadata.Meta{block0hto1hExt1Ulid: blocks[block0hto1hExt1Ulid], block0hto1hExt2Ulid: blocks[block0hto1hExt2Ulid], block0to1hExt3Ulid: blocks[block0to1hExt3Ulid]},
 			expected: [][]ulid.ULID{},
 		},
 		"test smallest range first": {
 			ranges: []time.Duration{2 * time.Hour, 4 * time.Hour},
-			blocks: map[ulid.ULID]*metadata.Meta{block1ulid: blocks[block1ulid], block2ulid: blocks[block2ulid], block3ulid: blocks[block3ulid], block4ulid: blocks[block4ulid], block10ulid: blocks[block10ulid], block11ulid: blocks[block11ulid]},
+			blocks: map[ulid.ULID]*metadata.Meta{block1hto2hExt1Ulid: blocks[block1hto2hExt1Ulid], block3hto4hExt1Ulid: blocks[block3hto4hExt1Ulid], block0hto1hExt1Ulid: blocks[block0hto1hExt1Ulid], block2hto3hExt1Ulid: blocks[block2hto3hExt1Ulid], block4hto6hExt2Ulid: blocks[block4hto6hExt2Ulid], block6hto8hExt2Ulid: blocks[block6hto8hExt2Ulid]},
 			expected: [][]ulid.ULID{
-				{block1ulid, block3ulid},
-				{block2ulid, block4ulid},
-				{block10ulid, block11ulid},
+				{block1hto2hExt1Ulid, block0hto1hExt1Ulid},
+				{block3hto4hExt1Ulid, block2hto3hExt1Ulid},
+				{block4hto6hExt2Ulid, block6hto8hExt2Ulid},
 			},
 		},
 		"test oldest min time first": {
 			ranges: []time.Duration{2 * time.Hour, 4 * time.Hour},
-			blocks: map[ulid.ULID]*metadata.Meta{block1ulid: blocks[block1ulid], block2ulid: blocks[block2ulid], block3ulid: blocks[block3ulid], block4ulid: blocks[block4ulid], block12ulid: blocks[block12ulid]},
+			blocks: map[ulid.ULID]*metadata.Meta{block1hto2hExt1Ulid: blocks[block1hto2hExt1Ulid], block3hto4hExt1Ulid: blocks[block3hto4hExt1Ulid], block0hto1hExt1Ulid: blocks[block0hto1hExt1Ulid], block2hto3hExt1Ulid: blocks[block2hto3hExt1Ulid], block1hto2hExt1UlidCopy: blocks[block1hto2hExt1UlidCopy]},
 			expected: [][]ulid.ULID{
-				{block1ulid, block3ulid, block12ulid},
-				{block2ulid, block4ulid},
+				{block1hto2hExt1Ulid, block0hto1hExt1Ulid, block1hto2hExt1UlidCopy},
+				{block3hto4hExt1Ulid, block2hto3hExt1Ulid},
 			},
 		},
 		"test overlapping blocks": {
-			ranges:   []time.Duration{20 * time.Hour, 40 * time.Hour},
-			blocks:   map[ulid.ULID]*metadata.Meta{block13ulid: blocks[block13ulid], block14ulid: blocks[block14ulid], block15ulid: blocks[block15ulid]},
+			ranges: []time.Duration{20 * time.Hour, 40 * time.Hour},
+			blocks: map[ulid.ULID]*metadata.Meta{block0hto20hExt1Ulid: blocks[block0hto20hExt1Ulid], block21hto40hExt1Ulid: blocks[block21hto40hExt1Ulid], block21hto40hExt1UlidCopy: blocks[block21hto40hExt1UlidCopy]},
+			expected: [][]ulid.ULID{
+				{block21hto40hExt1Ulid, block21hto40hExt1UlidCopy},
+			},
+		},
+		"test imperfect maxTime blocks": {
+			ranges: []time.Duration{2 * time.Hour},
+			blocks: map[ulid.ULID]*metadata.Meta{block0hto1h30mExt1Ulid: blocks[block0hto1h30mExt1Ulid], block0hto45mExt1Ulid: blocks[block0hto45mExt1Ulid]},
+			expected: [][]ulid.ULID{
+				{block0hto45mExt1Ulid, block0hto1h30mExt1Ulid},
+			},
+		},
+		"test prematurely created blocks": {
+			ranges:   []time.Duration{2 * time.Hour},
+			blocks:   map[ulid.ULID]*metadata.Meta{blocklast1hExt1UlidCopy: blocks[blocklast1hExt1UlidCopy], blocklast1hExt1Ulid: blocks[blocklast1hExt1Ulid]},
 			expected: [][]ulid.ULID{},
 		},
 	}
@@ -374,21 +398,6 @@ func TestGroupBlocksByCompactableRanges(t *testing.T) {
 				}},
 			},
 		},
-		"a range containing the most recent block shouldn't be prematurely compacted if doesn't cover the full range": {
-			ranges: []int64{10, 20, 40},
-			blocks: []*metadata.Meta{
-				{BlockMeta: tsdb.BlockMeta{MinTime: 5, MaxTime: 8}},
-				{BlockMeta: tsdb.BlockMeta{MinTime: 7, MaxTime: 9}},
-				{BlockMeta: tsdb.BlockMeta{MinTime: 10, MaxTime: 12}},
-				{BlockMeta: tsdb.BlockMeta{MinTime: 13, MaxTime: 15}},
-			},
-			expected: []blocksGroup{
-				{rangeStart: 0, rangeEnd: 10, blocks: []*metadata.Meta{
-					{BlockMeta: tsdb.BlockMeta{MinTime: 5, MaxTime: 8}},
-					{BlockMeta: tsdb.BlockMeta{MinTime: 7, MaxTime: 9}},
-				}},
-			},
-		},
 	}
 
 	for testName, testData := range tests {