Skip to content

Commit f7a9d27

Browse files
harry671003harry671003
authored
Ensure that compaction continues for block with incomplete time-range (#4771)
* Ensure that compaction continues for block with incomplete time-range Signed-off-by: 🌲 Harry 🌊 John 🏔 <[email protected]> * Ensure that compaction continues for block with incomplete time-range Signed-off-by: 🌲 Harry 🌊 John 🏔 <[email protected]> * Ensure that compaction continues for block with incomplete time-range Signed-off-by: 🌲 Harry 🌊 John 🏔 <[email protected]> * Ensure that compaction continues for block with incomplete time-range Signed-off-by: 🌲 Harry 🌊 John 🏔 <[email protected]>
1 parent eb84489 commit f7a9d27

File tree

3 files changed

+94
-78
lines changed

3 files changed

+94
-78
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# Changelog
22

33
## master / unreleased
4+
[BUGFIX] Compactor: Fixes #4770 - an edge case in compactor where compaction stops when a tenant stops ingesting samples. #4771
45

56
## 1.13.0 in progress
67
* [CHANGE] Changed default for `-ingester.min-ready-duration` from 1 minute to 15 seconds. #4539

‎pkg/compactor/shuffle_sharding_grouper.go

+7-1
Original file line numberDiff line numberDiff line change
@@ -361,7 +361,6 @@ func groupBlocksByCompactableRanges(blocks []*metadata.Meta, ranges []int64) []b
361361
// the same size still fits in the range. To do it, we consider valid a group only
362362
// if it's before the most recent block or if it fully covers the range.
363363
highestMinTime := blocks[len(blocks)-1].MinTime
364-
365364
for idx := 0; idx < len(groups); {
366365
group := groups[idx]
367366

@@ -377,6 +376,13 @@ func groupBlocksByCompactableRanges(blocks []*metadata.Meta, ranges []int64) []b
377376
continue
378377
}
379378

379+
// If the group's maxTime is after 1 block range, we can compact assuming that
380+
// all the required blocks have already been uploaded.
381+
if int64(ulid.Now()) > group.maxTime()+group.rangeLength() {
382+
idx++
383+
continue
384+
}
385+
380386
// We hit into a group which would compact recent blocks prematurely,
381387
// so we need to filter it out.
382388

‎pkg/compactor/shuffle_sharding_grouper_test.go

+86-77
Original file line numberDiff line numberDiff line change
@@ -18,82 +18,92 @@ import (
1818
)
1919

2020
func TestShuffleShardingGrouper_Groups(t *testing.T) {
21-
block1ulid := ulid.MustNew(1, nil)
22-
block2ulid := ulid.MustNew(2, nil)
23-
block3ulid := ulid.MustNew(3, nil)
24-
block4ulid := ulid.MustNew(4, nil)
25-
block5ulid := ulid.MustNew(5, nil)
26-
block6ulid := ulid.MustNew(6, nil)
27-
block7ulid := ulid.MustNew(7, nil)
28-
block8ulid := ulid.MustNew(8, nil)
29-
block9ulid := ulid.MustNew(9, nil)
30-
block10ulid := ulid.MustNew(10, nil)
31-
block11ulid := ulid.MustNew(11, nil)
32-
block12ulid := ulid.MustNew(12, nil)
33-
block13ulid := ulid.MustNew(13, nil)
34-
block14ulid := ulid.MustNew(14, nil)
35-
block15ulid := ulid.MustNew(15, nil)
21+
block1hto2hExt1Ulid := ulid.MustNew(1, nil)
22+
block3hto4hExt1Ulid := ulid.MustNew(2, nil)
23+
block0hto1hExt1Ulid := ulid.MustNew(3, nil)
24+
block2hto3hExt1Ulid := ulid.MustNew(4, nil)
25+
block1hto2hExt2Ulid := ulid.MustNew(5, nil)
26+
block0hto1hExt2Ulid := ulid.MustNew(6, nil)
27+
block0to1hExt3Ulid := ulid.MustNew(7, nil)
28+
block4hto6hExt2Ulid := ulid.MustNew(8, nil)
29+
block6hto8hExt2Ulid := ulid.MustNew(9, nil)
30+
block1hto2hExt1UlidCopy := ulid.MustNew(10, nil)
31+
block0hto20hExt1Ulid := ulid.MustNew(11, nil)
32+
block21hto40hExt1Ulid := ulid.MustNew(12, nil)
33+
block21hto40hExt1UlidCopy := ulid.MustNew(13, nil)
34+
block0hto45mExt1Ulid := ulid.MustNew(14, nil)
35+
block0hto1h30mExt1Ulid := ulid.MustNew(15, nil)
36+
blocklast1hExt1Ulid := ulid.MustNew(16, nil)
37+
blocklast1hExt1UlidCopy := ulid.MustNew(17, nil)
3638

3739
blocks :=
3840
map[ulid.ULID]*metadata.Meta{
39-
block1ulid: {
40-
BlockMeta: tsdb.BlockMeta{ULID: block1ulid, MinTime: 1 * time.Hour.Milliseconds(), MaxTime: 2 * time.Hour.Milliseconds()},
41+
block1hto2hExt1Ulid: {
42+
BlockMeta: tsdb.BlockMeta{ULID: block1hto2hExt1Ulid, MinTime: 1 * time.Hour.Milliseconds(), MaxTime: 2 * time.Hour.Milliseconds()},
4143
Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}},
4244
},
43-
block2ulid: {
44-
BlockMeta: tsdb.BlockMeta{ULID: block2ulid, MinTime: 3 * time.Hour.Milliseconds(), MaxTime: 4 * time.Hour.Milliseconds()},
45+
block1hto2hExt1UlidCopy: {
46+
BlockMeta: tsdb.BlockMeta{ULID: block1hto2hExt1UlidCopy, MinTime: 1 * time.Hour.Milliseconds(), MaxTime: 2 * time.Hour.Milliseconds()},
4547
Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}},
4648
},
47-
block3ulid: {
48-
BlockMeta: tsdb.BlockMeta{ULID: block3ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()},
49+
block3hto4hExt1Ulid: {
50+
BlockMeta: tsdb.BlockMeta{ULID: block3hto4hExt1Ulid, MinTime: 3 * time.Hour.Milliseconds(), MaxTime: 4 * time.Hour.Milliseconds()},
4951
Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}},
5052
},
51-
block4ulid: {
52-
BlockMeta: tsdb.BlockMeta{ULID: block4ulid, MinTime: 2 * time.Hour.Milliseconds(), MaxTime: 3 * time.Hour.Milliseconds()},
53+
block0hto1hExt1Ulid: {
54+
BlockMeta: tsdb.BlockMeta{ULID: block0hto1hExt1Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()},
5355
Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}},
5456
},
55-
block5ulid: {
56-
BlockMeta: tsdb.BlockMeta{ULID: block5ulid, MinTime: 1 * time.Hour.Milliseconds(), MaxTime: 2 * time.Hour.Milliseconds()},
57-
Thanos: metadata.Thanos{Labels: map[string]string{"external": "2"}},
57+
block2hto3hExt1Ulid: {
58+
BlockMeta: tsdb.BlockMeta{ULID: block2hto3hExt1Ulid, MinTime: 2 * time.Hour.Milliseconds(), MaxTime: 3 * time.Hour.Milliseconds()},
59+
Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}},
5860
},
59-
block6ulid: {
60-
BlockMeta: tsdb.BlockMeta{ULID: block6ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()},
61+
block1hto2hExt2Ulid: {
62+
BlockMeta: tsdb.BlockMeta{ULID: block1hto2hExt2Ulid, MinTime: 1 * time.Hour.Milliseconds(), MaxTime: 2 * time.Hour.Milliseconds()},
6163
Thanos: metadata.Thanos{Labels: map[string]string{"external": "2"}},
6264
},
63-
block7ulid: {
64-
BlockMeta: tsdb.BlockMeta{ULID: block7ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()},
65-
Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}},
66-
},
67-
block8ulid: {
68-
BlockMeta: tsdb.BlockMeta{ULID: block8ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()},
65+
block0hto1hExt2Ulid: {
66+
BlockMeta: tsdb.BlockMeta{ULID: block0hto1hExt2Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()},
6967
Thanos: metadata.Thanos{Labels: map[string]string{"external": "2"}},
7068
},
71-
block9ulid: {
72-
BlockMeta: tsdb.BlockMeta{ULID: block9ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()},
69+
block0to1hExt3Ulid: {
70+
BlockMeta: tsdb.BlockMeta{ULID: block0to1hExt3Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()},
7371
Thanos: metadata.Thanos{Labels: map[string]string{"external": "3"}},
7472
},
75-
block10ulid: {
76-
BlockMeta: tsdb.BlockMeta{ULID: block10ulid, MinTime: 4 * time.Hour.Milliseconds(), MaxTime: 6 * time.Hour.Milliseconds()},
73+
block4hto6hExt2Ulid: {
74+
BlockMeta: tsdb.BlockMeta{ULID: block4hto6hExt2Ulid, MinTime: 4 * time.Hour.Milliseconds(), MaxTime: 6 * time.Hour.Milliseconds()},
7775
Thanos: metadata.Thanos{Labels: map[string]string{"external": "2"}},
7876
},
79-
block11ulid: {
80-
BlockMeta: tsdb.BlockMeta{ULID: block11ulid, MinTime: 6 * time.Hour.Milliseconds(), MaxTime: 8 * time.Hour.Milliseconds()},
77+
block6hto8hExt2Ulid: {
78+
BlockMeta: tsdb.BlockMeta{ULID: block6hto8hExt2Ulid, MinTime: 6 * time.Hour.Milliseconds(), MaxTime: 8 * time.Hour.Milliseconds()},
8179
Thanos: metadata.Thanos{Labels: map[string]string{"external": "2"}},
8280
},
83-
block12ulid: {
84-
BlockMeta: tsdb.BlockMeta{ULID: block12ulid, MinTime: 1 * time.Hour.Milliseconds(), MaxTime: 2 * time.Hour.Milliseconds()},
81+
block0hto20hExt1Ulid: {
82+
BlockMeta: tsdb.BlockMeta{ULID: block0hto20hExt1Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 20 * time.Hour.Milliseconds()},
83+
Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}},
84+
},
85+
block21hto40hExt1Ulid: {
86+
BlockMeta: tsdb.BlockMeta{ULID: block21hto40hExt1Ulid, MinTime: 21 * time.Hour.Milliseconds(), MaxTime: 40 * time.Hour.Milliseconds()},
8587
Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}},
8688
},
87-
block13ulid: {
88-
BlockMeta: tsdb.BlockMeta{ULID: block13ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 20 * time.Hour.Milliseconds()},
89+
block21hto40hExt1UlidCopy: {
90+
BlockMeta: tsdb.BlockMeta{ULID: block21hto40hExt1UlidCopy, MinTime: 21 * time.Hour.Milliseconds(), MaxTime: 40 * time.Hour.Milliseconds()},
8991
Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}},
9092
},
91-
block14ulid: {
92-
BlockMeta: tsdb.BlockMeta{ULID: block14ulid, MinTime: 21 * time.Hour.Milliseconds(), MaxTime: 40 * time.Hour.Milliseconds()},
93+
block0hto45mExt1Ulid: {
94+
BlockMeta: tsdb.BlockMeta{ULID: block0hto45mExt1Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 45 * time.Minute.Milliseconds()},
9395
Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}},
9496
},
95-
block15ulid: {
96-
BlockMeta: tsdb.BlockMeta{ULID: block15ulid, MinTime: 21 * time.Hour.Milliseconds(), MaxTime: 40 * time.Hour.Milliseconds()},
97+
block0hto1h30mExt1Ulid: {
98+
BlockMeta: tsdb.BlockMeta{ULID: block0hto1h30mExt1Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1*time.Hour.Milliseconds() + 30*time.Minute.Milliseconds()},
99+
Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}},
100+
},
101+
blocklast1hExt1Ulid: {
102+
BlockMeta: tsdb.BlockMeta{ULID: blocklast1hExt1Ulid, MinTime: int64(ulid.Now()) - 1*time.Hour.Milliseconds(), MaxTime: int64(ulid.Now())},
103+
Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}},
104+
},
105+
blocklast1hExt1UlidCopy: {
106+
BlockMeta: tsdb.BlockMeta{ULID: blocklast1hExt1UlidCopy, MinTime: int64(ulid.Now()) - 1*time.Hour.Milliseconds(), MaxTime: int64(ulid.Now())},
97107
Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}},
98108
},
99109
}
@@ -105,38 +115,52 @@ func TestShuffleShardingGrouper_Groups(t *testing.T) {
105115
}{
106116
"test basic grouping": {
107117
ranges: []time.Duration{2 * time.Hour, 4 * time.Hour},
108-
blocks: map[ulid.ULID]*metadata.Meta{block1ulid: blocks[block1ulid], block2ulid: blocks[block2ulid], block3ulid: blocks[block3ulid], block4ulid: blocks[block4ulid], block5ulid: blocks[block5ulid], block6ulid: blocks[block6ulid]},
118+
blocks: map[ulid.ULID]*metadata.Meta{block1hto2hExt1Ulid: blocks[block1hto2hExt1Ulid], block3hto4hExt1Ulid: blocks[block3hto4hExt1Ulid], block0hto1hExt1Ulid: blocks[block0hto1hExt1Ulid], block2hto3hExt1Ulid: blocks[block2hto3hExt1Ulid], block1hto2hExt2Ulid: blocks[block1hto2hExt2Ulid], block0hto1hExt2Ulid: blocks[block0hto1hExt2Ulid]},
109119
expected: [][]ulid.ULID{
110-
{block5ulid, block6ulid},
111-
{block1ulid, block3ulid},
112-
{block2ulid, block4ulid},
120+
{block1hto2hExt2Ulid, block0hto1hExt2Ulid},
121+
{block1hto2hExt1Ulid, block0hto1hExt1Ulid},
122+
{block3hto4hExt1Ulid, block2hto3hExt1Ulid},
113123
},
114124
},
115125
"test no compaction": {
116126
ranges: []time.Duration{2 * time.Hour, 4 * time.Hour},
117-
blocks: map[ulid.ULID]*metadata.Meta{block7ulid: blocks[block7ulid], block8ulid: blocks[block8ulid], block9ulid: blocks[block9ulid]},
127+
blocks: map[ulid.ULID]*metadata.Meta{block0hto1hExt1Ulid: blocks[block0hto1hExt1Ulid], block0hto1hExt2Ulid: blocks[block0hto1hExt2Ulid], block0to1hExt3Ulid: blocks[block0to1hExt3Ulid]},
118128
expected: [][]ulid.ULID{},
119129
},
120130
"test smallest range first": {
121131
ranges: []time.Duration{2 * time.Hour, 4 * time.Hour},
122-
blocks: map[ulid.ULID]*metadata.Meta{block1ulid: blocks[block1ulid], block2ulid: blocks[block2ulid], block3ulid: blocks[block3ulid], block4ulid: blocks[block4ulid], block10ulid: blocks[block10ulid], block11ulid: blocks[block11ulid]},
132+
blocks: map[ulid.ULID]*metadata.Meta{block1hto2hExt1Ulid: blocks[block1hto2hExt1Ulid], block3hto4hExt1Ulid: blocks[block3hto4hExt1Ulid], block0hto1hExt1Ulid: blocks[block0hto1hExt1Ulid], block2hto3hExt1Ulid: blocks[block2hto3hExt1Ulid], block4hto6hExt2Ulid: blocks[block4hto6hExt2Ulid], block6hto8hExt2Ulid: blocks[block6hto8hExt2Ulid]},
123133
expected: [][]ulid.ULID{
124-
{block1ulid, block3ulid},
125-
{block2ulid, block4ulid},
126-
{block10ulid, block11ulid},
134+
{block1hto2hExt1Ulid, block0hto1hExt1Ulid},
135+
{block3hto4hExt1Ulid, block2hto3hExt1Ulid},
136+
{block4hto6hExt2Ulid, block6hto8hExt2Ulid},
127137
},
128138
},
129139
"test oldest min time first": {
130140
ranges: []time.Duration{2 * time.Hour, 4 * time.Hour},
131-
blocks: map[ulid.ULID]*metadata.Meta{block1ulid: blocks[block1ulid], block2ulid: blocks[block2ulid], block3ulid: blocks[block3ulid], block4ulid: blocks[block4ulid], block12ulid: blocks[block12ulid]},
141+
blocks: map[ulid.ULID]*metadata.Meta{block1hto2hExt1Ulid: blocks[block1hto2hExt1Ulid], block3hto4hExt1Ulid: blocks[block3hto4hExt1Ulid], block0hto1hExt1Ulid: blocks[block0hto1hExt1Ulid], block2hto3hExt1Ulid: blocks[block2hto3hExt1Ulid], block1hto2hExt1UlidCopy: blocks[block1hto2hExt1UlidCopy]},
132142
expected: [][]ulid.ULID{
133-
{block1ulid, block3ulid, block12ulid},
134-
{block2ulid, block4ulid},
143+
{block1hto2hExt1Ulid, block0hto1hExt1Ulid, block1hto2hExt1UlidCopy},
144+
{block3hto4hExt1Ulid, block2hto3hExt1Ulid},
135145
},
136146
},
137147
"test overlapping blocks": {
138-
ranges: []time.Duration{20 * time.Hour, 40 * time.Hour},
139-
blocks: map[ulid.ULID]*metadata.Meta{block13ulid: blocks[block13ulid], block14ulid: blocks[block14ulid], block15ulid: blocks[block15ulid]},
148+
ranges: []time.Duration{20 * time.Hour, 40 * time.Hour},
149+
blocks: map[ulid.ULID]*metadata.Meta{block0hto20hExt1Ulid: blocks[block0hto20hExt1Ulid], block21hto40hExt1Ulid: blocks[block21hto40hExt1Ulid], block21hto40hExt1UlidCopy: blocks[block21hto40hExt1UlidCopy]},
150+
expected: [][]ulid.ULID{
151+
{block21hto40hExt1Ulid, block21hto40hExt1UlidCopy},
152+
},
153+
},
154+
"test imperfect maxTime blocks": {
155+
ranges: []time.Duration{2 * time.Hour},
156+
blocks: map[ulid.ULID]*metadata.Meta{block0hto1h30mExt1Ulid: blocks[block0hto1h30mExt1Ulid], block0hto45mExt1Ulid: blocks[block0hto45mExt1Ulid]},
157+
expected: [][]ulid.ULID{
158+
{block0hto45mExt1Ulid, block0hto1h30mExt1Ulid},
159+
},
160+
},
161+
"test prematurely created blocks": {
162+
ranges: []time.Duration{2 * time.Hour},
163+
blocks: map[ulid.ULID]*metadata.Meta{blocklast1hExt1UlidCopy: blocks[blocklast1hExt1UlidCopy], blocklast1hExt1Ulid: blocks[blocklast1hExt1Ulid]},
140164
expected: [][]ulid.ULID{},
141165
},
142166
}
@@ -374,21 +398,6 @@ func TestGroupBlocksByCompactableRanges(t *testing.T) {
374398
}},
375399
},
376400
},
377-
"a range containing the most recent block shouldn't be prematurely compacted if doesn't cover the full range": {
378-
ranges: []int64{10, 20, 40},
379-
blocks: []*metadata.Meta{
380-
{BlockMeta: tsdb.BlockMeta{MinTime: 5, MaxTime: 8}},
381-
{BlockMeta: tsdb.BlockMeta{MinTime: 7, MaxTime: 9}},
382-
{BlockMeta: tsdb.BlockMeta{MinTime: 10, MaxTime: 12}},
383-
{BlockMeta: tsdb.BlockMeta{MinTime: 13, MaxTime: 15}},
384-
},
385-
expected: []blocksGroup{
386-
{rangeStart: 0, rangeEnd: 10, blocks: []*metadata.Meta{
387-
{BlockMeta: tsdb.BlockMeta{MinTime: 5, MaxTime: 8}},
388-
{BlockMeta: tsdb.BlockMeta{MinTime: 7, MaxTime: 9}},
389-
}},
390-
},
391-
},
392401
}
393402

394403
for testName, testData := range tests {

0 commit comments

Comments
 (0)