Skip to content

Commit 3929bdb

Browse files
harry671003harry671003
committed
Ensure that compaction continues for block with incomplete time-range
Signed-off-by: 🌲 Harry 🌊 John 🏔 <[email protected]>
1 parent c6ca0a2 commit 3929bdb

File tree

1 file changed

+54
-54
lines changed

1 file changed

+54
-54
lines changed

‎pkg/compactor/shuffle_sharding_grouper_test.go

+54-54
Original file line numberDiff line numberDiff line change
@@ -21,20 +21,20 @@ func TestShuffleShardingGrouper_Groups(t *testing.T) {
2121
block1hto2hExt1Ulid := ulid.MustNew(1, nil)
2222
block3hto4hExt1Ulid := ulid.MustNew(2, nil)
2323
block0hto1hExt1Ulid := ulid.MustNew(3, nil)
24-
block_2hto3hExt1Ulid := ulid.MustNew(4, nil)
25-
block_1hto2hExt2Ulid := ulid.MustNew(5, nil)
26-
block_0hto1hExt2Ulid := ulid.MustNew(6, nil)
24+
block2hto3hExt1Ulid := ulid.MustNew(4, nil)
25+
block1hto2hExt2Ulid := ulid.MustNew(5, nil)
26+
block0hto1hExt2Ulid := ulid.MustNew(6, nil)
2727
block0to1hExt3Ulid := ulid.MustNew(7, nil)
28-
block_4hto6hExt2Ulid := ulid.MustNew(8, nil)
29-
block_6hto8hExt2Ulid := ulid.MustNew(9, nil)
30-
block_1hto2hExt_1_ulid := ulid.MustNew(10, nil)
31-
block_0hto20hExt1Ulid := ulid.MustNew(11, nil)
32-
block_21hto40hExt1Ulid := ulid.MustNew(12, nil)
33-
block_21hto40hExt1Ulid_copy := ulid.MustNew(13, nil)
34-
block_0hto45mExt1Ulid := ulid.MustNew(14, nil)
35-
block_0hto1h30mExt1Ulid := ulid.MustNew(15, nil)
36-
block_last1hExt1Ulid := ulid.MustNew(16, nil)
37-
block_last1hExt1Ulid_copy := ulid.MustNew(17, nil)
28+
block4hto6hExt2Ulid := ulid.MustNew(8, nil)
29+
block6hto8hExt2Ulid := ulid.MustNew(9, nil)
30+
block1hto2hExt_1_ulid := ulid.MustNew(10, nil)
31+
block0hto20hExt1Ulid := ulid.MustNew(11, nil)
32+
block21hto40hExt1Ulid := ulid.MustNew(12, nil)
33+
block21hto40hExt1Ulid_copy := ulid.MustNew(13, nil)
34+
block0hto45mExt1Ulid := ulid.MustNew(14, nil)
35+
block0hto1h30mExt1Ulid := ulid.MustNew(15, nil)
36+
blocklast1hExt1Ulid := ulid.MustNew(16, nil)
37+
blocklast1hExt1Ulid_copy := ulid.MustNew(17, nil)
3838

3939
blocks :=
4040
map[ulid.ULID]*metadata.Meta{
@@ -50,60 +50,60 @@ func TestShuffleShardingGrouper_Groups(t *testing.T) {
5050
BlockMeta: tsdb.BlockMeta{ULID: block0hto1hExt1Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()},
5151
Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}},
5252
},
53-
block_2hto3hExt1Ulid: {
54-
BlockMeta: tsdb.BlockMeta{ULID: block_2hto3hExt1Ulid, MinTime: 2 * time.Hour.Milliseconds(), MaxTime: 3 * time.Hour.Milliseconds()},
53+
block2hto3hExt1Ulid: {
54+
BlockMeta: tsdb.BlockMeta{ULID: block2hto3hExt1Ulid, MinTime: 2 * time.Hour.Milliseconds(), MaxTime: 3 * time.Hour.Milliseconds()},
5555
Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}},
5656
},
57-
block_1hto2hExt2Ulid: {
58-
BlockMeta: tsdb.BlockMeta{ULID: block_1hto2hExt2Ulid, MinTime: 1 * time.Hour.Milliseconds(), MaxTime: 2 * time.Hour.Milliseconds()},
57+
block1hto2hExt2Ulid: {
58+
BlockMeta: tsdb.BlockMeta{ULID: block1hto2hExt2Ulid, MinTime: 1 * time.Hour.Milliseconds(), MaxTime: 2 * time.Hour.Milliseconds()},
5959
Thanos: metadata.Thanos{Labels: map[string]string{"external": "2"}},
6060
},
61-
block_0hto1hExt2Ulid: {
62-
BlockMeta: tsdb.BlockMeta{ULID: block_0hto1hExt2Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()},
61+
block0hto1hExt2Ulid: {
62+
BlockMeta: tsdb.BlockMeta{ULID: block0hto1hExt2Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()},
6363
Thanos: metadata.Thanos{Labels: map[string]string{"external": "2"}},
6464
},
6565
block0to1hExt3Ulid: {
6666
BlockMeta: tsdb.BlockMeta{ULID: block0to1hExt3Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()},
6767
Thanos: metadata.Thanos{Labels: map[string]string{"external": "3"}},
6868
},
69-
block_4hto6hExt2Ulid: {
70-
BlockMeta: tsdb.BlockMeta{ULID: block_4hto6hExt2Ulid, MinTime: 4 * time.Hour.Milliseconds(), MaxTime: 6 * time.Hour.Milliseconds()},
69+
block4hto6hExt2Ulid: {
70+
BlockMeta: tsdb.BlockMeta{ULID: block4hto6hExt2Ulid, MinTime: 4 * time.Hour.Milliseconds(), MaxTime: 6 * time.Hour.Milliseconds()},
7171
Thanos: metadata.Thanos{Labels: map[string]string{"external": "2"}},
7272
},
73-
block_6hto8hExt2Ulid: {
74-
BlockMeta: tsdb.BlockMeta{ULID: block_6hto8hExt2Ulid, MinTime: 6 * time.Hour.Milliseconds(), MaxTime: 8 * time.Hour.Milliseconds()},
73+
block6hto8hExt2Ulid: {
74+
BlockMeta: tsdb.BlockMeta{ULID: block6hto8hExt2Ulid, MinTime: 6 * time.Hour.Milliseconds(), MaxTime: 8 * time.Hour.Milliseconds()},
7575
Thanos: metadata.Thanos{Labels: map[string]string{"external": "2"}},
7676
},
77-
block_1hto2hExt_1_ulid: {
78-
BlockMeta: tsdb.BlockMeta{ULID: block_1hto2hExt_1_ulid, MinTime: 1 * time.Hour.Milliseconds(), MaxTime: 2 * time.Hour.Milliseconds()},
77+
block1hto2hExt_1_ulid: {
78+
BlockMeta: tsdb.BlockMeta{ULID: block1hto2hExt_1_ulid, MinTime: 1 * time.Hour.Milliseconds(), MaxTime: 2 * time.Hour.Milliseconds()},
7979
Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}},
8080
},
81-
block_0hto20hExt1Ulid: {
82-
BlockMeta: tsdb.BlockMeta{ULID: block_0hto20hExt1Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 20 * time.Hour.Milliseconds()},
81+
block0hto20hExt1Ulid: {
82+
BlockMeta: tsdb.BlockMeta{ULID: block0hto20hExt1Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 20 * time.Hour.Milliseconds()},
8383
Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}},
8484
},
85-
block_21hto40hExt1Ulid: {
86-
BlockMeta: tsdb.BlockMeta{ULID: block_21hto40hExt1Ulid, MinTime: 21 * time.Hour.Milliseconds(), MaxTime: 40 * time.Hour.Milliseconds()},
85+
block21hto40hExt1Ulid: {
86+
BlockMeta: tsdb.BlockMeta{ULID: block21hto40hExt1Ulid, MinTime: 21 * time.Hour.Milliseconds(), MaxTime: 40 * time.Hour.Milliseconds()},
8787
Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}},
8888
},
89-
block_21hto40hExt1Ulid_copy: {
90-
BlockMeta: tsdb.BlockMeta{ULID: block_21hto40hExt1Ulid_copy, MinTime: 21 * time.Hour.Milliseconds(), MaxTime: 40 * time.Hour.Milliseconds()},
89+
block21hto40hExt1Ulid_copy: {
90+
BlockMeta: tsdb.BlockMeta{ULID: block21hto40hExt1Ulid_copy, MinTime: 21 * time.Hour.Milliseconds(), MaxTime: 40 * time.Hour.Milliseconds()},
9191
Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}},
9292
},
93-
block_0hto45mExt1Ulid: {
94-
BlockMeta: tsdb.BlockMeta{ULID: block_0hto45mExt1Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 45 * time.Minute.Milliseconds()},
93+
block0hto45mExt1Ulid: {
94+
BlockMeta: tsdb.BlockMeta{ULID: block0hto45mExt1Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 45 * time.Minute.Milliseconds()},
9595
Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}},
9696
},
97-
block_0hto1h30mExt1Ulid: {
98-
BlockMeta: tsdb.BlockMeta{ULID: block_0hto1h30mExt1Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1*time.Hour.Milliseconds() + 30*time.Minute.Milliseconds()},
97+
block0hto1h30mExt1Ulid: {
98+
BlockMeta: tsdb.BlockMeta{ULID: block0hto1h30mExt1Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1*time.Hour.Milliseconds() + 30*time.Minute.Milliseconds()},
9999
Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}},
100100
},
101-
block_last1hExt1Ulid: {
102-
BlockMeta: tsdb.BlockMeta{ULID: block_last1hExt1Ulid, MinTime: int64(ulid.Now()) - 1*time.Hour.Milliseconds(), MaxTime: int64(ulid.Now())},
101+
blocklast1hExt1Ulid: {
102+
BlockMeta: tsdb.BlockMeta{ULID: blocklast1hExt1Ulid, MinTime: int64(ulid.Now()) - 1*time.Hour.Milliseconds(), MaxTime: int64(ulid.Now())},
103103
Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}},
104104
},
105-
block_last1hExt1Ulid_copy: {
106-
BlockMeta: tsdb.BlockMeta{ULID: block_last1hExt1Ulid_copy, MinTime: int64(ulid.Now()) - 1*time.Hour.Milliseconds(), MaxTime: int64(ulid.Now())},
105+
blocklast1hExt1Ulid_copy: {
106+
BlockMeta: tsdb.BlockMeta{ULID: blocklast1hExt1Ulid_copy, MinTime: int64(ulid.Now()) - 1*time.Hour.Milliseconds(), MaxTime: int64(ulid.Now())},
107107
Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}},
108108
},
109109
}
@@ -115,52 +115,52 @@ func TestShuffleShardingGrouper_Groups(t *testing.T) {
115115
}{
116116
"test basic grouping": {
117117
ranges: []time.Duration{2 * time.Hour, 4 * time.Hour},
118-
blocks: map[ulid.ULID]*metadata.Meta{block1hto2hExt1Ulid: blocks[block1hto2hExt1Ulid], block3hto4hExt1Ulid: blocks[block3hto4hExt1Ulid], block0hto1hExt1Ulid: blocks[block0hto1hExt1Ulid], block_2hto3hExt1Ulid: blocks[block_2hto3hExt1Ulid], block_1hto2hExt2Ulid: blocks[block_1hto2hExt2Ulid], block_0hto1hExt2Ulid: blocks[block_0hto1hExt2Ulid]},
118+
blocks: map[ulid.ULID]*metadata.Meta{block1hto2hExt1Ulid: blocks[block1hto2hExt1Ulid], block3hto4hExt1Ulid: blocks[block3hto4hExt1Ulid], block0hto1hExt1Ulid: blocks[block0hto1hExt1Ulid], block2hto3hExt1Ulid: blocks[block2hto3hExt1Ulid], block1hto2hExt2Ulid: blocks[block1hto2hExt2Ulid], block0hto1hExt2Ulid: blocks[block0hto1hExt2Ulid]},
119119
expected: [][]ulid.ULID{
120-
{block_1hto2hExt2Ulid, block_0hto1hExt2Ulid},
120+
{block1hto2hExt2Ulid, block0hto1hExt2Ulid},
121121
{block1hto2hExt1Ulid, block0hto1hExt1Ulid},
122-
{block3hto4hExt1Ulid, block_2hto3hExt1Ulid},
122+
{block3hto4hExt1Ulid, block2hto3hExt1Ulid},
123123
},
124124
},
125125
"test no compaction": {
126126
ranges: []time.Duration{2 * time.Hour, 4 * time.Hour},
127-
blocks: map[ulid.ULID]*metadata.Meta{block0hto1hExt1Ulid: blocks[block0hto1hExt1Ulid], block_0hto1hExt2Ulid: blocks[block_0hto1hExt2Ulid], block0to1hExt3Ulid: blocks[block0to1hExt3Ulid]},
127+
blocks: map[ulid.ULID]*metadata.Meta{block0hto1hExt1Ulid: blocks[block0hto1hExt1Ulid], block0hto1hExt2Ulid: blocks[block0hto1hExt2Ulid], block0to1hExt3Ulid: blocks[block0to1hExt3Ulid]},
128128
expected: [][]ulid.ULID{},
129129
},
130130
"test smallest range first": {
131131
ranges: []time.Duration{2 * time.Hour, 4 * time.Hour},
132-
blocks: map[ulid.ULID]*metadata.Meta{block1hto2hExt1Ulid: blocks[block1hto2hExt1Ulid], block3hto4hExt1Ulid: blocks[block3hto4hExt1Ulid], block0hto1hExt1Ulid: blocks[block0hto1hExt1Ulid], block_2hto3hExt1Ulid: blocks[block_2hto3hExt1Ulid], block_4hto6hExt2Ulid: blocks[block_4hto6hExt2Ulid], block_6hto8hExt2Ulid: blocks[block_6hto8hExt2Ulid]},
132+
blocks: map[ulid.ULID]*metadata.Meta{block1hto2hExt1Ulid: blocks[block1hto2hExt1Ulid], block3hto4hExt1Ulid: blocks[block3hto4hExt1Ulid], block0hto1hExt1Ulid: blocks[block0hto1hExt1Ulid], block2hto3hExt1Ulid: blocks[block2hto3hExt1Ulid], block4hto6hExt2Ulid: blocks[block4hto6hExt2Ulid], block6hto8hExt2Ulid: blocks[block6hto8hExt2Ulid]},
133133
expected: [][]ulid.ULID{
134134
{block1hto2hExt1Ulid, block0hto1hExt1Ulid},
135-
{block3hto4hExt1Ulid, block_2hto3hExt1Ulid},
136-
{block_4hto6hExt2Ulid, block_6hto8hExt2Ulid},
135+
{block3hto4hExt1Ulid, block2hto3hExt1Ulid},
136+
{block4hto6hExt2Ulid, block6hto8hExt2Ulid},
137137
},
138138
},
139139
"test oldest min time first": {
140140
ranges: []time.Duration{2 * time.Hour, 4 * time.Hour},
141-
blocks: map[ulid.ULID]*metadata.Meta{block1hto2hExt1Ulid: blocks[block1hto2hExt1Ulid], block3hto4hExt1Ulid: blocks[block3hto4hExt1Ulid], block0hto1hExt1Ulid: blocks[block0hto1hExt1Ulid], block_2hto3hExt1Ulid: blocks[block_2hto3hExt1Ulid], block_1hto2hExt_1_ulid: blocks[block_1hto2hExt_1_ulid]},
141+
blocks: map[ulid.ULID]*metadata.Meta{block1hto2hExt1Ulid: blocks[block1hto2hExt1Ulid], block3hto4hExt1Ulid: blocks[block3hto4hExt1Ulid], block0hto1hExt1Ulid: blocks[block0hto1hExt1Ulid], block2hto3hExt1Ulid: blocks[block2hto3hExt1Ulid], block1hto2hExt_1_ulid: blocks[block1hto2hExt_1_ulid]},
142142
expected: [][]ulid.ULID{
143-
{block1hto2hExt1Ulid, block0hto1hExt1Ulid, block_1hto2hExt_1_ulid},
144-
{block3hto4hExt1Ulid, block_2hto3hExt1Ulid},
143+
{block1hto2hExt1Ulid, block0hto1hExt1Ulid, block1hto2hExt_1_ulid},
144+
{block3hto4hExt1Ulid, block2hto3hExt1Ulid},
145145
},
146146
},
147147
"test overlapping blocks": {
148148
ranges: []time.Duration{20 * time.Hour, 40 * time.Hour},
149-
blocks: map[ulid.ULID]*metadata.Meta{block_0hto20hExt1Ulid: blocks[block_0hto20hExt1Ulid], block_21hto40hExt1Ulid: blocks[block_21hto40hExt1Ulid], block_21hto40hExt1Ulid_copy: blocks[block_21hto40hExt1Ulid_copy]},
149+
blocks: map[ulid.ULID]*metadata.Meta{block0hto20hExt1Ulid: blocks[block0hto20hExt1Ulid], block21hto40hExt1Ulid: blocks[block21hto40hExt1Ulid], block21hto40hExt1Ulid_copy: blocks[block21hto40hExt1Ulid_copy]},
150150
expected: [][]ulid.ULID{
151-
{block_21hto40hExt1Ulid, block_21hto40hExt1Ulid_copy},
151+
{block21hto40hExt1Ulid, block21hto40hExt1Ulid_copy},
152152
},
153153
},
154154
"test imperfect maxTime blocks": {
155155
ranges: []time.Duration{2 * time.Hour},
156-
blocks: map[ulid.ULID]*metadata.Meta{block_0hto1h30mExt1Ulid: blocks[block_0hto1h30mExt1Ulid], block_0hto45mExt1Ulid: blocks[block_0hto45mExt1Ulid]},
156+
blocks: map[ulid.ULID]*metadata.Meta{block0hto1h30mExt1Ulid: blocks[block0hto1h30mExt1Ulid], block0hto45mExt1Ulid: blocks[block0hto45mExt1Ulid]},
157157
expected: [][]ulid.ULID{
158-
{block_0hto45mExt1Ulid, block_0hto1h30mExt1Ulid},
158+
{block0hto45mExt1Ulid, block0hto1h30mExt1Ulid},
159159
},
160160
},
161161
"test prematurely created blocks": {
162162
ranges: []time.Duration{2 * time.Hour},
163-
blocks: map[ulid.ULID]*metadata.Meta{block_last1hExt1Ulid_copy: blocks[block_last1hExt1Ulid_copy], block_last1hExt1Ulid: blocks[block_last1hExt1Ulid]},
163+
blocks: map[ulid.ULID]*metadata.Meta{blocklast1hExt1Ulid_copy: blocks[blocklast1hExt1Ulid_copy], blocklast1hExt1Ulid: blocks[blocklast1hExt1Ulid]},
164164
expected: [][]ulid.ULID{},
165165
},
166166
}

0 commit comments

Comments
 (0)