Skip to content

Commit f2b0021

Browse files
bwplotkayeya24GiedriusS
authored
Bring 0.19.0 rc fixes to main. (#3978)
* tools: Fix partial and empty matchers in rewrite (#3891) * Fix partial and empty matchers match Signed-off-by: yeya24 <[email protected]> * add testcase for non-equal matchers Signed-off-by: yeya24 <[email protected]> * v0.19.0 patch: Added receive benchmark; Fixed Receiver excessive mem usage introduced in 0.17 (#3943) * Added receive benchmark, baseline. ``` goos: linux goarch: amd64 pkg: github.com/thanos-io/thanos/pkg/receive BenchmarkHandlerReceiveHTTP BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them. BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them./OK BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them./OK-12 22260 1550152 ns/op 1380340 B/op 6093 allocs/op BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them./conflict_errors BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them./conflict_errors-12 6619 6430408 ns/op 4522487 B/op 26118 allocs/op BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them. BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them./OK BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them./OK-12 2695 17208794 ns/op 15072963 B/op 60441 allocs/op BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them./conflict_errors BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them./conflict_errors-12 474 72533286 ns/op 46396932 B/op 260141 allocs/op BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK-12 270 137050518 ns/op 226595379 B/op 132 allocs/op BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors-12 21 1616025443 ns/op 698724321 B/op 408 allocs/op PASS Process finished with exit code 0 ``` Signed-off-by: Bartlomiej Plotka <[email protected]> * Copy labels. ``` GOROOT=/home/bwplotka/.gvm/gos/go1.15 #gosetup GOPATH=/home/bwplotka/Repos/thanosgopath #gosetup /home/bwplotka/.gvm/gos/go1.15/bin/go test -c -o /tmp/___BenchmarkHandlerReceiveHTTP_in_github_com_thanos_io_thanos_pkg_receive github.com/thanos-io/thanos/pkg/receive #gosetup /tmp/___BenchmarkHandlerReceiveHTTP_in_github_com_thanos_io_thanos_pkg_receive -test.v -test.bench ^\QBenchmarkHandlerReceiveHTTP\E$ -test.run ^$ -test.benchmem -test.benchtime=30s goos: linux goarch: amd64 pkg: github.com/thanos-io/thanos/pkg/receive BenchmarkHandlerReceiveHTTP BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/OK BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/OK-12 25887 1537262 ns/op 1380023 B/op 6092 allocs/op BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/conflict_errors BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/conflict_errors-12 4237 7547968 ns/op 4522583 B/op 26118 allocs/op BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/OK BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/OK-12 2205 16513380 ns/op 15071092 B/op 60420 allocs/op BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/conflict_errors BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/conflict_errors-12 525 67278233 ns/op 46396645 B/op 260141 allocs/op BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK-12 285 148049189 ns/op 226596168 B/op 132 allocs/op BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors-12 20 1731361499 ns/op 698722550 B/op 401 allocs/op PASS Process finished with exit code 0 ``` Signed-off-by: Bartlomiej Plotka <[email protected]> * Addded bench., Signed-off-by: Bartlomiej Plotka <[email protected]> * Fix. Signed-off-by: Bartlomiej Plotka <[email protected]> * Improved API. Signed-off-by: Bartlomiej Plotka <[email protected]> * Changelog. Signed-off-by: Bartlomiej Plotka <[email protected]> * Addressed Lucas comments. Signed-off-by: Bartlomiej Plotka <[email protected]> # Conflicts: # CHANGELOG.md * compact: clean up directories thoroughly (#3869) * compact: clean up directories properly I couldn't stop thinking about this code for some reason and I have figured that I had missed one case in #3031. We need to also clean up the directories in compaction groups. A compaction could fail leaving some new directory with a random ULID on the disk. Before attempting to do another compaction loop, we need to remove it as well because the compaction process always produces a new, unique directory. Signed-off-by: Giedrius Statkevičius <[email protected]> * Remove all non expected dirs. Signed-off-by: Bartlomiej Plotka <[email protected]> * Addressed comment. Signed-off-by: Bartlomiej Plotka <[email protected]> Co-authored-by: Bartlomiej Plotka <[email protected]> * Added benchmark, Moved minio-deps to fork without race fix we don't need. (#3968) Fixes: thanos-io/thanos#3917 Long term fix: thanos-io/thanos#3967 Signed-off-by: Bartlomiej Plotka <[email protected]> # Conflicts: # go.sum * Changelog fix. Signed-off-by: Bartlomiej Plotka <[email protected]> * Build fix. Signed-off-by: Bartlomiej Plotka <[email protected]> Co-authored-by: Ben Ye <[email protected]> Co-authored-by: Giedrius Statkevičius <[email protected]>
1 parent ba2c5e1 commit f2b0021

File tree

3 files changed

+58
-5
lines changed

3 files changed

+58
-5
lines changed

s3/s3.go

+2-4
Original file line numberDiff line numberDiff line change
@@ -66,9 +66,7 @@ var DefaultConfig = Config{
6666
MaxIdleConnsPerHost: 100,
6767
MaxConnsPerHost: 0,
6868
},
69-
// Minimum file size after which an HTTP multipart request should be used to upload objects to storage.
70-
// Set to 128 MiB as in the minio client.
71-
PartSize: 1024 * 1024 * 128,
69+
PartSize: 1024 * 1024 * 64, // 64MB.
7270
}
7371

7472
// Config stores the configuration for s3 bucket.
@@ -85,6 +83,7 @@ type Config struct {
8583
TraceConfig TraceConfig `yaml:"trace"`
8684
ListObjectsVersion string `yaml:"list_objects_version"`
8785
// PartSize used for multipart upload. Only used if uploaded object size is known and larger than configured PartSize.
86+
// NOTE we need to make sure this number does not produce more parts than 10 000.
8887
PartSize uint64 `yaml:"part_size"`
8988
SSEConfig SSEConfig `yaml:"sse_config"`
9089
}
@@ -449,7 +448,6 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error {
449448
size = -1
450449
}
451450

452-
// partSize cannot be larger than object size.
453451
partSize := b.partSize
454452
if size < int64(partSize) {
455453
partSize = 0

s3/s3_e2e_test.go

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
// Copyright (c) The Thanos Authors.
2+
// Licensed under the Apache License 2.0.
3+
4+
package s3_test
5+
6+
import (
7+
"bytes"
8+
"context"
9+
"strings"
10+
"testing"
11+
12+
"github.com/cortexproject/cortex/integration/e2e"
13+
e2edb "github.com/cortexproject/cortex/integration/e2e/db"
14+
"github.com/go-kit/kit/log"
15+
"github.com/thanos-io/thanos/pkg/objstore/s3"
16+
"github.com/thanos-io/thanos/test/e2e/e2ethanos"
17+
18+
"github.com/thanos-io/thanos/pkg/testutil"
19+
)
20+
21+
// Regression benchmark for https://github.com/thanos-io/thanos/issues/3917.
22+
func BenchmarkUpload(b *testing.B) {
23+
b.ReportAllocs()
24+
ctx := context.Background()
25+
26+
s, err := e2e.NewScenario("e2e_bench_mino_client")
27+
testutil.Ok(b, err)
28+
b.Cleanup(e2ethanos.CleanScenario(b, s))
29+
30+
const bucket = "test"
31+
m := e2edb.NewMinio(8080, bucket)
32+
testutil.Ok(b, s.StartAndWaitReady(m))
33+
34+
bkt, err := s3.NewBucketWithConfig(log.NewNopLogger(), s3.Config{
35+
Bucket: bucket,
36+
AccessKey: e2edb.MinioAccessKey,
37+
SecretKey: e2edb.MinioSecretKey,
38+
Endpoint: m.HTTPEndpoint(),
39+
Insecure: true,
40+
}, "test-feed")
41+
testutil.Ok(b, err)
42+
43+
buf := bytes.Buffer{}
44+
buf.Grow(1028 * 1028 * 100) // 100MB.
45+
word := "abcdefghij"
46+
for i := 0; i < buf.Cap()/len(word); i++ {
47+
_, _ = buf.WriteString(word)
48+
}
49+
str := buf.String()
50+
51+
b.ResetTimer()
52+
for i := 0; i < b.N; i++ {
53+
testutil.Ok(b, bkt.Upload(ctx, "test", strings.NewReader(str)))
54+
}
55+
}

s3/s3_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,7 @@ http_config:
208208

209209
cfg, err := parseConfig(input)
210210
testutil.Ok(t, err)
211-
testutil.Assert(t, cfg.PartSize == 1024*1024*128, "when part size not set it should default to 128MiB")
211+
testutil.Assert(t, cfg.PartSize == 1024*1024*64, "when part size not set it should default to 128MiB")
212212

213213
input2 := []byte(`bucket: "bucket-name"
214214
endpoint: "s3-endpoint"

0 commit comments

Comments
 (0)