From 3cb75ec717875fd537a44c4a9e21217c11063c81 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 13 Apr 2021 22:28:39 +0200 Subject: [PATCH 1/7] receive: Improved efficiency of multitsdb appends. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Release vs current main looks the same: ``` benchstat -delta-test=none ../_dev/thanos/2021/receive/5.txt ../_dev/thanos/2021/receive2/main-go1.15.txt name old time/op new time/op delta HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/OK-12 1.56ms ± 0% 1.45ms ± 0% -7.12% HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/conflict_errors-12 6.49ms ± 0% 7.14ms ± 0% +9.92% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/OK-12 16.0ms ± 0% 16.4ms ± 0% +2.79% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/conflict_errors-12 71.7ms ± 0% 69.4ms ± 0% -3.20% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK-12 138ms ± 0% 131ms ± 0% -4.79% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors-12 1.58s ± 0% 1.68s ± 0% +6.11% name old alloc/op new alloc/op delta HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/OK-12 1.70MB ± 0% 1.70MB ± 0% +0.12% HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/conflict_errors-12 4.84MB ± 0% 4.84MB ± 0% +0.04% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/OK-12 18.3MB ± 0% 18.2MB ± 0% -0.19% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/conflict_errors-12 49.6MB ± 0% 49.6MB ± 0% +0.00% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK-12 331MB ± 0% 331MB ± 0% -0.00% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors-12 804MB ± 0% 804MB ± 0% +0.00% name old allocs/op new allocs/op delta HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/OK-12 15.6k ± 0% 15.6k ± 0% +0.04% HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/conflict_errors-12 35.6k ± 0% 35.6k ± 0% +0.01% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/OK-12 155k ± 0% 155k ± 0% -0.08% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/conflict_errors-12 355k ± 0% 355k ± 0% +0.00% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK-12 147 ± 0% 145 ± 0% -1.36% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors-12 417 ± 0% 421 ± 0% +0.96% ``` Unfortunately go1.16 introduces more allocs overall (not that much more): ``` benchstat -delta-test=none ../_dev/thanos/2021/receive2/main-go1.15.txt ../_dev/thanos/2021/receive2/main-go1.16.3.txt name old time/op new time/op delta HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/OK-12 1.45ms ± 0% 1.62ms ± 0% +11.87% HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/conflict_errors-12 7.14ms ± 0% 6.47ms ± 0% -9.40% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/OK-12 16.4ms ± 0% 15.8ms ± 0% -3.87% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/conflict_errors-12 69.4ms ± 0% 66.4ms ± 0% -4.35% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK-12 131ms ± 0% 141ms ± 0% +7.59% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors-12 1.68s ± 0% 1.67s ± 0% -0.49% name old alloc/op new alloc/op delta HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/OK-12 1.70MB ± 0% 1.75MB ± 0% +2.50% HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/conflict_errors-12 4.84MB ± 0% 4.89MB ± 0% +0.88% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/OK-12 18.2MB ± 0% 18.8MB ± 0% +3.07% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/conflict_errors-12 49.6MB ± 0% 50.1MB ± 0% +1.09% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK-12 331MB ± 0% 343MB ± 0% +3.63% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors-12 804MB ± 0% 816MB ± 0% +1.50% name old allocs/op new allocs/op delta HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/OK-12 15.6k ± 0% 15.6k ± 0% -0.01% HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/conflict_errors-12 35.6k ± 0% 35.6k ± 0% +0.01% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/OK-12 155k ± 0% 155k ± 0% +0.08% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/conflict_errors-12 355k ± 0% 355k ± 0% +0.00% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK-12 145 ± 0% 166 ± 0% +14.48% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors-12 421 ± 0% 440 ± 0% +4.51% ``` Signed-off-by: Bartlomiej Plotka --- pkg/receive/handler_test.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/receive/handler_test.go b/pkg/receive/handler_test.go index a806947061..a7f13aed0b 100644 --- a/pkg/receive/handler_test.go +++ b/pkg/receive/handler_test.go @@ -1166,7 +1166,10 @@ func benchmarkHandlerMultiTSDBReceiveRemoteWrite(b testutil.TB) { for i := 0; i < n; i++ { r := httptest.NewRecorder() handler.receiveHTTP(r, &http.Request{ContentLength: int64(len(tcase.writeRequest)), Body: ioutil.NopCloser(bytes.NewReader(tcase.writeRequest))}) - testutil.Equals(b, http.StatusConflict, r.Code, "%v", i) + testutil.Equals(b, http.StatusConflict, r.Code, "%v-%s", i, func() string { + b, _ := ioutil.ReadAll(r.Body) + return string(b) + }()) } }) }) @@ -1174,7 +1177,7 @@ func benchmarkHandlerMultiTSDBReceiveRemoteWrite(b testutil.TB) { runtime.GC() // Take snapshot at the end to reveal how much memory we keep in TSDB. - testutil.Ok(b, Heap("../../")) + testutil.Ok(b, Heap("../../../_dev/thanos/2021/receive2")) } @@ -1183,7 +1186,7 @@ func Heap(dir string) (err error) { return err } - f, err := os.Create(filepath.Join(dir, "mem.pprof")) + f, err := os.Create(filepath.Join(dir, "main-go16.pprof")) if err != nil { return err } From f2a091df07ace88280f8ec9f26a75cf3cfff85ab Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Mon, 19 Apr 2021 17:03:48 +0200 Subject: [PATCH 2/7] Prometheus upgrade. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No difference. ``` benchstat -delta-test=none ../_dev/thanos/2021/receive2/main-go1.16.3.txt ../_dev/thanos/2021/receive2/impr-go1.16.3-promup.txt name old time/op new time/op delta HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/OK-12 1.62ms ± 0% 1.77ms ± 0% +9.57% HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/conflict_errors-12 6.47ms ± 0% 5.71ms ± 0% -11.76% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/OK-12 15.8ms ± 0% 15.2ms ± 0% -3.83% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/conflict_errors-12 66.4ms ± 0% 59.5ms ± 0% -10.37% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK-12 141ms ± 0% 129ms ± 0% -8.60% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors-12 1.67s ± 0% 1.41s ± 0% -15.58% name old alloc/op new alloc/op delta HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/OK-12 1.75MB ± 0% 1.75MB ± 0% +0.04% HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/conflict_errors-12 4.89MB ± 0% 4.89MB ± 0% +0.02% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/OK-12 18.8MB ± 0% 18.8MB ± 0% -0.05% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/conflict_errors-12 50.1MB ± 0% 50.1MB ± 0% +0.00% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK-12 343MB ± 0% 344MB ± 0% +0.00% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors-12 816MB ± 0% 816MB ± 0% -0.00% name old allocs/op new allocs/op delta HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/OK-12 15.6k ± 0% 15.6k ± 0% +0.01% HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/conflict_errors-12 35.6k ± 0% 35.6k ± 0% +0.00% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/OK-12 155k ± 0% 155k ± 0% -0.06% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/conflict_errors-12 355k ± 0% 355k ± 0% +0.00% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK-12 166 ± 0% 169 ± 0% +1.81% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors-12 440 ± 0% 435 ± 0% -1.14% ``` Signed-off-by: Bartlomiej Plotka --- go.mod | 22 +++---- go.sum | 123 +++++++++++++++++++++++------------- pkg/receive/handler_test.go | 100 ++++++++++++++++++++++++++++- pkg/receive/writer.go | 88 -------------------------- 4 files changed, 190 insertions(+), 143 deletions(-) diff --git a/go.mod b/go.mod index e3670164b7..943218e9da 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,7 @@ module github.com/thanos-io/thanos require ( - cloud.google.com/go v0.74.0 + cloud.google.com/go v0.79.0 cloud.google.com/go/storage v1.10.0 github.com/Azure/azure-pipeline-go v0.2.2 github.com/Azure/azure-storage-blob-go v0.8.0 @@ -33,7 +33,7 @@ require ( github.com/leanovate/gopter v0.2.4 github.com/lightstep/lightstep-tracer-go v0.18.1 github.com/lovoo/gcloud-opentracing v0.3.0 - github.com/miekg/dns v1.1.38 + github.com/miekg/dns v1.1.41 github.com/minio/minio-go/v7 v7.0.10 github.com/mozillazg/go-cos v0.13.0 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f @@ -48,8 +48,8 @@ require ( github.com/prometheus/alertmanager v0.21.1-0.20210310093010-0f9cab6991e6 github.com/prometheus/client_golang v1.10.0 github.com/prometheus/client_model v0.2.0 - github.com/prometheus/common v0.18.0 - github.com/prometheus/prometheus v1.8.2-0.20210315220929-1cba1741828b + github.com/prometheus/common v0.20.0 + github.com/prometheus/prometheus v1.8.2-0.20210413124018-62afcabd01ea github.com/uber/jaeger-client-go v2.25.0+incompatible github.com/uber/jaeger-lib v2.4.0+incompatible github.com/weaveworks/common v0.0.0-20210112142934-23c8d7fa6120 @@ -58,13 +58,13 @@ require ( go.uber.org/atomic v1.7.0 go.uber.org/automaxprocs v1.2.0 go.uber.org/goleak v1.1.10 - golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9 - golang.org/x/oauth2 v0.0.0-20210210192628-66670185b0cd - golang.org/x/sync v0.0.0-20201207232520-09787c993a3a + golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 + golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/text v0.3.5 - google.golang.org/api v0.39.0 - google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d - google.golang.org/grpc v1.34.0 + google.golang.org/api v0.42.0 + google.golang.org/genproto v0.0.0-20210312152112-fc591d9ea70f + google.golang.org/grpc v1.36.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/fsnotify.v1 v1.4.7 gopkg.in/yaml.v2 v2.4.0 @@ -81,7 +81,7 @@ replace ( // TODO: Remove this: https://github.com/thanos-io/thanos/issues/3967. github.com/minio/minio-go/v7 => github.com/bwplotka/minio-go/v7 v7.0.11-0.20210324165441-f9927e5255a6 // Make sure Prometheus version is pinned as Prometheus semver does not include Go APIs. - github.com/prometheus/prometheus => github.com/prometheus/prometheus v1.8.2-0.20210315220929-1cba1741828b + github.com/prometheus/prometheus => github.com/prometheus/prometheus v1.8.2-0.20210413124018-62afcabd01ea github.com/sercand/kuberesolver => github.com/sercand/kuberesolver v2.4.0+incompatible google.golang.org/grpc => google.golang.org/grpc v1.29.1 diff --git a/go.sum b/go.sum index ffc352be04..442cf22d01 100644 --- a/go.sum +++ b/go.sum @@ -17,8 +17,10 @@ cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0 h1:kpgPA77kSSbjSs+fWHkPTxQ6J5Z2Qkruo5jfXEkHxNQ= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0 h1:oqqswrt4x6b9OGBnNqdssxBl1xf0rSUNjU2BR4BZar0= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -47,8 +49,8 @@ github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiU github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-sdk-for-go v36.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v51.2.0+incompatible h1:qQNk//OOHK0GZcgMMgdJ4tZuuh0zcOeUkpTxjvKFpSQ= -github.com/Azure/azure-sdk-for-go v51.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v52.5.0+incompatible h1:/NLBWHCnIHtZyLPc1P7WIqi4Te4CC23kIQyK3Ep/7lA= +github.com/Azure/azure-sdk-for-go v52.5.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-storage-blob-go v0.8.0 h1:53qhf0Oxa0nOjgbDeeYPUeyiNmafAFEY95rZLK0Tj6o= github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= @@ -57,12 +59,11 @@ github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest v0.9.3-0.20191028180845-3492b2aff503/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= github.com/Azure/go-autorest/autorest/adal v0.8.1-0.20191028180845-3492b2aff503/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= -github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= @@ -73,7 +74,6 @@ github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSY github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= -github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/to v0.3.1-0.20191028180845-3492b2aff503/go.mod h1:MgwOyqaIuKdG4TL/2ywSsIWKAfJfgHDo8ObuUk3t5sA= @@ -168,8 +168,9 @@ github.com/aws/aws-sdk-go v1.33.12/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZve github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aws/aws-sdk-go v1.35.5/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= github.com/aws/aws-sdk-go v1.35.31/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.37.8 h1:9kywcbuz6vQuTf+FD+U7FshafrHzmqUCjgAEiLuIJ8U= github.com/aws/aws-sdk-go v1.37.8/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.38.3 h1:QCL/le04oAz2jELMRSuJVjGT7H+4hhoQc66eMPCfU/k= +github.com/aws/aws-sdk-go v1.38.3/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= @@ -282,8 +283,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cu github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dgryski/go-sip13 v0.0.0-20200911182023-62edffca9245/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dhui/dktest v0.3.0/go.mod h1:cyzIUfGsBEbZ6BT7tnXqAShHSXCZhSNmFl70sZ7c1yc= -github.com/digitalocean/godo v1.57.0 h1:uCpe0sRIZ/sJWxWDsJyBPBjUfSvxop+WHkHiSf+tjjM= -github.com/digitalocean/godo v1.57.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU= +github.com/digitalocean/godo v1.58.0 h1:Iy8ULTvgCAxH8dlxZ54qRYpm5uTEb2deUqijywLH7Lo= +github.com/digitalocean/godo v1.58.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU= github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/docker/distribution v2.7.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= @@ -291,8 +292,8 @@ github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BU github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.7.3-0.20190103212154-2b7e084dc98b/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v0.7.3-0.20190817195342-4760db040282/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v20.10.3+incompatible h1:+HS4XO73J41FpA260ztGujJ+0WibrA2TPJEnWNSyGNE= -github.com/docker/docker v20.10.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.5+incompatible h1:o5WL5onN4awYGwrW7+oTn5x9AF2prw7V0Ox8ZEkoCdg= +github.com/docker/docker v20.10.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= @@ -542,6 +543,7 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -581,8 +583,10 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -605,8 +609,10 @@ github.com/google/pprof v0.0.0-20200615235658-03e1cf38a040/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210208152844-1612e9be7af6 h1:38TDCVodvyooskjOFh+Ve8EY37rS8ZNzEIPHMWZaY/Y= -github.com/google/pprof v0.0.0-20210208152844-1612e9be7af6/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210323184331-8eee2492667d h1:Rwivyny4wymF1qWzOk800eSVa/n9njfdOm+kHjiQhZQ= +github.com/google/pprof v0.0.0-20210323184331-8eee2492667d/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -624,8 +630,8 @@ github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3i github.com/gophercloud/gophercloud v0.6.0/go.mod h1:GICNByuaEBibcjmjvI7QvYJSZEbGkcYwAR7EZK2WMqM= github.com/gophercloud/gophercloud v0.12.0/go.mod h1:gmC5oQqMDOMO1t1gq5DquX/yAU808e/4mzjjDA76+Ss= github.com/gophercloud/gophercloud v0.13.0/go.mod h1:VX0Ibx85B60B5XOrZr6kaNwrmPUzcmMpwxvQ1WQIIWM= -github.com/gophercloud/gophercloud v0.15.0 h1:jQeAWj0s1p83+TrUXhJhEOK4oe2g6YcBcFwEyMNIjEk= -github.com/gophercloud/gophercloud v0.15.0/go.mod h1:VX0Ibx85B60B5XOrZr6kaNwrmPUzcmMpwxvQ1WQIIWM= +github.com/gophercloud/gophercloud v0.16.0 h1:sWjPfypuzxRxjVbk3/MsU4H8jS0NNlyauZtIUl78BPU= +github.com/gophercloud/gophercloud v0.16.0/go.mod h1:wRtmUelyIIv3CSSDI47aUwbs075O6i+LY+pXsKCBsb4= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20191106031601-ce3c9ade29de h1:F7WD09S8QB4LrkEpka0dFPLSotH11HRpCsLIbIcJ7sU= github.com/gopherjs/gopherjs v0.0.0-20191106031601-ce3c9ade29de/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -723,8 +729,8 @@ github.com/hashicorp/serf v0.9.0/go.mod h1:YL0HO+FifKOW2u1ke99DGVu1zhcpZzNwrLIqB github.com/hashicorp/serf v0.9.3/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/hashicorp/serf v0.9.5 h1:EBWvyu9tcRszt3Bxp3KNssBMP1KuHWyO51lz9+786iM= github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= -github.com/hetznercloud/hcloud-go v1.23.1 h1:SkYdCa6x458cMSDz5GI18iPz5j2hicACiDP6J/s/bTs= -github.com/hetznercloud/hcloud-go v1.23.1/go.mod h1:xng8lbDUg+xM1dgc0yGHX5EeqbwIq7UYlMWMTx3SQVg= +github.com/hetznercloud/hcloud-go v1.24.0 h1:/CeHDzhH3Fhm83pjxvE3xNNLbvACl0Lu1/auJ83gG5U= +github.com/hetznercloud/hcloud-go v1.24.0/go.mod h1:3YmyK8yaZZ48syie6xpm3dt26rtB6s65AisBHylXYFA= github.com/hodgesds/perf-utils v0.0.8/go.mod h1:F6TfvsbtrF88i++hou29dTXlI2sfsJv+gRZDtmTJkAs= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= @@ -886,8 +892,9 @@ github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKju github.com/miekg/dns v1.1.30/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/miekg/dns v1.1.38 h1:MtIY+fmHUVVgv1AXzmKMWcwdCYxTRPG1EDjpqF4RCEw= github.com/miekg/dns v1.1.38/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/minio-go/v6 v6.0.44/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= @@ -911,6 +918,7 @@ github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -1075,8 +1083,9 @@ github.com/prometheus/common v0.11.1/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16 github.com/prometheus/common v0.12.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.14.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/common v0.18.0 h1:WCVKW7aL6LEe1uryfI9dnEc2ZqNB1Fn0ok930v0iL1Y= github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= +github.com/prometheus/common v0.20.0 h1:pfeDeUdQcIxOMutNjCejsEFp7qeP+/iltHSSmLpE+hU= +github.com/prometheus/common v0.20.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/exporter-toolkit v0.5.0/go.mod h1:OCkM4805mmisBhLmVFw858QYi3v0wKdY6/UxrT0pZVg= github.com/prometheus/exporter-toolkit v0.5.1/go.mod h1:OCkM4805mmisBhLmVFw858QYi3v0wKdY6/UxrT0pZVg= github.com/prometheus/node_exporter v1.0.0-rc.0.0.20200428091818-01054558c289 h1:dTUS1vaLWq+Y6XKOTnrFpoVsQKLCbCp1OLj24TDi7oM= @@ -1097,8 +1106,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/prometheus v1.8.2-0.20210315220929-1cba1741828b h1:TTOvmIV3W6IUIj3pYFs9gfCgueHlriLStMGBsnNdEX4= -github.com/prometheus/prometheus v1.8.2-0.20210315220929-1cba1741828b/go.mod h1:MS/bpdil77lPbfQeKk6OqVQ9OLnpN3Rszd0hka0EOWE= +github.com/prometheus/prometheus v1.8.2-0.20210413124018-62afcabd01ea h1:8nUryfJZAbrzfqV6PirGwRRjzYXu04PhEz5/cN9LC6o= +github.com/prometheus/prometheus v1.8.2-0.20210413124018-62afcabd01ea/go.mod h1:8lddSGaFjEe/U0H1l34rzhc+l2M3C2/usvMvHhT+U3I= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1/go.mod h1:JaY6n2sDr+z2WTsXkOmNRUfDy6FN0L6Nk7x06ndm4tY= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= @@ -1302,8 +1311,9 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/otel v0.11.0 h1:IN2tzQa9Gc4ZVKnTaMbPVcHjvzOdg5n9QfnmlqiET7E= go.opentelemetry.io/otel v0.11.0/go.mod h1:G8UCk+KooF2HLkgo8RHX9epABH/aRGYET7gQOqBVdB0= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -1353,8 +1363,10 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9 h1:sYNJzB4J8toYPQTM6pAkcmBRgw9SnQKP9oXCHfgy604= golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1393,8 +1405,9 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1452,8 +1465,11 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210324051636-2c4c8ecb7826 h1:lNRDRnwZWawoPHDS50ebYHTOHjctRMLSrUSQFcAHiW4= +golang.org/x/net v0.0.0-20210324051636-2c4c8ecb7826/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1464,8 +1480,12 @@ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210210192628-66670185b0cd h1:2arJsLyTCJGek+eeptQ3z49Rqndm0f+zvvpwNIXWNIA= golang.org/x/oauth2 v0.0.0-20210210192628-66670185b0cd/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558 h1:D7nTwh4J0i+5mW4Zjzn5omvlr6YBcWywE6KOcatyNxY= +golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1477,8 +1497,9 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200930132711-30421366ff76/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1555,18 +1576,25 @@ golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210314195730-07df6a141424/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492 h1:Paq34FxTluEPvVyayQqMPgHm+vTOrIifmcYxFBx9TLg= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1581,8 +1609,9 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1693,8 +1722,11 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513 google.golang.org/api v0.32.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.39.0 h1:zHCTXf0NeDdKTgcSQpT+ZflWAqHsEp1GmdpxW09f3YM= google.golang.org/api v0.39.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.42.0 h1:uqATLkpxiBrhrvFoebXUjvyzE9nQf+pVyy0Z0IHE+fc= +google.golang.org/api v0.42.0/go.mod h1:+Oj4s6ch2SEGtPjGqfUfZonBH0GjQH89gTeKKAEGZKI= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1749,8 +1781,12 @@ google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d h1:HV9Z9qMhQEsdlvxNFELgQ11RkMzO3CMkjEySjCtuLes= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210312152112-fc591d9ea70f h1:YRBxgxUW6GFi+AKsn8WGA9k1SZohK+gGuEqdeT5aoNQ= +google.golang.org/genproto v0.0.0-20210312152112-fc591d9ea70f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -1822,18 +1858,18 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= k8s.io/api v0.0.0-20191115095533-47f6de673b26/go.mod h1:iA/8arsvelvo4IDqIhX4IbjTEKBGgvsf2OraTuRtLFU= -k8s.io/api v0.20.2 h1:y/HR22XDZY3pniu9hIFDLpUCPq2w5eQ6aV/VFQ7uJMw= -k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= +k8s.io/api v0.21.0 h1:gu5iGF4V6tfVCQ/R+8Hc0h7H1JuEhzyEi9S4R5LM8+Y= +k8s.io/api v0.21.0/go.mod h1:+YbrhBBGgsxbF6o6Kj4KJPJnBmAKuXDeS3E18bgHNVU= k8s.io/apimachinery v0.0.0-20191115015347-3c7067801da2/go.mod h1:dXFS2zaQR8fyzuvRdJDHw2Aerij/yVGJSre0bZQSVJA= -k8s.io/apimachinery v0.20.2 h1:hFx6Sbt1oG0n6DZ+g4bFt5f6BoMkOjKWsQFu077M3Vg= -k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/client-go v0.20.2 h1:uuf+iIAbfnCSw8IGAv/Rg0giM+2bOzHLOsbbrwrdhNQ= -k8s.io/client-go v0.20.2/go.mod h1:kH5brqWqp7HDxUFKoEgiI4v8G1xzbe9giaCenUWJzgE= +k8s.io/apimachinery v0.21.0 h1:3Fx+41if+IRavNcKOz09FwEXDBG6ORh6iMsTSelhkMA= +k8s.io/apimachinery v0.21.0/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= +k8s.io/client-go v0.21.0 h1:n0zzzJsAQmJngpC0IhgFcApZyoGXPrDIAD601HD09ag= +k8s.io/client-go v0.21.0/go.mod h1:nNBytTF9qPFDEhoqgEPaarobC8QPae13bElIVHzIglA= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= +k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= k8s.io/utils v0.0.0-20191114200735-6ca3b61696b6/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= @@ -1843,8 +1879,9 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e h1:4Z09Hglb792X0kfOBBJUPFEyvVfQWrYT/l8h5EKA6JQ= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8= +sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/pkg/receive/handler_test.go b/pkg/receive/handler_test.go index a7f13aed0b..09116da3fd 100644 --- a/pkg/receive/handler_test.go +++ b/pkg/receive/handler_test.go @@ -26,6 +26,7 @@ import ( "github.com/golang/snappy" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/pkg/exemplar" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb" @@ -185,6 +186,103 @@ func TestDetermineWriteErrorCause(t *testing.T) { } } +type fakeTenantAppendable struct { + f *fakeAppendable +} + +func newFakeTenantAppendable(f *fakeAppendable) *fakeTenantAppendable { + return &fakeTenantAppendable{f: f} +} + +func (t *fakeTenantAppendable) TenantAppendable(_ string) (Appendable, error) { + return t.f, nil +} + +type fakeAppendable struct { + appender storage.Appender + appenderErr func() error +} + +var _ Appendable = &fakeAppendable{} + +func nilErrFn() error { + return nil +} + +func (f *fakeAppendable) Appender(_ context.Context) (storage.Appender, error) { + errf := f.appenderErr + if errf == nil { + errf = nilErrFn + } + return f.appender, errf() +} + +type fakeAppender struct { + sync.Mutex + samples map[uint64][]prompb.Sample + exemplars map[uint64][]exemplar.Exemplar + appendErr func() error + commitErr func() error + rollbackErr func() error +} + +var _ storage.Appender = &fakeAppender{} + +func newFakeAppender(appendErr, commitErr, rollbackErr func() error) *fakeAppender { //nolint:unparam + if appendErr == nil { + appendErr = nilErrFn + } + if commitErr == nil { + commitErr = nilErrFn + } + if rollbackErr == nil { + rollbackErr = nilErrFn + } + return &fakeAppender{ + samples: make(map[uint64][]prompb.Sample), + appendErr: appendErr, + commitErr: commitErr, + rollbackErr: rollbackErr, + } +} + +func (f *fakeAppender) Get(l labels.Labels) []prompb.Sample { + f.Lock() + defer f.Unlock() + s := f.samples[l.Hash()] + res := make([]prompb.Sample, len(s)) + copy(res, s) + return res +} + +func (f *fakeAppender) Append(ref uint64, l labels.Labels, t int64, v float64) (uint64, error) { + f.Lock() + defer f.Unlock() + if ref == 0 { + ref = l.Hash() + } + f.samples[ref] = append(f.samples[ref], prompb.Sample{Timestamp: t, Value: v}) + return ref, f.appendErr() +} + +func (f *fakeAppender) AppendExemplar(ref uint64, l labels.Labels, e exemplar.Exemplar) (uint64, error) { + f.Lock() + defer f.Unlock() + if ref == 0 { + ref = l.Hash() + } + f.exemplars[ref] = append(f.exemplars[ref], e) + return ref, f.appendErr() +} + +func (f *fakeAppender) Commit() error { + return f.commitErr() +} + +func (f *fakeAppender) Rollback() error { + return f.rollbackErr() +} + func newTestHandlerHashring(appendables []*fakeAppendable, replicationFactor uint64) ([]*Handler, Hashring) { var ( cfg = []HashringConfig{{Hashring: "test"}} @@ -1186,7 +1284,7 @@ func Heap(dir string) (err error) { return err } - f, err := os.Create(filepath.Join(dir, "main-go16.pprof")) + f, err := os.Create(filepath.Join(dir, "impr2-go1.16.3.pprof")) if err != nil { return err } diff --git a/pkg/receive/writer.go b/pkg/receive/writer.go index d58ac35f95..ed03c1e3b9 100644 --- a/pkg/receive/writer.go +++ b/pkg/receive/writer.go @@ -5,12 +5,10 @@ package receive import ( "context" - "sync" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/pkg/errors" - "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb" "github.com/thanos-io/thanos/pkg/store/labelpb" @@ -107,89 +105,3 @@ func (r *Writer) Write(ctx context.Context, tenantID string, wreq *prompb.WriteR return errs.Err() } - -type fakeTenantAppendable struct { - f *fakeAppendable -} - -func newFakeTenantAppendable(f *fakeAppendable) *fakeTenantAppendable { - return &fakeTenantAppendable{f: f} -} - -func (t *fakeTenantAppendable) TenantAppendable(tenantID string) (Appendable, error) { - return t.f, nil -} - -type fakeAppendable struct { - appender storage.Appender - appenderErr func() error -} - -var _ Appendable = &fakeAppendable{} - -func nilErrFn() error { - return nil -} - -func (f *fakeAppendable) Appender(_ context.Context) (storage.Appender, error) { - errf := f.appenderErr - if errf == nil { - errf = nilErrFn - } - return f.appender, errf() -} - -type fakeAppender struct { - sync.Mutex - samples map[uint64][]prompb.Sample - appendErr func() error - commitErr func() error - rollbackErr func() error -} - -var _ storage.Appender = &fakeAppender{} - -func newFakeAppender(appendErr, commitErr, rollbackErr func() error) *fakeAppender { //nolint:unparam - if appendErr == nil { - appendErr = nilErrFn - } - if commitErr == nil { - commitErr = nilErrFn - } - if rollbackErr == nil { - rollbackErr = nilErrFn - } - return &fakeAppender{ - samples: make(map[uint64][]prompb.Sample), - appendErr: appendErr, - commitErr: commitErr, - rollbackErr: rollbackErr, - } -} - -func (f *fakeAppender) Get(l labels.Labels) []prompb.Sample { - f.Lock() - defer f.Unlock() - s := f.samples[l.Hash()] - res := make([]prompb.Sample, len(s)) - copy(res, s) - return res -} - -func (f *fakeAppender) Append(ref uint64, l labels.Labels, t int64, v float64) (uint64, error) { - f.Lock() - defer f.Unlock() - if ref == 0 { - ref = l.Hash() - } - f.samples[ref] = append(f.samples[ref], prompb.Sample{Timestamp: t, Value: v}) - return ref, f.appendErr() -} - -func (f *fakeAppender) Commit() error { - return f.commitErr() -} - -func (f *fakeAppender) Rollback() error { - return f.rollbackErr() -} From faf59b02f7cea8f4a98c6233da27d403974a61dc Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Mon, 19 Apr 2021 18:03:12 +0200 Subject: [PATCH 3/7] ReadAll to Grow + Copy. Signed-off-by: Bartlomiej Plotka --- pkg/receive/handler.go | 32 ++++++++++++++++++++------------ pkg/receive/handler_test.go | 2 +- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/pkg/receive/handler.go b/pkg/receive/handler.go index 21e7161754..7900bd1be6 100644 --- a/pkg/receive/handler.go +++ b/pkg/receive/handler.go @@ -4,10 +4,11 @@ package receive import ( + "bytes" "context" "crypto/tls" "fmt" - "io/ioutil" + "io" stdlog "log" "net" "net/http" @@ -265,7 +266,7 @@ func (h *Handler) handleRequest(ctx context.Context, rep uint64, tenant string, replicated: rep != 0, } - // on-the-wire format is 1-indexed and in-code is 0-indexed so we decrement the value if it was already replicated. + // On the wire, format is 1-indexed and in-code is 0-indexed so we decrement the value if it was already replicated. if r.replicated { r.n-- } @@ -280,17 +281,24 @@ func (h *Handler) receiveHTTP(w http.ResponseWriter, r *http.Request) { span, ctx := tracing.StartSpan(r.Context(), "receive_http") defer span.Finish() - // TODO(bwplotka): Optimize readAll https://github.com/thanos-io/thanos/pull/3334/files. - compressed, err := ioutil.ReadAll(r.Body) + // ioutil.ReadAll dynamically adjust the byte slice for read data, starting from 512B. + // Since this is receive hot path, grow upfront saving allocations and CPU time. + compressed := bytes.Buffer{} + if r.ContentLength >= 0 { + compressed.Grow(int(r.ContentLength)) + } else { + compressed.Grow(512) + } + _, err := io.Copy(&compressed, r.Body) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + http.Error(w, errors.Wrap(err, "read compressed request body").Error(), http.StatusInternalServerError) return } - reqBuf, err := snappy.Decode(nil, compressed) + reqBuf, err := snappy.Decode(nil, compressed.Bytes()) if err != nil { level.Error(h.logger).Log("msg", "snappy decode error", "err", err) - http.Error(w, err.Error(), http.StatusBadRequest) + http.Error(w, errors.Wrap(err, "snappy decode error").Error(), http.StatusBadRequest) return } @@ -413,9 +421,9 @@ func (h *Handler) fanoutForward(pctx context.Context, tenant string, replicas ma } }() - logger := log.With(h.logger, "tenant", tenant) + logTags := []interface{}{"tenant", tenant} if id, ok := middleware.RequestIDFromContext(pctx); ok { - logger = log.With(logger, "request-id", id) + logTags = append(logTags, "request-id", id) } ec := make(chan error) @@ -465,7 +473,7 @@ func (h *Handler) fanoutForward(pctx context.Context, tenant string, replicas ma if err != nil { // When a MultiError is added to another MultiError, the error slices are concatenated, not nested. // To avoid breaking the counting logic, we need to flatten the error. - level.Debug(h.logger).Log("msg", "local tsdb write failed", "err", err.Error()) + level.Debug(h.logger).Log(append(logTags, "msg", "local tsdb write failed", "err", err.Error())) ec <- errors.Wrapf(determineWriteErrorCause(err, 1), "store locally for endpoint %v", endpoint) return } @@ -528,7 +536,7 @@ func (h *Handler) fanoutForward(pctx context.Context, tenant string, replicas ma b.attempt++ dur := h.expBackoff.ForAttempt(b.attempt) b.nextAllowed = time.Now().Add(dur) - level.Debug(h.logger).Log("msg", "target unavailable backing off", "for", dur) + level.Debug(h.logger).Log(append(logTags, "msg", "target unavailable backing off", "for", dur)) } else { h.peerStates[endpoint] = &retryState{nextAllowed: time.Now().Add(h.expBackoff.ForAttempt(0))} } @@ -557,7 +565,7 @@ func (h *Handler) fanoutForward(pctx context.Context, tenant string, replicas ma go func() { for err := range ec { if err != nil { - level.Debug(logger).Log("msg", "request failed, but not needed to achieve quorum", "err", err) + level.Debug(h.logger).Log(append(logTags, "msg", "request failed, but not needed to achieve quorum", "err", err)) } } }() diff --git a/pkg/receive/handler_test.go b/pkg/receive/handler_test.go index 09116da3fd..a2d5a647fc 100644 --- a/pkg/receive/handler_test.go +++ b/pkg/receive/handler_test.go @@ -1284,7 +1284,7 @@ func Heap(dir string) (err error) { return err } - f, err := os.Create(filepath.Join(dir, "impr2-go1.16.3.pprof")) + f, err := os.Create(filepath.Join(dir, "impr3-go1.16.3.pprof")) if err != nil { return err } From a09ac4fc514c7777c5f4bf636fae8fbd7f6242b0 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Mon, 19 Apr 2021 18:14:47 +0200 Subject: [PATCH 4/7] Moved hashring to optimized hash function. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` benchstat -delta-test=none ../_dev/thanos/2021/receive2/impr1-go1.16.3.txt ../_dev/thanos/2021/receive2/impr3-go1.16.3.txt name old time/op new time/op delta HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/OK-12 1.54ms ± 0% 1.64ms ± 0% +6.54% HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/conflict_errors-12 6.96ms ± 0% 8.02ms ± 0% +15.23% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/OK-12 16.1ms ± 0% 16.5ms ± 0% +2.77% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/conflict_errors-12 65.4ms ± 0% 65.1ms ± 0% -0.49% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK-12 168ms ± 0% 119ms ± 0% -29.49% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors-12 1.69s ± 0% 1.37s ± 0% -19.05% name old alloc/op new alloc/op delta HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/OK-12 1.75MB ± 0% 1.63MB ± 0% -6.43% HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/conflict_errors-12 4.89MB ± 0% 4.77MB ± 0% -2.50% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/OK-12 18.8MB ± 0% 17.6MB ± 0% -6.55% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/conflict_errors-12 50.1MB ± 0% 48.9MB ± 0% -2.55% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK-12 344MB ± 0% 225MB ± 0% -34.63% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors-12 816MB ± 0% 697MB ± 0% -14.59% name old allocs/op new allocs/op delta HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/OK-12 15.6k ± 0% 13.6k ± 0% -12.85% HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/conflict_errors-12 35.6k ± 0% 33.6k ± 0% -5.64% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/OK-12 155k ± 0% 135k ± 0% -12.95% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/conflict_errors-12 355k ± 0% 335k ± 0% -5.64% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK-12 170 ± 0% 101 ± 0% -40.59% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors-12 439 ± 0% 372 ± 0% -15.26% ``` Signed-off-by: Bartlomiej Plotka --- go.mod | 1 + pkg/receive/hashring.go | 22 +---------- pkg/receive/hashring_test.go | 23 ----------- pkg/store/labelpb/label.go | 30 ++++++++++++++ pkg/store/labelpb/label_test.go | 69 +++++++++++++++++++++++++++++++++ 5 files changed, 102 insertions(+), 43 deletions(-) diff --git a/go.mod b/go.mod index 943218e9da..9c3c53a0d3 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b github.com/cespare/xxhash v1.1.0 + github.com/cespare/xxhash/v2 v2.1.1 github.com/chromedp/cdproto v0.0.0-20200424080200-0de008e41fa0 github.com/chromedp/chromedp v0.5.3 github.com/cortexproject/cortex v1.7.1-0.20210316085356-3fedc1108a49 diff --git a/pkg/receive/hashring.go b/pkg/receive/hashring.go index cb5ceb71a5..497b42eca8 100644 --- a/pkg/receive/hashring.go +++ b/pkg/receive/hashring.go @@ -6,11 +6,10 @@ package receive import ( "context" "fmt" - "sort" "sync" - "github.com/cespare/xxhash" "github.com/pkg/errors" + "github.com/thanos-io/thanos/pkg/store/labelpb" "github.com/thanos-io/thanos/pkg/store/storepb/prompb" ) @@ -39,23 +38,6 @@ type Hashring interface { GetN(tenant string, timeSeries *prompb.TimeSeries, n uint64) (string, error) } -// hash returns a hash for the given tenant and time series. -func hash(tenant string, ts *prompb.TimeSeries) uint64 { - // Sort labelset to ensure a stable hash. - sort.Slice(ts.Labels, func(i, j int) bool { return ts.Labels[i].Name < ts.Labels[j].Name }) - - b := make([]byte, 0, 1024) - b = append(b, []byte(tenant)...) - b = append(b, sep) - for _, v := range ts.Labels { - b = append(b, v.Name...) - b = append(b, sep) - b = append(b, v.Value...) - b = append(b, sep) - } - return xxhash.Sum64(b) -} - // SingleNodeHashring always returns the same node. type SingleNodeHashring string @@ -85,7 +67,7 @@ func (s simpleHashring) GetN(tenant string, ts *prompb.TimeSeries, n uint64) (st if n >= uint64(len(s)) { return "", &insufficientNodesError{have: uint64(len(s)), want: n + 1} } - return s[(hash(tenant, ts)+n)%uint64(len(s))], nil + return s[(labelpb.HashWithPrefix(tenant, ts.Labels)+n)%uint64(len(s))], nil } // multiHashring represents a set of hashrings. diff --git a/pkg/receive/hashring_test.go b/pkg/receive/hashring_test.go index d822bb0e21..f5a1411a5b 100644 --- a/pkg/receive/hashring_test.go +++ b/pkg/receive/hashring_test.go @@ -10,29 +10,6 @@ import ( "github.com/thanos-io/thanos/pkg/store/storepb/prompb" ) -func TestHash(t *testing.T) { - ts := &prompb.TimeSeries{ - Labels: []labelpb.ZLabel{ - { - Name: "foo", - Value: "bar", - }, - { - Name: "baz", - Value: "qux", - }, - }, - } - - ts2 := &prompb.TimeSeries{ - Labels: []labelpb.ZLabel{ts.Labels[1], ts.Labels[0]}, - } - - if hash("", ts) != hash("", ts2) { - t.Errorf("expected hashes to be independent of label order") - } -} - func TestHashringGet(t *testing.T) { ts := &prompb.TimeSeries{ Labels: []labelpb.ZLabel{ diff --git a/pkg/store/labelpb/label.go b/pkg/store/labelpb/label.go index 99083f82ab..2aa778d71c 100644 --- a/pkg/store/labelpb/label.go +++ b/pkg/store/labelpb/label.go @@ -14,6 +14,7 @@ import ( "strings" "unsafe" + "github.com/cespare/xxhash/v2" "github.com/pkg/errors" "github.com/prometheus/prometheus/pkg/labels" ) @@ -315,6 +316,35 @@ func DeepCopy(lbls []ZLabel) []ZLabel { return ret } +var sep = []byte{'\xff'} + +// HashWithPrefix returns a hash for the given prefix and labels. +func HashWithPrefix(prefix string, lbls []ZLabel) uint64 { + // Use xxhash.Sum64(b) for fast path as it's faster. + b := make([]byte, 0, 1024) + b = append(b, prefix...) + + for i, v := range lbls { + if len(b)+len(v.Name)+len(v.Value)+2 >= cap(b) { + // If labels entry is 1KB allocate do not allocate whole entry. + h := xxhash.New() + _, _ = h.Write(b) + for _, v := range lbls[i:] { + _, _ = h.WriteString(v.Name) + _, _ = h.Write(sep) + _, _ = h.WriteString(v.Value) + _, _ = h.Write(sep) + } + return h.Sum64() + } + b = append(b, v.Name...) + b = append(b, sep[0]) + b = append(b, v.Value...) + b = append(b, sep[0]) + } + return xxhash.Sum64(b) +} + // ZLabelSets is a sortable list of ZLabelSet. It assumes the label pairs in each ZLabelSet element are already sorted. type ZLabelSets []ZLabelSet diff --git a/pkg/store/labelpb/label_test.go b/pkg/store/labelpb/label_test.go index ee80b5ad47..09e94ac9cd 100644 --- a/pkg/store/labelpb/label_test.go +++ b/pkg/store/labelpb/label_test.go @@ -8,6 +8,7 @@ import ( ioutil "io/ioutil" "reflect" "sort" + "strings" "testing" "github.com/prometheus/prometheus/pkg/labels" @@ -352,3 +353,71 @@ func TestSortZLabelSets(t *testing.T) { sort.Sort(list) reflect.DeepEqual(expectedResult, list) } + +func TestHashWithPrefix(t *testing.T) { + lbls := []ZLabel{ + {Name: "foo", Value: "bar"}, + {Name: "baz", Value: "qux"}, + } + testutil.Equals(t, HashWithPrefix("a", lbls), HashWithPrefix("a", lbls)) + testutil.Assert(t, HashWithPrefix("a", lbls) != HashWithPrefix("a", []ZLabel{lbls[0]})) + testutil.Assert(t, HashWithPrefix("a", lbls) != HashWithPrefix("a", []ZLabel{lbls[1], lbls[0]})) + testutil.Assert(t, HashWithPrefix("a", lbls) != HashWithPrefix("b", lbls)) +} + +var benchmarkLabelsResult uint64 + +func BenchmarkHasWithPrefix(b *testing.B) { + for _, tcase := range []struct { + name string + lbls []ZLabel + }{ + { + name: "typical labels under 1KB", + lbls: func() []ZLabel { + lbls := make([]ZLabel, 10) + for i := 0; i < len(lbls); i++ { + // ZLabel ~20B name, 50B value. + lbls[i] = ZLabel{Name: fmt.Sprintf("abcdefghijabcdefghijabcdefghij%d", i), Value: fmt.Sprintf("abcdefghijabcdefghijabcdefghijabcdefghijabcdefghij%d", i)} + } + return lbls + }(), + }, + { + name: "bigger labels over 1KB", + lbls: func() []ZLabel { + lbls := make([]ZLabel, 10) + for i := 0; i < len(lbls); i++ { + //ZLabel ~50B name, 50B value. + lbls[i] = ZLabel{Name: fmt.Sprintf("abcdefghijabcdefghijabcdefghijabcdefghijabcdefghij%d", i), Value: fmt.Sprintf("abcdefghijabcdefghijabcdefghijabcdefghijabcdefghij%d", i)} + } + return lbls + }(), + }, + { + name: "extremely large label value 10MB", + lbls: func() []ZLabel { + lbl := &strings.Builder{} + lbl.Grow(1024 * 1024 * 10) // 10MB. + word := "abcdefghij" + for i := 0; i < lbl.Cap()/len(word); i++ { + _, _ = lbl.WriteString(word) + } + return []ZLabel{{Name: "__name__", Value: lbl.String()}} + }(), + }, + } { + b.Run(tcase.name, func(b *testing.B) { + var h uint64 + + const prefix = "test-" + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + h = HashWithPrefix(prefix, tcase.lbls) + } + benchmarkLabelsResult = h + }) + } +} From b30021e05a9fbe20de71943e01315c3a79acbecc Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Mon, 19 Apr 2021 18:31:39 +0200 Subject: [PATCH 5/7] Used Prometheus GetRef to avoid reallocating the same series. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` benchstat -delta-test=none ../_dev/thanos/2021/receive2/impr3-go1.16.3.txt ../_dev/thanos/2021/receive2/impr4-go1.16.3.txt name old time/op new time/op delta HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/OK-12 1.64ms ± 0% 1.15ms ± 0% -30.02% HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/conflict_errors-12 8.02ms ± 0% 5.57ms ± 0% -30.53% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/OK-12 16.5ms ± 0% 11.5ms ± 0% -30.28% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/conflict_errors-12 65.1ms ± 0% 58.8ms ± 0% -9.66% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK-12 119ms ± 0% 114ms ± 0% -3.56% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors-12 1.37s ± 0% 1.43s ± 0% +4.58% name old alloc/op new alloc/op delta HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/OK-12 1.63MB ± 0% 1.15MB ± 0% -29.48% HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/conflict_errors-12 4.77MB ± 0% 4.29MB ± 0% -10.07% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/OK-12 17.6MB ± 0% 12.8MB ± 0% -27.20% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/conflict_errors-12 48.9MB ± 0% 44.1MB ± 0% -9.82% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK-12 225MB ± 0% 120MB ± 0% -46.70% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors-12 697MB ± 0% 592MB ± 0% -15.05% name old allocs/op new allocs/op delta HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/OK-12 13.6k ± 0% 3.6k ± 0% -73.58% HandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/conflict_errors-12 33.6k ± 0% 23.6k ± 0% -29.75% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/OK-12 135k ± 0% 35k ± 0% -73.84% HandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/conflict_errors-12 335k ± 0% 235k ± 0% -29.84% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK-12 101 ± 0% 79 ± 0% -21.78% HandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors-12 372 ± 0% 360 ± 0% -3.23% ``` Signed-off-by: Bartlomiej Plotka --- pkg/receive/handler_test.go | 11 ++++++++++- pkg/receive/writer.go | 24 +++++++++++++++--------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/pkg/receive/handler_test.go b/pkg/receive/handler_test.go index a2d5a647fc..98d54409da 100644 --- a/pkg/receive/handler_test.go +++ b/pkg/receive/handler_test.go @@ -227,6 +227,7 @@ type fakeAppender struct { } var _ storage.Appender = &fakeAppender{} +var _ storage.GetRef = &fakeAppender{} func newFakeAppender(appendErr, commitErr, rollbackErr func() error) *fakeAppender { //nolint:unparam if appendErr == nil { @@ -275,6 +276,10 @@ func (f *fakeAppender) AppendExemplar(ref uint64, l labels.Labels, e exemplar.Ex return ref, f.appendErr() } +func (f *fakeAppender) GetRef(l labels.Labels) (uint64, labels.Labels) { + return l.Hash(), l +} + func (f *fakeAppender) Commit() error { return f.commitErr() } @@ -1108,6 +1113,10 @@ func (a *tsOverrideAppender) Append(ref uint64, l labels.Labels, _ int64, v floa return a.Appender.Append(ref, l, cnt, v) } +func (a *tsOverrideAppender) GetRef(lset labels.Labels) (uint64, labels.Labels) { + return a.Appender.(storage.GetRef).GetRef(lset) +} + // serializeSeriesWithOneSample returns marshaled and compressed remote write requests like it would // be send to Thanos receive. // It has one sample and allow passing multiple series, in same manner as typical Prometheus would batch it. @@ -1284,7 +1293,7 @@ func Heap(dir string) (err error) { return err } - f, err := os.Create(filepath.Join(dir, "impr3-go1.16.3.pprof")) + f, err := os.Create(filepath.Join(dir, "impr4-go1.16.3.pprof")) if err != nil { return err } diff --git a/pkg/receive/writer.go b/pkg/receive/writer.go index ed03c1e3b9..5e315c590b 100644 --- a/pkg/receive/writer.go +++ b/pkg/receive/writer.go @@ -57,22 +57,28 @@ func (r *Writer) Write(ctx context.Context, tenantID string, wreq *prompb.WriteR if err != nil { return errors.Wrap(err, "get appender") } + getRef := app.(storage.GetRef) var errs errutil.MultiError for _, t := range wreq.Timeseries { - // Copy labels so we allocate memory only for labels, nothing else. - labelpb.ReAllocZLabelsStrings(&t.Labels) - - // TODO(bwplotka): Use improvement https://github.com/prometheus/prometheus/pull/8600, so we do that only when - // we need it (when we store labels for longer). - lset := labelpb.ZLabelsToPromLabels(t.Labels) + var ( + ref uint64 = 0 + lset = labelpb.ZLabelsToPromLabels(t.Labels) + ) + + // Check if the TSDB has cached reference for those labels. + ref, lset = getRef.GetRef(lset) + if ref == 0 { + // If not, copy labels, as TSDB will hold those strings long term. Given no + // copy unmarshal we don't want to keep memory for whole protobuf, only for labels. + labelpb.ReAllocZLabelsStrings(&t.Labels) + lset = labelpb.ZLabelsToPromLabels(t.Labels) + } // Append as many valid samples as possible, but keep track of the errors. for _, s := range t.Samples { - _, err = app.Append(0, lset, s.Timestamp, s.Value) + ref, err = app.Append(ref, lset, s.Timestamp, s.Value) switch err { - case nil: - continue case storage.ErrOutOfOrderSample: numOutOfOrder++ level.Debug(r.logger).Log("msg", "Out of order sample", "lset", lset.String(), "sample", s.String()) From 0c73663da788f9acba89fce6a7ad038bd3c4b434 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 20 Apr 2021 11:36:19 +0200 Subject: [PATCH 6/7] Build fixes. Signed-off-by: Bartlomiej Plotka --- pkg/receive/hashring.go | 2 -- pkg/receive/writer.go | 10 +++++----- pkg/rules/manager_test.go | 10 +++++++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/pkg/receive/hashring.go b/pkg/receive/hashring.go index 497b42eca8..0de4dbefe1 100644 --- a/pkg/receive/hashring.go +++ b/pkg/receive/hashring.go @@ -14,8 +14,6 @@ import ( "github.com/thanos-io/thanos/pkg/store/storepb/prompb" ) -const sep = '\xff' - // insufficientNodesError is returned when a hashring does not // have enough nodes to satisfy a request for a node. type insufficientNodesError struct { diff --git a/pkg/receive/writer.go b/pkg/receive/writer.go index 5e315c590b..913da33029 100644 --- a/pkg/receive/writer.go +++ b/pkg/receive/writer.go @@ -59,12 +59,12 @@ func (r *Writer) Write(ctx context.Context, tenantID string, wreq *prompb.WriteR } getRef := app.(storage.GetRef) - var errs errutil.MultiError + var ( + ref uint64 + errs errutil.MultiError + ) for _, t := range wreq.Timeseries { - var ( - ref uint64 = 0 - lset = labelpb.ZLabelsToPromLabels(t.Labels) - ) + lset := labelpb.ZLabelsToPromLabels(t.Labels) // Check if the TSDB has cached reference for those labels. ref, lset = getRef.GetRef(lset) diff --git a/pkg/rules/manager_test.go b/pkg/rules/manager_test.go index 5f04764367..d4230da74c 100644 --- a/pkg/rules/manager_test.go +++ b/pkg/rules/manager_test.go @@ -17,6 +17,7 @@ import ( "github.com/go-kit/kit/log" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/pkg/exemplar" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/rules" @@ -35,9 +36,12 @@ func (n nopAppendable) Appender(_ context.Context) storage.Appender { return nop type nopAppender struct{} func (n nopAppender) Append(uint64, labels.Labels, int64, float64) (uint64, error) { return 0, nil } -func (n nopAppender) Commit() error { return nil } -func (n nopAppender) Rollback() error { return nil } -func (n nopAppender) Appender(_ context.Context) (storage.Appender, error) { return n, nil } +func (n nopAppender) AppendExemplar(uint64, labels.Labels, exemplar.Exemplar) (uint64, error) { + return 0, nil +} +func (n nopAppender) Commit() error { return nil } +func (n nopAppender) Rollback() error { return nil } +func (n nopAppender) Appender(_ context.Context) (storage.Appender, error) { return n, nil } type nopQueryable struct{} From e90e75300a54204f603fa403fdfd83caf413f607 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 20 Apr 2021 16:34:50 +0200 Subject: [PATCH 7/7] Fixes. Signed-off-by: Bartlomiej Plotka --- pkg/receive/handler_test.go | 2 +- pkg/receive/hashring.go | 5 +++++ pkg/store/labelpb/label.go | 5 +++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/receive/handler_test.go b/pkg/receive/handler_test.go index 98d54409da..f7dcc7cae2 100644 --- a/pkg/receive/handler_test.go +++ b/pkg/receive/handler_test.go @@ -1293,7 +1293,7 @@ func Heap(dir string) (err error) { return err } - f, err := os.Create(filepath.Join(dir, "impr4-go1.16.3.pprof")) + f, err := os.Create(filepath.Join(dir, "impr5-go1.16.3.pprof")) if err != nil { return err } diff --git a/pkg/receive/hashring.go b/pkg/receive/hashring.go index 0de4dbefe1..b368bf30e7 100644 --- a/pkg/receive/hashring.go +++ b/pkg/receive/hashring.go @@ -6,6 +6,7 @@ package receive import ( "context" "fmt" + "sort" "sync" "github.com/pkg/errors" @@ -65,6 +66,10 @@ func (s simpleHashring) GetN(tenant string, ts *prompb.TimeSeries, n uint64) (st if n >= uint64(len(s)) { return "", &insufficientNodesError{have: uint64(len(s)), want: n + 1} } + + // TODO(bwplotka): This might be not needed, double check. + sort.Slice(ts.Labels, func(i, j int) bool { return ts.Labels[i].Name < ts.Labels[j].Name }) + return s[(labelpb.HashWithPrefix(tenant, ts.Labels)+n)%uint64(len(s))], nil } diff --git a/pkg/store/labelpb/label.go b/pkg/store/labelpb/label.go index 2aa778d71c..5d6e7378cb 100644 --- a/pkg/store/labelpb/label.go +++ b/pkg/store/labelpb/label.go @@ -19,6 +19,8 @@ import ( "github.com/prometheus/prometheus/pkg/labels" ) +var sep = []byte{'\xff'} + func noAllocString(buf []byte) string { return *(*string)(unsafe.Pointer(&buf)) } @@ -316,13 +318,12 @@ func DeepCopy(lbls []ZLabel) []ZLabel { return ret } -var sep = []byte{'\xff'} - // HashWithPrefix returns a hash for the given prefix and labels. func HashWithPrefix(prefix string, lbls []ZLabel) uint64 { // Use xxhash.Sum64(b) for fast path as it's faster. b := make([]byte, 0, 1024) b = append(b, prefix...) + b = append(b, sep[0]) for i, v := range lbls { if len(b)+len(v.Name)+len(v.Value)+2 >= cap(b) {