Re-adjust histogram buckets (thanos-io#1769)

Signed-off-by: Kemal Akkoyun <[email protected]>
Signed-off-by: Aleksey Sin <[email protected]>

Author: kakkoyun

cmd/thanos/query.go

@@ -168,7 +168,7 @@ func registerQuery(m map[string]setupFunc, app *kingpin.Application) {
 func storeClientGRPCOpts(logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, secure bool, cert, key, caCert, serverName string) ([]grpc.DialOption, error) {
 	grpcMets := grpc_prometheus.NewClientMetrics()
 	grpcMets.EnableClientHandlingTimeHistogram(
-		grpc_prometheus.WithHistogramBuckets(prometheus.ExponentialBuckets(0.001, 2, 15)),
+		grpc_prometheus.WithHistogramBuckets([]float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}),
 	)
 	dialOpts := []grpc.DialOption{
 		// We want to make sure that we can receive huge gRPC messages from storeAPI.

pkg/compact/compact.go

@@ -83,7 +83,7 @@ func newSyncerMetrics(reg prometheus.Registerer) *syncerMetrics {
 	m.syncMetaDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
 		Name:    "thanos_compact_sync_meta_duration_seconds",
 		Help:    "Time it took to sync meta files.",
-		Buckets: prometheus.ExponentialBuckets(0.001, 2, 15),
+		Buckets: []float64{0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120, 240, 360, 720},
 	})
 
 	m.garbageCollectedBlocks = prometheus.NewCounter(prometheus.CounterOpts{
@@ -101,7 +101,7 @@ func newSyncerMetrics(reg prometheus.Registerer) *syncerMetrics {
 	m.garbageCollectionDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
 		Name:    "thanos_compact_garbage_collection_duration_seconds",
 		Help:    "Time it took to perform garbage collection iteration.",
-		Buckets: prometheus.ExponentialBuckets(0.001, 2, 15),
+		Buckets: []float64{0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120, 240, 360, 720},
 	})
 
 	m.compactions = prometheus.NewCounterVec(prometheus.CounterOpts{

pkg/extprom/http/instrument_server.go

@@ -41,7 +41,7 @@ func NewInstrumentationMiddleware(reg prometheus.Registerer) InstrumentationMidd
 			prometheus.HistogramOpts{
 				Name:    "http_request_duration_seconds",
 				Help:    "Tracks the latencies for HTTP requests.",
-				Buckets: prometheus.ExponentialBuckets(0.001, 2, 17),
+				Buckets: []float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120},
 			},
 			[]string{"code", "handler", "method"},
 		),

pkg/objstore/objstore.go

@@ -204,7 +204,7 @@ func BucketWithMetrics(name string, b Bucket, r prometheus.Registerer) Bucket {
 			Name:        "thanos_objstore_bucket_operation_duration_seconds",
 			Help:        "Duration of operations against the bucket",
 			ConstLabels: prometheus.Labels{"bucket": name},
-			Buckets:     prometheus.ExponentialBuckets(0.001, 2, 17),
+			Buckets:     []float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120},
 		}, []string{"operation"}),
 		lastSuccessfullUploadTime: prometheus.NewGaugeVec(prometheus.GaugeOpts{
 			Name: "thanos_objstore_bucket_last_successful_upload_time",

pkg/server/grpc/grpc.go

@@ -43,7 +43,7 @@ func New(logger log.Logger, reg prometheus.Registerer, tracer opentracing.Tracer
 
 	met := grpc_prometheus.NewServerMetrics()
 	met.EnableHandlingTimeHistogram(
-		grpc_prometheus.WithHistogramBuckets(prometheus.ExponentialBuckets(0.001, 2, 15)),
+		grpc_prometheus.WithHistogramBuckets([]float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}),
 	)
 	panicsTotal := prometheus.NewCounter(prometheus.CounterOpts{
 		Name: "thanos_grpc_req_panics_recovered_total",

pkg/store/bucket.go

@@ -137,12 +137,12 @@ func newBucketStoreMetrics(reg prometheus.Registerer) *bucketStoreMetrics {
 	m.seriesGetAllDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
 		Name:    "thanos_bucket_store_series_get_all_duration_seconds",
 		Help:    "Time it takes until all per-block prepares and preloads for a query are finished.",
-		Buckets: prometheus.ExponentialBuckets(0.01, 2, 15),
+		Buckets: []float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120},
 	})
 	m.seriesMergeDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
 		Name:    "thanos_bucket_store_series_merge_duration_seconds",
 		Help:    "Time it takes to merge sub-results from all queried blocks into a single result.",
-		Buckets: prometheus.ExponentialBuckets(0.01, 2, 15),
+		Buckets: []float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120},
 	})
 	m.resultSeriesCount = prometheus.NewSummary(prometheus.SummaryOpts{
 		Name: "thanos_bucket_store_series_result_series",

pkg/store/gate.go

@@ -26,7 +26,7 @@ func NewGate(maxConcurrent int, reg prometheus.Registerer) *Gate {
 		gateTiming: prometheus.NewHistogram(prometheus.HistogramOpts{
 			Name:    "gate_duration_seconds",
 			Help:    "How many seconds it took for queries to wait at the gate.",
-			Buckets: prometheus.ExponentialBuckets(0.1, 2, 15),
+			Buckets: []float64{0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120, 240, 360, 720},
 		}),
 	}