Store: If request ctx has an error we do not increment opsFailures counter (#3179)

* Store: If request ctx has an error we do not increment opsFailures counter

Signed-off-by: Jarod Watkins <[email protected]>

* Using context.Canceled

Signed-off-by: Jarod Watkins <[email protected]>

* Updating CHANGELOG

Signed-off-by: Jarod Watkins <[email protected]>

Author: ipstatic

CHANGELOG.md

@@ -26,6 +26,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#3136](https://github.com/thanos-io/thanos/pull/3136) Sidecar: Add metric `thanos_sidecar_reloader_config_apply_operations_total` and rename metric `thanos_sidecar_reloader_config_apply_errors_total` to `thanos_sidecar_reloader_config_apply_operations_failed_total`.
 - [#3154](https://github.com/thanos-io/thanos/pull/3154) Query: Add metric `thanos_query_gate_queries_max`. Remove metric `thanos_query_concurrent_selects_gate_queries_in_flight`.
 - [#3154](https://github.com/thanos-io/thanos/pull/3154) Store: Rename metric `thanos_bucket_store_queries_concurrent_max` to `thanos_bucket_store_series_gate_queries_max`.
+- [#3179](https://github.com/thanos-io/thanos/pull/3179) Store: context.Canceled will not increase `thanos_objstore_bucket_operation_failures_total`.
 
 ## [v0.15.0](https://github.com/thanos-io/thanos/releases) - 2020.09.07
 

pkg/objstore/objstore.go

@@ -312,8 +312,10 @@ func (b *metricBucket) Iter(ctx context.Context, dir string, f func(name string)
 	b.ops.WithLabelValues(op).Inc()
 
 	err := b.bkt.Iter(ctx, dir, f)
-	if err != nil && !b.isOpFailureExpected(err) {
-		b.opsFailures.WithLabelValues(op).Inc()
+	if err != nil {
+		if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled {
+			b.opsFailures.WithLabelValues(op).Inc()
+		}
 	}
 	return err
 }
@@ -325,7 +327,7 @@ func (b *metricBucket) Attributes(ctx context.Context, name string) (ObjectAttri
 	start := time.Now()
 	attrs, err := b.bkt.Attributes(ctx, name)
 	if err != nil {
-		if !b.isOpFailureExpected(err) {
+		if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled {
 			b.opsFailures.WithLabelValues(op).Inc()
 		}
 		return attrs, err
@@ -340,7 +342,7 @@ func (b *metricBucket) Get(ctx context.Context, name string) (io.ReadCloser, err
 
 	rc, err := b.bkt.Get(ctx, name)
 	if err != nil {
-		if !b.isOpFailureExpected(err) {
+		if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled {
 			b.opsFailures.WithLabelValues(op).Inc()
 		}
 		return nil, err
@@ -360,7 +362,7 @@ func (b *metricBucket) GetRange(ctx context.Context, name string, off, length in
 
 	rc, err := b.bkt.GetRange(ctx, name, off, length)
 	if err != nil {
-		if !b.isOpFailureExpected(err) {
+		if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled {
 			b.opsFailures.WithLabelValues(op).Inc()
 		}
 		return nil, err
@@ -381,7 +383,7 @@ func (b *metricBucket) Exists(ctx context.Context, name string) (bool, error) {
 	start := time.Now()
 	ok, err := b.bkt.Exists(ctx, name)
 	if err != nil {
-		if !b.isOpFailureExpected(err) {
+		if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled {
 			b.opsFailures.WithLabelValues(op).Inc()
 		}
 		return false, err
@@ -396,7 +398,7 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err
 
 	start := time.Now()
 	if err := b.bkt.Upload(ctx, name, r); err != nil {
-		if !b.isOpFailureExpected(err) {
+		if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled {
 			b.opsFailures.WithLabelValues(op).Inc()
 		}
 		return err
@@ -412,7 +414,7 @@ func (b *metricBucket) Delete(ctx context.Context, name string) error {
 
 	start := time.Now()
 	if err := b.bkt.Delete(ctx, name); err != nil {
-		if !b.isOpFailureExpected(err) {
+		if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled {
 			b.opsFailures.WithLabelValues(op).Inc()
 		}
 		return err