Add timeout on http requests for kv store DDB client

Author: anna-tran

CHANGELOG.md

@@ -7,6 +7,7 @@
 * [BUGFIX] Ruler: Validate if rule group can be safely converted back to rule group yaml from protobuf message #5265
 * [BUGFIX] Querier: Convert gRPC `ResourceExhausted` status code from store gateway to 422 limit error. #5286
 * [BUGFIX] Alertmanager: Route web-ui requests to the alertmanager distributor when sharding is enabled. #5293
+* [BUGFIX] Add timeout on http requests for kv store DynamoDB client. #6211
 
 ## 1.15.1 2023-04-26
 

docs/configuration/config-file-reference.md

@@ -314,6 +314,10 @@ sharding_ring:
       # CLI flag: -alertmanager.sharding-ring.dynamodb.table-name
       [table_name: <string> | default = ""]
 
+      # HTTP timeout when talking to dynamodb
+      # CLI flag: -alertmanager.sharding-ring.dynamodb.client-timeout
+      [http_client_timeout: <duration> | default = 20s]
+
       # Time to expire items on dynamodb.
       # CLI flag: -alertmanager.sharding-ring.dynamodb.ttl-time
       [ttl: <duration> | default = 0s]
@@ -1871,6 +1875,10 @@ sharding_ring:
       # CLI flag: -compactor.ring.dynamodb.table-name
       [table_name: <string> | default = ""]
 
+      # HTTP timeout when talking to dynamodb
+      # CLI flag: -compactor.ring.dynamodb.client-timeout
+      [http_client_timeout: <duration> | default = 20s]
+
       # Time to expire items on dynamodb.
       # CLI flag: -compactor.ring.dynamodb.ttl-time
       [ttl: <duration> | default = 0s]
@@ -2106,6 +2114,10 @@ ha_tracker:
       # CLI flag: -distributor.ha-tracker.dynamodb.table-name
       [table_name: <string> | default = ""]
 
+      # HTTP timeout when talking to dynamodb
+      # CLI flag: -distributor.ha-tracker.dynamodb.client-timeout
+      [http_client_timeout: <duration> | default = 20s]
+
       # Time to expire items on dynamodb.
       # CLI flag: -distributor.ha-tracker.dynamodb.ttl-time
       [ttl: <duration> | default = 0s]
@@ -2183,6 +2195,10 @@ ring:
       # CLI flag: -distributor.ring.dynamodb.table-name
       [table_name: <string> | default = ""]
 
+      # HTTP timeout when talking to dynamodb
+      # CLI flag: -distributor.ring.dynamodb.client-timeout
+      [http_client_timeout: <duration> | default = 20s]
+
       # Time to expire items on dynamodb.
       # CLI flag: -distributor.ring.dynamodb.ttl-time
       [ttl: <duration> | default = 0s]
@@ -2471,6 +2487,10 @@ lifecycler:
         # CLI flag: -dynamodb.table-name
         [table_name: <string> | default = ""]
 
+        # HTTP timeout when talking to dynamodb
+        # CLI flag: -dynamodb.client-timeout
+        [http_client_timeout: <duration> | default = 20s]
+
         # Time to expire items on dynamodb.
         # CLI flag: -dynamodb.ttl-time
         [ttl: <duration> | default = 0s]
@@ -3842,6 +3862,10 @@ ring:
       # CLI flag: -ruler.ring.dynamodb.table-name
       [table_name: <string> | default = ""]
 
+      # HTTP timeout when talking to dynamodb
+      # CLI flag: -ruler.ring.dynamodb.client-timeout
+      [http_client_timeout: <duration> | default = 20s]
+
       # Time to expire items on dynamodb.
       # CLI flag: -ruler.ring.dynamodb.ttl-time
       [ttl: <duration> | default = 0s]
@@ -4437,6 +4461,10 @@ sharding_ring:
       # CLI flag: -store-gateway.sharding-ring.dynamodb.table-name
       [table_name: <string> | default = ""]
 
+      # HTTP timeout when talking to dynamodb
+      # CLI flag: -store-gateway.sharding-ring.dynamodb.client-timeout
+      [http_client_timeout: <duration> | default = 20s]
+
       # Time to expire items on dynamodb.
       # CLI flag: -store-gateway.sharding-ring.dynamodb.ttl-time
       [ttl: <duration> | default = 0s]

pkg/ring/kv/dynamodb/client.go

@@ -15,11 +15,12 @@ import (
 	"github.com/cortexproject/cortex/pkg/util/backoff"
 )
 
-// Config to create a ConsulClient
+// Config to create a DynamoDBClient
 type Config struct {
-	Region    string        `yaml:"region"`
-	TableName string        `yaml:"table_name"`
-	TTL       time.Duration `yaml:"ttl"`
+	Region            string        `yaml:"region"`
+	TableName         string        `yaml:"table_name"`
+	HTTPClientTimeout time.Duration `yaml:"http_client_timeout"`
+	TTL               time.Duration `yaml:"ttl"`
 }
 
 type Client struct {
@@ -52,6 +53,7 @@ var (
 func (cfg *Config) RegisterFlags(f *flag.FlagSet, prefix string) {
 	f.StringVar(&cfg.Region, prefix+"dynamodb.region", "", "Region to access dynamodb.")
 	f.StringVar(&cfg.TableName, prefix+"dynamodb.table-name", "", "Table name to use on dynamodb.")
+	f.DurationVar(&cfg.TTL, prefix+"dynamodb.client-timeout", 20*time.Second, "Timeout for HTTP requests to dynamodb.")
 	f.DurationVar(&cfg.TTL, prefix+"dynamodb.ttl-time", 0, "Time to expire items on dynamodb.")
 }
 

pkg/ring/kv/dynamodb/dynamodb.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"math"
+	"net/http"
 	"strconv"
 	"time"
 
@@ -59,7 +60,8 @@ func newDynamodbKV(cfg Config, logger log.Logger) (dynamodbKV, error) {
 
 	if len(cfg.Region) > 0 {
 		sess.Config = &aws.Config{
-			Region: aws.String(cfg.Region),
+			Region:     aws.String(cfg.Region),
+			HTTPClient: &http.Client{Timeout: cfg.HTTPClientTimeout},
 		}
 	}