Introduce http config settings in Azure storage (#4581)

* Introduce `http` config settings in Azure storage

Cortex v1.11.0 included thanos-io/thanos#3970, which added configuration
options to Azure's http client and transport, replacing usage of
`http.DefaultClient`. Unfortunately since Cortex was not setting this
config, Cortex implicitly switched from `http.DefaultClient` to all
empty values (e.g. `MaxIdleConns: 0` rather than 100).

Introduce `http` config settings to Azure storage. This motivated moving
`s3.HTTPConfig` into a new `pkg/storage/bucket/config` package, to allow
`azure` and `s3` to share it.

Also update the instructions for running the website to include
installing `embedmd`.

Signed-off-by: Andrew Seigner <[email protected]>

* feedback: `config.HTTP` -> `http.Config`

also back out changelog cleanup

Signed-off-by: Andrew Seigner <[email protected]>

* Back out accidental changelog addition

Signed-off-by: Andrew Seigner <[email protected]>

Author: siggy

CHANGELOG.md

@@ -11,6 +11,7 @@
 * [ENHANCEMENT] Upgraded Docker base images to `alpine:3.14`. #4514
 * [ENHANCEMENT] Updated Prometheus to latest. Includes changes from prometheus#9239, adding 15 new functions. Multiple TSDB bugfixes prometheus#9438 & prometheus#9381. #4524
 * [ENHANCEMENT] Query Frontend: Add setting `-frontend.forward-headers-list` in frontend  to configure the set of headers from the requests to be forwarded to downstream requests. #4486
+* [ENHANCEMENT] Blocks storage: Add `-blocks-storage.azure.http.*`, `-alertmanager-storage.azure.http.*`, and `-ruler-storage.azure.http.*` to configure the Azure storage client. #4581
 * [BUGFIX] AlertManager: remove stale template files. #4495
 * [BUGFIX] Distributor: fix bug in query-exemplar where some results would get dropped. #4582
 

docs/blocks-storage/querier.md

@@ -278,8 +278,8 @@ blocks_storage:
       # CLI flag: -blocks-storage.s3.http.response-header-timeout
       [response_header_timeout: <duration> | default = 2m]
 
-      # If the client connects to S3 via HTTPS and this option is enabled, the
-      # client will accept any certificate and hostname.
+      # If the client connects via HTTPS and this option is enabled, the client
+      # will accept any certificate and hostname.
       # CLI flag: -blocks-storage.s3.http.insecure-skip-verify
       [insecure_skip_verify: <boolean> | default = false]
 
@@ -340,6 +340,44 @@ blocks_storage:
     # CLI flag: -blocks-storage.azure.max-retries
     [max_retries: <int> | default = 20]
 
+    http:
+      # The time an idle connection will remain idle before closing.
+      # CLI flag: -blocks-storage.azure.http.idle-conn-timeout
+      [idle_conn_timeout: <duration> | default = 1m30s]
+
+      # The amount of time the client will wait for a servers response headers.
+      # CLI flag: -blocks-storage.azure.http.response-header-timeout
+      [response_header_timeout: <duration> | default = 2m]
+
+      # If the client connects via HTTPS and this option is enabled, the client
+      # will accept any certificate and hostname.
+      # CLI flag: -blocks-storage.azure.http.insecure-skip-verify
+      [insecure_skip_verify: <boolean> | default = false]
+
+      # Maximum time to wait for a TLS handshake. 0 means no limit.
+      # CLI flag: -blocks-storage.azure.tls-handshake-timeout
+      [tls_handshake_timeout: <duration> | default = 10s]
+
+      # The time to wait for a server's first response headers after fully
+      # writing the request headers if the request has an Expect header. 0 to
+      # send the request body immediately.
+      # CLI flag: -blocks-storage.azure.expect-continue-timeout
+      [expect_continue_timeout: <duration> | default = 1s]
+
+      # Maximum number of idle (keep-alive) connections across all hosts. 0
+      # means no limit.
+      # CLI flag: -blocks-storage.azure.max-idle-connections
+      [max_idle_connections: <int> | default = 100]
+
+      # Maximum number of idle (keep-alive) connections to keep per-host. If 0,
+      # a built-in default value is used.
+      # CLI flag: -blocks-storage.azure.max-idle-connections-per-host
+      [max_idle_connections_per_host: <int> | default = 100]
+
+      # Maximum number of connections per host. 0 means no limit.
+      # CLI flag: -blocks-storage.azure.max-connections-per-host
+      [max_connections_per_host: <int> | default = 0]
+
   swift:
     # OpenStack Swift authentication API version. 0 to autodetect.
     # CLI flag: -blocks-storage.swift.auth-version

docs/blocks-storage/store-gateway.md

@@ -342,8 +342,8 @@ blocks_storage:
       # CLI flag: -blocks-storage.s3.http.response-header-timeout
       [response_header_timeout: <duration> | default = 2m]
 
-      # If the client connects to S3 via HTTPS and this option is enabled, the
-      # client will accept any certificate and hostname.
+      # If the client connects via HTTPS and this option is enabled, the client
+      # will accept any certificate and hostname.
       # CLI flag: -blocks-storage.s3.http.insecure-skip-verify
       [insecure_skip_verify: <boolean> | default = false]
 
@@ -404,6 +404,44 @@ blocks_storage:
     # CLI flag: -blocks-storage.azure.max-retries
     [max_retries: <int> | default = 20]
 
+    http:
+      # The time an idle connection will remain idle before closing.
+      # CLI flag: -blocks-storage.azure.http.idle-conn-timeout
+      [idle_conn_timeout: <duration> | default = 1m30s]
+
+      # The amount of time the client will wait for a servers response headers.
+      # CLI flag: -blocks-storage.azure.http.response-header-timeout
+      [response_header_timeout: <duration> | default = 2m]
+
+      # If the client connects via HTTPS and this option is enabled, the client
+      # will accept any certificate and hostname.
+      # CLI flag: -blocks-storage.azure.http.insecure-skip-verify
+      [insecure_skip_verify: <boolean> | default = false]
+
+      # Maximum time to wait for a TLS handshake. 0 means no limit.
+      # CLI flag: -blocks-storage.azure.tls-handshake-timeout
+      [tls_handshake_timeout: <duration> | default = 10s]
+
+      # The time to wait for a server's first response headers after fully
+      # writing the request headers if the request has an Expect header. 0 to
+      # send the request body immediately.
+      # CLI flag: -blocks-storage.azure.expect-continue-timeout
+      [expect_continue_timeout: <duration> | default = 1s]
+
+      # Maximum number of idle (keep-alive) connections across all hosts. 0
+      # means no limit.
+      # CLI flag: -blocks-storage.azure.max-idle-connections
+      [max_idle_connections: <int> | default = 100]
+
+      # Maximum number of idle (keep-alive) connections to keep per-host. If 0,
+      # a built-in default value is used.
+      # CLI flag: -blocks-storage.azure.max-idle-connections-per-host
+      [max_idle_connections_per_host: <int> | default = 100]
+
+      # Maximum number of connections per host. 0 means no limit.
+      # CLI flag: -blocks-storage.azure.max-connections-per-host
+      [max_connections_per_host: <int> | default = 0]
+
   swift:
     # OpenStack Swift authentication API version. 0 to autodetect.
     # CLI flag: -blocks-storage.swift.auth-version

docs/configuration/config-file-reference.md

@@ -1714,8 +1714,8 @@ s3:
     # CLI flag: -ruler-storage.s3.http.response-header-timeout
     [response_header_timeout: <duration> | default = 2m]
 
-    # If the client connects to S3 via HTTPS and this option is enabled, the
-    # client will accept any certificate and hostname.
+    # If the client connects via HTTPS and this option is enabled, the client
+    # will accept any certificate and hostname.
     # CLI flag: -ruler-storage.s3.http.insecure-skip-verify
     [insecure_skip_verify: <boolean> | default = false]
 
@@ -1776,6 +1776,44 @@ azure:
   # CLI flag: -ruler-storage.azure.max-retries
   [max_retries: <int> | default = 20]
 
+  http:
+    # The time an idle connection will remain idle before closing.
+    # CLI flag: -ruler-storage.azure.http.idle-conn-timeout
+    [idle_conn_timeout: <duration> | default = 1m30s]
+
+    # The amount of time the client will wait for a servers response headers.
+    # CLI flag: -ruler-storage.azure.http.response-header-timeout
+    [response_header_timeout: <duration> | default = 2m]
+
+    # If the client connects via HTTPS and this option is enabled, the client
+    # will accept any certificate and hostname.
+    # CLI flag: -ruler-storage.azure.http.insecure-skip-verify
+    [insecure_skip_verify: <boolean> | default = false]
+
+    # Maximum time to wait for a TLS handshake. 0 means no limit.
+    # CLI flag: -ruler-storage.azure.tls-handshake-timeout
+    [tls_handshake_timeout: <duration> | default = 10s]
+
+    # The time to wait for a server's first response headers after fully writing
+    # the request headers if the request has an Expect header. 0 to send the
+    # request body immediately.
+    # CLI flag: -ruler-storage.azure.expect-continue-timeout
+    [expect_continue_timeout: <duration> | default = 1s]
+
+    # Maximum number of idle (keep-alive) connections across all hosts. 0 means
+    # no limit.
+    # CLI flag: -ruler-storage.azure.max-idle-connections
+    [max_idle_connections: <int> | default = 100]
+
+    # Maximum number of idle (keep-alive) connections to keep per-host. If 0, a
+    # built-in default value is used.
+    # CLI flag: -ruler-storage.azure.max-idle-connections-per-host
+    [max_idle_connections_per_host: <int> | default = 100]
+
+    # Maximum number of connections per host. 0 means no limit.
+    # CLI flag: -ruler-storage.azure.max-connections-per-host
+    [max_connections_per_host: <int> | default = 0]
+
 swift:
   # OpenStack Swift authentication API version. 0 to autodetect.
   # CLI flag: -ruler-storage.swift.auth-version
@@ -2258,8 +2296,8 @@ s3:
     # CLI flag: -alertmanager-storage.s3.http.response-header-timeout
     [response_header_timeout: <duration> | default = 2m]
 
-    # If the client connects to S3 via HTTPS and this option is enabled, the
-    # client will accept any certificate and hostname.
+    # If the client connects via HTTPS and this option is enabled, the client
+    # will accept any certificate and hostname.
     # CLI flag: -alertmanager-storage.s3.http.insecure-skip-verify
     [insecure_skip_verify: <boolean> | default = false]
 
@@ -2320,6 +2358,44 @@ azure:
   # CLI flag: -alertmanager-storage.azure.max-retries
   [max_retries: <int> | default = 20]
 
+  http:
+    # The time an idle connection will remain idle before closing.
+    # CLI flag: -alertmanager-storage.azure.http.idle-conn-timeout
+    [idle_conn_timeout: <duration> | default = 1m30s]
+
+    # The amount of time the client will wait for a servers response headers.
+    # CLI flag: -alertmanager-storage.azure.http.response-header-timeout
+    [response_header_timeout: <duration> | default = 2m]
+
+    # If the client connects via HTTPS and this option is enabled, the client
+    # will accept any certificate and hostname.
+    # CLI flag: -alertmanager-storage.azure.http.insecure-skip-verify
+    [insecure_skip_verify: <boolean> | default = false]
+
+    # Maximum time to wait for a TLS handshake. 0 means no limit.
+    # CLI flag: -alertmanager-storage.azure.tls-handshake-timeout
+    [tls_handshake_timeout: <duration> | default = 10s]
+
+    # The time to wait for a server's first response headers after fully writing
+    # the request headers if the request has an Expect header. 0 to send the
+    # request body immediately.
+    # CLI flag: -alertmanager-storage.azure.expect-continue-timeout
+    [expect_continue_timeout: <duration> | default = 1s]
+
+    # Maximum number of idle (keep-alive) connections across all hosts. 0 means
+    # no limit.
+    # CLI flag: -alertmanager-storage.azure.max-idle-connections
+    [max_idle_connections: <int> | default = 100]
+
+    # Maximum number of idle (keep-alive) connections to keep per-host. If 0, a
+    # built-in default value is used.
+    # CLI flag: -alertmanager-storage.azure.max-idle-connections-per-host
+    [max_idle_connections_per_host: <int> | default = 100]
+
+    # Maximum number of connections per host. 0 means no limit.
+    # CLI flag: -alertmanager-storage.azure.max-connections-per-host
+    [max_connections_per_host: <int> | default = 0]
+
 swift:
   # OpenStack Swift authentication API version. 0 to autodetect.
   # CLI flag: -alertmanager-storage.swift.auth-version
@@ -4565,8 +4641,8 @@ s3:
     # CLI flag: -blocks-storage.s3.http.response-header-timeout
     [response_header_timeout: <duration> | default = 2m]
 
-    # If the client connects to S3 via HTTPS and this option is enabled, the
-    # client will accept any certificate and hostname.
+    # If the client connects via HTTPS and this option is enabled, the client
+    # will accept any certificate and hostname.
     # CLI flag: -blocks-storage.s3.http.insecure-skip-verify
     [insecure_skip_verify: <boolean> | default = false]
 
@@ -4627,6 +4703,44 @@ azure:
   # CLI flag: -blocks-storage.azure.max-retries
   [max_retries: <int> | default = 20]
 
+  http:
+    # The time an idle connection will remain idle before closing.
+    # CLI flag: -blocks-storage.azure.http.idle-conn-timeout
+    [idle_conn_timeout: <duration> | default = 1m30s]
+
+    # The amount of time the client will wait for a servers response headers.
+    # CLI flag: -blocks-storage.azure.http.response-header-timeout
+    [response_header_timeout: <duration> | default = 2m]
+
+    # If the client connects via HTTPS and this option is enabled, the client
+    # will accept any certificate and hostname.
+    # CLI flag: -blocks-storage.azure.http.insecure-skip-verify
+    [insecure_skip_verify: <boolean> | default = false]
+
+    # Maximum time to wait for a TLS handshake. 0 means no limit.
+    # CLI flag: -blocks-storage.azure.tls-handshake-timeout
+    [tls_handshake_timeout: <duration> | default = 10s]
+
+    # The time to wait for a server's first response headers after fully writing
+    # the request headers if the request has an Expect header. 0 to send the
+    # request body immediately.
+    # CLI flag: -blocks-storage.azure.expect-continue-timeout
+    [expect_continue_timeout: <duration> | default = 1s]
+
+    # Maximum number of idle (keep-alive) connections across all hosts. 0 means
+    # no limit.
+    # CLI flag: -blocks-storage.azure.max-idle-connections
+    [max_idle_connections: <int> | default = 100]
+
+    # Maximum number of idle (keep-alive) connections to keep per-host. If 0, a
+    # built-in default value is used.
+    # CLI flag: -blocks-storage.azure.max-idle-connections-per-host
+    [max_idle_connections_per_host: <int> | default = 100]
+
+    # Maximum number of connections per host. 0 means no limit.
+    # CLI flag: -blocks-storage.azure.max-connections-per-host
+    [max_connections_per_host: <int> | default = 0]
+
 swift:
   # OpenStack Swift authentication API version. 0 to autodetect.
   # CLI flag: -blocks-storage.swift.auth-version

docs/contributing/how-to-run-website-locally.md

@@ -18,7 +18,11 @@ The following initial setup is required only once:
    ```
    cd website && npm install && cd -
    ```
-4. Run `make BUILD_IN_CONTAINER=false web-build`
+4. Install [embedmd](https://github.com/campoy/embedmd) `v1.0.0`:
+   ```
+   go install github.com/campoy/[email protected]
+   ```
+5. Run `make BUILD_IN_CONTAINER=false web-build`
 
 
 ## Run it

pkg/storage/bucket/azure/bucket_client.go

@@ -2,6 +2,7 @@ package azure
 
 import (
 	"github.com/go-kit/log"
+	"github.com/prometheus/common/model"
 	"github.com/thanos-io/thanos/pkg/objstore"
 	"github.com/thanos-io/thanos/pkg/objstore/azure"
 	yaml "gopkg.in/yaml.v2"
@@ -14,6 +15,16 @@ func NewBucketClient(cfg Config, name string, logger log.Logger) (objstore.Bucke
 		ContainerName:      cfg.ContainerName,
 		Endpoint:           cfg.Endpoint,
 		MaxRetries:         cfg.MaxRetries,
+		HTTPConfig: azure.HTTPConfig{
+			IdleConnTimeout:       model.Duration(cfg.IdleConnTimeout),
+			ResponseHeaderTimeout: model.Duration(cfg.ResponseHeaderTimeout),
+			InsecureSkipVerify:    cfg.InsecureSkipVerify,
+			TLSHandshakeTimeout:   model.Duration(cfg.TLSHandshakeTimeout),
+			ExpectContinueTimeout: model.Duration(cfg.ExpectContinueTimeout),
+			MaxIdleConns:          cfg.MaxIdleConns,
+			MaxIdleConnsPerHost:   cfg.MaxIdleConnsPerHost,
+			MaxConnsPerHost:       cfg.MaxConnsPerHost,
+		},
 	}
 
 	// Thanos currently doesn't support passing the config as is, but expects a YAML,

pkg/storage/bucket/azure/config.go

@@ -4,6 +4,8 @@ import (
 	"flag"
 
 	"github.com/grafana/dskit/flagext"
+
+	"github.com/cortexproject/cortex/pkg/storage/bucket/http"
 )
 
 // Config holds the config options for an Azure backend
@@ -13,6 +15,8 @@ type Config struct {
 	ContainerName      string         `yaml:"container_name"`
 	Endpoint           string         `yaml:"endpoint_suffix"`
 	MaxRetries         int            `yaml:"max_retries"`
+
+	http.Config `yaml:"http"`
 }
 
 // RegisterFlags registers the flags for Azure storage
@@ -27,4 +31,5 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 	f.StringVar(&cfg.ContainerName, prefix+"azure.container-name", "", "Azure storage container name")
 	f.StringVar(&cfg.Endpoint, prefix+"azure.endpoint-suffix", "", "Azure storage endpoint suffix without schema. The account name will be prefixed to this value to create the FQDN")
 	f.IntVar(&cfg.MaxRetries, prefix+"azure.max-retries", 20, "Number of retries for recoverable errors")
+	cfg.Config.RegisterFlagsWithPrefix(prefix+"azure.", f)
 }