Add TLS support to HTTP/GRPC clients

cmd/query-tee/main.go

@@ -3,23 +3,23 @@ package main
 import (
 	"flag"
 	"os"
-	"time"
 
 	"github.com/go-kit/kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/weaveworks/common/logging"
 	"github.com/weaveworks/common/server"
 
 	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/cortexproject/cortex/pkg/util/httpclient"
 )
 
 type Config struct {
-	ServerServicePort  int
-	ServerMetricsPort  int
-	BackendEndpoints   string
-	PreferredBackend   string
-	BackendReadTimeout time.Duration
-	LogLevel           logging.Level
+	ServerServicePort int
+	ServerMetricsPort int
+	BackendEndpoints  string
+	PreferredBackend  string
+	ClientConfig      httpclient.Config
+	LogLevel          logging.Level
 }
 
 func main() {
@@ -29,7 +29,7 @@ func main() {
 	flag.IntVar(&cfg.ServerMetricsPort, "server.metrics-port", 9900, "The port where metrics are exposed.")
 	flag.StringVar(&cfg.BackendEndpoints, "backend.endpoints", "", "Comma separated list of backend endpoints to query.")
 	flag.StringVar(&cfg.PreferredBackend, "backend.preferred", "", "The hostname of the preferred backend when selecting the response to send back to the client.")
-	flag.DurationVar(&cfg.BackendReadTimeout, "backend.read-timeout", 90*time.Second, "The timeout when reading the response from a backend.")
+	cfg.ClientConfig.RegisterFlags(flag.CommandLine)
 	cfg.LogLevel.RegisterFlags(flag.CommandLine)
 	flag.Parse()
 

cmd/query-tee/proxy.go

@@ -11,6 +11,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/cortexproject/cortex/pkg/util/flagext"
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
 	"github.com/gorilla/mux"
@@ -69,7 +70,9 @@ func NewProxy(cfg Config, logger log.Logger, registerer prometheus.Registerer) (
 			preferred = preferredIdx == idx
 		}
 
-		p.backends = append(p.backends, NewProxyBackend(name, u, cfg.BackendReadTimeout, preferred))
+		clientConfig := cfg.ClientConfig
+		clientConfig.Endpoint = flagext.URLValue{URL: u}
+		p.backends = append(p.backends, NewProxyBackend(name, &clientConfig, preferred))
 	}
 
 	// At least 1 backend is required

cmd/query-tee/proxy_backend.go

@@ -5,46 +5,50 @@ import (
 	"io/ioutil"
 	"net"
 	"net/http"
-	"net/url"
 	"path"
 	"time"
 
 	"github.com/pkg/errors"
+
+	"github.com/cortexproject/cortex/pkg/util/httpclient"
 )
 
 // ProxyBackend holds the information of a single backend.
 type ProxyBackend struct {
-	name     string
-	endpoint *url.URL
-	client   *http.Client
-	timeout  time.Duration
+	name         string
+	clientConfig *httpclient.Config
+	client       *http.Client
 
 	// Whether this is the preferred backend from which picking up
 	// the response and sending it back to the client.
 	preferred bool
 }
 
 // NewProxyBackend makes a new ProxyBackend
-func NewProxyBackend(name string, endpoint *url.URL, timeout time.Duration, preferred bool) *ProxyBackend {
+func NewProxyBackend(name string, config *httpclient.Config, preferred bool) *ProxyBackend {
+	roundTripper := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   config.ClientTimeout,
+			KeepAlive: 30 * time.Second,
+		}).DialContext,
+		MaxIdleConns:        100,
+		MaxIdleConnsPerHost: 100, // see https://github.com/golang/go/issues/13801
+		IdleConnTimeout:     config.BackendReadTimeout,
+	}
+	if tlsConfig := config.GetTLSConfig(); tlsConfig != nil {
+		roundTripper.TLSClientConfig = tlsConfig
+	}
+
 	return &ProxyBackend{
-		name:      name,
-		endpoint:  endpoint,
-		timeout:   timeout,
-		preferred: preferred,
+		name:         name,
+		clientConfig: config,
+		preferred:    preferred,
 		client: &http.Client{
 			CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
 				return errors.New("the query-tee proxy does not follow redirects")
 			},
-			Transport: &http.Transport{
-				Proxy: http.ProxyFromEnvironment,
-				DialContext: (&net.Dialer{
-					Timeout:   30 * time.Second,
-					KeepAlive: 30 * time.Second,
-				}).DialContext,
-				MaxIdleConns:        100,
-				MaxIdleConnsPerHost: 100, // see https://github.com/golang/go/issues/13801
-				IdleConnTimeout:     90 * time.Second,
-			},
+			Transport: roundTripper,
 		},
 	}
 }
@@ -65,19 +69,19 @@ func (b *ProxyBackend) createBackendRequest(orig *http.Request) (*http.Request,
 	}
 
 	// Replace the endpoint with the backend one.
-	req.URL.Scheme = b.endpoint.Scheme
-	req.URL.Host = b.endpoint.Host
+	req.URL.Scheme = b.clientConfig.Endpoint.Scheme
+	req.URL.Host = b.clientConfig.Endpoint.Host
 
 	// Prepend the endpoint path to the request path.
-	req.URL.Path = path.Join(b.endpoint.Path, req.URL.Path)
+	req.URL.Path = path.Join(b.clientConfig.Endpoint.Path, req.URL.Path)
 
 	// Replace the auth:
 	// - If the endpoint has user and password, use it.
 	// - If the endpoint has user only, keep it and use the request password (if any).
 	// - If the endpoint has no user and no password, use the request auth (if any).
 	clientUser, clientPass, clientAuth := orig.BasicAuth()
-	endpointUser := b.endpoint.User.Username()
-	endpointPass, _ := b.endpoint.User.Password()
+	endpointUser := b.clientConfig.Endpoint.User.Username()
+	endpointPass, _ := b.clientConfig.Endpoint.User.Password()
 
 	if endpointUser != "" && endpointPass != "" {
 		req.SetBasicAuth(endpointUser, endpointPass)
@@ -92,7 +96,7 @@ func (b *ProxyBackend) createBackendRequest(orig *http.Request) (*http.Request,
 
 func (b *ProxyBackend) doBackendRequest(req *http.Request) (int, []byte, error) {
 	// Honor the read timeout.
-	ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
+	ctx, cancel := context.WithTimeout(context.Background(), b.clientConfig.ClientTimeout)
 	defer cancel()
 
 	// Execute the request.

cmd/query-tee/proxy_backend_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/cortexproject/cortex/pkg/util/httpclient"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -61,7 +62,7 @@ func Test_ProxyBackend_createBackendRequest_HTTPBasicAuthentication(t *testing.T
 			orig := httptest.NewRequest("GET", "/test", nil)
 			orig.SetBasicAuth(testData.clientUser, testData.clientPass)
 
-			b := NewProxyBackend("test", u, time.Second, false)
+			b := NewProxyBackend("test", httpclient.ConfigFromURLAndTimeout(u, time.Second), false)
 			r, err := b.createBackendRequest(orig)
 			require.NoError(t, err)
 

cmd/query-tee/proxy_test.go

@@ -13,6 +13,8 @@ import (
 	"github.com/go-kit/kit/log"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/cortexproject/cortex/pkg/util/httpclient"
 )
 
 func Test_NewProxy(t *testing.T) {
@@ -146,11 +148,13 @@ func Test_Proxy_RequestsForwarding(t *testing.T) {
 
 			// Start the proxy.
 			cfg := Config{
-				BackendEndpoints:   strings.Join(backendURLs, ","),
-				PreferredBackend:   strconv.Itoa(testData.preferredBackendIdx),
-				ServerServicePort:  0,
-				ServerMetricsPort:  0,
-				BackendReadTimeout: time.Second,
+				BackendEndpoints:  strings.Join(backendURLs, ","),
+				PreferredBackend:  strconv.Itoa(testData.preferredBackendIdx),
+				ServerServicePort: 0,
+				ServerMetricsPort: 0,
+				ClientConfig: httpclient.Config{
+					ClientTimeout: time.Second,
+				},
 			}
 
 			p, err := NewProxy(cfg, log.NewNopLogger(), nil)

docs/production/tls.md

@@ -0,0 +1,108 @@
+---
+title: "Securing communication between cortex components with TLS"
+linkTitle: "Securing communication between cortex components with TLS"
+weight: 5
+slug: tls
+---
+
+Cortex is a distributed system with significant traffic between its services. 
+To allow for secure communication, Cortex supports TLS between all its 
+components. This guide describes the process of setting up TLS.
+
+### Generation of certs to configure TLS
+
+The first step to securing inter-service communication in Cortex with TLS is 
+generating certificates. A Certifying Authority (CA) will be used for this 
+purpose which should be private to the organization, as any certificates signed 
+by this CA will have permissions to communicate with the cluster. 
+
+We will use the following script to generate self signed certs for the cluster:
+
+```
+# Refer: https://github.com/joe-elliott/cert-exporter/blob/69d3d7230378325a1de4fa313432d3d6ced4a518/test/files/genCerts.sh
+
+# keys
+openssl genrsa -out root.key
+openssl genrsa -out client.key
+openssl genrsa -out server.key
+
+# root cert / certifying authority
+openssl req -x509 -new -nodes -key root.key -subj "/C=US/ST=KY/O=Org/CN=root" -sha256 -days 100000 -out root.crt
+
+# csrs - certificate signing requests
+openssl req -new -sha256 -key client.key -subj "/C=US/ST=KY/O=Org/CN=client" -out client.csr
+openssl req -new -sha256 -key server.key -subj "/C=US/ST=KY/O=Org/CN=localhost" -out server.csr
+
+# certificates
+openssl x509 -req -in client.csr -CA root.crt -CAkey root.key -CAcreateserial -out client.crt -days 100000 -sha256
+openssl x509 -req -in server.csr -CA root.crt -CAkey root.key -CAcreateserial -out server.crt -days 100000 -sha256
+```
+
+Note that the above script generates certificates that are valid for 100000 days.
+This can be changed by adjusting the `-days` option in the above commands.
+It is recommended that the certs be replaced atleast once every 2 years.
+
+The above script generates keys `client.key, server.key` and certs 
+`client.crt, server.crt` for both the client and server. The CA cert is 
+generated as `root.crt`. 
+
+### Load certs into the HTTP/GRPC server/client
+
+Every HTTP/GRPC link between Cortex components supports TLS configuration 
+through the following config parameters:
+
+#### Server flags
+
+```
+    # Path to the TLS Cert for the HTTP Server
+    -server.http-tls-cert-path=/path/to/server.crt
+
+    # Path to the TLS Key for the HTTP Server
+    -server.http-tls-key-path=/path/to/server.key
+
+    # Type of Client Auth for the HTTP Server
+    -server.http-tls-client-auth="RequireAndVerifyClientCert"
+
+    # Path to the Client CA Cert for the HTTP Server
+    -server.http-tls-ca-path="/path/to/root.crt"
+
+    # Path to the TLS Cert for the GRPC Server
+    -server.grpc-tls-cert-path=/path/to/server.crt
+
+    # Path to the TLS Key for the GRPC Server
+    -server.grpc-tls-key-path=/path/to/server.key
+
+    # Type of Client Auth for the GRPC Server
+    -server.grpc-tls-client-auth="RequireAndVerifyClientCert"
+
+    # Path to the Client CA Cert for the GRPC Server
+    -server.grpc-tls-ca-path=/path/to/root.crt
+```
+
+#### Client flags
+
+Client flags are component specific. 
+
+For an HTTP client in the alertmanager:
+```
+    # Path to the TLS Cert for the HTTP Client
+    -alertmanager.client.http-tls-cert-path=/path/to/client.crt
+
+    # Path to the TLS Key for the HTTP Client
+    -alertmanager.client.http-tls-key-path=/path/to/client.key
+
+    # Path to the TLS CA for the HTTP Client
+    -alertmanager.client.http-tls-ca-path=/path/to/root.crt
+```
+
+For a GRPC client in the ruler:
+```
+    # Path to the TLS Cert for the GRPC Client
+    -ruler.grpc-tls-cert-path=/path/to/client.crt
+
+    # Path to the TLS Key for the GRPC Client
+    -ruler.grpc-tls-key-path=/path/to/client.key
+
+    # Path to the TLS CA for the GRPC Client
+    -ruler.grpc-tls-ca-path=/path/to/root.crt
+```

integration/certs/genCerts.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Copied from https://github.com/joe-elliott/cert-exporter/blob/5ce49ebf6bfcdcb178d31145ae2a460f3b348cf5/test/files/genCerts.sh
+# Copyright [2020] [cert-exporter authors]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+certFolder=$1
+days=$2
+
+pushd $certFolder
+
+# keys
+openssl genrsa -out root.key
+openssl genrsa -out client.key
+openssl genrsa -out server.key
+
+# root cert
+openssl req -x509 -new -nodes -key root.key -subj "/C=US/ST=KY/O=Org/CN=root" -sha256 -days $days -out root.crt
+
+# csrs
+openssl req -new -sha256 -key client.key -subj "/C=US/ST=KY/O=Org/CN=client" -out client.csr
+openssl req -new -sha256 -key server.key -subj "/C=US/ST=KY/O=Org/CN=localhost" -out server.csr
+
+openssl x509 -req -in client.csr -CA root.crt -CAkey root.key -CAcreateserial -out client.crt -days $days -sha256
+openssl x509 -req -in server.csr -CA root.crt -CAkey root.key -CAcreateserial -out server.crt -days $days -sha256
+
+popd

integration/configs.go

@@ -22,16 +22,22 @@ const (
 	cortexConfigFile       = "config.yaml"
 	cortexSchemaConfigFile = "schema.yaml"
 	blocksStorageEngine    = "tsdb"
+	certsFolder            = "integration/certs/"
+	clientCertFile         = "certs/client.crt"
+	clientKeyFile          = "certs/client.key"
+	rootCertFile           = "certs/root.crt"
+	serverCertFile         = "certs/server.crt"
+	serverKeyFile          = "certs/server.key"
 	storeConfigTemplate    = `
 - from: {{.From}}
   store: {{.IndexStore}}
   schema: v9
   index:
     prefix: cortex_
-    period: 168h 
+    period: 168h
   chunks:
     prefix: cortex_chunks_
-    period: 168h 
+    period: 168h
 `
 
 	cortexAlertmanagerUserConfigYaml = `route:
@@ -120,7 +126,7 @@ storage:
 
 table_manager:
   poll_interval:    1m
-  retention_period: 168h 
+  retention_period: 168h
 
 schema:
 {{.SchemaConfig}}