Randomize gossip node name by adding random suffix.

CHANGELOG.md

@@ -48,6 +48,7 @@
   * `cortex_<service>_blocks_sync_seconds` => `cortex_bucket_stores_blocks_sync_seconds{component="<service>"}`
   * `cortex_<service>_blocks_last_successful_sync_timestamp_seconds` => `cortex_bucket_stores_blocks_last_successful_sync_timestamp_seconds{component="<service>"}`
 * [CHANGE] Available command-line flags are printed to stdout, and only when requested via `-help`. Using invalid flag no longer causes printing of all available flags. #2691
+* [CHANGE] Experimental Memberlist ring: randomize gossip node names to avoid conflicts when running multiple clients on the same host, or reusing host names (eg. pods in statefulset). #2715
 * [FEATURE] TLS config options added for GRPC clients in Querier (Query-frontend client & Ingester client), Ruler, Store Gateway, as well as HTTP client in Config store client. #2502
 * [FEATURE] The flag `-frontend.max-cache-freshness` is now supported within the limits overrides, to specify per-tenant max cache freshness values. The corresponding YAML config parameter has been changed from `results_cache.max_freshness` to `limits_config.max_cache_freshness`. The legacy YAML config parameter (`results_cache.max_freshness`) will continue to be supported till Cortex release `v1.4.0`. #2609
 * [FEATURE] Experimental gRPC Store: Added support to 3rd parties index and chunk stores using gRPC client/server plugin mechanism. #2220

docs/configuration/config-file-reference.md

@@ -2267,6 +2267,10 @@ The `memberlist_config` configures the Gossip memberlist.
 # CLI flag: -memberlist.nodename
 [node_name: <string> | default = ""]
 
+# Add random suffix to the node name.
+# CLI flag: -memberlist.randomize-node-name
+[randomize_node_name: <boolean> | default = true]
+
 # The timeout for establishing a connection with a remote node, and for
 # read/write operations. Uses memberlist LAN defaults if 0.
 # CLI flag: -memberlist.stream-timeout

docs/configuration/single-process-config-blocks-gossip-1.yaml

@@ -59,8 +59,6 @@ ingester:
       replication_factor: 1
 
 memberlist:
-  # defaults to hostname
-  node_name: "Ingester 1"
   bind_port: 7946
   join_members:
     - localhost:7947

docs/configuration/single-process-config-blocks-gossip-2.yaml

@@ -58,8 +58,6 @@ ingester:
       replication_factor: 1
 
 memberlist:
-  # defaults to hostname
-  node_name: "Ingester 2"
   bind_port: 7947
   join_members:
     - localhost:7946

pkg/ring/kv/memberlist/memberlist_client.go

@@ -3,6 +3,7 @@ package memberlist
 import (
 	"bytes"
 	"context"
+	"crypto/rand"
 	"encoding/binary"
 	"errors"
 	"flag"
@@ -81,6 +82,7 @@ func (c *Client) WatchPrefix(ctx context.Context, prefix string, f func(string,
 type KVConfig struct {
 	// Memberlist options.
 	NodeName            string        `yaml:"node_name"`
+	RandomizeNodeName   bool          `yaml:"randomize_node_name"`
 	StreamTimeout       time.Duration `yaml:"stream_timeout"`
 	RetransmitMult      int           `yaml:"retransmit_factor"`
 	PushPullInterval    time.Duration `yaml:"pull_push_interval"`
@@ -113,6 +115,7 @@ type KVConfig struct {
 func (cfg *KVConfig) RegisterFlags(f *flag.FlagSet, prefix string) {
 	// "Defaults to hostname" -- memberlist sets it to hostname by default.
 	f.StringVar(&cfg.NodeName, prefix+"memberlist.nodename", "", "Name of the node in memberlist cluster. Defaults to hostname.") // memberlist.DefaultLANConfig will put hostname here.
+	f.BoolVar(&cfg.RandomizeNodeName, prefix+"memberlist.randomize-node-name", true, "Add random suffix to the node name.")
 	f.DurationVar(&cfg.StreamTimeout, prefix+"memberlist.stream-timeout", 0, "The timeout for establishing a connection with a remote node, and for read/write operations. Uses memberlist LAN defaults if 0.")
 	f.IntVar(&cfg.RetransmitMult, prefix+"memberlist.retransmit-factor", 0, "Multiplication factor used when sending out messages (factor * log(N+1)).")
 	f.Var(&cfg.JoinMembers, prefix+"memberlist.join", "Other cluster members to join. Can be specified multiple times. Memberlist store is EXPERIMENTAL.")
@@ -128,6 +131,16 @@ func (cfg *KVConfig) RegisterFlags(f *flag.FlagSet, prefix string) {
 	cfg.TCPTransport.RegisterFlags(f, prefix)
 }
 
+func generateRandomSuffix() string {
+	suffix := make([]byte, 4)
+	_, err := rand.Read(suffix)
+	if err != nil {
+		level.Error(util.Logger).Log("msg", "failed to generate random suffix", "err", err)
+		return "error"
+	}
+	return fmt.Sprintf("%2x", suffix)
+}
+
 // KV implements Key-Value store on top of memberlist library. KV store has API similar to kv.Client,
 // except methods also need explicit codec for each operation.
 type KV struct {
@@ -240,6 +253,10 @@ func NewKV(cfg KVConfig) (*KV, error) {
 	if cfg.NodeName != "" {
 		mlCfg.Name = cfg.NodeName
 	}
+	if cfg.RandomizeNodeName {
+		mlCfg.Name = mlCfg.Name + "-" + generateRandomSuffix()
+		level.Info(util.Logger).Log("msg", "Using memberlist cluster node name", "name", mlCfg.Name)
+	}
 
 	mlCfg.LogOutput = newMemberlistLoggerAdapter(util.Logger, false)
 	mlCfg.Transport = tr

pkg/ring/kv/memberlist/memberlist_client_test.go

@@ -807,3 +807,12 @@ func TestMultipleCodecs(t *testing.T) {
 	require.NotNil(t, val)
 	require.Equal(t, 5, val.(distributedCounter)["test"])
 }
+
+func TestGenerateRandomSuffix(t *testing.T) {
+	h1 := generateRandomSuffix()
+	h2 := generateRandomSuffix()
+	h3 := generateRandomSuffix()
+
+	require.NotEqual(t, h1, h2)
+	require.NotEqual(t, h2, h3)
+}