Prevent CPLB VIPs to be etcd's peer address

This fixes only the automatic address detection for etcd. If a user
configures them manually it doesn't attempt to prevent the error.

Signed-off-by: Juan-Luis de Sousa-Valadas Castaño <[email protected]>

Author: juanluisvaladas

cmd/controller/controller.go

@@ -34,6 +34,7 @@ import (
 	workercmd "github.com/k0sproject/k0s/cmd/worker"
 	"github.com/k0sproject/k0s/internal/pkg/dir"
 	"github.com/k0sproject/k0s/internal/pkg/file"
+	"github.com/k0sproject/k0s/internal/pkg/iface"
 	k0slog "github.com/k0sproject/k0s/internal/pkg/log"
 	"github.com/k0sproject/k0s/internal/pkg/sysinfo"
 	"github.com/k0sproject/k0s/internal/sync/value"
@@ -231,6 +232,9 @@ func (c *command) start(ctx context.Context) error {
 			K0sVars:     c.K0sVars,
 			LogLevel:    c.LogLevels.Etcd,
 		}
+		// if cplb is disabled use the default primary address
+		storageBackend.(*controller.Etcd).Config.PeerAddress = etcdPeerAddress(nodeConfig.Spec.Network.ControlPlaneLoadBalancing)
+
 	default:
 		return fmt.Errorf("invalid storage type: %s", nodeConfig.Spec.Storage.Type)
 	}
@@ -641,6 +645,27 @@ func (c *command) start(ctx context.Context) error {
 	return nil
 }
 
+func etcdPeerAddress(cplb *v1beta1.ControlPlaneLoadBalancingSpec) string {
+	if cplb != nil && cplb.Enabled {
+		cplbAddresses := cplb.VirtualIPs()
+		addresses, err := iface.GetPublicAddresses()
+		if err != nil {
+			logrus.WithError(err).Errorf("Failed to get the primary address, using %s", addresses[0])
+			return addresses[0]
+		}
+		for _, addr := range addresses {
+			if !slices.Contains(cplbAddresses, addr) {
+				return addr
+			}
+		}
+	}
+	addr, err := iface.FirstPublicAddress()
+	if err != nil {
+		logrus.WithError(err).Errorf("Failed to get the primary address, using %s", addr)
+	}
+	return addr
+}
+
 func (c *command) startWorker(ctx context.Context, profile string, nodeConfig *v1beta1.ClusterConfig) error {
 	var bootstrapConfig string
 	if !file.Exists(c.K0sVars.KubeletAuthConfigPath) {

internal/pkg/iface/iface.go

@@ -59,11 +59,18 @@ func CollectAllIPs() (addresses []net.IP, err error) {
 // FirstPublicAddress return the first found non-local IPv4 address that's not part of pod network
 // if any interface does not have any IPv4 address then return the first found non-local IPv6 address
 func FirstPublicAddress() (string, error) {
+	addr, err := GetPublicAddresses()
+	return addr[0], err
+}
+
+// GetPublicAddresses returns a list of all public network addresses on a node.
+// The slice contains the ipv4 addresses followed by the ipv6 addresses
+func GetPublicAddresses() ([]string, error) {
 	ifs, err := net.Interfaces()
 	if err != nil {
-		return "127.0.0.1", fmt.Errorf("failed to list network interfaces: %w", err)
+		return []string{"127.0.0.1"}, fmt.Errorf("failed to list network interfaces: %w", err)
 	}
-	ipv6addr := ""
+	publicAddresses := []string{}
 	for _, i := range ifs {
 		switch {
 		// Skip calico CNI interface
@@ -91,18 +98,23 @@ func FirstPublicAddress() (string, error) {
 			// check the address type and skip if loopback
 			if ipnet, ok := a.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
 				if ipnet.IP.To4() != nil {
-					return ipnet.IP.String(), nil
+					publicAddresses = append(publicAddresses, ipnet.IP.String())
 				}
-				if ipnet.IP.To16() != nil && ipv6addr == "" {
-					ipv6addr = ipnet.IP.String()
+			}
+		}
+		for _, a := range addresses {
+			// check the address type and skip if loopback
+			if ipnet, ok := a.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
+				if ipnet.IP.To16() != nil {
+					publicAddresses = append(publicAddresses, ipnet.IP.String())
 				}
 			}
 		}
 	}
-	if ipv6addr != "" {
-		return ipv6addr, nil
+	if len(publicAddresses) > 0 {
+		return publicAddresses, nil
 	}
 
 	logrus.Warn("failed to find any non-local, non podnetwork addresses on host, defaulting public address to 127.0.0.1")
-	return "127.0.0.1", nil
+	return []string{"127.0.0.1"}, nil
 }

pkg/apis/k0s/v1beta1/clusterconfig_types_test.go

@@ -20,8 +20,6 @@ import (
 	"encoding/json"
 	"testing"
 
-	"github.com/k0sproject/k0s/internal/pkg/iface"
-
 	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/yaml"
 
@@ -55,9 +53,6 @@ metadata:
 	c, err := ConfigFromString(yamlData)
 	assert.NoError(t, err)
 	assert.Equal(t, EtcdStorageType, c.Spec.Storage.Type)
-	addr, err := iface.FirstPublicAddress()
-	assert.NoError(t, err)
-	assert.Equal(t, addr, c.Spec.Storage.Etcd.PeerAddress)
 }
 
 func TestEmptyClusterSpec(t *testing.T) {
@@ -135,9 +130,6 @@ spec:
 	c, err := ConfigFromString(yamlData)
 	assert.NoError(t, err)
 	assert.Equal(t, EtcdStorageType, c.Spec.Storage.Type)
-	addr, err := iface.FirstPublicAddress()
-	assert.NoError(t, err)
-	assert.Equal(t, addr, c.Spec.Storage.Etcd.PeerAddress)
 }
 
 func TestNetworkValidation_Custom(t *testing.T) {

pkg/apis/k0s/v1beta1/cplb.go

@@ -309,3 +309,12 @@ func (k *KeepalivedSpec) Validate() (errs []error) {
 
 	return errs
 }
+
+// VirtualIPs returns a list of virtual IPs used by the VRRP instances.
+func (c *ControlPlaneLoadBalancingSpec) VirtualIPs() []string {
+	var vips []string
+	for _, vrrp := range c.Keepalived.VRRPInstances {
+		vips = append(vips, vrrp.VirtualIPs...)
+	}
+	return vips
+}

pkg/apis/k0s/v1beta1/storage.go

@@ -27,13 +27,10 @@ import (
 	"slices"
 	"strings"
 
-	"github.com/k0sproject/k0s/internal/pkg/iface"
 	"github.com/k0sproject/k0s/pkg/config/kine"
 	"github.com/k0sproject/k0s/pkg/constant"
 
 	"k8s.io/apimachinery/pkg/util/validation/field"
-
-	"github.com/sirupsen/logrus"
 )
 
 var _ Validateable = (*StorageSpec)(nil)
@@ -158,14 +155,8 @@ type ExternalCluster struct {
 
 // DefaultEtcdConfig creates EtcdConfig with sane defaults
 func DefaultEtcdConfig() *EtcdConfig {
-	addr, err := iface.FirstPublicAddress()
-	if err != nil {
-		logrus.Warnf("failed to resolve etcd peering address automatically, using loopback")
-		addr = "127.0.0.1"
-	}
 	return &EtcdConfig{
 		ExternalCluster: nil,
-		PeerAddress:     addr,
 		ExtraArgs:       make(map[string]string),
 	}
 }