Merge pull request #4213 from jnummelin/feature/etcd-member-mgmt

Manage Etcd peers using custom resources

Author: jnummelin

Makefile

@@ -124,7 +124,12 @@ controllergen_targets += pkg/apis/autopilot/v1beta2/.controller-gen.stamp
 pkg/apis/autopilot/v1beta2/.controller-gen.stamp: $(shell find pkg/apis/autopilot/v1beta2/ -maxdepth 1 -type f -name \*.go)
 pkg/apis/autopilot/v1beta2/.controller-gen.stamp: gen_output_dir = autopilot
 
+controllergen_targets += pkg/apis/etcd/v1beta1/.controller-gen.stamp
+pkg/apis/etcd/v1beta1/.controller-gen.stamp: $(shell find pkg/apis/etcd/v1beta1/ -maxdepth 1 -type f -name \*.go)
+pkg/apis/etcd/v1beta1/.controller-gen.stamp: gen_output_dir = etcd
+
 codegen_targets += $(controllergen_targets)
+
 pkg/apis/%/.controller-gen.stamp: .k0sbuild.docker-image.k0s hack/tools/boilerplate.go.txt hack/tools/Makefile.variables
 	rm -rf 'static/manifests/$(gen_output_dir)/CustomResourceDefinition'
 	mkdir -p 'static/manifests/$(gen_output_dir)'
@@ -137,7 +142,7 @@ pkg/apis/%/.controller-gen.stamp: .k0sbuild.docker-image.k0s hack/tools/boilerpl
 	  && mv -f -- "$$gendir"/zz_generated.deepcopy.go '$(dir $@).'
 	touch -- '$@'
 
-clientset_input_dirs := pkg/apis/autopilot/v1beta2 pkg/apis/k0s/v1beta1 pkg/apis/helm/v1beta1
+clientset_input_dirs := pkg/apis/autopilot/v1beta2 pkg/apis/k0s/v1beta1 pkg/apis/helm/v1beta1 pkg/apis/etcd/v1beta1
 codegen_targets += pkg/client/clientset/.client-gen.stamp
 pkg/client/clientset/.client-gen.stamp: $(shell find $(clientset_input_dirs) -type f -name \*.go -not -name \*_test.go -not -name zz_\*)
 pkg/client/clientset/.client-gen.stamp: .k0sbuild.docker-image.k0s hack/tools/boilerplate.go.txt embedded-bins/Makefile.variables
@@ -163,6 +168,7 @@ static/zz_generated_assets.go: .k0sbuild.docker-image.k0s hack/tools/Makefile.va
 	  static/manifests/helm/CustomResourceDefinition/... \
 	  static/manifests/v1beta1/CustomResourceDefinition/... \
 	  static/manifests/autopilot/CustomResourceDefinition/... \
+	  static/manifests/etcd/CustomResourceDefinition/... \
 	  static/manifests/calico/... \
 	  static/manifests/windows/... \
 	  static/misc/...

cmd/controller/controller.go

@@ -325,6 +325,19 @@ func (c *command) start(ctx context.Context) error {
 		CertManager: worker.NewCertificateManager(ctx, c.K0sVars.KubeletAuthConfigPath),
 	})
 
+	if nodeConfig.Spec.Storage.Type == v1beta1.EtcdStorageType && !nodeConfig.Spec.Storage.Etcd.IsExternalClusterUsed() {
+		etcdReconciler, err := controller.NewEtcdMemberReconciler(adminClientFactory, c.K0sVars, nodeConfig.Spec.Storage.Etcd, leaderElector)
+		if err != nil {
+			return err
+		}
+		etcdCRDSaver, err := controller.NewManifestsSaver("etcd-member", c.K0sVars.DataDir)
+		if err != nil {
+			return fmt.Errorf("failed to initialize etcd-member manifests saver: %w", err)
+		}
+		clusterComponents.Add(ctx, controller.NewCRD(etcdCRDSaver, []string{"etcd"}))
+		nodeComponents.Add(ctx, etcdReconciler)
+	}
+
 	perfTimer.Checkpoint("starting-certificates-init")
 	certs := &Certificates{
 		ClusterSpec: nodeConfig.Spec,

docs/remove_controller.md

@@ -41,6 +41,61 @@ k0s reset
 reboot
 ```
 
+### Declarative Etcd member management
+
+Starting from version 1.30, k0s also supports a declarative way to remove an  
+etcd member. Since in k0s the etcd cluster is set up so that the etcd API is  
+**NOT** exposed outside the nodes, it makes it difficult for external automation  
+like Cluster API, Terraform, etc. to handle controller node replacements.
+
+Each controller manages their own `EtcdMember` object.
+
+```shell
+k0s kubectl get etcdmember
+NAME          PEER ADDRESS   MEMBER ID           JOINED   RECONCILE STATUS
+controller0   172.17.0.2     b8e14bda2255bc24    True     
+controller1   172.17.0.3     cb242476916c8a58    True     
+controller2   172.17.0.4     9c90504b1bc867bb    True 
+```
+
+By marking an `EtcdMember` object to leave the etcd cluster, k0s will handle the  
+interaction with etcd. For example, in a 3 controller HA setup, you can  
+remove a member by flagging it to leave:
+
+```console
+$ kubectl patch etcdmember controller2 -p '{"spec":{"leave":true}}' --type merge
+etcdmember.etcd.k0sproject.io/controller2 patched
+```
+
+The join/leave status is tracked in the object's conditions. This allows you to  
+wait for the leave to actually happen:
+
+```console
+$ kubectl wait etcdmember controller2 --for condition=Joined=False
+etcdmember.etcd.k0sproject.io/controller2 condition met
+```
+
+You'll see the node left etcd cluster:
+
+```console
+$ k0s kc get etcdmember
+NAME          PEER ADDRESS   MEMBER ID           JOINED   RECONCILE STATUS
+controller0   172.17.0.2     b8e14bda2255bc24    True     
+controller1   172.17.0.3     cb242476916c8a58    True     
+controller2   172.17.0.4     9c90504b1bc867bb    False    Success
+```
+
+```console
+$ k0s etcd member-list
+{"members":{"controller0":"https://172.17.0.2:2380","controller1":"https://172.17.0.3:2380"}}
+```
+
+The objects for members that have already left the etcd cluster are kept  
+available for tracking purposes. Once the member has left the cluster, the  
+object status will reflect that it is safe to remove it.
+
+**Note:** If you re-join same node without removing the corresponding `etcdmember` object the desired state will be updated back to `spec.leave: false` automatically. This is since currently in k0s there's no easy way to prevent a node joining etcd cluster.
+
 ## Replace a controller
 
 To replace a controller, you first remove the old controller (like described above) then follow the [manual installation procedure](k0s-multi-node.md) to add the new one.

internal/testutil/kube_client.go

@@ -18,6 +18,7 @@ package testutil
 
 import (
 	"fmt"
+
 	"k8s.io/client-go/rest"
 
 	"k8s.io/apimachinery/pkg/runtime"
@@ -32,9 +33,13 @@ import (
 	restfake "k8s.io/client-go/rest/fake"
 	kubetesting "k8s.io/client-go/testing"
 
+	etcdMemberClient "github.com/k0sproject/k0s/pkg/client/clientset/typed/etcd/v1beta1"
 	cfgClient "github.com/k0sproject/k0s/pkg/client/clientset/typed/k0s/v1beta1"
+	kubeutil "github.com/k0sproject/k0s/pkg/kubernetes"
 )
 
+var _ kubeutil.ClientFactoryInterface = (*FakeClientFactory)(nil)
+
 // NewFakeClientFactory creates new client factory which uses internally only the kube fake client interface
 func NewFakeClientFactory(objects ...runtime.Object) FakeClientFactory {
 	rawDiscovery := &discoveryfake.FakeDiscovery{Fake: &kubetesting.Fake{}}
@@ -89,3 +94,7 @@ func (f FakeClientFactory) GetRESTClient() (rest.Interface, error) {
 func (f FakeClientFactory) GetRESTConfig() *rest.Config {
 	return &rest.Config{}
 }
+
+func (f FakeClientFactory) GetEtcdMemberClient() (etcdMemberClient.EtcdMemberInterface, error) {
+	return nil, fmt.Errorf("NOT IMPLEMENTED")
+}

inttest/Makefile.variables

@@ -34,6 +34,7 @@ smoketests := \
 	check-dualstack \
 	check-dualstack-dynamicconfig \
 	check-embedded-binaries \
+	check-etcdmember \
 	check-externaletcd \
 	check-extraargs \
 	check-hacontrolplane \

inttest/common/bootloosesuite.go

@@ -43,6 +43,7 @@ import (
 
 	"github.com/k0sproject/k0s/internal/pkg/file"
 	apclient "github.com/k0sproject/k0s/pkg/client/clientset"
+	etcdmemberclient "github.com/k0sproject/k0s/pkg/client/clientset/typed/etcd/v1beta1"
 	"github.com/k0sproject/k0s/pkg/constant"
 	"github.com/k0sproject/k0s/pkg/k0scontext"
 	"github.com/k0sproject/k0s/pkg/kubernetes/watch"
@@ -774,6 +775,18 @@ func (s *BootlooseSuite) StopController(name string) error {
 	return s.launchDelegate.StopController(s.Context(), ssh)
 }
 
+func (s *BootlooseSuite) RestartController(name string) error {
+	ssh, err := s.SSH(s.Context(), name)
+	s.Require().NoError(err)
+	defer ssh.Disconnect()
+	s.T().Log("killing k0s")
+	err = s.launchDelegate.StopController(s.Context(), ssh)
+	if err != nil {
+		return err
+	}
+	return s.launchDelegate.StartController(s.Context(), ssh)
+}
+
 func (s *BootlooseSuite) StartController(name string) error {
 	ssh, err := s.SSH(s.Context(), name)
 	s.Require().NoError(err)
@@ -896,6 +909,16 @@ func (s *BootlooseSuite) ExtensionsClient(node string, k0sKubeconfigArgs ...stri
 	return extclient.NewForConfig(cfg)
 }
 
+// EtcdMemberClient return a client for accessing etcd member CRDs
+func (s *BootlooseSuite) EtcdMemberClient(node string, k0sKubeconfigArgs ...string) (*etcdmemberclient.EtcdV1beta1Client, error) {
+	cfg, err := s.GetKubeConfig(node, k0sKubeconfigArgs...)
+	if err != nil {
+		return nil, err
+	}
+
+	return etcdmemberclient.NewForConfig(cfg)
+}
+
 // WaitForNodeReady wait that we see the given node in "Ready" state in kubernetes API
 func (s *BootlooseSuite) WaitForNodeReady(name string, kc kubernetes.Interface) error {
 	s.T().Logf("waiting to see %s ready in kube API", name)

inttest/common/util.go

@@ -334,6 +334,16 @@ func VerifyKubeletMetrics(ctx context.Context, kc *kubernetes.Clientset, node st
 	})
 }
 
+func ResetNode(name string, suite *BootlooseSuite) error {
+	ssh, err := suite.SSH(suite.Context(), name)
+	if err != nil {
+		return err
+	}
+	defer ssh.Disconnect()
+	_, err = ssh.ExecWithOutput(suite.Context(), fmt.Sprintf("%s reset --debug", suite.K0sFullPath))
+	return err
+}
+
 // Retrieves the LogfFn stored in context, falling back to use testing.T's Logf
 // if the context has a *testing.T or logrus's Infof as a last resort.
 func logfFrom(ctx context.Context) LogfFn {