Merge pull request #4238 from twz123/reset-inttests

Add integration tests around k0s reset

Author: twz123

inttest/Makefile.variables

@@ -54,6 +54,7 @@ smoketests := \
 	check-noderole-single \
 	check-openebs\
 	check-psp \
+	check-reset \
 	check-singlenode \
 	check-statussocket \
 	check-upgrade \

inttest/backup/backup_test.go

@@ -96,8 +96,8 @@ func (s *BackupSuite) TestK0sGetsUp() {
 	s.Require().NoError(s.StopController(s.ControllerNode(0)))
 	_ = s.StopController(s.ControllerNode(1)) // No error check as k0s might have actually exited since etcd is not really happy
 
-	s.Require().NoError(s.Reset(s.ControllerNode(0)))
-	s.Require().NoError(s.Reset(s.ControllerNode(1)))
+	s.reset(s.ControllerNode(0))
+	s.reset(s.ControllerNode(1))
 
 	s.Require().NoError(s.restoreFunc())
 	s.Require().NoError(s.InitController(0, "--enable-worker"))
@@ -122,6 +122,13 @@ func (s *BackupSuite) TestK0sGetsUp() {
 	s.Require().NoError(s.VerifyFileSystemRestore())
 }
 
+func (s *BackupSuite) reset(name string) {
+	ssh, err := s.SSH(s.Context(), name)
+	s.Require().NoError(err)
+	defer ssh.Disconnect()
+	s.Require().NoError(ssh.Exec(s.Context(), "k0s reset --debug", common.SSHStreams{}))
+}
+
 type snapshot struct {
 	namespaces map[types.UID]string
 	services   map[types.UID]string

inttest/byocri/byocri_test.go

@@ -18,6 +18,7 @@ package byocri
 
 import (
 	"fmt"
+	"strings"
 	"testing"
 
 	"github.com/avast/retry-go"
@@ -31,21 +32,51 @@ type BYOCRISuite struct {
 	common.BootlooseSuite
 }
 
-func (s *BYOCRISuite) TestK0sGetsUp() {
+func (s *BYOCRISuite) TestBringYourOwnCRI() {
+	ctx := s.Context()
+	workerNode := s.WorkerNode(0)
 
-	s.NoError(s.InitController(0))
-	s.Require().NoError(s.runDockerWorker())
+	if ok := s.Run("k0s gets up", func() {
+		s.NoError(s.InitController(0))
+		s.Require().NoError(s.runDockerWorker())
 
-	kc, err := s.KubeClient(s.ControllerNode(0))
-	s.Require().NoError(err)
+		kc, err := s.KubeClient(s.ControllerNode(0))
+		s.Require().NoError(err)
 
-	err = s.WaitForNodeReady(s.WorkerNode(0), kc)
-	s.NoError(err)
+		err = s.WaitForNodeReady(workerNode, kc)
+		s.NoError(err)
 
-	s.AssertSomeKubeSystemPods(kc)
+		s.AssertSomeKubeSystemPods(kc)
 
-	s.T().Log("waiting to see CNI pods ready")
-	s.NoError(common.WaitForKubeRouterReady(s.Context(), kc), "CNI did not start")
+		s.T().Log("waiting to see CNI pods ready")
+		s.NoError(common.WaitForKubeRouterReady(ctx, kc), "CNI did not start")
+	}); !ok {
+		return
+	}
+
+	s.Run("k0s reset terminates Docker containers", func() {
+		ssh, err := s.SSH(ctx, workerNode)
+		s.Require().NoError(err)
+		defer ssh.Disconnect()
+
+		var containerIDs strings.Builder
+		if s.NoError(ssh.Exec(ctx, "docker ps -q", common.SSHStreams{Out: &containerIDs}), "Failed to get running Docker containers") {
+			s.NotEmpty(containerIDs.String(), "Expected some running Docker containers")
+		}
+
+		s.NoError(s.StopWorker(workerNode), "Failed to stop k0s")
+
+		resetCmd := "k0s reset --debug --cri-socket remote:unix:///var/run/cri-dockerd.sock"
+		streams, flushStreams := common.TestLogStreams(s.T(), "reset")
+		err = ssh.Exec(ctx, resetCmd, streams)
+		flushStreams()
+		s.NoError(err, "k0s reset didn't exit cleanly")
+
+		containerIDs.Reset()
+		if s.NoError(ssh.Exec(ctx, "docker ps -q", common.SSHStreams{Out: &containerIDs}), "Failed to get running Docker containers") {
+			s.Empty(containerIDs.String(), "Expected no running Docker containers")
+		}
+	})
 }
 
 func (s *BYOCRISuite) runDockerWorker() error {

inttest/common/bootloosesuite.go

@@ -795,15 +795,6 @@ func (s *BootlooseSuite) StopWorker(name string) error {
 	return s.launchDelegate.StopWorker(s.Context(), ssh)
 }
 
-func (s *BootlooseSuite) Reset(name string) error {
-	ssh, err := s.SSH(s.Context(), name)
-	s.Require().NoError(err)
-	defer ssh.Disconnect()
-	resetCommand := fmt.Sprintf("%s reset --debug", s.K0sFullPath)
-	_, err = ssh.ExecWithOutput(s.Context(), resetCommand)
-	return err
-}
-
 // KubeClient return kube client by loading the admin access config from given node
 func (s *BootlooseSuite) GetKubeConfig(node string, k0sKubeconfigArgs ...string) (*rest.Config, error) {
 	machine, err := s.MachineForName(node)

inttest/common/ssh.go

@@ -27,6 +27,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"testing"
 
 	"github.com/mitchellh/go-homedir"
 	"golang.org/x/crypto/ssh"
@@ -193,6 +194,16 @@ func (c *SSHConnection) Exec(ctx context.Context, cmd string, streams SSHStreams
 	}
 }
 
+// Returns SSH streams that log lines to the test log.
+func TestLogStreams(t *testing.T, prefix string) (_ SSHStreams, flush func()) {
+	out := LineWriter{WriteLine: func(line []byte) { t.Logf("%s stdout: %s", prefix, string(line)) }}
+	err := LineWriter{WriteLine: func(line []byte) { t.Logf("%s stderr: %s", prefix, string(line)) }}
+	return SSHStreams{Out: &out, Err: &err}, func() {
+		out.Flush()
+		err.Flush()
+	}
+}
+
 func newWriterBuffer() (io.WriteCloser, func() []byte) {
 	var mu sync.Mutex
 	var buf bytes.Buffer

inttest/common/util.go

@@ -345,3 +345,44 @@ func logfFrom(ctx context.Context) LogfFn {
 	}
 	return logrus.Infof
 }
+
+type LineWriter struct {
+	WriteLine func([]byte)
+	buf       []byte
+}
+
+// Write implements [io.Writer].
+func (s *LineWriter) Write(in []byte) (int, error) {
+	s.buf = append(s.buf, in...)
+	s.logLines()
+	return len(in), nil
+}
+
+// Logs each complete line and discards the used data.
+func (s *LineWriter) logLines() {
+	var off int
+	for {
+		n := bytes.IndexByte(s.buf[off:], '\n')
+		if n < 0 {
+			break
+		}
+
+		s.WriteLine(s.buf[off : off+n])
+		off += n + 1
+	}
+
+	// Move the unprocessed data to the beginning of the buffer and reset the length.
+	if off > 0 {
+		len := copy(s.buf, s.buf[off:])
+		s.buf = s.buf[:len]
+	}
+}
+
+// Logs any remaining data in the buffer that doesn't end with a newline.
+func (s *LineWriter) Flush() {
+	if len(s.buf) > 0 {
+		s.WriteLine(s.buf)
+		// Reset the length and keep the underlying array.
+		s.buf = s.buf[:0]
+	}
+}

inttest/reset/reset_test.go

@@ -0,0 +1,82 @@
+/*
+Copyright 2024 k0s authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package reset
+
+import (
+	"testing"
+
+	testifysuite "github.com/stretchr/testify/suite"
+
+	"github.com/k0sproject/k0s/inttest/common"
+)
+
+type suite struct {
+	common.BootlooseSuite
+}
+
+func (s *suite) TestReset() {
+	ctx := s.Context()
+	workerNode := s.WorkerNode(0)
+
+	if ok := s.Run("k0s gets up", func() {
+		s.Require().NoError(s.InitController(0, "--disable-components=konnectivity-server,metrics-server"))
+		s.Require().NoError(s.RunWorkers())
+
+		kc, err := s.KubeClient(s.ControllerNode(0))
+		s.Require().NoError(err)
+
+		err = s.WaitForNodeReady(workerNode, kc)
+		s.NoError(err)
+
+		s.T().Log("waiting to see CNI pods ready")
+		s.NoError(common.WaitForKubeRouterReady(ctx, kc), "CNI did not start")
+	}); !ok {
+		return
+	}
+
+	s.Run("k0s reset", func() {
+		ssh, err := s.SSH(ctx, workerNode)
+		s.Require().NoError(err)
+		defer ssh.Disconnect()
+
+		s.NoError(ssh.Exec(ctx, "test -d /var/lib/k0s", common.SSHStreams{}), "/var/lib/k0s is not a directory")
+		s.NoError(ssh.Exec(ctx, "test -d /run/k0s", common.SSHStreams{}), "/run/k0s is not a directory")
+
+		s.NoError(ssh.Exec(ctx, "pidof containerd-shim-runc-v2 >&2", common.SSHStreams{}), "Expected some running containerd shims")
+
+		s.NoError(s.StopWorker(workerNode), "Failed to stop k0s")
+
+		streams, flushStreams := common.TestLogStreams(s.T(), "reset")
+		err = ssh.Exec(ctx, "k0s reset --debug", streams)
+		flushStreams()
+		s.NoError(err, "k0s reset didn't exit cleanly")
+
+		// /var/lib/k0s is a mount point in the Docker container and can't be deleted, so it must be empty
+		s.NoError(ssh.Exec(ctx, `x="$(ls -A /var/lib/k0s)" && echo "$x" >&2 && [ -z "$x" ]`, common.SSHStreams{}), "/var/lib/k0s is not empty")
+		s.NoError(ssh.Exec(ctx, "! test -e /run/k0s", common.SSHStreams{}), "/run/k0s still exists")
+		s.NoError(ssh.Exec(ctx, "! pidof containerd-shim-runc-v2 >&2", common.SSHStreams{}), "Expected no running containerd shims")
+	})
+}
+
+func TestResetSuite(t *testing.T) {
+	testifysuite.Run(t, &suite{
+		common.BootlooseSuite{
+			ControllerCount: 1,
+			WorkerCount:     1,
+		},
+	})
+}