Improve debugging experience for k0s inttests

Implements an optional, heavier debug image with delve and strace, a way
to replace the command in the BootlooseSuite for a debugging command and
documents it.

Signed-off-by: Juan-Luis de Sousa-Valadas Castaño <[email protected]>

Author: juanluisvaladas

README.md

@@ -181,3 +181,39 @@ To run a basic smoke test after build:
 ```shell
 make check-basic
 ```
+
+### Debugging smoke tests
+
+Every smoke test can be executed independently, the complete list of smoke tests is
+available in `inttest/Makefile.variables``
+
+It's possible to attach a debugger to a smoke test but it requires some modifications:
+
+1. k0s must be compiled with symbols: `DEBUG=true make k0s`
+2. The source code must be present in `go/src/github.com/k0sproject/` and
+the go cache in `/run/k0s-build/go`
+
+   ```shell
+   sudo mkdir -p /go/src/github.com/k0sproject/ /run/k0s-build/
+   sudo chown -R $(whoami):$(whoami) /go/src/github.com/k0sproject/ /run/k0s-build/
+   ln -s <K0s source dir> /go/src/github.com/k0sproject/k0s
+   ln -s <K0s source dir>/build/cache/go /run/k0s-build/go
+   ```
+
+3. Modify the inttest to include to run the command that you want to run:
+
+   ```go
+   // SetK0sCommand is only evaluated when Controllers or Workers are initialized
+   // feel free to call it where it's appropriate and change the position.
+   // This example enables the debugger for the controllers but not the workers
+   s.SetK0sCommand("PATH=/usr/local/go/bin:$PATH /go/bin/dlv exec --headless --listen :4040 -- /usr/local/bin/k0s")
+   s.Require().NoError(s.InitController(0, controllerArgs...))
+   s.SetK0sCommand("/usr/local/bin/k0s")
+   s.Require().NoError(s.RunWorkers())
+   ```
+
+4. Launch the tests with `K0S_DEBUG_TEST=true` so that it uses the correct image:
+`K0S_KEEP_AFTER_TESTS=always DEBUG=true K0S_DEBUG_TESTS=true make check-<test name>`
+
+5. Connect to the remote debugger using the correct container IP
+`dlv -connect <container ip address>:4040`

inttest/Makefile

@@ -20,16 +20,25 @@ bin/sonobuoy: | bin
 	$(curl) $(sonobuoy_url) | tar -C bin/ -zxv $(notdir $@)
 
 bootloose_alpine_build_cmdline := \
-	--build-arg GOLANG_IMAGE=$(golang_buildimage) \
-	--build-arg ALPINE_VERSION=$(alpine_patch_version) \
 	--build-arg ETCD_VERSION=$(etcd_version) \
 	--build-arg HELM_VERSION=$(helm_version) \
-	-t bootloose-alpine \
+	  --build-arg TARGETARCH=$(ARCH) \
 	-f bootloose-alpine/Dockerfile \
 	bootloose-alpine
 
 .bootloose-alpine.stamp: $(shell find bootloose-alpine -type f)
-	docker build --progress=plain --build-arg TARGETARCH=$(ARCH) $(bootloose_alpine_build_cmdline)
+	docker build --progress=plain \
+	  --build-arg BOOTLOOSE_BASE=docker.io/library/alpine:$(alpine_patch_version) \
+	  -t bootloose-alpine \
+	  $(bootloose_alpine_build_cmdline)
+	touch $@
+
+.bootloose-debug.stamp: $(shell find bootloose-alpine -type f)
+	docker build --progress=plain \
+	  -t bootloose-debug \
+	  --build-arg INCLUDE_DEBUG_TOOLS=true \
+	  --build-arg BOOTLOOSE_BASE=$(golang_buildimage) \
+	  $(bootloose_alpine_build_cmdline)
 	touch $@
 
 # This is a special target to test the bootloose alpine image locally for all supported platforms.
@@ -133,15 +142,25 @@ check-nllb: TIMEOUT=15m
 .PHONY: $(smoketests)
 include Makefile.variables
 
+# Determine the bootloose stamp dependency based on K0S_DEBUG
+# Default to .bootloose-alpine.stamp (the "otherwise" case from the request)
+BOOTLOOSE_STAMP_DEP := .bootloose-alpine.stamp
+ifeq ($(K0S_DEBUG_TESTS),true)
+  BOOTLOOSE_STAMP_DEP := .bootloose-debug.stamp
+  BOOTLOOSE_IMAGE ?= bootloose-debug
+else
+endif
+
 $(smoketests): K0S_PATH ?= $(realpath ../k0s)
 $(smoketests): K0S_IMAGES_BUNDLE ?= $(realpath ../airgap-image-bundle-linux-$(ARCH).tar)
-$(smoketests): .bootloose-alpine.stamp
+$(smoketests): $(BOOTLOOSE_STAMP_DEP)
 $(smoketests): TEST_PACKAGE ?= $(subst check-,,$@)
 $(smoketests):
 	K0S_PATH='$(K0S_PATH)' \
 	K0S_UPDATE_FROM_PATH='$(K0S_UPDATE_FROM_PATH)' \
 	K0S_IMAGES_BUNDLE='$(K0S_IMAGES_BUNDLE)' \
 	K0S_UPDATE_TO_VERSION='$(K0S_UPDATE_TO_VERSION)' \
+	$(if $(BOOTLOOSE_IMAGE),BOOTLOOSE_IMAGE='$(BOOTLOOSE_IMAGE)') \
 	go test -count=1 -v -timeout $(TIMEOUT) github.com/k0sproject/k0s/inttest/$(TEST_PACKAGE)
 .PHONY: clean
 

inttest/bootloose-alpine/Dockerfile

@@ -1,13 +1,13 @@
-ARG ALPINE_VERSION
-ARG GOLANG_IMAGE
+ARG BOOTLOOSE_BASE
 
-FROM docker.io/library/alpine:$ALPINE_VERSION
+FROM $BOOTLOOSE_BASE
 
 ARG TARGETARCH
 ARG CRI_DOCKERD_VERSION=0.3.16
 ARG ETCD_VERSION
 ARG TROUBLESHOOT_VERSION=v0.115.1
 ARG HELM_VERSION
+ARG INCLUDE_DEBUG_TOOLS=false
 
 # Apply our changes to the image
 COPY root/ /
@@ -50,22 +50,27 @@ RUN curl --proto '=https' --tlsv1.2 -L https://get.helm.sh/helm-v$HELM_VERSION-l
 # Install etcd for smoke tests with external etcd
 # No arm or riscv64 binaries available (check-externaletcd won't work on ARMv7 or RISC-V)
 RUN if [ "$TARGETARCH" != arm ] && [ "$TARGETARCH" != riscv64 ]; then \
-    curl --proto '=https' --tlsv1.2 -L https://github.com/etcd-io/etcd/releases/download/v$ETCD_VERSION/etcd-v$ETCD_VERSION-linux-$TARGETARCH.tar.gz \
-      | tar xz -C /opt --strip-components=1; \
+  curl --proto '=https' --tlsv1.2 -L https://github.com/etcd-io/etcd/releases/download/v$ETCD_VERSION/etcd-v$ETCD_VERSION-linux-$TARGETARCH.tar.gz \
+  | tar xz -C /opt --strip-components=1; \
   fi
 
 # Install cri-dockerd shim for custom CRI testing
 # No arm or riscv64 binaries available (check-byocri won't work on ARMv7 or RISC-V)
 RUN if [ "$TARGETARCH" != arm ] && [ "$TARGETARCH" != riscv64 ]; then \
-    curl --proto '=https' --tlsv1.2 --retry 5 --retry-all-errors -sSLfo /tmp/cri-dockerd.tgz https://github.com/Mirantis/cri-dockerd/releases/download/v$CRI_DOCKERD_VERSION/cri-dockerd-$CRI_DOCKERD_VERSION.$TARGETARCH.tgz \
-      && tar xf /tmp/cri-dockerd.tgz --directory /tmp/ \
-      && mv /tmp/cri-dockerd/cri-dockerd /usr/local/bin/cri-dockerd \
-      && rm -rf /tmp/cri-dockerd \
-      && chmod 755 /usr/local/bin/cri-dockerd; \
+  curl --proto '=https' --tlsv1.2 --retry 5 --retry-all-errors -sSLfo /tmp/cri-dockerd.tgz https://github.com/Mirantis/cri-dockerd/releases/download/v$CRI_DOCKERD_VERSION/cri-dockerd-$CRI_DOCKERD_VERSION.$TARGETARCH.tgz \
+  && tar xf /tmp/cri-dockerd.tgz --directory /tmp/ \
+  && mv /tmp/cri-dockerd/cri-dockerd /usr/local/bin/cri-dockerd \
+  && rm -rf /tmp/cri-dockerd \
+  && chmod 755 /usr/local/bin/cri-dockerd; \
   fi
 
 RUN for u in etcd kube-apiserver kube-scheduler konnectivity-server; do \
-    adduser --system --shell /sbin/nologin --no-create-home --home /var/lib/k0s --disabled-password --gecos '' "$u"; \
+  adduser --system --shell /sbin/nologin --no-create-home --home /var/lib/k0s --disabled-password --gecos '' "$u"; \
   done
 
+RUN if [ "$INCLUDE_DEBUG_TOOLS" = "true" ]; then \
+  apk add --no-cache strace \
+  && go install github.com/go-delve/delve/cmd/dlv@latest; \
+  fi
+
 ADD cri-dockerd.sh /etc/init.d/cri-dockerd

inttest/common/bootloosesuite.go

@@ -642,6 +642,12 @@ func (s *BootlooseSuite) InitController(idx int, k0sArgs ...string) error {
 	return s.WaitForKubeAPI(controllerNode, dataDirOpt)
 }
 
+// SetK0sCommand allows to set the command used to launch k0s, so that it
+// can be launched with some debugging tools such as strace or delve
+func (s *BootlooseSuite) SetK0sCommand(command string) {
+	s.Require().NoError(s.launchDelegate.SetK0sCommand(command))
+}
+
 // GetJoinToken generates join token for the asked role
 func (s *BootlooseSuite) GetJoinToken(role string, extraArgs ...string) (string, error) {
 	// assume we have main on node 0 always

inttest/common/launchdelegate.go

@@ -43,12 +43,13 @@ type launchDelegate interface {
 	StartWorker(ctx context.Context, conn *SSHConnection) error
 	StopWorker(ctx context.Context, conn *SSHConnection) error
 	ReadK0sLogs(ctx context.Context, conn *SSHConnection, out, err io.Writer) error
+	SetK0sCommand(prefix string) error
 }
 
 // standaloneLaunchDelegate is a launchDelegate that starts controllers and
 // workers in "standalone" mode, i.e. not via some service manager.
 type standaloneLaunchDelegate struct {
-	k0sFullPath     string
+	k0sCommand      string
 	controllerUmask int
 }
 
@@ -64,7 +65,7 @@ func (s *standaloneLaunchDelegate) InitController(ctx context.Context, conn *SSH
 		fmt.Fprintf(&script, "umask %d\n", s.controllerUmask)
 	}
 	fmt.Fprintf(&script, "export ETCD_UNSUPPORTED_ARCH='%s'\n", runtime.GOARCH)
-	fmt.Fprintf(&script, "%s controller --debug %s </dev/null >>/tmp/k0s-controller.log 2>&1 &\n", s.k0sFullPath, strings.Join(k0sArgs, " "))
+	fmt.Fprintf(&script, "%s controller --debug %s </dev/null >>/tmp/k0s-controller.log 2>&1 &\n", s.k0sCommand, strings.Join(k0sArgs, " "))
 	fmt.Fprintln(&script, "disown %1")
 
 	if err := conn.Exec(ctx, "cat >/tmp/start-k0s && chmod +x /tmp/start-k0s", SSHStreams{
@@ -76,6 +77,13 @@ func (s *standaloneLaunchDelegate) InitController(ctx context.Context, conn *SSH
 	return s.StartController(ctx, conn)
 }
 
+// SetControllerPrefixCmd allows to set some command before k0s so that it can
+// be launched with some debugging tools such as strace or delve
+func (s *standaloneLaunchDelegate) SetK0sCommand(command string) error {
+	s.k0sCommand = command
+	return nil
+}
+
 // StartController starts a k0s controller that was initialized in "standalone" mode.
 func (s *standaloneLaunchDelegate) StartController(ctx context.Context, conn *SSHConnection) error {
 	const cmd = "/tmp/start-k0s"
@@ -100,7 +108,7 @@ func (s *standaloneLaunchDelegate) InitWorker(ctx context.Context, conn *SSHConn
 	var script strings.Builder
 	fmt.Fprintln(&script, "#!/usr/bin/env bash")
 	fmt.Fprintln(&script, "set -eu")
-	fmt.Fprintf(&script, "%s worker --debug %s \"$@\" </dev/null >>/tmp/k0s-worker.log 2>&1 &\n", s.k0sFullPath, strings.Join(k0sArgs, " "))
+	fmt.Fprintf(&script, "%s worker --debug %s \"$@\" </dev/null >>/tmp/k0s-worker.log 2>&1 &\n", s.k0sCommand, strings.Join(k0sArgs, " "))
 	fmt.Fprintln(&script, "disown %1")
 
 	if err := conn.Exec(ctx, "cat >/tmp/start-k0s-worker && chmod +x /tmp/start-k0s-worker", SSHStreams{
@@ -134,7 +142,7 @@ func (s *standaloneLaunchDelegate) ReadK0sLogs(ctx context.Context, conn *SSHCon
 }
 
 func (s *standaloneLaunchDelegate) killK0s(ctx context.Context, conn *SSHConnection) error {
-	stopCommand := fmt.Sprintf("kill $(pidof %s | tr \" \" \"\\n\" | sort -n | head -n1) && while pidof %s; do sleep 0.1s; done", s.k0sFullPath, s.k0sFullPath)
+	stopCommand := fmt.Sprintf("kill $(pidof %s | tr \" \" \"\\n\" | sort -n | head -n1) && while pidof %s; do sleep 0.1s; done", s.k0sCommand, s.k0sCommand)
 	if err := conn.Exec(ctx, stopCommand, SSHStreams{}); err != nil {
 		return fmt.Errorf("unable to execute %q: %w", stopCommand, err)
 	}
@@ -171,6 +179,11 @@ func (o *openRCLaunchDelegate) InitController(ctx context.Context, conn *SSHConn
 	return o.StartController(ctx, conn)
 }
 
+// SetK0sCommand isn't implemented in "OpenRC" mode
+func (o *openRCLaunchDelegate) SetK0sCommand(_ string) error {
+	return errors.New("openRCLaunchDelegate.SetK0sCommand isn't implemented")
+}
+
 // StartController starts a k0s controller that was started using OpenRC.
 func (o *openRCLaunchDelegate) StartController(ctx context.Context, conn *SSHConnection) error {
 	const cmd = "/etc/init.d/k0scontroller start"