enhancement: support target-container procfs correct (#83)

* enhancement: support target-container procfs correct

* update Makefile 

remove some wrong adjustments for my own dev environment

* Update cmd.go

fix my typo error I made for travis CI

* fix: agent_daemonset yaml support lxcfs

* add:使用alpine linux,减小debug-agent镜像大小

* fix: 修复nsenter在alpine发行版中的一些error

* fix: reinforce the compatibility of debug-agent for kubectl-debug && fix typo && go fmt

Author: markzhang0928

.dockerignore

@@ -1,3 +1,3 @@
 *
 !debug-agent
-
+!./scripts/start.sh

Dockerfile

@@ -1,6 +1,21 @@
-FROM gcr.io/distroless/static
+FROM ubuntu:xenial as build
 
+RUN apt-get update && apt-get install libcgmanager-dev libnih-dbus-dev libnih-dev libfuse-dev automake libtool libpam-dev wget gcc automake -y && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+ENV LXCFS_VERSION 3.1.2
+RUN wget https://linuxcontainers.org/downloads/lxcfs/lxcfs-$LXCFS_VERSION.tar.gz && \
+	mkdir /lxcfs && tar xzvf lxcfs-$LXCFS_VERSION.tar.gz -C /lxcfs  --strip-components=1 && \
+	cd /lxcfs && ./configure && make
+
+FROM alpine:3.10
+
+COPY --from=build /lxcfs/lxcfs /usr/local/bin/lxcfs
+COPY --from=build /lxcfs/.libs/liblxcfs.so /usr/local/lib/lxcfs/liblxcfs.so
+COPY --from=build /lxcfs/lxcfs /lxcfs/lxcfs
+COPY --from=build /lxcfs/.libs/liblxcfs.so /lxcfs/liblxcfs.so
+COPY ./scripts/start.sh /
 COPY ./debug-agent /bin/debug-agent
+
 EXPOSE 10027
 
-ENTRYPOINT ["/bin/debug-agent"]
+CMD ["/start.sh"]

Makefile

@@ -12,7 +12,7 @@ plugin:
 	GO111MODULE=on CGO_ENABLED=0 go build -ldflags '$(LDFLAGS)' -o kubectl-debug cmd/plugin/main.go
 
 agent-docker: agent
-	docker build . -t aylei/debug-agent:latest
+	docker build . -t aylei/debug-agent:0.0.2
 
 agent:
 	$(GO) build -ldflags '$(LDFLAGS)' -o debug-agent cmd/agent/main.go

pkg/agent/lxcfs.go

@@ -0,0 +1,62 @@
+package agent
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+)
+
+// List of LXC filesystem files
+const (
+	MemFile       string = "/proc/meminfo"
+	CpuFile       string = "/proc/cpuinfo"
+	UpTimeFile    string = "/proc/uptime"
+	SwapsFile     string = "/proc/swaps"
+	StatFile      string = "/proc/stat"
+	DiskStatsFile string = "/proc/diskstats"
+	LoadavgFile   string = "/proc/loadavg"
+)
+
+var (
+	// IsLxcfsEnabled means whether to enable lxcfs
+	LxcfsEnabled bool
+
+	// LxcfsRootDir
+	LxcfsRootDir = "/var/lib/lxc"
+
+	// LxcfsHomeDir means /var/lib/lxc/lxcfs
+	LxcfsHomeDir = "/var/lib/lxc/lxcfs"
+
+	// LxcfsFiles is a list of LXC files
+	LxcfsProcFiles = []string{MemFile, CpuFile, UpTimeFile, SwapsFile, StatFile, DiskStatsFile, LoadavgFile}
+)
+
+// CheckLxcfsMount check if the the mount point of lxcfs exists
+func CheckLxcfsMount() error {
+	isMount := false
+	f, err := os.Open("/proc/1/mountinfo")
+	if err != nil {
+		return fmt.Errorf("Check lxcfs mounts failed: %v", err)
+	}
+	fr := bufio.NewReader(f)
+	for {
+		line, err := fr.ReadBytes('\n')
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return fmt.Errorf("Check lxcfs mounts failed: %v", err)
+		}
+
+		if bytes.Contains(line, []byte(LxcfsHomeDir)) {
+			isMount = true
+			break
+		}
+	}
+	if !isMount {
+		return fmt.Errorf("%s is not a mount point, please run \" lxcfs %s \" before debug", LxcfsHomeDir, LxcfsHomeDir)
+	}
+	return nil
+}

pkg/agent/runtime.go

@@ -9,7 +9,8 @@ import (
 	"log"
 	"time"
 
-	term "github.com/aylei/kubectl-debug/pkg/util"
+	"github.com/aylei/kubectl-debug/pkg/nsenter"
+	"github.com/aylei/kubectl-debug/pkg/util"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/strslice"
@@ -41,11 +42,12 @@ func NewRuntimeManager(host string, timeout time.Duration) (*RuntimeManager, err
 // DebugAttacher implements Attacher
 // we use this struct in order to inject debug info (image, command) in the debug procedure
 type DebugAttacher struct {
-	runtime *RuntimeManager
-	image   string
-	authStr string
-	command []string
-	client  *dockerclient.Client
+	runtime      *RuntimeManager
+	image        string
+	authStr      string
+	lxcfsEnabled bool
+	command      []string
+	client       *dockerclient.Client
 
 	// control the preparing of debug container
 	stopListenEOF chan struct{}
@@ -58,11 +60,12 @@ func (a *DebugAttacher) AttachContainer(name string, uid kubetype.UID, container
 }
 
 // GetAttacher returns an implementation of Attacher
-func (m *RuntimeManager) GetAttacher(image string, authStr string, command []string, context context.Context, cancel context.CancelFunc) kubeletremote.Attacher {
+func (m *RuntimeManager) GetAttacher(image, authStr string, lxcfsEnabled bool, command []string, context context.Context, cancel context.CancelFunc) kubeletremote.Attacher {
 	return &DebugAttacher{
 		runtime:       m,
 		image:         image,
 		authStr:       authStr,
+		lxcfsEnabled:  lxcfsEnabled,
 		command:       command,
 		context:       context,
 		client:        m.client,
@@ -101,6 +104,17 @@ func (m *DebugAttacher) DebugContainer(container, image string, authStr string,
 	//		}
 	//	}
 	//} ()
+	// step 0: set container procfs correct by lxcfs
+	stdout.Write([]byte(fmt.Sprintf("set container procfs correct %t .. \n\r", m.lxcfsEnabled)))
+	if m.lxcfsEnabled {
+		if err := CheckLxcfsMount(); err != nil {
+			return err
+		}
+
+		if err := m.SetContainerLxcfs(container); err != nil {
+			return err
+		}
+	}
 
 	// step 1: pull image
 	stdout.Write([]byte(fmt.Sprintf("pulling image %s... \n\r", image)))
@@ -129,6 +143,33 @@ func (m *DebugAttacher) DebugContainer(container, image string, authStr string,
 	return nil
 }
 
+func (m *DebugAttacher) SetContainerLxcfs(container string) error {
+	ctx, cancel := m.getContextWithTimeout()
+	defer cancel()
+	containerInstance, err := m.client.ContainerInspect(ctx, container)
+	if err != nil {
+		return err
+	}
+	for _, mount := range containerInstance.Mounts {
+		if mount.Destination == LxcfsRootDir {
+			log.Printf("remount lxcfs when the rootdir of lxcfs of target container has been mounted. \n\t ")
+			for _, procfile := range LxcfsProcFiles {
+				nsenter := &nsenter.MountNSEnter{
+					Target:     containerInstance.State.Pid,
+					MountLxcfs: true,
+				}
+				_, stderr, err := nsenter.Execute("--", "mount", "-B", LxcfsHomeDir+procfile, procfile)
+				if err != nil {
+					log.Printf("bind mount lxcfs files failed. \n\t reason: %s", stderr)
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
 // Run a new container, this container will join the network,
 // mount, and pid namespace of the given container
 func (m *DebugAttacher) RunDebugContainer(targetId string, image string, command []string) (string, error) {

pkg/agent/server.go

@@ -97,6 +97,12 @@ func (s *Server) ServeDebug(w http.ResponseWriter, req *http.Request) {
 		Stderr: false,
 		TTY:    true,
 	}
+	lxcfsEnabled := req.FormValue("lxcfsEnabled")
+	if lxcfsEnabled == "" || lxcfsEnabled == "false" {
+		LxcfsEnabled = false
+	} else if lxcfsEnabled == "true" {
+		LxcfsEnabled = true
+	}
 
 	context, cancel := context.WithCancel(req.Context())
 	defer cancel()
@@ -105,7 +111,7 @@ func (s *Server) ServeDebug(w http.ResponseWriter, req *http.Request) {
 	kubeletremote.ServeAttach(
 		w,
 		req,
-		s.runtimeApi.GetAttacher(image, authStr, commandSlice, context, cancel),
+		s.runtimeApi.GetAttacher(image, authStr, LxcfsEnabled, commandSlice, context, cancel),
 		"",
 		"",
 		dockerContainerId,

pkg/nsenter/nsenter.go

@@ -0,0 +1,61 @@
+package nsenter
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os/exec"
+	"strconv"
+)
+
+// MountNSEnter is the client used to enter the mount namespace
+type MountNSEnter struct {
+	Target     int    // target PID (required)
+	MountLxcfs bool   // enter mount namespace or not
+	MountFile  string // Mount namespace location, default to /proc/PID/ns/mnt
+}
+
+// Execute runs the given command with a default background context
+func (cli *MountNSEnter) Execute(command string, args ...string) (stdout, stderr string, err error) {
+	return cli.ExecuteContext(context.Background(), command, args...)
+}
+
+// ExecuteContext the given command using the specific nsenter config
+func (cli *MountNSEnter) ExecuteContext(ctx context.Context, command string, args ...string) (string, string, error) {
+	cmd, err := cli.setCommand(ctx)
+	if err != nil {
+		return "", "", fmt.Errorf("Error when set command: %v", err)
+	}
+
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	cmd.Args = append(cmd.Args, command)
+	cmd.Args = append(cmd.Args, args...)
+
+	err = cmd.Run()
+	if err != nil {
+		return stdout.String(), stderr.String(), fmt.Errorf("Error while executing command: %v", err)
+	}
+
+	return stdout.String(), stderr.String(), nil
+}
+
+func (cli *MountNSEnter) setCommand(ctx context.Context) (*exec.Cmd, error) {
+	if cli.Target == 0 {
+		return nil, fmt.Errorf("Target must be specified")
+	}
+	var args []string
+	args = append(args, "--target", strconv.Itoa(cli.Target))
+
+	if cli.MountLxcfs {
+		if cli.MountFile != "" {
+			args = append(args, fmt.Sprintf("--mount=%s", cli.MountFile))
+		} else {
+			args = append(args, "--mount")
+		}
+	}
+
+	cmd := exec.CommandContext(ctx, "/usr/bin/nsenter", args...)
+	return cmd, nil
+}

pkg/plugin/cmd.go

@@ -85,6 +85,7 @@ You may set default configuration such as image and command in the config file,
 
 	defaultPortForward = true
 	defaultAgentless   = true
+	defaultLxcfsEnable = true
 )
 
 // DebugOptions specify how to run debug container in a running pod
@@ -114,6 +115,8 @@ type DebugOptions struct {
 	AgentPodNamespace string
 	AgentPodNode      string
 	AgentPodResource  agentPodResources
+	// enable lxcfs
+	IsLxcfsEnabled bool
 
 	Flags      *genericclioptions.ConfigFlags
 	CoreClient coreclient.CoreV1Interface
@@ -214,6 +217,8 @@ func NewDebugCmd(streams genericclioptions.IOStreams) *cobra.Command {
 		fmt.Sprintf("Agentless mode, agent pod cpu limits, default is not set"))
 	cmd.Flags().StringVar(&opts.AgentPodResource.MemoryLimits, "agent-pod-memory-limits", "",
 		fmt.Sprintf("Agentless mode, agent pod memory limits, default is not set"))
+	cmd.Flags().BoolVarP(&opts.IsLxcfsEnabled, "enable-lxcfs", "", true,
+		fmt.Sprintf("Enable Lxcfs, the target container can use its proc files, default to %t", defaultLxcfsEnable))
 	opts.Flags.AddFlags(cmd.Flags())
 
 	return cmd
@@ -372,6 +377,12 @@ func (o *DebugOptions) Complete(cmd *cobra.Command, args []string, argsLenAtDash
 		}
 	}
 
+	if o.IsLxcfsEnabled {
+		o.IsLxcfsEnabled = config.IsLxcfsEnabled
+	} else {
+		o.IsLxcfsEnabled = defaultLxcfsEnable
+	}
+
 	if config.PortForward {
 		o.PortForward = true
 	}
@@ -534,6 +545,11 @@ func (o *DebugOptions) Run() error {
 		params := url.Values{}
 		params.Add("image", o.Image)
 		params.Add("container", containerID)
+		if o.IsLxcfsEnabled {
+			params.Add("lxcfsEnabled", "true")
+		} else {
+			params.Add("lxcfsEnabled", "false")
+		}
 		var authStr string
 		registrySecret, err := o.CoreClient.Secrets(o.RegistrySecretNamespace).Get(o.RegistrySecretName, v1.GetOptions{})
 		if err != nil {
@@ -725,6 +741,9 @@ func (o *DebugOptions) launchPod(pod *corev1.Pod) (*corev1.Pod, error) {
 
 // getAgentPod construnct agentPod from agent pod template
 func (o *DebugOptions) getAgentPod() *corev1.Pod {
+	prop := corev1.MountPropagationBidirectional
+	directoryCreate := corev1.HostPathDirectoryOrCreate
+	priveleged := true
 	agentPod := &corev1.Pod{
 		TypeMeta: v1.TypeMeta{
 			Kind:       "Pod",
@@ -735,6 +754,7 @@ func (o *DebugOptions) getAgentPod() *corev1.Pod {
 			Namespace: o.AgentPodNamespace,
 		},
 		Spec: corev1.PodSpec{
+			HostPID:  true,
 			NodeName: o.AgentPodNode,
 			Containers: []corev1.Container{
 				{
@@ -754,12 +774,24 @@ func (o *DebugOptions) getAgentPod() *corev1.Pod {
 						TimeoutSeconds:      1,
 						FailureThreshold:    3,
 					},
+					SecurityContext: &corev1.SecurityContext{
+						Privileged: &priveleged,
+					},
 					Resources: o.buildAgentResourceRequirements(),
 					VolumeMounts: []corev1.VolumeMount{
 						{
 							Name:      "docker",
 							MountPath: "/var/run/docker.sock",
 						},
+						{
+							Name:      "cgroup",
+							MountPath: "/sys/fs/cgroup",
+						},
+						{
+							Name:             "lxcfs",
+							MountPath:        "/var/lib/lxc/lxcfs",
+							MountPropagation: &prop,
+						},
 					},
 					Ports: []corev1.ContainerPort{
 						{
@@ -779,6 +811,23 @@ func (o *DebugOptions) getAgentPod() *corev1.Pod {
 						},
 					},
 				},
+				{
+					Name: "cgroup",
+					VolumeSource: corev1.VolumeSource{
+						HostPath: &corev1.HostPathVolumeSource{
+							Path: "/sys/fs/cgroup",
+						},
+					},
+				},
+				{
+					Name: "lxcfs",
+					VolumeSource: corev1.VolumeSource{
+						HostPath: &corev1.HostPathVolumeSource{
+							Path: "/var/lib/lxc/lxcfs",
+							Type: &directoryCreate,
+						},
+					},
+				},
 			},
 			RestartPolicy: corev1.RestartPolicyNever,
 		},