Support collect logs for failed agents and controller for supportbundle

Signed-off-by: Hang Yan <[email protected]>

Author: hangyan

pkg/antctl/raw/supportbundle/command.go

@@ -33,10 +33,15 @@ import (
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/time/rate"
 	"gopkg.in/yaml.v2"
+
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+
+	corev1 "k8s.io/api/core/v1"
+
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8sruntime "k8s.io/apimachinery/pkg/runtime"
+	utilerror "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
@@ -46,7 +51,10 @@ import (
 	"antrea.io/antrea/pkg/apis/crd/v1beta1"
 	systemv1beta1 "antrea.io/antrea/pkg/apis/system/v1beta1"
 	antrea "antrea.io/antrea/pkg/client/clientset/versioned"
+
 	systemclientset "antrea.io/antrea/pkg/client/clientset/versioned/typed/system/v1beta1"
+
+	"antrea.io/antrea/pkg/util/compress"
 )
 
 const (
@@ -584,6 +592,20 @@ func controllerRemoteRunE(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("failed to create clientset: %w", err)
 	}
 
+	if err := os.MkdirAll(option.dir, 0700|os.ModeDir); err != nil {
+		return fmt.Errorf("error when creating output dir: %w", err)
+	}
+
+	f, err := os.Create(filepath.Join(option.dir, "clusterinfo"))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	err = getClusterInfo(f, k8sClientset)
+	if err != nil {
+		return err
+	}
+
 	var controllerClient systemclientset.SupportBundleInterface
 	var agentClients map[string]systemclientset.SupportBundleInterface
 
@@ -628,29 +650,17 @@ func controllerRemoteRunE(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("no matched Nodes found to collect agent bundles")
 	}
 
-	if err := os.MkdirAll(option.dir, 0700|os.ModeDir); err != nil {
-		return fmt.Errorf("error when creating output dir: %w", err)
-	}
 	amount := len(agentClients) * 2
 	if controllerClient != nil {
 		amount += 2
 	}
 	bar := barTmpl.Start(amount)
 	defer bar.Finish()
 	defer bar.Set("prefix", "Finish ")
-	f, err := os.Create(filepath.Join(option.dir, "clusterinfo"))
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	err = getClusterInfo(f, k8sClientset)
-	if err != nil {
-		return err
-	}
 
 	results := requestAll(ctx, agentClients, controllerClient, bar)
 	results = downloadAll(ctx, agentClients, controllerClient, dir, bar, results)
-	return processResults(results, dir)
+	return processResults(antreaClientset, k8sClientset, results, dir)
 }
 
 func genErrorMsg(resultMap map[string]error) string {
@@ -662,8 +672,9 @@ func genErrorMsg(resultMap map[string]error) string {
 }
 
 // processResults will output the failed nodes and their reasons if any. If no data was collected,
-// error is returned, otherwise will return nil.
-func processResults(resultMap map[string]error, dir string) error {
+// error is returned, otherwise will return nil. For failed nodes and controller, will also trying to get logs from
+// kubernetes api.
+func processResults(antreaClientset antrea.Interface, k8sClient kubernetes.Interface, resultMap map[string]error, dir string) error {
 	resultStr := ""
 	var failedNodes []string
 	allFailed := true
@@ -679,7 +690,8 @@ func processResults(resultMap map[string]error, dir string) error {
 		}
 	}
 
-	if resultMap[""] != nil {
+	controllerFail := resultMap[""] != nil
+	if controllerFail {
 		fmt.Println("Controller Info Failed Reason: " + resultMap[""].Error())
 	}
 
@@ -692,9 +704,149 @@ func processResults(resultMap map[string]error, dir string) error {
 		err = writeFailedNodes(dir, failedNodes)
 	}
 
+	// download logs from kubernetes api
+	if failedNodes != nil || controllerFail {
+		err := downloadPodInfoFromKubernetes(antreaClientset, k8sClient, failedNodes, controllerFail, dir)
+		if err != nil {
+			fmt.Println("Failed to download logs from kubernetes api: " + err.Error())
+		} else {
+			allFailed = false
+		}
+	}
+
 	if allFailed {
 		return fmt.Errorf("no data was collected: %s", genErrorMsg(resultMap))
 	} else {
 		return err
 	}
 }
+
+// downloadLogsFromKubernetes will try to download pod logs from kubernetes api for failed nodes and controller, as well as controllerinfo/agentinfo...
+func downloadPodInfoFromKubernetes(antreaClientset antrea.Interface, k8sClient kubernetes.Interface, failedNodes []string, isControllerFail bool, dir string) error {
+	agentInfoList, err := antreaClientset.CrdV1beta1().AntreaAgentInfos().List(context.TODO(), metav1.ListOptions{ResourceVersion: "0"})
+	if err != nil {
+		return err
+	}
+
+	agentInfoMap := map[string]v1beta1.AntreaAgentInfo{}
+	for _, agentInfo := range agentInfoList.Items {
+		agentInfoMap[agentInfo.Name] = agentInfo
+	}
+
+	pods, err := k8sClient.CoreV1().Pods("kube-system").List(context.TODO(), metav1.ListOptions{
+		ResourceVersion: "0",
+		LabelSelector:   "app=antrea",
+	})
+	if err != nil {
+		return err
+	}
+
+	failedNodesMap := make(map[string]struct{})
+	for _, node := range failedNodes {
+		failedNodesMap[node] = struct{}{}
+	}
+
+	var errors []error
+
+	for _, pod := range pods.Items {
+
+		tmpDir, err := afero.TempDir(defaultFS, "", "bundle_tmp_")
+		if err != nil {
+			errors = append(errors, err)
+			continue
+		}
+		defer defaultFS.RemoveAll(tmpDir)
+
+		if pod.Labels["component"] == "antrea-controller" && isControllerFail {
+			controllerInfo, err := antreaClientset.CrdV1beta1().AntreaControllerInfos().Get(context.TODO(), "antrea-controller", metav1.GetOptions{})
+			if err != nil {
+				errors = append(errors, err)
+				continue
+			}
+			data, err := yaml.Marshal(controllerInfo)
+			if err != nil {
+				errors = append(errors, err)
+				continue
+			}
+			err = afero.WriteFile(defaultFS, filepath.Join(tmpDir, "controllerinfo"), data, 0644)
+			errors = append(errors, err)
+
+			err = downloadPodLogs(k8sClient, "controller", pod.Namespace, pod.Name, []string{"antrea-controller"}, dir, tmpDir)
+			errors = append(errors, err)
+
+		}
+
+		if _, exist := failedNodesMap[pod.Spec.NodeName]; !exist {
+			continue
+		}
+
+		if pod.Labels["component"] == "antrea-agent" {
+			if agentInfo, ok := agentInfoMap[pod.Spec.NodeName]; ok {
+				data, err := yaml.Marshal(agentInfo)
+				if err != nil {
+					errors = append(errors, err)
+					continue
+				}
+				err = afero.WriteFile(defaultFS, filepath.Join(tmpDir, "agentinfo"), data, 0644)
+				errors = append(errors, err)
+			}
+
+			err = downloadPodLogs(k8sClient, "agent_"+pod.Spec.NodeName, pod.Namespace, pod.Name, []string{"antrea-agent", "antrea-ovs", "install-cni"}, dir, tmpDir)
+			errors = append(errors, err)
+
+		}
+	}
+	return utilerror.NewAggregate(errors)
+}
+
+func downloadPodLogs(k8sClient kubernetes.Interface, comp string, namespace string, podName string, containers []string, dir string, tmpDir string) error {
+	var errors []error
+
+	for _, containerName := range containers {
+		containerDirName := containerName
+		if strings.HasPrefix(containerName, "antrea-") {
+			containerDirName = strings.ReplaceAll(containerName, "antrea-", "")
+		}
+
+		podLogDir := filepath.Join(tmpDir, "logs", containerDirName)
+		err := os.MkdirAll(podLogDir, 0755)
+		if err != nil {
+			return err
+		}
+
+		fileName := filepath.Join(podLogDir, containerName+".log")
+		f, err := os.Create(fileName)
+		if err != nil {
+			errors = append(errors, err)
+			continue
+		}
+
+		logOption := &corev1.PodLogOptions{
+			Container: containerName,
+		}
+		logs := k8sClient.CoreV1().Pods(namespace).GetLogs(podName, logOption)
+		logStream, err := logs.Stream(context.TODO())
+		if err != nil {
+			errors = append(errors, err)
+			continue
+		}
+
+		_, err = io.Copy(f, logStream)
+		errors = append(errors, err)
+		err = logStream.Close()
+		errors = append(errors, err)
+	}
+
+	gzFileName := filepath.Join(dir, comp+".tar.gz")
+
+	f, err := os.Create(gzFileName)
+	if err != nil {
+		errors = append(errors, err)
+	} else {
+		_, err := compress.PackDir(defaultFS, tmpDir, f)
+		errors = append(errors, err)
+	}
+
+	return utilerror.NewAggregate(errors)
+
+}