Debug blocking ipsec upgrade due to mcp pause

We noticied network operator goes ahead with deploying newer ipsec dameon
though some of the custom machine config pools are in paused state (which
means IPsec machine configs are not deployed on those pool nodes yet).
machine config pool label selectors and processing of these labels in cno
seem to be okay, so need to find out which causes cno to continue with
rendering newer ipsec daemon set and it fails on these pool nodes because
ipsec packages are not installed, hence networks co goes into progressing
state.

Signed-off-by: Periyasamy Palanisamy <[email protected]>

Author: pperiyasamy

pkg/network/ovn_kubernetes.go

@@ -632,6 +632,8 @@ func shouldRenderIPsec(conf *operv1.OVNKubernetesConfig, bootstrapResult *bootst
 	// While OVN ipsec is being upgraded and IPsec MachineConfigs deployment is in progress
 	// (or) IPsec config in OVN is being disabled, then ipsec deployment is not updated.
 	renderIPsecDaemonSetAsCreateWaitOnly = isIPsecMachineConfigNotActiveOnUpgrade || (isOVNIPsecActive && !renderIPsecOVN)
+	klog.Infof("IPsec upgrade block: isIpsecUpgrade=%t, isIPsecMachineConfigNotActiveOnUpgrade=%t, renderIPsecDaemonSetAsCreateWaitOnly=%t",
+		isIpsecUpgrade, isIPsecMachineConfigNotActiveOnUpgrade, renderIPsecDaemonSetAsCreateWaitOnly)
 
 	return
 }
@@ -1473,13 +1475,17 @@ func isIPsecMachineConfigActive(infra bootstrap.InfraStatus) bool {
 	}
 	for _, masterMCPStatus := range infra.MasterMCPStatuses {
 		if !ipSecPluginOnPool(masterMCPStatus, infra.MasterIPsecMachineConfigs) {
+			klog.Infof("IPsec upgrade block: master mcp %v is not ready with ipsec plugin", masterMCPStatus)
 			return false
 		}
+		klog.Infof("IPsec upgrade block: master mcp %v is ready with ipsec plugin", masterMCPStatus)
 	}
 	for _, workerMCPStatus := range infra.WorkerMCPStatuses {
 		if !ipSecPluginOnPool(workerMCPStatus, infra.WorkerIPsecMachineConfigs) {
+			klog.Infof("IPsec upgrade block: worker mcp %v is not ready with ipsec plugin", workerMCPStatus)
 			return false
 		}
+		klog.Infof("IPsec upgrade block: worker mcp %v is ready with ipsec plugin", workerMCPStatus)
 	}
 	return true
 }

pkg/platform/platform.go

@@ -250,13 +250,16 @@ func getMachineConfigPoolStatuses(ctx context.Context, client cnoclient.Client,
 	}
 	var mcpStatuses []mcfgv1.MachineConfigPoolStatus
 	for _, mcp := range mcpList.Items {
+		klog.Infof("IPsec upgrade block: retrieved machine config pool: %s", mcp.Name)
 		mcSelector, err := metav1.LabelSelectorAsSelector(mcp.Spec.MachineConfigSelector)
 		if err != nil {
 			return nil, fmt.Errorf("invalid machine config label selector in %s pool", mcp.Name)
 		}
 		if mcSelector.Matches(mcLabel) {
+			klog.Infof("IPsec upgrade block: machine config pool %s matches with label: %s", mcp.Name, mcLabel.String())
 			mcpStatuses = append(mcpStatuses, mcp.Status)
 		}
 	}
+	klog.Infof("IPsec upgrade block: %d number of machine config pools matches with label: %s", len(mcpStatuses), mcLabel.String())
 	return mcpStatuses, nil
 }