feat: cleanup orphaned instances while evicting node

Signed-off-by: Raphanus Lo <[email protected]>

Author: COLDTURNIP

controller/instance_manager_controller.go

@@ -1004,6 +1004,17 @@ func (imc *InstanceManagerController) deleteInstanceManagerPDB(im *longhorn.Inst
 }
 
 func (imc *InstanceManagerController) syncOrphans(im *longhorn.InstanceManager) error {
+	var isNodeDeletedOrEvictionRequested bool
+	node, err := imc.ds.GetNode(im.Spec.NodeID)
+	switch {
+	case datastore.ErrorIsNotFound(err):
+		isNodeDeletedOrEvictionRequested = true
+	case err != nil:
+		return errors.Wrapf(err, "failed to check node eviction status on %v", im.Spec.NodeID)
+	default:
+		isNodeDeletedOrEvictionRequested = node.Spec.EvictionRequested
+	}
+
 	// Instances are live inside the instance manager pod.
 	// Remove the instance orphan CRs when the instance manager pod is deleted or not running.
 	var isInstanceManagerPodDeletedOrNotRunning bool
@@ -1025,7 +1036,7 @@ func (imc *InstanceManagerController) syncOrphans(im *longhorn.InstanceManager)
 	// Remove the corresponding orphan CRs, and they will be resync after the owner instance manager gets back.
 	instanceManagerOwnershipChanged := im.Spec.NodeID != imc.controllerID
 
-	isInstanceManagerTerminating := isInstanceManagerPodDeletedOrNotRunning || instanceManagerOwnershipChanged
+	isInstanceManagerTerminating := isNodeDeletedOrEvictionRequested || isInstanceManagerPodDeletedOrNotRunning || instanceManagerOwnershipChanged
 	return imc.deleteOrphans(im, isInstanceManagerTerminating)
 }
 

controller/node_controller.go

@@ -1305,15 +1305,15 @@ func (nc *NodeController) syncOrphans(node *longhorn.Node, collectedDataInfo map
 		newOrphanedReplicaDataStores, missingOrphanedReplicaDataStores :=
 			nc.getNewAndMissingOrphanedReplicaDataStores(diskName, diskInfo.DiskUUID, diskInfo.Path, diskInfo.OrphanedReplicaDataStores)
 
-		if err := nc.createOrphans(node, diskName, diskInfo, newOrphanedReplicaDataStores); err != nil {
+		if err := nc.createDataOrphans(node, diskName, diskInfo, newOrphanedReplicaDataStores); err != nil {
 			return errors.Wrapf(err, "failed to create orphans for disk %v", diskName)
 		}
-		if err := nc.deleteOrphans(node, diskName, diskInfo, missingOrphanedReplicaDataStores); err != nil {
+		if err := nc.deleteDataOrphans(node, diskName, diskInfo, missingOrphanedReplicaDataStores); err != nil {
 			return errors.Wrapf(err, "failed to delete orphans for disk %v", diskName)
 		}
 	}
 
-	return nil
+	return nc.deleteInstanceOrphans(node)
 }
 
 func (nc *NodeController) getNewAndMissingOrphanedReplicaDataStores(diskName, diskUUID, diskPath string, replicaDataStores map[string]string) (map[string]string, map[string]string) {
@@ -1356,7 +1356,33 @@ func (nc *NodeController) getNewAndMissingOrphanedReplicaDataStores(diskName, di
 	return newOrphanedReplicaDataStores, missingOrphanedReplicaDataStores
 }
 
-func (nc *NodeController) deleteOrphans(node *longhorn.Node, diskName string, diskInfo *monitor.CollectedDiskInfo, missingOrphanedReplicaDataStores map[string]string) error {
+func (nc *NodeController) deleteInstanceOrphans(node *longhorn.Node) error {
+	if !node.Spec.EvictionRequested {
+		return nil
+	}
+	nc.logger.Debugf("Deleting orphaned instances to evict node %v", node.Name)
+
+	orphans, err := nc.ds.ListOrphansByNodeRO(node.Name)
+	if err != nil {
+		return errors.Wrapf(err, "failed to list orphans to evict node %v", node.Name)
+	}
+
+	multiError := util.NewMultiError()
+	for _, orphan := range orphans {
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance, longhorn.OrphanTypeReplicaInstance:
+			if err := nc.ds.DeleteOrphan(orphan.Name); err != nil {
+				multiError.Append(util.NewMultiError(fmt.Sprintf("%v: %v", orphan.Name, err)))
+			}
+		}
+	}
+	if len(multiError) > 0 {
+		return fmt.Errorf("node controller failed to delete instance orphan CR: %v", multiError.Join())
+	}
+	return nil
+}
+
+func (nc *NodeController) deleteDataOrphans(node *longhorn.Node, diskName string, diskInfo *monitor.CollectedDiskInfo, missingOrphanedReplicaDataStores map[string]string) error {
 	autoDeletionResourceTypes, err := nc.ds.GetSettingOrphanResourceAutoDeletion()
 	if err != nil {
 		return errors.Wrapf(err, "failed to get %v setting", types.SettingNameOrphanResourceAutoDeletion)
@@ -1394,7 +1420,7 @@ func (nc *NodeController) deleteOrphans(node *longhorn.Node, diskName string, di
 	return nil
 }
 
-func (nc *NodeController) createOrphans(node *longhorn.Node, diskName string, diskInfo *monitor.CollectedDiskInfo, newOrphanedReplicaDataStores map[string]string) error {
+func (nc *NodeController) createDataOrphans(node *longhorn.Node, diskName string, diskInfo *monitor.CollectedDiskInfo, newOrphanedReplicaDataStores map[string]string) error {
 	for dataStore := range newOrphanedReplicaDataStores {
 		if err := nc.createOrphan(node, diskName, dataStore, diskInfo); err != nil && !apierrors.IsAlreadyExists(err) {
 			return errors.Wrapf(err, "failed to create orphan for orphaned replica data store %v in disk %v on node %v",