fix(orphan): finalize orphan instance CR when instance manager unavailable

Signed-off-by: Raphanus Lo <[email protected]>

Author: COLDTURNIP

controller/orphan_controller.go

@@ -315,7 +315,7 @@ func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (is
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
-	// Make sure if the orphan nodeID and controller ID are same.
+	// Make sure if the orphan nodeID and controller ID are the same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
 		log.WithFields(logrus.Fields{
@@ -370,7 +370,7 @@ func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orpha
 	} else {
 		spec = &engineCR.Spec.InstanceSpec
 	}
-	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeEngine, spec)
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeEngine, spec), nil
 }
 
 func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
@@ -394,7 +394,7 @@ func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orph
 	} else {
 		spec = &replicaCR.Spec.InstanceSpec
 	}
-	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeReplica, spec)
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeReplica, spec), nil
 }
 
 func (oc *OrphanController) extractOrphanedInstanceInfo(orphan *longhorn.Orphan) (name, instanceManager string, err error) {
@@ -418,18 +418,21 @@ func (oc *OrphanController) extractOrphanedInstanceInfo(orphan *longhorn.Orphan)
 	return name, instanceManager, nil
 }
 
-func (oc *OrphanController) cleanupOrphanedInstance(orphan *longhorn.Orphan, instance, imName string, imType longhorn.InstanceManagerType, instanceCRSpec *longhorn.InstanceSpec) (isCleanupComplete bool, err error) {
+func (oc *OrphanController) cleanupOrphanedInstance(orphan *longhorn.Orphan, instance, imName string, imType longhorn.InstanceManagerType, instanceCRSpec *longhorn.InstanceSpec) bool {
 	if instanceCRSpec != nil && instanceCRSpec.NodeID == orphan.Spec.NodeID {
 		oc.logger.Infof("Orphan instance %v is scheduled back to current node %v. Skip cleaning up the instance resource and finalize the orphan CR.", instance, orphan.Spec.NodeID)
-		return true, nil
+		return true
 	}
 
+	// If the instance manager client is unavailable or failed to delete the instance, continue finalizing the orphan.
+	// Later if the orphaned instance is still reachable, the orphan will be recreated.
 	imc, err := oc.getRunningInstanceManagerClientForOrphan(orphan, imName)
 	if err != nil {
-		return false, errors.Wrapf(err, "failed to get running instance manager client for orphan %v", orphan.Name)
+		oc.logger.WithError(err).Warnf("Failed to delete orphan instance %v due to instance manager client initialization failure. Continue to finalize orphan %v", instance, orphan.Name)
+		return false
 	} else if imc == nil {
 		oc.logger.WithField("orphanInstanceNode", orphan.Spec.NodeID).Warnf("No running instance manager for deleting orphan instance %v", orphan.Name)
-		return true, nil
+		return true
 	}
 	defer func() {
 		if closeErr := imc.Close(); closeErr != nil {
@@ -438,13 +441,21 @@ func (oc *OrphanController) cleanupOrphanedInstance(orphan *longhorn.Orphan, ins
 	}()
 
 	if err := oc.deleteInstance(imc, instance, imType, orphan.Spec.DataEngine); err != nil {
-		return false, err
+		oc.logger.WithError(err).Warnf("Failed to delete orphan instance %v due to instance manager client error. Continue to finalize orphan %v", instance, orphan.Name)
+		return true
 	}
 	isCleanupComplete, cleanupErr := oc.confirmOrphanInstanceCleanup(imc, instance, imType, orphan.Spec.DataEngine)
-	if cleanupErr == nil && !isCleanupComplete {
-		oc.logger.Infof("Orphan instance %v cleanup in progress", instance)
+	switch {
+	case cleanupErr != nil:
+		oc.logger.WithError(err).Warnf("Failed to confirm deletion of orphan instance %v due to instance manager client error. Continue to finalize orphan %v", instance, orphan.Name)
+		return true
+	case !isCleanupComplete:
+		oc.logger.Infof("Orphan instance %v cleanup in progress, waiting for instance state update", instance)
+		return false
+	default:
+		oc.logger.Infof("Orphan instance %v cleanup complete", instance)
+		return true
 	}
-	return isCleanupComplete, cleanupErr
 }
 
 func (oc *OrphanController) getRunningInstanceManagerClientForOrphan(orphan *longhorn.Orphan, imName string) (*engineapi.InstanceManagerClient, error) {