fix: rebuilding gets stuck if verification fails and longhorn-manager crashes

Longhorn 8589

Signed-off-by: Shuo Wu <[email protected]>
(cherry picked from commit 921f63f)

Author: shuo-wu

pkg/controller/rebuild.go

@@ -35,12 +35,20 @@ func (c *Controller) getCurrentAndRWReplica(address string) (*types.Replica, *ty
 	return current, rwReplica, nil
 }
 
-func (c *Controller) VerifyRebuildReplica(address, instanceName string) error {
+func (c *Controller) VerifyRebuildReplica(address, instanceName string) (err error) {
 	// Prevent snapshot happens at the same time, as well as prevent
 	// writing from happening since we're updating revision counter
 	c.Lock()
 	defer c.Unlock()
 
+	// TODO: It's unreasonable to set the replica mode to ERR for some of the below errors. Need to improve in the future.
+	defer func() {
+		if err != nil {
+			logrus.WithError(err).Errorf("Failed to verify rebuild replica %v for volume %v", address, c.VolumeName)
+			c.setReplicaModeNoLock(address, types.ERR)
+		}
+	}()
+
 	replica, rwReplica, err := c.getCurrentAndRWReplica(address)
 	if err != nil {
 		return err

pkg/sync/rpc/server.go

@@ -430,6 +430,7 @@ func (s *SyncAgentServer) FilesSync(ctx context.Context, req *enginerpc.FilesSyn
 			s.RebuildStatus.State = types.ProcessStateError
 			logrus.WithError(err).Error("Sync agent gRPC server failed to rebuild replica/sync files")
 		} else {
+			s.RebuildStatus.Progress = 100
 			s.RebuildStatus.State = types.ProcessStateComplete
 			logrus.Infof("Sync agent gRPC server finished rebuilding replica/sync files for replica %v", req.ToHost)
 		}