INDY-1112: change primeries election procedure for backup instances.

Now primaries for backup instances are choosen in round-robin
manner always starting from primary. If the next node is a primary
for some instance then this node is skipped. So the first non-primary
node is choosen as primary for current instance. Such approach allows
to avoid election of instances of the same node as a primeries for
different instances.
The election procedure of the primary for master instance is not changed.

Signed-off-by: Sergey Shilov <[email protected]>

Author: Sergey Shilov

plenum/server/node.py

@@ -2354,15 +2354,42 @@ def lost_master_primary(self):
         self._schedule_view_change()
 
     def select_primaries(self, nodeReg: Dict[str, HA]=None):
+        primaries = set()
+        primary_rank = None
+        '''
+        Build a set of names of primaries, it is needed to avoid
+        duplicates of primary nodes for different replicas.
+        '''
+        for instance_id, replica in enumerate(self.replicas):
+            if replica.primaryName is not None:
+                name = replica.primaryName.split(":", 1)[0]
+                primaries.add(name)
+                '''
+                Remember the rank of primary of master instance, it is needed
+                for calculation of primaries for backup instances.
+                '''
+                if instance_id == 0:
+                    primary_rank = self.poolManager.get_rank_by_name(
+                        replica.primaryName.split(":", 1)[0], nodeReg)
+
         for instance_id, replica in enumerate(self.replicas):
             if replica.primaryName is not None:
                 logger.debug('{} already has a primary'.format(replica))
                 continue
-            new_primary_name = self.elector.next_primary_replica_name(
-                instance_id, nodeReg=nodeReg)
+            if instance_id == 0:
+                new_primary_name, new_primary_instance_name =\
+                    self.elector.next_primary_replica_name_for_master(nodeReg=nodeReg)
+                primary_rank = self.poolManager.get_rank_by_name(
+                    new_primary_name, nodeReg)
+            else:
+                assert primary_rank != None
+                new_primary_name, new_primary_instance_name =\
+                    self.elector.next_primary_replica_name_for_backup(
+                        instance_id, primary_rank, primaries, nodeReg=nodeReg)
+            primaries.add(new_primary_name)
             logger.display("{}{} selected primary {} for instance {} (view {})"
                            .format(PRIMARY_SELECTION_PREFIX, replica,
-                                   new_primary_name, instance_id, self.viewNo),
+                                   new_primary_instance_name, instance_id, self.viewNo),
                            extra={"cli": "ANNOUNCE",
                                   "tags": ["node-election"]})
             if instance_id == 0:
@@ -2372,7 +2399,7 @@ def select_primaries(self, nodeReg: Dict[str, HA]=None):
                 # participating.
                 self.start_participating()
 
-            replica.primaryChanged(new_primary_name)
+            replica.primaryChanged(new_primary_instance_name)
             self.primary_selected(instance_id)
 
             logger.display("{}{} declares view change {} as completed for "
@@ -2383,7 +2410,7 @@ def select_primaries(self, nodeReg: Dict[str, HA]=None):
                                    replica,
                                    self.viewNo,
                                    instance_id,
-                                   new_primary_name,
+                                   new_primary_instance_name,
                                    self.ledger_summary),
                            extra={"cli": "ANNOUNCE",
                                   "tags": ["node-election"]})

plenum/server/pool_manager.py

@@ -411,6 +411,11 @@ def get_rank_of(self, node_id, nodeReg=None) -> Optional[int]:
             return None
         return self._get_rank(node_id, self.node_ids_ordered_by_rank(nodeReg))
 
+    def get_rank_by_name(self, name, nodeReg=None) -> Optional[int]:
+        for nym, nm in self._ordered_node_ids.items():
+            if name == nm:
+                return self.get_rank_of(nym, nodeReg)
+
     def get_name_by_rank(self, rank, nodeReg=None) -> Optional[str]:
         try:
             nym = self.node_ids_ordered_by_rank(nodeReg)[rank]

plenum/server/primary_selector.py

@@ -54,14 +54,56 @@ def next_primary_node_name(self, instance_id, nodeReg=None):
 
         return name
 
-    def next_primary_replica_name(self, instance_id, nodeReg=None):
+    def next_primary_node_name_for_master(self, nodeReg=None):
+        if nodeReg is None:
+            nodeReg = self.node.nodeReg
+        rank = self._get_primary_id(self.viewNo, 0, len(nodeReg))
+        name = self.node.get_name_by_rank(rank, nodeReg=nodeReg)
+
+        logger.trace("{} selected {} as next primary node for master instance, "
+                     "viewNo {} with rank {}, nodeReg {}".format(
+                         self, name, self.viewNo, rank, nodeReg))
+        assert name, "{} failed to get next primary node name for master instance".format(self)
+
+        return name
+
+    def next_primary_node_name_for_backup(self, instance_id, nodeReg=None):
+        if nodeReg is None:
+            nodeReg = self.node.nodeReg
+        rank = self._get_primary_id(self.viewNo, instance_id, len(nodeReg))
+        name = self.node.get_name_by_rank(rank, nodeReg=nodeReg)
+
+        logger.trace("{} selected {} as next primary node for instId {}, "
+                     "viewNo {} with rank {}, nodeReg {}".format(
+                         self, name, instance_id, self.viewNo, rank, nodeReg))
+        assert name, "{} failed to get next primary node name".format(self)
+
+        return name
+
+    def next_primary_replica_name_for_master(self, nodeReg=None):
         """
         Returns name of the next node which is supposed to be a new Primary
         in round-robin fashion
         """
-        return Replica.generateName(
-            nodeName=self.next_primary_node_name(instance_id, nodeReg=nodeReg),
-            instId=instance_id)
+        name = self.next_primary_node_name_for_master(nodeReg)
+        return name, Replica.generateName(nodeName=name, instId=0)
+
+    def next_primary_replica_name_for_backup(self, instance_id, master_primary_rank,
+                                             primaries, nodeReg=None):
+        """
+        Returns name of the next node which is supposed to be a new Primary
+        for backup instance in round-robin fashion starting from primary of
+        master instance.
+        """
+        if nodeReg is None:
+            nodeReg = self.node.nodeReg
+        total_nodes = len(nodeReg)
+        rank = (master_primary_rank + 1) % total_nodes
+        name = self.node.get_name_by_rank(rank, nodeReg=nodeReg)
+        while name in primaries:
+            rank = (rank + 1) % total_nodes
+            name = self.node.get_name_by_rank(rank, nodeReg=nodeReg)
+        return name, Replica.generateName(nodeName=name, instId=instance_id)
 
     # overridden method of PrimaryDecider
     def start_election_for_instance(self, instance_id):