Merge pull request #89 from hyperledger/master

Master

Author: spivachuk

plenum/server/node.py

@@ -1118,10 +1118,11 @@ def onConnsChanged(self, joined: Set[str], left: Set[str]):
 
         if self.isReady():
             self.checkInstances()
-            for node in joined:
-                self.send_current_state_to_lagging_node(node)
+        else:
+            logger.debug("{} joined nodes {} but status is {}".format(self, joined, self.status))
         # Send ledger status whether ready (connected to enough nodes) or not
         for node in joined:
+            self.send_current_state_to_lagging_node(node)
             self.send_ledger_status_to_newly_connected_node(node)
 
     def request_ledger_status_from_nodes(self, ledger_id):
@@ -2567,7 +2568,7 @@ def updateSeqNoMap(self, committedTxns):
 
     def commitAndSendReplies(self, reqHandler, ppTime, reqs: List[Request],
                              stateRoot, txnRoot) -> List:
-        committedTxns = reqHandler.commit(len(reqs), stateRoot, txnRoot)
+        committedTxns = reqHandler.commit(len(reqs), stateRoot, txnRoot, ppTime)
         self.updateSeqNoMap(committedTxns)
         self.sendRepliesToClients(
             map(self.update_txn_with_extra_data, committedTxns),

plenum/server/pool_manager.py

@@ -200,7 +200,7 @@ def executePoolTxnBatch(self, ppTime, reqs, stateRoot, txnRoot) -> List:
         :param ppTime: PrePrepare request time
         :param reqs: request
         """
-        committedTxns = self.reqHandler.commit(len(reqs), stateRoot, txnRoot)
+        committedTxns = self.reqHandler.commit(len(reqs), stateRoot, txnRoot, ppTime)
         self.node.updateSeqNoMap(committedTxns)
         for txn in committedTxns:
             t = deepcopy(txn)

plenum/server/replica.py

@@ -48,7 +48,7 @@ def updateState(self, txns, isCommitted=False):
         not committed transactions
         """
 
-    def commit(self, txnCount, stateRoot, txnRoot) -> List:
+    def commit(self, txnCount, stateRoot, txnRoot, ppTime) -> List:
         """
         :param txnCount: The number of requests to commit (The actual requests
         are picked up from the uncommitted list from the ledger)

plenum/server/req_handler.py

@@ -1,15 +1,15 @@
 import types
 import pytest
-from plenum.common.exceptions import UnauthorizedClientRequest
+from plenum.common.exceptions import UnauthorizedClientRequest, RequestRejectedException
 from plenum.test.batching_3pc.helper import checkNodesHaveSameRoots
-from plenum.test.helper import sendRandomRequests, checkRejectWithReason, waitForSufficientRepliesForRequests
+from plenum.test.helper import sdk_send_random_requests, sdk_get_and_check_replies
 from stp_core.loop.eventually import eventually
 from plenum.common.exceptions import InvalidClientRequest
 from plenum.test.helper import sdk_sign_request_from_dict, sdk_send_random_and_check
 from plenum.common.request import Request
 
 
-def testRequestStaticValidation(tconf, looper,txnPoolNodeSet,
+def testRequestStaticValidation(tconf, looper, txnPoolNodeSet,
                                 sdk_wallet_client):
     """
     Check that for requests which fail static validation, REQNACK is sent
@@ -59,7 +59,7 @@ def testTreeRootsCorrectAfterEachBatch(tconf, looper, txnPoolNodeSet,
 
 
 def testRequestDynamicValidation(tconf, looper, txnPoolNodeSet,
-                                 client, wallet1):
+                                 sdk_pool_handle, sdk_wallet_client):
     """
     Check that for requests which fail dynamic (state based) validation,
     REJECT is sent to the client
@@ -82,16 +82,14 @@ def rejectingMethod(self, req):
         origMethods.append(node.doDynamicValidation)
         node.doDynamicValidation = types.MethodType(rejectingMethod, node)
 
-    reqs = sendRandomRequests(wallet1, client, tconf.Max3PCBatchSize)
-    waitForSufficientRepliesForRequests(looper, client, requests=reqs[:-1])
+    reqs = sdk_send_random_requests(looper, sdk_pool_handle,
+                                    sdk_wallet_client,
+                                    tconf.Max3PCBatchSize)
+    sdk_get_and_check_replies(looper, reqs[:-1])
+    with pytest.raises(RequestRejectedException) as e:
+        sdk_get_and_check_replies(looper, reqs[-1:])
 
-    with pytest.raises(AssertionError):
-        waitForSufficientRepliesForRequests(looper, client, requests=reqs[-1:])
-
-    for node in txnPoolNodeSet:
-        looper.run(eventually(checkRejectWithReason, client,
-                              'Simulated rejection', node.clientstack.name,
-                              retryWait=1))
+    assert 'Simulated rejection' in e._excinfo[1].args[0]
 
     for i, node in enumerate(txnPoolNodeSet):
         node.doDynamicValidation = origMethods[i]

plenum/test/batching_3pc/test_basic_batching.py

@@ -13,6 +13,7 @@
 from plenum.common.util import getCallableName
 from plenum.test.test_client import TestClient
 
+DEFAULT_DELAY = 600
 
 def delayer(seconds, op, senderFilter=None, instFilter: int = None):
     def inner(rx):
@@ -74,83 +75,83 @@ def inner(action_pair):
     return inner
 
 
-def nom_delay(delay: float, inst_id=None, sender_filter: str=None):
+def nom_delay(delay: float = DEFAULT_DELAY, inst_id=None, sender_filter: str=None):
     # Delayer of NOMINATE requests
     return delayerMsgTuple(
         delay, Nomination, instFilter=inst_id, senderFilter=sender_filter)
 
 
-def prim_delay(delay: float, inst_id=None, sender_filter: str=None):
+def prim_delay(delay: float = DEFAULT_DELAY, inst_id=None, sender_filter: str=None):
     # Delayer of PRIMARY requests
     return delayerMsgTuple(
         delay, Primary, instFilter=inst_id, senderFilter=sender_filter)
 
 
-def rel_delay(delay: float, inst_id=None, sender_filter: str=None):
+def rel_delay(delay: float = DEFAULT_DELAY, inst_id=None, sender_filter: str=None):
     # Delayer of REELECTION requests
     return delayerMsgTuple(
         delay, Reelection, instFilter=inst_id, senderFilter=sender_filter)
 
 
-def ppgDelay(delay: float, sender_filter: str=None):
+def ppgDelay(delay: float = DEFAULT_DELAY, sender_filter: str=None):
     # Delayer of PROPAGATE requests
     return delayerMsgTuple(delay, Propagate, senderFilter=sender_filter)
 
 
-def ppDelay(delay: float, instId: int=None, sender_filter: str=None):
+def ppDelay(delay: float = DEFAULT_DELAY, instId: int=None, sender_filter: str=None):
     # Delayer of PRE-PREPARE requests from a particular instance
     return delayerMsgTuple(delay, PrePrepare, instFilter=instId,
                            senderFilter=sender_filter)
 
 
-def pDelay(delay: float, instId: int=None, sender_filter: str=None):
+def pDelay(delay: float = DEFAULT_DELAY, instId: int=None, sender_filter: str=None):
     # Delayer of PREPARE requests from a particular instance
     return delayerMsgTuple(
         delay, Prepare, instFilter=instId, senderFilter=sender_filter)
 
 
-def cDelay(delay: float, instId: int=None, sender_filter: str=None):
+def cDelay(delay: float = DEFAULT_DELAY, instId: int=None, sender_filter: str=None):
     # Delayer of COMMIT requests from a particular instance
     return delayerMsgTuple(
         delay, Commit, instFilter=instId, senderFilter=sender_filter)
 
 
-def icDelay(delay: float):
+def icDelay(delay: float = DEFAULT_DELAY):
     # Delayer of INSTANCE-CHANGE requests
     return delayerMsgTuple(delay, InstanceChange)
 
 
-def vcd_delay(delay: float):
+def vcd_delay(delay: float = DEFAULT_DELAY):
     # Delayer of VIEW_CHANGE_DONE requests
     return delayerMsgTuple(delay, ViewChangeDone)
 
 
-def lsDelay(delay: float):
+def lsDelay(delay: float = DEFAULT_DELAY):
     # Delayer of LEDGER_STATUSES requests
     return delayerMsgTuple(delay, LedgerStatus)
 
 
-def cpDelay(delay: float):
+def cpDelay(delay: float = DEFAULT_DELAY):
     # Delayer of CONSISTENCY_PROOFS requests
     return delayerMsgTuple(delay, ConsistencyProof)
 
 
-def cqDelay(delay: float):
+def cqDelay(delay: float = DEFAULT_DELAY):
     # Delayer of CATCHUP_REQ requests
     return delayerMsgTuple(delay, CatchupReq)
 
 
-def cr_delay(delay: float):
+def cr_delay(delay: float = DEFAULT_DELAY):
     # Delayer of CATCHUP_REP requests
     return delayerMsgTuple(delay, CatchupRep)
 
 
-def req_delay(delay: float):
+def req_delay(delay: float = DEFAULT_DELAY):
     # Delayer of Request requests
     return delayerMsgTuple(delay, Request)
 
 
-def msg_req_delay(delay: float, types_to_delay: List=None):
+def msg_req_delay(delay: float = DEFAULT_DELAY, types_to_delay: List=None):
     # Delayer of MessageReq messages
     def specific_msgs(msg):
         if isinstance(
@@ -162,7 +163,7 @@ def specific_msgs(msg):
     return specific_msgs
 
 
-def msg_rep_delay(delay: float, types_to_delay: List=None):
+def msg_rep_delay(delay: float = DEFAULT_DELAY, types_to_delay: List=None):
     # Delayer of MessageRep messages
     def specific_msgs(msg):
         if isinstance(

plenum/test/delayers.py

@@ -11,9 +11,7 @@
     checkNodeRemotes
 from plenum.test.test_stack import CONNECTED, JOINED_NOT_ALLOWED
 from plenum.test import waits
-
-
-whitelist = ['discarding message']
+from plenum.test.pool_transactions.conftest import looper
 
 logger = getlogger()
 
@@ -59,7 +57,9 @@ def addNodeBackAndCheck(nodeIdx: int, expectedStatus: Status):
                 addNodeBack(nodeSet, looper, nodeNames[nodeIdx])
 
                 timeout = waits.expectedNodeStartUpTimeout() + \
-                    waits.expectedPoolInterconnectionTime(len(nodeSet))
+                          waits.expectedPoolInterconnectionTime(len(nodeSet))
+                # TODO: Probably it's better to modify waits.* functions
+                timeout *= 1.5
                 looper.run(eventually(checkNodeStatusRemotesAndF,
                                       expectedStatus,
                                       nodeIdx,

plenum/test/instances/test_instance_cannot_become_active_with_less_than_four_servers.py

@@ -6,27 +6,30 @@
 from plenum.test.delayers import delayerMsgTuple
 from plenum.test.test_node import TestNode
 from plenum.test import waits
-
+from plenum.test.pool_transactions.conftest import looper
+from plenum.test.node_request.conftest import committed1, \
+    prepared1, preprepared1, propagated1, reqAcked1, \
+    sent1, noRetryReq, faultyNodes
 
 nodeCount = 4
 
 faultyNodes = 1
 
 
 @pytest.fixture()
-def configNodeSet(nodeSet):
-    A, B, C, D = nodeSet.nodes.values()
+def configNodeSet(txnPoolNodeSet):
+    A, B, C, D = txnPoolNodeSet
     # Nodes C and D delay Commit request from node A for protocol instance 0
     for n in [C, D]:
         n.nodeIbStasher.delay(delayerMsgTuple(30,
                                               Commit,
                                               senderFilter=A.name,
                                               instFilter=0))
-    return nodeSet
+    return txnPoolNodeSet
 
 
 def testMsgFromInstanceDelay(configNodeSet, looper, prepared1):
-    A, B, C, D = configNodeSet.nodes.values()
+    A, B, C, D = configNodeSet
 
     def getCommits(node: TestNode, instId: int):
         replica = node.replicas[instId]  # type: Replica
@@ -41,6 +44,6 @@ def checkPresence():
             assert len(commReqs) > 0
             assert Replica.generateName(A.name, 1) in commReqs[0][0]
 
-    numOfNodes = len(configNodeSet.nodes)
+    numOfNodes = len(configNodeSet)
     timeout = waits.expectedClientRequestPropagationTime(numOfNodes)
     looper.run(eventually(checkPresence, retryWait=.5, timeout=timeout))

plenum/test/instances/test_msgs_from_slow_instances.py

@@ -12,15 +12,16 @@
     sendDuplicate3PhaseMsg
 from plenum.test.test_node import getNonPrimaryReplicas, getPrimaryReplica
 from plenum.test import waits
-
-whitelist = [Suspicions.DUPLICATE_CM_SENT.reason,
-             'cannot process incoming COMMIT']
+from plenum.test.pool_transactions.conftest import looper
+from plenum.test.node_request.conftest import committed1, \
+    prepared1, preprepared1, propagated1, reqAcked1, \
+    sent1, noRetryReq, faultyNodes
 
 
 @pytest.fixture("module")
-def setup(nodeSet, up):
-    primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
-        getNonPrimaryReplicas(nodeSet, 0)
+def setup(txnPoolNodeSet):
+    primaryRep, nonPrimaryReps = getPrimaryReplica(txnPoolNodeSet, 0), \
+                                 getNonPrimaryReplicas(txnPoolNodeSet, 0)
 
     faultyRep = nonPrimaryReps[0]
     makeNodeFaulty(faultyRep.node, partial(sendDuplicate3PhaseMsg,
@@ -32,13 +33,14 @@ def setup(nodeSet, up):
     # want to check for a particular suspicion
 
     whitelistNode(faultyRep.node.name,
-                  [node for node in nodeSet if node != faultyRep.node],
+                  [node for node in txnPoolNodeSet if node != faultyRep.node],
                   Suspicions.DUPLICATE_CM_SENT.code)
 
     # If the request is ordered then COMMIT will be rejected much earlier
     for r in [primaryRep, *nonPrimaryReps]:
         def do_nothing(self, commit):
             pass
+
         r.doOrder = types.MethodType(do_nothing, r)
 
     return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps,
@@ -53,7 +55,7 @@ def testMultipleCommit(setup, looper, sent1):
     should count only one COMMIT from that sender
     """
     primaryRep, nonPrimaryReps, faultyRep = setup.primaryRep, \
-        setup.nonPrimaryReps, setup.faultyRep
+                                            setup.nonPrimaryReps, setup.faultyRep
 
     def chkSusp():
         for r in (primaryRep, *nonPrimaryReps):

plenum/test/instances/test_multiple_commit.py

@@ -9,7 +9,6 @@
 from plenum.test.spy_helpers import getAllArgs
 from plenum.test import waits
 
-
 nodeCount = 7
 
 

plenum/test/instances/test_multiple_instance_change_msgs.py

@@ -12,21 +12,16 @@
     sendDuplicate3PhaseMsg
 from plenum.test.test_node import getNonPrimaryReplicas, getPrimaryReplica
 from plenum.test import waits
-
-
-whitelist = [Suspicions.DUPLICATE_PPR_SENT.reason,
-             'cannot process incoming PRE-PREPARE',
-             Suspicions.UNKNOWN_PR_SENT.reason,
-             'Invalid prepare message received',
-             'cannot process incoming PREPARE',
-             Suspicions.UNKNOWN_CM_SENT.reason,
-             'cannot process incoming COMMIT']
+from plenum.test.pool_transactions.conftest import looper
+from plenum.test.node_request.conftest import committed1, \
+    prepared1, preprepared1, propagated1, reqAcked1, \
+    sent1, noRetryReq, faultyNodes
 
 
 @pytest.fixture("module")
-def setup(nodeSet, up):
-    primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
-        getNonPrimaryReplicas(nodeSet, 0)
+def setup(txnPoolNodeSet):
+    primaryRep, nonPrimaryReps = getPrimaryReplica(txnPoolNodeSet, 0), \
+                                 getNonPrimaryReplicas(txnPoolNodeSet, 0)
 
     # The primary replica would send 3 duplicate PRE-PREPARE requests to
     # non primary replicas