Merge pull request #86 from hyperledger/master

Master

Author: spivachuk

Jenkinsfile

@@ -78,7 +78,6 @@ def stpTestUbuntu = {
             testHelpers.install()
 
             echo 'Ubuntu Test: Test'
-            testHelpers.testJUnit([testDir: 'stp_raet', resFile: "test-result-stp-raet.${NODE_NAME}.xml"])
             testHelpers.testJUnit([testDir: 'stp_zmq', resFile: "test-result-stp-zmq.${NODE_NAME}.xml"])
         }
     }

Jenkinsfile.cd

@@ -78,7 +78,6 @@ def stpTestUbuntu = {
             testHelpers.install()
 
             echo 'Ubuntu Test: Test'
-            testHelpers.testJUnit([testDir: 'stp_raet', resFile: "test-result-stp-raet.${NODE_NAME}.xml"])
             testHelpers.testJUnit([testDir: 'stp_zmq', resFile: "test-result-stp-zmq.${NODE_NAME}.xml"])
         }
     }

Jenkinsfile.ci

@@ -128,7 +128,6 @@ def staticCodeValidation() {
 
 def tests = [
     stp: { python ->
-        test(testDir: 'stp_raet', resFile: "test-result-stp-raet.${NODE_NAME}.xml", python: python)
         test(testDir: 'stp_zmq', resFile: "test-result-stp-zmq.${NODE_NAME}.xml", python: python)
     },
     ledger: { python ->

README.md

@@ -49,8 +49,7 @@ mechanism that Plenum needs.
     - state storage using python 3 version of Ethereum's Patricia Trie
 - stp:
     - secure transport abstraction
-    - it has two implementations: RAET and ZeroMQ
-    - Although RAET implementation is there, it's not supported anymore, and [ZeroMQ](http://zeromq.org/) is the default secure transport in plenum. 
+    - it has [ZeroMQ](http://zeromq.org/) implementations
 - storage:
     - key-value storage abstractions
     - contains [leveldb](http://leveldb.org/) implementation as the main key-valued storage used in Plenum (for ledger, state, etc.)

build-scripts/ubuntu-1604/build-3rd-parties.sh

@@ -44,7 +44,8 @@ build_from_pypi base58 0.2.4
 build_from_pypi prompt-toolkit 0.57
 build_from_pypi rlp 0.5.1
 build_from_pypi sha3 0.2.1
-build_from_pypi raet 0.6.6
+build_from_pypi libnacl 1.6.1
+build_from_pypi six 1.11.0
 build_from_pypi pyzmq 16.0.2
 build_from_pypi intervaltree 2.1.0
 build_from_pypi portalocker 0.5.7

ci/ubuntu.dockerfile

@@ -8,7 +8,7 @@ ARG venv=venv
 RUN apt-get update -y && apt-get install -y \
     python3-nacl \
     libindy-crypto=0.2.0 \
-    libindy
+    libindy=1.3.1~403
 
 RUN indy_ci_add_user $uid $user $venv
 

plenum/cli/cli.py

@@ -40,7 +40,6 @@
 from stp_core.crypto.util import cleanSeed, seedFromHex
 from stp_core.network.port_dispenser import genHa
 from stp_core.types import HA
-from stp_raet.util import getLocalEstateData
 from plenum.common.config_helper import PNodeConfigHelper
 
 import configparser
@@ -231,20 +230,13 @@ def __init__(self, looper, basedirpath: str, ledger_base_dir: str, nodeReg=None,
             eventloop=eventloop,
             output=out)
 
-        # RAETVerbosity = getRAETLogLevelFromConfig("RAETLogLevelCli",
-        #                                           Console.Wordage.mute,
-        #                                           self.config)
-        # RAETLogFile = getRAETLogFilePath("RAETLogFilePathCli", self.config)
         # Patch stdout in something that will always print *above* the prompt
         # when something is written to stdout.
         sys.stdout = self.cli.stdout_proxy()
 
         if logFileName:
             Logger().enableFileLogging(logFileName)
 
-        # TODO: If we want RAET logging in CLI we need fix this. See INDY-315.
-        # Logger().setupRaet(RAETVerbosity, RAETLogFile)
-
         self.logger = getlogger("cli")
         self.print("\n{}-CLI (c) 2017 Evernym, Inc.".format(self.properName))
         self._actions = []
@@ -1062,8 +1054,7 @@ def newClient(self, clientName,
             if not areKeysSetup(clientName, self.basedirpath):
                 client_addr = genHa(ip='0.0.0.0')
             else:
-                client_addr = tuple(getLocalEstateData(clientName,
-                                                       self.basedirpath)['ha'])
+                raise Exception("Usage of deprecated code")
             nodeReg = None if self.nodeRegLoadedFromFile else self.cliNodeReg
             client = self.ClientClass(clientName,
                                       ha=client_addr,

plenum/client/client.py

@@ -107,7 +107,6 @@ def __init__(self,
         self.keys_dir = os.path.join(_keys_dir, "keys")
 
         cha = None
-        # If client information already exists is RAET then use that
         if self.exists(self.stackName, self.keys_dir):
             cha = self.nodeStackClass.getHaFromLocal(
                 self.stackName, self.keys_dir)

plenum/common/batched.py

@@ -141,8 +141,11 @@ def flushOutBoxes(self) -> None:
             del self.outBoxes[rid]
 
     def _make_batch(self, msgs):
-        batch = Batch(msgs, None)
-        serialized_batch = self.sign_and_serialize(batch)
+        if len(msgs) > 1:
+            batch = Batch(msgs, None)
+            serialized_batch = self.sign_and_serialize(batch)
+        else:
+            serialized_batch = msgs[0]
         return serialized_batch
 
     def _test_batch_len(self, batch_len):

plenum/common/ledger_info.py

@@ -47,7 +47,8 @@ def set_defaults(self):
 
         # Key of the 3PC-batch ordered by the master instance that contained
         # the last transaction of this node's ledger
-        self.last_txn_3PC_key = None
+        # This is a map of last 3PC for each received LedgerStatus
+        self.last_txn_3PC_key = {}
 
         # Dictionary of consistency proofs received for the ledger
         # in process of catching up
@@ -83,7 +84,7 @@ def done_syncing(self):
         self.canSync = False
         self.state = LedgerState.synced
         self.ledgerStatusOk = set()
-        self.last_txn_3PC_key = None
+        self.last_txn_3PC_key = {}
         self.recvdConsistencyProofs = {}
         self.receivedCatchUpReplies = []
         self.recvdCatchupRepliesFrm = {}

plenum/common/ledger_manager.py

@@ -1,42 +1,39 @@
 import heapq
+import math
 import operator
-from collections import Callable
+import time
+from collections import Callable, Counter
 from functools import partial
 from random import shuffle
 from typing import Any, List, Dict, Tuple
-import math
 from typing import Optional
 
-import time
-from plenum.common.ledger import Ledger
 from ledger.merkle_verifier import MerkleVerifier
 from ledger.util import F
-
-from plenum.common.messages.node_messages import LedgerStatus, CatchupRep, \
-    ConsistencyProof, f, CatchupReq
+from plenum.common.config_util import getConfig
 from plenum.common.constants import POOL_LEDGER_ID, LedgerState, DOMAIN_LEDGER_ID, \
     CONSISTENCY_PROOF, CATCH_UP_PREFIX
-from plenum.common.util import compare_3PC_keys, SortedDict
-from plenum.common.config_util import getConfig
+from plenum.common.ledger import Ledger
+from plenum.common.ledger_info import LedgerInfo
+from plenum.common.messages.node_messages import LedgerStatus, CatchupRep, \
+    ConsistencyProof, f, CatchupReq
+from plenum.common.txn_util import reqToTxn
+from plenum.common.util import compare_3PC_keys, SortedDict, mostCommonElement, min_3PC_key
+from plenum.server.has_action_queue import HasActionQueue
 from plenum.server.quorums import Quorums
 from stp_core.common.constants import CONNECTION_PREFIX
 from stp_core.common.log import getlogger
-from plenum.server.has_action_queue import HasActionQueue
-from plenum.common.ledger_info import LedgerInfo
-from plenum.common.txn_util import reqToTxn
-
 
 logger = getlogger()
 
 
 class LedgerManager(HasActionQueue):
-
     def __init__(self,
                  owner,
-                 ownedByNode: bool=True,
-                 postAllLedgersCaughtUp: Optional[Callable]=None,
-                 preCatchupClbk: Optional[Callable]=None,
-                 ledger_sync_order: Optional[List]=None):
+                 ownedByNode: bool = True,
+                 postAllLedgersCaughtUp: Optional[Callable] = None,
+                 preCatchupClbk: Optional[Callable] = None,
+                 ledger_sync_order: Optional[List] = None):
         # If ledger_sync_order is not provided (is None), it is assumed that
         # `postCatchupCompleteClbk` of the LedgerInfo will be used
         self.owner = owner
@@ -52,7 +49,7 @@ def __init__(self,
 
         # Holds ledgers of different types with
         # their info like callbacks, state, etc
-        self.ledgerRegistry = {}   # type: Dict[int, LedgerInfo]
+        self.ledgerRegistry = {}  # type: Dict[int, LedgerInfo]
 
         # Largest 3 phase key received during catchup.
         # This field is needed to discard any stashed 3PC messages or
@@ -67,11 +64,11 @@ def service(self):
         return self._serviceActions()
 
     def addLedger(self, iD: int, ledger: Ledger,
-                  preCatchupStartClbk: Callable=None,
-                  postCatchupStartClbk: Callable=None,
-                  preCatchupCompleteClbk: Callable=None,
-                  postCatchupCompleteClbk: Callable=None,
-                  postTxnAddedToLedgerClbk: Callable=None):
+                  preCatchupStartClbk: Callable = None,
+                  postCatchupStartClbk: Callable = None,
+                  preCatchupCompleteClbk: Callable = None,
+                  postCatchupCompleteClbk: Callable = None,
+                  postTxnAddedToLedgerClbk: Callable = None):
 
         if iD in self.ledgerRegistry:
             logger.error("{} already present in ledgers "
@@ -257,7 +254,7 @@ def processLedgerStatus(self, status: LedgerStatus, frm: str):
         ledgerStatus = LedgerStatus(*status)
         if ledgerStatus.txnSeqNo < 0:
             self.discard(status, reason="Received negative sequence number "
-                         "from {}".format(frm),
+                                        "from {}".format(frm),
                          logMethod=logger.warning)
             return
         ledgerId = getattr(status, f.LEDGER_ID.nm)
@@ -309,10 +306,8 @@ def processLedgerStatus(self, status: LedgerStatus, frm: str):
         ledgerInfo.recvdConsistencyProofs[frm] = None
         ledgerInfo.ledgerStatusOk.add(frm)
 
-        if self.isLedgerSame(ledgerStatus) \
-                and ledgerStatus.viewNo is not None \
-                and ledgerStatus.ppSeqNo is not None:
-            ledgerInfo.last_txn_3PC_key = \
+        if self.isLedgerSame(ledgerStatus):
+            ledgerInfo.last_txn_3PC_key[frm] = \
                 (ledgerStatus.viewNo, ledgerStatus.ppSeqNo)
 
         if self.has_ledger_status_quorum(
@@ -329,11 +324,30 @@ def processLedgerStatus(self, status: LedgerStatus, frm: str):
                 self.do_pre_catchup(ledgerId)
                 # Any state cleanup that is part of pre-catchup should be
                 # done
-                self.catchupCompleted(ledgerId, ledgerInfo.last_txn_3PC_key)
+                last_3PC_key = self._get_last_txn_3PC_key(ledgerInfo)
+                self.catchupCompleted(ledgerId, last_3PC_key)
             else:
                 # Ledger was already synced
                 self.mark_ledger_synced(ledgerId)
 
+    def _get_last_txn_3PC_key(self, ledgerInfo):
+        quorum = Quorums(self.owner.totalNodes)
+        quorumed_3PC_keys = \
+            [
+                most_common_element
+                for most_common_element, freq in
+                Counter(ledgerInfo.last_txn_3PC_key.values()).most_common()
+                if quorum.ledger_status_last_3PC.is_reached(freq) and
+                most_common_element[0] is not None and
+                most_common_element[1] is not None
+            ]
+
+        if len(quorumed_3PC_keys) == 0:
+            return None
+
+        min_quorumed_3PC_key = min_3PC_key(quorumed_3PC_keys)
+        return min_quorumed_3PC_key
+
     @staticmethod
     def has_ledger_status_quorum(leger_status_num, total_nodes):
         quorum = Quorums(total_nodes).ledger_status
@@ -577,7 +591,7 @@ def _transform(self, txn):
         # Certain transactions might need to be
         # transformed to certain format before applying to the ledger
         txn = reqToTxn(txn)
-        z = txn if not self.ownedByNode else  \
+        z = txn if not self.ownedByNode else \
             self.owner.transform_txn_for_ledger(txn)
         return z
 
@@ -636,7 +650,7 @@ def _getCatchupReplyForSeqNo(self, ledgerId, seqNo):
         for k, catchupReps in ledgerInfo.recvdCatchupRepliesFrm.items():
             for rep in catchupReps:
                 txns = getattr(rep, f.TXNS.nm)
-                # Transfers of odcits in RAET converts integer keys to string
+
                 if str(seqNo) in txns:
                     return k, rep
 
@@ -726,7 +740,7 @@ def canStartCatchUpProcess(self, ledgerId: int):
 
         logger.debug(
             "{} cannot start catchup since received only {} "
-            "consistency proofs but need at least {}". format(
+            "consistency proofs but need at least {}".format(
                 self,
                 len(recvdConsProof),
                 adjustedQuorum.consistency_proof.value))
@@ -852,7 +866,7 @@ def startCatchUpProcess(self, ledgerId: int, proof: ConsistencyProof):
     def _getCatchupTimeout(self, numRequest, batchSize):
         return numRequest * self.config.CatchupTransactionsTimeout
 
-    def catchupCompleted(self, ledgerId: int, last_3PC: Optional[Tuple]=None):
+    def catchupCompleted(self, ledgerId: int, last_3PC: Optional[Tuple] = None):
         if ledgerId not in self.ledgerRegistry:
             logger.error("{}{} called catchup completed for ledger {}".
                          format(CATCH_UP_PREFIX, self, ledgerId))
@@ -975,7 +989,7 @@ def _get_merged_catchup_txns(existing_txns, new_txns):
                                 key=operator.itemgetter(0)))
 
     def getConsistencyProof(self, status: LedgerStatus):
-        ledger = self.getLedgerForMsg(status)    # type: Ledger
+        ledger = self.getLedgerForMsg(status)  # type: Ledger
         ledgerId = getattr(status, f.LEDGER_ID.nm)
         seqNoStart = getattr(status, f.TXN_SEQ_NO.nm)
         seqNoEnd = ledger.size

plenum/common/messages/fields.py

@@ -102,6 +102,13 @@ def _specific_validation(self, val):
         return
 
 
+class IntegerField(FieldBase):
+    _base_types = (int,)
+
+    def _specific_validation(self, val):
+        return
+
+
 class NonEmptyStringField(FieldBase):
     _base_types = (str,)
 

plenum/common/script_helper.py

@@ -14,7 +14,6 @@
 from stp_core.loop.eventually import eventually
 from stp_core.network.port_dispenser import genHa
 from stp_core.types import HA
-from stp_raet.util import getLocalVerKey, getLocalPubKey
 
 NodeInfoFile = "node-info"
 GenTxnFile = "genesis_txn"
@@ -107,10 +106,6 @@ def storeGenTxns(baseDir, txn):
                 isLineNoKey=True)
 
 
-def getStewardKeyFromName(baseDir, name):
-    return getLocalVerKey(name, baseDir)
-
-
 def getAddGenesisHAs(nodeip, nodeport, clientip, clientport):
     vnodeip = nodeip if nodeip else "127.0.0.1"
     vnodeport = nodeport if nodeport else "9701"
@@ -182,41 +177,6 @@ def printGenTxn(txn, displayTxn):
         print('\n' + txn)
 
 
-def getVerKeyFromName(baseDir, roleName):
-    return getLocalVerKey(roleName, baseDir)
-
-
-def getPubKeyFromName(baseDir, roleName):
-    return getLocalPubKey(roleName, baseDir)
-
-
-def exportNodeGenTxn(baseDir, displayTxn, name):
-    nodeInfo = getNodeInfo(baseDir, name)
-    nodeVerKey = getVerKeyFromName(baseDir, name)
-    stewardKey = nodeInfo.get('steward')
-    nodeAddr = nodeInfo.get('nodeAddr')
-    clientAddr = nodeInfo.get('clientAddr')
-
-    txn = 'add genesis transaction {node} with data {"'.format(node=PlenumTransactions.NODE.name) + name + '": {' \
-        '"verkey":' \
-        ' "' + \
-          nodeVerKey + \
-          '", "node_address": "' + nodeAddr + \
-          '", "client_address": "' + clientAddr + '"}, "by":"' + stewardKey + \
-          '"}'
-    storeExportedTxns(baseDir, txn)
-    printGenTxn(txn, displayTxn)
-
-
-def exportStewardGenTxn(baseDir, displayTxn, name):
-    verkey = getLocalVerKey(name, baseDir)
-    txn = 'add genesis transaction {nym} with data  {"'.format(nym=PlenumTransactions.NYM.name) + name + '": {' \
-        '"verkey": "' + verkey + '"} role={role}'.format(
-        role=Roles.STEWARD.name)
-    storeExportedTxns(baseDir, txn)
-    printGenTxn(txn, displayTxn)
-
-
 def submitNodeIpChange(client, stewardWallet, name: str, nym: str,
                        nodeStackHa: HA, clientStackHa: HA):
     (nodeIp, nodePort), (clientIp, clientPort) = nodeStackHa, clientStackHa