Master

Jenkinsfile.ci

@@ -89,7 +89,7 @@ def test(options=[:]) {
 
     try {
         if (options.useRunner) {
-            sh "PYTHONASYNCIODEBUG='0' $options.python runner.py --pytest \"$options.python -m pytest\" --dir $options.testDir --output \"$options.resFile\" --test-only-slice \"$options.testOnlySlice\""
+            sh "RUSTPYTHONASYNCIODEBUG='0' $options.python runner.py --pytest \"$options.python -m pytest\" --dir $options.testDir --output \"$options.resFile\" --test-only-slice \"$options.testOnlySlice\""
         } else {
             sh "$options.python -m pytest --junitxml=$options.resFile $options.testDir"
         }

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright {yyyy} {name of copyright owner}
+   Copyright 2016 Sovrin Foundation
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

ci/code-validation.dockerfile

@@ -11,8 +11,7 @@ RUN apt-get update -y && apt-get install -y \
 	python3-pip \
 	python-setuptools \
 	python3-nacl
-RUN pip3 install -U \ 
-	pip \ 
+RUN pip3 install -U \
 	setuptools \
 	pep8 \
 	pep8-naming \

ci/ubuntu.dockerfile

@@ -9,8 +9,8 @@ RUN echo "To invalidate cache"
 
 RUN apt-get update -y && apt-get install -y \
     python3-nacl \
-    libindy-crypto=0.2.0 \
-    libindy=1.3.1~403 \
+    libindy-crypto=0.4.0 \
+    libindy=1.3.1~469 \
 # rocksdb python wrapper
     libbz2-dev \
     zlib1g-dev \

plenum/cli/cli.py

@@ -927,14 +927,10 @@ def newNode(self, nodeName: str):
         nodes = []
         for name in names:
             try:
-                nodeRegistry = None if self.nodeRegLoadedFromFile \
-                    else self.nodeRegistry
-
                 config_helper = PNodeConfigHelper(name, self.config, chroot=self.nodes_chroot)
                 learnKeysFromOthers(config_helper.keys_dir, name,
                                     self.nodes.values())
                 node = self.NodeClass(name,
-                                      nodeRegistry=nodeRegistry,
                                       config_helper=config_helper,
                                       pluginPaths=self.pluginPaths,
                                       config=self.config)

plenum/common/ledger.py

@@ -81,10 +81,10 @@ def discardTxns(self, count: int):
             self.uncommittedTree = self.treeWithAppliedTxns(
                 self.uncommittedTxns)
             self.uncommittedRootHash = self.uncommittedTree.root_hash
-        logger.debug('Discarding {} txns and root hash {} and new root hash '
-                     'is {}. {} are still uncommitted'.
-                     format(count, old_hash, self.uncommittedRootHash,
-                            len(self.uncommittedTxns)))
+        logger.info('Discarding {} txns and root hash {} and new root hash '
+                    'is {}. {} are still uncommitted'.
+                    format(count, old_hash, self.uncommittedRootHash,
+                           len(self.uncommittedTxns)))
 
     def treeWithAppliedTxns(self, txns: List, currentTree=None):
         """

plenum/common/ledger_manager.py

@@ -100,7 +100,8 @@ def request_CPs_if_needed(self, ledgerId):
 
         quorum = Quorums(self.owner.totalNodes)
         groupedProofs, null_proofs_count = self._groupConsistencyProofs(proofs)
-        if quorum.same_consistency_proof.is_reached(null_proofs_count):
+        if quorum.same_consistency_proof.is_reached(null_proofs_count)\
+                or len(groupedProofs) == 0:
             return
         result = self._latestReliableProof(groupedProofs, ledgerInfo.ledger)
         if not result:
@@ -723,6 +724,7 @@ def canStartCatchUpProcess(self, ledgerId: int):
             # from other nodes, see `request_CPs_if_needed`
 
             ledgerInfo.consistencyProofsTimer = time.perf_counter()
+            # TODO: find appropriate moment to unschedule this event!
             self._schedule(partial(self.request_CPs_if_needed, ledgerId),
                            self.config.ConsistencyProofsTimeout * (
                                self.owner.totalNodes - 1))
@@ -1017,7 +1019,7 @@ def _buildConsistencyProof(self, ledgerId, seqNoStart, seqNoEnd):
                          .format(self, seqNoEnd, ledgerSize))
             return
         if seqNoEnd < seqNoStart:
-            self.error(
+            logger.error(
                 '{} cannot build consistency proof since end {} is '
                 'lesser than start {}'.format(
                     self, seqNoEnd, seqNoStart))
@@ -1090,8 +1092,8 @@ def appendToLedger(self, ledgerId: int, txn: Any) -> Dict:
         return ledgerInfo.ledger.append(txn)
 
     def stashLedgerStatus(self, ledgerId: int, status, frm: str):
-        logger.debug("{} stashing ledger status {} from {}".
-                     format(self, status, frm))
+        logger.info("{} stashing ledger status {} from {}".
+                    format(self, status, frm))
         ledgerInfo = self.getLedgerInfoByType(ledgerId)
         ledgerInfo.stashedLedgerStatuses.append((status, frm))
 

plenum/common/messages/fields.py

@@ -194,14 +194,28 @@ def _specific_validation(self, val):
 class IterableField(FieldBase):
     _base_types = (list, tuple)
 
-    def __init__(self, inner_field_type: FieldValidator, **kwargs):
+    def __init__(self, inner_field_type: FieldValidator, min_length=None,
+                 max_length=None, **kwargs):
         assert inner_field_type
         assert isinstance(inner_field_type, FieldValidator)
+        for m in (min_length, max_length):
+            if m is not None:
+                assert isinstance(m, int)
+                assert m > 0
 
         self.inner_field_type = inner_field_type
+        self.min_length = min_length
+        self.max_length = max_length
         super().__init__(**kwargs)
 
     def _specific_validation(self, val):
+        if self.min_length is not None:
+            if len(val) < self.min_length:
+                return 'length should be at least {}'.format(self.min_length)
+        if self.max_length is not None:
+            if len(val) > self.max_length:
+                return 'length should be at most {}'.format(self.max_length)
+
         for v in val:
             check_er = self.inner_field_type.validate(v)
             if check_er:

plenum/common/messages/node_messages.py

@@ -1,8 +1,11 @@
 from typing import TypeVar, NamedTuple
 
-from plenum.common.constants import NOMINATE, BATCH, REELECTION, PRIMARY, BLACKLIST, REQACK, REQNACK, REJECT, \
-    POOL_LEDGER_TXNS, ORDERED, PROPAGATE, PREPREPARE, PREPARE, COMMIT, CHECKPOINT, THREE_PC_STATE, CHECKPOINT_STATE, \
-    REPLY, INSTANCE_CHANGE, LEDGER_STATUS, CONSISTENCY_PROOF, CATCHUP_REQ, CATCHUP_REP, VIEW_CHANGE_DONE, CURRENT_STATE, \
+from plenum.common.constants import NOMINATE, BATCH, REELECTION, PRIMARY, \
+    BLACKLIST, REQACK, REQNACK, REJECT, \
+    POOL_LEDGER_TXNS, ORDERED, PROPAGATE, PREPREPARE, PREPARE, COMMIT, \
+    CHECKPOINT, THREE_PC_STATE, CHECKPOINT_STATE, \
+    REPLY, INSTANCE_CHANGE, LEDGER_STATUS, CONSISTENCY_PROOF, CATCHUP_REQ, \
+    CATCHUP_REP, VIEW_CHANGE_DONE, CURRENT_STATE, \
     MESSAGE_REQUEST, MESSAGE_RESPONSE, OBSERVED_DATA, BATCH_COMMITTED
 from plenum.common.messages.client_request import ClientMessageValidator
 from plenum.common.messages.fields import NonNegativeNumberField, IterableField, \

plenum/common/types.py

@@ -45,6 +45,7 @@ class f:  # provides a namespace for reusable field constants
     ELECTION_DATA = Field('electionData', Any)
     TXN_ID = Field('txnId', str)
     REASON = Field('reason', Any)
+    IS_SUCCESS = Field('isSuccess', Any)
     SENDER_CLIENT = Field('senderClient', str)
     PP_TIME = Field("ppTime", float)
     REQ_IDR = Field("reqIdr", List[Tuple[str, int]])

plenum/config.py

@@ -86,6 +86,7 @@
 
 # Monitoring configuration
 PerfCheckFreq = 10
+UnorderedCheckFreq = 60
 
 # Temporarily reducing DELTA till the calculations for extra work are not
 # incorporated
@@ -100,17 +101,19 @@
 LatencyGraphDuration = 240
 notifierEventTriggeringConfig = {
     'clusterThroughputSpike': {
-        'coefficient': 3,
-        'minCnt': 100,
+        'borders_coeff': 10,
+        'min_cnt': 15,
         'freq': 60,
-        'minActivityThreshold': 2,
+        'min_activity_threshold': 10,
+        'use_weighted_borders_coeff': True,
         'enabled': True
     },
     'nodeRequestSpike': {
-        'coefficient': 3,
-        'minCnt': 100,
+        'borders_coeff': 10,
+        'min_cnt': 15,
         'freq': 60,
-        'minActivityThreshold': 2,
+        'min_activity_threshold': 10,
+        'use_weighted_borders_coeff': True,
         'enabled': True
     }
 }
@@ -149,10 +152,11 @@
 
 
 # Log configuration
-logRotationWhen = 'D'
+logRotationWhen = 'W6'
 logRotationInterval = 1
-logRotationBackupCount = 10
+logRotationBackupCount = 50
 logRotationMaxBytes = 100 * 1024 * 1024
+logRotationCompress = True
 logFormat = '{asctime:s} | {levelname:8s} | {filename:20s} ({lineno: >4}) | {funcName:s} | {message:s}'
 logFormatStyle = '{'
 logLevel = logging.NOTSET

plenum/server/action_req_handler.py

@@ -0,0 +1,16 @@
+from plenum.common.request import Request
+from plenum.server.req_handler import RequestHandler
+
+
+class ActionReqHandler(RequestHandler):
+    def __init__(self):
+        super().__init__()
+
+    def doStaticValidation(self, request: Request):
+        pass
+
+    def validate(self, req: Request):
+        pass
+
+    def apply(self, req: Request, cons_time: int):
+        pass

plenum/server/client_authn.py

@@ -14,6 +14,7 @@
     InsufficientSignatures, InsufficientCorrectSignatures
 from plenum.common.types import f
 from plenum.common.verifier import DidVerifier, Verifier
+from plenum.server.action_req_handler import ActionReqHandler
 from plenum.server.domain_req_handler import DomainRequestHandler
 from plenum.server.pool_req_handler import PoolRequestHandler
 from stp_core.common.log import getlogger
@@ -179,13 +180,18 @@ class CoreAuthMixin:
     ).union(
         DomainRequestHandler.query_types
     )
+    action_types = ActionReqHandler.operation_types
 
     def is_query(self, typ):
         return typ in self.query_types
 
     def is_write(self, typ):
         return typ in self.write_types
 
+    @classmethod
+    def is_action(cls, typ):
+        return typ in cls.action_types
+
     @staticmethod
     def _extract_signature(msg):
         if f.SIG.nm not in msg:

plenum/server/config_req_handler.py

@@ -1,6 +1,19 @@
-from plenum.server.req_handler import RequestHandler
+from plenum.common.request import Request
+from plenum.server.ledger_req_handler import LedgerRequestHandler
 
 
-class ConfigReqHandler(RequestHandler):
+class ConfigReqHandler(LedgerRequestHandler):
     def __init__(self, ledger, state):
         super().__init__(ledger, state)
+
+    def doStaticValidation(self, request: Request):
+        pass
+
+    def get_query_response(self, request):
+        pass
+
+    def validate(self, req: Request):
+        pass
+
+    def apply(self, req: Request, cons_time: int):
+        pass

plenum/server/domain_req_handler.py

@@ -11,13 +11,14 @@
 from plenum.common.txn_util import reqToTxn
 from plenum.common.types import f
 from plenum.persistence.util import txnsWithSeqNo
+from plenum.server.ledger_req_handler import LedgerRequestHandler
 from plenum.server.req_handler import RequestHandler
 from stp_core.common.log import getlogger
 
 logger = getlogger()
 
 
-class DomainRequestHandler(RequestHandler):
+class DomainRequestHandler(LedgerRequestHandler):
     stateSerializer = domain_state_serializer
     write_types = {NYM, }
 

plenum/server/ledger_req_handler.py

@@ -0,0 +1,78 @@
+from abc import ABCMeta, abstractmethod
+from typing import List
+
+import base58
+
+from plenum.common.ledger import Ledger
+from plenum.common.request import Request
+from plenum.persistence.util import txnsWithSeqNo
+from plenum.server.req_handler import RequestHandler
+from stp_core.common.log import getlogger
+
+from state.state import State
+
+logger = getlogger()
+
+
+class LedgerRequestHandler(RequestHandler, metaclass=ABCMeta):
+    """
+    Base class for request handlers
+    Declares methods for validation, application of requests and
+    state control
+    """
+
+    query_types = set()
+    write_types = set()
+
+    def __init__(self, ledger: Ledger, state: State):
+        self.state = state
+        self.ledger = ledger
+
+    def updateState(self, txns, isCommitted=False):
+        """
+        Updates current state with a number of committed or
+        not committed transactions
+        """
+
+    def commit(self, txnCount, stateRoot, txnRoot, ppTime) -> List:
+        """
+        :param txnCount: The number of requests to commit (The actual requests
+        are picked up from the uncommitted list from the ledger)
+        :param stateRoot: The state trie root after the txns are committed
+        :param txnRoot: The txn merkle root after the txns are committed
+
+        :return: list of committed transactions
+        """
+
+        (seqNoStart, seqNoEnd), committedTxns = \
+            self.ledger.commitTxns(txnCount)
+        stateRoot = base58.b58decode(stateRoot.encode())
+        # Probably the following assertion fail should trigger catchup
+        assert self.ledger.root_hash == txnRoot, '{} {}'.format(
+            self.ledger.root_hash, txnRoot)
+        self.state.commit(rootHash=stateRoot)
+        return txnsWithSeqNo(seqNoStart, seqNoEnd, committedTxns)
+
+    def onBatchCreated(self, state_root):
+        pass
+
+    def onBatchRejected(self):
+        pass
+
+    @abstractmethod
+    def doStaticValidation(self, request: Request):
+        pass
+
+    def is_query(self, txn_type):
+        return txn_type in self.query_types
+
+    def get_query_response(self, request):
+        raise NotImplementedError
+
+    @staticmethod
+    def transform_txn_for_ledger(txn):
+        return txn
+
+    @property
+    def operation_types(self) -> set:
+        return self.write_types.union(self.query_types)