INDY-877: Protocol version support (#407)

* add protocolVersion parameter to Request

Signed-off-by: ashcherbakov <[email protected]>

* set current protocol version for all client Requests

Signed-off-by: ashcherbakov <[email protected]>

* remove deprecated method

Signed-off-by: ashcherbakov <[email protected]>

* static code validation

Signed-off-by: ashcherbakov <[email protected]>

* fix tests

Signed-off-by: ashcherbakov <[email protected]>

Author: ashcherbakov

common/serializers/serialization.py

@@ -17,6 +17,7 @@
 proof_nodes_serializer = Base64Serializer()
 
 
+# TODO: separate data, metadata and signature, so that we don't need to have topLevelKeysToIgnore
 def serialize_msg_for_signing(msg: Mapping, topLevelKeysToIgnore=None):
     """
     Serialize a message for signing.

plenum/common/constants.py

@@ -1,6 +1,7 @@
 # inter-node communication
 from enum import IntEnum, unique
 
+from plenum.common.plenum_protocol_version import PlenumProtocolVersion
 from plenum.common.roles import Roles
 from plenum.common.transactions import PlenumTransactions
 
@@ -154,3 +155,5 @@ class LedgerState(IntEnum):
 PLUGIN_BASE_DIR_PATH = "PluginBaseDirPath"
 POOL_LEDGER_ID = 0
 DOMAIN_LEDGER_ID = 1
+
+CURRENT_PROTOCOL_VERSION = PlenumProtocolVersion.STATE_PROOF_SUPPORT.value

plenum/common/messages/client_request.py

@@ -2,7 +2,7 @@
     TARGET_NYM, VERKEY, ROLE, NODE, NYM, GET_TXN, VALIDATOR, BLS_KEY
 from plenum.common.messages.fields import NetworkIpAddressField, NetworkPortField, IterableField, \
     ChooseField, ConstantField, DestNodeField, VerkeyField, DestNymField, RoleField, TxnSeqNoField, IdentifierField, \
-    NonNegativeNumberField, SignatureField, LimitedLengthStringField
+    NonNegativeNumberField, SignatureField, LimitedLengthStringField, ProtocolVersionField
 from plenum.common.messages.message_base import MessageValidator
 from plenum.common.types import OPERATION, f
 from plenum.config import ALIAS_FIELD_LIMIT, DIGEST_FIELD_LIMIT, SIGNATURE_FIELD_LIMIT, BLS_KEY_LIMIT
@@ -108,4 +108,5 @@ def __init__(self, operation_schema_is_strict, *args, **kwargs):
         (OPERATION, ClientOperationField()),
         (f.SIG.nm, SignatureField(max_length=SIGNATURE_FIELD_LIMIT, optional=True)),
         (f.DIGEST.nm, LimitedLengthStringField(max_length=DIGEST_FIELD_LIMIT, optional=True)),
+        (f.PROTOCOL_VERSION.nm, ProtocolVersionField(optional=True)),
     )

plenum/common/messages/fields.py

@@ -5,6 +5,7 @@
 
 import base58
 from plenum.common.constants import DOMAIN_LEDGER_ID, POOL_LEDGER_ID
+from plenum.common.plenum_protocol_version import PlenumProtocolVersion
 from plenum.config import BLS_MULTI_SIG_LIMIT
 
 
@@ -124,12 +125,13 @@ def _specific_validation(self, val):
 class SignatureField(LimitedLengthStringField):
     _base_types = (str, type(None))
 
-    # TODO do nothing because EmptySignature should be raised somehow
-
     def _specific_validation(self, val):
-        if val and len(val) > 0:
-            return super()._specific_validation(val)
-        return
+        if val is None:
+            # TODO do nothing because EmptySignature should be raised somehow
+            return
+        if len(val) == 0:
+            return "signature can not be empty"
+        return super()._specific_validation(val)
 
 
 class RoleField(FieldBase):
@@ -523,3 +525,13 @@ def _specific_validation(self, val):
             return err
         if len(participants) == 0:
             return "multi-signature participants list is empty"
+
+
+class ProtocolVersionField(FieldBase):
+    _base_types = (int, type(None))
+
+    def _specific_validation(self, val):
+        if val is None:
+            return
+        if not PlenumProtocolVersion.has_value(val):
+            return 'Unknown protocol version value {}'.format(val)

plenum/common/messages/message_base.py

@@ -12,6 +12,7 @@ class MessageValidator(FieldValidator):
 
     schema = ()
     optional = False
+    schema_is_strict = True
 
     def __init__(self, schema_is_strict=True):
         self.schema_is_strict = schema_is_strict

plenum/common/plenum_protocol_version.py

@@ -0,0 +1,18 @@
+from enum import Enum, unique
+
+
+@unique
+class PlenumProtocolVersion(Enum):
+    #  These numeric constants CANNOT be changed once they have been used
+    STATE_PROOF_SUPPORT = 1
+
+    def __str__(self):
+        return self.name
+
+    @staticmethod
+    def has_value(value):
+        try:
+            PlenumProtocolVersion(value)
+            return True
+        except ValueError:
+            return False

plenum/common/request.py

@@ -2,32 +2,41 @@
 from typing import Mapping, NamedTuple
 
 from common.serializers.serialization import serialize_msg_for_signing
-from plenum.common.constants import REQDIGEST, REQKEY, FORCE
+from plenum.common.constants import REQDIGEST, REQKEY, FORCE, CURRENT_PROTOCOL_VERSION
 from plenum.common.messages.client_request import ClientMessageValidator
 from plenum.common.types import f, OPERATION
 from stp_core.types import Identifier
 
 
 class Request:
     def __init__(self,
-                 identifier: Identifier=None,
-                 reqId: int=None,
-                 operation: Mapping=None,
-                 signature: str=None):
+                 identifier: Identifier = None,
+                 reqId: int = None,
+                 operation: Mapping = None,
+                 signature: str = None,
+                 protocolVersion: int = CURRENT_PROTOCOL_VERSION):
         self.identifier = identifier
         self.reqId = reqId
         self.operation = operation
-        self.digest = self.getDigest()
+        self.protocolVersion = protocolVersion
         self.signature = signature
+        # TODO: getDigest must be called after all initialization above! refactor it
+        self.digest = self.getDigest()
 
     @property
     def as_dict(self):
-        return {
+        # TODO: as of now, the keys below must be equal to the class fields name (see SafeRequest)
+        dct = {
             f.IDENTIFIER.nm: self.identifier,
             f.REQ_ID.nm: self.reqId,
-            OPERATION: self.operation,
-            f.SIG.nm: self.signature
+            OPERATION: self.operation
         }
+        if self.signature is not None:
+            dct[f.SIG.nm] = self.signature
+        if self.protocolVersion is not None:
+            dct[f.PROTOCOL_VERSION.nm] = self.protocolVersion
+
+        return dct
 
     def __eq__(self, other):
         return self.as_dict == other.as_dict
@@ -51,11 +60,15 @@ def __getstate__(self):
 
     @property
     def signingState(self):
-        return {
+        # TODO: separate data, metadata and signature, so that we don't need to have this kind of messages
+        dct = {
             f.IDENTIFIER.nm: self.identifier,
             f.REQ_ID.nm: self.reqId,
             OPERATION: self.operation
         }
+        if self.protocolVersion is not None:
+            dct[f.PROTOCOL_VERSION.nm] = self.protocolVersion
+        return dct
 
     def __setstate__(self, state):
         self.__dict__.update(state)
@@ -91,7 +104,6 @@ class ReqKey(NamedTuple(REQKEY, [f.IDENTIFIER, f.REQ_ID])):
 
 
 class SafeRequest(Request, ClientMessageValidator):
-
     def __init__(self, **kwargs):
         self.validate(kwargs)
         super().__init__(**kwargs)

plenum/common/types.py

@@ -37,6 +37,7 @@ class f:  # provides a namespace for reusable field constants
     IS_STABLE = Field('isStable', bool)
     MSGS = Field('messages', List[Mapping])
     SIG = Field('signature', Optional[str])
+    PROTOCOL_VERSION = Field('protocolVersion', int)
     SUSP_CODE = Field('suspicionCode', int)
     ELECTION_DATA = Field('electionData', Any)
     TXN_ID = Field('txnId', str)

plenum/req_handler/__init__.py

@@ -2,7 +2,7 @@
 from plenum.common.keygen_utils import initRemoteKeys
 
 from stp_core.loop.eventually import eventually
-from plenum.common.exceptions import EmptySignature
+from plenum.common.exceptions import MissingSignature
 from plenum.common.exceptions import NotConnectedToAny
 from stp_core.common.log import getlogger
 from plenum.common.constants import OP_FIELD_NAME, REPLY, REQACK
@@ -107,15 +107,15 @@ async def go(ctx):
             params = n.spylog.getLastParams(Node.handleInvalidClientMsg)
             ex = params['ex']
             msg, _ = params['wrappedMsg']
-            assert isinstance(ex, EmptySignature)
+            assert isinstance(ex, MissingSignature)
             assert msg.get(f.IDENTIFIER.nm) == request.identifier
 
             params = n.spylog.getLastParams(Node.discard)
             reason = params["reason"]
             (msg, frm) = params["msg"]
             assert msg == request.as_dict
             assert msg.get(f.IDENTIFIER.nm) == request.identifier
-            assert "EmptySignature" in reason
+            assert "MissingSignature" in reason
 
     pool.run(go)
 

plenum/test/client/test_client.py

@@ -0,0 +1,48 @@
+from plenum.common.constants import CURRENT_PROTOCOL_VERSION
+from plenum.test.helper import sendRandomRequests, waitForSufficientRepliesForRequests, signed_random_requests, \
+    send_signed_requests, checkReqNackWithReason
+from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \
+    client1, wallet1, client1Connected
+from stp_core.loop.eventually import eventually
+
+
+def test_request_no_protocol_version(tconf, looper, txnPoolNodeSet,
+                                     client1, client1Connected,
+                                     wallet1):
+    reqs = signed_random_requests(wallet1, 2)
+    for req in reqs:
+        req.protocolVersion = None
+    send_signed_requests(client1, reqs)
+    waitForSufficientRepliesForRequests(looper, client1, requests=reqs)
+
+
+def test_version_set_by_default(tconf, looper, txnPoolNodeSet,
+                                client1, client1Connected,
+                                wallet1):
+    reqs = signed_random_requests(wallet1, 1)
+    assert reqs[0].protocolVersion
+    send_signed_requests(client1, reqs)
+    waitForSufficientRepliesForRequests(looper, client1, requests=reqs)
+
+
+def test_request_with_correct_version(tconf, looper,
+                                      txnPoolNodeSet, client1, client1Connected,
+                                      wallet1):
+    reqs = signed_random_requests(wallet1, 2)
+    for req in reqs:
+        req.protocolVersion = CURRENT_PROTOCOL_VERSION
+    send_signed_requests(client1, reqs)
+    waitForSufficientRepliesForRequests(looper, client1, requests=reqs)
+
+
+def test_request_with_invalid_version(tconf, looper, txnPoolNodeSet,
+                                      client1, client1Connected,
+                                      wallet1):
+    reqs = signed_random_requests(wallet1, 2)
+    for req in reqs:
+        req.protocolVersion = -1
+    send_signed_requests(client1, reqs)
+    for node in txnPoolNodeSet:
+        looper.run(eventually(checkReqNackWithReason, client1,
+                              'Unknown protocol version value -1',
+                              node.clientstack.name, retryWait=1))

plenum/test/client/test_protocol_version.py

@@ -73,39 +73,6 @@ def check_sufficient_replies_received(client: Client,
                          .format(full_request_id))
 
 
-def checkSufficientRepliesReceived(receivedMsgs: Iterable,
-                                   reqId: int,
-                                   fValue: int):
-    """
-    Checks number of replies for request with specified id in given inbox and
-    if this number is lower than number of malicious nodes (fValue) -
-    raises exception
-
-    If you do not need response ponder on using
-    waitForSufficientRepliesForRequests instead
-
-    :returns: response for request
-    """
-
-    receivedReplies = getRepliesFromClientInbox(inbox=receivedMsgs,
-                                                reqId=reqId)
-    logger.debug("received replies for reqId {}: {}".
-                 format(reqId, receivedReplies))
-    assert len(receivedReplies) > fValue, "Received {} replies but expected " \
-                                          "at-least {} for reqId {}". \
-        format(len(receivedReplies), fValue + 1, reqId)
-    result = checkIfMoreThanFSameItems([reply[f.RESULT.nm] for reply in
-                                        receivedReplies], fValue)
-    assert result, "reqId {}: found less than in {} same replies".format(
-        reqId, fValue)
-
-    assert all([r[f.RESULT.nm][f.REQ_ID.nm] == reqId for r in receivedReplies]
-               ), "not all replies have got reqId {}".format(reqId)
-
-    return result
-    # TODO add test case for what happens when replies don't have the same data
-
-
 def waitForSufficientRepliesForRequests(looper,
                                         client,
                                         *,  # To force usage of names

plenum/test/helper.py

@@ -0,0 +1,19 @@
+from plenum.common.messages.fields import ProtocolVersionField
+from plenum.common.plenum_protocol_version import PlenumProtocolVersion
+
+validator = ProtocolVersionField()
+
+
+def test_valid():
+    assert not validator.validate(1)
+    assert not validator.validate(PlenumProtocolVersion.STATE_PROOF_SUPPORT.value)
+    assert not validator.validate(None)  # version can be None (for backward compatibility)
+
+
+def test_invalid():
+    assert validator.validate(2)
+    assert validator.validate("1")
+    assert validator.validate("")
+    assert validator.validate(0)
+    assert validator.validate(1.0)
+    assert validator.validate(0.1)

plenum/test/input_validation/fields_validation/test_protocol_version_field.py

@@ -0,0 +1,25 @@
+from collections import OrderedDict
+
+from plenum.common.messages.client_request import ClientMessageValidator, ClientOperationField
+from plenum.common.messages.fields import LimitedLengthStringField, SignatureField, NonNegativeNumberField, \
+    IdentifierField, ProtocolVersionField
+
+EXPECTED_ORDERED_FIELDS = OrderedDict([
+    ("identifier", IdentifierField),
+    ("reqId", NonNegativeNumberField),
+    ("operation", ClientOperationField),
+    ("signature", SignatureField),
+    ("digest", LimitedLengthStringField),
+    ("protocolVersion", ProtocolVersionField),
+])
+
+
+def test_has_expected_fields():
+    actual_field_names = OrderedDict(ClientMessageValidator.schema).keys()
+    assert list(actual_field_names) == list(EXPECTED_ORDERED_FIELDS.keys())
+
+
+def test_has_expected_validators():
+    schema = dict(ClientMessageValidator.schema)
+    for field, validator in EXPECTED_ORDERED_FIELDS.items():
+        assert isinstance(schema[field], validator)