Merge pull request #81 from hyperledger/master

Master

Author: spivachuk

common/error.py

@@ -0,0 +1,9 @@
+def error(msg: str, exc_type: Exception=Exception) -> Exception:
+    """
+    Wrapper to get around Python's distinction between statements and expressions
+    Can be used in lambdas and expressions such as: a if b else error(c)
+
+    :param msg: error message
+    :param exc_type: type of exception to raise
+    """
+    raise exc_type(msg)

common/serializers/signing_serializer.py

@@ -22,7 +22,7 @@
 """
 from collections import Iterable
 
-from plenum.common.error import error
+from common.error import error
 from stp_core.common.log import getlogger
 
 logger = getlogger()
@@ -32,7 +32,8 @@
 
 
 class SigningSerializer:
-    def serialize(self, obj, level=0, objname=None, topLevelKeysToIgnore=None, toBytes=True):
+    def serialize(self, obj, level=0, objname=None, topLevelKeysToIgnore=None,
+                  toBytes=True):
         """
         Create a string representation of the given object.
 

docs/main.md

@@ -1,2 +1,3 @@
 Outline of the system
-Node 
+# TODO
+

docs/plugins.md

@@ -0,0 +1,59 @@
+# Plugins
+
+Plugins allow for supporting new transactions without modifying/adding to the core parts of the system. 
+Supporting any new transaction requires:
+-   Defining new transaction types
+-   Defining new message validation rules or updates to any message validation rules
+-   Logic to handle new transactions, validation, making changes to state, etc (for more detail look at the doc on [request handling](request_handling.md)).
+-   Storage of transactions in existing ledger or introducing new ledgers
+-   Changes to existing state or introducing new states  
+-   Defining new constants
+-   Defining any new config variables
+
+
+### Conventions that need to be followed when defining a new plugin:
+1. Each plugin is a separate python package that lives under the package defined in `PLUGIN_ROOT` in `config.py`. It is currently set to `plenum.server.plugin`
+2. The list of enabled plugins is defined in `ENABLED_PLUGINS` in `config.py`.
+3. Each plugin has unique name, this is the name in the list `ENABLED_PLUGINS` and has to match the package name of the plugin.
+4. The plugin package contains defines several items in the `__init__.py` file
+    -   It contains a ledger id for each new ledger that the plugin introduces, see `LEDGER_IDS`
+    -   For each new field that needs to be added in the `Request`, the plugin defines the name and its corresponding input validation function in a dict, see `CLIENT_REQUEST_FIELDS`
+    -   Each write or query transaction type is defined in `AcceptableWriteTypes` and `AcceptableQueryTypes` respectively
+5. Each plugin package contains a `main.py` that contains a method `integrate_plugin_in_node` to initialise and register any new ledgers, states, authenticators, 
+    request handlers, etc.
+6. If the plugin requires a new authenticator, it should be defined in `client_authn.py` in the plugin package.
+7. New storage layers should be defined in `storage.py` in the package
+8. New transactions need to be in `transactions.py` in the plugin package. To avoid conflicts in transaction type values, each plugin defines a unique prefix to be added before 
+    the value of the transaction type, see `PREFIX`.
+9. Any new config variables that have to be introduced are introduced in `get_config` in `config.py` in the plugin package.
+
+
+### Plugin setup
+When plenum initialises, initialises all enabled plugins; this is done through `setup_plugins` method defined in `plenum/__init__.py` Plenum defines 2 project level globals:
+`PLUGIN_LEDGER_IDS` which would be the set of ids of all new ledgers defined by any enabled plugin and `PLUGIN_CLIENT_REQUEST_FIELDS` would be a dictionary of all new `Request` fields
+defined by any plugin and its corresponding validator.
+`setup_plugins` iterates over each of the enabled plugin packages and updates `PLUGIN_LEDGER_IDS` and `PLUGIN_CLIENT_REQUEST_FIELDS`. 
+It then reloads some modules which defines input validation objects and schemas, this is done since set of valid ledger ids of request fields might have changed.
+
+
+### Demo plugin
+The Demo plugin serves as an example of how a plugin should be written, though it does some extra setup actions that an actual plugin will not do.
+Since the configuration has been already loaded completely and the input validation objects have been initialised, the changes made by this plugin need to be reflected in them.
+Therefore after updating the loaded config in `tconf` in `plenum/test/plugin/demo_plugin/conftest.py`, in the `do_plugin_initialisation_for_tests`, all such modules are reloaded and `do_plugin_initialisation_for_tests` is called after the plugin is setup (`setup_plugins`) 
+The demo plugin is located at `plenum/test/plugin/demo_plugin`. The demo plugin models a very trivial auction functionality.
+1. The plugin defines new transaction in `plenum/test/plugin/demo_plugin/transactions.py`
+2. The plugin defines constants denoting transaction types in `plenum/test/plugin/demo_plugin/constants.py`
+3. The demo plugin defines a new ledger for which it declares the ledger id `AUCTION_LEDGER_ID` in `plenum/test/plugin/demo_plugin/__init__.py`. 
+    Also the plugin defines a new field in request called `fix_length_dummy` and corresponding input validator for that.
+4. New ledger, hash store and state are defined in `plenum/test/plugin/demo_plugin/storage.py`
+5. A request handler for the plugin introduced transactions are defined, `AuctionReqHandler` in `plenum/test/plugin/demo_plugin/auction_req_handler.py`. 
+    The request handler defines methods for static validation, dynamic validation and state changes.
+6. New config entries for ledger name, state name, storage type are defined in `plenum/test/plugin/demo_plugin/config.py`
+7. An authenticator `AuctionAuthNr` is defined in `plenum/test/plugin/demo_plugin/client_authnr.py`, that specifies the query and write types.
+8. The integration of this plugin happens in `integrate_plugin_in_node` in `plenum/test/plugin/demo_plugin/main.py` which takes the node as an argument. In this integration method:
+    -   Update the node's config with the plugin's config
+    -   Initialise the plugin's ledger, state and other storage components
+    -   Register the storage components in the node.
+    -   Initialise the authenticator and register it with the node
+    -   Initialise and register the request handler with the node.
+9. Now the above method can be called on any initialised node. For the tests, the `TestNode`s are updated in the fixture `do_post_node_creation` in `plenum/test/plugin/demo_plugin/conftest.py`

ledger/test/conftest.py

@@ -6,8 +6,8 @@
 from common.serializers.msgpack_serializer import MsgPackSerializer
 from common.serializers.signing_serializer import SigningSerializer
 from ledger.genesis_txn.genesis_txn_file_util import create_genesis_txn_init_ledger
-from ledger.test.helper import create_ledger, create_ledger_text_file_storage, create_ledger_chunked_file_storage, \
-    create_ledger_leveldb_file_storage
+from ledger.test.helper import create_ledger, create_ledger_text_file_storage, \
+    create_ledger_chunked_file_storage, create_ledger_leveldb_file_storage
 
 
 @pytest.fixture(scope='module')

ledger/test/test_ledger.py

@@ -320,29 +320,10 @@ def test_add_get_txns(ledger_no_genesis):
 
     check_ledger_generator(ledger)
 
-    for s, t in ledger.getAllTxn(frm=1, to=20):
-        assert txns[s - 1] == t
-
-    for s, t in ledger.getAllTxn(frm=3, to=8):
-        assert txns[s - 1] == t
-
-    for s, t in ledger.getAllTxn(frm=5, to=17):
-        assert txns[s - 1] == t
-
-    for s, t in ledger.getAllTxn(frm=6, to=10):
-        assert txns[s - 1] == t
-
-    for s, t in ledger.getAllTxn(frm=3, to=3):
-        assert txns[s - 1] == t
-
-    for s, t in ledger.getAllTxn(frm=3):
-        assert txns[s - 1] == t
-
-    for s, t in ledger.getAllTxn(to=10):
-        assert txns[s - 1] == t
-
-    for s, t in ledger.getAllTxn():
-        assert txns[s - 1] == t
+    for frm, to in [(1, 20), (3, 8), (5, 17), (6, 10), (3, 3),
+                    (3, None), (None, 10), (None, None)]:
+        for s, t in ledger.getAllTxn(frm=frm, to=to):
+            assert txns[s - 1] == t
 
     # with pytest.raises(AssertionError):
     #     list(ledger.getAllTxn(frm=3, to=1))

plenum/__init__.py

@@ -9,24 +9,34 @@
     raise ImportError("Python 3.5.0 or later required.")
 
 import os   # noqa
+import importlib    # noqa
 from importlib.util import module_from_spec, spec_from_file_location    # noqa: E402
 
 import plenum   # noqa: E402
 import plenum.server.plugin     # noqa: E402
+
 from plenum.common.config_util import getConfigOnce   # noqa: E402
 
 PLUGIN_LEDGER_IDS = set()
 PLUGIN_CLIENT_REQUEST_FIELDS = {}
 
 
 def setup_plugins():
+    # TODO: Should have a check to make sure no plugin defines any conflicting ledger id or request field
     global PLUGIN_LEDGER_IDS
+    global PLUGIN_CLIENT_REQUEST_FIELDS
 
     config = getConfigOnce()
 
-    ENABLED_PLUGINS = config.ENABLED_PLUGINS
-    for plugin_name in ENABLED_PLUGINS:
-        plugin_path = os.path.join(plenum.server.plugin.__path__[0],
+    plugin_root = config.PLUGIN_ROOT
+    try:
+        plugin_root = importlib.import_module(plugin_root)
+    except ImportError:
+        raise ImportError('Incorrect plugin root {}. No such package found'.
+                          format(plugin_root))
+    enabled_plugins = config.ENABLED_PLUGINS
+    for plugin_name in enabled_plugins:
+        plugin_path = os.path.join(plugin_root.__path__[0],
                                    plugin_name, '__init__.py')
         spec = spec_from_file_location('__init__.py', plugin_path)
         init = module_from_spec(spec)
@@ -37,6 +47,12 @@ def setup_plugins():
         if 'CLIENT_REQUEST_FIELDS' in plugin_globals:
             PLUGIN_CLIENT_REQUEST_FIELDS.update(plugin_globals['CLIENT_REQUEST_FIELDS'])
 
+    # Reloading message types since some some schemas would have been changed
+    import plenum.common.messages.node_messages
+    import plenum.common.messages.node_message_factory
+    importlib.reload(plenum.common.messages.node_messages)
+    importlib.reload(plenum.common.messages.node_message_factory)
+
 
 setup_plugins()
 

plenum/client/client.py

@@ -42,8 +42,7 @@
     MULTI_SIGNATURE_SIGNATURE, MULTI_SIGNATURE_VALUE
 from plenum.common.txn_util import idr_from_req_data
 from plenum.common.types import f
-from plenum.common.util import getMaxFailures, checkIfMoreThanFSameItems, \
-    rawToFriendly, mostCommonElement
+from plenum.common.util import getMaxFailures, rawToFriendly, mostCommonElement
 from plenum.persistence.client_req_rep_store_file import ClientReqRepStoreFile
 from plenum.persistence.client_txn_log import ClientTxnLog
 from plenum.server.has_action_queue import HasActionQueue
@@ -275,7 +274,7 @@ def start(self, loop):
             self.nodestack.start()
             self.nodestack.maintainConnections(force=True)
             if self.ledger:
-                self.ledgerManager.setLedgerCanSync(0, True)
+                self.ledgerManager.setLedgerCanSync(POOL_LEDGER_ID, True)
                 self.mode = Mode.starting
 
     async def prod(self, limit) -> int:

plenum/client/wallet.py

@@ -172,7 +172,6 @@ def signMsg(self,
         :param otherIdentifier:
         :return: signature that then can be assigned to request
         """
-
         idr = self.requiredIdr(idr=identifier or otherIdentifier)
         signer = self._signerById(idr)
         signature = signer.sign(msg)
@@ -198,7 +197,7 @@ def signRequest(self,
         # QUESTION: `self.ids[idr]` would be overwritten if same identifier
         # is used to send 2 requests, why is `IdData` persisted?
         # self.ids[idr] = IdData(idData.signer, req.reqId)
-        req.signature = self.signMsg(msg=req.signingState(),
+        req.signature = self.signMsg(msg=req.signingState(identifier=idr),
                                      identifier=idr,
                                      otherIdentifier=req.identifier)
 

plenum/common/batched.py

@@ -1,5 +1,5 @@
 from collections import deque
-from typing import Any, Iterable
+from typing import Any, Iterable, Dict
 
 from plenum.common.constants import BATCH, OP_FIELD_NAME
 from plenum.common.prepare_batch import split_messages_on_batches
@@ -117,8 +117,8 @@ def flushOutBoxes(self) -> None:
                                 timeout=self.messageTimeout,
                                 serialized=True)
                     else:
-                        logger.warning("Cannot create batch(es) for {}".format(
-                            self, dest))
+                        logger.warning("{} cannot create batch(es) for {}"
+                                       .format(self, dest))
                 else:
                     while msgs:
                         msg = msgs.popleft()

plenum/common/config_helper.py

@@ -1,7 +1,7 @@
 import os
 
 
-class PConfigHelper():
+class PConfigHelper:
 
     def __init__(self, config, *, chroot=None):
         assert config is not None
@@ -47,4 +47,5 @@ def __init__(self, name: str, config, *, chroot=None):
 
     @property
     def ledger_dir(self):
-        return self.chroot_if_needed(os.path.join(self.config.LEDGER_DIR, self.name))
+        return self.chroot_if_needed(os.path.join(self.config.LEDGER_DIR,
+                                                  self.name))

plenum/common/config_util.py

@@ -77,13 +77,14 @@ def _getConfig(general_config_dir: str = None):
     config = stp_config
     config.__dict__.update(plenum_config.__dict__)
 
-    if (general_config_dir):
+    if general_config_dir:
         config.GENERAL_CONFIG_DIR = general_config_dir
 
     if not config.GENERAL_CONFIG_DIR:
         raise Exception('GENERAL_CONFIG_DIR must be set')
 
-    extend_with_external_config(config, (config.GENERAL_CONFIG_DIR, config.GENERAL_CONFIG_FILE))
+    extend_with_external_config(config, (config.GENERAL_CONFIG_DIR,
+                                         config.GENERAL_CONFIG_FILE))
 
     # "unsafe" is a set of attributes that can set certain behaviors that
     # are not safe, for example, 'disable_view_change' disables view changes

plenum/common/constants.py

@@ -4,6 +4,7 @@
 from plenum.common.plenum_protocol_version import PlenumProtocolVersion
 from plenum.common.roles import Roles
 from plenum.common.transactions import PlenumTransactions
+from plenum.common.util import UniqueSet
 
 NOMINATE = "NOMINATE"
 REELECTION = "REELECTION"
@@ -144,8 +145,10 @@ class KeyValueStorageType(IntEnum):
 @unique
 class LedgerState(IntEnum):
     not_synced = 1  # Still gathering consistency proofs
-    syncing = 2  # Got sufficient consistency proofs, will be sending catchup requests and waiting for their replies
-    synced = 3  # Got replies for all catchup requests, indicating catchup complete for the ledger
+    syncing = 2  # Got sufficient consistency proofs, will be sending catchup
+    # requests and waiting for their replies
+    synced = 3  # Got replies for all catchup requests, indicating catchup
+    # complete for the ledger
 
 
 OP_FIELD_NAME = "op"
@@ -165,36 +168,31 @@ class LedgerState(IntEnum):
 PLUGIN_BASE_DIR_PATH = "PluginBaseDirPath"
 POOL_LEDGER_ID = 0
 DOMAIN_LEDGER_ID = 1
+CONFIG_LEDGER_ID = 2
 
-VALID_LEDGER_IDS = (POOL_LEDGER_ID, DOMAIN_LEDGER_ID)
+VALID_LEDGER_IDS = (POOL_LEDGER_ID, DOMAIN_LEDGER_ID, CONFIG_LEDGER_ID)
 
 CURRENT_PROTOCOL_VERSION = PlenumProtocolVersion.STATE_PROOF_SUPPORT.value
 
-PRE_STATIC_VALIDATION = 1
-POST_STATIC_VALIDATION = 2
-PRE_SIG_VERIFICATION = 3
-POST_SIG_VERIFICATION = 4
-PRE_DYNAMIC_VALIDATION = 5
-POST_DYNAMIC_VALIDATION = 6
-PRE_REQUEST_APPLICATION = 7
-POST_REQUEST_APPLICATION = 8
-PRE_REQUEST_COMMIT = 9
-POST_REQUEST_COMMIT = 10
-
-NODE_HOOKS = [PRE_STATIC_VALIDATION, POST_STATIC_VALIDATION,
-              PRE_SIG_VERIFICATION, POST_SIG_VERIFICATION,
-              PRE_DYNAMIC_VALIDATION, POST_DYNAMIC_VALIDATION,
-              PRE_REQUEST_APPLICATION, POST_REQUEST_APPLICATION,
-              PRE_REQUEST_COMMIT, POST_REQUEST_COMMIT]
-
-
-CREATE_PPR = 1
-CREATE_PR = 2
-CREATE_CM = 3
-CREATE_ORD = 4
-RECV_PPR = 5
-RECV_PR = 6
-RECV_CM = 7
-
-REPLICA_HOOKS = [CREATE_PPR, CREATE_PR, CREATE_CM, CREATE_ORD, RECV_PPR,
-                 RECV_PR, RECV_CM]
+
+class NodeHooks(UniqueSet):
+    PRE_STATIC_VALIDATION = 1
+    POST_STATIC_VALIDATION = 2
+    PRE_SIG_VERIFICATION = 3
+    POST_SIG_VERIFICATION = 4
+    PRE_DYNAMIC_VALIDATION = 5
+    POST_DYNAMIC_VALIDATION = 6
+    PRE_REQUEST_APPLICATION = 7
+    POST_REQUEST_APPLICATION = 8
+    PRE_REQUEST_COMMIT = 9
+    POST_REQUEST_COMMIT = 10
+
+
+class ReplicaHooks(UniqueSet):
+    CREATE_PPR = 1
+    CREATE_PR = 2
+    CREATE_CM = 3
+    CREATE_ORD = 4
+    RECV_PPR = 5
+    RECV_PR = 6
+    RECV_CM = 7

plenum/common/error.py

@@ -4,14 +4,3 @@
 def fault(ex: Exception, msg: str):
     from stp_core.common.log import getlogger
     getlogger().error(msg, exc_info=ex)
-
-
-def error(msg: str, exc_type: Exception=Exception) -> Exception:
-    """
-    Wrapper to get around Python's distinction between statements and expressions
-    Can be used in lambdas and expressions such as: a if b else error(c)
-
-    :param msg: error message
-    :param exc_type: type of exception to raise
-    """
-    raise exc_type(msg)

plenum/common/hook_manager.py

@@ -1,6 +1,6 @@
 from typing import Callable, Dict, List
 
-from plenum.common.error import error
+from common.error import error
 
 
 class HookManager: