Change serialisation format for service maps (#2301)

Author: jumaffre

CHANGELOG.md

@@ -7,12 +7,19 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [0.19.2]
 
+### Added
+
+- New `get_user_data_v1` and `get_member_data_v1` C++ API calls have been added to retrieve the data associated with users/members. The user/member data is no longer included in the `AuthnIdentity` caller struct (#2301).
+- New `get_user_cert_v1` and `get_member_cert_v1` C++ API calls have been added to retrieve the PEM certificate of the users/members. The user/member certificate is no longer included in the `AuthnIdentity` caller struct (#2301).
+
 ### Changed
 
-- String values in query parameters no longer need to be quoted. For instance, you should now call `/network/nodes?host=127.0.0.1` rather than `/network/nodes?host="127.0.0.1"`.
-- Schema documentation for query parameters should now be added with `add_query_parameter`, rather than `set_auto_schema`. The `In` type of `set_auto_schema` should only be used to describe the request body.
-- `json_adapter` will no longer try to convert query parameters to a JSON object. The JSON passed as an argument to these handlers will now be populated only by the request body. The query string should be parsed separately, and `http::parse_query(s)` is added as a starting point. This means strings in query parameters no longer need to be quoted.
+- String values in query parameters no longer need to be quoted. For instance, you should now call `/network/nodes?host=127.0.0.1` rather than `/network/nodes?host="127.0.0.1"` (#2309).
+- Schema documentation for query parameters should now be added with `add_query_parameter`, rather than `set_auto_schema`. The `In` type of `set_auto_schema` should only be used to describe the request body (#2309).
+- `json_adapter` will no longer try to convert query parameters to a JSON object. The JSON passed as an argument to these handlers will now be populated only by the request body. The query string should be parsed separately, and `http::parse_query(s)` is added as a starting point. This means strings in query parameters no longer need to be quoted (#2309).
 - Enum values returned by built-in REST API endpoints are now PascalCase. Lua governance scripts that use enum values need to be updated as well, for example, `"ACTIVE"` becomes `"Active"` for member info. The same applies when using the `/gov/query` endpoint (#2152).
+- Most service tables (e.g. for nodes and signatures) are now serialised as JSON instead of msgpack. Some tables (e.g. user and member certificates) are serialised as raw bytes for performance reasons (#2301).
+- The users and members tables have been split into `public:ccf.gov.users.certs`/`public:ccf.gov.users.info` and `public:ccf.gov.members.certs`/`public:ccf.gov.members.encryption_public_keys`/`public:ccf.gov.members.info` respectively (#2301).
 
 ## [0.19.1]
 

CMakeLists.txt

@@ -269,6 +269,7 @@ if(BUILD_TESTS)
       ${CMAKE_CURRENT_SOURCE_DIR}/src/ds/test/hash.cpp
       ${CMAKE_CURRENT_SOURCE_DIR}/src/ds/test/thread_messaging.cpp
       ${CMAKE_CURRENT_SOURCE_DIR}/src/ds/test/lru.cpp
+      ${CMAKE_CURRENT_SOURCE_DIR}/src/ds/test/hex.cpp
     )
     target_link_libraries(ds_test PRIVATE ${CMAKE_THREAD_LIBS_INIT})
 

doc/schemas/app_openapi.json

@@ -56,6 +56,7 @@
         "type": "object"
       },
       "EntityId": {
+        "format": "hex",
         "pattern": "^[a-f0-9]{64}$",
         "type": "string"
       },

doc/schemas/gov_openapi.json

@@ -56,6 +56,7 @@
         "type": "object"
       },
       "EntityId": {
+        "format": "hex",
         "pattern": "^[a-f0-9]{64}$",
         "type": "string"
       },
@@ -286,7 +287,7 @@
       "Script": {
         "properties": {
           "bytecode": {
-            "$ref": "#/components/schemas/uint8_array"
+            "$ref": "#/components/schemas/base64string"
           },
           "text": {
             "$ref": "#/components/schemas/string"
@@ -351,6 +352,10 @@
         ],
         "type": "object"
       },
+      "base64string": {
+        "format": "base64",
+        "type": "string"
+      },
       "int64": {
         "maximum": 9223372036854775807,
         "minimum": -9223372036854775808,
@@ -364,17 +369,6 @@
         "maximum": 18446744073709551615,
         "minimum": 0,
         "type": "integer"
-      },
-      "uint8": {
-        "maximum": 255,
-        "minimum": 0,
-        "type": "integer"
-      },
-      "uint8_array": {
-        "items": {
-          "$ref": "#/components/schemas/uint8"
-        },
-        "type": "array"
       }
     },
     "securitySchemes": {

doc/schemas/node_openapi.json

@@ -56,6 +56,7 @@
         "type": "object"
       },
       "EntityId": {
+        "format": "hex",
         "pattern": "^[a-f0-9]{64}$",
         "type": "string"
       },
@@ -306,7 +307,7 @@
       "Quote": {
         "properties": {
           "endorsements": {
-            "$ref": "#/components/schemas/string"
+            "$ref": "#/components/schemas/base64string"
           },
           "format": {
             "$ref": "#/components/schemas/QuoteFormat"
@@ -318,7 +319,7 @@
             "$ref": "#/components/schemas/EntityId"
           },
           "raw": {
-            "$ref": "#/components/schemas/string"
+            "$ref": "#/components/schemas/base64string"
           }
         },
         "required": [
@@ -363,6 +364,10 @@
         ],
         "type": "string"
       },
+      "base64string": {
+        "format": "base64",
+        "type": "string"
+      },
       "boolean": {
         "type": "boolean"
       },

getting_started/setup_vm/roles/ccf_build/vars/common.yml

@@ -22,7 +22,6 @@ debs:
   - sudo
   - curl
   - shellcheck
-  - xxd
 
 mbedtls_ver: "2.16.8"
 mbedtls_dir: "mbedtls-{{ mbedtls_ver }}"

python/ccf/clients.py

@@ -35,7 +35,7 @@ def escape_loguru_tags(s):
     return loguru_tag_regex.sub(lambda match: f"\\{match[0]}", s)
 
 
-def truncate(string: str, max_len: int = 128):
+def truncate(string: str, max_len: int = 256):
     if len(string) > max_len:
         return f"{string[: max_len]} + {len(string) - max_len} chars"
     else:

python/ccf/ledger.py

@@ -5,11 +5,12 @@
 import struct
 import os
 
-from typing import BinaryIO, NamedTuple, Optional, Set
-from enum import IntEnum
+from typing import BinaryIO, NamedTuple, Optional
 
 # Default implementation has buggy interaction between read_bytes and tell, so use fallback
 import msgpack.fallback as msgpack  # type: ignore
+import json
+import base64
 
 from loguru import logger as LOG  # type: ignore
 from cryptography.x509 import load_pem_x509_certificate
@@ -73,7 +74,6 @@ class PublicDomain:
     _version: int
     _max_conflict_version: int
     _tables: dict
-    _msgpacked_tables: Set[str]
 
     def __init__(self, buffer: io.BytesIO):
         self._buffer = buffer
@@ -83,12 +83,6 @@ def __init__(self, buffer: io.BytesIO):
         self._version = self._read_next()
         self._max_conflict_version = self._read_next()
         self._tables = {}
-        # Keys and Values may have custom serialisers.
-        # Store most as raw bytes, only decode a few which we know are msgpack and are required for ledger verification.
-        self._msgpacked_tables = {
-            SIGNATURE_TX_TABLE_NAME,
-            NODES_TABLE_NAME,
-        }
         self._read()
 
     def _read_next(self):
@@ -124,19 +118,12 @@ def _read(self):
                 for _ in range(write_count):
                     k = self._read_next_entry()
                     val = self._read_next_entry()
-                    if map_name in self._msgpacked_tables:
-                        k = msgpack.unpackb(k, **UNPACK_ARGS)
-                        val = msgpack.unpackb(val, **UNPACK_ARGS)
-                    if map_name == NODES_TABLE_NAME:
-                        k = str(k)
                     records[k] = val
 
             remove_count = self._read_next()
             if remove_count:
                 for _ in range(remove_count):
                     k = self._read_next_entry()
-                    if map_name in self._msgpacked_tables:
-                        k = msgpack.unpackb(k, **UNPACK_ARGS)
                     records[k] = None
 
             LOG.trace(
@@ -162,14 +149,6 @@ def _byte_read_safe(file, num_of_bytes):
     return ret
 
 
-class NodeStatus(IntEnum):
-    """These are the corresponding status meanings from the ccf.nodes table"""
-
-    PENDING = 0
-    TRUSTED = 1
-    RETIRED = 2
-
-
 class TxBundleInfo(NamedTuple):
     """Bundle for transaction information required for validation """
 
@@ -178,7 +157,7 @@ class TxBundleInfo(NamedTuple):
     node_cert: bytes
     signature: bytes
     node_activity: dict
-    signing_node: bytes
+    signing_node: str
 
 
 class LedgerValidator:
@@ -187,27 +166,11 @@ class LedgerValidator:
     It has the ability to take transactions and it maintains a MerkleTree data structure similar to CCF.
 
     Ledger is valid and hasn't been tampered with if following conditions are met:
-        1) The merkle proof is signed by a TRUSTED node in the given network
+        1) The merkle proof is signed by a Trusted node in the given network
         2) The merkle root and signature are verified with the node cert
         3) The merkle proof is correct for each set of transactions
     """
 
-    # The node that is expected to sign the signature transaction
-    # The certificate used to sign the signature transaction
-    # https://github.com/microsoft/CCF/blob/main/src/node/nodes.h
-    EXPECTED_NODE_CERT_INDEX = 1
-    # The current network trust status of the Node at the time of the current transaction
-    EXPECTED_NODE_STATUS_INDEX = 4
-
-    # Signature table contains PrimarySignature which extends NodeSignature. NodeId should be at index 1 in the serialized Node
-    # https://github.com/microsoft/CCF/blob/main/src/node/signatures.h
-    EXPECTED_NODE_SIGNATURE_INDEX = 0
-    EXPECTED_NODE_SEQNO_INDEX = 1
-    EXPECTED_NODE_VIEW_INDEX = 2
-    EXPECTED_ROOT_INDEX = 5
-    # https://github.com/microsoft/CCF/blob/main/src/node/node_signature.h
-    EXPECTED_SIGNING_NODE_ID_INDEX = 1
-    EXPECTED_SIGNATURE_INDEX = 0
     # Constant for the size of a hashed transaction
     SHA_256_HASH_SIZE = 32
 
@@ -240,34 +203,29 @@ def add_transaction(self, transaction):
         # Add contributing nodes certs and update nodes network trust status for verification
         if NODES_TABLE_NAME in tables:
             node_table = tables[NODES_TABLE_NAME]
-            for nodeid, values in node_table.items():
+            for node_id, node_info in node_table.items():
+                node_id = node_id.decode()
+                node_info = json.loads(node_info)
                 # Add the nodes certificate
-                self.node_certificates[nodeid] = values[self.EXPECTED_NODE_CERT_INDEX]
+                self.node_certificates[node_id] = node_info["cert"].encode()
                 # Update node trust status
-                self.node_activity_status[nodeid] = NodeStatus(
-                    values[self.EXPECTED_NODE_STATUS_INDEX]
-                )
+                self.node_activity_status[node_id] = node_info["status"]
 
         # This is a merkle root/signature tx if the table exists
         if SIGNATURE_TX_TABLE_NAME in tables:
             self.signature_count += 1
             signature_table = tables[SIGNATURE_TX_TABLE_NAME]
 
-            for nodeid, values in signature_table.items():
-                current_seqno = values[self.EXPECTED_NODE_SEQNO_INDEX]
-                current_view = values[self.EXPECTED_NODE_VIEW_INDEX]
-                signing_node = str(
-                    values[self.EXPECTED_NODE_SIGNATURE_INDEX][
-                        self.EXPECTED_SIGNING_NODE_ID_INDEX
-                    ]
-                )
+            for _, signature in signature_table.items():
+                signature = json.loads(signature)
+                current_seqno = signature["seqno"]
+                current_view = signature["view"]
+                signing_node = signature["node"]
 
                 # Get binary representations for the cert, existing root, and signature
-                cert = b"".join(self.node_certificates[signing_node])
-                existing_root = b"".join(values[self.EXPECTED_ROOT_INDEX])
-                signature = values[self.EXPECTED_NODE_SIGNATURE_INDEX][
-                    self.EXPECTED_SIGNATURE_INDEX
-                ]
+                cert = self.node_certificates[signing_node]
+                existing_root = bytes.fromhex(signature["root"])
+                signature = base64.b64decode(signature["sig"])
 
                 tx_info = TxBundleInfo(
                     self.merkle,
@@ -295,7 +253,7 @@ def _verify_tx_set(self, tx_info: TxBundleInfo):
         """
         Verify items 1, 2, and 3 for all the transactions up until a signature.
         """
-        # 1) The merkle root is signed by a TRUSTED node in the given network, else throws
+        # 1) The merkle root is signed by a Trusted node in the given network, else throws
         self._verify_node_status(tx_info)
         # 2) The merkle root and signature are verified with the node cert, else throws
         self._verify_root_signature(tx_info)
@@ -304,8 +262,8 @@ def _verify_tx_set(self, tx_info: TxBundleInfo):
 
     @staticmethod
     def _verify_node_status(tx_info: TxBundleInfo):
-        """Verify item 1, The merkle root is signed by a TRUSTED node in the given network"""
-        if tx_info.node_activity[tx_info.signing_node] != NodeStatus.TRUSTED:
+        """Verify item 1, The merkle root is signed by a Trusted node in the given network"""
+        if tx_info.node_activity[tx_info.signing_node] != "Trusted":
             LOG.error(
                 f"The signing node {tx_info.signing_node!r} is not trusted by the network"
             )
@@ -535,10 +493,6 @@ def __iter__(self):
         return self
 
 
-def extract_msgpacked_data(data: bytes):
-    return msgpack.unpackb(data, **UNPACK_ARGS)
-
-
 class InvalidRootException(Exception):
     """MerkleTree root doesn't match with the root reported in the signature's table"""
 

python/ccf/proposal_generator.py

@@ -347,7 +347,7 @@ def set_module(module_name: str, module_path: str, **kwargs):
     if module_name_.suffix == ".js":
         with open(module_path) as f:
             js = f.read()
-        proposal_args = {"name": module_name, "module": {"js": js}}
+        proposal_args = {"name": module_name, "module": js}
     else:
         raise ValueError("module name must end with .js")
     return build_proposal("set_module", proposal_args, **kwargs)
@@ -367,7 +367,7 @@ def read_modules(modules_path: str) -> List[dict]:
         rel_module_name = rel_module_name.replace("\\", "/")  # Windows support
         with open(path) as f:
             js = f.read()
-            modules.append({"name": rel_module_name, "module": {"js": js}})
+            modules.append({"name": rel_module_name, "module": js})
     return modules
 
 

python/ledger_tutorial.py

@@ -4,6 +4,7 @@
 import ccf.ledger
 import sys
 from loguru import logger as LOG
+import json
 
 # Note: It is safer to run the ledger tutorial when the service has stopped
 # as all ledger files will have been written to.
@@ -30,8 +31,6 @@
 target_table = "public:ccf.gov.nodes.info"
 
 # SNIPPET_START: iterate_over_ledger
-target_table_changes = 0  # Simple counter
-
 for chunk in ledger:
     for transaction in chunk:
         # Retrieve all public tables changed in transaction
@@ -41,10 +40,8 @@
         if target_table in public_tables:
             # Ledger verification is happening implicitly in ccf.ledger.Ledger()
             for key, value in public_tables[target_table].items():
-                target_table_changes += 1  # A key was changed
-                # Log the key and value for the transaction on the target table
-                # The target_table: 'public:ccf.gov.nodes.info' has already been decoded in ledger.py
-                # For other tables knowledge of serialization scheme used is important.
-                # If the table was using msgpack, use ccf.ledger.extract_msgpacked_data(data)
-                LOG.info(f"{key} : {value}")
+                # Note: `key` and `value` are raw bytes here.
+                # This code needs to have knowledge of the serialisation format for each table.
+                # In this case, the target table 'public:ccf.gov.nodes.info' is raw bytes to JSON.
+                LOG.info(f"{key.decode()} : {json.loads(value)}")
 # SNIPPET_END: iterate_over_ledger

python/utils/verify_quote.sh

@@ -65,8 +65,8 @@ trap cleanup EXIT
 
 curl_output=$(curl -sS --fail -X GET "${node_address}"/node/quotes/self "${@}")
 
-echo "${curl_output}" | jq -r .raw | xxd -r -p > "${tmp_dir}/${quote_file_name}"
-echo "${curl_output}" | jq -r .endorsements | xxd -r -p > "${tmp_dir}/${endorsements_file_name}"
+echo "${curl_output}" | jq -r .raw | base64 --decode > "${tmp_dir}/${quote_file_name}"
+echo "${curl_output}" | jq -r .endorsements | base64 --decode > "${tmp_dir}/${endorsements_file_name}"
 
 if [ ! -s "${tmp_dir}/${quote_file_name}" ]; then
     echo "Error: Node quote is empty. Virtual mode does not support SGX quotes."