Remove /gov/read and /gov/query endpoints

CHANGELOG.md

@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [1.0]
+
+### Added
+
+- The service certificate is now returned as part of the `/node/network/` endpoint response.
+
+### Removed
+
+- `/gov/query` and `/gov/read` governance endpoints are removed.
+
 ## [0.99.0]
 
 This is a bridging release to simplify the upgrade to 1.0. It includes the new JS constitution, but also supports the existing Lua governance so that users can upgrade in 2 steps - first implementing all of the changes below with their existing Lua governance, then upgrading to the JS governance. Lua governance will be removed in CCF 1.0. See [temporary docs](https://microsoft.github.io/CCF/ccf-0.99.0/governance/js_gov.html) for help with transitioning from Lua to JS.

doc/schemas/gov_openapi.json

@@ -161,21 +161,6 @@
         ],
         "type": "object"
       },
-      "KVRead__In": {
-        "properties": {
-          "key": {
-            "$ref": "#/components/schemas/json"
-          },
-          "table": {
-            "$ref": "#/components/schemas/string"
-          }
-        },
-        "required": [
-          "table",
-          "key"
-        ],
-        "type": "object"
-      },
       "Proposal": {
         "properties": {
           "actions": {
@@ -912,68 +897,6 @@
         ]
       }
     },
-    "/query": {
-      "post": {
-        "requestBody": {
-          "content": {
-            "application/json": {
-              "schema": {
-                "$ref": "#/components/schemas/Script"
-              }
-            }
-          },
-          "description": "Auto-generated request body schema"
-        },
-        "responses": {
-          "200": {
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/json"
-                }
-              }
-            },
-            "description": "Default response description"
-          }
-        },
-        "security": [
-          {
-            "member_signature": []
-          }
-        ]
-      }
-    },
-    "/read": {
-      "post": {
-        "requestBody": {
-          "content": {
-            "application/json": {
-              "schema": {
-                "$ref": "#/components/schemas/KVRead__In"
-              }
-            }
-          },
-          "description": "Auto-generated request body schema"
-        },
-        "responses": {
-          "200": {
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/json"
-                }
-              }
-            },
-            "description": "Default response description"
-          }
-        },
-        "security": [
-          {
-            "member_signature": []
-          }
-        ]
-      }
-    },
     "/receipt": {
       "get": {
         "parameters": [

doc/schemas/node_openapi.json

@@ -111,12 +111,16 @@
           "primary_id": {
             "$ref": "#/components/schemas/EntityId"
           },
+          "service_certificate": {
+            "$ref": "#/components/schemas/Pem"
+          },
           "service_status": {
             "$ref": "#/components/schemas/ServiceStatus"
           }
         },
         "required": [
           "service_status",
+          "service_certificate",
           "current_view",
           "primary_id"
         ],
@@ -252,6 +256,9 @@
         ],
         "type": "string"
       },
+      "Pem": {
+        "type": "string"
+      },
       "Quote": {
         "properties": {
           "endorsements": {

python/ccf/ledger.py

@@ -5,7 +5,7 @@
 import struct
 import os
 
-from typing import BinaryIO, NamedTuple, Optional
+from typing import BinaryIO, NamedTuple, Optional, Tuple, Dict
 
 import json
 import base64
@@ -31,6 +31,9 @@
 SIGNATURE_TX_TABLE_NAME = "public:ccf.internal.signatures"
 NODES_TABLE_NAME = "public:ccf.gov.nodes.info"
 
+# Key used by CCF to record single-key tables
+WELL_KNOWN_SINGLETON_TABLE_KEY = bytes(bytearray(8))
+
 
 def to_uint_32(buffer):
     return struct.unpack("@I", buffer)[0]
@@ -420,8 +423,7 @@ def __next__(self):
             self._ledger_validator.add_transaction(self)
 
             return self
-        except Exception as exception:
-            LOG.exception(f"Encountered exception: {exception}")
+        except:
             raise
 
 
@@ -466,10 +468,14 @@ class Ledger:
     _current_chunk: LedgerChunk
     _ledger_validator: LedgerValidator
 
+    def _reset_iterators(self):
+        self._fileindex = -1
+        # Initialize LedgerValidator instance which will be passed to LedgerChunks.
+        self._ledger_validator = LedgerValidator()
+
     def __init__(self, directory: str):
 
         self._filenames = []
-        self._fileindex = -1
 
         ledgers = os.listdir(directory)
         # Sorts the list based off the first number after ledger_ so that
@@ -485,8 +491,7 @@ def __init__(self, directory: str):
             if os.path.isfile(os.path.join(directory, chunk)):
                 self._filenames.append(os.path.join(directory, chunk))
 
-        # Initialize LedgerValidator instance which will be passed to LedgerChunks.
-        self._ledger_validator = LedgerValidator()
+        self._reset_iterators()
 
     def __next__(self) -> LedgerChunk:
         self._fileindex += 1
@@ -513,6 +518,68 @@ def last_verified_txid(self):
             self._ledger_validator.last_verified_seqno,
         )
 
+    def get_transaction(self, seqno: int) -> Transaction:
+        """
+        Returns the :py:class:`ccf.Ledger.Transaction` recorded in the ledger at the given sequence number
+
+        Note that the transaction returned may not yet be verified by a
+        signature transaction nor committed by the service.
+
+        :param int seqno: Sequence number of the transaction to fetch
+
+        :return: :py:class:`ccf.Ledger.Transaction`
+        """
+        if seqno < 1:
+            raise ValueError("Ledger first seqno is 1")
+
+        self._reset_iterators()
+
+        transaction = None
+        try:
+            for chunk in self:
+                for tx in chunk:
+                    public_transaction = tx.get_public_domain()
+                    if public_transaction.get_seqno() == seqno:
+                        return tx
+        finally:
+            self._reset_iterators()
+
+        if transaction is None:
+            raise UnknownTransaction(
+                f"Transaction at seqno {seqno} does not exist in ledger"
+            )
+        return transaction
+
+    def get_latest_public_state(self) -> Tuple[dict, int]:
+        """
+        Returns the current public state of the service.
+
+        Note that the public state returned may not yet be verified by a
+        signature transaction nor committed by the service.
+
+        :return: Tuple[Dict, int]: Tuple containing a dictionnary of public tables and their values and the seqno of the state read from the ledger.
+        """
+        self._reset_iterators()
+
+        public_tables: Dict[str, Dict] = {}
+        latest_seqno = 0
+        for chunk in self:
+            for tx in chunk:
+                latest_seqno = tx.get_public_domain().get_seqno()
+                for table_name, records in tx.get_public_domain().get_tables().items():
+                    if table_name in public_tables:
+                        public_tables[table_name].update(records)
+                        # Remove deleted keys
+                        public_tables[table_name] = {
+                            k: v
+                            for k, v in public_tables[table_name].items()
+                            if v is not None
+                        }
+                    else:
+                        public_tables[table_name] = records
+
+        return public_tables, latest_seqno
+
 
 class InvalidRootException(Exception):
     """MerkleTree root doesn't match with the root reported in the signature's table"""
@@ -528,3 +595,7 @@ class CommitIdRangeException(Exception):
 
 class UntrustedNodeException(Exception):
     """The signing node wasn't part of the network"""
+
+
+class UnknownTransaction(Exception):
+    """The transaction at seqno does not exist in ledger"""

python/ledger_tutorial.py

@@ -4,6 +4,7 @@
 import sys
 from loguru import logger as LOG
 import json
+import random
 
 # Note: It is safer to run the ledger tutorial when the service has stopped
 # as all ledger files will have been written to.
@@ -44,3 +45,13 @@
                 # In this case, the target table 'public:ccf.gov.nodes.info' is raw bytes to JSON.
                 LOG.info(f"{key.decode()} : {json.loads(value)}")
 # SNIPPET_END: iterate_over_ledger
+
+# Read state of ledger
+_, latest_seqno = ledger.get_latest_public_state()
+
+seqnos = [1, 2, 3, latest_seqno // 2, latest_seqno]
+random.shuffle(seqnos)
+for seqno in seqnos:
+    transaction = ledger.get_transaction(seqno)
+
+ledger.get_latest_public_state()

src/crypto/pem.h

@@ -116,6 +116,16 @@ namespace crypto
         fmt::format("Unable to parse pem from this JSON: {}", j.dump()));
     }
   }
+
+  inline std::string schema_name(const Pem&)
+  {
+    return "Pem";
+  }
+
+  inline void fill_json_schema(nlohmann::json& schema, const Pem&)
+  {
+    schema["type"] = "string";
+  }
 }
 
 namespace std

src/node/rpc/call_types.h

@@ -54,6 +54,7 @@ namespace ccf
     struct Out
     {
       ServiceStatus service_status;
+      crypto::Pem service_certificate;
       std::optional<ccf::View> current_view;
       std::optional<NodeId> primary_id;
     };