feat(unrecoverable-error): implement halting of full node execution

Author: glevco

hathor/conf/settings.py

@@ -389,9 +389,6 @@ def GENESIS_TX2_TIMESTAMP(self) -> int:
     # Identifier used in metadata's voided_by to mark a tx as soft-voided.
     SOFT_VOIDED_ID: bytes = b'tx-non-grata'
 
-    # Identifier used in metadata's voided_by when an unexpected exception occurs at consensus.
-    CONSENSUS_FAIL_ID: bytes = b'consensus-fail'
-
     # Identifier used in metadata's voided_by to mark a tx as partially validated.
     PARTIALLY_VALIDATED_ID: bytes = b'pending-validation'
 

hathor/consensus/consensus.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import os
 
 from structlog import get_logger
 
@@ -76,10 +77,12 @@ def update(self, base: BaseTransaction) -> None:
         try:
             self._unsafe_update(base)
         except Exception:
-            meta.add_voided_by(self._settings.CONSENSUS_FAIL_ID)
-            assert base.storage is not None
-            base.storage.save_transaction(base, only_metadata=True)
-            raise
+            self.log.critical(
+                'Consensus update failed, causing the full node to halt execution. Manual intervention is required.',
+                tx=base.hash_hex,
+                exc_info=True
+            )
+            os._exit(-1)
 
     def _unsafe_update(self, base: BaseTransaction) -> None:
         """Run a consensus update with its own context, indexes will be updated accordingly."""
@@ -154,8 +157,6 @@ def filter_out_soft_voided_entries(self, tx: BaseTransaction, voided_by: set[byt
         for h in voided_by:
             if h == self._settings.SOFT_VOIDED_ID:
                 continue
-            if h == self._settings.CONSENSUS_FAIL_ID:
-                continue
             if h == tx.hash:
                 continue
             if h in self.soft_voided_tx_ids:

hathor/manager.py

@@ -1003,13 +1003,7 @@ def on_new_tx(self, tx: BaseTransaction, *, conn: Optional[HathorProtocol] = Non
         tx.update_initial_metadata(save=False)
         self.tx_storage.save_transaction(tx)
         self.tx_storage.add_to_indexes(tx)
-        try:
-            self.consensus_algorithm.update(tx)
-        except HathorError as e:
-            if not fails_silently:
-                raise InvalidNewTransaction('consensus update failed') from e
-            self.log.warn('on_new_tx(): consensus update failed', tx=tx.hash_hex, exc_info=True)
-            return False
+        self.consensus_algorithm.update(tx)
 
         assert self.verification_service.validate_full(
             tx,

tests/consensus/test_consensus.py

@@ -1,4 +1,5 @@
-from unittest.mock import MagicMock
+import os
+from unittest.mock import MagicMock, patch
 
 from hathor.conf import HathorSettings
 from hathor.transaction.storage import TransactionMemoryStorage
@@ -25,7 +26,7 @@ def setUp(self):
         self.genesis_blocks = [tx for tx in self.genesis if tx.is_block]
         self.genesis_txs = [tx for tx in self.genesis if not tx.is_block]
 
-    def test_unhandled_exception(self):
+    def test_unhandled_exception(self) -> None:
         manager = self.create_peer('testnet', tx_storage=self.tx_storage)
 
         # Mine a few blocks in a row with no transaction but the genesis
@@ -41,12 +42,9 @@ class MyError(Exception):
 
         manager.consensus_algorithm._unsafe_update = MagicMock(side_effect=MyError)
 
-        with self.assertRaises(MyError):
+        with patch.object(os, '_exit') as exit_mock:
             manager.propagate_tx(tx, fails_silently=False)
-
-        tx2 = manager.tx_storage.get_transaction(tx.hash)
-        meta2 = tx2.get_metadata()
-        self.assertEqual({settings.CONSENSUS_FAIL_ID}, meta2.voided_by)
+            exit_mock.assert_called_once_with(-1)
 
     def test_revert_block_high_weight(self):
         """ A conflict transaction will be propagated. At first, it will be voided.