refactor(metadata): pre-refactors for migrating feature states to static metadata [part 1/2]

Author: glevco

hathor/builder/builder.py

@@ -435,6 +435,7 @@ def _get_or_create_indexes_manager(self) -> IndexesManager:
 
     def _get_or_create_tx_storage(self) -> TransactionStorage:
         indexes = self._get_or_create_indexes_manager()
+        settings = self._get_or_create_settings()
 
         if self._tx_storage is not None:
             # If a tx storage is provided, set the indexes manager to it.
@@ -446,11 +447,11 @@ def _get_or_create_tx_storage(self) -> TransactionStorage:
             store_indexes = None
 
         if self._storage_type == StorageType.MEMORY:
-            self._tx_storage = TransactionMemoryStorage(indexes=store_indexes)
+            self._tx_storage = TransactionMemoryStorage(indexes=store_indexes, settings=settings)
 
         elif self._storage_type == StorageType.ROCKSDB:
             rocksdb_storage = self._get_or_create_rocksdb_storage()
-            self._tx_storage = TransactionRocksDBStorage(rocksdb_storage, indexes=store_indexes)
+            self._tx_storage = TransactionRocksDBStorage(rocksdb_storage, indexes=store_indexes, settings=settings)
 
         else:
             raise NotImplementedError
@@ -460,7 +461,13 @@ def _get_or_create_tx_storage(self) -> TransactionStorage:
             kwargs: dict[str, Any] = {}
             if self._tx_storage_cache_capacity is not None:
                 kwargs['capacity'] = self._tx_storage_cache_capacity
-            self._tx_storage = TransactionCacheStorage(self._tx_storage, reactor, indexes=indexes, **kwargs)
+            self._tx_storage = TransactionCacheStorage(
+                self._tx_storage,
+                reactor,
+                indexes=indexes,
+                settings=settings,
+                **kwargs
+            )
 
         return self._tx_storage
 
@@ -504,10 +511,7 @@ def _get_or_create_feature_service(self) -> FeatureService:
         if self._feature_service is None:
             settings = self._get_or_create_settings()
             tx_storage = self._get_or_create_tx_storage()
-            self._feature_service = FeatureService(
-                feature_settings=settings.FEATURE_ACTIVATION,
-                tx_storage=tx_storage
-            )
+            self._feature_service = FeatureService(settings=settings, tx_storage=tx_storage)
 
         return self._feature_service
 
@@ -518,7 +522,7 @@ def _get_or_create_bit_signaling_service(self) -> BitSignalingService:
             feature_service = self._get_or_create_feature_service()
             feature_storage = self._get_or_create_feature_storage()
             self._bit_signaling_service = BitSignalingService(
-                feature_settings=settings.FEATURE_ACTIVATION,
+                settings=settings,
                 feature_service=feature_service,
                 tx_storage=tx_storage,
                 support_features=self._support_features,

hathor/builder/cli_builder.py

@@ -145,7 +145,7 @@ def create_manager(self, reactor: Reactor) -> HathorManager:
             else:
                 indexes = RocksDBIndexesManager(self.rocksdb_storage)
 
-            kwargs = {}
+            kwargs: dict[str, Any] = {}
             if not self._args.cache:
                 # We should only pass indexes if cache is disabled. Otherwise,
                 # only TransactionCacheStorage should have indexes.
@@ -257,13 +257,10 @@ def create_manager(self, reactor: Reactor) -> HathorManager:
             self.log.info('--x-enable-event-queue flag provided. '
                           'The events detected by the full node will be stored and can be retrieved by clients')
 
-        self.feature_service = FeatureService(
-            feature_settings=settings.FEATURE_ACTIVATION,
-            tx_storage=tx_storage
-        )
+        self.feature_service = FeatureService(settings=settings, tx_storage=tx_storage)
 
         bit_signaling_service = BitSignalingService(
-            feature_settings=settings.FEATURE_ACTIVATION,
+            settings=settings,
             feature_service=self.feature_service,
             tx_storage=tx_storage,
             support_features=self._args.signal_support,

hathor/builder/resources_builder.py

@@ -206,7 +206,7 @@ def create_resources(self) -> server.Site:
             (
                 b'feature',
                 FeatureResource(
-                    feature_settings=settings.FEATURE_ACTIVATION,
+                    settings=settings,
                     feature_service=self._feature_service,
                     tx_storage=self.manager.tx_storage
                 ),

hathor/feature_activation/bit_signaling_service.py

@@ -14,11 +14,11 @@
 
 from structlog import get_logger
 
+from hathor.conf.settings import HathorSettings
 from hathor.feature_activation.feature import Feature
 from hathor.feature_activation.feature_service import FeatureService
 from hathor.feature_activation.model.criteria import Criteria
 from hathor.feature_activation.model.feature_state import FeatureState
-from hathor.feature_activation.settings import Settings as FeatureSettings
 from hathor.feature_activation.storage.feature_activation_storage import FeatureActivationStorage
 from hathor.transaction import Block
 from hathor.transaction.storage import TransactionStorage
@@ -29,7 +29,7 @@
 class BitSignalingService:
     __slots__ = (
         '_log',
-        '_feature_settings',
+        '_settings',
         '_feature_service',
         '_tx_storage',
         '_support_features',
@@ -40,15 +40,15 @@ class BitSignalingService:
     def __init__(
         self,
         *,
-        feature_settings: FeatureSettings,
+        settings: HathorSettings,
         feature_service: FeatureService,
         tx_storage: TransactionStorage,
         support_features: set[Feature],
         not_support_features: set[Feature],
         feature_storage: FeatureActivationStorage | None,
     ) -> None:
         self._log = logger.new()
-        self._feature_settings = feature_settings
+        self._settings = settings
         self._feature_service = feature_service
         self._tx_storage = tx_storage
         self._support_features = support_features
@@ -163,14 +163,14 @@ def _log_signal_bits(self, feature: Feature, enable_bit: bool, support: bool, no
 
     def _get_signaling_features(self, block: Block) -> dict[Feature, Criteria]:
         """Given a specific block, return all features that are in a signaling state for that block."""
-        feature_descriptions = self._feature_service.get_bits_description(block=block)
+        feature_infos = self._feature_service.get_feature_infos(block=block)
         signaling_features = {
-            feature: description.criteria
-            for feature, description in feature_descriptions.items()
-            if description.state in FeatureState.get_signaling_states()
+            feature: feature_info.criteria
+            for feature, feature_info in feature_infos.items()
+            if feature_info.state in FeatureState.get_signaling_states()
         }
 
-        assert len(signaling_features) <= self._feature_settings.max_signal_bits, (
+        assert len(signaling_features) <= self._settings.FEATURE_ACTIVATION.max_signal_bits, (
             'Invalid state. Signaling more features than the allowed maximum.'
         )
 

hathor/feature_activation/feature_service.py

@@ -15,10 +15,10 @@
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Optional, TypeAlias
 
+from hathor.conf.settings import HathorSettings
 from hathor.feature_activation.feature import Feature
-from hathor.feature_activation.model.feature_description import FeatureDescription
+from hathor.feature_activation.model.feature_info import FeatureInfo
 from hathor.feature_activation.model.feature_state import FeatureState
-from hathor.feature_activation.settings import Settings as FeatureSettings
 
 if TYPE_CHECKING:
     from hathor.feature_activation.bit_signaling_service import BitSignalingService
@@ -44,8 +44,8 @@ class BlockIsMissingSignal:
 class FeatureService:
     __slots__ = ('_feature_settings', '_tx_storage', 'bit_signaling_service')
 
-    def __init__(self, *, feature_settings: FeatureSettings, tx_storage: 'TransactionStorage') -> None:
-        self._feature_settings = feature_settings
+    def __init__(self, *, settings: HathorSettings, tx_storage: 'TransactionStorage') -> None:
+        self._feature_settings = settings.FEATURE_ACTIVATION
         self._tx_storage = tx_storage
         self.bit_signaling_service: Optional['BitSignalingService'] = None
 
@@ -64,11 +64,11 @@ def is_signaling_mandatory_features(self, block: 'Block') -> BlockSignalingState
         height = block.static_metadata.height
         offset_to_boundary = height % self._feature_settings.evaluation_interval
         remaining_blocks = self._feature_settings.evaluation_interval - offset_to_boundary - 1
-        descriptions = self.get_bits_description(block=block)
+        feature_infos = self.get_feature_infos(block=block)
 
         must_signal_features = (
-            feature for feature, description in descriptions.items()
-            if description.state is FeatureState.MUST_SIGNAL
+            feature for feature, feature_info in feature_infos.items()
+            if feature_info.state is FeatureState.MUST_SIGNAL
         )
 
         for feature in must_signal_features:
@@ -192,12 +192,12 @@ def _calculate_new_state(
         if previous_state is FeatureState.FAILED:
             return FeatureState.FAILED
 
-        raise ValueError(f'Unknown previous state: {previous_state}')
+        raise NotImplementedError(f'Unknown previous state: {previous_state}')
 
-    def get_bits_description(self, *, block: 'Block') -> dict[Feature, FeatureDescription]:
+    def get_feature_infos(self, *, block: 'Block') -> dict[Feature, FeatureInfo]:
         """Returns the criteria definition and feature state for all features at a certain block."""
         return {
-            feature: FeatureDescription(
+            feature: FeatureInfo(
                 criteria=criteria,
                 state=self.get_state(block=block, feature=feature)
             )
@@ -223,9 +223,11 @@ def _get_ancestor_at_height(self, *, block: 'Block', ancestor_height: int) -> 'B
         if parent_block.static_metadata.height == ancestor_height:
             return parent_block
 
-        if not parent_metadata.voided_by and (ancestor := self._tx_storage.get_block_by_height(ancestor_height)):
-            from hathor.transaction import Block
-            assert isinstance(ancestor, Block)
+        if not parent_metadata.voided_by:
+            ancestor = self._tx_storage.get_block_by_height(ancestor_height)
+            assert ancestor is not None, (
+                'it is guaranteed that the ancestor of a fully connected and non-voided block is in the height index'
+            )
             return ancestor
 
         return self._get_ancestor_iteratively(block=parent_block, ancestor_height=ancestor_height)

hathor/feature_activation/model/feature_description.py renamed to hathor/feature_activation/model/feature_info.py

@@ -18,7 +18,7 @@
 from hathor.feature_activation.model.feature_state import FeatureState
 
 
-class FeatureDescription(NamedTuple):
+class FeatureInfo(NamedTuple):
     """Represents all information related to one feature, that is, its criteria and state."""
     criteria: Criteria
     state: FeatureState

hathor/feature_activation/model/feature_state.py

@@ -12,10 +12,11 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from enum import Enum
+from enum import Enum, unique
 
 
-class FeatureState(Enum):
+@unique
+class FeatureState(str, Enum):
     """
     Possible states a feature can be in, for each block.
 
@@ -35,6 +36,10 @@ class FeatureState(Enum):
     ACTIVE = 'ACTIVE'
     FAILED = 'FAILED'
 
+    def is_active(self) -> bool:
+        """Return whether the state is active."""
+        return self is FeatureState.ACTIVE
+
     @staticmethod
     def get_signaling_states() -> set['FeatureState']:
         """

hathor/feature_activation/resources/feature.py

@@ -18,10 +18,10 @@
 
 from hathor.api_util import Resource, set_cors
 from hathor.cli.openapi_files.register import register_resource
+from hathor.conf.settings import HathorSettings
 from hathor.feature_activation.feature import Feature
 from hathor.feature_activation.feature_service import FeatureService
 from hathor.feature_activation.model.feature_state import FeatureState
-from hathor.feature_activation.settings import Settings as FeatureSettings
 from hathor.transaction import Block
 from hathor.transaction.storage import TransactionStorage
 from hathor.utils.api import ErrorResponse, QueryParams, Response
@@ -36,12 +36,12 @@ class FeatureResource(Resource):
     def __init__(
         self,
         *,
-        feature_settings: FeatureSettings,
+        settings: HathorSettings,
         feature_service: FeatureService,
         tx_storage: TransactionStorage
     ) -> None:
         super().__init__()
-        self._feature_settings = feature_settings
+        self._feature_settings = settings.FEATURE_ACTIVATION
         self._feature_service = feature_service
         self.tx_storage = tx_storage
 
@@ -68,17 +68,17 @@ def get_block_features(self, request: Request) -> bytes:
             return error.json_dumpb()
 
         signal_bits = []
-        feature_descriptions = self._feature_service.get_bits_description(block=block)
+        feature_infos = self._feature_service.get_feature_infos(block=block)
 
-        for feature, description in feature_descriptions.items():
-            if description.state not in FeatureState.get_signaling_states():
+        for feature, feature_info in feature_infos.items():
+            if feature_info.state not in FeatureState.get_signaling_states():
                 continue
 
             block_feature = GetBlockFeatureResponse(
-                bit=description.criteria.bit,
-                signal=block.get_feature_activation_bit_value(description.criteria.bit),
+                bit=feature_info.criteria.bit,
+                signal=block.get_feature_activation_bit_value(feature_info.criteria.bit),
                 feature=feature,
-                feature_state=description.state.name
+                feature_state=feature_info.state.name
             )
 
             signal_bits.append(block_feature)
@@ -90,10 +90,12 @@ def get_block_features(self, request: Request) -> bytes:
     def get_features(self) -> bytes:
         best_block = self.tx_storage.get_best_block()
         bit_counts = best_block.static_metadata.feature_activation_bit_counts
+        feature_infos = self._feature_service.get_feature_infos(block=best_block)
         features = []
 
-        for feature, criteria in self._feature_settings.features.items():
-            state = self._feature_service.get_state(block=best_block, feature=feature)
+        for feature, feature_info in feature_infos.items():
+            state = feature_info.state
+            criteria = feature_info.criteria
             threshold_count = criteria.get_threshold(self._feature_settings)
             threshold_percentage = threshold_count / self._feature_settings.evaluation_interval
             acceptance_percentage = None

hathor/transaction/__init__.py

@@ -19,6 +19,7 @@
     TxInput,
     TxOutput,
     TxVersion,
+    Vertex,
     sum_weights,
 )
 from hathor.transaction.block import Block
@@ -29,6 +30,7 @@
 __all__ = [
     'Transaction',
     'BitcoinAuxPow',
+    'Vertex',
     'BaseTransaction',
     'Block',
     'MergeMinedBlock',

hathor/transaction/static_metadata.py

@@ -12,9 +12,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-import dataclasses
 from abc import ABC
-from dataclasses import dataclass
 from itertools import chain, starmap, zip_longest
 from operator import add
 from typing import TYPE_CHECKING, Callable
@@ -25,15 +23,15 @@
 from hathor.feature_activation.feature import Feature
 from hathor.feature_activation.model.feature_state import FeatureState
 from hathor.types import VertexId
-from hathor.util import json_dumpb, json_loadb
+from hathor.util import json_loadb
+from hathor.utils.pydantic import BaseModel
 
 if TYPE_CHECKING:
     from hathor.transaction import BaseTransaction, Block, Transaction
     from hathor.transaction.storage import TransactionStorage
 
 
-@dataclass(slots=True, frozen=True, kw_only=True)
-class VertexStaticMetadata(ABC):
+class VertexStaticMetadata(ABC, BaseModel):
     """
     Static Metadata represents vertex attributes that are not intrinsic to the vertex data, but can be calculated from
     only the vertex itself and its dependencies, and whose values never change.
@@ -47,10 +45,6 @@ class VertexStaticMetadata(ABC):
     # metadata (that does not have this calculated, from a tx with a new format that does have this calculated)
     min_height: int
 
-    def to_bytes(self) -> bytes:
-        """Convert this static metadata instance to a json bytes representation."""
-        return json_dumpb(dataclasses.asdict(self))
-
     @classmethod
     def from_bytes(cls, data: bytes, *, target: 'BaseTransaction') -> 'VertexStaticMetadata':
         """Create a static metadata instance from a json bytes representation, with a known vertex type target."""
@@ -66,11 +60,10 @@ def from_bytes(cls, data: bytes, *, target: 'BaseTransaction') -> 'VertexStaticM
         raise NotImplementedError
 
 
-@dataclass(slots=True, frozen=True, kw_only=True)
 class BlockStaticMetadata(VertexStaticMetadata):
     height: int
 
-    # A list of feature activation bit counts. Must only be used by Blocks, is None otherwise.
+    # A list of feature activation bit counts.
     # Each list index corresponds to a bit position, and its respective value is the rolling count of active bits from
     # the previous boundary block up to this block, including it. LSB is on the left.
     feature_activation_bit_counts: list[int]
@@ -178,7 +171,6 @@ def _get_previous_feature_activation_bit_counts(
         return parent_block.static_metadata.feature_activation_bit_counts
 
 
-@dataclass(slots=True, frozen=True, kw_only=True)
 class TransactionStaticMetadata(VertexStaticMetadata):
     @classmethod
     def create_from_storage(cls, tx: 'Transaction', settings: HathorSettings, storage: 'TransactionStorage') -> Self: