diff --git a/hathor/builder/builder.py b/hathor/builder/builder.py index 6fe3c1a91..b5cb7a0c0 100644 --- a/hathor/builder/builder.py +++ b/hathor/builder/builder.py @@ -16,6 +16,7 @@ from typing import Any, Callable, NamedTuple, Optional, TypeAlias from structlog import get_logger +from typing_extensions import assert_never from hathor.checkpoint import Checkpoint from hathor.conf.get_settings import get_global_settings @@ -29,6 +30,7 @@ from hathor.feature_activation.bit_signaling_service import BitSignalingService from hathor.feature_activation.feature import Feature from hathor.feature_activation.feature_service import FeatureService +from hathor.feature_activation.storage.feature_activation_storage import FeatureActivationStorage from hathor.indexes import IndexesManager, MemoryIndexesManager, RocksDBIndexesManager from hathor.manager import HathorManager from hathor.mining.cpu_mining_service import CpuMiningService @@ -473,12 +475,14 @@ def _get_or_create_bit_signaling_service(self) -> BitSignalingService: settings = self._get_or_create_settings() tx_storage = self._get_or_create_tx_storage() feature_service = self._get_or_create_feature_service() + feature_storage = self._get_or_create_feature_storage() self._bit_signaling_service = BitSignalingService( feature_settings=settings.FEATURE_ACTIVATION, feature_service=feature_service, tx_storage=tx_storage, support_features=self._support_features, not_support_features=self._not_support_features, + feature_storage=feature_storage, ) return self._bit_signaling_service @@ -490,6 +494,15 @@ def _get_or_create_verification_service(self) -> VerificationService: return self._verification_service + def _get_or_create_feature_storage(self) -> FeatureActivationStorage | None: + match self._storage_type: + case StorageType.MEMORY: return None + case StorageType.ROCKSDB: return FeatureActivationStorage( + settings=self._get_or_create_settings(), + rocksdb_storage=self._get_or_create_rocksdb_storage() + ) + case _: assert_never(self._storage_type) + def _get_or_create_vertex_verifiers(self) -> VertexVerifiers: if self._vertex_verifiers is None: settings = self._get_or_create_settings() diff --git a/hathor/builder/cli_builder.py b/hathor/builder/cli_builder.py index 0ab658a25..6c106c8e4 100644 --- a/hathor/builder/cli_builder.py +++ b/hathor/builder/cli_builder.py @@ -30,6 +30,7 @@ from hathor.execution_manager import ExecutionManager from hathor.feature_activation.bit_signaling_service import BitSignalingService from hathor.feature_activation.feature_service import FeatureService +from hathor.feature_activation.storage.feature_activation_storage import FeatureActivationStorage from hathor.indexes import IndexesManager, MemoryIndexesManager, RocksDBIndexesManager from hathor.manager import HathorManager from hathor.mining.cpu_mining_service import CpuMiningService @@ -120,6 +121,7 @@ def create_manager(self, reactor: Reactor) -> HathorManager: tx_storage: TransactionStorage event_storage: EventStorage indexes: IndexesManager + feature_storage: FeatureActivationStorage | None = None self.rocksdb_storage: Optional[RocksDBStorage] = None self.event_ws_factory: Optional[EventWebsocketFactory] = None @@ -152,6 +154,7 @@ def create_manager(self, reactor: Reactor) -> HathorManager: kwargs['indexes'] = indexes tx_storage = TransactionRocksDBStorage(self.rocksdb_storage, **kwargs) event_storage = EventRocksDBStorage(self.rocksdb_storage) + feature_storage = FeatureActivationStorage(settings=settings, rocksdb_storage=self.rocksdb_storage) self.log.info('with storage', storage_class=type(tx_storage).__name__, path=self._args.data) if self._args.cache: @@ -260,7 +263,8 @@ def create_manager(self, reactor: Reactor) -> HathorManager: feature_service=self.feature_service, tx_storage=tx_storage, support_features=self._args.signal_support, - not_support_features=self._args.signal_not_support + not_support_features=self._args.signal_not_support, + feature_storage=feature_storage, ) test_mode = TestMode.DISABLED diff --git a/hathor/cli/main.py b/hathor/cli/main.py index a9c287cbf..a1ab960d2 100644 --- a/hathor/cli/main.py +++ b/hathor/cli/main.py @@ -49,6 +49,7 @@ def __init__(self) -> None: quick_test, replay_logs, reset_event_queue, + reset_feature_settings, run_node, shell, stratum_mining, @@ -81,6 +82,8 @@ def __init__(self) -> None: self.add_cmd('oracle', 'oracle-encode-data', oracle_encode_data, 'Encode data and sign it with a private key') self.add_cmd('events', 'reset-event-queue', reset_event_queue, 'Delete all events and related data from the ' 'database') + self.add_cmd('features', 'reset-feature-settings', reset_feature_settings, 'Delete existing Feature ' + 'Activation settings from the database') self.add_cmd('dev', 'shell', shell, 'Run a Python shell') self.add_cmd('dev', 'quick_test', quick_test, 'Similar to run_node but will quit after receiving a tx') self.add_cmd('dev', 'generate_nginx_config', nginx_config, 'Generate nginx config from OpenAPI json') diff --git a/hathor/cli/reset_feature_settings.py b/hathor/cli/reset_feature_settings.py new file mode 100644 index 000000000..a4c8ea9e0 --- /dev/null +++ b/hathor/cli/reset_feature_settings.py @@ -0,0 +1,49 @@ +# Copyright 2023 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from argparse import ArgumentParser, Namespace + +from structlog import get_logger + +logger = get_logger() + + +def create_parser() -> ArgumentParser: + from hathor.cli.util import create_parser + + parser = create_parser() + parser.add_argument('--data', help='Data directory') + + return parser + + +def execute(args: Namespace) -> None: + from hathor.conf.get_settings import get_global_settings + from hathor.feature_activation.storage.feature_activation_storage import FeatureActivationStorage + from hathor.storage import RocksDBStorage + + assert args.data is not None, '--data is required' + + rocksdb_storage = RocksDBStorage(path=args.data) + feature_storage = FeatureActivationStorage(settings=get_global_settings(), rocksdb_storage=rocksdb_storage) + + logger.info('removing feature activation settings...') + feature_storage.reset_settings() + logger.info('reset complete') + + +def main(): + parser = create_parser() + args = parser.parse_args() + execute(args) diff --git a/hathor/feature_activation/bit_signaling_service.py b/hathor/feature_activation/bit_signaling_service.py index a8f7f09a4..6585086b4 100644 --- a/hathor/feature_activation/bit_signaling_service.py +++ b/hathor/feature_activation/bit_signaling_service.py @@ -19,6 +19,7 @@ from hathor.feature_activation.model.criteria import Criteria from hathor.feature_activation.model.feature_state import FeatureState from hathor.feature_activation.settings import Settings as FeatureSettings +from hathor.feature_activation.storage.feature_activation_storage import FeatureActivationStorage from hathor.transaction import Block from hathor.transaction.storage import TransactionStorage @@ -32,7 +33,8 @@ class BitSignalingService: '_feature_service', '_tx_storage', '_support_features', - '_not_support_features' + '_not_support_features', + '_feature_storage', ) def __init__( @@ -42,7 +44,8 @@ def __init__( feature_service: FeatureService, tx_storage: TransactionStorage, support_features: set[Feature], - not_support_features: set[Feature] + not_support_features: set[Feature], + feature_storage: FeatureActivationStorage | None, ) -> None: self._log = logger.new() self._feature_settings = feature_settings @@ -50,6 +53,7 @@ def __init__( self._tx_storage = tx_storage self._support_features = support_features self._not_support_features = not_support_features + self._feature_storage = feature_storage self._validate_support_intersection() @@ -58,6 +62,9 @@ def start(self) -> None: Log information related to bit signaling. Must be called after the storage is ready and migrations have been applied. """ + if self._feature_storage: + self._feature_storage.validate_settings() + best_block = self._tx_storage.get_best_block() self._warn_non_signaling_features(best_block) diff --git a/hathor/feature_activation/feature.py b/hathor/feature_activation/feature.py index 56082def8..05b08226e 100644 --- a/hathor/feature_activation/feature.py +++ b/hathor/feature_activation/feature.py @@ -16,7 +16,7 @@ @unique -class Feature(Enum): +class Feature(str, Enum): """ An enum containing all features that participate in the feature activation process, past or future, activated or not, for all networks. Features should NOT be removed from this enum, to preserve history. Their values diff --git a/hathor/feature_activation/settings.py b/hathor/feature_activation/settings.py index aa4c119b4..3d36e052b 100644 --- a/hathor/feature_activation/settings.py +++ b/hathor/feature_activation/settings.py @@ -83,7 +83,7 @@ def _validate_conflicting_bits(cls, features: dict[Feature, Criteria]) -> dict[F first, second = overlap raise ValueError( f'At least one pair of Features have the same bit configured for an overlapping interval: ' - f'{first.feature} and {second.feature}' + f'{first.feature.value} and {second.feature.value}' ) return features diff --git a/hathor/feature_activation/storage/__init__.py b/hathor/feature_activation/storage/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/hathor/feature_activation/storage/feature_activation_storage.py b/hathor/feature_activation/storage/feature_activation_storage.py new file mode 100644 index 000000000..101f213dd --- /dev/null +++ b/hathor/feature_activation/storage/feature_activation_storage.py @@ -0,0 +1,101 @@ +# Copyright 2023 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from structlog import get_logger + +from hathor.conf.settings import HathorSettings +from hathor.exception import InitializationError +from hathor.feature_activation.feature import Feature +from hathor.feature_activation.model.criteria import Criteria +from hathor.feature_activation.settings import Settings as FeatureActivationSettings +from hathor.storage import RocksDBStorage + +_CF_NAME_META = b'feature-activation-metadata' +_KEY_SETTINGS = b'feature-activation-settings' + +logger = get_logger() + + +class FeatureActivationStorage: + __slots__ = ('_log', '_settings', '_db', '_cf_meta') + + def __init__(self, *, settings: HathorSettings, rocksdb_storage: RocksDBStorage) -> None: + self._log = logger.new() + self._settings = settings + self._db = rocksdb_storage.get_db() + self._cf_meta = rocksdb_storage.get_or_create_column_family(_CF_NAME_META) + + def reset_settings(self) -> None: + """Reset feature settings from the database.""" + self._db.delete((self._cf_meta, _KEY_SETTINGS)) + + def validate_settings(self) -> None: + """Validate new feature settings against the previous configuration from the database.""" + new_settings = self._settings.FEATURE_ACTIVATION + db_settings_bytes: bytes | None = self._db.get((self._cf_meta, _KEY_SETTINGS)) + + if not db_settings_bytes: + self._save_settings(new_settings) + return + + db_settings: FeatureActivationSettings = FeatureActivationSettings.parse_raw(db_settings_bytes) + db_basic_settings = db_settings.copy(deep=True, exclude={'features'}) + new_basic_settings = new_settings.copy(deep=True, exclude={'features'}) + + self._validate_basic_settings(db_basic_settings=db_basic_settings, new_basic_settings=new_basic_settings) + self._validate_features(db_features=db_settings.features, new_features=new_settings.features) + self._save_settings(new_settings) + + def _validate_basic_settings( + self, + *, + db_basic_settings: FeatureActivationSettings, + new_basic_settings: FeatureActivationSettings + ) -> None: + """Validate that the basic feature settings are the same.""" + if new_basic_settings != db_basic_settings: + self._log.error( + 'Feature Activation basic settings are incompatible with previous settings.', + previous_settings=db_basic_settings, new_settings=new_basic_settings + ) + raise InitializationError() + + def _validate_features( + self, + *, + db_features: dict[Feature, Criteria], + new_features: dict[Feature, Criteria] + ) -> None: + """Validate that all previous features exist and are the same.""" + for db_feature, db_criteria in db_features.items(): + new_criteria = new_features.get(db_feature) + + if not new_criteria: + self._log.error( + 'Configuration for existing feature missing in new settings.', + feature=db_feature, previous_features=db_features, new_features=new_features + ) + raise InitializationError() + + if new_criteria != db_criteria: + self._log.error( + 'Criteria for feature is different than previous settings.', + feature=db_feature, previous_criteria=db_criteria, new_criteria=new_criteria + ) + raise InitializationError() + + def _save_settings(self, settings: FeatureActivationSettings) -> None: + """Save feature settings to the database.""" + settings_bytes = settings.json_dumpb() + + self._db.put((self._cf_meta, _KEY_SETTINGS), settings_bytes) diff --git a/tests/feature_activation/test_bit_signaling_service.py b/tests/feature_activation/test_bit_signaling_service.py index f3b24e140..c6aab1c18 100644 --- a/tests/feature_activation/test_bit_signaling_service.py +++ b/tests/feature_activation/test_bit_signaling_service.py @@ -173,7 +173,8 @@ def _test_generate_signal_bits( feature_service=feature_service, tx_storage=Mock(), support_features=support_features, - not_support_features=not_support_features + not_support_features=not_support_features, + feature_storage=Mock(), ) return service.generate_signal_bits(block=Mock()) @@ -216,6 +217,7 @@ def test_support_intersection_validation( tx_storage=Mock(), support_features=support_features, not_support_features=not_support_features, + feature_storage=Mock(), ) message = str(e.value) @@ -270,6 +272,7 @@ def get_bits_description_mock(block): tx_storage=tx_storage, support_features=support_features, not_support_features=not_support_features, + feature_storage=Mock(), ) logger_mock = Mock() service._log = logger_mock diff --git a/tests/feature_activation/test_settings.py b/tests/feature_activation/test_settings.py index 04af34229..7159533a1 100644 --- a/tests/feature_activation/test_settings.py +++ b/tests/feature_activation/test_settings.py @@ -121,7 +121,7 @@ def test_conflicting_bits(features): errors = e.value.errors() assert errors[0]['msg'] == 'At least one pair of Features have the same bit configured for an overlapping ' \ - 'interval: Feature.NOP_FEATURE_1 and Feature.NOP_FEATURE_2' + 'interval: NOP_FEATURE_1 and NOP_FEATURE_2' @pytest.mark.parametrize( diff --git a/tests/others/test_metrics.py b/tests/others/test_metrics.py index 41c4ddb25..f799fc961 100644 --- a/tests/others/test_metrics.py +++ b/tests/others/test_metrics.py @@ -109,6 +109,7 @@ def _init_manager(): b'migrations': 0.0, b'event': 0.0, b'event-metadata': 0.0, + b'feature-activation-metadata': 0.0, }) manager.tx_storage.pre_init() @@ -161,6 +162,7 @@ def _init_manager(): b'migrations': 0.0, b'event': 0.0, b'event-metadata': 0.0, + b'feature-activation-metadata': 0.0, }) manager.tx_storage.pre_init()