refactor(storage): change storages to use injected settings

Author: glevco

hathor/builder/builder.py

@@ -435,6 +435,7 @@ def _get_or_create_indexes_manager(self) -> IndexesManager:
 
     def _get_or_create_tx_storage(self) -> TransactionStorage:
         indexes = self._get_or_create_indexes_manager()
+        settings = self._get_or_create_settings()
 
         if self._tx_storage is not None:
             # If a tx storage is provided, set the indexes manager to it.
@@ -446,11 +447,11 @@ def _get_or_create_tx_storage(self) -> TransactionStorage:
             store_indexes = None
 
         if self._storage_type == StorageType.MEMORY:
-            self._tx_storage = TransactionMemoryStorage(indexes=store_indexes)
+            self._tx_storage = TransactionMemoryStorage(indexes=store_indexes, settings=settings)
 
         elif self._storage_type == StorageType.ROCKSDB:
             rocksdb_storage = self._get_or_create_rocksdb_storage()
-            self._tx_storage = TransactionRocksDBStorage(rocksdb_storage, indexes=store_indexes)
+            self._tx_storage = TransactionRocksDBStorage(rocksdb_storage, indexes=store_indexes, settings=settings)
 
         else:
             raise NotImplementedError

hathor/builder/cli_builder.py

@@ -127,7 +127,7 @@ def create_manager(self, reactor: Reactor) -> HathorManager:
             self.check_or_raise(not self._args.data, '--data should not be used with --memory-storage')
             # if using MemoryStorage, no need to have cache
             indexes = MemoryIndexesManager()
-            tx_storage = TransactionMemoryStorage(indexes)
+            tx_storage = TransactionMemoryStorage(indexes, settings=settings)
             event_storage = EventMemoryStorage()
             self.check_or_raise(not self._args.x_rocksdb_indexes, 'RocksDB indexes require RocksDB data')
             self.log.info('with storage', storage_class=type(tx_storage).__name__)
@@ -150,14 +150,14 @@ def create_manager(self, reactor: Reactor) -> HathorManager:
                 # We should only pass indexes if cache is disabled. Otherwise,
                 # only TransactionCacheStorage should have indexes.
                 kwargs['indexes'] = indexes
-            tx_storage = TransactionRocksDBStorage(self.rocksdb_storage, **kwargs)
+            tx_storage = TransactionRocksDBStorage(self.rocksdb_storage, settings=settings, **kwargs)
             event_storage = EventRocksDBStorage(self.rocksdb_storage)
             feature_storage = FeatureActivationStorage(settings=settings, rocksdb_storage=self.rocksdb_storage)
 
         self.log.info('with storage', storage_class=type(tx_storage).__name__, path=self._args.data)
         if self._args.cache:
             self.check_or_raise(not self._args.memory_storage, '--cache should not be used with --memory-storage')
-            tx_storage = TransactionCacheStorage(tx_storage, reactor, indexes=indexes)
+            tx_storage = TransactionCacheStorage(tx_storage, reactor, indexes=indexes, settings=settings)
             if self._args.cache_size:
                 tx_storage.capacity = self._args.cache_size
             if self._args.cache_interval:

hathor/transaction/storage/cache_storage.py

@@ -17,6 +17,7 @@
 
 from twisted.internet import threads
 
+from hathor.conf.settings import HathorSettings
 from hathor.indexes import IndexesManager
 from hathor.reactor import ReactorProtocol as Reactor
 from hathor.transaction import BaseTransaction
@@ -32,8 +33,17 @@ class TransactionCacheStorage(BaseTransactionStorage):
     cache: OrderedDict[bytes, BaseTransaction]
     dirty_txs: set[bytes]
 
-    def __init__(self, store: 'BaseTransactionStorage', reactor: Reactor, interval: int = 5,
-                 capacity: int = 10000, *, indexes: Optional[IndexesManager], _clone_if_needed: bool = False):
+    def __init__(
+        self,
+        store: 'BaseTransactionStorage',
+        reactor: Reactor,
+        interval: int = 5,
+        capacity: int = 10000,
+        *,
+        settings: HathorSettings,
+        indexes: Optional[IndexesManager],
+        _clone_if_needed: bool = False,
+    ) -> None:
         """
         :param store: a subclass of BaseTransactionStorage
         :type store: :py:class:`hathor.transaction.storage.BaseTransactionStorage`
@@ -68,7 +78,7 @@ def __init__(self, store: 'BaseTransactionStorage', reactor: Reactor, interval:
 
         # we need to use only one weakref dict, so we must first initialize super, and then
         # attribute the same weakref for both.
-        super().__init__(indexes=indexes)
+        super().__init__(indexes=indexes, settings=settings)
         self._tx_weakref = store._tx_weakref
         # XXX: just to make sure this isn't being used anywhere, setters/getters should be used instead
         del self._allow_scope

hathor/transaction/storage/memory_storage.py

@@ -14,6 +14,7 @@
 
 from typing import Any, Iterator, Optional, TypeVar
 
+from hathor.conf.settings import HathorSettings
 from hathor.indexes import IndexesManager
 from hathor.transaction.storage.exceptions import TransactionDoesNotExist
 from hathor.transaction.storage.migrations import MigrationState
@@ -25,7 +26,13 @@
 
 
 class TransactionMemoryStorage(BaseTransactionStorage):
-    def __init__(self, indexes: Optional[IndexesManager] = None, *, _clone_if_needed: bool = False) -> None:
+    def __init__(
+        self,
+        indexes: Optional[IndexesManager] = None,
+        *,
+        settings: HathorSettings,
+        _clone_if_needed: bool = False,
+    ) -> None:
         """
         :param _clone_if_needed: *private parameter*, defaults to True, controls whether to clone
                                  transaction/blocks/metadata when returning those objects.
@@ -36,7 +43,7 @@ def __init__(self, indexes: Optional[IndexesManager] = None, *, _clone_if_needed
         # Store custom key/value attributes
         self.attributes: dict[str, Any] = {}
         self._clone_if_needed = _clone_if_needed
-        super().__init__(indexes=indexes)
+        super().__init__(indexes=indexes, settings=settings)
 
     def _check_and_set_network(self) -> None:
         # XXX: does not apply to memory storage, can safely be ignored

hathor/transaction/storage/rocksdb_storage.py

@@ -16,6 +16,7 @@
 
 from structlog import get_logger
 
+from hathor.conf.settings import HathorSettings
 from hathor.indexes import IndexesManager
 from hathor.storage import RocksDBStorage
 from hathor.transaction.storage.exceptions import TransactionDoesNotExist
@@ -43,15 +44,21 @@ class TransactionRocksDBStorage(BaseTransactionStorage):
     It uses Protobuf serialization internally.
     """
 
-    def __init__(self, rocksdb_storage: RocksDBStorage, indexes: Optional[IndexesManager] = None):
+    def __init__(
+        self,
+        rocksdb_storage: RocksDBStorage,
+        indexes: Optional[IndexesManager] = None,
+        *,
+        settings: HathorSettings,
+    ) -> None:
         self._cf_tx = rocksdb_storage.get_or_create_column_family(_CF_NAME_TX)
         self._cf_meta = rocksdb_storage.get_or_create_column_family(_CF_NAME_META)
         self._cf_attr = rocksdb_storage.get_or_create_column_family(_CF_NAME_ATTR)
         self._cf_migrations = rocksdb_storage.get_or_create_column_family(_CF_NAME_MIGRATIONS)
 
         self._rocksdb_storage = rocksdb_storage
         self._db = rocksdb_storage.get_db()
-        super().__init__(indexes=indexes)
+        super().__init__(indexes=indexes, settings=settings)
 
     def _load_from_bytes(self, tx_data: bytes, meta_data: bytes) -> 'BaseTransaction':
         from hathor.transaction.base_transaction import tx_or_block_from_bytes

hathor/transaction/storage/transaction_storage.py

@@ -23,7 +23,7 @@
 from intervaltree.interval import Interval
 from structlog import get_logger
 
-from hathor.conf.get_settings import get_global_settings
+from hathor.conf.settings import HathorSettings
 from hathor.execution_manager import ExecutionManager
 from hathor.indexes import IndexesManager
 from hathor.indexes.height_index import HeightInfo
@@ -104,8 +104,8 @@ class TransactionStorage(ABC):
 
     _migrations: list[BaseMigration]
 
-    def __init__(self) -> None:
-        self._settings = get_global_settings()
+    def __init__(self, *, settings: HathorSettings) -> None:
+        self._settings = settings
         # Weakref is used to guarantee that there is only one instance of each transaction in memory.
         self._tx_weakref: WeakValueDictionary[bytes, BaseTransaction] = WeakValueDictionary()
         self._tx_weakref_disabled: bool = False
@@ -1165,8 +1165,14 @@ def get_block(self, block_id: VertexId) -> Block:
 class BaseTransactionStorage(TransactionStorage):
     indexes: Optional[IndexesManager]
 
-    def __init__(self, indexes: Optional[IndexesManager] = None, pubsub: Optional[Any] = None) -> None:
-        super().__init__()
+    def __init__(
+        self,
+        indexes: Optional[IndexesManager] = None,
+        pubsub: Optional[Any] = None,
+        *,
+        settings: HathorSettings,
+    ) -> None:
+        super().__init__(settings=settings)
 
         # Pubsub is used to publish tx voided and winner but it's optional
         self.pubsub = pubsub

tests/consensus/test_consensus.py

@@ -13,7 +13,7 @@ class BaseConsensusTestCase(unittest.TestCase):
 
     def setUp(self) -> None:
         super().setUp()
-        self.tx_storage = TransactionMemoryStorage()
+        self.tx_storage = TransactionMemoryStorage(settings=self._settings)
         self.genesis = self.tx_storage.get_all_genesis()
         self.genesis_blocks = [tx for tx in self.genesis if tx.is_block]
         self.genesis_txs = [tx for tx in self.genesis if not tx.is_block]

tests/tx/test_genesis.py

@@ -34,7 +34,7 @@ def setUp(self) -> None:
         self._daa = DifficultyAdjustmentAlgorithm(settings=self._settings)
         verifiers = VertexVerifiers.create_defaults(settings=self._settings, daa=self._daa, feature_service=Mock())
         self._verification_service = VerificationService(settings=self._settings, verifiers=verifiers)
-        self.storage = TransactionMemoryStorage()
+        self.storage = TransactionMemoryStorage(settings=settings)
 
     def test_pow(self):
         verifier = VertexVerifier(settings=self._settings)