refactor(mypy): add stricter rules to specific modules (#969)

Author: glevco

hathor/builder/builder.py

@@ -47,7 +47,8 @@
     TransactionStorage,
 )
 from hathor.util import Random, get_environment_info, not_none
-from hathor.verification.verification_service import VerificationService, VertexVerifiers
+from hathor.verification.verification_service import VerificationService
+from hathor.verification.vertex_verifiers import VertexVerifiers
 from hathor.wallet import BaseWallet, Wallet
 
 logger = get_logger()

hathor/cli/mining.py

@@ -139,7 +139,8 @@ def execute(args: Namespace) -> None:
 
             from hathor.conf.get_settings import get_global_settings
             from hathor.daa import DifficultyAdjustmentAlgorithm
-            from hathor.verification.verification_service import VerificationService, VertexVerifiers
+            from hathor.verification.verification_service import VerificationService
+            from hathor.verification.vertex_verifiers import VertexVerifiers
             settings = get_global_settings()
             daa = DifficultyAdjustmentAlgorithm(settings=settings)
             verifiers = VertexVerifiers.create_defaults(settings=settings, daa=daa, feature_service=Mock())

hathor/consensus/block_consensus.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 from itertools import chain
-from typing import TYPE_CHECKING, Iterable, Optional, cast
+from typing import TYPE_CHECKING, Any, Iterable, Optional, cast
 
 from structlog import get_logger
 
@@ -39,7 +39,7 @@ def __init__(self, context: 'ConsensusAlgorithmContext') -> None:
         self.context = context
 
     @classproperty
-    def log(cls):
+    def log(cls) -> Any:
         """ This is a workaround because of a bug on structlog (or abc).
 
         See: https://github.com/hynek/structlog/issues/229

hathor/consensus/transaction_consensus.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TYPE_CHECKING, Iterable, cast
+from typing import TYPE_CHECKING, Any, Iterable, cast
 
 from structlog import get_logger
 
@@ -38,7 +38,7 @@ def __init__(self, context: 'ConsensusAlgorithmContext') -> None:
         self.context = context
 
     @classproperty
-    def log(cls):
+    def log(cls) -> Any:
         """ This is a workaround because of a bug on structlog (or abc).
 
         See: https://github.com/hynek/structlog/issues/229

hathor/event/model/base_event.py

@@ -12,11 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Optional
+from typing import Any, Optional
 
 from pydantic import NonNegativeInt, validator
 
-from hathor.event.model.event_data import EventData
+from hathor.event.model.event_data import BaseEventData, EventData
 from hathor.event.model.event_type import EventType
 from hathor.pubsub import EventArguments
 from hathor.utils.pydantic import BaseModel
@@ -58,7 +58,7 @@ def from_event_arguments(
         )
 
     @validator('data')
-    def data_type_must_match_event_type(cls, v, values):
+    def data_type_must_match_event_type(cls, v: BaseEventData, values: dict[str, Any]) -> BaseEventData:
         event_type = EventType(values['type'])
         expected_data_type = event_type.data_type()
 

hathor/event/resources/event.py

@@ -16,6 +16,7 @@
 from typing import Optional
 
 from pydantic import Field, NonNegativeInt
+from twisted.web.http import Request
 
 from hathor.api_util import Resource, set_cors
 from hathor.cli.openapi_files.register import register_resource
@@ -35,7 +36,7 @@ def __init__(self, event_manager: Optional[EventManager]):
         super().__init__()
         self.event_manager = event_manager
 
-    def render_GET(self, request):
+    def render_GET(self, request: Request) -> bytes:
         request.setHeader(b'content-type', b'application/json; charset=utf-8')
         set_cors(request, 'GET')
 

hathor/event/websocket/protocol.py

@@ -12,13 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TYPE_CHECKING, Callable, Optional
+from typing import TYPE_CHECKING, Optional
 
 from autobahn.exception import Disconnected
 from autobahn.twisted.websocket import WebSocketServerProtocol
 from autobahn.websocket import ConnectionRequest
 from pydantic import ValidationError
 from structlog import get_logger
+from typing_extensions import assert_never
 
 from hathor.event.websocket.request import AckRequest, Request, RequestWrapper, StartStreamRequest, StopStreamRequest
 from hathor.event.websocket.response import EventResponse, InvalidRequestResponse, InvalidRequestType, Response
@@ -50,7 +51,7 @@ class EventWebsocketProtocol(WebSocketServerProtocol):
     # Whether the stream is enabled or not.
     _stream_is_active: bool = False
 
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__()
         self.log = logger.new()
 
@@ -102,18 +103,11 @@ def onMessage(self, payload: bytes, isBinary: bool) -> None:
 
     def _handle_request(self, request: Request) -> None:
         """Handles a request message according to its type."""
-        # This could be a pattern match in Python 3.10
-        request_type = type(request)
-        handlers: dict[type, Callable] = {
-            StartStreamRequest: self._handle_start_stream_request,
-            AckRequest: self._handle_ack_request,
-            StopStreamRequest: lambda _: self._handle_stop_stream_request()
-        }
-        handle_fn = handlers.get(request_type)
-
-        assert handle_fn is not None, f'cannot handle request of unknown type "{request_type}"'
-
-        handle_fn(request)
+        match request:
+            case StartStreamRequest(): self._handle_start_stream_request(request)
+            case AckRequest(): self._handle_ack_request(request)
+            case StopStreamRequest(): self._handle_stop_stream_request()
+            case _: assert_never(request)
 
     def _handle_start_stream_request(self, request: StartStreamRequest) -> None:
         """

hathor/event/websocket/request.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Annotated, Literal, Optional, Union
+from typing import Annotated, Literal, Optional
 
 from pydantic import Field, NonNegativeInt
 
@@ -54,7 +54,7 @@ class StopStreamRequest(BaseModel):
     type: Literal['STOP_STREAM']
 
 
-Request = Annotated[Union[StartStreamRequest, AckRequest, StopStreamRequest], Field(discriminator='type')]
+Request = Annotated[StartStreamRequest | AckRequest | StopStreamRequest, Field(discriminator='type')]
 
 
 class RequestWrapper(BaseModel):

hathor/sysctl/sysctl.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Any, Callable, Iterator, NamedTuple, Optional
+from typing import Any, Callable, Iterator, NamedTuple, Optional, ParamSpec, TypeVar
 
 from pydantic import validate_arguments
 from structlog import get_logger
@@ -21,16 +21,18 @@
 
 Getter = Callable[[], Any]
 Setter = Callable[..., None]
+P = ParamSpec('P')
+T = TypeVar('T')
 
 logger = get_logger()
 
 
-def signal_handler_safe(f):
+def signal_handler_safe(f: Callable[P, T]) -> Callable[P, T]:
     """Decorator to mark methods as signal handler safe.
 
     It should only be used if that method can be executed during a signal handling.
     Notice that a signal handling can pause the code execution at any point and the execution will resume after."""
-    f._signal_handler_safe = True
+    f._signal_handler_safe = True  # type: ignore[attr-defined]
     return f
 
 

hathor/types.py

@@ -12,13 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import TypeAlias
+
 # XXX There is a lot of refactor to be done before we can use `NewType`.
 #     So, let's skip using NewType until everything is refactored.
 
-VertexId = bytes        # NewType('TxId', bytes)
-Address = bytes         # NewType('Address', bytes)
-AddressB58 = str
-TxOutputScript = bytes  # NewType('TxOutputScript', bytes)
-Timestamp = int         # NewType('Timestamp', int)
-TokenUid = VertexId     # NewType('TokenUid', VertexId)
-Amount = int            # NewType('Amount', int)
+VertexId: TypeAlias = bytes        # NewType('TxId', bytes)
+Address: TypeAlias = bytes         # NewType('Address', bytes)
+AddressB58: TypeAlias = str
+TxOutputScript: TypeAlias = bytes  # NewType('TxOutputScript', bytes)
+Timestamp: TypeAlias = int         # NewType('Timestamp', int)
+TokenUid: TypeAlias = VertexId     # NewType('TokenUid', VertexId)
+Amount: TypeAlias = int            # NewType('Amount', int)

poetry.lock

@@ -38,8 +38,8 @@ hathor-cli = 'hathor.cli.main:main'
 [tool.poetry.dev-dependencies]
 flake8 = "~6.1.0"
 isort = {version = "~5.12.0", extras = ["colors"]}
-mypy = {version = "^1.5.1", markers = "implementation_name == 'cpython'"}
-mypy-zope = {version = "^1.0.1", markers = "implementation_name == 'cpython'"}
+mypy = {version = "^1.9.0", markers = "implementation_name == 'cpython'"}
+mypy-zope = {version = "^1.0.4", markers = "implementation_name == 'cpython'"}
 pytest = "~7.4.3"
 pytest-cov = "~4.1.0"
 flaky = "~3.7.0"
@@ -97,6 +97,8 @@ multi_line_output = 3
 pretty = true
 disallow_incomplete_defs = true
 no_implicit_optional = true
+extra_checks = true
+disallow_untyped_decorators = true
 warn_redundant_casts = true
 warn_unused_configs = true
 warn_unused_ignores = true
@@ -131,6 +133,27 @@ module = [
 ]
 ignore_missing_imports = true
 
+# This override enables stricter rules for some specific modules.
+# Currently, we have only two options from strict-mode that are disabled, but we have to opt-in instead of opt-out
+# because setting strict=true doesn't work for module-level settings.
+# Reference: https://mypy.readthedocs.io/en/stable/existing_code.html#introduce-stricter-options
+[[tool.mypy.overrides]]
+module = [
+    "hathor.consensus.*",
+    "hathor.feature_activation.*",
+    "hathor.event.*",
+    "hathor.verification.*"
+]
+strict_equality = true
+strict_concatenate = true
+check_untyped_defs = true
+disallow_any_generics = true
+disallow_untyped_defs = true
+no_implicit_reexport = true
+warn_return_any = true
+# disallow_subclassing_any = true
+# disallow_untyped_calls = true
+
 [tool.pydantic-mypy]
 init_typed = true
 init_forbid_extra = true

pyproject.toml

@@ -13,7 +13,7 @@
 
 class SysctlTest(unittest.TestCase):
     # We need this patch because pydantic.validate_arguments fails when it gets a mock function.
-    @patch('hathor.sysctl.sysctl.validate_arguments', new=lambda x: x)
+    @patch('hathor.sysctl.sysctl.validate_arguments', new=lambda x: x)  # type: ignore
     def setUp(self) -> None:
         super().setUp()