feat(debug): make it possible to connect a remote ipython shell

Author: jansegre

hathor/cli/run_node.py

@@ -34,7 +34,8 @@ class RunNode:
         ('--x-sync-bridge', lambda args: bool(args.x_sync_bridge)),
         ('--x-sync-v2-only', lambda args: bool(args.x_sync_v2_only)),
         ('--x-enable-event-queue', lambda args: bool(args.x_enable_event_queue)),
-        ('--x-asyncio-reactor', lambda args: bool(args.x_asyncio_reactor))
+        ('--x-asyncio-reactor', lambda args: bool(args.x_asyncio_reactor)),
+        ('--x-ipython-kernel', lambda args: bool(args.x_ipython_kernel)),
     ]
 
     @classmethod
@@ -120,6 +121,9 @@ def create_parser(cls) -> ArgumentParser:
                             help=f'Signal not support for a feature. One of {possible_features}')
         parser.add_argument('--x-asyncio-reactor', action='store_true',
                             help='Use asyncio reactor instead of Twisted\'s default.')
+        # XXX: this is temporary, should be added as a sysctl instead before merging
+        parser.add_argument('--x-ipython-kernel', action='store_true',
+                            help='Launch embedded IPython kernel for remote debugging')
         return parser
 
     def prepare(self, *, register_resources: bool = True) -> None:
@@ -227,13 +231,29 @@ def register_signal_handlers(self) -> None:
         if sigusr1 is not None:
             # USR1 is available in this OS.
             signal.signal(sigusr1, self.signal_usr1_handler)
+        sigquit = getattr(signal, 'SIGQUIT', None)
+        if sigquit is not None:
+            # SIGQUIT is available in this OS.
+            signal.signal(sigquit, self.signal_quit_handler)
 
     def signal_usr1_handler(self, sig: int, frame: Any) -> None:
         """Called when USR1 signal is received."""
         self.log.warn('USR1 received. Killing all connections...')
         if self.manager and self.manager.connections:
             self.manager.connections.disconnect_all_peers(force=True)
 
+    def signal_quit_handler(self, sig: int, frame: Any) -> None:
+        self.log.warn('QUIT received. Exiting...')
+        if self.manager.is_started:
+            self.log.info('stop manager')
+            self.manager.stop()
+        if self.reactor.running:
+            self.log.info('stop reactor')
+            self.reactor.stop()
+        else:
+            self.log.warn('reactor already stopped, crashing it')
+            self.reactor.crash()
+
     def check_unsafe_arguments(self) -> None:
         unsafe_args_found = []
         for arg_cmdline, arg_test_fn in self.UNSAFE_ARGUMENTS:
@@ -415,7 +435,15 @@ def init_sysctl(self, description: str, sysctl_init_file: Optional[str] = None)
     def parse_args(self, argv: list[str]) -> Namespace:
         return self.parser.parse_args(argv)
 
+    def start_ipykernel(self) -> None:
+        os.environ['PYDEVD_DISABLE_FILE_VALIDATION'] = '1'  # silence ipykernel warning that does not apply
+        from hathor.ipykernel import embed_kernel
+        embed_kernel(self.manager, runtime_dir=self._args.data, extra_ns=dict(run_node=self))
+
     def run(self) -> None:
+        if self._args.x_ipython_kernel:
+            assert self._args.x_asyncio_reactor
+            self.start_ipykernel()
         self.reactor.run()
 
 

hathor/cli/run_node_args.py

@@ -73,3 +73,4 @@ class RunNodeArgs(BaseModel, extra=Extra.allow):
     signal_support: set[Feature]
     signal_not_support: set[Feature]
     x_asyncio_reactor: bool
+    x_ipython_kernel: bool

hathor/cli/util.py

@@ -206,18 +206,25 @@ def setup_logging(
                     'level': 'INFO' if debug else 'WARN',
                     'propagate': False,
                 },
-                '': {
+                'tornado': {
                     'handlers': handlers,
-                    'level': 'DEBUG' if debug else 'INFO',
+                    'level': 'INFO' if debug else 'WARN',
+                    'propagate': False,
                 },
                 'hathor.p2p.sync_v1': {
                     'handlers': handlers,
                     'level': 'DEBUG' if debug_sync else 'INFO',
+                    'propagate': False,
                 },
                 'hathor.p2p.sync_v2': {
                     'handlers': handlers,
                     'level': 'DEBUG' if debug_sync else 'INFO',
-                }
+                    'propagate': False,
+                },
+                '': {
+                    'handlers': handlers,
+                    'level': 'DEBUG' if debug else 'INFO',
+                },
             }
     })
 

hathor/ipykernel.py

@@ -0,0 +1,64 @@
+# Copyright 2024 Hathor Labs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from logging import getLogger
+from typing import TYPE_CHECKING, Any, Optional
+
+from ipykernel.kernelapp import IPKernelApp as OriginalIPKernelApp
+
+if TYPE_CHECKING:
+    from hathor.manager import HathorManager
+
+
+class IPKernelApp(OriginalIPKernelApp):
+    def __init__(self, runtime_dir: Optional[str] = None):
+        super().__init__()
+        if runtime_dir is not None:
+            self.connection_dir = runtime_dir
+        self.connection_file = 'kernel.json'
+        self.logging_config: dict[str, Any] = {}  # empty out logging config
+        self.default_extensions: list[str] = []  # disable default extensions
+        self.no_stderr = True  # disable forwarding of stderr (because we use it for logging)
+        self.log = getLogger('hathor.ipykernel')
+
+    # ignore registering of signals
+    def init_signal(self) -> None:
+        pass
+
+    # custom logging of connection file
+    def log_connection_info(self) -> None:
+        self.log.info(f'ipykernel connection enabled at {self.abs_connection_file}')
+
+    # disable original logging setup
+    def get_default_logging_config(self) -> dict[str, Any]:
+        return {"version": 1, "disable_existing_loggers": False}
+
+    def configure_tornado_logger(self) -> None:
+        pass
+
+    def start(self):
+        self.kernel.start()
+
+
+def embed_kernel(manager: 'HathorManager', *,
+                 runtime_dir: Optional[str] = None, extra_ns: dict[str, Any] = {}) -> None:
+    # get the app if it exists, or set it up if it doesn't
+    if IPKernelApp.initialized():
+        app = IPKernelApp.instance()
+    else:
+        app = IPKernelApp.instance(runtime_dir=runtime_dir)
+        app.initialize([])
+    app.kernel.user_ns = dict(manager=manager) | extra_ns
+    app.shell.set_completer_frame()
+    app.start()

hathor/manager.py

@@ -140,7 +140,7 @@ def __init__(self,
         self.reactor = reactor
         add_system_event_trigger = getattr(self.reactor, 'addSystemEventTrigger', None)
         if add_system_event_trigger is not None:
-            add_system_event_trigger('after', 'shutdown', self.stop)
+            add_system_event_trigger('after', 'shutdown', self.try_stop)
 
         self.state: Optional[HathorManager.NodeState] = None
         self.profiler: Optional[Any] = None
@@ -315,6 +315,10 @@ def start(self) -> None:
         # Start running
         self.tx_storage.start_running_manager()
 
+    def try_stop(self) -> None:
+        if self.is_started:
+            self.stop()
+
     def stop(self) -> Deferred:
         if not self.is_started:
             raise Exception('HathorManager is already stopped')