refactor(side-dag): refactor node processes management

Author: glevco

Makefile

@@ -1,4 +1,4 @@
-py_sources = hathor/ tests/
+py_sources = hathor/ tests/ extras/custom_tests/
 
 .PHONY: all
 all: check tests
@@ -49,8 +49,12 @@ tests-genesis:
 tests-ci:
 	pytest $(tests_ci)
 
+.PHONY: tests-custom
+tests-custom:
+	bash ./extras/custom_tests.sh
+
 .PHONY: tests
-tests: tests-cli tests-lib tests-genesis tests-ci
+tests: tests-cli tests-lib tests-genesis tests-custom tests-ci
 
 .PHONY: tests-full
 tests-full:
@@ -60,11 +64,11 @@ tests-full:
 
 .PHONY: mypy
 mypy:
-	mypy -p hathor -p tests
+	mypy -p hathor -p tests -p extras.custom_tests
 
 .PHONY: dmypy
 dmypy:
-	dmypy run --timeout 86400 -- -p hathor -p tests
+	dmypy run --timeout 86400 -- -p hathor -p tests -p extras.custom_tests
 
 .PHONY: flake8
 flake8:

extras/custom_tests.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Define colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+TESTS_DIR="extras/custom_tests"
+
+# List of test scripts to be executed
+tests=(
+	/side_dag/test_one_fails.py
+	/side_dag/test_both_fail.py
+)
+
+# Initialize a variable to track if any test fails
+any_test_failed=0
+
+# Loop over all tests
+for test in "${tests[@]}"; do
+  echo -e "${BLUE}Testing $test${NC}"
+	PYTHONPATH=$TESTS_DIR python $TESTS_DIR/$test
+  result=$?
+	if [ $result -ne 0 ]; then
+		echo -e "${RED}Test $test FAILED${NC}"
+		any_test_failed=1
+	else
+		echo -e "${GREEN}Test $test PASSED${NC}"
+	fi
+done
+
+# Exit with code 0 if no test failed, otherwise exit with code 1
+if [ $any_test_failed -eq 0 ]; then
+	echo -e "${GREEN}All tests PASSED${NC}"
+	exit 0
+else
+	echo -e "${RED}Some tests FAILED${NC}"
+	exit 1
+fi

extras/custom_tests/__init__.py

@@ -0,0 +1,98 @@
+#  Copyright 2024 Hathor Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os
+import shlex
+import signal
+import subprocess
+import sys
+
+from utils import (  # type: ignore[import-not-found]
+    COMMAND,
+    HATHOR_PROCESS_NAME,
+    KILL_WAIT_DELAY,
+    MONITOR_PROCESS_NAME,
+    SIDE_DAG_PROCESS_NAME,
+    get_pid_by_name,
+    is_alive,
+    popen_is_alive,
+    wait_seconds,
+)
+
+if sys.platform == 'win32':
+    print("test skipped on windows")
+    sys.exit(0)
+
+
+def test_both_fail() -> None:
+    # Assert that there are no existing processes
+    assert get_pid_by_name(MONITOR_PROCESS_NAME) is None
+    assert get_pid_by_name(HATHOR_PROCESS_NAME) is None
+    assert get_pid_by_name(SIDE_DAG_PROCESS_NAME) is None
+
+    # Run the python command
+    args = shlex.split(COMMAND)
+    monitor_process = subprocess.Popen(args)
+    print(f'running "run_node_with_side_dag" in the background with pid: {monitor_process.pid}')
+    print('awaiting subprocesses initialization...')
+    wait_seconds(5)
+
+    monitor_process_pid = get_pid_by_name(MONITOR_PROCESS_NAME)
+    hathor_process_pid = get_pid_by_name(HATHOR_PROCESS_NAME)
+    side_dag_process_pid = get_pid_by_name(SIDE_DAG_PROCESS_NAME)
+
+    # Assert that the processes exist and are alive
+    assert monitor_process_pid == monitor_process.pid
+    assert monitor_process_pid is not None
+    assert hathor_process_pid is not None
+    assert side_dag_process_pid is not None
+
+    assert is_alive(monitor_process_pid)
+    assert is_alive(hathor_process_pid)
+    assert is_alive(side_dag_process_pid)
+
+    print('processes are running:')
+    print(f'  "{MONITOR_PROCESS_NAME}" pid: {monitor_process_pid}')
+    print(f'  "{HATHOR_PROCESS_NAME}" pid: {hathor_process_pid}')
+    print(f'  "{SIDE_DAG_PROCESS_NAME}" pid: {side_dag_process_pid}')
+    print('letting processes run for a while...')
+    wait_seconds(10)
+
+    # Terminate both subprocess
+    print('terminating subprocesses...')
+    os.kill(hathor_process_pid, signal.SIGTERM)
+    os.kill(side_dag_process_pid, signal.SIGTERM)
+    print('awaiting processes termination...')
+    wait_seconds(KILL_WAIT_DELAY, break_function=lambda: not popen_is_alive(monitor_process))
+
+    # Assert that all process are terminated
+    assert not popen_is_alive(monitor_process)
+    assert not is_alive(monitor_process_pid)
+    assert not is_alive(hathor_process_pid)
+    assert not is_alive(side_dag_process_pid)
+
+    print('all processes are dead. test succeeded!')
+
+
+try:
+    test_both_fail()
+except Exception:
+    if monitor_process_pid := get_pid_by_name(MONITOR_PROCESS_NAME):
+        os.kill(monitor_process_pid, signal.SIGKILL)
+    if hathor_process_pid := get_pid_by_name(HATHOR_PROCESS_NAME):
+        os.kill(hathor_process_pid, signal.SIGKILL)
+    if side_dag_process_pid := get_pid_by_name(SIDE_DAG_PROCESS_NAME):
+        os.kill(side_dag_process_pid, signal.SIGKILL)
+
+    raise

extras/custom_tests/side_dag/__init__.py

@@ -0,0 +1,97 @@
+#  Copyright 2024 Hathor Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os
+import shlex
+import signal
+import subprocess
+import sys
+
+from utils import (  # type: ignore[import-not-found]
+    COMMAND,
+    HATHOR_PROCESS_NAME,
+    KILL_WAIT_DELAY,
+    MONITOR_PROCESS_NAME,
+    SIDE_DAG_PROCESS_NAME,
+    get_pid_by_name,
+    is_alive,
+    popen_is_alive,
+    wait_seconds,
+)
+
+if sys.platform == 'win32':
+    print("test skipped on windows")
+    sys.exit(0)
+
+
+def test_one_fails() -> None:
+    # Assert that there are no existing processes
+    assert get_pid_by_name(MONITOR_PROCESS_NAME) is None
+    assert get_pid_by_name(HATHOR_PROCESS_NAME) is None
+    assert get_pid_by_name(SIDE_DAG_PROCESS_NAME) is None
+
+    # Run the python command
+    args = shlex.split(COMMAND)
+    monitor_process = subprocess.Popen(args)
+    print(f'running "run_node_with_side_dag" in the background with pid: {monitor_process.pid}')
+    print('awaiting subprocesses initialization...')
+    wait_seconds(5)
+
+    monitor_process_pid = get_pid_by_name(MONITOR_PROCESS_NAME)
+    hathor_process_pid = get_pid_by_name(HATHOR_PROCESS_NAME)
+    side_dag_process_pid = get_pid_by_name(SIDE_DAG_PROCESS_NAME)
+
+    # Assert that the processes exist and are alive
+    assert monitor_process_pid == monitor_process.pid
+    assert monitor_process_pid is not None
+    assert hathor_process_pid is not None
+    assert side_dag_process_pid is not None
+
+    assert is_alive(monitor_process_pid)
+    assert is_alive(hathor_process_pid)
+    assert is_alive(side_dag_process_pid)
+
+    print('processes are running:')
+    print(f'  "{MONITOR_PROCESS_NAME}" pid: {monitor_process_pid}')
+    print(f'  "{HATHOR_PROCESS_NAME}" pid: {hathor_process_pid}')
+    print(f'  "{SIDE_DAG_PROCESS_NAME}" pid: {side_dag_process_pid}')
+    print('letting processes run for a while...')
+    wait_seconds(10)
+
+    # Terminate one subprocess
+    print('terminating side-dag process...')
+    os.kill(side_dag_process_pid, signal.SIGTERM)
+    print('awaiting process termination...')
+    wait_seconds(KILL_WAIT_DELAY, break_function=lambda: not popen_is_alive(monitor_process))
+
+    # Assert that all process are terminated
+    assert not popen_is_alive(monitor_process)
+    assert not is_alive(monitor_process_pid)
+    assert not is_alive(hathor_process_pid)
+    assert not is_alive(side_dag_process_pid)
+
+    print('all processes are dead. test succeeded!')
+
+
+try:
+    test_one_fails()
+except Exception:
+    if monitor_process_pid := get_pid_by_name(MONITOR_PROCESS_NAME):
+        os.kill(monitor_process_pid, signal.SIGKILL)
+    if hathor_process_pid := get_pid_by_name(HATHOR_PROCESS_NAME):
+        os.kill(hathor_process_pid, signal.SIGKILL)
+    if side_dag_process_pid := get_pid_by_name(SIDE_DAG_PROCESS_NAME):
+        os.kill(side_dag_process_pid, signal.SIGKILL)
+
+    raise

extras/custom_tests/side_dag/test_both_fail.py

@@ -0,0 +1,78 @@
+#  Copyright 2024 Hathor Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os
+import subprocess
+import time
+from typing import Callable
+
+MONITOR_PROCESS_NAME = 'hathor-core: monitor'
+PROCNAME_SUFFIX = 'hathor-core'
+HATHOR_PROCESS_PREFIX = 'hathor:'
+SIDE_DAG_PROCESS_PREFIX = 'side-dag:'
+HATHOR_PROCESS_NAME = HATHOR_PROCESS_PREFIX + PROCNAME_SUFFIX
+SIDE_DAG_PROCESS_NAME = SIDE_DAG_PROCESS_PREFIX + PROCNAME_SUFFIX
+KILL_WAIT_DELAY = 305
+
+COMMAND = f"""
+    python -m hathor run_node_with_side_dag
+        --disable-logs
+        --testnet
+        --memory-storage
+        --x-localhost-only
+        --procname-prefix {HATHOR_PROCESS_PREFIX}
+        --side-dag-testnet
+        --side-dag-memory-storage
+        --side-dag-x-localhost-only
+        --side-dag-procname-prefix {SIDE_DAG_PROCESS_PREFIX}
+"""
+
+
+def wait_seconds(seconds: int, *, break_function: Callable[[], bool] | None = None) -> None:
+    while seconds > 0:
+        print(f'waiting {seconds} seconds...')
+        time.sleep(1)
+        seconds -= 1
+        if break_function and break_function():
+            break
+
+
+def get_pid_by_name(process_name: str) -> int | None:
+    try:
+        output = subprocess.check_output(['pgrep', '-f', process_name], text=True)
+    except subprocess.CalledProcessError:
+        return None
+    pids = output.strip().split()
+    assert len(pids) <= 1
+    try:
+        return int(pids[0])
+    except IndexError:
+        return None
+
+
+def is_alive(pid: int) -> bool:
+    try:
+        os.kill(pid, 0)
+    except OSError:
+        return False
+    return True
+
+
+def popen_is_alive(popen: subprocess.Popen) -> bool:
+    try:
+        popen.wait(0)
+    except subprocess.TimeoutExpired:
+        return True
+    assert popen.returncode is not None
+    return False