[launch_pytest] publishing error: cannot use Destroyable because destruction was requested

[Rezenders]

Bug report

Required Info:

• Operating System:
  • Ubuntu 22.04
• Installation type:
  • binaries
• Version or commit hash:
  • Humble
• DDS implementation:
  • default
• Client library (if applicable):
  • rclpy

Steps to reproduce issue

Similar to this question, I am writing some pytests to test a Node I am developing. I am using launch_pytest to setup the ROS Nodes I want to test. And within a test, I am creating a helper node to publish some messages.

However, I am getting the following error when I publish something with the helper node: The following exception was never retrieved: cannot use Destroyable because destruction was requested. The error is triggered with this node.diagnostics_pub.publish(diag_msg)

I don' t get any errors when running the node normally, only when testing.

import launch
import launch_pytest
import launch_ros

from pathlib import Path

import pytest
import rclpy
import traceback

from threading import Event
from threading import Thread

import sys

from diagnostic_msgs.msg import DiagnosticArray
from diagnostic_msgs.msg import DiagnosticStatus
from diagnostic_msgs.msg import KeyValue
from lifecycle_msgs.srv import ChangeState
from lifecycle_msgs.srv import GetState
from rclpy.callback_groups import ReentrantCallbackGroup
from rclpy.node import Node
from ros_typedb_msgs.srv import Query


@launch_pytest.fixture
def generate_test_description():
    path_to_pkg = Path(__file__).parents[1]
    path_config = path_to_pkg / 'config'
    path_test_data = path_to_pkg / 'test' / 'test_data'

    metacontrol_kb_node = launch_ros.actions.Node(
        executable=sys.executable,
        arguments=[
            str(path_to_pkg / 'metacontrol_kb' / 'metacontrol_kb_typedb.py')],
        additional_env={'PYTHONUNBUFFERED': '1'},
        name='metacontrol_kb',
        output='screen',
        parameters=[{
            'schema_path': str(path_config / 'schema.tql'),
            'data_path': str(path_test_data / 'test_data.tql')
        }]
    )

    return launch.LaunchDescription([
        metacontrol_kb_node,
    ])


@pytest.mark.launch(fixture=generate_test_description)
def test_metacontrol_kb_diagnostics():
    rclpy.init()
    traceback_logger = rclpy.logging.get_logger('node_class_traceback_logger')
    try:
        node = MakeTestNode()
        node.start_node()
        node.activate_metacontrol_kb()

        status_msg = DiagnosticStatus()
        status_msg.level = DiagnosticStatus.OK
        status_msg.name = ''
        key_value = KeyValue()
        key_value.key = 'ea_measurement'
        key_value.value = str(1.72)
        status_msg.values.append(key_value)
        status_msg.message = 'QA status'

        diag_msg = DiagnosticArray()
        diag_msg.header.stamp = node.get_clock().now().to_msg()
        diag_msg.status.append(status_msg)

        node.diagnostics_pub.publish(diag_msg)

        query_req = Query.Request()
        query_req.query_type = 'match'
        query_req.query = """
            match $ea isa Attribute,
                has attribute-name "ea_measurement",
                has attribute-measurement $measurement;
            get $measurement;
        """
        query_res = node.call_service(node.query_srv, query_req)

        correct_measurement = False
        for r in query_res.result:
            if r.attribute_name == 'attribute-measurement' \
               and r.value.double_value == 1.72:
                correct_measurement = True

        assert correct_measurement
    except Exception as exception:
        traceback_logger.error(traceback.format_exc())
        raise exception
    finally:
        rclpy.shutdown()


class MakeTestNode(Node):

    def __init__(self, name='test_node'):
        super().__init__(name)

        self.diagnostics_pub = self.create_publisher(
            DiagnosticArray,
            '/diagnostics',
            1)

        self.change_state_srv = self.create_client(
            ChangeState, '/metacontrol_kb/change_state')

        self.get_state_srv = self.create_client(
            GetState, '/metacontrol_kb/get_state')

        self.query_srv = self.create_client(
            Query, '/typedb/query')

    def start_node(self):
        self.ros_spin_thread = Thread(
            target=lambda node: rclpy.spin(node),
            args=(self,))
        self.ros_spin_thread.start()

    def change_node_state(self, transition_id):
        change_state_req = ChangeState.Request()
        change_state_req.transition.id = transition_id
        return self.call_service(self.change_state_srv, change_state_req)

    def get_node_state(self):
        get_state_req = GetState.Request()
        return self.call_service(self.get_state_srv, get_state_req)

    def call_service(self, cli, request):
        if cli.wait_for_service(timeout_sec=5.0) is False:
            self.get_logger().error(
                'service not available {}'.format(cli.srv_name))
            return None
        future = cli.call_async(request)
        if self.executor.spin_until_future_complete(
                future, timeout_sec=5.0) is False:
            self.get_logger().error(
                'Future not completed {}'.format(cli.srv_name))
            return None

        return future.result()

    def activate_metacontrol_kb(self):
        state_req = 1
        res = self.change_node_state(state_req)
        if res.success is True:
            state_req = 3
            res = self.change_node_state(state_req)
        if res.success is False:
            self.get_logger().error(
                'State change  req error. Requested {}'.format(state_req))

Expected behavior

No error returned

Actual behavior

Error: [python3-1] The following exception was never retrieved: cannot use Destroyable because destruction was requested

[clalancette]

We think this is likely a bug somewhere in rclpy (where the Destroyable class is implemented). I'll move it over there for now.

[dcconner]

I sometimes get this error after trying to destroy_subscription

[start_behavior-1] Exception in executor       at 2023-08-02 14:08:13.545086 - ! <class 'rclpy._rclpy_pybind11.InvalidHandle'>
[start_behavior-1]   cannot use Destroyable because destruction was requested
[start_behavior-1] Traceback (most recent call last):
[start_behavior-1]   File "/home/ros_ws/install/flexbe_onboard/lib/python3.10/site-packages/flexbe_onboard/start_behavior.py", line 57, in main
[start_behavior-1]     executor.spin()
[start_behavior-1]   File "/opt/ros/rolling/lib/python3.10/site-packages/rclpy/executors.py", line 293, in spin
[start_behavior-1]     self.spin_once()
[start_behavior-1]   File "/opt/ros/rolling/lib/python3.10/site-packages/rclpy/executors.py", line 829, in spin_once
[start_behavior-1]     self._spin_once_impl(timeout_sec)
[start_behavior-1]   File "/opt/ros/rolling/lib/python3.10/site-packages/rclpy/executors.py", line 808, in _spin_once_impl
[start_behavior-1]     handler, entity, node = self.wait_for_ready_callbacks(
[start_behavior-1]   File "/opt/ros/rolling/lib/python3.10/site-packages/rclpy/executors.py", line 724, in wait_for_ready_callbacks
[start_behavior-1]     return next(self._cb_iter)
[start_behavior-1]   File "/opt/ros/rolling/lib/python3.10/site-packages/rclpy/executors.py", line 618, in _wait_for_ready_callbacks
[start_behavior-1]     waitable.add_to_wait_set(wait_set)
[start_behavior-1]   File "/opt/ros/rolling/lib/python3.10/site-packages/rclpy/event_handler.py", line 124, in add_to_wait_set
[start_behavior-1]     with self.__event:
[start_behavior-1] rclpy._rclpy_pybind11.InvalidHandle: cannot use Destroyable because destruction was requested

I noticed a similar question by @Rezenders that was apparently solved by setting to QoS to unreliable. That would NOT be a good option for my case, but maybe it could point in direction of solution. Can @Rezenders comment on the answer he gave to that question?

At a minimum, can we not catch this particular error and ignore? it seems to be case of subscription trying to process a message after destroy has been requested, so I would think just skipping that processing would be the way to go.

Is there an easy way to check the if destroy requested and ignore processing after that?

I will note that my error above and the original referenced question used a python Thread outside the executors. I'm in the process of reworking my code to avoid this, and only use multithreaded executor and callback groups. It is possible that non-executor thread is the primary trigger for this issue, but seems to be possible with any multithreaded node.

I'm willing to do some testing on adding try-except block for my use case, but would like @adityapande-1995 or @clalancette to chime in as this is pretty deep in the weeds for me. No sense wasting time if there are clear reasons they don't want to go that route.

[dchatarpaul]

I am also seeing this same problem in a ros2 tester I'm working on which has to handle services/actions/topics that may not be available. I notice when a service isn't available and I destroy the node with the client while it is waiting for service to respond I hit this error. The node is destroyed in another thread and catching the error and moving on seems OK for my use-case. Using a multi-threaded executor does avoid this problem without any try-exception loops.

[aarsht7]

Hello,

I faced a somewhat similar error while trying to remove a node from MultiThreadedExecutor and then trying to destroy the same node. The error only appears while trying to destroy a node and not removing it from MultiThreadedExecutor

rclpy._rclpy_pybind11.InvalidHandle: cannot use Destroyable because destruction was requested

However, I don't think it is affecting any other node in the MultiThreadedExecutor or affecting any other process for me. It is just an error thrown when I try to destroy a node from the executor.

[zacharyyamaoka]

I am also facing challenges with this error:

When using:

success, msg = wait_for_message(aruco_msgs.MarkerArray, self, f"{self.get_namespace()}/aruco_info", time_to_wait=1)

It works 90% of the time, but will occasionally error out.

Traceback (most recent call last):
  File "/home/bam-gpu/bam_ws/src/bam_calibration/scripts/conveyor_calibration.py", line 249, in <module>
    main()
  File "/home/bam-gpu/bam_ws/src/bam_calibration/scripts/conveyor_calibration.py", line 242, in main
    executor.spin()
  File "/opt/ros/iron/lib/python3.10/site-packages/rclpy/executors.py", line 295, in spin
    self.spin_once()
  File "/opt/ros/iron/lib/python3.10/site-packages/rclpy/executors.py", line 852, in spin_once
    self._spin_once_impl(timeout_sec)
  File "/opt/ros/iron/lib/python3.10/site-packages/rclpy/executors.py", line 833, in _spin_once_impl
    handler, entity, node = self.wait_for_ready_callbacks(
  File "/opt/ros/iron/lib/python3.10/site-packages/rclpy/executors.py", line 762, in wait_for_ready_callbacks
    return next(self._cb_iter)
  File "/opt/ros/iron/lib/python3.10/site-packages/rclpy/executors.py", line 663, in _wait_for_ready_callbacks
    waitable.add_to_wait_set(wait_set)
  File "/opt/ros/iron/lib/python3.10/site-packages/rclpy/event_handler.py", line 124, in add_to_wait_set
    with self.__event:
rclpy._rclpy_pybind11.InvalidHandle: cannot use Destroyable because destruction was requested

I am calling wait_for_message() from a MultiThreadedExecutor() node inside a MutuallyExclusiveCallbackGroup() service callback

        self.service_cb_group = MutuallyExclusiveCallbackGroup()
        self.service = self.create_service(Trigger, f'{self.get_name()}/calibrate', self.service_callback, callback_group=self.service_cb_group)