[chassis] accomodate test_memory_exhaustion.py if random dut is chassis sup (sonic-net#9075)

Author: wenyiz2021

tests/common/platform/processes_utils.py

@@ -63,6 +63,10 @@ def wait_critical_processes(dut):
     @param dut: The AnsibleHost object of DUT. For interacting with DUT.
     """
     timeout = reset_timeout(dut)
+    # No matter what we set in inventory file, we always set sup timeout to 900
+    # because most SUPs have 10+ dockers that need to come up
+    if dut.is_supervisor_node():
+        timeout = 900
     logging.info("Wait until all critical processes are healthy in {} sec"
                  .format(timeout))
     pytest_assert(wait_until(timeout, 20, 0, _all_critical_processes_healthy, dut),

tests/platform_tests/test_memory_exhaustion.py

@@ -4,7 +4,7 @@
 
 from tests.common.helpers.assertions import pytest_assert
 from tests.common.platform.processes_utils import wait_critical_processes
-from tests.common.reboot import SONIC_SSH_PORT, SONIC_SSH_REGEX
+from tests.common.reboot import SONIC_SSH_PORT, SONIC_SSH_REGEX, wait_for_startup
 
 pytestmark = [
     pytest.mark.disable_loganalyzer,
@@ -22,9 +22,17 @@ class TestMemoryExhaustion:
     """
     This test case is used to verify that DUT will reboot when it runs out of memory.
     """
+    def wait_lc_healthy_if_sup(self, duthost, duthosts, localhost):
+        # For sup, we also need to ensure linecards are back and healthy for following tests
+        is_sup = duthost.get_facts().get("modular_chassis") and duthost.is_supervisor_node()
+        if is_sup:
+            for lc in duthosts.frontend_nodes:
+                wait_for_startup(lc, localhost, delay=10, timeout=300)
+                wait_critical_processes(lc)
 
     @pytest.fixture(autouse=True)
-    def tearDown(self, duthosts, enum_rand_one_per_hwsku_hostname, localhost, pdu_controller):
+    def tearDown(self, duthosts, enum_rand_one_per_hwsku_hostname,
+                 localhost, pdu_controller):
         yield
         # If the SSH connection is not established, or any critical process is exited,
         # try to recover the DUT by PDU reboot.
@@ -41,6 +49,7 @@ def tearDown(self, duthosts, enum_rand_one_per_hwsku_hostname, localhost, pdu_co
                           'Recover {} by PDU reboot failed'.format(hostname))
             # Wait until all critical processes are healthy.
             wait_critical_processes(duthost)
+            self.wait_lc_healthy_if_sup(duthost, duthosts)
 
     def test_memory_exhaustion(self, duthosts, enum_rand_one_per_hwsku_hostname, localhost):
         duthost = duthosts[enum_rand_one_per_hwsku_hostname]
@@ -68,6 +77,7 @@ def test_memory_exhaustion(self, duthosts, enum_rand_one_per_hwsku_hostname, loc
                       'DUT {} did not startup'.format(hostname))
         # Wait until all critical processes are healthy.
         wait_critical_processes(duthost)
+        self.wait_lc_healthy_if_sup(duthost, duthosts)
         # Verify DUT uptime is later than the time when the test case started running.
         dut_uptime = duthost.get_up_time()
         pytest_assert(dut_uptime > dut_datetime, "Device {} did not reboot".format(hostname))