[advanced-reboot] Set timeout for reboot test to prevent hung tests (#5365)

Advanced-reboot testcases sometimes gets hung inside ptf.

This was earlier handled as part of #4532
Although PR 4532 sets a timeout for PTF test, the inbuilt --test-case-timeout interrupts the advanced-reboot test as expected, but sometimes that is not enough.
After the timeout interrupt, the test still proceeds to collect logs and analyze the pcap files.
In an unlikely event when the analysis logic is hung, the whole testcase stays hung indefinitely.
This commit fixes this testcase hung issue.

Author: vaibhavhd

tests/common/fixtures/advanced_reboot.py

@@ -13,6 +13,7 @@
 from tests.common.helpers.sad_path import SadOperation
 from tests.ptf_runner import ptf_runner
 from tests.common.helpers.assertions import pytest_assert
+from tests.common.utilities import InterruptableThread
 
 logger = logging.getLogger(__name__)
 
@@ -448,9 +449,21 @@ def runRebootTest(self):
                     pre_reboot_analysis, post_reboot_analysis = self.advanceboot_loganalyzer
                     marker = pre_reboot_analysis()
                 self.__setupRebootOper(rebootOper)
-                result = self.__runPtfRunner(rebootOper)
+                thread = InterruptableThread(
+                    target=self.__runPtfRunner,
+                    kwargs={"rebootOper": rebootOper})
+                thread.daemon = True
+                thread.start()
+                # give the test REBOOT_CASE_TIMEOUT (1800s) to complete the reboot with IO,
+                # and then additional 300s to examine the pcap, logs and generate reports
+                ptf_timeout = REBOOT_CASE_TIMEOUT + 300
+                thread.join(timeout=ptf_timeout, suppress_exception=True)
+                self.ptfhost.shell("pkill -f 'ptftests advanced-reboot.ReloadTest'", module_ignore_errors=True)
+                # the thread might still be running, and to catch any exceptions after pkill allow 10s to join
+                thread.join(timeout=10)
                 self.__verifyRebootOper(rebootOper)
             except Exception:
+                logger.error("Exception caught while running advanced-reboot test on ptf")
                 failed_list.append(rebootOper)
             finally:
                 # always capture the test logs