Improve ptf_runner exception and timeout handling (#4586)

The failures seen in ptf (tests started by ptf_runner) are difficult to examine. Make it easier by bringing out the exceptions and print the traceback message.

Also, make the exeception handling generic in the ptf_runner. Earlier in the PR #4532 similar try-catch was added only for advanced_reboot cases.

Author: vaibhavhd

tests/common/fixtures/advanced_reboot.py

@@ -11,6 +11,7 @@
 from tests.common.reboot import reboot as rebootDut
 from tests.common.helpers.sad_path import SadOperation
 from tests.ptf_runner import ptf_runner
+from tests.common.helpers.assertions import pytest_assert
 
 logger = logging.getLogger(__name__)
 
@@ -424,25 +425,25 @@ def runRebootTest(self):
         # Run advanced-reboot.ReloadTest for item in preboot/inboot list
         count = 0
         result = True
-        final_result = True
+        failed_list = list()
         for rebootOper in self.rebootData['sadList']:
             count += 1
             try:
                 self.__setupRebootOper(rebootOper)
                 result = self.__runPtfRunner(rebootOper)
                 self.__verifyRebootOper(rebootOper)
             except Exception:
-                result = False
+                failed_list.append(rebootOper)
             finally:
                 # always capture the test logs
                 self.__fetchTestLogs(rebootOper)
                 self.__clearArpAndFdbTables()
                 self.__revertRebootOper(rebootOper)
-            if not result:
-                final_result = False
             if len(self.rebootData['sadList']) > 1 and count != len(self.rebootData['sadList']):
                 time.sleep(TIME_BETWEEN_SUCCESSIVE_TEST_OPER)
-        return final_result
+        pytest_assert(len(failed_list) == 0,\
+            "Advanced-reboot failure. Failed cases: {}".format(failed_list))
+        return result
 
     def runRebootTestcase(self, prebootList=None, inbootList=None, prebootFiles=None):
         '''
@@ -536,23 +537,18 @@ def __runPtfRunner(self, rebootOper=None):
         self.__updateAndRestartArpResponder(rebootOper)
 
         logger.info('Run advanced-reboot ReloadTest on the PTF host')
-        try:
-            result = ptf_runner(
-                self.ptfhost,
-                "ptftests",
-                "advanced-reboot.ReloadTest",
-                qlen=PTFRUNNER_QLEN,
-                platform_dir="ptftests",
-                platform="remote",
-                params=params,
-                log_file=u'/tmp/advanced-reboot.ReloadTest.log',
-                module_ignore_errors=self.moduleIgnoreErrors,
-                timeout=REBOOT_CASE_TIMEOUT
-            )
-        except Exception as err:
-            logger.error("Timed out after {}s of executing advance reboot case: {}.".format(
-                REBOOT_CASE_TIMEOUT, str(rebootOper)) + ". Error message: {}".format(err.message))
-            raise Exception
+        result = ptf_runner(
+            self.ptfhost,
+            "ptftests",
+            "advanced-reboot.ReloadTest",
+            qlen=PTFRUNNER_QLEN,
+            platform_dir="ptftests",
+            platform="remote",
+            params=params,
+            log_file=u'/tmp/advanced-reboot.ReloadTest.log',
+            module_ignore_errors=self.moduleIgnoreErrors,
+            timeout=REBOOT_CASE_TIMEOUT
+        )
 
         return result
 

tests/ptf_runner.py

@@ -1,4 +1,9 @@
 import pipes
+import traceback
+import logging
+
+logger = logging.getLogger(__name__)
+
 
 def ptf_runner(host, testdir, testname, platform_dir=None, params={},
                platform="remote", qlen=0, relax=True, debug_level="info",
@@ -38,8 +43,14 @@ def ptf_runner(host, testdir, testname, platform_dir=None, params={},
     if timeout:
         cmd += " --test-case-timeout {}".format(int(timeout))
 
-    result = host.shell(cmd, chdir="/root", module_ignore_errors=module_ignore_errors)
-    if module_ignore_errors:
-        if result["rc"] != 0:
-            return result
+    try:
+        result = host.shell(cmd, chdir="/root", module_ignore_errors=module_ignore_errors)
+        if module_ignore_errors:
+            if result["rc"] != 0:
+                return result
+    except Exception:
+        traceback_msg = traceback.format_exc()
+        logger.error("Exception caught while executing case: {}. Error message: {}"\
+            .format(testname, traceback_msg))
+        raise Exception
     return True