Add CPU usage test for the optimization of port-buffer-drop counterpoll (#5024)

There is a performance optimization for port buffer drop counter. Adjust port-buffer-drop counter default interval from 60000ms to 10000ms.

Test is skipped on 201811, 201911 and 202012. It should work on 202106 and later images
Test will check DUT memory usage and process cpu usage are within threshold for 60s.

- Disable all counterpoll types except for the tested one
- Collect memory and CPU usage for 60 secs
- Compare the memory usage with the memory threshold
- Compare the average cpu usage with the cpu threshold for the specified progress(for nvidia, the progress is sx_sdk)
- Restore counterpoll interval and status to state before the test

Author: JibinBao

tests/platform_tests/args/counterpoll_cpu_usage_args.py

@@ -0,0 +1,11 @@
+def add_counterpoll_cpu_usage_args(parser):
+    '''
+    Adding arguments required for counterpoll cpu usage test
+    '''
+    parser.addoption(
+        "--port_buffer_drop_cpu_usage_threshold",
+        action="store",
+        type=int,
+        default=10,
+        help="Port buffer drop cpu usage threshold",
+    )

tests/platform_tests/conftest.py

@@ -13,6 +13,7 @@
 from tests.common.broadcom_data import is_broadcom_device
 from tests.common.plugins.loganalyzer.loganalyzer import LogAnalyzer
 from tests.common.plugins.sanity_check.recover import neighbor_vm_restore
+from .args.counterpoll_cpu_usage_args import add_counterpoll_cpu_usage_args
 
 
 TEMPLATES_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates")
@@ -592,3 +593,7 @@ def pytest_generate_tests(metafunc):
                 metafunc.parametrize('power_off_delay', delay_list)
             except ValueError:
                 metafunc.parametrize('power_off_delay', default_delay_list)
+
+
+def pytest_addoption(parser):
+     add_counterpoll_cpu_usage_args(parser)

tests/platform_tests/counterpoll/__init__.py

@@ -0,0 +1,41 @@
+
+class CounterpollConstants:
+    COUNTERPOLL_SHOW = 'counterpoll show'
+    COUNTERPOLL_DISABLE = 'counterpoll {} disable'
+    COUNTERPOLL_ENABLE = 'counterpoll {} enable'
+    COUNTERPOLL_RESTORE = 'counterpoll {} {}'
+    COUNTERPOLL_INTERVAL_STR = 'counterpoll {} interval {}'
+    COUNTERPOLL_QUEST = 'counterpoll --help'
+    EXCLUDE_COUNTER_SUB_COMMAND = ['show', 'config-db', "flowcnt-trap", "tunnel"]
+    INTERVAL = 'interval (in ms)'
+    TYPE = 'type'
+    STATUS = 'status'
+    STDOUT = 'stdout'
+    PG_DROP = 'pg-drop'
+    PG_DROP_STAT_TYPE = 'PG_DROP_STAT'
+    QUEUE_STAT_TYPE = 'QUEUE_STAT'
+    QUEUE = 'queue'
+    PORT_STAT_TYPE = 'PORT_STAT'
+    PORT = 'port'
+    PORT_BUFFER_DROP_TYPE = 'PORT_BUFFER_DROP'
+    PORT_BUFFER_DROP = 'port-buffer-drop'
+    RIF_STAT_TYPE = 'RIF_STAT'
+    RIF = 'rif'
+    WATERMARK = 'watermark'
+    QUEUE_WATERMARK_STAT_TYPE = 'QUEUE_WATERMARK_STAT'
+    PG_WATERMARK_STAT_TYPE = 'PG_WATERMARK_STAT'
+    BUFFER_POOL_WATERMARK_STAT_TYPE = 'BUFFER_POOL_WATERMARK_STAT'
+    ACL = 'acl'
+    ACL_TYPE = "ACL"
+    COUNTERPOLL_MAPPING = {PG_DROP_STAT_TYPE: PG_DROP,
+                           QUEUE_STAT_TYPE: QUEUE,
+                           PORT_STAT_TYPE: PORT,
+                           PORT_BUFFER_DROP_TYPE: PORT_BUFFER_DROP,
+                           RIF_STAT_TYPE: RIF,
+                           BUFFER_POOL_WATERMARK_STAT_TYPE: WATERMARK,
+                           QUEUE_WATERMARK_STAT_TYPE: WATERMARK,
+                           PG_WATERMARK_STAT_TYPE: WATERMARK,
+                           ACL_TYPE: ACL}
+    PORT_BUFFER_DROP_INTERVAL = '10000'
+    COUNTERPOLL_INTERVAL = {PORT_BUFFER_DROP: 10000}
+    SX_SDK = 'sx_sdk'

tests/platform_tests/counterpoll/counterpoll_constants.py

@@ -0,0 +1,52 @@
+from tests.platform_tests.counterpoll.counterpoll_constants import CounterpollConstants
+
+
+class ConterpollHelper:
+    @staticmethod
+    def get_counterpoll_show_output(duthost):
+        return duthost.show_and_parse(CounterpollConstants.COUNTERPOLL_SHOW)
+
+    @staticmethod
+    def get_available_counterpoll_types(duthost):
+        available_option_list = []
+        COMMANDS = 'Commands:'
+        counterpoll_show = duthost.command(CounterpollConstants.COUNTERPOLL_QUEST)[CounterpollConstants.STDOUT]
+        index = counterpoll_show.find(COMMANDS) + len(COMMANDS) + 1
+        for line in counterpoll_show[index:].splitlines():
+            available_option_list.append(line.split()[0])
+        return [option for option in available_option_list if option not in CounterpollConstants.EXCLUDE_COUNTER_SUB_COMMAND]
+
+    @staticmethod
+    def get_parsed_counterpoll_show(counterpoll_show):
+        parsed_counterpoll = {}
+        for counterpoll in counterpoll_show:
+            parsed_counterpoll[counterpoll[CounterpollConstants.TYPE]] = {
+                CounterpollConstants.INTERVAL: counterpoll[CounterpollConstants.INTERVAL],
+                CounterpollConstants.STATUS: counterpoll[CounterpollConstants.STATUS]}
+        return parsed_counterpoll
+
+    @staticmethod
+    def restore_counterpoll_status(duthost, counterpoll_before, counterpoll_after):
+        for counterpoll, value in counterpoll_after.items():
+            if counterpoll not in counterpoll_before:
+                continue
+            else:
+                if counterpoll_after[counterpoll][CounterpollConstants.STATUS] \
+                        != counterpoll_before[counterpoll][CounterpollConstants.STATUS]:
+                    duthost.command(CounterpollConstants.COUNTERPOLL_RESTORE.format(
+                        CounterpollConstants.COUNTERPOLL_MAPPING[counterpoll],
+                        counterpoll_before[counterpoll][CounterpollConstants.STATUS]))
+
+    @staticmethod
+    def disable_counterpoll(duthost, counter_type_list):
+        for counterpoll_type in counter_type_list:
+            duthost.command(CounterpollConstants.COUNTERPOLL_DISABLE.format(counterpoll_type))
+
+    @staticmethod
+    def enable_counterpoll(duthost, counter_type_list):
+        for counterpoll_type in counter_type_list:
+            duthost.command(CounterpollConstants.COUNTERPOLL_ENABLE.format(counterpoll_type))
+
+    @staticmethod
+    def set_counterpoll_interval(duthost, counterpoll_type, interval):
+        duthost.command(CounterpollConstants.COUNTERPOLL_INTERVAL_STR.format(counterpoll_type, interval))

tests/platform_tests/counterpoll/counterpoll_helper.py

@@ -0,0 +1,23 @@
+import pytest
+
+from tests.platform_tests.counterpoll.counterpoll_constants import CounterpollConstants
+from tests.platform_tests.counterpoll.counterpoll_helper import ConterpollHelper
+from tests.common.utilities import skip_release
+
+
+@pytest.fixture(params=[CounterpollConstants.PORT_BUFFER_DROP])
+def counterpoll_type(request):
+    return request.param
+
+
+@pytest.fixture()
+def restore_counter_poll(duthosts, enum_rand_one_per_hwsku_hostname):
+    duthost = duthosts[enum_rand_one_per_hwsku_hostname]
+    skip_release(duthost, ["201811", "201911", "202012"])
+
+    counter_poll_show = ConterpollHelper.get_counterpoll_show_output(duthost)
+    parsed_counterpoll_before = ConterpollHelper.get_parsed_counterpoll_show(counter_poll_show)
+    yield
+    counter_poll_show = ConterpollHelper.get_counterpoll_show_output(duthost)
+    parsed_counterpoll_after = ConterpollHelper.get_parsed_counterpoll_show(counter_poll_show)
+    ConterpollHelper.restore_counterpoll_status(duthost, parsed_counterpoll_before, parsed_counterpoll_after)

tests/platform_tests/counterpoll/cpu_memory_helper.py

@@ -2,6 +2,11 @@
 import pytest
 
 from collections import namedtuple, Counter
+from tests.platform_tests.counterpoll.cpu_memory_helper import restore_counter_poll  # lgtm [py/unused-import]
+from tests.platform_tests.counterpoll.cpu_memory_helper import counterpoll_type      # lgtm [py/unused-import]
+from tests.platform_tests.counterpoll.counterpoll_helper import ConterpollHelper
+from tests.platform_tests.counterpoll.counterpoll_constants import CounterpollConstants
+from tests.common.mellanox_data import is_mellanox_device
 
 
 pytestmark = [
@@ -22,6 +27,7 @@ def setup_thresholds(duthosts, enum_rand_one_per_hwsku_hostname):
         high_cpu_consume_procs['syncd'] = 80
     return memory_threshold, cpu_threshold, high_cpu_consume_procs
 
+
 def test_cpu_memory_usage(duthosts, enum_rand_one_per_hwsku_hostname, setup_thresholds):
     """Check DUT memory usage and process cpu usage are within threshold."""
     duthost = duthosts[enum_rand_one_per_hwsku_hostname]
@@ -35,36 +41,172 @@ def test_cpu_memory_usage(duthosts, enum_rand_one_per_hwsku_hostname, setup_thre
     outstanding_procs_counter = Counter()
     for i, monit_result in enumerate(MonitResult(*_) for _ in monit_results):
         logging.debug("------ Iteration %d ------", i)
-        if monit_result.memory['used_percent'] > memory_threshold:
-            logging.debug("system memory usage exceeds %d%%: %s",
-                          memory_threshold, monit_result.memory)
-            outstanding_mem_polls[i] = monit_result.memory
+        check_memory(i, memory_threshold, monit_result, outstanding_mem_polls)
         for proc in monit_result.processes:
             cpu_threshold = normal_cpu_threshold
-            if high_cpu_consume_procs.has_key(proc['name']):
-                cpu_threshold = high_cpu_consume_procs[proc['name']]                
-            if proc['cpu_percent'] >= cpu_threshold:
-                logging.debug("process %s(%d) cpu usage exceeds %d%%.",
-                              proc['name'], proc['pid'], cpu_threshold)
-                outstanding_procs[proc['pid']] = proc['name']
-                outstanding_procs_counter[proc['pid']] += 1
+            if proc['name'] in high_cpu_consume_procs:
+                cpu_threshold = high_cpu_consume_procs[proc['name']]
+            check_cpu_usage(cpu_threshold, outstanding_procs, outstanding_procs_counter, proc)
+
+    analyse_monitoring_results(cpu_threshold, memory_threshold, outstanding_mem_polls, outstanding_procs,
+                               outstanding_procs_counter, persist_threshold)
+
 
+def analyse_monitoring_results(cpu_threshold, memory_threshold, outstanding_mem_polls, outstanding_procs,
+                               outstanding_procs_counter, persist_threshold):
     persist_outstanding_procs = []
     for pid, freq in outstanding_procs_counter.most_common():
         if freq <= persist_threshold:
             break
         persist_outstanding_procs.append(pid)
-
     if outstanding_mem_polls or persist_outstanding_procs:
-        failure_message = ""
-
         if outstanding_mem_polls:
-            failure_message += "System memory usage exceeds {}%".format(memory_threshold)
-            if persist_outstanding_procs:
-                failure_message += "; "
-
+            logging.error("system memory usage exceeds %d%%", memory_threshold)
         if persist_outstanding_procs:
-            failure_message += "Processes that persistently exceed CPU usage ({}%): {}".format(
-                cpu_threshold, [outstanding_procs[p] for p in persist_outstanding_procs])
+            logging.error(
+                "processes that persistently exceeds cpu usage %d%%: %s",
+                cpu_threshold,
+                [outstanding_procs[p] for p in persist_outstanding_procs]
+            )
+        pytest.fail("system cpu and memory usage check fails")
+
+
+@pytest.fixture(scope='module')
+def counterpoll_cpu_threshold(duthosts, request):
+    counterpoll_cpu_usage_threshold = {"port-buffer-drop": request.config.getoption("--port_buffer_drop_cpu_usage_threshold")}
+    return counterpoll_cpu_usage_threshold
+
+
+def test_cpu_memory_usage_counterpoll(duthosts, enum_rand_one_per_hwsku_hostname,
+                                      setup_thresholds, restore_counter_poll, counterpoll_type, counterpoll_cpu_threshold):
+    """Check DUT memory usage and process cpu usage are within threshold.
+    Disable all counterpoll types except tested one
+    Collect memory and CPUs usage for 60 secs
+    Compare the memory usage with the memory threshold
+    Compare the average cpu usage with the cpu threshold for the specified progress
+    Restore counterpolls status
+    """
+    duthost = duthosts[enum_rand_one_per_hwsku_hostname]
+    program_to_check = get_manufacturer_program_to_check(duthost)
+    if program_to_check is None:
+        pytest.skip("Skip no program is offered to check")
+
+    memory_threshold, _, _ = setup_thresholds
+    counterpoll_cpu_usage_threshold = counterpoll_cpu_threshold[counterpoll_type]
+
+    MonitResult = namedtuple('MonitResult', ['processes', 'memory'])
+    disable_all_counterpoll_type_except_tested(duthost, counterpoll_type)
+    monit_results = duthost.monit_process(iterations=60, delay_interval=1)['monit_results']
+    poll_interval = CounterpollConstants.COUNTERPOLL_INTERVAL[counterpoll_type] // 1000
+
+    outstanding_mem_polls = {}
+    outstanding_procs = {}
+    outstanding_procs_counter = Counter()
+
+    cpu_usage_program_to_check = []
+
+    prepare_ram_cpu_usage_results(MonitResult, counterpoll_cpu_usage_threshold, memory_threshold, monit_results, outstanding_mem_polls,
+                                  outstanding_procs, outstanding_procs_counter, program_to_check,
+                                  cpu_usage_program_to_check)
+
+    log_cpu_usage_by_vendor(cpu_usage_program_to_check, counterpoll_type)
+
+    cpu_usage_average = caculate_cpu_usge_average_value(extract_valid_cpu_usage_data(cpu_usage_program_to_check, poll_interval), cpu_usage_program_to_check)
+    logging.info("Average cpu_usage is {}".format(cpu_usage_average))
+    assert cpu_usage_average < counterpoll_cpu_usage_threshold, "cpu_usage_average of {} exceeds the cpu threshold:{}".format(program_to_check, counterpoll_cpu_usage_threshold)
+    assert not outstanding_mem_polls, " Memory {} exceeds the memory threshold {} ".format(outstanding_mem_polls, memory_threshold)
+
+
+def log_cpu_usage_by_vendor(cpu_usage_program_to_check, counterpoll_type):
+    if cpu_usage_program_to_check:
+        logging.info('CPU usage for counterpoll type {} : {}'.format(counterpoll_type, cpu_usage_program_to_check))
+
+
+def get_manufacturer_program_to_check(duthost):
+    if is_mellanox_device(duthost):
+        return CounterpollConstants.SX_SDK
+
+
+def prepare_ram_cpu_usage_results(MonitResult, cpu_threshold, memory_threshold, monit_results, outstanding_mem_polls,
+                                  outstanding_procs, outstanding_procs_counter, program_to_check,
+                                  program_to_check_cpu_usage):
+    for i, monit_result in enumerate(MonitResult(*_) for _ in monit_results):
+        logging.debug("------ Iteration %d ------", i)
+        check_memory(i, memory_threshold, monit_result, outstanding_mem_polls)
+        for proc in monit_result.processes:
+            update_cpu_usage_desired_program(proc, program_to_check, program_to_check_cpu_usage)
+
+
+def extract_valid_cpu_usage_data(program_to_check_cpu_usage, poll_interval):
+    """
+    This method it to extract the valid cpu usage data according to the poll_interval
+    1. Find the index for the max one for every poll interval,
+    2. Discard the data if the index is on the edge(0 o the length of program_to_check_cpu_usage -1)
+    3. If the index is closed in the neighbour interval, only keep the former one
+    4. Return all indexes
+    For example:
+    poll_interval = 10
+    7, 1, 0, 1, 0, 1, 5, 1, 1,2, 0, 1, 0, 1, 0, 6, 1, 1, 1,2
+    return [15]
+    0, 1, 0, 1, 0, 1, 0, 1, 0, 8, 7, 1, 0, 1, 0, 6, 1, 1, 1,2
+    return [9]
+    """
+    valid_cpu_usage_center_index_list = []
+    poll_number = len(program_to_check_cpu_usage) // poll_interval
+
+    def find_max_cpu_usage(cpu_usage_list, poll_times):
+        max_cpu_usage = cpu_usage_list[0]
+        max_cpu_usage_index = 0
+        for i, cpu_usage in enumerate(cpu_usage_list):
+            if cpu_usage > max_cpu_usage:
+                max_cpu_usage = cpu_usage
+                max_cpu_usage_index = i
+        return [max_cpu_usage, max_cpu_usage_index + poll_times * poll_interval]
+
+    for i in range(0, poll_number):
+        max_cpu_usage, max_cpu_usage_index = find_max_cpu_usage(
+            program_to_check_cpu_usage[poll_interval * i:poll_interval * (i + 1)], i)
+        if max_cpu_usage_index == 0 or max_cpu_usage_index == len(program_to_check_cpu_usage) - 1:
+            logging.info("The data is on the edge:{}, discard it ".format(max_cpu_usage_index))
+        else:
+            if valid_cpu_usage_center_index_list and valid_cpu_usage_center_index_list[-1] + 1 == max_cpu_usage_index:
+                continue
+            valid_cpu_usage_center_index_list.append(max_cpu_usage_index)
+
+    return valid_cpu_usage_center_index_list
+
+
+def caculate_cpu_usge_average_value(valid_cpu_usage_center_index_list, program_to_check_cpu_usage):
+    len_valid_cpu_usage = len(valid_cpu_usage_center_index_list)
+    cpu_usage_average = 0.0
+    for i in valid_cpu_usage_center_index_list:
+        cpu_usage_average += sum(program_to_check_cpu_usage[i - 1: i + 2])
+        logging.info("cpu usage center index:{}: cpu usage:{}".format(i, program_to_check_cpu_usage[i - 1:i + 2]))
+    return cpu_usage_average / len_valid_cpu_usage / 3.0 if len_valid_cpu_usage != 0 else 0
+
+
+def check_cpu_usage(cpu_threshold, outstanding_procs, outstanding_procs_counter, proc):
+    if proc['cpu_percent'] >= cpu_threshold:
+        logging.debug("process %s(%d) cpu usage exceeds %d%%.",
+                      proc['name'], proc['pid'], cpu_threshold)
+        outstanding_procs[proc['pid']] = proc['name']
+        outstanding_procs_counter[proc['pid']] += 1
+
+
+def update_cpu_usage_desired_program(proc, program_to_check, program_to_check_cpu_usage):
+    if program_to_check:
+        if proc['name'] == program_to_check:
+            program_to_check_cpu_usage.append(proc['cpu_percent'])
+
+
+def check_memory(i, memory_threshold, monit_result, outstanding_mem_polls):
+    if monit_result.memory['used_percent'] > memory_threshold:
+        logging.debug("system memory usage exceeds %d%%: %s",
+                      memory_threshold, monit_result.memory)
+        outstanding_mem_polls[i] = monit_result.memory
+
 
-        pytest.fail(failure_message)
+def disable_all_counterpoll_type_except_tested(duthost, counterpoll_type):
+    available_types = ConterpollHelper.get_available_counterpoll_types(duthost)
+    available_types.remove(counterpoll_type)
+    ConterpollHelper.disable_counterpoll(duthost, available_types)