[action] [PR:18033] Generate PFC storm using PFC backpressure on additional Arista devices (#222)

<!--
Please make sure you've read and understood our contributing guidelines;
https://github.com/sonic-net/SONiC/blob/gh-pages/CONTRIBUTING.md

Please provide following information to help code review process a bit easier:
-->
### Description of PR
NOTE: This is a duplicate of sonic-net/sonic-mgmt#17971, as @veronica-arista is away.

Add a pfc_gen script for Arista fanout devices to generate pfc pause frames by setting PFC backpressure status in hardware. Enable the storm generation to use this script on interfaces that are connected to Arista fanout devices running EOS or Sonic. Currently supports the following Arista devices:
Arista-7060X6
Arista-7060DX5
Arista-7060PX5
Arista-7060CX
Arista-7260CX3
Arista-7260QX3

This is implemented on top of PR sonic-net/sonic-mgmt#15594

Summary:
Fixes # (issue)

### Type of change

<!--
- Fill x for your type of change.
- e.g.
- [x] Bug fix
-->

- [ ] Bug fix
- [ ] Testbed and Framework(new/improvement)
- [ ] New Test case
 - [ ] Skipped for non-supported platforms
- [x] Test case improvement

### Back port request
- [x] 202012
- [ ] 202205
- [ ] 202305
- [ ] 202311
- [ ] 202405
- [x] 202411

### Approach
#### What is the motivation for this PR?
Adds more reliable PFC storm generation for interfaces connected to Arista fanout devices

#### How did you do it?

#### How did you verify/test it?
Ran sonic-mgmt pfcwd tests on testbed with Arista fanout switches of the listed SKUs running EOS.
Ran the pfc_gen script manually on Arista switches running Sonic.

#### Any platform specific information?

#### Supported testbed topology if it's a new test case?

### Documentation
<!--
(If it's a new feature, new test case)
Did you update documentation/Wiki relevant to your implementation?
Link to the wiki page?
-->

Author: mssonicbld

tests/common/helpers/pfc_gen_brcm_xgs.py

@@ -0,0 +1,246 @@
+#!/usr/bin/env python
+
+"""
+Script to generate PFC storm.
+
+"""
+import sys
+import optparse
+import logging
+import logging.handlers
+import re
+import signal
+import subprocess
+import time
+
+
+logger = logging.getLogger('MyLogger')
+logger.setLevel(logging.DEBUG)
+
+
+class SignalCleanup():
+    def __init__(self, fanoutPfcStorm, endMsg):
+        self.fanoutPfcStorm = fanoutPfcStorm
+        self.endMsg = endMsg
+        signal.signal(signal.SIGTERM, self.sigHandler)
+
+    def sigHandler(self, *args):
+        self.fanoutPfcStorm.endAllPfcStorm()
+
+        logger.debug(self.endMsg)
+        sys.exit(0)
+
+
+class FanoutPfcStorm():
+    '''
+    For eos this class expects all interfaces to be in the front panel interface format
+    ex. Ethernet1/1 and not et1_1
+    For sonic, the interfaces are the default format
+    '''
+    def __init__(self, priority, chipName, os):
+        self.intfsEnabled = []
+        self.priority = priority
+        self.switchChip = chipName
+        self.os = os
+        if os == 'sonic':
+            self.intfToMmuPort, self.intfToPort = self._parseInterfaceMapFullSonic()
+        else:
+            self.intfToMmuPort, self.intfToPort = self._parseInterfaceMapFull()
+
+    def _shellCmd(self, cmd):
+        output = ""
+        result = subprocess.run([f"{cmd}"], capture_output=True, text=True, shell=True)
+        if result.returncode == 0:
+            output = result.stdout
+        return output
+
+    def _cliCmd(self, cmd):
+        output = ""
+        if self.os == 'sonic':
+            result = subprocess.run([f"bcmcmd '{cmd}'"], capture_output=True, text=True, shell=True)
+        else:
+            result = subprocess.run(
+                ["Cli", "-c", f"{cmd}"], capture_output=True, text=True)
+        if result.returncode == 0:
+            output = result.stdout
+        return output
+
+    def _bcmltshellCmd(self, cmd):
+        if self.os == 'sonic':
+            return self._cliCmd(f"bsh -c \"{cmd}\"")
+        else:
+            return self._cliCmd(f"en\nplatform trident shell\nbcmltshell\n{cmd}")
+
+    def _bcmshellCmd(self, cmd):
+        return self._cliCmd(f"en\nplatform trident shell\n{cmd}")
+
+    def _parseInterfaceMapFull(self):
+        intfToMmuPort = {}
+        intfToPort = {}
+
+        output = self._cliCmd("en\nshow platform trident interface map full")
+
+        for line in output.splitlines():
+            mo = re.search(
+                 r'Intf: (?P<intf>Ethernet\S+).{1,50}Port: {1,3}(?P<port>\S+)'
+                 r'.{1,100}P2M\[ {0,3}\d+\]: {1,3}(?P<mmu>\S+)',
+                 line
+            )
+            if mo is None:
+                continue
+            intfToMmuPort[mo.group('intf')] = mo.group('mmu')
+            intfToPort[mo.group('intf')] = mo.group('port')
+
+        return intfToMmuPort, intfToPort
+
+    def _parseInterfaceMapFullSonic(self):
+        intfToMmuPort = {}
+        intfTolPort = {}
+        lPortToIntf = {}
+
+        if self.switchChip.startswith(("Tomahawk5", "Tomahawk4")):
+            output = self._bcmltshellCmd('knet netif info')
+            for info in output.split("Network interface Info:"):
+                mo = re.search(r"Name: (?P<intf>Ethernet\d+)[\s\S]{1,100}Port: (?P<lport>\d+)", info)
+                if mo is None:
+                    continue
+                lPortToIntf[mo.group('lport')] = mo.group('intf')
+                intfTolPort[mo.group('intf')] = mo.group('lport')
+            output = self._cliCmd("show portmap")
+            for line in output.splitlines():
+                entries = line.split()
+                if len(entries) == 7:
+                    lport = entries[2]
+                    mmuPort = entries[4]
+                    if lport in lPortToIntf:
+                        intfToMmuPort[lPortToIntf[lport]] = mmuPort
+                    intfTolPort[mo.group('intf')] = mo.group('lport')
+        else:
+            output = self._cliCmd('knet netif show')
+            for info in output.split("Interface ID"):
+                mo = re.search(r"name=(?P<intf>Ethernet\d+)[\s\S]{1,100}port=(?P<lport>\S+)", info)
+                if mo is None:
+                    continue
+                lPortToIntf[mo.group('lport')] = mo.group('intf')
+                intfTolPort[mo.group('intf')] = mo.group('lport')
+            output = self._cliCmd("show portmap")
+            for line in output.splitlines():
+                entries = line.split()
+                if len(entries) == 9:
+                    lport = entries[0]
+                    mmuPort = entries[4]
+                    if lport in lPortToIntf:
+                        intfToMmuPort[lPortToIntf[lport]] = mmuPort
+                    intfTolPort[mo.group('intf')] = mo.group('lport')
+
+        return intfToMmuPort, intfTolPort
+
+    def _endPfcStorm(self, intf):
+        '''
+        Intf format is Ethernet1/1
+
+        The users of this class are only expected to call
+        startPfcStorm and endAllPfcStorm
+        '''
+        mmuPort = self.intfToMmuPort[intf]
+        port = self.intfToPort[intf]
+        if self.switchChip.startswith(("Tomahawk5", "Tomahawk4")):
+            self._bcmltshellCmd(f"pt MMU_INTFO_XPORT_BKP_HW_UPDATE_DISr set BCMLT_PT_PORT={mmuPort} PAUSE_PFC_BKP=0")
+            self._bcmltshellCmd(f"pt MMU_INTFO_TO_XPORT_BKPr set BCMLT_PT_PORT={mmuPort} PAUSE_PFC_BKP=0")
+        else:
+            self._bcmshellCmd(f"setreg CHFC2PFC_STATE.{port} PRI_BKP=0")
+        self._cliCmd(f"en\nconf\n\nint {intf}\nno priority-flow-control on")
+        if self.os == 'sonic':
+            for prio in range(8):
+                self._cliCmd(f"config interface pfc priority {intf} {prio} off")
+        else:
+            for prio in range(8):
+                self._cliCmd(f"en\nconf\n\nint {intf}\nno priority-flow-control priority {prio} no-drop")
+
+    def startPfcStorm(self, intf):
+        if intf in self.intfsEnabled:
+            return
+        self.intfsEnabled.append(intf)
+
+        mmuPort = self.intfToMmuPort[intf]
+        port = self.intfToPort[intf]
+        if self.os == 'sonic':
+            for prio in range(8):
+                if (1 << prio) & self.priority:
+                    self._shellCmd(f"config interface pfc priority {intf} {prio} on")
+        else:
+            self._cliCmd(f"en\nconf\n\nint {intf}\npriority-flow-control on")
+            for prio in range(8):
+                if (1 << prio) & self.priority:
+                    self._cliCmd(f"en\nconf\n\nint {intf}\npriority-flow-control priority {prio} no-drop")
+
+        if self.switchChip.startswith(("Tomahawk5", "Tomahawk4")):
+            self._bcmltshellCmd(f"pt MMU_INTFO_XPORT_BKP_HW_UPDATE_DISr set BCMLT_PT_PORT={mmuPort} PAUSE_PFC_BKP=1")
+            self._bcmltshellCmd(f"pt MMU_INTFO_TO_XPORT_BKPr set BCMLT_PT_PORT={mmuPort} PAUSE_PFC_BKP={self.priority}")
+        else:
+            self._bcmshellCmd(f"setreg CHFC2PFC_STATE.{port} PRI_BKP={self.priority}")
+
+    def endAllPfcStorm(self):
+        for intf in self.intfsEnabled:
+            self._endPfcStorm(intf)
+
+
+def main():
+    usage = "usage: %prog [options] arg1 arg2"
+    parser = optparse.OptionParser(usage=usage)
+    parser.add_option("-i", "--interface", type="string", dest="interface",
+                      help="Interface list to send packets, separated by ','", metavar="Interface")
+    parser.add_option('-p', "--priority", type="int", dest="priority",
+                      help="PFC class enable bitmap.", metavar="Priority", default=-1)
+    parser.add_option("-r", "--rsyslog-server", type="string", dest="rsyslog_server",
+                      default="127.0.0.1", help="Rsyslog server IPv4 address", metavar="IPAddress")
+    parser.add_option("-c", "--chipName", type="string", dest="chipName", metavar="ChipName",
+                      help="Name of chip in the switch, i.e. Tomahawk5")
+    parser.add_option("-o", "--os", type="string", dest="os",
+                      help="Operating system (eos or sonic)", default="eos")
+
+    (options, args) = parser.parse_args()
+
+    if options.interface is None:
+        print("Need to specify the interface to send PFC pause frame packets.")
+        parser.print_help()
+        sys.exit(1)
+
+    if options.chipName is None:
+        print("Need to specify the ChipName to determine what cli to generate PFC pause frame packets.")
+        parser.print_help()
+        sys.exit(1)
+
+    if options.priority > 255 or options.priority < 0:
+        print("Enable class bitmap is not valid. Need to be in range 0-255.")
+        parser.print_help()
+        sys.exit(1)
+
+    # Configure logging
+    handler = logging.handlers.SysLogHandler(address=(options.rsyslog_server, 514))
+    logger.addHandler(handler)
+
+    # List of front panel kernel intfs
+    interfaces = options.interface.split(',')
+
+    fs = FanoutPfcStorm(options.priority, options.chipName, options.os)
+    SignalCleanup(fs, 'PFC_STORM_END')
+
+    logger.debug('PFC_STORM_DEBUG')
+    for intf in interfaces:
+        if options.os == 'eos':
+            intf = frontPanelIntfFromKernelIntfName(intf)
+        fs.startPfcStorm(intf)
+    logger.debug('PFC_STORM_START')
+
+    # wait forever until stop
+    while True:
+        time.sleep(100)
+
+
+def frontPanelIntfFromKernelIntfName(intf):
+    return intf.replace("et", "Ethernet").replace("_", "/")
+
+
+if __name__ == "__main__":
+    main()

tests/common/helpers/pfc_storm.py

@@ -13,6 +13,26 @@
 logger = logging.getLogger(__name__)
 
 
+def get_chip_name_if_asic_pfc_storm_supported(fanout):
+    hwSkuInfo = {
+        "Arista DCS-7060DX5": "Tomahawk4",
+        "Arista DCS-7060PX5": "Tomahawk4",
+        "Arista DCS-7060X6": "Tomahawk5",
+        "Arista-7060X6": "Tomahawk5",
+        "Arista DCS-7060CX": "Tomahawk",
+        "Arista-7060CX": "Tomahawk",
+        "Arista DCS-7260CX3": "Tomahawk2",
+        "Arista-7260CX3": "Tomahawk2",
+        "Arista-7260QX3": "Tomahawk2",
+        }
+
+    for sku, chip in hwSkuInfo.items():
+        if fanout.startswith(sku):
+            return chip
+
+    return None
+
+
 class PFCStorm(object):
     """ PFC storm/start on different interfaces on a fanout connected to the DUT"""
 
@@ -29,6 +49,7 @@ def __init__(self, duthost, fanout_graph_facts, fanouthosts, **kwargs):
             fanouthosts(AnsibleHost) : fanout instance
             kwargs(dict):
                 peer_info(dict): keys are 'peerdevice', 'pfc_fanout_interface'. Optional: 'hwsku'
+                pfc_gen_chip_name(string) : chip name of switch where PFC frames are generated. default: None
                 pfc_queue_index(int) : queue on which the PFC storm should be generated. default: 3
                 pfc_frames_number(int) : Number of PFC frames to generate. default: 100000
                 pfc_gen_file(string): Script which generates the PFC traffic. default: 'pfc_gen.py'
@@ -43,6 +64,7 @@ def __init__(self, duthost, fanout_graph_facts, fanouthosts, **kwargs):
         self.fanout_hosts = fanouthosts
         self.pfc_gen_file = kwargs.pop('pfc_gen_file', "pfc_gen.py")
         self.pfc_gen_multiprocess = kwargs.pop('pfc_gen_multiprocess', False)
+        self.pfc_gen_chip_name = None
         self.pfc_queue_idx = kwargs.pop('pfc_queue_index', 3)
         self.pfc_frames_number = kwargs.pop('pfc_frames_number', 100000)
         self.send_pfc_frame_interval = kwargs.pop('send_pfc_frame_interval', 0)
@@ -125,13 +147,39 @@ def _create_pfc_gen(self):
         if not out['stat']['exists'] or not out['stat']['isdir']:
             self.peer_device.file(path=pfc_gen_fpath, state="touch")
 
+    def _get_eos_fanout_version(self):
+        """
+        Get version info for eos fanout device
+        """
+        cmd = 'Cli -c "show version"'
+        return self.peer_device.shell(cmd)['stdout_lines']
+
+    def _get_sonic_fanout_hwsku(self):
+        """
+        Get hwsku for sonic fanout device
+        """
+        cmd = 'show version'
+        out_lines = self.peer_device.shell(cmd)['stdout_lines']
+        for line in out_lines:
+            if line.startswith('HwSKU:'):
+                return line.split()[1]
+
     def deploy_pfc_gen(self):
         """
         Deploy the pfc generation file on the fanout
         """
         if self.asic_type == 'vs':
             return
         if self.peer_device.os in ('eos', 'sonic'):
+            chip_name = None
+            if self.peer_device.os == 'eos':
+                chip_name = get_chip_name_if_asic_pfc_storm_supported(self._get_eos_fanout_version()[0])
+            elif self.peer_device.os == 'sonic':
+                chip_name = get_chip_name_if_asic_pfc_storm_supported(self._get_sonic_fanout_hwsku())
+            if self.peer_device.os == 'eos' and chip_name:
+                self.pfc_gen_file = "pfc_gen_brcm_xgs.py"
+                self.pfc_gen_file_test_name = "pfc_gen_brcm_xgs.py"
+                self.pfc_gen_chip_name = chip_name
             src_pfc_gen_file = "common/helpers/{}".format(self.pfc_gen_file)
             self._create_pfc_gen()
             if self.fanout_asic_type == 'mellanox':
@@ -185,6 +233,7 @@ def _update_template_args(self):
             "pfc_queue_index": self.pfc_queue_idx,
             "pfc_frames_number": self.pfc_frames_number,
             "pfc_fanout_interface": self.peer_info['pfc_fanout_interface'] if self.asic_type != 'vs' else "",
+            "pfc_gen_chip_name": self.pfc_gen_chip_name,
             "ansible_eth0_ipv4_addr": self.ip_addr,
             "peer_hwsku": self.peer_info['hwsku'] if self.asic_type != 'vs' else "",
             "send_pfc_frame_interval": self.send_pfc_frame_interval,
@@ -210,15 +259,21 @@ def _prepare_start_template(self):
         Populates the pfc storm start template
         """
         self._update_template_args()
-        if self.dut.topo_type == 't2' and self.peer_device.os == 'sonic':
+        if self.asic_type == 'vs':
+            self.pfc_start_template = os.path.join(
+                TEMPLATES_DIR, "pfc_storm_eos.j2")
+        elif self.dut.topo_type == 't2' and self.peer_device.os == 'sonic':
             self.pfc_start_template = os.path.join(
                 TEMPLATES_DIR, "pfc_storm_{}_t2.j2".format(self.peer_device.os))
         elif self.fanout_asic_type == 'mellanox' and self.peer_device.os == 'sonic':
             self.pfc_start_template = os.path.join(
                 TEMPLATES_DIR, "pfc_storm_mlnx_{}.j2".format(self.peer_device.os))
-        elif self.asic_type == 'vs':
+        elif ((self.peer_device.os == 'eos' and
+               get_chip_name_if_asic_pfc_storm_supported(self._get_eos_fanout_version()[0])) or
+              (self.peer_device.os == 'sonic' and
+               get_chip_name_if_asic_pfc_storm_supported(self._get_sonic_fanout_hwsku()))):
             self.pfc_start_template = os.path.join(
-                TEMPLATES_DIR, "pfc_storm_eos.j2")
+                TEMPLATES_DIR, "pfc_storm_arista_{}.j2".format(self.peer_device.os))
         else:
             self.pfc_start_template = os.path.join(
                 TEMPLATES_DIR, "pfc_storm_{}.j2".format(self.peer_device.os))
@@ -229,15 +284,21 @@ def _prepare_stop_template(self):
         Populates the pfc storm stop template
         """
         self._update_template_args()
-        if self.dut.topo_type == 't2' and self.peer_device.os == 'sonic':
+        if self.asic_type == 'vs':
+            self.pfc_stop_template = os.path.join(
+                TEMPLATES_DIR, "pfc_storm_stop_eos.j2")
+        elif self.dut.topo_type == 't2' and self.peer_device.os == 'sonic':
             self.pfc_stop_template = os.path.join(
                 TEMPLATES_DIR, "pfc_storm_stop_{}_t2.j2".format(self.peer_device.os))
         elif self.fanout_asic_type == 'mellanox' and self.peer_device.os == 'sonic':
             self.pfc_stop_template = os.path.join(
                 TEMPLATES_DIR, "pfc_storm_stop_mlnx_{}.j2".format(self.peer_device.os))
-        elif self.asic_type == 'vs':
+        elif ((self.peer_device.os == 'eos' and
+               get_chip_name_if_asic_pfc_storm_supported(self._get_eos_fanout_version()[0])) or
+              (self.peer_device.os == 'sonic' and
+               get_chip_name_if_asic_pfc_storm_supported(self._get_sonic_fanout_hwsku()))):
             self.pfc_stop_template = os.path.join(
-                TEMPLATES_DIR, "pfc_storm_stop_eos.j2")
+                TEMPLATES_DIR, "pfc_storm_stop_arista_{}.j2".format(self.peer_device.os))
         else:
             self.pfc_stop_template = os.path.join(
                 TEMPLATES_DIR, "pfc_storm_stop_{}.j2".format(self.peer_device.os))

tests/common/templates/pfc_storm_arista_eos.j2

@@ -0,0 +1,9 @@
+bash
+cd {{pfc_gen_dir}}
+{% if (pfc_asym is defined) and (pfc_asym == True) %}
+{% if pfc_storm_defer_time is defined %}sleep {{pfc_storm_defer_time}} &&{% endif %} sudo python3 {{pfc_gen_file}} -c {{pfc_gen_chip_name}} -p {{pfc_queue_index}} -i {{pfc_fanout_interface | replace("Ethernet", "et") | replace("/", "_")}} > /dev/null 2>&1 &
+{% else %}
+{% if pfc_storm_defer_time is defined %}sleep {{pfc_storm_defer_time}} &&{% endif %} sudo python3 {{pfc_gen_file}} -c {{pfc_gen_chip_name}} -p {{(1).__lshift__(pfc_queue_index)}} -i {{pfc_fanout_interface | replace("Ethernet", "et") | replace("/", "_")}} -r {{ansible_eth0_ipv4_addr}} > /dev/null 2>&1 &
+{% endif %}
+exit
+exit