From bf7bd74fd4ca1ef06199618dcc778df01e785b93 Mon Sep 17 00:00:00 2001 From: Sihui Han Date: Tue, 6 Mar 2018 01:36:29 +0000 Subject: [PATCH 1/3] [pfcwd]: Iterate functional test over all ports Signed-off-by: Sihui Han --- ansible/roles/test/files/ptftests/pfc_wd.py | 94 +++++++--- ansible/roles/test/tasks/pfc_wd.yml | 54 +++++- .../tasks/pfc_wd/config_test/config_test.yml | 3 + .../config_test/config_test_ignore_messages | 2 + .../functional_test/functional_test.yml | 172 +++++++++++------- .../functional_test/ignore_pfc_wd_messages | 2 + .../pfc_wd/functional_test/storm_all_test.yml | 2 + .../test/tasks/pfc_wd/iterate_interfaces.yml | 45 +++++ .../tasks/pfc_wd/iterate_portchannels.yml | 73 ++++++++ .../roles/test/tasks/pfc_wd/iterate_vlans.yml | 9 + 10 files changed, 361 insertions(+), 95 deletions(-) create mode 100644 ansible/roles/test/tasks/pfc_wd/iterate_interfaces.yml create mode 100644 ansible/roles/test/tasks/pfc_wd/iterate_portchannels.yml create mode 100644 ansible/roles/test/tasks/pfc_wd/iterate_vlans.yml diff --git a/ansible/roles/test/files/ptftests/pfc_wd.py b/ansible/roles/test/files/ptftests/pfc_wd.py index b25f26fb769..e3d07c7272e 100644 --- a/ansible/roles/test/files/ptftests/pfc_wd.py +++ b/ansible/roles/test/files/ptftests/pfc_wd.py @@ -3,6 +3,9 @@ import random import socket import sys +import struct +import ipaddress +import re import ptf import ptf.packet as scapy @@ -13,6 +16,7 @@ from ptf.mask import Mask from ptf.testutils import * + class PfcWdTest(BaseTest): def __init__(self): BaseTest.__init__(self) @@ -24,8 +28,9 @@ def setUp(self): self.queue_index = int(self.test_params['queue_index']) self.pkt_count = int(self.test_params['pkt_count']) self.port_src = int(self.test_params['port_src']) - self.ip_src = self.test_params['ip_src'] + self.port_dst = self.test_params['port_dst'] self.ip_dst = self.test_params['ip_dst'] + self.port_type = self.test_params['port_type'] self.wd_action = self.test_params.get('wd_action', 'drop') def runTest(self): @@ -33,32 +38,73 @@ def runTest(self): dscp = self.queue_index tos = dscp << 2 tos |= ecn - dst_port_list = range(0,32) - sport = random.randint(0, 65535) - dport = random.randint(0, 65535) + + matches = re.findall('\[([\d\s]+)\]', self.port_dst) + + dst_port_list = [] + for match in matches: + for port in match.split(): + dst_port_list.append(int(port)) src_mac = self.dataplane.get_mac(0, 0) - pkt = simple_tcp_packet( - eth_dst=self.router_mac, - eth_src=src_mac, - ip_src=self.ip_src, - ip_dst=self.ip_dst, - ip_tos = tos, - tcp_sport=sport, - tcp_dport=dport, - ip_ttl=64) - exp_pkt = simple_tcp_packet( - eth_src=self.router_mac, - ip_src=self.ip_src, - ip_dst=self.ip_dst, - ip_tos = tos, - tcp_sport=sport, - tcp_dport=dport, - ip_ttl=63) - masked_exp_pkt = Mask(exp_pkt) - masked_exp_pkt.set_do_not_care_scapy(scapy.Ether, "dst") + if self.port_type == "portchannel": + for x in range(0, self.pkt_count): + sport = random.randint(0, 65535) + dport = random.randint(0, 65535) + ip_src = socket.inet_ntoa(struct.pack('>I', random.randint(1, 0xffffffff))) + ip_src =ipaddress.IPv4Address(unicode(ip_src,'utf-8')) + while ip_src == ipaddress.IPv4Address(unicode(self.ip_dst,'utf-8')) or ip_src.is_multicast or ip_src.is_private or ip_src.is_global or ip_src.is_reserved: + ip_src = socket.inet_ntoa(struct.pack('>I', random.randint(1, 0xffffffff))) + ip_src =ipaddress.IPv4Address(unicode(ip_src,'utf-8')) + + ip_src = str(ip_src) + pkt = simple_tcp_packet( + eth_dst=self.router_mac, + eth_src=src_mac, + ip_src=ip_src, + ip_dst=self.ip_dst, + ip_tos = tos, + tcp_sport=sport, + tcp_dport=dport, + ip_ttl=64) + exp_pkt = simple_tcp_packet( + eth_src=self.router_mac, + ip_src=ip_src, + ip_dst=self.ip_dst, + ip_tos = tos, + tcp_sport=sport, + tcp_dport=dport, + ip_ttl=63) + masked_exp_pkt = Mask(exp_pkt) + masked_exp_pkt.set_do_not_care_scapy(scapy.Ether, "dst") + + send_packet(self, self.port_src, pkt, 1) + else: + sport = random.randint(0, 65535) + dport = random.randint(0, 65535) + ip_src = "1.1.1.1" + + pkt = simple_tcp_packet( + eth_dst=self.router_mac, + eth_src=src_mac, + ip_src=ip_src, + ip_dst=self.ip_dst, + ip_tos = tos, + tcp_sport=sport, + tcp_dport=dport, + ip_ttl=64) + exp_pkt = simple_tcp_packet( + eth_src=self.router_mac, + ip_src=ip_src, + ip_dst=self.ip_dst, + ip_tos = tos, + tcp_sport=sport, + tcp_dport=dport, + ip_ttl=63) + masked_exp_pkt = Mask(exp_pkt) + masked_exp_pkt.set_do_not_care_scapy(scapy.Ether, "dst") - send_packet(self, self.port_src, pkt, self.pkt_count) + send_packet(self, self.port_src, pkt, self.pkt_count) if self.wd_action == 'drop': return verify_no_packet_any(self, masked_exp_pkt, dst_port_list) diff --git a/ansible/roles/test/tasks/pfc_wd.yml b/ansible/roles/test/tasks/pfc_wd.yml index 7e3635ee328..af13809e62e 100644 --- a/ansible/roles/test/tasks/pfc_wd.yml +++ b/ansible/roles/test/tasks/pfc_wd.yml @@ -11,6 +11,7 @@ # # 3. Set pfc_storm_template and pfc_storm_stop_template variables to platform-specific template names. # in roles/test/tasks/pfc_wd/functional_test/set_pfc_storm_templates.yml +# #--------------------------------------------- - set_fact: @@ -39,24 +40,65 @@ - set_fact: neighbors: "{{device_conn}}" +- set_fact: + used: false + first_pair: false + pfc_wd_rx_port: "rx" + pfc_wd_test_port: "test" + +- set_fact: + test_ports: {} + +- name: Iterate all interfaces to get all interface ports info if exists + include: roles/test/tasks/pfc_wd/iterate_interfaces.yml + with_items: "{{minigraph_interfaces}}" + when: item['addr'] | ipv4 + +- set_fact: + restart: True + when: minigraph_portchannels| length > 0 + +- name: Iterate all the portchannels to get all portchanel member ports info if exists. + include: roles/test/tasks/pfc_wd/iterate_portchannels.yml + with_dict: "{{minigraph_portchannels}}" + +- name: Generate ips in VLAN range + get_ip_in_range: num=1 prefix="{{minigraph_vlan_interfaces[0]['addr']}}/{{minigraph_vlan_interfaces[0]['prefixlen']}}" exclude_ips="{{minigraph_vlan_interfaces[0]['addr']}}" + become: no + connection: local + failed_when: False + when: minigraph_vlans | length >0 + +- name: Set unique MACs to PTF interfaces + script: roles/test/files/helpers/change_mac.sh + delegate_to: "{{ptf_host}}" + when: minigraph_vlans | length >0 + +- set_fact: + vlan_members: "{{minigraph_vlans[minigraph_vlans.keys()[0]]['members']}}" + when: minigraph_vlans | length >0 + +- name: Iterate all vlan members to get all vlan member ports info if exists. + include: roles/test/tasks/pfc_wd/iterate_vlans.yml + with_items: "{{vlan_members | default([])}}" + when: minigraph_vlans | length >0 + +- debug: msg="{{test_ports}}" + + #****************************************# # Start tests # #****************************************# - block: - - name: Choose random port for testing - include: roles/test/tasks/pfc_wd/choose_test_port.yml - - name: Test PFC WD configuration validation. vars: pfc_wd_template: roles/test/templates/pfc_wd_config.j2 include: roles/test/tasks/pfc_wd/config_test/config_test.yml - name: Test PFC WD Functional tests. - vars: - pfc_wd_template: roles/test/templates/pfc_wd_config.j2 - peer_device: "{{ neighbors[pfc_wd_test_port]['peerdevice'] }}" include: roles/test/tasks/pfc_wd/functional_test/functional_test.yml + with_dict: "{{test_ports}}" - name: Test PFC WD extreme case when all ports have storm include: roles/test/tasks/pfc_wd/functional_test/storm_all_test.yml diff --git a/ansible/roles/test/tasks/pfc_wd/config_test/config_test.yml b/ansible/roles/test/tasks/pfc_wd/config_test/config_test.yml index 3af6086af0f..38acf66ce24 100644 --- a/ansible/roles/test/tasks/pfc_wd/config_test/config_test.yml +++ b/ansible/roles/test/tasks/pfc_wd/config_test/config_test.yml @@ -1,3 +1,6 @@ +- set_fact: + pfc_wd_test_port: "{{test_ports.keys()[0]}}" + - name: Generate config file for forward action test. vars: pfc_wd_interface_list: "{{ pfc_wd_test_port }}" diff --git a/ansible/roles/test/tasks/pfc_wd/config_test/config_test_ignore_messages b/ansible/roles/test/tasks/pfc_wd/config_test/config_test_ignore_messages index 2f649b28be0..017bda0795b 100644 --- a/ansible/roles/test/tasks/pfc_wd/config_test/config_test_ignore_messages +++ b/ansible/roles/test/tasks/pfc_wd/config_test/config_test_ignore_messages @@ -1,4 +1,6 @@ r, ".* Port counter .* not implemented" r, ".* Port counter .* not supported" r, ".* Invalid port counter .*" +r, ".* Unknown stat.*" +r, ".* snmpd .*" r, ".* Trying to remove nonexisting queue from flex counter .*" diff --git a/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test.yml b/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test.yml index ee13e9dc73c..277e44e814b 100644 --- a/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test.yml +++ b/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test.yml @@ -1,3 +1,28 @@ +- name: Set fact for item values + set_fact: + peer_device: "{{item.value.peer_device}}" + pfc_wd_test_port: "{{item.key}}" + pfc_wd_rx_port: "{{item.value.rx_port}}" + pfc_wd_test_neighbor_addr: "{{item.value.test_neighbor_addr}}" + pfc_wd_rx_neighbor_addr: "{{item.value.rx_neighbor_addr}}" + pfc_wd_test_port_id: "{{item.value.test_port_id}}" + pfc_wd_rx_port_id: "{{item.value.rx_port_id}}" + port_type: "{{item.value.test_port_type}}" + +- name: Remove existing IPs from PTF host + script: roles/test/files/helpers/remove_ip.sh + delegate_to: "{{ptf_host}}" + when: port_type == "vlan" + +- name: "Set {{pfc_wd_test_neighbor_addr}} to eth{{pfc_wd_test_port_id}} on PTF host" + shell: ifconfig eth{{pfc_wd_test_port_id}} {{pfc_wd_test_neighbor_addr}} + delegate_to: "{{ptf_host}}" + when: port_type == "vlan" + +- name: "Update ARP entry on DUT" + shell: docker exec -i swss arping {{pfc_wd_test_neighbor_addr}} -c 5 + when: port_type == "vlan" + - conn_graph_facts: host={{ peer_device }} connection: local become: no @@ -6,7 +31,7 @@ set_fact: pfc_queue_index: 4 pfc_frames_number: 100000 - pfc_wd_test_pkt_count: 10000 + pfc_wd_test_pkt_count: 100 pfc_fanout_interface: "{{neighbors[pfc_wd_test_port]['peerport']}}" peer_hwsku: "{{device_info['HwSku']}}" peer_mgmt: "{{device_info['mgmtip']}}" @@ -20,9 +45,6 @@ - set_fact: class_enable: "{{(1).__lshift__(pfc_queue_index)}}" -- set_fact: pfc_wd_test_neighbor_addr="{{((minigraph_bgp|selectattr('peer_addr', 'equalto', pfc_wd_test_port_addr))|list)[0]['addr']}}" -- set_fact: pfc_wd_rx_neighbor_addr="{{((minigraph_bgp|selectattr('peer_addr', 'equalto', pfc_wd_rx_port_addr))|list)[0]['addr']}}" - - set_fact: peer_login: "{{switch_login[hwsku_map[peer_hwsku]]}}" @@ -34,7 +56,15 @@ pfc_wd_detect_time: 200 pfc_wd_restore_time: 200 pfc_wd_restore_time_large: 50000 - pfc_wd_poll_time: 100 + pfc_wd_poll_time: 200 + +- name: Set timers if user has specified + set_fact: + pfc_wd_restore_time_large: "{{restore_time}}" + when: restore_time is defined + +- name: Set polling interval {{ pfc_wd_poll_time }}. + shell: "pfcwd interval {{ pfc_wd_poll_time }}" - set_fact: pfc_gen_file: pfc_gen.py @@ -77,20 +107,21 @@ - testbed_type='{{testbed_type}}' - router_mac='{{ansible_Ethernet0['macaddress']}}' - queue_index='{{pfc_queue_index}}' - - pkt_count='1000' - - port_src='{{pfc_wd_rx_port_id}}' - - ip_src='{{pfc_wd_rx_neighbor_addr}}' + - pkt_count='{{pfc_wd_test_pkt_count}}' + - port_src='{{pfc_wd_rx_port_id[0]}}' + - port_dst='[{{pfc_wd_test_port_id}}]' - ip_dst='{{pfc_wd_test_neighbor_addr}}' + - port_type='{{port_type}}' - wd_action='drop' ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log " - set_fact: test_expect_file: "expect_pfc_wd_detect" + test_ignore_file: "ignore_pfc_wd_messages" - name: Initialize loganalyzer include: roles/test/files/tools/loganalyzer/loganalyzer_init.yml - - name: Apply drop config to port {{ pfc_wd_test_port }}. shell: "pfcwd start --action drop --restoration-time {{ pfc_wd_restore_time_large }} {{ ports }} {{ pfc_wd_detect_time }}" @@ -120,9 +151,10 @@ - router_mac='{{ansible_Ethernet0['macaddress']}}' - queue_index='{{pfc_queue_index}}' - pkt_count='{{pfc_wd_test_pkt_count}}' - - port_src='{{pfc_wd_rx_port_id}}' - - ip_src='{{pfc_wd_rx_neighbor_addr}}' + - port_src='{{pfc_wd_rx_port_id[0]}}' + - port_dst='[{{pfc_wd_test_port_id}}]' - ip_dst='{{pfc_wd_test_neighbor_addr}}' + - port_type='{{port_type}}' - wd_action='drop' ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log " @@ -139,12 +171,13 @@ - queue_index='{{pfc_queue_index}}' - pkt_count='{{pfc_wd_test_pkt_count}}' - port_src='{{pfc_wd_test_port_id}}' - - ip_src='{{pfc_wd_test_neighbor_addr}}' + - port_dst='[{{pfc_wd_rx_port_id | join(' ')}}]' - ip_dst='{{pfc_wd_rx_neighbor_addr}}' + - port_type='{{port_type}}' - wd_action='drop' ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log " - - name: "Send 10K packets via {{pfc_wd_test_port}} to verify that other queue is not affected" + - name: "Send packets via {{pfc_wd_test_port}} to verify that other queue is not affected" include: roles/test/tasks/ptf_runner.yml vars: ptf_test_name: PFC WD test @@ -156,13 +189,14 @@ - router_mac='{{ansible_Ethernet0['macaddress']}}' - queue_index='{{pfc_queue_index - 1}}' - pkt_count='{{pfc_wd_test_pkt_count}}' - - port_src='{{pfc_wd_rx_port_id}}' - - ip_src='{{pfc_wd_rx_neighbor_addr}}' + - port_src='{{pfc_wd_rx_port_id[0]}}' + - port_dst='[{{pfc_wd_test_port_id}}]' - ip_dst='{{pfc_wd_test_neighbor_addr}}' + - port_type='{{port_type}}' - wd_action='forward' ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log " - - name: "Send 10K packets to {{pfc_wd_test_port}} to verify that other pg is not affected" + - name: "Send packets to {{pfc_wd_test_port}} to verify that other pg is not affected" include: roles/test/tasks/ptf_runner.yml vars: ptf_test_name: PFC WD test @@ -175,8 +209,9 @@ - queue_index='{{pfc_queue_index - 1}}' - pkt_count='{{pfc_wd_test_pkt_count}}' - port_src='{{pfc_wd_test_port_id}}' - - ip_src='{{pfc_wd_test_neighbor_addr}}' + - port_dst='[{{pfc_wd_rx_port_id | join(' ')}}]' - ip_dst='{{pfc_wd_rx_neighbor_addr}}' + - port_type='{{port_type}}' - wd_action='forward' ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log " @@ -195,7 +230,7 @@ - name: Wait for queue to recover from PFC storm pause: - seconds: "{{(pfc_wd_restore_time_large / 1000 * 2) | int}}" + seconds: "{{(pfc_wd_restore_time_large | int / 1000 * 2) | int}}" - name: Check if logs contain message that PFC WD restored from deadlock include: roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml @@ -209,8 +244,8 @@ - name: Verify tx drop counter fail: - msg: "Number of tx dropped packets is not equal to {{ pfc_wd_test_pkt_count }}" - when: "{{ pfc_wd_tx_drop_after.stdout | int - pfc_wd_tx_drop_before.stdout | int != pfc_wd_test_pkt_count }}" + msg: "Number of tx dropped packets {{pfc_wd_tx_drop_after.stdout | int - pfc_wd_tx_drop_before.stdout | int}} is not equal to {{ pfc_wd_test_pkt_count }}" + when: (pfc_wd_tx_drop_after.stdout | int - pfc_wd_tx_drop_before.stdout | int != pfc_wd_test_pkt_count and port_type in ["vlan", "interface"]) or (pfc_wd_tx_drop_after.stdout | int - pfc_wd_tx_drop_before.stdout | int<= 0) - name: Get PFC_WD_QUEUE_STATS_TX_DROPPED_PACKETS_LAST after test shell: "docker exec -i database redis-cli -n 2 HGET COUNTERS:{{ queue_oid.stdout }} PFC_WD_QUEUE_STATS_TX_DROPPED_PACKETS_LAST" @@ -218,8 +253,8 @@ - name: Verify last tx drop counter fail: - msg: "Number of last tx dropped packets is not equal to {{ pfc_wd_test_pkt_count }}" - when: "{{ pfc_wd_tx_drop_last.stdout | int != pfc_wd_test_pkt_count }}" + msg: "Number of last tx dropped packets {{pfc_wd_tx_drop_last.stdout | int}} is not equal to {{ pfc_wd_test_pkt_count }}" + when: (pfc_wd_tx_drop_last.stdout | int != pfc_wd_test_pkt_count and port_type in ["vlan", "interface"]) or (pfc_wd_tx_drop_last.stdout | int <= 0) - name: Get PFC_WD_QUEUE_STATS_RX_DROPPED_PACKETS after test shell: "docker exec -i database redis-cli -n 2 HGET COUNTERS:{{ queue_oid.stdout }} PFC_WD_QUEUE_STATS_RX_DROPPED_PACKETS" @@ -228,7 +263,7 @@ - name: Verify rx drop counter fail: msg: "Number of rx dropped packets is not equal to {{ pfc_wd_test_pkt_count }}" - when: "{{ pfc_wd_rx_drop_after.stdout | int - pfc_wd_rx_drop_before.stdout | int != pfc_wd_test_pkt_count }}" + when: (pfc_wd_rx_drop_after.stdout | int - pfc_wd_rx_drop_before.stdout | int != pfc_wd_test_pkt_count and port_type in ["vlan", "interface"]) or (pfc_wd_rx_drop_after.stdout | int - pfc_wd_rx_drop_before.stdout | int <= 0) - name: Get PFC_WD_QUEUE_STATS_RX_DROPPED_PACKETS_LAST after test shell: "docker exec -i database redis-cli -n 2 HGET COUNTERS:{{ queue_oid.stdout }} PFC_WD_QUEUE_STATS_RX_DROPPED_PACKETS_LAST" @@ -237,7 +272,7 @@ - name: Verify last rx drop counter fail: msg: "Number of last rx dropped packets is not equal to {{ pfc_wd_test_pkt_count }}" - when: "{{ pfc_wd_rx_drop_last.stdout | int != pfc_wd_test_pkt_count }}" + when: (pfc_wd_rx_drop_last.stdout | int != pfc_wd_test_pkt_count and port_type in ["vlan", "interface"]) or (pfc_wd_rx_drop_last.stdout | int <= 0) always: - name: Stop PFC storm on fanout switch @@ -258,7 +293,6 @@ - name: Apply drop config to {{ pfc_wd_test_port }}. vars: command_to_run: "pfcwd start --action drop --restoration-time {{ pfc_wd_restore_time_large }} {{ ports }} {{ pfc_wd_detect_time }}" - test_ignore_file: ignore_pfc_wd_messages errors_expected: false include: roles/test/tasks/run_command_with_log_analyzer.yml @@ -292,7 +326,7 @@ - name: Check if logs contain message that PFC WD detected deadlock include: roles/test/files/tools/loganalyzer/loganalyzer_end.yml - - name: "Send 10K packets via {{pfc_wd_test_port}}" + - name: "Send packets via {{pfc_wd_test_port}}" include: roles/test/tasks/ptf_runner.yml vars: ptf_test_name: PFC WD test @@ -304,13 +338,14 @@ - router_mac='{{ansible_Ethernet0['macaddress']}}' - queue_index='{{pfc_queue_index}}' - pkt_count='{{pfc_wd_test_pkt_count}}' - - port_src='{{pfc_wd_rx_port_id}}' - - ip_src='{{pfc_wd_rx_neighbor_addr}}' + - port_src='{{pfc_wd_rx_port_id[0]}}' + - port_dst='[{{pfc_wd_test_port_id}}]' - ip_dst='{{pfc_wd_test_neighbor_addr}}' + - port_type='{{port_type}}' - wd_action='drop' ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log " - - name: "Send 10K packets to {{pfc_wd_test_port}}" + - name: "Send packets to {{pfc_wd_test_port}}" include: roles/test/tasks/ptf_runner.yml vars: ptf_test_name: PFC WD test @@ -323,12 +358,13 @@ - queue_index='{{pfc_queue_index}}' - pkt_count='{{pfc_wd_test_pkt_count}}' - port_src='{{pfc_wd_test_port_id}}' - - ip_src='{{pfc_wd_test_neighbor_addr}}' + - port_dst='[{{pfc_wd_rx_port_id | join(' ')}}]' - ip_dst='{{pfc_wd_rx_neighbor_addr}}' + - port_type='{{port_type}}' - wd_action='drop' ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log " - - name: "Send 10K packets via {{pfc_wd_test_port}} to verify that other queue is not affected" + - name: "Send packets via {{pfc_wd_test_port}} to verify that other queue is not affected" include: roles/test/tasks/ptf_runner.yml vars: ptf_test_name: PFC WD test @@ -340,13 +376,14 @@ - router_mac='{{ansible_Ethernet0['macaddress']}}' - queue_index='{{pfc_queue_index - 1}}' - pkt_count='{{pfc_wd_test_pkt_count}}' - - port_src='{{pfc_wd_rx_port_id}}' - - ip_src='{{pfc_wd_rx_neighbor_addr}}' + - port_src='{{pfc_wd_rx_port_id[0]}}' + - port_dst='[{{pfc_wd_test_port_id}}]' - ip_dst='{{pfc_wd_test_neighbor_addr}}' + - port_type='{{port_type}}' - wd_action='forward' ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log " - - name: "Send 10K packets to {{pfc_wd_test_port}} to verify that other pg is not affected" + - name: "Send packets to {{pfc_wd_test_port}} to verify that other pg is not affected" include: roles/test/tasks/ptf_runner.yml vars: ptf_test_name: PFC WD test @@ -359,8 +396,9 @@ - queue_index='{{pfc_queue_index - 1}}' - pkt_count='{{pfc_wd_test_pkt_count}}' - port_src='{{pfc_wd_test_port_id}}' - - ip_src='{{pfc_wd_test_neighbor_addr}}' + - port_dst='[{{pfc_wd_rx_port_id | join(' ')}}]' - ip_dst='{{pfc_wd_rx_neighbor_addr}}' + - port_type='{{port_type}}' - wd_action='forward' ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log " @@ -379,7 +417,7 @@ - name: Wait for queue to recover from PFC storm pause: - seconds: "{{(pfc_wd_restore_time_large / 1000 * 2) | int}}" + seconds: "{{(pfc_wd_restore_time_large | int / 1000 * 2) | int}}" - name: Check if logs contain message that PFC WD restored from deadlock include: roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml @@ -393,8 +431,8 @@ - name: Verify tx drop counter fail: - msg: "Number of tx dropped packets is not equal to {{ pfc_wd_test_pkt_count }}" - when: "{{ pfc_wd_tx_drop_after.stdout | int - pfc_wd_tx_drop_before.stdout | int != pfc_wd_test_pkt_count }}" + msg: "Number of tx dropped packets {{pfc_wd_tx_drop_after.stdout | int - pfc_wd_tx_drop_before.stdout | int}} is not equal to {{ pfc_wd_test_pkt_count }}" + when: (pfc_wd_tx_drop_after.stdout | int - pfc_wd_tx_drop_before.stdout | int != pfc_wd_test_pkt_count and port_type in ["vlan", "interface"]) or (pfc_wd_tx_drop_after.stdout | int - pfc_wd_tx_drop_before.stdout | int <= 0) - name: Get PFC_WD_QUEUE_STATS_TX_DROPPED_PACKETS_LAST after test shell: "docker exec -i database redis-cli -n 2 HGET COUNTERS:{{ queue_oid.stdout }} PFC_WD_QUEUE_STATS_TX_DROPPED_PACKETS_LAST" @@ -402,8 +440,8 @@ - name: Verify last tx drop counter fail: - msg: "Number of last tx dropped packets is not equal to {{ pfc_wd_test_pkt_count }}" - when: "{{ pfc_wd_tx_drop_last.stdout | int != pfc_wd_test_pkt_count }}" + msg: "Number of last tx dropped packets {{(pfc_wd_tx_drop_last.stdout | int}} is not equal to {{ pfc_wd_test_pkt_count }}" + when: (pfc_wd_tx_drop_last.stdout | int != pfc_wd_test_pkt_count and port_type in ["vlan", "interface"]) or (pfc_wd_tx_drop_last.stdout | int <= 0) - name: Get PFC_WD_QUEUE_STATS_RX_DROPPED_PACKETS after test shell: "docker exec -i database redis-cli -n 2 HGET COUNTERS:{{ queue_oid.stdout }} PFC_WD_QUEUE_STATS_RX_DROPPED_PACKETS" @@ -411,8 +449,8 @@ - name: Verify rx drop counter fail: - msg: "Number of rx dropped packets is not equal to {{ pfc_wd_test_pkt_count }}" - when: "{{ pfc_wd_rx_drop_after.stdout | int - pfc_wd_rx_drop_before.stdout | int != pfc_wd_test_pkt_count }}" + msg: "Number of rx dropped packets {{pfc_wd_rx_drop_after.stdout | int - pfc_wd_rx_drop_before.stdout | int}} is not equal to {{ pfc_wd_test_pkt_count }}" + when: (pfc_wd_rx_drop_after.stdout | int - pfc_wd_rx_drop_before.stdout | int != pfc_wd_test_pkt_count and port_type in ["vlan", "interface"]) or (pfc_wd_rx_drop_after.stdout | int - pfc_wd_rx_drop_before.stdout | int <= 0) - name: Get PFC_WD_QUEUE_STATS_RX_DROPPED_PACKETS_LAST after test shell: "docker exec -i database redis-cli -n 2 HGET COUNTERS:{{ queue_oid.stdout }} PFC_WD_QUEUE_STATS_RX_DROPPED_PACKETS_LAST" @@ -420,8 +458,12 @@ - name: Verify last rx drop counter fail: - msg: "Number of last rx dropped packets is not equal to {{ pfc_wd_test_pkt_count }}" - when: "{{ pfc_wd_rx_drop_last.stdout | int != pfc_wd_test_pkt_count }}" + msg: "Number of last rx dropped packets {{pfc_wd_rx_drop_last.stdout | int}} is not equal to {{ pfc_wd_test_pkt_count }}" + when: (pfc_wd_rx_drop_last.stdout | int != pfc_wd_test_pkt_count and port_type in ["vlan", "interface"]) or (pfc_wd_rx_drop_last.stdout | int <= 0) + + - name: Get PFC_WD_QUEUE_STATS_TX_DROPPED_PACKETS after test + shell: "docker exec -i database redis-cli -n 2 HGET COUNTERS:{{ queue_oid.stdout }} PFC_WD_QUEUE_STATS_TX_DROPPED_PACKETS" + register: pfc_wd_tx_drop_after always: - name: Stop PFC storm on fanout switch @@ -434,7 +476,6 @@ - name: Clean up config vars: command_to_run: "pfcwd stop" - test_ignore_file: ignore_pfc_wd_messages errors_expected: false include: roles/test/tasks/run_command_with_log_analyzer.yml @@ -480,7 +521,7 @@ - name: Check if logs contain message that PFC WD detected deadlock include: roles/test/files/tools/loganalyzer/loganalyzer_end.yml - - name: "Send 10K packets via {{pfc_wd_test_port}}" + - name: "Send packets via {{pfc_wd_test_port}}" include: roles/test/tasks/ptf_runner.yml vars: ptf_test_name: PFC WD test @@ -491,14 +532,15 @@ - testbed_type='{{testbed_type}}' - router_mac='{{ansible_Ethernet0['macaddress']}}' - queue_index='{{pfc_queue_index}}' - - pkt_count='10000' - - port_src='{{pfc_wd_rx_port_id}}' - - ip_src='{{pfc_wd_rx_neighbor_addr}}' + - pkt_count='{{pfc_wd_test_pkt_count}}' + - port_src='{{pfc_wd_rx_port_id[0]}}' + - port_dst='[{{pfc_wd_test_port_id}}]' - ip_dst='{{pfc_wd_test_neighbor_addr}}' + - port_type='{{port_type}}' - wd_action='forward' ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log " - - name: "Send 10K packets to {{pfc_wd_test_port}}" + - name: "Send packets to {{pfc_wd_test_port}}" include: roles/test/tasks/ptf_runner.yml vars: ptf_test_name: PFC WD test @@ -509,10 +551,11 @@ - testbed_type='{{testbed_type}}' - router_mac='{{ansible_Ethernet0['macaddress']}}' - queue_index='{{pfc_queue_index}}' - - pkt_count='10000' + - pkt_count='{{pfc_wd_test_pkt_count}}' - port_src='{{pfc_wd_test_port_id}}' - - ip_src='{{pfc_wd_test_neighbor_addr}}' + - port_dst='[{{pfc_wd_rx_port_id | join(' ')}}]' - ip_dst='{{pfc_wd_rx_neighbor_addr}}' + - port_type='{{port_type}}' - wd_action='forward' ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log " @@ -531,7 +574,7 @@ - name: Wait for queue to recover from PFC storm pause: - seconds: "{{(pfc_wd_restore_time_large / 1000 * 2) | int}}" + seconds: "{{(pfc_wd_restore_time_large | int / 1000 * 2) | int}}" - name: Check if logs contain message that PFC WD restored from deadlock include: roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml @@ -546,7 +589,7 @@ - name: Verify tx counter fail: msg: "Number of tx packets is not equal to {{ pfc_wd_test_pkt_count }}" - when: "{{ pfc_wd_tx_after.stdout | int - pfc_wd_tx_before.stdout | int != pfc_wd_test_pkt_count }}" + when: (pfc_wd_tx_after.stdout | int - pfc_wd_tx_before.stdout | int != pfc_wd_test_pkt_count and port_type in ["vlan", "interface"]) or (pfc_wd_tx_after.stdout | int - pfc_wd_tx_before.stdout | int <= 0) - name: Get PFC_WD_QUEUE_STATS_TX_PACKETS_LAST after test shell: "docker exec -i database redis-cli -n 2 HGET COUNTERS:{{ queue_oid.stdout }} PFC_WD_QUEUE_STATS_TX_PACKETS_LAST" @@ -555,7 +598,7 @@ - name: Verify last tx counter fail: msg: "Number of last tx packets is not equal to {{ pfc_wd_test_pkt_count }}" - when: "{{ pfc_wd_tx_last.stdout | int != pfc_wd_test_pkt_count }}" + when: (pfc_wd_tx_last.stdout | int != pfc_wd_test_pkt_count and port_type in ["vlan", "interface"]) or (pfc_wd_tx_last.stdout | int <= 0) - name: Get PFC_WD_QUEUE_STATS_RX_PACKETS after test shell: "docker exec -i database redis-cli -n 2 HGET COUNTERS:{{ queue_oid.stdout }} PFC_WD_QUEUE_STATS_RX_PACKETS" @@ -564,7 +607,7 @@ - name: Verify rx counter fail: msg: "Number of rx packets is not equal to {{ pfc_wd_test_pkt_count }}" - when: "{{ pfc_wd_rx_after.stdout | int - pfc_wd_rx_before.stdout | int != pfc_wd_test_pkt_count }}" + when: (pfc_wd_rx_after.stdout | int - pfc_wd_rx_before.stdout | int != pfc_wd_test_pkt_count and port_typein ["vlan", "interface"]) or (pfc_wd_rx_after.stdout | int - pfc_wd_rx_before.stdout | int <= 0) - name: Get PFC_WD_QUEUE_STATS_RX_PACKETS_LAST after test shell: "docker exec -i database redis-cli -n 2 HGET COUNTERS:{{ queue_oid.stdout }} PFC_WD_QUEUE_STATS_RX_PACKETS_LAST" @@ -573,7 +616,7 @@ - name: Verify last rx counter fail: msg: "Number of last rx packets is not equal to {{ pfc_wd_test_pkt_count }}" - when: "{{ pfc_wd_rx_last.stdout | int != pfc_wd_test_pkt_count }}" + when: (pfc_wd_rx_last.stdout | int != pfc_wd_test_pkt_count and port_type in ["vlan", "interface"]) or (pfc_wd_rx_last.stdout | int <= 0) rescue: - name: Remove all temporary files from local host. @@ -604,7 +647,6 @@ - name: Apply config with proper timers to {{ pfc_wd_test_port }}. vars: command_to_run: "pfcwd start --action drop --restoration-time {{ pfc_wd_restore_time }} {{ ports }} {{ pfc_wd_detect_time }}" - test_ignore_file: ignore_pfc_wd_messages errors_expected: false include: roles/test/tasks/run_command_with_log_analyzer.yml @@ -629,7 +671,7 @@ - name: Calculate detection and restoration timings include: roles/test/tasks/pfc_wd/functional_test/timer_test.yml - with_sequence: start=1 end=3 + with_sequence: start=1 end=5 - debug: var: "{{item}}" @@ -640,22 +682,22 @@ - name: Verify that real detection time is not greater than configured fail: msg: Real detection time is greater than configured - when: "{{(detect_time_list | sum) / 3 > pfc_wd_detect_time + pfc_wd_poll_time}}" + when: "{{(detect_time_list | sum) / 5 > pfc_wd_detect_time + pfc_wd_poll_time}}" - name: Verify that real detection time is not less than configured fail: msg: Real detection time is less than configured - when: "{{(detect_time_list | sum) / 3 < pfc_wd_detect_time}}" + when: "{{(detect_time_list | sum) / 5 < pfc_wd_detect_time}}" - name: Verify that real restoration time is not less than configured fail: - msg: Real restoration time is less than configured - when: "{{(restore_time_list | sum) / 3 < pfc_wd_restore_time}}" + msg: Real restoration time is less than configured + when: "{{(restore_time_list | sum) / 5 < pfc_wd_restore_time}}" - name: Verify that real restoration time is less than configured fail: - msg: Real restoration time is greater than configured - when: "{{(restore_time_list | sum) / 3 > pfc_wd_restore_time + pfc_wd_poll_time}}" + msg: Real restoration time is greater than configured + when: "{{(restore_time_list | sum) / 5 > pfc_wd_restore_time + pfc_wd_poll_time}}" always: - name: Clean up config diff --git a/ansible/roles/test/tasks/pfc_wd/functional_test/ignore_pfc_wd_messages b/ansible/roles/test/tasks/pfc_wd/functional_test/ignore_pfc_wd_messages index 2f649b28be0..017bda0795b 100644 --- a/ansible/roles/test/tasks/pfc_wd/functional_test/ignore_pfc_wd_messages +++ b/ansible/roles/test/tasks/pfc_wd/functional_test/ignore_pfc_wd_messages @@ -1,4 +1,6 @@ r, ".* Port counter .* not implemented" r, ".* Port counter .* not supported" r, ".* Invalid port counter .*" +r, ".* Unknown stat.*" +r, ".* snmpd .*" r, ".* Trying to remove nonexisting queue from flex counter .*" diff --git a/ansible/roles/test/tasks/pfc_wd/functional_test/storm_all_test.yml b/ansible/roles/test/tasks/pfc_wd/functional_test/storm_all_test.yml index 9f3c8e05719..8313a444df8 100644 --- a/ansible/roles/test/tasks/pfc_wd/functional_test/storm_all_test.yml +++ b/ansible/roles/test/tasks/pfc_wd/functional_test/storm_all_test.yml @@ -41,6 +41,7 @@ - name: Start to copy and send PFC packets from all peers vars: test_expect_file: "expect_pfc_wd_detect" + test_ignore_file: "ignore_pfc_wd_messages" storm_action: "start" include: roles/test/tasks/pfc_wd/functional_test/storm_all_action.yml with_dict: "{{combined_dict}}" @@ -48,6 +49,7 @@ - name: Stop PFC packets from all peers vars: test_expect_file: "expect_pfc_wd_restore" + test_ignore_file: "ignore_pfc_wd_messages" storm_action: "stop" include: roles/test/tasks/pfc_wd/functional_test/storm_all_action.yml with_dict: "{{combined_dict}}" diff --git a/ansible/roles/test/tasks/pfc_wd/iterate_interfaces.yml b/ansible/roles/test/tasks/pfc_wd/iterate_interfaces.yml new file mode 100644 index 00000000000..4b88c007464 --- /dev/null +++ b/ansible/roles/test/tasks/pfc_wd/iterate_interfaces.yml @@ -0,0 +1,45 @@ +- debug: msg="running interface" + +- set_fact: + rx_port: "{{item['attachto']}}" + used: true + when: rx_port is not defined + +- set_fact: + test_port: "{{item['attachto']}}" + first_pair: true + used: true + when: not used and test_port is not defined + +- set_fact: + test_port: "{{item['attachto']}}" + when: not used + +- set_fact: + pfc_wd_test_port: "{{test_port}}" + pfc_wd_rx_port: "{{rx_port}}" + when: rx_port is defined and test_port is defined + +- set_fact: + pfc_wd_test_port_addr: "{%for p in minigraph_interfaces%}{%if p['attachto']==pfc_wd_test_port and p['addr']|ipv4%}{{p['addr']}}{%endif %}{%endfor%}" + pfc_wd_rx_port_addr: "{%for p in minigraph_interfaces%}{%if p['attachto']==pfc_wd_rx_port and p['addr']|ipv4%}{{p['addr']}}{%endif %}{%endfor%}" + when: rx_port is defined and test_port is defined + +- set_fact: + pfc_wd_test_neighbor_addr: "{%for p in minigraph_bgp%}{%if p['peer_addr']==pfc_wd_test_port_addr and p['addr']|ipv4%}{{p['addr']}}{%endif %}{%endfor%}" + pfc_wd_rx_neighbor_addr: "{%for p in minigraph_bgp%}{%if p['peer_addr']==pfc_wd_rx_port_addr and p['addr']|ipv4%}{{p['addr']}}{%endif %}{%endfor%}" + pfc_wd_rx_port_id: "{{minigraph_port_indices[pfc_wd_rx_port]}}" + pfc_wd_test_port_id: "{{minigraph_port_indices[pfc_wd_test_port]}}" + when: rx_port is defined and test_port is defined + +- set_fact: + test_ports: "{{ test_ports | combine( {pfc_wd_test_port:{'test_neighbor_addr':pfc_wd_test_neighbor_addr, 'rx_port': [pfc_wd_rx_port], 'rx_neighbor_addr': pfc_wd_rx_neighbor_addr, 'peer_device':neighbors[pfc_wd_test_port]['peerdevice'], 'test_port_id': pfc_wd_test_port_id, 'rx_port_id':[pfc_wd_rx_port_id], 'test_port_type':'interface' }}) }}" + when: rx_port is defined and test_port is defined + +- set_fact: + test_ports: "{{ test_ports | combine( {pfc_wd_rx_port:{'test_neighbor_addr':pfc_wd_rx_neighbor_addr, 'rx_port': [pfc_wd_test_port], 'rx_neighbor_addr': pfc_wd_test_neighbor_addr, 'peer_device':neighbors[pfc_wd_rx_port]['peerdevice'], 'test_port_id': pfc_wd_rx_port_id, 'rx_port_id':[pfc_wd_test_port_id], 'test_port_type':'interface' }}) }}" + when: first_pair + +- set_fact: + used: false + first_pair: false diff --git a/ansible/roles/test/tasks/pfc_wd/iterate_portchannels.yml b/ansible/roles/test/tasks/pfc_wd/iterate_portchannels.yml new file mode 100644 index 00000000000..1227fa849d6 --- /dev/null +++ b/ansible/roles/test/tasks/pfc_wd/iterate_portchannels.yml @@ -0,0 +1,73 @@ +- set_fact: + rx_pc_port: "{{item.value.members}}" + rx_portchannel: "{{item.key}}" + used: true + when: rx_pc_port is not defined + +- set_fact: + test_pc_port: "{{item.value.members }}" + test_portchannel: "{{item.key}}" + first_pair: true + used: true + when: not used and test_pc_port is not defined + +- set_fact: + test_pc_port: "{{item.value.members}}" + test_portchannel: "{{item.key}}" + when: not used + +- set_fact: + pfc_wd_test_port: "{{test_pc_port}}" + pfc_wd_rx_port: "{{rx_pc_port}}" + pfc_wd_test_portchannel: "{{test_portchannel}}" + pfc_wd_rx_portchannel: "{{rx_portchannel}}" + when: rx_pc_port is defined and test_pc_port is defined + +- set_fact: + pfc_wd_test_port_addr: "{%for p in minigraph_portchannel_interfaces%}{%if p['attachto']==pfc_wd_test_portchannel and p['addr']|ipv4%}{{p['addr']}}{%endif %}{%endfor%}" + pfc_wd_rx_port_addr: "{%for p in minigraph_portchannel_interfaces%}{%if p['attachto']==pfc_wd_rx_portchannel and p['addr']|ipv4%}{{p['addr']}}{%endif %}{%endfor%}" + peer_device: "{{ neighbors[pfc_wd_test_port[0]]['peerdevice'] }}" + pfc_wd_rx_port_id: "{%for port in pfc_wd_rx_port %}{{minigraph_port_indices[port]}}{%if not loop.last%} {%endif%}{%endfor%}" + pfc_wd_test_port_id: "{%for port in pfc_wd_test_port %}{{minigraph_port_indices[port]}}{%if not loop.last%} {%endif%}{%endfor%}" + when: rx_pc_port is defined and test_pc_port is defined + +- set_fact: + pfc_wd_test_neighbor_addr: "{%for p in minigraph_bgp%}{%if p['peer_addr']==pfc_wd_test_port_addr and p['addr']|ipv4%}{{p['addr']}}{%endif %}{%endfor%}" + pfc_wd_rx_neighbor_addr: "{%for p in minigraph_bgp%}{%if p['peer_addr']==pfc_wd_rx_port_addr and p['addr']|ipv4%}{{p['addr']}}{%endif %}{%endfor%}" + when: rx_pc_port is defined and test_pc_port is defined + +- set_fact: + p: "{{pfc_wd_test_port[0]}}" + when: rx_pc_port is defined and test_pc_port is defined + +- set_fact: + test_ports: "{{ test_ports | combine( {p:{'test_neighbor_addr':pfc_wd_test_neighbor_addr, 'rx_port':pfc_wd_rx_port, 'rx_neighbor_addr': pfc_wd_rx_neighbor_addr, 'peer_device':neighbors[p]['peerdevice'], 'test_port_id': minigraph_port_indices[p], 'rx_port_id': pfc_wd_rx_port_id.split(' ') | list, 'test_port_type':'portchannel' }}) }}" + when: rx_pc_port is defined and test_pc_port is defined + +- set_fact: + p: "{{pfc_wd_test_port[1]}}" + when: rx_pc_port is defined and test_pc_port is defined and testbed_type in ['t0-64', 't0-112', 't1-lag'] + +- set_fact: + test_ports: "{{ test_ports | combine( {p:{'test_neighbor_addr':pfc_wd_test_neighbor_addr, 'rx_port':pfc_wd_rx_port, 'rx_neighbor_addr': pfc_wd_rx_neighbor_addr, 'peer_device':neighbors[p]['peerdevice'], 'test_port_id': minigraph_port_indices[p], 'rx_port_id': pfc_wd_rx_port_id.split(' ') | list, 'test_port_type':'portchannel' }}) }}" + when: rx_pc_port is defined and test_pc_port is defined and testbed_type in ['t0-64', 't0-112', 't1-lag'] + +- set_fact: + p: "{{pfc_wd_rx_port[0]}}" + when: first_pair + +- set_fact: + test_ports: "{{ test_ports | combine( {p:{'test_neighbor_addr':pfc_wd_rx_neighbor_addr, 'rx_port':pfc_wd_test_port, 'rx_neighbor_addr': pfc_wd_test_neighbor_addr, 'peer_device':neighbors[p]['peerdevice'], 'test_port_id': minigraph_port_indices[p], 'rx_port_id': pfc_wd_test_port_id.split(' ') | list , 'test_port_type':'portchannel' }}) }}" + when: first_pair + +- set_fact: + p: "{{pfc_wd_rx_port[1]}}" + when: first_pair and testbed_type in ['t0-64', 't0-112'] + +- set_fact: + test_ports: "{{ test_ports | combine( {p:{'test_neighbor_addr':pfc_wd_rx_neighbor_addr, 'rx_port':pfc_wd_test_port, 'rx_neighbor_addr': pfc_wd_test_neighbor_addr, 'peer_device':neighbors[p]['peerdevice'], 'test_port_id': minigraph_port_indices[p], 'rx_port_id': pfc_wd_test_port_id.split(' ') | list , 'test_port_type':'portchannel' }}) }}" + when: first_pair and testbed_type in ['t0-64', 't0-112'] + +- set_fact: + used: false + first_pair: false diff --git a/ansible/roles/test/tasks/pfc_wd/iterate_vlans.yml b/ansible/roles/test/tasks/pfc_wd/iterate_vlans.yml new file mode 100644 index 00000000000..a0e8e3bb7a8 --- /dev/null +++ b/ansible/roles/test/tasks/pfc_wd/iterate_vlans.yml @@ -0,0 +1,9 @@ +- debug: msg="vlan running" +- debug: msg="{{pfc_wd_rx_port}}" + + #- name: flush the ip on ptf + + + #- name: assgin ip to ptf neightbor +- set_fact: + test_ports: "{{ test_ports | combine( {item:{'test_neighbor_addr':generated_ips[0].split('/')[0], 'rx_port': pfc_wd_rx_port, 'rx_neighbor_addr': pfc_wd_rx_neighbor_addr, 'peer_device':neighbors[item]['peerdevice'], 'test_port_id': minigraph_port_indices[item], 'rx_port_id': pfc_wd_rx_port_id.split(' ') | list, 'test_port_type':'vlan' }}) }}" From 1c4540de66f01e90d8da5b35e5f502df178220e4 Mon Sep 17 00:00:00 2001 From: Sihui Han Date: Tue, 13 Mar 2018 00:34:45 +0000 Subject: [PATCH 2/3] stablize the test and support t0-64 topo --- ansible/roles/test/tasks/interface.yml | 5 + ansible/roles/test/tasks/pfc_wd.yml | 24 +++- .../check_timer_accuracy_test.yml | 66 +++++++++ .../functional_test/functional_test.yml | 127 +++++++----------- .../functional_test/storm_all_action.yml | 2 +- .../pfc_wd/functional_test/timer_test.yml | 13 +- .../tasks/pfc_wd/iterate_portchannels.yml | 8 +- .../roles/test/templates/pfc_storm_arista.j2 | 3 +- .../roles/test/templates/pfc_storm_mlnx.j2 | 2 +- 9 files changed, 157 insertions(+), 93 deletions(-) create mode 100644 ansible/roles/test/tasks/pfc_wd/functional_test/check_timer_accuracy_test.yml diff --git a/ansible/roles/test/tasks/interface.yml b/ansible/roles/test/tasks/interface.yml index 4913ae95256..602f55ed466 100644 --- a/ansible/roles/test/tasks/interface.yml +++ b/ansible/roles/test/tasks/interface.yml @@ -1,3 +1,8 @@ +- name: Gather minigraph facts about the device + minigraph_facts: host={{inventory_hostname}} + become: no + connection: local + - name: Get interface facts interface_facts: up_ports={{minigraph_ports}} diff --git a/ansible/roles/test/tasks/pfc_wd.yml b/ansible/roles/test/tasks/pfc_wd.yml index af13809e62e..e52b4640a98 100644 --- a/ansible/roles/test/tasks/pfc_wd.yml +++ b/ansible/roles/test/tasks/pfc_wd.yml @@ -11,7 +11,6 @@ # # 3. Set pfc_storm_template and pfc_storm_stop_template variables to platform-specific template names. # in roles/test/tasks/pfc_wd/functional_test/set_pfc_storm_templates.yml -# #--------------------------------------------- - set_fact: @@ -40,6 +39,10 @@ - set_fact: neighbors: "{{device_conn}}" +- set_fact: + ansible_eth0_ipv4_addr: "{{ansible_eth0['ipv4']['address']}}" + ansible_ethernet0_mac_addr: "{{ansible_Ethernet0['macaddress']}}" + - set_fact: used: false first_pair: false @@ -83,8 +86,22 @@ with_items: "{{vlan_members | default([])}}" when: minigraph_vlans | length >0 -- debug: msg="{{test_ports}}" +- name: Flush vlan ips route + command: ip route flush {{generated_ips[0].split('/')[0]}}/32 + become: yes + when: minigraph_vlans | length >0 + +- name: Add vlan ips route + command: ip route add {{generated_ips[0].split('/')[0]}}/32 dev {{minigraph_vlan_interfaces[0]['attachto']}} + become: yes + when: minigraph_vlans | length >0 + +- name: "Update ARP entry on DUT" + shell: ping {{minigraph_vlan_interfaces[0]['addr']}} -c 5 + delegate_to: "{{ptf_host}}" + when: minigraph_vlans | length >0 +- debug: msg="{{test_ports}}" #****************************************# # Start tests # @@ -100,6 +117,9 @@ include: roles/test/tasks/pfc_wd/functional_test/functional_test.yml with_dict: "{{test_ports}}" + - name: Test PFC WD Timer accuracy. + include: roles/test/tasks/pfc_wd/functional_test/check_timer_accuracy_test.yml + - name: Test PFC WD extreme case when all ports have storm include: roles/test/tasks/pfc_wd/functional_test/storm_all_test.yml diff --git a/ansible/roles/test/tasks/pfc_wd/functional_test/check_timer_accuracy_test.yml b/ansible/roles/test/tasks/pfc_wd/functional_test/check_timer_accuracy_test.yml new file mode 100644 index 00000000000..2cffa317d3f --- /dev/null +++ b/ansible/roles/test/tasks/pfc_wd/functional_test/check_timer_accuracy_test.yml @@ -0,0 +1,66 @@ +# Verify timers +- set_fact: + pfc_wd_test_port: "{{test_ports.keys()[0]}}" + +- block: + - name: Apply config with proper timers to {{ pfc_wd_test_port }}. + vars: + command_to_run: "pfcwd start --action drop --restoration-time {{ pfc_wd_restore_time }} {{ ports }} {{ pfc_wd_detect_time }}" + errors_expected: false + include: roles/test/tasks/run_command_with_log_analyzer.yml + + - name: Enable routing from mgmt interface to localhost + sysctl: + name: net.ipv4.conf.eth0.route_localnet + value: 1 + sysctl_set: yes + become: true + + # Cannot use iptables module, because it does not support to_destination in 2.0 + - name: Add iptables rule to forward syslog packets from mgmt interface to localhost + shell: "iptables -t nat -I PREROUTING -p udp -d {{ansible_eth0_ipv4_addr}} --dport 514 -j DNAT --to-destination 127.0.0.1:514" + become: true + + - set_fact: + detect_time_list: [] + restore_time_list: [] + + - set_fact: + pfc_frames_number: 300000 + + - name: Calculate detection and restoration timings + include: roles/test/tasks/pfc_wd/functional_test/timer_test.yml + with_sequence: start=1 end=20 + + - debug: + var: "{{item}}" + with_items: + - detect_time_list + - restore_time_list + + - name: Verify that real detection time is not greater than configured + fail: + msg: Real detection time is greater than configured + when: "{{(detect_time_list | sum)/20 > pfc_wd_detect_time + pfc_wd_poll_time}}" + + - name: Verify that real detection time is not less than configured + fail: + msg: Real detection time is less than configured + when: "{{(detect_time_list | sum)/20 < pfc_wd_detect_time}}" + + - name: Verify that real restoration time is not less than configured + fail: + msg: Real restoration time is less than configured + when: "{{(restore_time_list | sum)/20 < pfc_wd_restore_time}}" + + - name: Verify that real restoration time is less than configured + fail: + msg: Real restoration time is greater than configured + when: "{{(restore_time_list | sum)/20 > pfc_wd_restore_time + pfc_wd_poll_time}}" + + always: + - name: Clean up config + vars: + command_to_run: "pfcwd stop" + errors_expected: false + include: roles/test/tasks/run_command_with_log_analyzer.yml diff --git a/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test.yml b/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test.yml index 277e44e814b..8a7ca0c2b50 100644 --- a/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test.yml +++ b/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test.yml @@ -9,6 +9,15 @@ pfc_wd_rx_port_id: "{{item.value.rx_port_id}}" port_type: "{{item.value.test_port_type}}" +- name: Set fact for test port ids and include portchannel other members if exists + set_fact: + pfc_wd_test_port_ids: "[{{item.value.test_portchannel_members | join(' ')}}]" + when: port_type == "portchannel" + +- set_fact: + pfc_wd_test_port_ids: "[{{pfc_wd_test_port_id}}]" + when: port_type in ["vlan", "interface"] + - name: Remove existing IPs from PTF host script: roles/test/files/helpers/remove_ip.sh delegate_to: "{{ptf_host}}" @@ -19,6 +28,11 @@ delegate_to: "{{ptf_host}}" when: port_type == "vlan" + #- name: "Update ARP entry on DUT" + #shell: ping {{minigraph_vlan_interfaces[0]['addr']}} -c 5 -I eth{{pfc_wd_test_port_id}} + #delegate_to: "{{ptf_host}}" + #when: port_type == "vlan" + - name: "Update ARP entry on DUT" shell: docker exec -i swss arping {{pfc_wd_test_neighbor_addr}} -c 5 when: port_type == "vlan" @@ -53,10 +67,10 @@ - name: Set timers set_fact: - pfc_wd_detect_time: 200 - pfc_wd_restore_time: 200 + pfc_wd_detect_time: 400 + pfc_wd_restore_time: 400 pfc_wd_restore_time_large: 50000 - pfc_wd_poll_time: 200 + pfc_wd_poll_time: 400 - name: Set timers if user has specified set_fact: @@ -103,9 +117,10 @@ ptf_test_dir: ptftests ptf_test_path: pfc_wd.PfcWdTest ptf_platform: remote + ptf_platform_dir: ptftests ptf_test_params: - testbed_type='{{testbed_type}}' - - router_mac='{{ansible_Ethernet0['macaddress']}}' + - router_mac='{{ansible_ethernet0_mac_addr}}' - queue_index='{{pfc_queue_index}}' - pkt_count='{{pfc_wd_test_pkt_count}}' - port_src='{{pfc_wd_rx_port_id[0]}}' @@ -125,6 +140,10 @@ - name: Apply drop config to port {{ pfc_wd_test_port }}. shell: "pfcwd start --action drop --restoration-time {{ pfc_wd_restore_time_large }} {{ ports }} {{ pfc_wd_detect_time }}" + - name: Let PFC storm happen for a while + pause: + seconds: 1 + - name: Check if logs contain message that PFC WD detected deadlock include: roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml @@ -146,9 +165,10 @@ ptf_test_dir: ptftests ptf_test_path: pfc_wd.PfcWdTest ptf_platform: remote + ptf_platform_dir: ptftests ptf_test_params: - testbed_type='{{testbed_type}}' - - router_mac='{{ansible_Ethernet0['macaddress']}}' + - router_mac='{{ansible_ethernet0_mac_addr}}' - queue_index='{{pfc_queue_index}}' - pkt_count='{{pfc_wd_test_pkt_count}}' - port_src='{{pfc_wd_rx_port_id[0]}}' @@ -165,9 +185,10 @@ ptf_test_dir: ptftests ptf_test_path: pfc_wd.PfcWdTest ptf_platform: remote + ptf_platform_dir: ptftests ptf_test_params: - testbed_type='{{testbed_type}}' - - router_mac='{{ansible_Ethernet0['macaddress']}}' + - router_mac='{{ansible_ethernet0_mac_addr}}' - queue_index='{{pfc_queue_index}}' - pkt_count='{{pfc_wd_test_pkt_count}}' - port_src='{{pfc_wd_test_port_id}}' @@ -184,13 +205,14 @@ ptf_test_dir: ptftests ptf_test_path: pfc_wd.PfcWdTest ptf_platform: remote + ptf_platform_dir: ptftests ptf_test_params: - testbed_type='{{testbed_type}}' - - router_mac='{{ansible_Ethernet0['macaddress']}}' + - router_mac='{{ansible_ethernet0_mac_addr}}' - queue_index='{{pfc_queue_index - 1}}' - pkt_count='{{pfc_wd_test_pkt_count}}' - port_src='{{pfc_wd_rx_port_id[0]}}' - - port_dst='[{{pfc_wd_test_port_id}}]' + - port_dst='{{pfc_wd_test_port_ids}}' - ip_dst='{{pfc_wd_test_neighbor_addr}}' - port_type='{{port_type}}' - wd_action='forward' @@ -203,9 +225,10 @@ ptf_test_dir: ptftests ptf_test_path: pfc_wd.PfcWdTest ptf_platform: remote + ptf_platform_dir: ptftests ptf_test_params: - testbed_type='{{testbed_type}}' - - router_mac='{{ansible_Ethernet0['macaddress']}}' + - router_mac='{{ansible_ethernet0_mac_addr}}' - queue_index='{{pfc_queue_index - 1}}' - pkt_count='{{pfc_wd_test_pkt_count}}' - port_src='{{pfc_wd_test_port_id}}' @@ -333,9 +356,10 @@ ptf_test_dir: ptftests ptf_test_path: pfc_wd.PfcWdTest ptf_platform: remote + ptf_platform_dir: ptftests ptf_test_params: - testbed_type='{{testbed_type}}' - - router_mac='{{ansible_Ethernet0['macaddress']}}' + - router_mac='{{ansible_ethernet0_mac_addr}}' - queue_index='{{pfc_queue_index}}' - pkt_count='{{pfc_wd_test_pkt_count}}' - port_src='{{pfc_wd_rx_port_id[0]}}' @@ -352,9 +376,10 @@ ptf_test_dir: ptftests ptf_test_path: pfc_wd.PfcWdTest ptf_platform: remote + ptf_platform_dir: ptftests ptf_test_params: - testbed_type='{{testbed_type}}' - - router_mac='{{ansible_Ethernet0['macaddress']}}' + - router_mac='{{ansible_ethernet0_mac_addr}}' - queue_index='{{pfc_queue_index}}' - pkt_count='{{pfc_wd_test_pkt_count}}' - port_src='{{pfc_wd_test_port_id}}' @@ -371,13 +396,14 @@ ptf_test_dir: ptftests ptf_test_path: pfc_wd.PfcWdTest ptf_platform: remote + ptf_platform_dir: ptftests ptf_test_params: - testbed_type='{{testbed_type}}' - - router_mac='{{ansible_Ethernet0['macaddress']}}' + - router_mac='{{ansible_ethernet0_mac_addr}}' - queue_index='{{pfc_queue_index - 1}}' - pkt_count='{{pfc_wd_test_pkt_count}}' - port_src='{{pfc_wd_rx_port_id[0]}}' - - port_dst='[{{pfc_wd_test_port_id}}]' + - port_dst='{{pfc_wd_test_port_ids}}' - ip_dst='{{pfc_wd_test_neighbor_addr}}' - port_type='{{port_type}}' - wd_action='forward' @@ -390,9 +416,10 @@ ptf_test_dir: ptftests ptf_test_path: pfc_wd.PfcWdTest ptf_platform: remote + ptf_platform_dir: ptftests ptf_test_params: - testbed_type='{{testbed_type}}' - - router_mac='{{ansible_Ethernet0['macaddress']}}' + - router_mac='{{ansible_ethernet0_mac_addr}}' - queue_index='{{pfc_queue_index - 1}}' - pkt_count='{{pfc_wd_test_pkt_count}}' - port_src='{{pfc_wd_test_port_id}}' @@ -528,13 +555,14 @@ ptf_test_dir: ptftests ptf_test_path: pfc_wd.PfcWdTest ptf_platform: remote + ptf_platform_dir: ptftests ptf_test_params: - testbed_type='{{testbed_type}}' - - router_mac='{{ansible_Ethernet0['macaddress']}}' + - router_mac='{{ansible_ethernet0_mac_addr}}' - queue_index='{{pfc_queue_index}}' - pkt_count='{{pfc_wd_test_pkt_count}}' - port_src='{{pfc_wd_rx_port_id[0]}}' - - port_dst='[{{pfc_wd_test_port_id}}]' + - port_dst='{{pfc_wd_test_port_ids}}' - ip_dst='{{pfc_wd_test_neighbor_addr}}' - port_type='{{port_type}}' - wd_action='forward' @@ -547,9 +575,10 @@ ptf_test_dir: ptftests ptf_test_path: pfc_wd.PfcWdTest ptf_platform: remote + ptf_platform_dir: ptftests ptf_test_params: - testbed_type='{{testbed_type}}' - - router_mac='{{ansible_Ethernet0['macaddress']}}' + - router_mac='{{ansible_ethernet0_mac_addr}}' - queue_index='{{pfc_queue_index}}' - pkt_count='{{pfc_wd_test_pkt_count}}' - port_src='{{pfc_wd_test_port_id}}' @@ -641,67 +670,3 @@ errors_expected: false include: roles/test/tasks/run_command_with_log_analyzer.yml -# Verify timers - -- block: - - name: Apply config with proper timers to {{ pfc_wd_test_port }}. - vars: - command_to_run: "pfcwd start --action drop --restoration-time {{ pfc_wd_restore_time }} {{ ports }} {{ pfc_wd_detect_time }}" - errors_expected: false - include: roles/test/tasks/run_command_with_log_analyzer.yml - - - name: Enable routing from mgmt interface to localhost - sysctl: - name: net.ipv4.conf.eth0.route_localnet - value: 1 - sysctl_set: yes - become: true - - # Cannot use iptables module, because it does not support to_destination in 2.0 - - name: Add iptables rule to forward syslog packets from mgmt interface to localhost - shell: "iptables -t nat -I PREROUTING -p udp -d {{ansible_eth0['ipv4']['address']}} --dport 514 -j DNAT --to-destination 127.0.0.1:514" - become: true - - - set_fact: - detect_time_list: [] - restore_time_list: [] - - - set_fact: - pfc_frames_number: 100000 - - - name: Calculate detection and restoration timings - include: roles/test/tasks/pfc_wd/functional_test/timer_test.yml - with_sequence: start=1 end=5 - - - debug: - var: "{{item}}" - with_items: - - detect_time_list - - restore_time_list - - - name: Verify that real detection time is not greater than configured - fail: - msg: Real detection time is greater than configured - when: "{{(detect_time_list | sum) / 5 > pfc_wd_detect_time + pfc_wd_poll_time}}" - - - name: Verify that real detection time is not less than configured - fail: - msg: Real detection time is less than configured - when: "{{(detect_time_list | sum) / 5 < pfc_wd_detect_time}}" - - - name: Verify that real restoration time is not less than configured - fail: - msg: Real restoration time is less than configured - when: "{{(restore_time_list | sum) / 5 < pfc_wd_restore_time}}" - - - name: Verify that real restoration time is less than configured - fail: - msg: Real restoration time is greater than configured - when: "{{(restore_time_list | sum) / 5 > pfc_wd_restore_time + pfc_wd_poll_time}}" - - always: - - name: Clean up config - vars: - command_to_run: "pfcwd stop" - errors_expected: false - include: roles/test/tasks/run_command_with_log_analyzer.yml diff --git a/ansible/roles/test/tasks/pfc_wd/functional_test/storm_all_action.yml b/ansible/roles/test/tasks/pfc_wd/functional_test/storm_all_action.yml index c98fc298583..1b64eff7296 100644 --- a/ansible/roles/test/tasks/pfc_wd/functional_test/storm_all_action.yml +++ b/ansible/roles/test/tasks/pfc_wd/functional_test/storm_all_action.yml @@ -27,7 +27,7 @@ - name: Initialize loganalyzer include: roles/test/files/tools/loganalyzer/loganalyzer_init.yml -- name: Generate PFC storm on fanout switch +- name: Take action on fanout switch action: apswitch template="{{storm_action_template}}" args: host: "{{peer_mgmt}}" diff --git a/ansible/roles/test/tasks/pfc_wd/functional_test/timer_test.yml b/ansible/roles/test/tasks/pfc_wd/functional_test/timer_test.yml index c4a668983c3..39e67b86d80 100644 --- a/ansible/roles/test/tasks/pfc_wd/functional_test/timer_test.yml +++ b/ansible/roles/test/tasks/pfc_wd/functional_test/timer_test.yml @@ -2,6 +2,11 @@ shell: logrotate -f /etc/logrotate.conf become: true +- name: Wait for queue to recover from PFC storm + pause: + seconds: 1 + + - name: Generate PFC storm on fanout switch action: apswitch template="{{pfc_wd_storm_template}}" args: @@ -16,6 +21,7 @@ - name: Find PFC storm start marker shell: grep "PFC_STORM_START" /var/log/syslog register: storm_start + become: true - name: Convert start marker time to milliseconds shell: "date -d {{storm_start.stdout.replace(' ',' ').split(' ')[2]}} +'%s%3N'" @@ -24,6 +30,7 @@ - name: Find PFC storm detect message shell: grep "detected PFC storm" /var/log/syslog register: storm_detect + become: true - name: Convert detect message time to milliseconds shell: "date -d {{storm_detect.stdout.replace(' ',' ').split(' ')[2]}} +'%s%3N'" @@ -32,14 +39,16 @@ - name: Find PFC storm end marker shell: grep "PFC_STORM_END" /var/log/syslog register: storm_end + become: true -- name: Convert start marker time to milliseconds +- name: Convert end marker time to milliseconds shell: "date -d {{storm_end.stdout.replace(' ',' ').split(' ')[2]}} +'%s%3N'" register: storm_end_millis - name: Find PFC storm restore message - shell: grep "PFC Watchdog storm restored" /var/log/syslog + shell: grep "storm restored" /var/log/syslog register: storm_restore + become: true - name: Convert restore message time to milliseconds shell: "date -d {{storm_restore.stdout.replace(' ',' ').split(' ')[2]}} +'%s%3N'" diff --git a/ansible/roles/test/tasks/pfc_wd/iterate_portchannels.yml b/ansible/roles/test/tasks/pfc_wd/iterate_portchannels.yml index 1227fa849d6..44444517c4e 100644 --- a/ansible/roles/test/tasks/pfc_wd/iterate_portchannels.yml +++ b/ansible/roles/test/tasks/pfc_wd/iterate_portchannels.yml @@ -41,7 +41,7 @@ when: rx_pc_port is defined and test_pc_port is defined - set_fact: - test_ports: "{{ test_ports | combine( {p:{'test_neighbor_addr':pfc_wd_test_neighbor_addr, 'rx_port':pfc_wd_rx_port, 'rx_neighbor_addr': pfc_wd_rx_neighbor_addr, 'peer_device':neighbors[p]['peerdevice'], 'test_port_id': minigraph_port_indices[p], 'rx_port_id': pfc_wd_rx_port_id.split(' ') | list, 'test_port_type':'portchannel' }}) }}" + test_ports: "{{ test_ports | combine( {p:{'test_neighbor_addr':pfc_wd_test_neighbor_addr, 'rx_port':pfc_wd_rx_port, 'rx_neighbor_addr': pfc_wd_rx_neighbor_addr, 'peer_device':neighbors[p]['peerdevice'], 'test_port_id': minigraph_port_indices[p], 'rx_port_id': pfc_wd_rx_port_id.split(' ') | list, 'test_portchannel_members': pfc_wd_test_port_id.split(' ') | list, 'test_port_type':'portchannel' }}) }}" when: rx_pc_port is defined and test_pc_port is defined - set_fact: @@ -49,7 +49,7 @@ when: rx_pc_port is defined and test_pc_port is defined and testbed_type in ['t0-64', 't0-112', 't1-lag'] - set_fact: - test_ports: "{{ test_ports | combine( {p:{'test_neighbor_addr':pfc_wd_test_neighbor_addr, 'rx_port':pfc_wd_rx_port, 'rx_neighbor_addr': pfc_wd_rx_neighbor_addr, 'peer_device':neighbors[p]['peerdevice'], 'test_port_id': minigraph_port_indices[p], 'rx_port_id': pfc_wd_rx_port_id.split(' ') | list, 'test_port_type':'portchannel' }}) }}" + test_ports: "{{ test_ports | combine( {p:{'test_neighbor_addr':pfc_wd_test_neighbor_addr, 'rx_port':pfc_wd_rx_port, 'rx_neighbor_addr': pfc_wd_rx_neighbor_addr, 'peer_device':neighbors[p]['peerdevice'], 'test_port_id': minigraph_port_indices[p], 'rx_port_id': pfc_wd_rx_port_id.split(' ') | list, 'test_portchannel_members': pfc_wd_test_port_id.split(' ') | list, 'test_port_type':'portchannel' }}) }}" when: rx_pc_port is defined and test_pc_port is defined and testbed_type in ['t0-64', 't0-112', 't1-lag'] - set_fact: @@ -57,7 +57,7 @@ when: first_pair - set_fact: - test_ports: "{{ test_ports | combine( {p:{'test_neighbor_addr':pfc_wd_rx_neighbor_addr, 'rx_port':pfc_wd_test_port, 'rx_neighbor_addr': pfc_wd_test_neighbor_addr, 'peer_device':neighbors[p]['peerdevice'], 'test_port_id': minigraph_port_indices[p], 'rx_port_id': pfc_wd_test_port_id.split(' ') | list , 'test_port_type':'portchannel' }}) }}" + test_ports: "{{ test_ports | combine( {p:{'test_neighbor_addr':pfc_wd_rx_neighbor_addr, 'rx_port':pfc_wd_test_port, 'rx_neighbor_addr': pfc_wd_test_neighbor_addr, 'peer_device':neighbors[p]['peerdevice'], 'test_port_id': minigraph_port_indices[p], 'rx_port_id': pfc_wd_test_port_id.split(' ') | list , 'test_portchannel_members': pfc_wd_rx_port_id.split(' ') | list, 'test_port_type':'portchannel' }}) }}" when: first_pair - set_fact: @@ -65,7 +65,7 @@ when: first_pair and testbed_type in ['t0-64', 't0-112'] - set_fact: - test_ports: "{{ test_ports | combine( {p:{'test_neighbor_addr':pfc_wd_rx_neighbor_addr, 'rx_port':pfc_wd_test_port, 'rx_neighbor_addr': pfc_wd_test_neighbor_addr, 'peer_device':neighbors[p]['peerdevice'], 'test_port_id': minigraph_port_indices[p], 'rx_port_id': pfc_wd_test_port_id.split(' ') | list , 'test_port_type':'portchannel' }}) }}" + test_ports: "{{ test_ports | combine( {p:{'test_neighbor_addr':pfc_wd_rx_neighbor_addr, 'rx_port':pfc_wd_test_port, 'rx_neighbor_addr': pfc_wd_test_neighbor_addr, 'peer_device':neighbors[p]['peerdevice'], 'test_port_id': minigraph_port_indices[p], 'rx_port_id': pfc_wd_test_port_id.split(' ') | list , 'test_portchannel_members': pfc_wd_rx_port_id.split(' ') | list, 'test_port_type':'portchannel' }}) }}" when: first_pair and testbed_type in ['t0-64', 't0-112'] - set_fact: diff --git a/ansible/roles/test/templates/pfc_storm_arista.j2 b/ansible/roles/test/templates/pfc_storm_arista.j2 index fa4b2cb2cf7..d74944a20f3 100644 --- a/ansible/roles/test/templates/pfc_storm_arista.j2 +++ b/ansible/roles/test/templates/pfc_storm_arista.j2 @@ -1,6 +1,5 @@ bash cd /mnt/flash -sudo python {{pfc_gen_file}} -p {{(1).__lshift__(pfc_queue_index)}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("Ethernet", "et")}} -r {{ansible_eth0['ipv4']['address']}} & - +sudo python {{pfc_gen_file}} -p {{(1).__lshift__(pfc_queue_index)}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("Ethernet", "et")}} -r {{ansible_eth0_ipv4_addr}} & exit exit diff --git a/ansible/roles/test/templates/pfc_storm_mlnx.j2 b/ansible/roles/test/templates/pfc_storm_mlnx.j2 index 93c15414099..ef9c6e0e03b 100644 --- a/ansible/roles/test/templates/pfc_storm_mlnx.j2 +++ b/ansible/roles/test/templates/pfc_storm_mlnx.j2 @@ -1,5 +1,5 @@ enable configure terminal -docker exec scapy "./pfcgen.py --p{{pfc_queue_index}} --q{{pfc_queue_index}}=65535 -i {{pfc_frames_number}} -d {{pfc_fanout_interface | replace("ernet 1/", "")}} -r {{ansible_eth0['ipv4']['address']}}" +docker exec scapy "./pfcgen.py --p{{pfc_queue_index}} --q{{pfc_queue_index}}=65535 -i {{pfc_frames_number}} -d {{pfc_fanout_interface | replace("ernet 1/", "")}} -r {{ansible_eth0_ipv4_addr}}" exit exit From 079886167f39831317b3d44b359db253f89f3094 Mon Sep 17 00:00:00 2001 From: Sihui Han Date: Tue, 27 Mar 2018 21:10:25 +0000 Subject: [PATCH 3/3] Get minigraph facts from DUT instead of local --- ansible/roles/test/tasks/interface.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ansible/roles/test/tasks/interface.yml b/ansible/roles/test/tasks/interface.yml index 602f55ed466..bd6f8dc639f 100644 --- a/ansible/roles/test/tasks/interface.yml +++ b/ansible/roles/test/tasks/interface.yml @@ -1,7 +1,5 @@ - name: Gather minigraph facts about the device minigraph_facts: host={{inventory_hostname}} - become: no - connection: local - name: Get interface facts interface_facts: up_ports={{minigraph_ports}}