[testbed-cli] Code change on add-topo and deploy-minigraph for deploying testbed with peers on multiple servers (sonic-net#15643)

In PR sonic-net#15547, we define server index in topology file which make topology file and testbed yaml file is coupled.
To decouple them, in new design sonic-net#15395, we remove server index in topology file and add dut_interfaces in testbed yaml, so there is no more change on topology file schema.

What is the motivation for this PR?
Decouple topology file and testbed file in design.

How did you do it?
Record all information in testbed schema

How did you verify/test it?
Deploy testbed

Author: w1nda

ansible/config_sonic_basedon_testbed.yml

@@ -71,7 +71,7 @@
 
     - name: set vm
       set_fact:
-        vm_base: "{% if testbed_facts['vm_base'] != '' %}{{ testbed_facts['vm_base'] }}{% else %}''{% endif %}"
+        vm_base: "{% if 'vm_base' in testbed_facts and testbed_facts['vm_base'] != '' %}{{ testbed_facts['vm_base'] }}{% else %}''{% endif %}"
     when: testbed_name is defined
 
   - name: find supervisor dut of testbed
@@ -222,7 +222,7 @@
       remote_dut: "{{ ansible_ssh_host }}"
 
   - name: gather testbed VM information
-    testbed_vm_info: base_vm={{ testbed_facts['vm_base'] }} topo={{ testbed_facts['topo'] }} vm_file={{ vm_file }}
+    testbed_vm_info: base_vm={{ testbed_facts['vm_base'] if 'vm_base' in testbed_facts else "" }} topo={{ testbed_facts['topo'] }} vm_file={{ vm_file }} servers_info={{ testbed_facts['servers'] | default({}) }}
     delegate_to: localhost
     when: "(VM_topo | bool) and ('cable' not in topo)"
 
@@ -433,7 +433,7 @@
 
       - name: Update TACACS server address to PTF IP
         set_fact:
-            tacacs_servers: ["{{ testbed_facts['ptf_ip'] }}"]
+            tacacs_servers: ["{{ testbed_facts['multi_servers_tacacs_ip'] if 'multi_servers_tacacs_ip' in testbed_facts else testbed_facts['ptf_ip'] }}"]
         when: use_ptf_tacacs_server is defined and use_ptf_tacacs_server|bool == true
 
       - debug: msg="tacacs_servers {{ tacacs_servers }}"

ansible/fanout_connect.yml

@@ -11,6 +11,7 @@
         - set_fact:
             server: "{{ inventory_hostname|lower }}"
             server_port: "{{ external_port }}"
+            clean_before_add: "{{ clean_before_add | default('y') }}"
 
         - set_fact: root_fanout_connect=true
           when: root_fanout_connect is not defined

ansible/library/announce_routes.py

@@ -15,6 +15,7 @@
 from multiprocessing.pool import ThreadPool
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.debug_utils import config_module_logging
+from ansible.module_utils.multi_servers_utils import MultiServersUtils
 
 if sys.version_info.major == 3:
     UNICODE_TYPE = str
@@ -1150,6 +1151,7 @@ def main():
             action=dict(required=False, type='str',
                         default='announce', choices=["announce", "withdraw"]),
             path=dict(required=False, type='str', default=''),
+            dut_interfaces=dict(required=False, type='str', default=''),
             log_path=dict(required=False, type='str', default='')
         ),
         supports_check_mode=False)
@@ -1160,11 +1162,17 @@ def main():
     topo_name = module.params['topo_name']
     ptf_ip = module.params['ptf_ip']
     action = module.params['action']
+    dut_interfaces = module.params['dut_interfaces']
     path = module.params['path']
 
     topo = read_topo(topo_name, path)
     if not topo:
         module.fail_json(msg='Unable to load topology "{}"'.format(topo_name))
+    if dut_interfaces:
+        topo['topology']['VMs'] = MultiServersUtils.get_vms_by_dut_interfaces(topo['topology']['VMs'], dut_interfaces)
+        for vm_name in topo['configuration'].keys():
+            if vm_name not in topo['topology']['VMs']:
+                topo['configuration'].pop(vm_name)
 
     is_storage_backend = "backend" in topo_name
 

ansible/library/test_facts.py

@@ -169,15 +169,18 @@ def _read_testbed_topo_from_yaml():
             with open(self.testbed_filename) as f:
                 tb_info = yaml.safe_load(f)
                 for tb in tb_info:
-                    if tb["ptf_ip"]:
+                    if "ptf_ip" in tb and tb["ptf_ip"]:
                         tb["ptf_ip"], tb["ptf_netmask"] = \
                             _cidr_to_ip_mask(tb["ptf_ip"])
-                    if tb["ptf_ipv6"]:
+                    if "ptf_ipv6" in tb and tb["ptf_ipv6"]:
                         tb["ptf_ipv6"], tb["ptf_netmask_v6"] = \
                             _cidr_to_ip_mask(tb["ptf_ipv6"])
                     tb["duts"] = tb.pop("dut")
                     tb["duts_map"] =  \
                         {dut: i for i, dut in enumerate(tb["duts"])}
+                    if 'servers' in tb:
+                        tb['multi_servers_tacacs_ip'], _ = \
+                            _cidr_to_ip_mask(tb['servers'].values()[0]["ptf_ip"])
                     self.testbed_topo[tb["conf-name"]] = tb
 
         if self.testbed_filename.endswith(".csv"):

ansible/library/testbed_vm_info.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.multi_servers_utils import MultiServersUtils
 import re
 import yaml
 import traceback
@@ -49,12 +50,13 @@ class TestbedVMFacts():
 
     """
 
-    def __init__(self, toponame, base_vm, vm_file):
+    def __init__(self, toponame, base_vm, vm_file, servers_info):
         CLET_SUFFIX = "-clet"
         self.toponame = re.sub(CLET_SUFFIX + "$", "", toponame)
         self.topofile = TOPO_PATH + 'topo_' + self.toponame + '.yml'
         self.base_vm = base_vm
         self.vm_file = vm_file
+        self.servers_info = servers_info
         if has_dataloader:
             self.inv_mgr = InventoryManager(
                 loader=DataLoader(), sources=self.vm_file)
@@ -65,6 +67,12 @@ def get_neighbor_eos(self):
             vm_topology = yaml.safe_load(f)
         self.topoall = vm_topology
 
+        if self.servers_info:
+            return MultiServersUtils.generate_vm_name_mapping(
+                self.servers_info,
+                vm_topology['topology']['VMs']
+            )
+
         if len(self.base_vm) > 2:
             vm_start_index = int(self.base_vm[2:])
             vm_name_fmt = 'VM%0{}d'.format(len(self.base_vm) - 2)
@@ -116,6 +124,7 @@ def main():
         argument_spec=dict(
             base_vm=dict(required=True, type='str'),
             topo=dict(required=True, type='str'),
+            servers_info=dict(required=False, type='dict', default={}),
             vm_file=dict(default=VM_INV_FILE, type='str')
         ),
         supports_check_mode=True
@@ -128,7 +137,7 @@ def main():
     vm_mgmt_ip = {}
     try:
         vm_facts = TestbedVMFacts(
-            m_args['topo'], m_args['base_vm'], m_args['vm_file'])
+            m_args['topo'], m_args['base_vm'], m_args['vm_file'], m_args['servers_info'])
         neighbor_eos = vm_facts.get_neighbor_eos()
         neighbor_eos.update(vm_facts.get_neighbor_dpu())
         if has_dataloader:

ansible/module_utils/multi_servers_utils.py

@@ -0,0 +1,66 @@
+class MultiServersUtils:
+    @staticmethod
+    def filter_by_dut_interfaces(values, dut_interfaces):
+        if not dut_interfaces:
+            return values
+
+        if isinstance(dut_interfaces, str) or isinstance(dut_interfaces, unicode):  # noqa F821
+            dut_interfaces = MultiServersUtils.parse_multi_servers_interface(dut_interfaces)
+
+        if isinstance(values, dict):
+            return {k: v for k, v in values.items() if int(k) in dut_interfaces}
+        elif isinstance(values, list):
+            return [v for v in values if int(v) in dut_interfaces]
+        else:
+            raise ValueError('Unsupported type "{}"'.format(type(values)))
+
+    @staticmethod
+    def parse_multi_servers_interface(intf_pattern):
+        intf_pattern = str(intf_pattern)
+        intfs = []
+        for intf in iter(map(str.strip, intf_pattern.split(','))):
+            if intf.isdigit():
+                intfs.append(int(intf))
+            elif '-' in intf:
+                intf_range = list(map(int, map(str.strip, intf.split('-'))))
+                assert len(intf_range) == 2, 'Invalid interface range "{}"'.format(intf)
+                intfs.extend(list(range(intf_range[0], intf_range[1]+1)))
+            else:
+                raise ValueError('Unsupported format "{}"'.format(intf_pattern))
+        if len(intfs) != len(set(intfs)):
+            raise ValueError('There are interface duplication/overlap in "{}"'.format(intf_pattern))
+        return intfs
+
+    @staticmethod
+    def get_vms_by_dut_interfaces(VMs, dut_interfaces):
+        if not dut_interfaces:
+            return VMs
+
+        if isinstance(dut_interfaces, str) or isinstance(dut_interfaces, unicode):  # noqa F821
+            dut_interfaces = MultiServersUtils.parse_multi_servers_interface(dut_interfaces)
+
+        result = {}
+        offset = 0
+        for hostname, attr in VMs.items():
+            if dut_interfaces and attr['vlans'][0] not in dut_interfaces:
+                continue
+            result[hostname] = attr
+            result[hostname]['vm_offset'] = offset
+            offset += 1
+        return result
+
+    @staticmethod
+    def generate_vm_name_mapping(servers_info, topo_vms):
+        _m = {}
+
+        for server_attr in servers_info.values():
+            if 'dut_interfaces' in server_attr:
+                filtered_vms = MultiServersUtils.get_vms_by_dut_interfaces(topo_vms, server_attr['dut_interfaces'])
+                vm_base = server_attr['vm_base']
+                vm_start_index = int(vm_base[2:])
+                vm_name_fmt = 'VM%0{}d'.format(len(vm_base) - 2)
+
+                for hostname, host_attr in filtered_vms.items():
+                    vm_name = vm_name_fmt % (vm_start_index + host_attr['vm_offset'])
+                    _m[hostname] = vm_name
+        return _m

ansible/plugins/filter/filters.py

@@ -8,6 +8,8 @@
 class FilterModule(object):
     def filters(self):
         return {
+            'filter_by_dut_interfaces': MultiServersUtils.filter_by_dut_interfaces,
+            'get_vms_by_dut_interfaces': MultiServersUtils.get_vms_by_dut_interfaces,
             'extract_by_prefix': extract_by_prefix,
             'filter_by_prefix': filter_by_prefix,
             'filter_vm_targets': filter_vm_targets,
@@ -88,7 +90,7 @@ def first_n_elements(values, num):
     return values[0:int(num)]
 
 
-def filter_vm_targets(values, topology, vm_base):
+def filter_vm_targets(values, topology, vm_base, dut_interfaces=None):
     """
     This function takes a list of host VMs as parameter 'values' and then extract a list of host VMs
     which starts with 'vm_base' and contains all VMs which mentioned in 'vm_offset' keys inside of 'topology' structure
@@ -114,17 +116,18 @@ def filter_vm_targets(values, topology, vm_base):
     if vm_base not in values:
         raise errors.AnsibleFilterError('Current vm_base: %s is not found in vm_list' % vm_base)
 
+    vms = MultiServersUtils.get_vms_by_dut_interfaces(topology, dut_interfaces) if dut_interfaces else topology
     result = []
     base = values.index(vm_base)
-    for hostname, attr in topology.items():
+    for hostname, attr in vms.items():
         if base + attr['vm_offset'] >= len(values):
             continue
         result.append(values[base + attr['vm_offset']])
 
     return result
 
 
-def extract_hostname(values, topology, vm_base, inventory_hostname):
+def extract_hostname(values, topology, vm_base, inventory_hostname, dut_interfaces=None):
     """
     This function takes a list of host VMs as parameter 'values' and then return 'inventory_hostname'
     corresponding EOS hostname based on 'topology' structure, 'vm_base' parameters
@@ -156,8 +159,9 @@ def extract_hostname(values, topology, vm_base, inventory_hostname):
     if vm_base not in values:
         raise errors.AnsibleFilterError('Current vm_base: %s is not found in vm_list' % vm_base)
 
+    vms = MultiServersUtils.get_vms_by_dut_interfaces(topology, dut_interfaces) if dut_interfaces else topology
     base = values.index(vm_base)
-    for hostname, attr in topology.items():
+    for hostname, attr in vms.items():
         if base + attr['vm_offset'] >= len(values):
             continue
         if inventory_hostname == values[base + attr['vm_offset']]:
@@ -209,3 +213,55 @@ def first_ip_of_subnet(value):
 def path_join(paths):
     """Join path strings."""
     return os.path.join(*paths)
+
+
+class MultiServersUtils:
+    @staticmethod
+    def filter_by_dut_interfaces(values, dut_interfaces):
+        if not dut_interfaces:
+            return values
+
+        if isinstance(dut_interfaces, str) or isinstance(dut_interfaces, unicode):  # noqa F821
+            dut_interfaces = MultiServersUtils.parse_multi_servers_interface(dut_interfaces)
+
+        if isinstance(values, dict):
+            return {k: v for k, v in values.items() if int(k) in dut_interfaces}
+        elif isinstance(values, list):
+            return [v for v in values if int(v) in dut_interfaces]
+        else:
+            raise ValueError('Unsupported type "{}"'.format(type(values)))
+
+    @staticmethod
+    def parse_multi_servers_interface(intf_pattern):
+        intf_pattern = str(intf_pattern)
+        intfs = []
+        for intf in iter(map(str.strip, intf_pattern.split(','))):
+            if intf.isdigit():
+                intfs.append(int(intf))
+            elif '-' in intf:
+                intf_range = list(map(int, map(str.strip, intf.split('-'))))
+                assert len(intf_range) == 2, 'Invalid interface range "{}"'.format(intf)
+                intfs.extend(list(range(intf_range[0], intf_range[1]+1)))
+            else:
+                raise ValueError('Unsupported format "{}"'.format(intf_pattern))
+        if len(intfs) != len(set(intfs)):
+            raise ValueError('There are interface duplication/overlap in "{}"'.format(intf_pattern))
+        return intfs
+
+    @staticmethod
+    def get_vms_by_dut_interfaces(VMs, dut_interfaces):
+        if not dut_interfaces:
+            return VMs
+
+        if isinstance(dut_interfaces, str) or isinstance(dut_interfaces, unicode):  # noqa F821
+            dut_interfaces = MultiServersUtils.parse_multi_servers_interface(dut_interfaces)
+
+        result = {}
+        offset = 0
+        for hostname, attr in VMs.items():
+            if dut_interfaces and attr['vlans'][0] not in dut_interfaces:
+                continue
+            result[hostname] = attr
+            result[hostname]['vm_offset'] = offset
+            offset += 1
+        return result

ansible/roles/eos/tasks/main.yml

@@ -17,7 +17,7 @@
   set_fact: VM_host={{ groups[current_server] | difference(VM_list) }}
 
 - name: Generate hostname for target VM
-  set_fact: hostname={{ VM_list | extract_hostname(topology['VMs'], VM_base, inventory_hostname) }}
+  set_fact: hostname={{ VM_list | extract_hostname(topology['VMs'], VM_base, inventory_hostname, dut_interfaces | default("")) }}
   when: topology['VMs'] is defined
 
 - fail:

ansible/roles/fanout/templates/arista_7260_connect.j2

@@ -10,8 +10,10 @@ vlan {{ dev_vlans | list | join(',') }}
 {% set peer_speed = root_conn[intf]['speed'] %}
 {% if peer_dev in lab_devices and 'Fanout' not in lab_devices[peer_dev]['Type'] and not deploy_leaf %}
 interface {{ intf }}
+{% if clean_before_add == 'y' %}
   switchport
   switchport trunk allowed vlan remove {{ dev_vlans | list | join(',') }}
+{% endif %}
 {% if peer_dev == server and peer_port == server_port %}
   switchport mode trunk
   switchport trunk allowed vlan add {{ dev_vlans | list | join(',') }}
@@ -20,8 +22,10 @@ interface {{ intf }}
 {% endif %}
 {% if peer_dev in lab_devices and 'Fanout' in lab_devices[peer_dev]['Type']  and deploy_leaf %}
 interface {{ intf }}
+{% if clean_before_add == 'y' %}
   switchport
   switchport trunk allowed vlan remove {{ dev_vlans | list | join(',') }}
+{% endif %}
 {% if peer_dev == leaf_name %}
   description {{ peer_dev }}-{{ peer_port }}
   speed forced {{ peer_speed }}full