X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Flibraries%2Fpython%2FQemuManager.py;h=766372ad9c64004d537e013f786a23466e78c977;hp=0ea6164cb7a3d65bb7fcdd2c29d451e9ba4e8e5e;hb=9780b57a9640e9ab40e40ec122ac80e09cd74c79;hpb=0e0cf38fcd1ff704293c9dd2c7a05f8682e11de9 diff --git a/resources/libraries/python/QemuManager.py b/resources/libraries/python/QemuManager.py index 0ea6164cb7..766372ad9c 100644 --- a/resources/libraries/python/QemuManager.py +++ b/resources/libraries/python/QemuManager.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -15,66 +15,19 @@ from collections import OrderedDict -from robot.libraries.BuiltIn import BuiltIn - from resources.libraries.python.Constants import Constants from resources.libraries.python.CpuUtils import CpuUtils from resources.libraries.python.QemuUtils import QemuUtils from resources.libraries.python.topology import NodeType, Topology -__all__ = ["QemuManager"] - - -def get_affinity_vm(nodes, node, nf_chains=1, nf_nodes=1, nf_chain=1, nf_node=1, - cpu_count_int=1, vnf_count_int=1): - """Get affinity of VM. Result will be used to compute the amount of - CPUs and also affinity. - - :param node: SUT nodes. - :param node: DUT node. - :param nf_chains: Number of NF chains. - :param nf_nodes: Number of NF nodes in chain. - :param nf_chain: Chain ID. - :param nf_node: Node ID. - :param cpu_count_int: Amount of Dataplane threads of vswitch. - :param vnf_count_int: Amount of Dataplane threads of vnf. - :type nodes: dict - :type node: dict - :type nf_chains: int - :type nf_nodes: int - :type nf_chain: int - :type nf_node: int - :type cpu_count_int: int - :type vnf_count_int: int - :returns: List of CPUs allocated to VM. - :rtype: list - """ - sut_sc = 1 - dut_mc = 1 - dut_dc = cpu_count_int - skip_cnt = sut_sc + dut_mc + dut_dc - dtc = vnf_count_int - - interface_list = [] - interface_list.append( - BuiltIn().get_variable_value('${{{node}_if1}}'.format(node=node))) - interface_list.append( - BuiltIn().get_variable_value('${{{node}_if2}}'.format(node=node))) - - cpu_node = Topology.get_interfaces_numa_node(nodes[node], *interface_list) - - nf_cpus = CpuUtils.cpu_slice_of_list_for_nf( - node=nodes[node], cpu_node=cpu_node, chains=nf_chains, - nodeness=nf_nodes, chain_id=nf_chain, node_id=nf_node, mtcr=2, dtcr=1, - dtc=dtc, skip_cnt=skip_cnt) - - return nf_cpus - -class QemuManager(object): +__all__ = [u"QemuManager"] + + +class QemuManager: """QEMU lifecycle management class""" # Use one instance of class per tests. - ROBOT_LIBRARY_SCOPE = 'TEST CASE' + ROBOT_LIBRARY_SCOPE = u"TEST CASE" def __init__(self, nodes): """Init QemuManager object.""" @@ -93,48 +46,50 @@ class QemuManager(object): :param kwargs: Named parameters. :type kwargs: dict """ - node = kwargs['node'] - nf_chains = int(kwargs['nf_chains']) - nf_nodes = int(kwargs['nf_nodes']) - queues = kwargs['rxq_count_int'] if kwargs['auto_scale'] else 1 - cpu_count_int = kwargs['cpu_count_int'] - vnf_count_int = kwargs['cpu_count_int'] if kwargs['auto_scale'] else 1 - - img = Constants.QEMU_VM_KERNEL + node = kwargs[u"node"] + nf_chains = int(kwargs[u"nf_chains"]) + nf_nodes = int(kwargs[u"nf_nodes"]) + queues = kwargs[u"rxq_count_int"] if kwargs[u"auto_scale"] else 1 + vs_dtc = kwargs[u"vs_dtc"] + nf_dtc = kwargs[u"nf_dtc"] + if kwargs[u"auto_scale"] and not kwargs[u"fixed_auto_scale"]: + nf_dtc = kwargs[u"vs_dtc"] + nf_dtcr = kwargs[u"nf_dtcr"] \ + if isinstance(kwargs[u"nf_dtcr"], int) else 2 for nf_chain in range(1, nf_chains + 1): for nf_node in range(1, nf_nodes + 1): qemu_id = (nf_chain - 1) * nf_nodes + nf_node - name = '{node}_{qemu_id}'.format(node=node, qemu_id=qemu_id) - sock1 = '/var/run/vpp/sock-{qemu_id}-1'.format(qemu_id=qemu_id) - sock2 = '/var/run/vpp/sock-{qemu_id}-2'.format(qemu_id=qemu_id) - vif1_mac = kwargs['tg_if1_mac'] if nf_node == 1 \ - else '52:54:00:00:{id:02x}:02'.format(id=qemu_id - 1) - vif2_mac = kwargs['tg_if2_mac'] if nf_node == nf_nodes \ - else '52:54:00:00:{id:02x}:01'.format(id=qemu_id + 1) - - self.machines_affinity[name] = get_affinity_vm( + name = f"{node}_{qemu_id}" + idx1 = (nf_chain - 1) * nf_nodes * 2 + nf_node * 2 - 1 + + vif1_mac = Topology.get_interface_mac( + self.nodes[node], f"vhost{idx1}" + ) if kwargs[u"vnf"] == u"testpmd_mac" \ + else kwargs[u"tg_pf1_mac"] if nf_node == 1 \ + else f"52:54:00:00:{(qemu_id - 1):02x}:02" + idx2 = (nf_chain - 1) * nf_nodes * 2 + nf_node * 2 + vif2_mac = Topology.get_interface_mac( + self.nodes[node], f"vhost{idx2}" + ) if kwargs[u"vnf"] == u"testpmd_mac" \ + else kwargs[u"tg_pf2_mac"] if nf_node == nf_nodes \ + else f"52:54:00:00:{(qemu_id + 1):02x}:01" + + self.machines_affinity[name] = CpuUtils.get_affinity_nf( nodes=self.nodes, node=node, nf_chains=nf_chains, nf_nodes=nf_nodes, nf_chain=nf_chain, nf_node=nf_node, - cpu_count_int=cpu_count_int, vnf_count_int=vnf_count_int) - - self.machines[name] = QemuUtils( - node=self.nodes[node], qemu_id=qemu_id, - smp=len(self.machines_affinity[name]), mem=4096, - vnf=kwargs['vnf'], img=img) - self.machines[name].configure_kernelvm_vnf( - mac1='52:54:00:00:{id:02x}:01'.format(id=qemu_id), - mac2='52:54:00:00:{id:02x}:02'.format(id=qemu_id), - vif1_mac=vif1_mac, - vif2_mac=vif2_mac, - queues=queues, - jumbo_frames=kwargs['jumbo']) - self.machines[name].qemu_add_vhost_user_if( - sock1, jumbo_frames=kwargs['jumbo'], queues=queues, - queue_size=1024) - self.machines[name].qemu_add_vhost_user_if( - sock2, jumbo_frames=kwargs['jumbo'], queues=queues, - queue_size=1024) + vs_dtc=vs_dtc, nf_dtc=nf_dtc, nf_dtcr=nf_dtcr + ) + + try: + getattr(self, f'_c_{kwargs["vnf"]}')( + qemu_id=qemu_id, name=name, queues=queues, **kwargs + ) + except AttributeError: + self._c_default( + qemu_id=qemu_id, name=name, queues=queues, + vif1_mac=vif1_mac, vif2_mac=vif2_mac, **kwargs + ) def construct_vms_on_all_nodes(self, **kwargs): """Construct 1..Mx1..N VMs(s) with specified name on all nodes. @@ -144,7 +99,7 @@ class QemuManager(object): """ self.initialize() for node in self.nodes: - if self.nodes[node]['type'] == NodeType.DUT: + if self.nodes[node][u"type"] == NodeType.DUT: self.construct_vms_on_node(node=node, **kwargs) def start_all_vms(self, pinning=False): @@ -153,16 +108,16 @@ class QemuManager(object): :param pinning: If True, then do also QEMU process pinning. :type pinning: bool """ - for machine, machine_affinity in zip(self.machines.values(), - self.machines_affinity.values()): - machine.qemu_start() + cpus = [] + for machine, machine_affinity in \ + zip(self.machines.values(), self.machines_affinity.values()): + index = list(self.machines.values()).index(machine) + name = list(self.machines.keys())[index] + self.nodes[name] = machine.qemu_start() if pinning: machine.qemu_set_affinity(*machine_affinity) - - def set_scheduler_all_vms(self): - """Set CFS scheduler policy on all VMs in manager.""" - for machine in self.machines.values(): - machine.qemu_set_scheduler_policy() + cpus.extend(machine_affinity) + return ",".join(str(cpu) for cpu in cpus) def kill_all_vms(self, force=False): """Kill all added VMs in manager. @@ -170,8 +125,333 @@ class QemuManager(object): :param force: Force kill all Qemu instances by pkill qemu if True. :type force: bool """ + for node in list(self.nodes.values()): + if node["type"] == NodeType.VM: + try: + self.nodes.popitem(node) + except TypeError: + pass for machine in self.machines.values(): if force: machine.qemu_kill_all() else: machine.qemu_kill() + + def _c_default(self, **kwargs): + """Instantiate one VM with default configuration. + + :param kwargs: Named parameters. + :type kwargs: dict + """ + qemu_id = kwargs[u"qemu_id"] + name = kwargs[u"name"] + virtio_feature_mask = kwargs[u"virtio_feature_mask"] \ + if u"virtio_feature_mask" in kwargs else None + + self.machines[name] = QemuUtils( + node=self.nodes[kwargs[u"node"]], + qemu_id=qemu_id, + smp=len(self.machines_affinity[name]), + mem=4096, + vnf=kwargs[u"vnf"], + img=Constants.QEMU_VM_KERNEL + ) + self.machines[name].add_default_params() + self.machines[name].add_kernelvm_params() + self.machines[name].configure_kernelvm_vnf( + mac1=f"52:54:00:00:{qemu_id:02x}:01", + mac2=f"52:54:00:00:{qemu_id:02x}:02", + vif1_mac=kwargs[u"vif1_mac"], + vif2_mac=kwargs[u"vif2_mac"], + queues=kwargs[u"queues"], + jumbo_frames=kwargs[u"jumbo"] + ) + self.machines[name].add_vhost_user_if( + f"/run/vpp/sock-{qemu_id}-1", + jumbo_frames=kwargs[u"jumbo"], + queues=kwargs[u"queues"], + queue_size=kwargs[u"perf_qemu_qsz"], + virtio_feature_mask=virtio_feature_mask + ) + self.machines[name].add_vhost_user_if( + f"/run/vpp/sock-{qemu_id}-2", + jumbo_frames=kwargs[u"jumbo"], + queues=kwargs[u"queues"], + queue_size=kwargs[u"perf_qemu_qsz"], + virtio_feature_mask=virtio_feature_mask + ) + + def _c_vpp_2vfpt_ip4base_plen24(self, **kwargs): + """Instantiate one VM with vpp_2vfpt_ip4base_plen24 configuration. + + :param kwargs: Named parameters. + :type kwargs: dict + """ + qemu_id = kwargs[u"qemu_id"] + name = kwargs[u"name"] + + self.machines[name] = QemuUtils( + node=self.nodes[kwargs[u"node"]], + qemu_id=qemu_id, + smp=len(self.machines_affinity[name]), + mem=4096, + vnf=kwargs[u"vnf"], + img=Constants.QEMU_VM_KERNEL + ) + self.machines[name].add_default_params() + self.machines[name].add_kernelvm_params() + if u"DUT1" in name: + self.machines[name].configure_kernelvm_vnf( + ip1=u"2.2.2.1/30", + ip2=u"1.1.1.2/30", + route1=u"20.0.0.0/24", + routeif1=u"avf-0/0/6/0", + nexthop1=u"2.2.2.2", + route2=u"10.0.0.0/24", + routeif2=u"avf-0/0/7/0", + nexthop2=u"1.1.1.1", + arpmac1=u"3c:fd:fe:d1:5c:d8", + arpip1=u"1.1.1.1", + arpif1=u"avf-0/0/7/0", + queues=kwargs[u"queues"], + jumbo_frames=kwargs[u"jumbo"] + ) + else: + self.machines[name].configure_kernelvm_vnf( + ip1=u"3.3.3.2/30", + ip2=u"2.2.2.2/30", + route1=u"10.0.0.0/24", + routeif1=u"avf-0/0/7/0", + nexthop1=u"2.2.2.1", + route2=u"20.0.0.0/24", + routeif2=u"avf-0/0/6/0", + nexthop2=u"3.3.3.1", + arpmac1=u"3c:fd:fe:d1:5c:d9", + arpip1=u"3.3.3.1", + arpif1=u"avf-0/0/6/0", + queues=kwargs[u"queues"], + jumbo_frames=kwargs[u"jumbo"] + ) + self.machines[name].add_vfio_pci_if( + pci=Topology.get_interface_pci_addr( + self.nodes[kwargs[u"node"]], kwargs[u"if2"]) + ) + self.machines[name].add_vfio_pci_if( + pci=Topology.get_interface_pci_addr( + self.nodes[kwargs[u"node"]], kwargs[u"if1"]) + ) + + def _c_vpp_2vfpt_ip4scale2k_plen30(self, **kwargs): + """Instantiate one VM with vpp_2vfpt_ip4scale2k_plen30 configuration. + + :param kwargs: Named parameters. + :type kwargs: dict + """ + qemu_id = kwargs[u"qemu_id"] + name = kwargs[u"name"] + + self.machines[name] = QemuUtils( + node=self.nodes[kwargs[u"node"]], + qemu_id=qemu_id, + smp=len(self.machines_affinity[name]), + mem=4096, + vnf=kwargs[u"vnf"], + img=Constants.QEMU_VM_KERNEL + ) + self.machines[name].add_default_params() + self.machines[name].add_kernelvm_params() + if u"DUT1" in name: + self.machines[name].configure_kernelvm_vnf( + ip1=u"2.2.2.1/30", + ip2=u"1.1.1.2/30", + route1=u"20.0.0.0/30", + routeif1=u"avf-0/0/6/0", + nexthop1=u"2.2.2.2", + route2=u"10.0.0.0/30", + routeif2=u"avf-0/0/7/0", + nexthop2=u"1.1.1.1", + arpmac1=u"3c:fd:fe:d1:5c:d8", + arpip1=u"1.1.1.1", + arpif1=u"avf-0/0/7/0", + queues=kwargs[u"queues"], + jumbo_frames=kwargs[u"jumbo"] + ) + else: + self.machines[name].configure_kernelvm_vnf( + ip1=u"3.3.3.2/30", + ip2=u"2.2.2.2/30", + route1=u"10.0.0.0/30", + routeif1=u"avf-0/0/7/0", + nexthop1=u"2.2.2.1", + route2=u"20.0.0.0/30", + routeif2=u"avf-0/0/6/0", + nexthop2=u"3.3.3.1", + arpmac1=u"3c:fd:fe:d1:5c:d9", + arpip1=u"3.3.3.1", + arpif1=u"avf-0/0/6/0", + queues=kwargs[u"queues"], + jumbo_frames=kwargs[u"jumbo"] + ) + self.machines[name].add_vfio_pci_if( + pci=Topology.get_interface_pci_addr( + self.nodes[kwargs[u"node"]], kwargs[u"if2"]) + ) + self.machines[name].add_vfio_pci_if( + pci=Topology.get_interface_pci_addr( + self.nodes[kwargs[u"node"]], kwargs[u"if1"]) + ) + + def _c_vpp_2vfpt_ip4scale20k_plen30(self, **kwargs): + """Instantiate one VM with vpp_2vfpt_ip4scale20k_plen30 configuration. + + :param kwargs: Named parameters. + :type kwargs: dict + """ + qemu_id = kwargs[u"qemu_id"] + name = kwargs[u"name"] + + self.machines[name] = QemuUtils( + node=self.nodes[kwargs[u"node"]], + qemu_id=qemu_id, + smp=len(self.machines_affinity[name]), + mem=4096, + vnf=kwargs[u"vnf"], + img=Constants.QEMU_VM_KERNEL + ) + self.machines[name].add_default_params() + self.machines[name].add_kernelvm_params() + if u"DUT1" in name: + self.machines[name].configure_kernelvm_vnf( + ip1=u"2.2.2.1/30", + ip2=u"1.1.1.2/30", + route1=u"20.0.0.0/30", + routeif1=u"avf-0/0/6/0", + nexthop1=u"2.2.2.2", + route2=u"10.0.0.0/30", + routeif2=u"avf-0/0/7/0", + nexthop2=u"1.1.1.1", + arpmac1=u"3c:fd:fe:d1:5c:d8", + arpip1=u"1.1.1.1", + arpif1=u"avf-0/0/7/0", + queues=kwargs[u"queues"], + jumbo_frames=kwargs[u"jumbo"] + ) + else: + self.machines[name].configure_kernelvm_vnf( + ip1=u"3.3.3.2/30", + ip2=u"2.2.2.2/30", + route1=u"10.0.0.0/30", + routeif1=u"avf-0/0/7/0", + nexthop1=u"2.2.2.1", + route2=u"20.0.0.0/30", + routeif2=u"avf-0/0/6/0", + nexthop2=u"3.3.3.1", + arpmac1=u"3c:fd:fe:d1:5c:d9", + arpip1=u"3.3.3.1", + arpif1=u"avf-0/0/6/0", + queues=kwargs[u"queues"], + jumbo_frames=kwargs[u"jumbo"] + ) + self.machines[name].add_vfio_pci_if( + pci=Topology.get_interface_pci_addr( + self.nodes[kwargs[u"node"]], kwargs[u"if2"]) + ) + self.machines[name].add_vfio_pci_if( + pci=Topology.get_interface_pci_addr( + self.nodes[kwargs[u"node"]], kwargs[u"if1"]) + ) + + def _c_vpp_2vfpt_ip4scale200k_plen30(self, **kwargs): + """Instantiate one VM with vpp_2vfpt_ip4scale200k_plen30 configuration. + + :param kwargs: Named parameters. + :type kwargs: dict + """ + qemu_id = kwargs[u"qemu_id"] + name = kwargs[u"name"] + + self.machines[name] = QemuUtils( + node=self.nodes[kwargs[u"node"]], + qemu_id=qemu_id, + smp=len(self.machines_affinity[name]), + mem=4096, + vnf=kwargs[u"vnf"], + img=Constants.QEMU_VM_KERNEL + ) + self.machines[name].add_default_params() + self.machines[name].add_kernelvm_params() + if u"DUT1" in name: + self.machines[name].configure_kernelvm_vnf( + ip1=u"2.2.2.1/30", + ip2=u"1.1.1.2/30", + route1=u"20.0.0.0/30", + routeif1=u"avf-0/0/6/0", + nexthop1=u"2.2.2.2", + route2=u"10.0.0.0/30", + routeif2=u"avf-0/0/7/0", + nexthop2=u"1.1.1.1", + arpmac1=u"3c:fd:fe:d1:5c:d8", + arpip1=u"1.1.1.1", + arpif1=u"avf-0/0/7/0", + queues=kwargs[u"queues"], + jumbo_frames=kwargs[u"jumbo"] + ) + else: + self.machines[name].configure_kernelvm_vnf( + ip1=u"3.3.3.2/30", + ip2=u"2.2.2.2/30", + route1=u"10.0.0.0/30", + routeif1=u"avf-0/0/7/0", + nexthop1=u"2.2.2.1", + route2=u"20.0.0.0/30", + routeif2=u"avf-0/0/6/0", + nexthop2=u"3.3.3.1", + arpmac1=u"3c:fd:fe:d1:5c:d9", + arpip1=u"3.3.3.1", + arpif1=u"avf-0/0/6/0", + queues=kwargs[u"queues"], + jumbo_frames=kwargs[u"jumbo"] + ) + self.machines[name].add_vfio_pci_if( + pci=Topology.get_interface_pci_addr( + self.nodes[kwargs[u"node"]], kwargs[u"if2"]) + ) + self.machines[name].add_vfio_pci_if( + pci=Topology.get_interface_pci_addr( + self.nodes[kwargs[u"node"]], kwargs[u"if1"]) + ) + + def _c_iperf3(self, **kwargs): + """Instantiate one VM with iperf3 configuration. + + :param kwargs: Named parameters. + :type kwargs: dict + """ + qemu_id = kwargs[u"qemu_id"] + name = kwargs[u"name"] + virtio_feature_mask = kwargs[u"virtio_feature_mask"] \ + if u"virtio_feature_mask" in kwargs else None + + self.machines[name] = QemuUtils( + node=self.nodes[kwargs[u"node"]], + qemu_id=qemu_id, + smp=len(self.machines_affinity[name]), + mem=4096, + vnf=kwargs[u"vnf"], + img=Constants.QEMU_VM_KERNEL + ) + self.machines[name].add_default_params() + self.machines[name].add_kernelvm_params() + self.machines[name].configure_kernelvm_vnf( + queues=kwargs[u"queues"], + jumbo_frames=kwargs[u"jumbo"] + ) + self.machines[name].add_net_user() + self.machines[name].add_vhost_user_if( + f"/run/vpp/sock-{qemu_id}-1", + server=False, + jumbo_frames=kwargs[u"jumbo"], + queues=kwargs[u"queues"], + queue_size=kwargs[u"perf_qemu_qsz"], + virtio_feature_mask=virtio_feature_mask + )