resources/libraries/python/QemuUtils.py

   1 # Copyright (c) 2021 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """QEMU utilities library."""
  15
  16 import json
  17
  18 from re import match
  19 from string import Template
  20 from time import sleep
  21
  22 from robot.api import logger
  23
  24 from resources.libraries.python.Constants import Constants
  25 from resources.libraries.python.DpdkUtil import DpdkUtil
  26 from resources.libraries.python.DUTSetup import DUTSetup
  27 from resources.libraries.python.OptionString import OptionString
  28 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
  29 from resources.libraries.python.topology import NodeType, Topology
  30 from resources.libraries.python.VhostUser import VirtioFeaturesFlags
  31 from resources.libraries.python.VhostUser import VirtioFeatureMask
  32 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
  33
  34 __all__ = [u"QemuUtils"]
  35
  36
  37 class QemuUtils:
  38     """QEMU utilities."""
  39
  40     # Use one instance of class per tests.
  41     ROBOT_LIBRARY_SCOPE = u"TEST CASE"
  42
  43     def __init__(
  44             self, node, qemu_id=1, smp=1, mem=512, vnf=None,
  45             img=Constants.QEMU_VM_IMAGE, page_size=u""):
  46         """Initialize QemuUtil class.
  47
  48         :param node: Node to run QEMU on.
  49         :param qemu_id: QEMU identifier.
  50         :param smp: Number of virtual SMP units (cores).
  51         :param mem: Amount of memory.
  52         :param vnf: Network function workload.
  53         :param img: QEMU disk image or kernel image path.
  54         :param page_size: Hugepage Size.
  55         :type node: dict
  56         :type qemu_id: int
  57         :type smp: int
  58         :type mem: int
  59         :type vnf: str
  60         :type img: str
  61         :type page_size: str
  62         """
  63         self._nic_id = 0
  64         self._node = node
  65         self._arch = Topology.get_node_arch(self._node)
  66         self._opt = dict()
  67
  68         # Architecture specific options
  69         if self._arch == u"aarch64":
  70             self._opt[u"machine_args"] = \
  71                 u"virt,accel=kvm,usb=off,mem-merge=off,gic-version=3"
  72             self._opt[u"console"] = u"ttyAMA0"
  73         else:
  74             self._opt[u"machine_args"] = u"pc,accel=kvm,usb=off,mem-merge=off"
  75             self._opt[u"console"] = u"ttyS0"
  76         self._testpmd_path = f"{Constants.QEMU_VM_DPDK}/build/app"
  77         self._vm_info = {
  78             u"host": node[u"host"],
  79             u"type": NodeType.VM,
  80             u"port": 10021 + qemu_id,
  81             u"serial": 4555 + qemu_id,
  82             u"username": 'testuser',
  83             u"password": 'Csit1234',
  84             u"interfaces": {},
  85         }
  86         if node[u"port"] != 22:
  87             self._vm_info[u"host_port"] = node[u"port"]
  88             self._vm_info[u"host_username"] = node[u"username"]
  89             self._vm_info[u"host_password"] = node[u"password"]
  90         # Input Options.
  91         self._opt[u"qemu_id"] = qemu_id
  92         self._opt[u"mem"] = int(mem)
  93         self._opt[u"smp"] = int(smp)
  94         self._opt[u"img"] = img
  95         self._opt[u"vnf"] = vnf
  96         self._opt[u"page_size"] = page_size
  97
  98         # Temporary files.
  99         self._temp = dict()
 100         self._temp[u"log"] = f"/tmp/serial_{qemu_id}.log"
 101         self._temp[u"pidfile"] = f"/run/qemu_{qemu_id}.pid"
 102         if img == Constants.QEMU_VM_IMAGE:
 103             self._temp[u"qmp"] = f"/run/qmp_{qemu_id}.sock"
 104             self._temp[u"qga"] = f"/run/qga_{qemu_id}.sock"
 105         elif img == Constants.QEMU_VM_KERNEL:
 106             self._opt[u"img"], _ = exec_cmd_no_error(
 107                 node, f"ls -1 {Constants.QEMU_VM_KERNEL}* | tail -1",
 108                 message=u"Qemu Kernel VM image not found!"
 109             )
 110             self._temp[u"ini"] = f"/etc/vm_init_{qemu_id}.conf"
 111             self._opt[u"initrd"], _ = exec_cmd_no_error(
 112                 node, f"ls -1 {Constants.QEMU_VM_KERNEL_INITRD}* | tail -1",
 113                 message=u"Qemu Kernel initrd image not found!"
 114             )
 115         else:
 116             raise RuntimeError(f"QEMU: Unknown VM image option: {img}")
 117         # Computed parameters for QEMU command line.
 118         self._params = OptionString(prefix=u"-")
 119
 120     def add_default_params(self):
 121         """Set default QEMU command line parameters."""
 122         mem_path = f"/dev/hugepages1G" \
 123             if self._opt[u"page_size"] == u"1G" else u"/dev/hugepages"
 124
 125         self._params.add(u"daemonize")
 126         self._params.add(u"nodefaults")
 127         self._params.add_with_value(
 128             u"name", f"vnf{self._opt.get(u'qemu_id')},debug-threads=on"
 129         )
 130         self._params.add(u"no-user-config")
 131         self._params.add(u"nographic")
 132         self._params.add(u"enable-kvm")
 133         self._params.add_with_value(u"pidfile", self._temp.get(u"pidfile"))
 134         self._params.add_with_value(u"cpu", u"host")
 135
 136         self._params.add_with_value(u"machine", self._opt.get(u"machine_args"))
 137         self._params.add_with_value(
 138             u"smp", f"{self._opt.get(u'smp')},sockets=1,"
 139             f"cores={self._opt.get(u'smp')},threads=1"
 140         )
 141         self._params.add_with_value(
 142             u"object", f"memory-backend-file,id=mem,"
 143             f"size={self._opt.get(u'mem')}M,"
 144             f"mem-path={mem_path},share=on"
 145         )
 146         self._params.add_with_value(u"m", f"{self._opt.get(u'mem')}M")
 147         self._params.add_with_value(u"numa", u"node,memdev=mem")
 148
 149     def add_net_user(self, net="10.0.2.0/24"):
 150         """Set managment port forwarding."""
 151         self._params.add_with_value(
 152             u"netdev", f"user,id=mgmt,net={net},"
 153             f"hostfwd=tcp::{self._vm_info[u'port']}-:22"
 154         )
 155         self._params.add_with_value(
 156             u"device", f"virtio-net,netdev=mgmt"
 157         )
 158
 159     def add_qmp_qga(self):
 160         """Set QMP, QGA management."""
 161         self._params.add_with_value(
 162             u"chardev", f"socket,path={self._temp.get(u'qga')},"
 163             f"server,nowait,id=qga0"
 164         )
 165         self._params.add_with_value(
 166             u"device", u"isa-serial,chardev=qga0"
 167         )
 168         self._params.add_with_value(
 169             u"qmp", f"unix:{self._temp.get(u'qmp')},server,nowait"
 170         )
 171
 172     def add_serial(self):
 173         """Set serial to file redirect."""
 174         self._params.add_with_value(
 175             u"chardev", f"socket,host=127.0.0.1,"
 176             f"port={self._vm_info[u'serial']},id=gnc0,server,nowait"
 177         )
 178         self._params.add_with_value(
 179             u"device", u"isa-serial,chardev=gnc0"
 180         )
 181         self._params.add_with_value(
 182             u"serial", f"file:{self._temp.get(u'log')}"
 183         )
 184
 185     def add_drive_cdrom(self, drive_file, index=None):
 186         """Set CD-ROM drive.
 187
 188         :param drive_file: Path to drive image.
 189         :param index: Drive index.
 190         :type drive_file: str
 191         :type index: int
 192         """
 193         index = f"index={index}," if index else u""
 194         self._params.add_with_value(
 195             u"drive", f"file={drive_file},{index}media=cdrom"
 196         )
 197
 198     def add_drive(self, drive_file, drive_format):
 199         """Set drive with custom format.
 200
 201         :param drive_file: Path to drive image.
 202         :param drive_format: Drive image format.
 203         :type drive_file: str
 204         :type drive_format: str
 205         """
 206         self._params.add_with_value(
 207             u"drive", f"file={drive_file},format={drive_format},"
 208             u"cache=none,if=virtio,file.locking=off"
 209         )
 210
 211     def add_kernelvm_params(self):
 212         """Set KernelVM QEMU parameters."""
 213         hugepages = 3 if self._opt[u"page_size"] == u"1G" else 512
 214
 215         self._params.add_with_value(
 216             u"serial", f"file:{self._temp.get(u'log')}"
 217         )
 218         self._params.add_with_value(
 219             u"fsdev", u"local,id=root9p,path=/,security_model=none"
 220         )
 221         self._params.add_with_value(
 222             u"device", u"virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot"
 223         )
 224         self._params.add_with_value(
 225             u"kernel", f"{self._opt.get(u'img')}"
 226         )
 227         self._params.add_with_value(
 228             u"initrd", f"{self._opt.get(u'initrd')}"
 229         )
 230         self._params.add_with_value(
 231             u"append", f"'ro rootfstype=9p rootflags=trans=virtio "
 232             f"root=virtioroot console={self._opt.get(u'console')} "
 233             f"tsc=reliable hugepages={hugepages} "
 234             f"hugepagesz={self._opt.get(u'page_size')} "
 235             f"init={self._temp.get(u'ini')} fastboot'"
 236         )
 237
 238     def add_vhost_user_if(
 239             self, socket, server=True, jumbo_frames=False, queue_size=None,
 240             queues=1, virtio_feature_mask=None):
 241         """Add Vhost-user interface.
 242
 243         :param socket: Path of the unix socket.
 244         :param server: If True the socket shall be a listening socket.
 245         :param jumbo_frames: Set True if jumbo frames are used in the test.
 246         :param queue_size: Vring queue size.
 247         :param queues: Number of queues.
 248         :param virtio_feature_mask: Mask of virtio features to be enabled.
 249         :type socket: str
 250         :type server: bool
 251         :type jumbo_frames: bool
 252         :type queue_size: int
 253         :type queues: int
 254         :type virtio_feature_mask: int
 255         """
 256         self._nic_id += 1
 257         if jumbo_frames:
 258             logger.debug(u"Jumbo frames temporarily disabled!")
 259         self._params.add_with_value(
 260             u"chardev", f"socket,id=char{self._nic_id},"
 261             f"path={socket}{u',server' if server is True else u''}"
 262         )
 263         self._params.add_with_value(
 264             u"netdev", f"vhost-user,id=vhost{self._nic_id},"
 265             f"chardev=char{self._nic_id},queues={queues}"
 266         )
 267         mac = f"52:54:00:00:{self._opt.get(u'qemu_id'):02x}:" \
 268             f"{self._nic_id:02x}"
 269         queue_size = f"rx_queue_size={queue_size},tx_queue_size={queue_size}" \
 270             if queue_size else u""
 271         gso = VirtioFeatureMask.is_feature_enabled(
 272             virtio_feature_mask, VirtioFeaturesFlags.VIRTIO_NET_F_API_GSO)
 273         csum = VirtioFeatureMask.is_feature_enabled(
 274             virtio_feature_mask, VirtioFeaturesFlags.VIRTIO_NET_F_API_CSUM)
 275
 276         self._params.add_with_value(
 277             u"device", f"virtio-net-pci,netdev=vhost{self._nic_id},mac={mac},"
 278             f"addr={self._nic_id+5}.0,mq=on,vectors={2 * queues + 2},"
 279             f"csum={u'on' if csum else u'off'},"
 280             f"gso={u'on' if gso else u'off'},"
 281             f"guest_tso4={u'on' if gso else u'off'},"
 282             f"guest_tso6={u'on' if gso else u'off'},"
 283             f"guest_ecn={u'on' if gso else u'off'},"
 284             f"{queue_size}"
 285         )
 286
 287         # Add interface MAC and socket to the node dict.
 288         if_data = {u"mac_address": mac, u"socket": socket}
 289         if_name = f"vhost{self._nic_id}"
 290         self._vm_info[u"interfaces"][if_name] = if_data
 291         # Add socket to temporary file list.
 292         self._temp[if_name] = socket
 293
 294     def add_vfio_pci_if(self, pci):
 295         """Add VFIO PCI interface.
 296
 297         :param pci: PCI address of interface.
 298         :type pci: str
 299         """
 300         self._nic_id += 1
 301         self._params.add_with_value(
 302             u"device", f"vfio-pci,host={pci},addr={self._nic_id+5}.0"
 303         )
 304
 305     def create_kernelvm_config_vpp(self, **kwargs):
 306         """Create QEMU VPP config files.
 307
 308         :param kwargs: Key-value pairs to replace content of VPP configuration
 309             file.
 310         :type kwargs: dict
 311         """
 312         startup = f"/etc/vpp/vm_startup_{self._opt.get(u'qemu_id')}.conf"
 313         running = f"/etc/vpp/vm_running_{self._opt.get(u'qemu_id')}.exec"
 314
 315         self._temp[u"startup"] = startup
 316         self._temp[u"running"] = running
 317         self._opt[u"vnf_bin"] = f"/usr/bin/vpp -c {startup}"
 318
 319         # Create VPP startup configuration.
 320         vpp_config = VppConfigGenerator()
 321         vpp_config.set_node(self._node)
 322         vpp_config.add_unix_nodaemon()
 323         vpp_config.add_unix_cli_listen()
 324         vpp_config.add_unix_exec(running)
 325         vpp_config.add_socksvr()
 326         vpp_config.add_main_heap_size(u"512M")
 327         vpp_config.add_main_heap_page_size(self._opt[u"page_size"])
 328         vpp_config.add_default_hugepage_size(self._opt[u"page_size"])
 329         vpp_config.add_statseg_size(u"512M")
 330         vpp_config.add_statseg_page_size(self._opt[u"page_size"])
 331         vpp_config.add_statseg_per_node_counters(u"on")
 332         vpp_config.add_buffers_per_numa(107520)
 333         vpp_config.add_cpu_main_core(u"0")
 334         if self._opt.get(u"smp") > 1:
 335             vpp_config.add_cpu_corelist_workers(f"1-{self._opt.get(u'smp')-1}")
 336         vpp_config.add_plugin(u"disable", u"default")
 337         vpp_config.add_plugin(u"enable", u"ping_plugin.so")
 338         if "2vfpt" in self._opt.get(u'vnf'):
 339             vpp_config.add_plugin(u"enable", u"avf_plugin.so")
 340         if "vhost" in self._opt.get(u'vnf'):
 341             vpp_config.add_plugin(u"enable", u"dpdk_plugin.so")
 342             vpp_config.add_dpdk_dev(u"0000:00:06.0", u"0000:00:07.0")
 343             vpp_config.add_dpdk_dev_default_rxq(kwargs[u"queues"])
 344             vpp_config.add_dpdk_log_level(u"debug")
 345             if not kwargs[u"jumbo_frames"]:
 346                 vpp_config.add_dpdk_no_multi_seg()
 347                 vpp_config.add_dpdk_no_tx_checksum_offload()
 348         if "ipsec" in self._opt.get(u'vnf'):
 349             vpp_config.add_plugin(u"enable", u"crypto_native_plugin.so")
 350             vpp_config.add_plugin(u"enable", u"crypto_ipsecmb_plugin.so")
 351             vpp_config.add_plugin(u"enable", u"crypto_openssl_plugin.so")
 352         if "nat" in self._opt.get(u'vnf'):
 353             vpp_config.add_nat(value=u"endpoint-dependent")
 354             vpp_config.add_plugin(u"enable", u"nat_plugin.so")
 355         vpp_config.write_config(startup)
 356
 357         # Create VPP running configuration.
 358         template = f"{Constants.RESOURCES_TPL}/vm/{self._opt.get(u'vnf')}.exec"
 359         exec_cmd_no_error(self._node, f"rm -f {running}", sudo=True)
 360
 361         with open(template, u"rt") as src_file:
 362             src = Template(src_file.read())
 363             exec_cmd_no_error(
 364                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 365                 f"sudo tee {running}"
 366             )
 367
 368     def create_kernelvm_config_testpmd_io(self, **kwargs):
 369         """Create QEMU testpmd-io command line.
 370
 371         :param kwargs: Key-value pairs to construct command line parameters.
 372         :type kwargs: dict
 373         """
 374         pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
 375         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 376             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 377             eal_driver=False,
 378             eal_pci_whitelist0=u"0000:00:06.0",
 379             eal_pci_whitelist1=u"0000:00:07.0",
 380             eal_in_memory=True,
 381             pmd_num_mbufs=32768,
 382             pmd_fwd_mode=u"io",
 383             pmd_nb_ports=u"2",
 384             pmd_portmask=u"0x3",
 385             pmd_max_pkt_len=pmd_max_pkt_len,
 386             pmd_mbuf_size=u"16384",
 387             pmd_rxq=kwargs[u"queues"],
 388             pmd_txq=kwargs[u"queues"],
 389             pmd_tx_offloads='0x0',
 390             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 391         )
 392
 393         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 394
 395     def create_kernelvm_config_testpmd_mac(self, **kwargs):
 396         """Create QEMU testpmd-mac command line.
 397
 398         :param kwargs: Key-value pairs to construct command line parameters.
 399         :type kwargs: dict
 400         """
 401         pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
 402         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 403             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 404             eal_driver=False,
 405             eal_pci_whitelist0=u"0000:00:06.0",
 406             eal_pci_whitelist1=u"0000:00:07.0",
 407             eal_in_memory=True,
 408             pmd_num_mbufs=32768,
 409             pmd_fwd_mode=u"mac",
 410             pmd_nb_ports=u"2",
 411             pmd_portmask=u"0x3",
 412             pmd_max_pkt_len=pmd_max_pkt_len,
 413             pmd_mbuf_size=u"16384",
 414             pmd_eth_peer_0=f"0,{kwargs[u'vif1_mac']}",
 415             pmd_eth_peer_1=f"1,{kwargs[u'vif2_mac']}",
 416             pmd_rxq=kwargs[u"queues"],
 417             pmd_txq=kwargs[u"queues"],
 418             pmd_tx_offloads=u"0x0",
 419             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 420         )
 421
 422         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 423
 424     def create_kernelvm_config_iperf3(self):
 425         """Create QEMU iperf3 command line."""
 426         self._opt[u"vnf_bin"] = f"mkdir /run/sshd; /usr/sbin/sshd -D -d"
 427
 428     def create_kernelvm_init(self, **kwargs):
 429         """Create QEMU init script.
 430
 431         :param kwargs: Key-value pairs to replace content of init startup file.
 432         :type kwargs: dict
 433         """
 434         init = self._temp.get(u"ini")
 435         exec_cmd_no_error(self._node, f"rm -f {init}", sudo=True)
 436
 437         with open(kwargs[u"template"], u"rt") as src_file:
 438             src = Template(src_file.read())
 439             exec_cmd_no_error(
 440                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 441                 f"sudo tee {init}"
 442             )
 443             exec_cmd_no_error(self._node, f"chmod +x {init}", sudo=True)
 444
 445     def configure_kernelvm_vnf(self, **kwargs):
 446         """Create KernelVM VNF configurations.
 447
 448         :param kwargs: Key-value pairs for templating configs.
 449         :type kwargs: dict
 450         """
 451         if u"vpp" in self._opt.get(u"vnf"):
 452             self.create_kernelvm_config_vpp(**kwargs)
 453             self.create_kernelvm_init(
 454                 template=f"{Constants.RESOURCES_TPL}/vm/init.sh",
 455                 vnf_bin=self._opt.get(u"vnf_bin")
 456             )
 457         elif u"testpmd_io" in self._opt.get(u"vnf"):
 458             self.create_kernelvm_config_testpmd_io(**kwargs)
 459             self.create_kernelvm_init(
 460                 template=f"{Constants.RESOURCES_TPL}/vm/init.sh",
 461                 vnf_bin=self._opt.get(u"vnf_bin")
 462             )
 463         elif u"testpmd_mac" in self._opt.get(u"vnf"):
 464             self.create_kernelvm_config_testpmd_mac(**kwargs)
 465             self.create_kernelvm_init(
 466                 template=f"{Constants.RESOURCES_TPL}/vm/init.sh",
 467                 vnf_bin=self._opt.get(u"vnf_bin")
 468             )
 469         elif u"iperf3" in self._opt.get(u"vnf"):
 470             qemu_id = self._opt.get(u'qemu_id') % 2
 471             self.create_kernelvm_config_iperf3()
 472             self.create_kernelvm_init(
 473                 template=f"{Constants.RESOURCES_TPL}/vm/init_iperf3.sh",
 474                 vnf_bin=self._opt.get(u"vnf_bin"),
 475                 ip_address_l=u"2.2.2.2/30" if qemu_id else u"1.1.1.1/30",
 476                 ip_address_r=u"2.2.2.1" if qemu_id else u"1.1.1.2",
 477                 ip_route_r=u"1.1.1.0/30" if qemu_id else u"2.2.2.0/30"
 478             )
 479         else:
 480             raise RuntimeError(u"QEMU: Unsupported VNF!")
 481
 482     def get_qemu_pids(self):
 483         """Get QEMU CPU pids.
 484
 485         :returns: List of QEMU CPU pids.
 486         :rtype: list of str
 487         """
 488         command = f"grep -rwl 'CPU' /proc/$(sudo cat " \
 489             f"{self._temp.get(u'pidfile')})/task/*/comm "
 490         command += r"| xargs dirname | sed -e 's/\/.*\///g' | uniq"
 491
 492         stdout, _ = exec_cmd_no_error(self._node, command)
 493         return stdout.splitlines()
 494
 495     def qemu_set_affinity(self, *host_cpus):
 496         """Set qemu affinity by getting thread PIDs via QMP and taskset to list
 497         of CPU cores. Function tries to execute 3 times to avoid race condition
 498         in getting thread PIDs.
 499
 500         :param host_cpus: List of CPU cores.
 501         :type host_cpus: list
 502         """
 503         for _ in range(3):
 504             try:
 505                 qemu_cpus = self.get_qemu_pids()
 506
 507                 if len(qemu_cpus) != len(host_cpus):
 508                     sleep(1)
 509                     continue
 510                 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
 511                     command = f"taskset -pc {host_cpu} {qemu_cpu}"
 512                     message = f"QEMU: Set affinity failed " \
 513                         f"on {self._node[u'host']}!"
 514                     exec_cmd_no_error(
 515                         self._node, command, sudo=True, message=message
 516                     )
 517                 break
 518             except (RuntimeError, ValueError):
 519                 self.qemu_kill_all()
 520                 raise
 521         else:
 522             self.qemu_kill_all()
 523             raise RuntimeError(u"Failed to set Qemu threads affinity!")
 524
 525     def qemu_set_scheduler_policy(self):
 526         """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
 527         processes.
 528
 529         :raises RuntimeError: Set scheduler policy failed.
 530         """
 531         try:
 532             qemu_cpus = self.get_qemu_pids()
 533
 534             for qemu_cpu in qemu_cpus:
 535                 command = f"chrt -r -p 1 {qemu_cpu}"
 536                 message = f"QEMU: Set SCHED_RR failed on {self._node[u'host']}"
 537                 exec_cmd_no_error(
 538                     self._node, command, sudo=True, message=message
 539                 )
 540         except (RuntimeError, ValueError):
 541             self.qemu_kill_all()
 542             raise
 543
 544     def _qemu_qmp_exec(self, cmd):
 545         """Execute QMP command.
 546
 547         QMP is JSON based protocol which allows to control QEMU instance.
 548
 549         :param cmd: QMP command to execute.
 550         :type cmd: str
 551         :returns: Command output in python representation of JSON format. The
 552             { "return": {} } response is QMP's success response. An error
 553             response will contain the "error" keyword instead of "return".
 554         """
 555         # To enter command mode, the qmp_capabilities command must be issued.
 556         command = f"echo \"{{{{ \\\"execute\\\": " \
 557             f"\\\"qmp_capabilities\\\" }}}}" \
 558             f"{{{{ \\\"execute\\\": \\\"{cmd}\\\" }}}}\" | " \
 559             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qmp')}"
 560         message = f"QMP execute '{cmd}' failed on {self._node[u'host']}"
 561
 562         stdout, _ = exec_cmd_no_error(
 563             self._node, command, sudo=False, message=message
 564         )
 565
 566         # Skip capabilities negotiation messages.
 567         out_list = stdout.splitlines()
 568         if len(out_list) < 3:
 569             raise RuntimeError(f"Invalid QMP output on {self._node[u'host']}")
 570         return json.loads(out_list[2])
 571
 572     def _qemu_qga_flush(self):
 573         """Flush the QGA parser state."""
 574         command = f"(printf \"\xFF\"; sleep 1) | sudo -S socat " \
 575             f"- UNIX-CONNECT:{self._temp.get(u'qga')}"
 576         message = f"QGA flush failed on {self._node[u'host']}"
 577         stdout, _ = exec_cmd_no_error(
 578             self._node, command, sudo=False, message=message
 579         )
 580
 581         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 582
 583     def _qemu_qga_exec(self, cmd):
 584         """Execute QGA command.
 585
 586         QGA provide access to a system-level agent via standard QMP commands.
 587
 588         :param cmd: QGA command to execute.
 589         :type cmd: str
 590         """
 591         command = f"(echo \"{{{{ \\\"execute\\\": " \
 592             f"\\\"{cmd}\\\" }}}}\"; sleep 1) | " \
 593             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qga')}"
 594         message = f"QGA execute '{cmd}' failed on {self._node[u'host']}"
 595         stdout, _ = exec_cmd_no_error(
 596             self._node, command, sudo=False, message=message
 597         )
 598
 599         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 600
 601     def _wait_until_vm_boot(self):
 602         """Wait until QEMU VM is booted."""
 603         try:
 604             getattr(self, f'_wait_{self._opt["vnf"]}')()
 605         except AttributeError:
 606             self._wait_default()
 607
 608     def _wait_default(self, retries=60):
 609         """Wait until QEMU with VPP is booted.
 610
 611         :param retries: Number of retries.
 612         :type retries: int
 613         """
 614         for _ in range(retries):
 615             command = f"tail -1 {self._temp.get(u'log')}"
 616             stdout = None
 617             try:
 618                 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 619                 sleep(1)
 620             except RuntimeError:
 621                 pass
 622             if "vpp " in stdout and "built by" in stdout:
 623                 break
 624             if u"Press enter to exit" in stdout:
 625                 break
 626             if u"reboot: Power down" in stdout:
 627                 raise RuntimeError(
 628                     f"QEMU: NF failed to run on {self._node[u'host']}!"
 629                 )
 630         else:
 631             raise RuntimeError(
 632                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 633             )
 634
 635     def _wait_nestedvm(self, retries=12):
 636         """Wait until QEMU with NestedVM is booted.
 637
 638         First try to flush qga until there is output.
 639         Then ping QEMU guest agent each 5s until VM booted or timeout.
 640
 641         :param retries: Number of retries with 5s between trials.
 642         :type retries: int
 643         """
 644         for _ in range(retries):
 645             out = None
 646             try:
 647                 out = self._qemu_qga_flush()
 648             except ValueError:
 649                 logger.trace(f"QGA qga flush unexpected output {out}")
 650             # Empty output - VM not booted yet
 651             if not out:
 652                 sleep(5)
 653             else:
 654                 break
 655         else:
 656             raise RuntimeError(
 657                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 658             )
 659         for _ in range(retries):
 660             out = None
 661             try:
 662                 out = self._qemu_qga_exec(u"guest-ping")
 663             except ValueError:
 664                 logger.trace(f"QGA guest-ping unexpected output {out}")
 665             # Empty output - VM not booted yet.
 666             if not out:
 667                 sleep(5)
 668             # Non-error return - VM booted.
 669             elif out.get(u"return") is not None:
 670                 break
 671             # Skip error and wait.
 672             elif out.get(u"error") is not None:
 673                 sleep(5)
 674             else:
 675                 # If there is an unexpected output from QGA guest-info, try
 676                 # again until timeout.
 677                 logger.trace(f"QGA guest-ping unexpected output {out}")
 678         else:
 679             raise RuntimeError(
 680                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 681             )
 682
 683     def _wait_iperf3(self, retries=60):
 684         """Wait until QEMU with iPerf3 is booted.
 685
 686         :param retries: Number of retries.
 687         :type retries: int
 688         """
 689         grep = u"Server listening on 0.0.0.0 port 22."
 690         cmd = f"fgrep '{grep}' {self._temp.get(u'log')}"
 691         message = f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 692         exec_cmd_no_error(
 693             self._node, cmd=cmd, sudo=True, message=message, retries=retries,
 694             include_reason=True
 695         )
 696
 697     def _update_vm_interfaces(self):
 698         """Update interface names in VM node dict."""
 699         # Send guest-network-get-interfaces command via QGA, output example:
 700         # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
 701         # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
 702         out = self._qemu_qga_exec(u"guest-network-get-interfaces")
 703         interfaces = out.get(u"return")
 704         mac_name = {}
 705         if not interfaces:
 706             raise RuntimeError(
 707                 f"Get VM interface list failed on {self._node[u'host']}"
 708             )
 709         # Create MAC-name dict.
 710         for interface in interfaces:
 711             if u"hardware-address" not in interface:
 712                 continue
 713             mac_name[interface[u"hardware-address"]] = interface[u"name"]
 714         # Match interface by MAC and save interface name.
 715         for interface in self._vm_info[u"interfaces"].values():
 716             mac = interface.get(u"mac_address")
 717             if_name = mac_name.get(mac)
 718             if if_name is None:
 719                 logger.trace(f"Interface name for MAC {mac} not found")
 720             else:
 721                 interface[u"name"] = if_name
 722
 723     def qemu_start(self):
 724         """Start QEMU and wait until VM boot.
 725
 726         :returns: VM node info.
 727         :rtype: dict
 728         """
 729         cmd_opts = OptionString()
 730         cmd_opts.add(f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch}")
 731         cmd_opts.extend(self._params)
 732         message = f"QEMU: Start failed on {self._node[u'host']}!"
 733         try:
 734             DUTSetup.check_huge_page(
 735                 self._node, self._opt.get(u"mem-path"),
 736                 int(self._opt.get(u"mem"))
 737             )
 738
 739             exec_cmd_no_error(
 740                 self._node, cmd_opts, timeout=300, sudo=True, message=message
 741             )
 742             self._wait_until_vm_boot()
 743         except RuntimeError:
 744             self.qemu_kill_all()
 745             raise
 746         return self._vm_info
 747
 748     def qemu_kill(self):
 749         """Kill qemu process."""
 750         exec_cmd(
 751             self._node, f"chmod +r {self._temp.get(u'pidfile')}", sudo=True
 752         )
 753         exec_cmd(
 754             self._node, f"kill -SIGKILL $(cat {self._temp.get(u'pidfile')})",
 755             sudo=True
 756         )
 757
 758         for value in self._temp.values():
 759             exec_cmd(self._node, f"cat {value}", sudo=True)
 760             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 761
 762     def qemu_kill_all(self):
 763         """Kill all qemu processes on DUT node if specified."""
 764         exec_cmd(self._node, u"pkill -SIGKILL qemu", sudo=True)
 765
 766         for value in self._temp.values():
 767             exec_cmd(self._node, f"cat {value}", sudo=True)
 768             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 769
 770     def qemu_version(self):
 771         """Return Qemu version.
 772
 773         :returns: Qemu version.
 774         :rtype: str
 775         """
 776         command = f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch} " \
 777             f"--version"
 778         try:
 779             stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 780             return match(r"QEMU emulator version ([\d.]*)", stdout).group(1)
 781         except RuntimeError:
 782             self.qemu_kill_all()
 783             raise