resources/libraries/python/QemuUtils.py

   1 # Copyright (c) 2021 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """QEMU utilities library."""
  15
  16 import json
  17
  18 from re import match
  19 from string import Template
  20 from time import sleep
  21
  22 from robot.api import logger
  23
  24 from resources.libraries.python.Constants import Constants
  25 from resources.libraries.python.DpdkUtil import DpdkUtil
  26 from resources.libraries.python.DUTSetup import DUTSetup
  27 from resources.libraries.python.OptionString import OptionString
  28 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
  29 from resources.libraries.python.topology import NodeType, Topology
  30 from resources.libraries.python.VhostUser import VirtioFeaturesFlags
  31 from resources.libraries.python.VhostUser import VirtioFeatureMask
  32 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
  33
  34 __all__ = [u"QemuUtils"]
  35
  36
  37 class QemuUtils:
  38     """QEMU utilities."""
  39
  40     # Use one instance of class per tests.
  41     ROBOT_LIBRARY_SCOPE = u"TEST CASE"
  42
  43     def __init__(
  44             self, node, qemu_id=1, smp=1, mem=512, vnf=None,
  45             img=Constants.QEMU_VM_IMAGE):
  46         """Initialize QemuUtil class.
  47
  48         :param node: Node to run QEMU on.
  49         :param qemu_id: QEMU identifier.
  50         :param smp: Number of virtual SMP units (cores).
  51         :param mem: Amount of memory.
  52         :param vnf: Network function workload.
  53         :param img: QEMU disk image or kernel image path.
  54         :type node: dict
  55         :type qemu_id: int
  56         :type smp: int
  57         :type mem: int
  58         :type vnf: str
  59         :type img: str
  60         """
  61         self._nic_id = 0
  62         self._node = node
  63         self._arch = Topology.get_node_arch(self._node)
  64         self._opt = dict()
  65
  66         # Architecture specific options
  67         if self._arch == u"aarch64":
  68             self._opt[u"machine_args"] = \
  69                 u"virt,accel=kvm,usb=off,mem-merge=off,gic-version=3"
  70             self._opt[u"console"] = u"ttyAMA0"
  71         else:
  72             self._opt[u"machine_args"] = u"pc,accel=kvm,usb=off,mem-merge=off"
  73             self._opt[u"console"] = u"ttyS0"
  74         self._testpmd_path = f"{Constants.QEMU_VM_DPDK}/build/app"
  75         self._vm_info = {
  76             u"host": node[u"host"],
  77             u"type": NodeType.VM,
  78             u"port": 10021 + qemu_id,
  79             u"serial": 4555 + qemu_id,
  80             u"username": 'testuser',
  81             u"password": 'Csit1234',
  82             u"interfaces": {},
  83         }
  84         if node[u"port"] != 22:
  85             self._vm_info[u"host_port"] = node[u"port"]
  86             self._vm_info[u"host_username"] = node[u"username"]
  87             self._vm_info[u"host_password"] = node[u"password"]
  88         # Input Options.
  89         self._opt[u"qemu_id"] = qemu_id
  90         self._opt[u"mem"] = int(mem)
  91         self._opt[u"smp"] = int(smp)
  92         self._opt[u"img"] = img
  93         self._opt[u"vnf"] = vnf
  94         # Temporary files.
  95         self._temp = dict()
  96         self._temp[u"log"] = f"/tmp/serial_{qemu_id}.log"
  97         self._temp[u"pidfile"] = f"/run/qemu_{qemu_id}.pid"
  98         if img == Constants.QEMU_VM_IMAGE:
  99             self._temp[u"qmp"] = f"/run/qmp_{qemu_id}.sock"
 100             self._temp[u"qga"] = f"/run/qga_{qemu_id}.sock"
 101         elif img == Constants.QEMU_VM_KERNEL:
 102             self._opt[u"img"], _ = exec_cmd_no_error(
 103                 node, f"ls -1 {Constants.QEMU_VM_KERNEL}* | tail -1",
 104                 message=u"Qemu Kernel VM image not found!"
 105             )
 106             self._temp[u"ini"] = f"/etc/vm_init_{qemu_id}.conf"
 107             self._opt[u"initrd"], _ = exec_cmd_no_error(
 108                 node, f"ls -1 {Constants.QEMU_VM_KERNEL_INITRD}* | tail -1",
 109                 message=u"Qemu Kernel initrd image not found!"
 110             )
 111         else:
 112             raise RuntimeError(f"QEMU: Unknown VM image option: {img}")
 113         # Computed parameters for QEMU command line.
 114         self._params = OptionString(prefix=u"-")
 115
 116     def add_default_params(self):
 117         """Set default QEMU command line parameters."""
 118         self._params.add(u"daemonize")
 119         self._params.add(u"nodefaults")
 120         self._params.add_with_value(
 121             u"name", f"vnf{self._opt.get(u'qemu_id')},debug-threads=on"
 122         )
 123         self._params.add(u"no-user-config")
 124         self._params.add(u"nographic")
 125         self._params.add(u"enable-kvm")
 126         self._params.add_with_value(u"pidfile", self._temp.get(u"pidfile"))
 127         self._params.add_with_value(u"cpu", u"host")
 128
 129         self._params.add_with_value(u"machine", self._opt.get(u"machine_args"))
 130         self._params.add_with_value(
 131             u"smp", f"{self._opt.get(u'smp')},sockets=1,"
 132             f"cores={self._opt.get(u'smp')},threads=1"
 133         )
 134         self._params.add_with_value(
 135             u"object", f"memory-backend-file,id=mem,"
 136             f"size={self._opt.get(u'mem')}M,mem-path=/dev/hugepages,share=on"
 137         )
 138         self._params.add_with_value(u"m", f"{self._opt.get(u'mem')}M")
 139         self._params.add_with_value(u"numa", u"node,memdev=mem")
 140
 141     def add_net_user(self, net="10.0.2.0/24"):
 142         """Set managment port forwarding."""
 143         self._params.add_with_value(
 144             u"netdev", f"user,id=mgmt,net={net},"
 145             f"hostfwd=tcp::{self._vm_info[u'port']}-:22"
 146         )
 147         self._params.add_with_value(
 148             u"device", f"virtio-net,netdev=mgmt"
 149         )
 150
 151     def add_qmp_qga(self):
 152         """Set QMP, QGA management."""
 153         self._params.add_with_value(
 154             u"chardev", f"socket,path={self._temp.get(u'qga')},"
 155             f"server,nowait,id=qga0"
 156         )
 157         self._params.add_with_value(
 158             u"device", u"isa-serial,chardev=qga0"
 159         )
 160         self._params.add_with_value(
 161             u"qmp", f"unix:{self._temp.get(u'qmp')},server,nowait"
 162         )
 163
 164     def add_serial(self):
 165         """Set serial to file redirect."""
 166         self._params.add_with_value(
 167             u"chardev", f"socket,host=127.0.0.1,"
 168             f"port={self._vm_info[u'serial']},id=gnc0,server,nowait"
 169         )
 170         self._params.add_with_value(
 171             u"device", u"isa-serial,chardev=gnc0"
 172         )
 173         self._params.add_with_value(
 174             u"serial", f"file:{self._temp.get(u'log')}"
 175         )
 176
 177     def add_drive_cdrom(self, drive_file, index=None):
 178         """Set CD-ROM drive.
 179
 180         :param drive_file: Path to drive image.
 181         :param index: Drive index.
 182         :type drive_file: str
 183         :type index: int
 184         """
 185         index = f"index={index}," if index else u""
 186         self._params.add_with_value(
 187             u"drive", f"file={drive_file},{index}media=cdrom"
 188         )
 189
 190     def add_drive(self, drive_file, drive_format):
 191         """Set drive with custom format.
 192
 193         :param drive_file: Path to drive image.
 194         :param drive_format: Drive image format.
 195         :type drive_file: str
 196         :type drive_format: str
 197         """
 198         self._params.add_with_value(
 199             u"drive", f"file={drive_file},format={drive_format},"
 200             u"cache=none,if=virtio,file.locking=off"
 201         )
 202
 203     def add_kernelvm_params(self):
 204         """Set KernelVM QEMU parameters."""
 205         self._params.add_with_value(
 206             u"serial", f"file:{self._temp.get(u'log')}"
 207         )
 208         self._params.add_with_value(
 209             u"fsdev", u"local,id=root9p,path=/,security_model=none"
 210         )
 211         self._params.add_with_value(
 212             u"device", u"virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot"
 213         )
 214         self._params.add_with_value(
 215             u"kernel", f"{self._opt.get(u'img')}"
 216         )
 217         self._params.add_with_value(
 218             u"initrd", f"{self._opt.get(u'initrd')}"
 219         )
 220         self._params.add_with_value(
 221             u"append", f"'ro rootfstype=9p rootflags=trans=virtio "
 222             f"root=virtioroot console={self._opt.get(u'console')} "
 223             f"tsc=reliable hugepages=512 "
 224             f"init={self._temp.get(u'ini')} fastboot'"
 225         )
 226
 227     def add_vhost_user_if(
 228             self, socket, server=True, jumbo_frames=False, queue_size=None,
 229             queues=1, virtio_feature_mask=None):
 230         """Add Vhost-user interface.
 231
 232         :param socket: Path of the unix socket.
 233         :param server: If True the socket shall be a listening socket.
 234         :param jumbo_frames: Set True if jumbo frames are used in the test.
 235         :param queue_size: Vring queue size.
 236         :param queues: Number of queues.
 237         :param virtio_feature_mask: Mask of virtio features to be enabled.
 238         :type socket: str
 239         :type server: bool
 240         :type jumbo_frames: bool
 241         :type queue_size: int
 242         :type queues: int
 243         :type virtio_feature_mask: int
 244         """
 245         self._nic_id += 1
 246         self._params.add_with_value(
 247             u"chardev", f"socket,id=char{self._nic_id},"
 248             f"path={socket}{u',server' if server is True else u''}"
 249         )
 250         self._params.add_with_value(
 251             u"netdev", f"vhost-user,id=vhost{self._nic_id},"
 252             f"chardev=char{self._nic_id},queues={queues}"
 253         )
 254         mac = f"52:54:00:00:{self._opt.get(u'qemu_id'):02x}:" \
 255             f"{self._nic_id:02x}"
 256         queue_size = f"rx_queue_size={queue_size},tx_queue_size={queue_size}" \
 257             if queue_size else u""
 258         gso = VirtioFeatureMask.is_feature_enabled(
 259             virtio_feature_mask, VirtioFeaturesFlags.VIRTIO_NET_F_API_GSO)
 260         csum = VirtioFeatureMask.is_feature_enabled(
 261             virtio_feature_mask, VirtioFeaturesFlags.VIRTIO_NET_F_API_CSUM)
 262
 263         self._params.add_with_value(
 264             u"device", f"virtio-net-pci,netdev=vhost{self._nic_id},mac={mac},"
 265             f"addr={self._nic_id+5}.0,mq=on,vectors={2 * queues + 2},"
 266             f"csum={u'on' if csum else u'off'},"
 267             f"gso={u'on' if gso else u'off'},"
 268             f"guest_tso4={u'on' if gso else u'off'},"
 269             f"guest_tso6={u'on' if gso else u'off'},"
 270             f"guest_ecn={u'on' if gso else u'off'},"
 271             f"{queue_size}"
 272         )
 273
 274         # Add interface MAC and socket to the node dict.
 275         if_data = {u"mac_address": mac, u"socket": socket}
 276         if_name = f"vhost{self._nic_id}"
 277         self._vm_info[u"interfaces"][if_name] = if_data
 278         # Add socket to temporary file list.
 279         self._temp[if_name] = socket
 280
 281     def add_vfio_pci_if(self, pci):
 282         """Add VFIO PCI interface.
 283
 284         :param pci: PCI address of interface.
 285         :type pci: str
 286         """
 287         self._nic_id += 1
 288         self._params.add_with_value(
 289             u"device", f"vfio-pci,host={pci},addr={self._nic_id+5}.0"
 290         )
 291
 292     def create_kernelvm_config_vpp(self, **kwargs):
 293         """Create QEMU VPP config files.
 294
 295         :param kwargs: Key-value pairs to replace content of VPP configuration
 296             file.
 297         :type kwargs: dict
 298         """
 299         startup = f"/etc/vpp/vm_startup_{self._opt.get(u'qemu_id')}.conf"
 300         running = f"/etc/vpp/vm_running_{self._opt.get(u'qemu_id')}.exec"
 301
 302         self._temp[u"startup"] = startup
 303         self._temp[u"running"] = running
 304         self._opt[u"vnf_bin"] = f"/usr/bin/vpp -c {startup}"
 305
 306         # Create VPP startup configuration.
 307         vpp_config = VppConfigGenerator()
 308         vpp_config.set_node(self._node)
 309         vpp_config.add_unix_nodaemon()
 310         vpp_config.add_unix_cli_listen()
 311         vpp_config.add_unix_exec(running)
 312         vpp_config.add_socksvr()
 313         vpp_config.add_main_heap_size(u"512M")
 314         vpp_config.add_main_heap_page_size(u"2M")
 315         vpp_config.add_statseg_size(u"512M")
 316         vpp_config.add_statseg_page_size(u"2M")
 317         vpp_config.add_statseg_per_node_counters(u"on")
 318         vpp_config.add_buffers_per_numa(107520)
 319         vpp_config.add_cpu_main_core(u"0")
 320         if self._opt.get(u"smp") > 1:
 321             vpp_config.add_cpu_corelist_workers(f"1-{self._opt.get(u'smp')-1}")
 322         vpp_config.add_plugin(u"disable", u"default")
 323         vpp_config.add_plugin(u"enable", u"ping_plugin.so")
 324         if "2vfpt" in self._opt.get(u'vnf'):
 325             vpp_config.add_plugin(u"enable", u"avf_plugin.so")
 326         if "vhost" in self._opt.get(u'vnf'):
 327             vpp_config.add_plugin(u"enable", u"dpdk_plugin.so")
 328             vpp_config.add_dpdk_dev(u"0000:00:06.0", u"0000:00:07.0")
 329             vpp_config.add_dpdk_dev_default_rxq(kwargs[u"queues"])
 330             vpp_config.add_dpdk_log_level(u"debug")
 331             if not kwargs[u"jumbo_frames"]:
 332                 vpp_config.add_dpdk_no_multi_seg()
 333                 vpp_config.add_dpdk_no_tx_checksum_offload()
 334         if "ipsec" in self._opt.get(u'vnf'):
 335             vpp_config.add_plugin(u"enable", u"crypto_native_plugin.so")
 336             vpp_config.add_plugin(u"enable", u"crypto_ipsecmb_plugin.so")
 337             vpp_config.add_plugin(u"enable", u"crypto_openssl_plugin.so")
 338         if "nat" in self._opt.get(u'vnf'):
 339             vpp_config.add_nat(value=u"endpoint-dependent")
 340             vpp_config.add_plugin(u"enable", u"nat_plugin.so")
 341         vpp_config.write_config(startup)
 342
 343         # Create VPP running configuration.
 344         template = f"{Constants.RESOURCES_TPL}/vm/{self._opt.get(u'vnf')}.exec"
 345         exec_cmd_no_error(self._node, f"rm -f {running}", sudo=True)
 346
 347         with open(template, u"rt") as src_file:
 348             src = Template(src_file.read())
 349             exec_cmd_no_error(
 350                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 351                 f"sudo tee {running}"
 352             )
 353
 354     def create_kernelvm_config_testpmd_io(self, **kwargs):
 355         """Create QEMU testpmd-io command line.
 356
 357         :param kwargs: Key-value pairs to construct command line parameters.
 358         :type kwargs: dict
 359         """
 360         pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
 361         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 362             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 363             eal_driver=False,
 364             eal_pci_whitelist0=u"0000:00:06.0",
 365             eal_pci_whitelist1=u"0000:00:07.0",
 366             eal_in_memory=True,
 367             pmd_num_mbufs=16384,
 368             pmd_fwd_mode=u"io",
 369             pmd_nb_ports=u"2",
 370             pmd_portmask=u"0x3",
 371             pmd_max_pkt_len=pmd_max_pkt_len,
 372             pmd_mbuf_size=u"16384",
 373             pmd_rxq=kwargs[u"queues"],
 374             pmd_txq=kwargs[u"queues"],
 375             pmd_tx_offloads='0x0',
 376             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 377         )
 378
 379         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 380
 381     def create_kernelvm_config_testpmd_mac(self, **kwargs):
 382         """Create QEMU testpmd-mac command line.
 383
 384         :param kwargs: Key-value pairs to construct command line parameters.
 385         :type kwargs: dict
 386         """
 387         pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
 388         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 389             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 390             eal_driver=False,
 391             eal_pci_whitelist0=u"0000:00:06.0",
 392             eal_pci_whitelist1=u"0000:00:07.0",
 393             eal_in_memory=True,
 394             pmd_num_mbufs=16384,
 395             pmd_fwd_mode=u"mac",
 396             pmd_nb_ports=u"2",
 397             pmd_portmask=u"0x3",
 398             pmd_max_pkt_len=pmd_max_pkt_len,
 399             pmd_mbuf_size=u"16384",
 400             pmd_eth_peer_0=f"0,{kwargs[u'vif1_mac']}",
 401             pmd_eth_peer_1=f"1,{kwargs[u'vif2_mac']}",
 402             pmd_rxq=kwargs[u"queues"],
 403             pmd_txq=kwargs[u"queues"],
 404             pmd_tx_offloads=u"0x0",
 405             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 406         )
 407
 408         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 409
 410     def create_kernelvm_config_iperf3(self):
 411         """Create QEMU iperf3 command line."""
 412         self._opt[u"vnf_bin"] = f"mkdir /run/sshd; /usr/sbin/sshd -D -d"
 413
 414     def create_kernelvm_init(self, **kwargs):
 415         """Create QEMU init script.
 416
 417         :param kwargs: Key-value pairs to replace content of init startup file.
 418         :type kwargs: dict
 419         """
 420         init = self._temp.get(u"ini")
 421         exec_cmd_no_error(self._node, f"rm -f {init}", sudo=True)
 422
 423         with open(kwargs[u"template"], u"rt") as src_file:
 424             src = Template(src_file.read())
 425             exec_cmd_no_error(
 426                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 427                 f"sudo tee {init}"
 428             )
 429             exec_cmd_no_error(self._node, f"chmod +x {init}", sudo=True)
 430
 431     def configure_kernelvm_vnf(self, **kwargs):
 432         """Create KernelVM VNF configurations.
 433
 434         :param kwargs: Key-value pairs for templating configs.
 435         :type kwargs: dict
 436         """
 437         if u"vpp" in self._opt.get(u"vnf"):
 438             self.create_kernelvm_config_vpp(**kwargs)
 439             self.create_kernelvm_init(
 440                 template=f"{Constants.RESOURCES_TPL}/vm/init.sh",
 441                 vnf_bin=self._opt.get(u"vnf_bin")
 442             )
 443         elif u"testpmd_io" in self._opt.get(u"vnf"):
 444             self.create_kernelvm_config_testpmd_io(**kwargs)
 445             self.create_kernelvm_init(
 446                 template=f"{Constants.RESOURCES_TPL}/vm/init.sh",
 447                 vnf_bin=self._opt.get(u"vnf_bin")
 448             )
 449         elif u"testpmd_mac" in self._opt.get(u"vnf"):
 450             self.create_kernelvm_config_testpmd_mac(**kwargs)
 451             self.create_kernelvm_init(
 452                 template=f"{Constants.RESOURCES_TPL}/vm/init.sh",
 453                 vnf_bin=self._opt.get(u"vnf_bin")
 454             )
 455         elif u"iperf3" in self._opt.get(u"vnf"):
 456             qemu_id = self._opt.get(u'qemu_id') % 2
 457             self.create_kernelvm_config_iperf3()
 458             self.create_kernelvm_init(
 459                 template=f"{Constants.RESOURCES_TPL}/vm/init_iperf3.sh",
 460                 vnf_bin=self._opt.get(u"vnf_bin"),
 461                 ip_address_l=u"2.2.2.2/30" if qemu_id else u"1.1.1.1/30",
 462                 ip_address_r=u"2.2.2.1" if qemu_id else u"1.1.1.2",
 463                 ip_route_r=u"1.1.1.0/30" if qemu_id else u"2.2.2.0/30"
 464             )
 465         else:
 466             raise RuntimeError(u"QEMU: Unsupported VNF!")
 467
 468     def get_qemu_pids(self):
 469         """Get QEMU CPU pids.
 470
 471         :returns: List of QEMU CPU pids.
 472         :rtype: list of str
 473         """
 474         command = f"grep -rwl 'CPU' /proc/$(sudo cat " \
 475             f"{self._temp.get(u'pidfile')})/task/*/comm "
 476         command += r"| xargs dirname | sed -e 's/\/.*\///g' | uniq"
 477
 478         stdout, _ = exec_cmd_no_error(self._node, command)
 479         return stdout.splitlines()
 480
 481     def qemu_set_affinity(self, *host_cpus):
 482         """Set qemu affinity by getting thread PIDs via QMP and taskset to list
 483         of CPU cores. Function tries to execute 3 times to avoid race condition
 484         in getting thread PIDs.
 485
 486         :param host_cpus: List of CPU cores.
 487         :type host_cpus: list
 488         """
 489         for _ in range(3):
 490             try:
 491                 qemu_cpus = self.get_qemu_pids()
 492
 493                 if len(qemu_cpus) != len(host_cpus):
 494                     sleep(1)
 495                     continue
 496                 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
 497                     command = f"taskset -pc {host_cpu} {qemu_cpu}"
 498                     message = f"QEMU: Set affinity failed " \
 499                         f"on {self._node[u'host']}!"
 500                     exec_cmd_no_error(
 501                         self._node, command, sudo=True, message=message
 502                     )
 503                 break
 504             except (RuntimeError, ValueError):
 505                 self.qemu_kill_all()
 506                 raise
 507         else:
 508             self.qemu_kill_all()
 509             raise RuntimeError(u"Failed to set Qemu threads affinity!")
 510
 511     def qemu_set_scheduler_policy(self):
 512         """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
 513         processes.
 514
 515         :raises RuntimeError: Set scheduler policy failed.
 516         """
 517         try:
 518             qemu_cpus = self.get_qemu_pids()
 519
 520             for qemu_cpu in qemu_cpus:
 521                 command = f"chrt -r -p 1 {qemu_cpu}"
 522                 message = f"QEMU: Set SCHED_RR failed on {self._node[u'host']}"
 523                 exec_cmd_no_error(
 524                     self._node, command, sudo=True, message=message
 525                 )
 526         except (RuntimeError, ValueError):
 527             self.qemu_kill_all()
 528             raise
 529
 530     def _qemu_qmp_exec(self, cmd):
 531         """Execute QMP command.
 532
 533         QMP is JSON based protocol which allows to control QEMU instance.
 534
 535         :param cmd: QMP command to execute.
 536         :type cmd: str
 537         :returns: Command output in python representation of JSON format. The
 538             { "return": {} } response is QMP's success response. An error
 539             response will contain the "error" keyword instead of "return".
 540         """
 541         # To enter command mode, the qmp_capabilities command must be issued.
 542         command = f"echo \"{{{{ \\\"execute\\\": " \
 543             f"\\\"qmp_capabilities\\\" }}}}" \
 544             f"{{{{ \\\"execute\\\": \\\"{cmd}\\\" }}}}\" | " \
 545             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qmp')}"
 546         message = f"QMP execute '{cmd}' failed on {self._node[u'host']}"
 547
 548         stdout, _ = exec_cmd_no_error(
 549             self._node, command, sudo=False, message=message
 550         )
 551
 552         # Skip capabilities negotiation messages.
 553         out_list = stdout.splitlines()
 554         if len(out_list) < 3:
 555             raise RuntimeError(f"Invalid QMP output on {self._node[u'host']}")
 556         return json.loads(out_list[2])
 557
 558     def _qemu_qga_flush(self):
 559         """Flush the QGA parser state."""
 560         command = f"(printf \"\xFF\"; sleep 1) | sudo -S socat " \
 561             f"- UNIX-CONNECT:{self._temp.get(u'qga')}"
 562         message = f"QGA flush failed on {self._node[u'host']}"
 563         stdout, _ = exec_cmd_no_error(
 564             self._node, command, sudo=False, message=message
 565         )
 566
 567         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 568
 569     def _qemu_qga_exec(self, cmd):
 570         """Execute QGA command.
 571
 572         QGA provide access to a system-level agent via standard QMP commands.
 573
 574         :param cmd: QGA command to execute.
 575         :type cmd: str
 576         """
 577         command = f"(echo \"{{{{ \\\"execute\\\": " \
 578             f"\\\"{cmd}\\\" }}}}\"; sleep 1) | " \
 579             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qga')}"
 580         message = f"QGA execute '{cmd}' failed on {self._node[u'host']}"
 581         stdout, _ = exec_cmd_no_error(
 582             self._node, command, sudo=False, message=message
 583         )
 584
 585         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 586
 587     def _wait_until_vm_boot(self):
 588         """Wait until QEMU VM is booted."""
 589         try:
 590             getattr(self, f'_wait_{self._opt["vnf"]}')()
 591         except AttributeError:
 592             self._wait_default()
 593
 594     def _wait_default(self, retries=60):
 595         """Wait until QEMU with VPP is booted.
 596
 597         :param retries: Number of retries.
 598         :type retries: int
 599         """
 600         for _ in range(retries):
 601             command = f"tail -1 {self._temp.get(u'log')}"
 602             stdout = None
 603             try:
 604                 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 605                 sleep(1)
 606             except RuntimeError:
 607                 pass
 608             if "vpp " in stdout and "built by" in stdout:
 609                 break
 610             if u"Press enter to exit" in stdout:
 611                 break
 612             if u"reboot: Power down" in stdout:
 613                 raise RuntimeError(
 614                     f"QEMU: NF failed to run on {self._node[u'host']}!"
 615                 )
 616         else:
 617             raise RuntimeError(
 618                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 619             )
 620
 621     def _wait_nestedvm(self, retries=12):
 622         """Wait until QEMU with NestedVM is booted.
 623
 624         First try to flush qga until there is output.
 625         Then ping QEMU guest agent each 5s until VM booted or timeout.
 626
 627         :param retries: Number of retries with 5s between trials.
 628         :type retries: int
 629         """
 630         for _ in range(retries):
 631             out = None
 632             try:
 633                 out = self._qemu_qga_flush()
 634             except ValueError:
 635                 logger.trace(f"QGA qga flush unexpected output {out}")
 636             # Empty output - VM not booted yet
 637             if not out:
 638                 sleep(5)
 639             else:
 640                 break
 641         else:
 642             raise RuntimeError(
 643                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 644             )
 645         for _ in range(retries):
 646             out = None
 647             try:
 648                 out = self._qemu_qga_exec(u"guest-ping")
 649             except ValueError:
 650                 logger.trace(f"QGA guest-ping unexpected output {out}")
 651             # Empty output - VM not booted yet.
 652             if not out:
 653                 sleep(5)
 654             # Non-error return - VM booted.
 655             elif out.get(u"return") is not None:
 656                 break
 657             # Skip error and wait.
 658             elif out.get(u"error") is not None:
 659                 sleep(5)
 660             else:
 661                 # If there is an unexpected output from QGA guest-info, try
 662                 # again until timeout.
 663                 logger.trace(f"QGA guest-ping unexpected output {out}")
 664         else:
 665             raise RuntimeError(
 666                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 667             )
 668
 669     def _wait_iperf3(self, retries=60):
 670         """Wait until QEMU with iPerf3 is booted.
 671
 672         :param retries: Number of retries.
 673         :type retries: int
 674         """
 675         grep = u"Server listening on 0.0.0.0 port 22."
 676         cmd = f"fgrep '{grep}' {self._temp.get(u'log')}"
 677         message = f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 678         exec_cmd_no_error(
 679             self._node, cmd=cmd, sudo=True, message=message, retries=retries,
 680             include_reason=True
 681         )
 682
 683     def _update_vm_interfaces(self):
 684         """Update interface names in VM node dict."""
 685         # Send guest-network-get-interfaces command via QGA, output example:
 686         # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
 687         # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
 688         out = self._qemu_qga_exec(u"guest-network-get-interfaces")
 689         interfaces = out.get(u"return")
 690         mac_name = {}
 691         if not interfaces:
 692             raise RuntimeError(
 693                 f"Get VM interface list failed on {self._node[u'host']}"
 694             )
 695         # Create MAC-name dict.
 696         for interface in interfaces:
 697             if u"hardware-address" not in interface:
 698                 continue
 699             mac_name[interface[u"hardware-address"]] = interface[u"name"]
 700         # Match interface by MAC and save interface name.
 701         for interface in self._vm_info[u"interfaces"].values():
 702             mac = interface.get(u"mac_address")
 703             if_name = mac_name.get(mac)
 704             if if_name is None:
 705                 logger.trace(f"Interface name for MAC {mac} not found")
 706             else:
 707                 interface[u"name"] = if_name
 708
 709     def qemu_start(self):
 710         """Start QEMU and wait until VM boot.
 711
 712         :returns: VM node info.
 713         :rtype: dict
 714         """
 715         cmd_opts = OptionString()
 716         cmd_opts.add(f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch}")
 717         cmd_opts.extend(self._params)
 718         message = f"QEMU: Start failed on {self._node[u'host']}!"
 719         try:
 720             DUTSetup.check_huge_page(
 721                 self._node, u"/dev/hugepages", int(self._opt.get(u"mem")))
 722
 723             exec_cmd_no_error(
 724                 self._node, cmd_opts, timeout=300, sudo=True, message=message
 725             )
 726             self._wait_until_vm_boot()
 727         except RuntimeError:
 728             self.qemu_kill_all()
 729             raise
 730         return self._vm_info
 731
 732     def qemu_kill(self):
 733         """Kill qemu process."""
 734         exec_cmd(
 735             self._node, f"chmod +r {self._temp.get(u'pidfile')}", sudo=True
 736         )
 737         exec_cmd(
 738             self._node, f"kill -SIGKILL $(cat {self._temp.get(u'pidfile')})",
 739             sudo=True
 740         )
 741
 742         for value in self._temp.values():
 743             exec_cmd(self._node, f"cat {value}", sudo=True)
 744             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 745
 746     def qemu_kill_all(self):
 747         """Kill all qemu processes on DUT node if specified."""
 748         exec_cmd(self._node, u"pkill -SIGKILL qemu", sudo=True)
 749
 750         for value in self._temp.values():
 751             exec_cmd(self._node, f"cat {value}", sudo=True)
 752             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 753
 754     def qemu_version(self):
 755         """Return Qemu version.
 756
 757         :returns: Qemu version.
 758         :rtype: str
 759         """
 760         command = f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch} " \
 761             f"--version"
 762         try:
 763             stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 764             return match(r"QEMU emulator version ([\d.]*)", stdout).group(1)
 765         except RuntimeError:
 766             self.qemu_kill_all()
 767             raise