resources/libraries/python/QemuUtils.py

   1 # Copyright (c) 2021 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """QEMU utilities library."""
  15
  16 import json
  17
  18 from re import match
  19 from string import Template
  20 from time import sleep
  21
  22 from robot.api import logger
  23
  24 from resources.libraries.python.Constants import Constants
  25 from resources.libraries.python.DpdkUtil import DpdkUtil
  26 from resources.libraries.python.DUTSetup import DUTSetup
  27 from resources.libraries.python.OptionString import OptionString
  28 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
  29 from resources.libraries.python.topology import NodeType, Topology
  30 from resources.libraries.python.VhostUser import VirtioFeaturesFlags
  31 from resources.libraries.python.VhostUser import VirtioFeatureMask
  32 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
  33
  34 __all__ = [u"QemuUtils"]
  35
  36
  37 class QemuUtils:
  38     """QEMU utilities."""
  39
  40     # Use one instance of class per tests.
  41     ROBOT_LIBRARY_SCOPE = u"TEST CASE"
  42
  43     def __init__(
  44             self, node, qemu_id=1, smp=1, mem=512, vnf=None,
  45             img=Constants.QEMU_VM_IMAGE):
  46         """Initialize QemuUtil class.
  47
  48         :param node: Node to run QEMU on.
  49         :param qemu_id: QEMU identifier.
  50         :param smp: Number of virtual SMP units (cores).
  51         :param mem: Amount of memory.
  52         :param vnf: Network function workload.
  53         :param img: QEMU disk image or kernel image path.
  54         :type node: dict
  55         :type qemu_id: int
  56         :type smp: int
  57         :type mem: int
  58         :type vnf: str
  59         :type img: str
  60         """
  61         self._nic_id = 0
  62         self._node = node
  63         self._arch = Topology.get_node_arch(self._node)
  64         self._opt = dict()
  65
  66         # Architecture specific options
  67         if self._arch == u"aarch64":
  68             self._opt[u"machine_args"] = \
  69                 u"virt,accel=kvm,usb=off,mem-merge=off,gic-version=3"
  70             self._opt[u"console"] = u"ttyAMA0"
  71         else:
  72             self._opt[u"machine_args"] = u"pc,accel=kvm,usb=off,mem-merge=off"
  73             self._opt[u"console"] = u"ttyS0"
  74         self._testpmd_path = f"{Constants.QEMU_VM_DPDK}/build/app"
  75         self._vm_info = {
  76             u"host": node[u"host"],
  77             u"type": NodeType.VM,
  78             u"port": 10021 + qemu_id,
  79             u"serial": 4555 + qemu_id,
  80             u"username": 'testuser',
  81             u"password": 'Csit1234',
  82             u"interfaces": {},
  83         }
  84         if node[u"port"] != 22:
  85             self._vm_info[u"host_port"] = node[u"port"]
  86             self._vm_info[u"host_username"] = node[u"username"]
  87             self._vm_info[u"host_password"] = node[u"password"]
  88         # Input Options.
  89         self._opt[u"qemu_id"] = qemu_id
  90         self._opt[u"mem"] = int(mem)
  91         self._opt[u"smp"] = int(smp)
  92         self._opt[u"img"] = img
  93         self._opt[u"vnf"] = vnf
  94         # Temporary files.
  95         self._temp = dict()
  96         self._temp[u"log"] = f"/tmp/serial_{qemu_id}.log"
  97         self._temp[u"pidfile"] = f"/run/qemu_{qemu_id}.pid"
  98         if img == Constants.QEMU_VM_IMAGE:
  99             self._temp[u"qmp"] = f"/run/qmp_{qemu_id}.sock"
 100             self._temp[u"qga"] = f"/run/qga_{qemu_id}.sock"
 101         elif img == Constants.QEMU_VM_KERNEL:
 102             self._opt[u"img"], _ = exec_cmd_no_error(
 103                 node, f"ls -1 {Constants.QEMU_VM_KERNEL}* | tail -1",
 104                 message=u"Qemu Kernel VM image not found!"
 105             )
 106             self._temp[u"ini"] = f"/etc/vm_init_{qemu_id}.conf"
 107             self._opt[u"initrd"], _ = exec_cmd_no_error(
 108                 node, f"ls -1 {Constants.QEMU_VM_KERNEL_INITRD}* | tail -1",
 109                 message=u"Qemu Kernel initrd image not found!"
 110             )
 111         else:
 112             raise RuntimeError(f"QEMU: Unknown VM image option: {img}")
 113         # Computed parameters for QEMU command line.
 114         self._params = OptionString(prefix=u"-")
 115
 116     def add_default_params(self):
 117         """Set default QEMU command line parameters."""
 118         self._params.add(u"daemonize")
 119         self._params.add(u"nodefaults")
 120         self._params.add_with_value(
 121             u"name", f"vnf{self._opt.get(u'qemu_id')},debug-threads=on"
 122         )
 123         self._params.add(u"no-user-config")
 124         self._params.add(u"nographic")
 125         self._params.add(u"enable-kvm")
 126         self._params.add_with_value(u"pidfile", self._temp.get(u"pidfile"))
 127         self._params.add_with_value(u"cpu", u"host")
 128
 129         self._params.add_with_value(u"machine", self._opt.get(u"machine_args"))
 130         self._params.add_with_value(
 131             u"smp", f"{self._opt.get(u'smp')},sockets=1,"
 132             f"cores={self._opt.get(u'smp')},threads=1"
 133         )
 134         self._params.add_with_value(
 135             u"object", f"memory-backend-file,id=mem,"
 136             f"size={self._opt.get(u'mem')}M,mem-path=/dev/hugepages,share=on"
 137         )
 138         self._params.add_with_value(u"m", f"{self._opt.get(u'mem')}M")
 139         self._params.add_with_value(u"numa", u"node,memdev=mem")
 140
 141     def add_net_user(self, net="10.0.2.0/24"):
 142         """Set managment port forwarding."""
 143         self._params.add_with_value(
 144             u"netdev", f"user,id=mgmt,net={net},"
 145             f"hostfwd=tcp::{self._vm_info[u'port']}-:22"
 146         )
 147         self._params.add_with_value(
 148             u"device", f"virtio-net,netdev=mgmt"
 149         )
 150
 151     def add_qmp_qga(self):
 152         """Set QMP, QGA management."""
 153         self._params.add_with_value(
 154             u"chardev", f"socket,path={self._temp.get(u'qga')},"
 155             f"server,nowait,id=qga0"
 156         )
 157         self._params.add_with_value(
 158             u"device", u"isa-serial,chardev=qga0"
 159         )
 160         self._params.add_with_value(
 161             u"qmp", f"unix:{self._temp.get(u'qmp')},server,nowait"
 162         )
 163
 164     def add_serial(self):
 165         """Set serial to file redirect."""
 166         self._params.add_with_value(
 167             u"chardev", f"socket,host=127.0.0.1,"
 168             f"port={self._vm_info[u'serial']},id=gnc0,server,nowait"
 169         )
 170         self._params.add_with_value(
 171             u"device", u"isa-serial,chardev=gnc0"
 172         )
 173         self._params.add_with_value(
 174             u"serial", f"file:{self._temp.get(u'log')}"
 175         )
 176
 177     def add_drive_cdrom(self, drive_file, index=None):
 178         """Set CD-ROM drive.
 179
 180         :param drive_file: Path to drive image.
 181         :param index: Drive index.
 182         :type drive_file: str
 183         :type index: int
 184         """
 185         index = f"index={index}," if index else u""
 186         self._params.add_with_value(
 187             u"drive", f"file={drive_file},{index}media=cdrom"
 188         )
 189
 190     def add_drive(self, drive_file, drive_format):
 191         """Set drive with custom format.
 192
 193         :param drive_file: Path to drive image.
 194         :param drive_format: Drive image format.
 195         :type drive_file: str
 196         :type drive_format: str
 197         """
 198         self._params.add_with_value(
 199             u"drive", f"file={drive_file},format={drive_format},"
 200             u"cache=none,if=virtio,file.locking=off"
 201         )
 202
 203     def add_kernelvm_params(self):
 204         """Set KernelVM QEMU parameters."""
 205         self._params.add_with_value(
 206             u"serial", f"file:{self._temp.get(u'log')}"
 207         )
 208         self._params.add_with_value(
 209             u"fsdev", u"local,id=root9p,path=/,security_model=none"
 210         )
 211         self._params.add_with_value(
 212             u"device", u"virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot"
 213         )
 214         self._params.add_with_value(
 215             u"kernel", f"{self._opt.get(u'img')}"
 216         )
 217         self._params.add_with_value(
 218             u"initrd", f"{self._opt.get(u'initrd')}"
 219         )
 220         self._params.add_with_value(
 221             u"append", f"'ro rootfstype=9p rootflags=trans=virtio "
 222             f"root=virtioroot console={self._opt.get(u'console')} "
 223             f"tsc=reliable hugepages=512 "
 224             f"init={self._temp.get(u'ini')} fastboot'"
 225         )
 226
 227     def add_vhost_user_if(
 228             self, socket, server=True, jumbo_frames=False, queue_size=None,
 229             queues=1, virtio_feature_mask=None):
 230         """Add Vhost-user interface.
 231
 232         :param socket: Path of the unix socket.
 233         :param server: If True the socket shall be a listening socket.
 234         :param jumbo_frames: Set True if jumbo frames are used in the test.
 235         :param queue_size: Vring queue size.
 236         :param queues: Number of queues.
 237         :param virtio_feature_mask: Mask of virtio features to be enabled.
 238         :type socket: str
 239         :type server: bool
 240         :type jumbo_frames: bool
 241         :type queue_size: int
 242         :type queues: int
 243         :type virtio_feature_mask: int
 244         """
 245         self._nic_id += 1
 246         if jumbo_frames:
 247             logger.debug(u"Jumbo frames temporarily disabled!")
 248         self._params.add_with_value(
 249             u"chardev", f"socket,id=char{self._nic_id},"
 250             f"path={socket}{u',server' if server is True else u''}"
 251         )
 252         self._params.add_with_value(
 253             u"netdev", f"vhost-user,id=vhost{self._nic_id},"
 254             f"chardev=char{self._nic_id},queues={queues}"
 255         )
 256         mac = f"52:54:00:00:{self._opt.get(u'qemu_id'):02x}:" \
 257             f"{self._nic_id:02x}"
 258         queue_size = f"rx_queue_size={queue_size},tx_queue_size={queue_size}" \
 259             if queue_size else u""
 260         gso = VirtioFeatureMask.is_feature_enabled(
 261             virtio_feature_mask, VirtioFeaturesFlags.VIRTIO_NET_F_API_GSO)
 262         csum = VirtioFeatureMask.is_feature_enabled(
 263             virtio_feature_mask, VirtioFeaturesFlags.VIRTIO_NET_F_API_CSUM)
 264
 265         self._params.add_with_value(
 266             u"device", f"virtio-net-pci,netdev=vhost{self._nic_id},mac={mac},"
 267             f"addr={self._nic_id+5}.0,mq=on,vectors={2 * queues + 2},"
 268             f"csum={u'on' if csum else u'off'},"
 269             f"gso={u'on' if gso else u'off'},"
 270             f"guest_tso4={u'on' if gso else u'off'},"
 271             f"guest_tso6={u'on' if gso else u'off'},"
 272             f"guest_ecn={u'on' if gso else u'off'},"
 273             f"{queue_size}"
 274         )
 275
 276         # Add interface MAC and socket to the node dict.
 277         if_data = {u"mac_address": mac, u"socket": socket}
 278         if_name = f"vhost{self._nic_id}"
 279         self._vm_info[u"interfaces"][if_name] = if_data
 280         # Add socket to temporary file list.
 281         self._temp[if_name] = socket
 282
 283     def add_vfio_pci_if(self, pci):
 284         """Add VFIO PCI interface.
 285
 286         :param pci: PCI address of interface.
 287         :type pci: str
 288         """
 289         self._nic_id += 1
 290         self._params.add_with_value(
 291             u"device", f"vfio-pci,host={pci},addr={self._nic_id+5}.0"
 292         )
 293
 294     def create_kernelvm_config_vpp(self, **kwargs):
 295         """Create QEMU VPP config files.
 296
 297         :param kwargs: Key-value pairs to replace content of VPP configuration
 298             file.
 299         :type kwargs: dict
 300         """
 301         startup = f"/etc/vpp/vm_startup_{self._opt.get(u'qemu_id')}.conf"
 302         running = f"/etc/vpp/vm_running_{self._opt.get(u'qemu_id')}.exec"
 303
 304         self._temp[u"startup"] = startup
 305         self._temp[u"running"] = running
 306         self._opt[u"vnf_bin"] = f"/usr/bin/vpp -c {startup}"
 307
 308         # Create VPP startup configuration.
 309         vpp_config = VppConfigGenerator()
 310         vpp_config.set_node(self._node)
 311         vpp_config.add_unix_nodaemon()
 312         vpp_config.add_unix_cli_listen()
 313         vpp_config.add_unix_exec(running)
 314         vpp_config.add_socksvr()
 315         vpp_config.add_main_heap_size(u"512M")
 316         vpp_config.add_main_heap_page_size(u"2M")
 317         vpp_config.add_statseg_size(u"512M")
 318         vpp_config.add_statseg_page_size(u"2M")
 319         vpp_config.add_statseg_per_node_counters(u"on")
 320         vpp_config.add_buffers_per_numa(107520)
 321         vpp_config.add_cpu_main_core(u"0")
 322         if self._opt.get(u"smp") > 1:
 323             vpp_config.add_cpu_corelist_workers(f"1-{self._opt.get(u'smp')-1}")
 324         vpp_config.add_plugin(u"disable", u"default")
 325         vpp_config.add_plugin(u"enable", u"ping_plugin.so")
 326         if "2vfpt" in self._opt.get(u'vnf'):
 327             vpp_config.add_plugin(u"enable", u"avf_plugin.so")
 328         if "vhost" in self._opt.get(u'vnf'):
 329             vpp_config.add_plugin(u"enable", u"dpdk_plugin.so")
 330             vpp_config.add_dpdk_dev(u"0000:00:06.0", u"0000:00:07.0")
 331             vpp_config.add_dpdk_dev_default_rxq(kwargs[u"queues"])
 332             vpp_config.add_dpdk_log_level(u"debug")
 333             if not kwargs[u"jumbo_frames"]:
 334                 vpp_config.add_dpdk_no_multi_seg()
 335                 vpp_config.add_dpdk_no_tx_checksum_offload()
 336         if "ipsec" in self._opt.get(u'vnf'):
 337             vpp_config.add_plugin(u"enable", u"crypto_native_plugin.so")
 338             vpp_config.add_plugin(u"enable", u"crypto_ipsecmb_plugin.so")
 339             vpp_config.add_plugin(u"enable", u"crypto_openssl_plugin.so")
 340         if "nat" in self._opt.get(u'vnf'):
 341             vpp_config.add_nat(value=u"endpoint-dependent")
 342             vpp_config.add_plugin(u"enable", u"nat_plugin.so")
 343         vpp_config.write_config(startup)
 344
 345         # Create VPP running configuration.
 346         template = f"{Constants.RESOURCES_TPL}/vm/{self._opt.get(u'vnf')}.exec"
 347         exec_cmd_no_error(self._node, f"rm -f {running}", sudo=True)
 348
 349         with open(template, u"rt") as src_file:
 350             src = Template(src_file.read())
 351             exec_cmd_no_error(
 352                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 353                 f"sudo tee {running}"
 354             )
 355
 356     def create_kernelvm_config_testpmd_io(self, **kwargs):
 357         """Create QEMU testpmd-io command line.
 358
 359         :param kwargs: Key-value pairs to construct command line parameters.
 360         :type kwargs: dict
 361         """
 362         pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
 363         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 364             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 365             eal_driver=False,
 366             eal_pci_whitelist0=u"0000:00:06.0",
 367             eal_pci_whitelist1=u"0000:00:07.0",
 368             eal_in_memory=True,
 369             pmd_num_mbufs=32768,
 370             pmd_fwd_mode=u"io",
 371             pmd_nb_ports=u"2",
 372             pmd_portmask=u"0x3",
 373             pmd_max_pkt_len=pmd_max_pkt_len,
 374             pmd_mbuf_size=u"16384",
 375             pmd_rxq=kwargs[u"queues"],
 376             pmd_txq=kwargs[u"queues"],
 377             pmd_tx_offloads='0x0',
 378             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 379         )
 380
 381         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 382
 383     def create_kernelvm_config_testpmd_mac(self, **kwargs):
 384         """Create QEMU testpmd-mac command line.
 385
 386         :param kwargs: Key-value pairs to construct command line parameters.
 387         :type kwargs: dict
 388         """
 389         pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
 390         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 391             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 392             eal_driver=False,
 393             eal_pci_whitelist0=u"0000:00:06.0",
 394             eal_pci_whitelist1=u"0000:00:07.0",
 395             eal_in_memory=True,
 396             pmd_num_mbufs=32768,
 397             pmd_fwd_mode=u"mac",
 398             pmd_nb_ports=u"2",
 399             pmd_portmask=u"0x3",
 400             pmd_max_pkt_len=pmd_max_pkt_len,
 401             pmd_mbuf_size=u"16384",
 402             pmd_eth_peer_0=f"0,{kwargs[u'vif1_mac']}",
 403             pmd_eth_peer_1=f"1,{kwargs[u'vif2_mac']}",
 404             pmd_rxq=kwargs[u"queues"],
 405             pmd_txq=kwargs[u"queues"],
 406             pmd_tx_offloads=u"0x0",
 407             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 408         )
 409
 410         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 411
 412     def create_kernelvm_config_iperf3(self):
 413         """Create QEMU iperf3 command line."""
 414         self._opt[u"vnf_bin"] = f"mkdir /run/sshd; /usr/sbin/sshd -D -d"
 415
 416     def create_kernelvm_init(self, **kwargs):
 417         """Create QEMU init script.
 418
 419         :param kwargs: Key-value pairs to replace content of init startup file.
 420         :type kwargs: dict
 421         """
 422         init = self._temp.get(u"ini")
 423         exec_cmd_no_error(self._node, f"rm -f {init}", sudo=True)
 424
 425         with open(kwargs[u"template"], u"rt") as src_file:
 426             src = Template(src_file.read())
 427             exec_cmd_no_error(
 428                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 429                 f"sudo tee {init}"
 430             )
 431             exec_cmd_no_error(self._node, f"chmod +x {init}", sudo=True)
 432
 433     def configure_kernelvm_vnf(self, **kwargs):
 434         """Create KernelVM VNF configurations.
 435
 436         :param kwargs: Key-value pairs for templating configs.
 437         :type kwargs: dict
 438         """
 439         if u"vpp" in self._opt.get(u"vnf"):
 440             self.create_kernelvm_config_vpp(**kwargs)
 441             self.create_kernelvm_init(
 442                 template=f"{Constants.RESOURCES_TPL}/vm/init.sh",
 443                 vnf_bin=self._opt.get(u"vnf_bin")
 444             )
 445         elif u"testpmd_io" in self._opt.get(u"vnf"):
 446             self.create_kernelvm_config_testpmd_io(**kwargs)
 447             self.create_kernelvm_init(
 448                 template=f"{Constants.RESOURCES_TPL}/vm/init.sh",
 449                 vnf_bin=self._opt.get(u"vnf_bin")
 450             )
 451         elif u"testpmd_mac" in self._opt.get(u"vnf"):
 452             self.create_kernelvm_config_testpmd_mac(**kwargs)
 453             self.create_kernelvm_init(
 454                 template=f"{Constants.RESOURCES_TPL}/vm/init.sh",
 455                 vnf_bin=self._opt.get(u"vnf_bin")
 456             )
 457         elif u"iperf3" in self._opt.get(u"vnf"):
 458             qemu_id = self._opt.get(u'qemu_id') % 2
 459             self.create_kernelvm_config_iperf3()
 460             self.create_kernelvm_init(
 461                 template=f"{Constants.RESOURCES_TPL}/vm/init_iperf3.sh",
 462                 vnf_bin=self._opt.get(u"vnf_bin"),
 463                 ip_address_l=u"2.2.2.2/30" if qemu_id else u"1.1.1.1/30",
 464                 ip_address_r=u"2.2.2.1" if qemu_id else u"1.1.1.2",
 465                 ip_route_r=u"1.1.1.0/30" if qemu_id else u"2.2.2.0/30"
 466             )
 467         else:
 468             raise RuntimeError(u"QEMU: Unsupported VNF!")
 469
 470     def get_qemu_pids(self):
 471         """Get QEMU CPU pids.
 472
 473         :returns: List of QEMU CPU pids.
 474         :rtype: list of str
 475         """
 476         command = f"grep -rwl 'CPU' /proc/$(sudo cat " \
 477             f"{self._temp.get(u'pidfile')})/task/*/comm "
 478         command += r"| xargs dirname | sed -e 's/\/.*\///g' | uniq"
 479
 480         stdout, _ = exec_cmd_no_error(self._node, command)
 481         return stdout.splitlines()
 482
 483     def qemu_set_affinity(self, *host_cpus):
 484         """Set qemu affinity by getting thread PIDs via QMP and taskset to list
 485         of CPU cores. Function tries to execute 3 times to avoid race condition
 486         in getting thread PIDs.
 487
 488         :param host_cpus: List of CPU cores.
 489         :type host_cpus: list
 490         """
 491         for _ in range(3):
 492             try:
 493                 qemu_cpus = self.get_qemu_pids()
 494
 495                 if len(qemu_cpus) != len(host_cpus):
 496                     sleep(1)
 497                     continue
 498                 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
 499                     command = f"taskset -pc {host_cpu} {qemu_cpu}"
 500                     message = f"QEMU: Set affinity failed " \
 501                         f"on {self._node[u'host']}!"
 502                     exec_cmd_no_error(
 503                         self._node, command, sudo=True, message=message
 504                     )
 505                 break
 506             except (RuntimeError, ValueError):
 507                 self.qemu_kill_all()
 508                 raise
 509         else:
 510             self.qemu_kill_all()
 511             raise RuntimeError(u"Failed to set Qemu threads affinity!")
 512
 513     def qemu_set_scheduler_policy(self):
 514         """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
 515         processes.
 516
 517         :raises RuntimeError: Set scheduler policy failed.
 518         """
 519         try:
 520             qemu_cpus = self.get_qemu_pids()
 521
 522             for qemu_cpu in qemu_cpus:
 523                 command = f"chrt -r -p 1 {qemu_cpu}"
 524                 message = f"QEMU: Set SCHED_RR failed on {self._node[u'host']}"
 525                 exec_cmd_no_error(
 526                     self._node, command, sudo=True, message=message
 527                 )
 528         except (RuntimeError, ValueError):
 529             self.qemu_kill_all()
 530             raise
 531
 532     def _qemu_qmp_exec(self, cmd):
 533         """Execute QMP command.
 534
 535         QMP is JSON based protocol which allows to control QEMU instance.
 536
 537         :param cmd: QMP command to execute.
 538         :type cmd: str
 539         :returns: Command output in python representation of JSON format. The
 540             { "return": {} } response is QMP's success response. An error
 541             response will contain the "error" keyword instead of "return".
 542         """
 543         # To enter command mode, the qmp_capabilities command must be issued.
 544         command = f"echo \"{{{{ \\\"execute\\\": " \
 545             f"\\\"qmp_capabilities\\\" }}}}" \
 546             f"{{{{ \\\"execute\\\": \\\"{cmd}\\\" }}}}\" | " \
 547             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qmp')}"
 548         message = f"QMP execute '{cmd}' failed on {self._node[u'host']}"
 549
 550         stdout, _ = exec_cmd_no_error(
 551             self._node, command, sudo=False, message=message
 552         )
 553
 554         # Skip capabilities negotiation messages.
 555         out_list = stdout.splitlines()
 556         if len(out_list) < 3:
 557             raise RuntimeError(f"Invalid QMP output on {self._node[u'host']}")
 558         return json.loads(out_list[2])
 559
 560     def _qemu_qga_flush(self):
 561         """Flush the QGA parser state."""
 562         command = f"(printf \"\xFF\"; sleep 1) | sudo -S socat " \
 563             f"- UNIX-CONNECT:{self._temp.get(u'qga')}"
 564         message = f"QGA flush failed on {self._node[u'host']}"
 565         stdout, _ = exec_cmd_no_error(
 566             self._node, command, sudo=False, message=message
 567         )
 568
 569         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 570
 571     def _qemu_qga_exec(self, cmd):
 572         """Execute QGA command.
 573
 574         QGA provide access to a system-level agent via standard QMP commands.
 575
 576         :param cmd: QGA command to execute.
 577         :type cmd: str
 578         """
 579         command = f"(echo \"{{{{ \\\"execute\\\": " \
 580             f"\\\"{cmd}\\\" }}}}\"; sleep 1) | " \
 581             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qga')}"
 582         message = f"QGA execute '{cmd}' failed on {self._node[u'host']}"
 583         stdout, _ = exec_cmd_no_error(
 584             self._node, command, sudo=False, message=message
 585         )
 586
 587         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 588
 589     def _wait_until_vm_boot(self):
 590         """Wait until QEMU VM is booted."""
 591         try:
 592             getattr(self, f'_wait_{self._opt["vnf"]}')()
 593         except AttributeError:
 594             self._wait_default()
 595
 596     def _wait_default(self, retries=60):
 597         """Wait until QEMU with VPP is booted.
 598
 599         :param retries: Number of retries.
 600         :type retries: int
 601         """
 602         for _ in range(retries):
 603             command = f"tail -1 {self._temp.get(u'log')}"
 604             stdout = None
 605             try:
 606                 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 607                 sleep(1)
 608             except RuntimeError:
 609                 pass
 610             if "vpp " in stdout and "built by" in stdout:
 611                 break
 612             if u"Press enter to exit" in stdout:
 613                 break
 614             if u"reboot: Power down" in stdout:
 615                 raise RuntimeError(
 616                     f"QEMU: NF failed to run on {self._node[u'host']}!"
 617                 )
 618         else:
 619             raise RuntimeError(
 620                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 621             )
 622
 623     def _wait_nestedvm(self, retries=12):
 624         """Wait until QEMU with NestedVM is booted.
 625
 626         First try to flush qga until there is output.
 627         Then ping QEMU guest agent each 5s until VM booted or timeout.
 628
 629         :param retries: Number of retries with 5s between trials.
 630         :type retries: int
 631         """
 632         for _ in range(retries):
 633             out = None
 634             try:
 635                 out = self._qemu_qga_flush()
 636             except ValueError:
 637                 logger.trace(f"QGA qga flush unexpected output {out}")
 638             # Empty output - VM not booted yet
 639             if not out:
 640                 sleep(5)
 641             else:
 642                 break
 643         else:
 644             raise RuntimeError(
 645                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 646             )
 647         for _ in range(retries):
 648             out = None
 649             try:
 650                 out = self._qemu_qga_exec(u"guest-ping")
 651             except ValueError:
 652                 logger.trace(f"QGA guest-ping unexpected output {out}")
 653             # Empty output - VM not booted yet.
 654             if not out:
 655                 sleep(5)
 656             # Non-error return - VM booted.
 657             elif out.get(u"return") is not None:
 658                 break
 659             # Skip error and wait.
 660             elif out.get(u"error") is not None:
 661                 sleep(5)
 662             else:
 663                 # If there is an unexpected output from QGA guest-info, try
 664                 # again until timeout.
 665                 logger.trace(f"QGA guest-ping unexpected output {out}")
 666         else:
 667             raise RuntimeError(
 668                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 669             )
 670
 671     def _wait_iperf3(self, retries=60):
 672         """Wait until QEMU with iPerf3 is booted.
 673
 674         :param retries: Number of retries.
 675         :type retries: int
 676         """
 677         grep = u"Server listening on 0.0.0.0 port 22."
 678         cmd = f"fgrep '{grep}' {self._temp.get(u'log')}"
 679         message = f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 680         exec_cmd_no_error(
 681             self._node, cmd=cmd, sudo=True, message=message, retries=retries,
 682             include_reason=True
 683         )
 684
 685     def _update_vm_interfaces(self):
 686         """Update interface names in VM node dict."""
 687         # Send guest-network-get-interfaces command via QGA, output example:
 688         # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
 689         # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
 690         out = self._qemu_qga_exec(u"guest-network-get-interfaces")
 691         interfaces = out.get(u"return")
 692         mac_name = {}
 693         if not interfaces:
 694             raise RuntimeError(
 695                 f"Get VM interface list failed on {self._node[u'host']}"
 696             )
 697         # Create MAC-name dict.
 698         for interface in interfaces:
 699             if u"hardware-address" not in interface:
 700                 continue
 701             mac_name[interface[u"hardware-address"]] = interface[u"name"]
 702         # Match interface by MAC and save interface name.
 703         for interface in self._vm_info[u"interfaces"].values():
 704             mac = interface.get(u"mac_address")
 705             if_name = mac_name.get(mac)
 706             if if_name is None:
 707                 logger.trace(f"Interface name for MAC {mac} not found")
 708             else:
 709                 interface[u"name"] = if_name
 710
 711     def qemu_start(self):
 712         """Start QEMU and wait until VM boot.
 713
 714         :returns: VM node info.
 715         :rtype: dict
 716         """
 717         cmd_opts = OptionString()
 718         cmd_opts.add(f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch}")
 719         cmd_opts.extend(self._params)
 720         message = f"QEMU: Start failed on {self._node[u'host']}!"
 721         try:
 722             DUTSetup.check_huge_page(
 723                 self._node, u"/dev/hugepages", int(self._opt.get(u"mem")))
 724
 725             exec_cmd_no_error(
 726                 self._node, cmd_opts, timeout=300, sudo=True, message=message
 727             )
 728             self._wait_until_vm_boot()
 729         except RuntimeError:
 730             self.qemu_kill_all()
 731             raise
 732         return self._vm_info
 733
 734     def qemu_kill(self):
 735         """Kill qemu process."""
 736         exec_cmd(
 737             self._node, f"chmod +r {self._temp.get(u'pidfile')}", sudo=True
 738         )
 739         exec_cmd(
 740             self._node, f"kill -SIGKILL $(cat {self._temp.get(u'pidfile')})",
 741             sudo=True
 742         )
 743
 744         for value in self._temp.values():
 745             exec_cmd(self._node, f"cat {value}", sudo=True)
 746             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 747
 748     def qemu_kill_all(self):
 749         """Kill all qemu processes on DUT node if specified."""
 750         exec_cmd(self._node, u"pkill -SIGKILL qemu", sudo=True)
 751
 752         for value in self._temp.values():
 753             exec_cmd(self._node, f"cat {value}", sudo=True)
 754             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 755
 756     def qemu_version(self):
 757         """Return Qemu version.
 758
 759         :returns: Qemu version.
 760         :rtype: str
 761         """
 762         command = f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch} " \
 763             f"--version"
 764         try:
 765             stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 766             return match(r"QEMU emulator version ([\d.]*)", stdout).group(1)
 767         except RuntimeError:
 768             self.qemu_kill_all()
 769             raise