resources/libraries/python/QemuUtils.py

   1 # Copyright (c) 2021 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """QEMU utilities library."""
  15
  16 import json
  17
  18 from re import match
  19 from string import Template
  20 from time import sleep
  21
  22 from robot.api import logger
  23
  24 from resources.libraries.python.Constants import Constants
  25 from resources.libraries.python.DpdkUtil import DpdkUtil
  26 from resources.libraries.python.DUTSetup import DUTSetup
  27 from resources.libraries.python.OptionString import OptionString
  28 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
  29 from resources.libraries.python.topology import NodeType, Topology
  30 from resources.libraries.python.VhostUser import VirtioFeaturesFlags
  31 from resources.libraries.python.VhostUser import VirtioFeatureMask
  32 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
  33
  34 __all__ = [u"QemuUtils"]
  35
  36
  37 class QemuUtils:
  38     """QEMU utilities."""
  39
  40     # Use one instance of class per tests.
  41     ROBOT_LIBRARY_SCOPE = u"TEST CASE"
  42
  43     def __init__(
  44             self, node, qemu_id=1, smp=1, mem=512, vnf=None,
  45             img=Constants.QEMU_VM_IMAGE):
  46         """Initialize QemuUtil class.
  47
  48         :param node: Node to run QEMU on.
  49         :param qemu_id: QEMU identifier.
  50         :param smp: Number of virtual SMP units (cores).
  51         :param mem: Amount of memory.
  52         :param vnf: Network function workload.
  53         :param img: QEMU disk image or kernel image path.
  54         :type node: dict
  55         :type qemu_id: int
  56         :type smp: int
  57         :type mem: int
  58         :type vnf: str
  59         :type img: str
  60         """
  61         self._nic_id = 0
  62         self._node = node
  63         self._arch = Topology.get_node_arch(self._node)
  64         self._opt = dict()
  65
  66         # Architecture specific options
  67         if self._arch == u"aarch64":
  68             dpdk_target = u"arm64-armv8a"
  69             self._opt[u"machine_args"] = \
  70                 u"virt,accel=kvm,usb=off,mem-merge=off,gic-version=3"
  71             self._opt[u"console"] = u"ttyAMA0"
  72         else:
  73             dpdk_target = u"x86_64-native"
  74             self._opt[u"machine_args"] = u"pc,accel=kvm,usb=off,mem-merge=off"
  75             self._opt[u"console"] = u"ttyS0"
  76         self._testpmd_path = f"{Constants.QEMU_VM_DPDK}/" \
  77             f"{dpdk_target}-linux-gcc/app"
  78         self._vm_info = {
  79             u"host": node[u"host"],
  80             u"type": NodeType.VM,
  81             u"port": 10021 + qemu_id,
  82             u"serial": 4555 + qemu_id,
  83             u"username": 'testuser',
  84             u"password": 'Csit1234',
  85             u"interfaces": {},
  86         }
  87         if node[u"port"] != 22:
  88             self._vm_info[u"host_port"] = node[u"port"]
  89             self._vm_info[u"host_username"] = node[u"username"]
  90             self._vm_info[u"host_password"] = node[u"password"]
  91         # Input Options.
  92         self._opt[u"qemu_id"] = qemu_id
  93         self._opt[u"mem"] = int(mem)
  94         self._opt[u"smp"] = int(smp)
  95         self._opt[u"img"] = img
  96         self._opt[u"vnf"] = vnf
  97         # Temporary files.
  98         self._temp = dict()
  99         self._temp[u"log"] = f"/tmp/serial_{qemu_id}.log"
 100         self._temp[u"pidfile"] = f"/run/qemu_{qemu_id}.pid"
 101         if img == Constants.QEMU_VM_IMAGE:
 102             self._temp[u"qmp"] = f"/run/qmp_{qemu_id}.sock"
 103             self._temp[u"qga"] = f"/run/qga_{qemu_id}.sock"
 104         elif img == Constants.QEMU_VM_KERNEL:
 105             self._opt[u"img"], _ = exec_cmd_no_error(
 106                 node, f"ls -1 {Constants.QEMU_VM_KERNEL}* | tail -1",
 107                 message=u"Qemu Kernel VM image not found!"
 108             )
 109             self._temp[u"ini"] = f"/etc/vm_init_{qemu_id}.conf"
 110             self._opt[u"initrd"], _ = exec_cmd_no_error(
 111                 node, f"ls -1 {Constants.QEMU_VM_KERNEL_INITRD}* | tail -1",
 112                 message=u"Qemu Kernel initrd image not found!"
 113             )
 114         else:
 115             raise RuntimeError(f"QEMU: Unknown VM image option: {img}")
 116         # Computed parameters for QEMU command line.
 117         self._params = OptionString(prefix=u"-")
 118
 119     def add_default_params(self):
 120         """Set default QEMU command line parameters."""
 121         self._params.add(u"daemonize")
 122         self._params.add(u"nodefaults")
 123         self._params.add_with_value(
 124             u"name", f"vnf{self._opt.get(u'qemu_id')},debug-threads=on"
 125         )
 126         self._params.add(u"no-user-config")
 127         self._params.add(u"nographic")
 128         self._params.add(u"enable-kvm")
 129         self._params.add_with_value(u"pidfile", self._temp.get(u"pidfile"))
 130         self._params.add_with_value(u"cpu", u"host")
 131
 132         self._params.add_with_value(u"machine", self._opt.get(u"machine_args"))
 133         self._params.add_with_value(
 134             u"smp", f"{self._opt.get(u'smp')},sockets=1,"
 135             f"cores={self._opt.get(u'smp')},threads=1"
 136         )
 137         self._params.add_with_value(
 138             u"object", f"memory-backend-file,id=mem,"
 139             f"size={self._opt.get(u'mem')}M,mem-path=/dev/hugepages,share=on"
 140         )
 141         self._params.add_with_value(u"m", f"{self._opt.get(u'mem')}M")
 142         self._params.add_with_value(u"numa", u"node,memdev=mem")
 143
 144     def add_net_user(self, net="10.0.2.0/24"):
 145         """Set managment port forwarding."""
 146         self._params.add_with_value(
 147             u"netdev", f"user,id=mgmt,net={net},"
 148             f"hostfwd=tcp::{self._vm_info[u'port']}-:22"
 149         )
 150         self._params.add_with_value(
 151             u"device", f"virtio-net,netdev=mgmt"
 152         )
 153
 154     def add_qmp_qga(self):
 155         """Set QMP, QGA management."""
 156         self._params.add_with_value(
 157             u"chardev", f"socket,path={self._temp.get(u'qga')},"
 158             f"server,nowait,id=qga0"
 159         )
 160         self._params.add_with_value(
 161             u"device", u"isa-serial,chardev=qga0"
 162         )
 163         self._params.add_with_value(
 164             u"qmp", f"unix:{self._temp.get(u'qmp')},server,nowait"
 165         )
 166
 167     def add_serial(self):
 168         """Set serial to file redirect."""
 169         self._params.add_with_value(
 170             u"chardev", f"socket,host=127.0.0.1,"
 171             f"port={self._vm_info[u'serial']},id=gnc0,server,nowait"
 172         )
 173         self._params.add_with_value(
 174             u"device", u"isa-serial,chardev=gnc0"
 175         )
 176         self._params.add_with_value(
 177             u"serial", f"file:{self._temp.get(u'log')}"
 178         )
 179
 180     def add_drive_cdrom(self, drive_file, index=None):
 181         """Set CD-ROM drive.
 182
 183         :param drive_file: Path to drive image.
 184         :param index: Drive index.
 185         :type drive_file: str
 186         :type index: int
 187         """
 188         index = f"index={index}," if index else u""
 189         self._params.add_with_value(
 190             u"drive", f"file={drive_file},{index}media=cdrom"
 191         )
 192
 193     def add_drive(self, drive_file, drive_format):
 194         """Set drive with custom format.
 195
 196         :param drive_file: Path to drive image.
 197         :param drive_format: Drive image format.
 198         :type drive_file: str
 199         :type drive_format: str
 200         """
 201         self._params.add_with_value(
 202             u"drive", f"file={drive_file},format={drive_format},"
 203             u"cache=none,if=virtio,file.locking=off"
 204         )
 205
 206     def add_kernelvm_params(self):
 207         """Set KernelVM QEMU parameters."""
 208         self._params.add_with_value(
 209             u"serial", f"file:{self._temp.get(u'log')}"
 210         )
 211         self._params.add_with_value(
 212             u"fsdev", u"local,id=root9p,path=/,security_model=none"
 213         )
 214         self._params.add_with_value(
 215             u"device", u"virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot"
 216         )
 217         self._params.add_with_value(
 218             u"kernel", f"{self._opt.get(u'img')}"
 219         )
 220         self._params.add_with_value(
 221             u"initrd", f"{self._opt.get(u'initrd')}"
 222         )
 223         self._params.add_with_value(
 224             u"append", f"'ro rootfstype=9p rootflags=trans=virtio "
 225             f"root=virtioroot console={self._opt.get(u'console')} "
 226             f"tsc=reliable hugepages=512 "
 227             f"init={self._temp.get(u'ini')} fastboot'"
 228         )
 229
 230     def add_vhost_user_if(
 231             self, socket, server=True, jumbo_frames=False, queue_size=None,
 232             queues=1, virtio_feature_mask=None):
 233         """Add Vhost-user interface.
 234
 235         :param socket: Path of the unix socket.
 236         :param server: If True the socket shall be a listening socket.
 237         :param jumbo_frames: Set True if jumbo frames are used in the test.
 238         :param queue_size: Vring queue size.
 239         :param queues: Number of queues.
 240         :param virtio_feature_mask: Mask of virtio features to be enabled.
 241         :type socket: str
 242         :type server: bool
 243         :type jumbo_frames: bool
 244         :type queue_size: int
 245         :type queues: int
 246         :type virtio_feature_mask: int
 247         """
 248         self._nic_id += 1
 249         self._params.add_with_value(
 250             u"chardev", f"socket,id=char{self._nic_id},"
 251             f"path={socket}{u',server' if server is True else u''}"
 252         )
 253         self._params.add_with_value(
 254             u"netdev", f"vhost-user,id=vhost{self._nic_id},"
 255             f"chardev=char{self._nic_id},queues={queues}"
 256         )
 257         mac = f"52:54:00:00:{self._opt.get(u'qemu_id'):02x}:" \
 258             f"{self._nic_id:02x}"
 259         queue_size = f"rx_queue_size={queue_size},tx_queue_size={queue_size}" \
 260             if queue_size else u""
 261         gso = VirtioFeatureMask.is_feature_enabled(
 262             virtio_feature_mask, VirtioFeaturesFlags.VIRTIO_NET_F_API_GSO)
 263         csum = VirtioFeatureMask.is_feature_enabled(
 264             virtio_feature_mask, VirtioFeaturesFlags.VIRTIO_NET_F_API_CSUM)
 265
 266         self._params.add_with_value(
 267             u"device", f"virtio-net-pci,netdev=vhost{self._nic_id},mac={mac},"
 268             f"addr={self._nic_id+5}.0,mq=on,vectors={2 * queues + 2},"
 269             f"csum={u'on' if csum else u'off'},"
 270             f"gso={u'on' if gso else u'off'},"
 271             f"guest_tso4={u'on' if gso else u'off'},"
 272             f"guest_tso6={u'on' if gso else u'off'},"
 273             f"guest_ecn={u'on' if gso else u'off'},"
 274             f"{queue_size}"
 275         )
 276
 277         # Add interface MAC and socket to the node dict.
 278         if_data = {u"mac_address": mac, u"socket": socket}
 279         if_name = f"vhost{self._nic_id}"
 280         self._vm_info[u"interfaces"][if_name] = if_data
 281         # Add socket to temporary file list.
 282         self._temp[if_name] = socket
 283
 284     def add_vfio_pci_if(self, pci):
 285         """Add VFIO PCI interface.
 286
 287         :param pci: PCI address of interface.
 288         :type pci: str
 289         """
 290         self._nic_id += 1
 291         self._params.add_with_value(
 292             u"device", f"vfio-pci,host={pci},addr={self._nic_id+5}.0"
 293         )
 294
 295     def create_kernelvm_config_vpp(self, **kwargs):
 296         """Create QEMU VPP config files.
 297
 298         :param kwargs: Key-value pairs to replace content of VPP configuration
 299             file.
 300         :type kwargs: dict
 301         """
 302         startup = f"/etc/vpp/vm_startup_{self._opt.get(u'qemu_id')}.conf"
 303         running = f"/etc/vpp/vm_running_{self._opt.get(u'qemu_id')}.exec"
 304
 305         self._temp[u"startup"] = startup
 306         self._temp[u"running"] = running
 307         self._opt[u"vnf_bin"] = f"/usr/bin/vpp -c {startup}"
 308
 309         # Create VPP startup configuration.
 310         vpp_config = VppConfigGenerator()
 311         vpp_config.set_node(self._node)
 312         vpp_config.add_unix_nodaemon()
 313         vpp_config.add_unix_cli_listen()
 314         vpp_config.add_unix_exec(running)
 315         vpp_config.add_socksvr()
 316         vpp_config.add_main_heap_size(u"512M")
 317         vpp_config.add_main_heap_page_size(u"2M")
 318         vpp_config.add_statseg_size(u"512M")
 319         vpp_config.add_statseg_page_size(u"2M")
 320         vpp_config.add_statseg_per_node_counters(u"on")
 321         vpp_config.add_buffers_per_numa(107520)
 322         vpp_config.add_cpu_main_core(u"0")
 323         if self._opt.get(u"smp") > 1:
 324             vpp_config.add_cpu_corelist_workers(f"1-{self._opt.get(u'smp')-1}")
 325         vpp_config.add_plugin(u"disable", u"default")
 326         vpp_config.add_plugin(u"enable", u"ping_plugin.so")
 327         if "2vfpt" in self._opt.get(u'vnf'):
 328             vpp_config.add_plugin(u"enable", u"avf_plugin.so")
 329         if "vhost" in self._opt.get(u'vnf'):
 330             vpp_config.add_plugin(u"enable", u"dpdk_plugin.so")
 331             vpp_config.add_dpdk_dev(u"0000:00:06.0", u"0000:00:07.0")
 332             vpp_config.add_dpdk_dev_default_rxq(kwargs[u"queues"])
 333             vpp_config.add_dpdk_log_level(u"debug")
 334             if not kwargs[u"jumbo_frames"]:
 335                 vpp_config.add_dpdk_no_multi_seg()
 336                 vpp_config.add_dpdk_no_tx_checksum_offload()
 337         if "ipsec" in self._opt.get(u'vnf'):
 338             vpp_config.add_plugin(u"enable", u"crypto_native_plugin.so")
 339             vpp_config.add_plugin(u"enable", u"crypto_ipsecmb_plugin.so")
 340             vpp_config.add_plugin(u"enable", u"crypto_openssl_plugin.so")
 341         if "nat" in self._opt.get(u'vnf'):
 342             vpp_config.add_nat(value=u"endpoint-dependent")
 343             vpp_config.add_plugin(u"enable", u"nat_plugin.so")
 344         vpp_config.write_config(startup)
 345
 346         # Create VPP running configuration.
 347         template = f"{Constants.RESOURCES_TPL}/vm/{self._opt.get(u'vnf')}.exec"
 348         exec_cmd_no_error(self._node, f"rm -f {running}", sudo=True)
 349
 350         with open(template, u"rt") as src_file:
 351             src = Template(src_file.read())
 352             exec_cmd_no_error(
 353                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 354                 f"sudo tee {running}"
 355             )
 356
 357     def create_kernelvm_config_testpmd_io(self, **kwargs):
 358         """Create QEMU testpmd-io command line.
 359
 360         :param kwargs: Key-value pairs to construct command line parameters.
 361         :type kwargs: dict
 362         """
 363         pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
 364         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 365             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 366             eal_driver=False,
 367             eal_pci_whitelist0=u"0000:00:06.0",
 368             eal_pci_whitelist1=u"0000:00:07.0",
 369             eal_in_memory=True,
 370             pmd_num_mbufs=16384,
 371             pmd_fwd_mode=u"io",
 372             pmd_nb_ports=u"2",
 373             pmd_portmask=u"0x3",
 374             pmd_max_pkt_len=pmd_max_pkt_len,
 375             pmd_mbuf_size=u"16384",
 376             pmd_rxq=kwargs[u"queues"],
 377             pmd_txq=kwargs[u"queues"],
 378             pmd_tx_offloads='0x0',
 379             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 380         )
 381
 382         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 383
 384     def create_kernelvm_config_testpmd_mac(self, **kwargs):
 385         """Create QEMU testpmd-mac command line.
 386
 387         :param kwargs: Key-value pairs to construct command line parameters.
 388         :type kwargs: dict
 389         """
 390         pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
 391         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 392             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 393             eal_driver=False,
 394             eal_pci_whitelist0=u"0000:00:06.0",
 395             eal_pci_whitelist1=u"0000:00:07.0",
 396             eal_in_memory=True,
 397             pmd_num_mbufs=16384,
 398             pmd_fwd_mode=u"mac",
 399             pmd_nb_ports=u"2",
 400             pmd_portmask=u"0x3",
 401             pmd_max_pkt_len=pmd_max_pkt_len,
 402             pmd_mbuf_size=u"16384",
 403             pmd_eth_peer_0=f"0,{kwargs[u'vif1_mac']}",
 404             pmd_eth_peer_1=f"1,{kwargs[u'vif2_mac']}",
 405             pmd_rxq=kwargs[u"queues"],
 406             pmd_txq=kwargs[u"queues"],
 407             pmd_tx_offloads=u"0x0",
 408             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 409         )
 410
 411         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 412
 413     def create_kernelvm_config_iperf3(self):
 414         """Create QEMU iperf3 command line."""
 415         self._opt[u"vnf_bin"] = f"mkdir /run/sshd; /usr/sbin/sshd -D -d"
 416
 417     def create_kernelvm_init(self, **kwargs):
 418         """Create QEMU init script.
 419
 420         :param kwargs: Key-value pairs to replace content of init startup file.
 421         :type kwargs: dict
 422         """
 423         init = self._temp.get(u"ini")
 424         exec_cmd_no_error(self._node, f"rm -f {init}", sudo=True)
 425
 426         with open(kwargs[u"template"], u"rt") as src_file:
 427             src = Template(src_file.read())
 428             exec_cmd_no_error(
 429                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 430                 f"sudo tee {init}"
 431             )
 432             exec_cmd_no_error(self._node, f"chmod +x {init}", sudo=True)
 433
 434     def configure_kernelvm_vnf(self, **kwargs):
 435         """Create KernelVM VNF configurations.
 436
 437         :param kwargs: Key-value pairs for templating configs.
 438         :type kwargs: dict
 439         """
 440         if u"vpp" in self._opt.get(u"vnf"):
 441             self.create_kernelvm_config_vpp(**kwargs)
 442             self.create_kernelvm_init(
 443                 template=f"{Constants.RESOURCES_TPL}/vm/init.sh",
 444                 vnf_bin=self._opt.get(u"vnf_bin")
 445             )
 446         elif u"testpmd_io" in self._opt.get(u"vnf"):
 447             self.create_kernelvm_config_testpmd_io(**kwargs)
 448             self.create_kernelvm_init(
 449                 template=f"{Constants.RESOURCES_TPL}/vm/init.sh",
 450                 vnf_bin=self._opt.get(u"vnf_bin")
 451             )
 452         elif u"testpmd_mac" in self._opt.get(u"vnf"):
 453             self.create_kernelvm_config_testpmd_mac(**kwargs)
 454             self.create_kernelvm_init(
 455                 template=f"{Constants.RESOURCES_TPL}/vm/init.sh",
 456                 vnf_bin=self._opt.get(u"vnf_bin")
 457             )
 458         elif u"iperf3" in self._opt.get(u"vnf"):
 459             qemu_id = self._opt.get(u'qemu_id') % 2
 460             self.create_kernelvm_config_iperf3()
 461             self.create_kernelvm_init(
 462                 template=f"{Constants.RESOURCES_TPL}/vm/init_iperf3.sh",
 463                 vnf_bin=self._opt.get(u"vnf_bin"),
 464                 ip_address_l=u"2.2.2.2/30" if qemu_id else u"1.1.1.1/30",
 465                 ip_address_r=u"2.2.2.1" if qemu_id else u"1.1.1.2",
 466                 ip_route_r=u"1.1.1.0/30" if qemu_id else u"2.2.2.0/30"
 467             )
 468         else:
 469             raise RuntimeError(u"QEMU: Unsupported VNF!")
 470
 471     def get_qemu_pids(self):
 472         """Get QEMU CPU pids.
 473
 474         :returns: List of QEMU CPU pids.
 475         :rtype: list of str
 476         """
 477         command = f"grep -rwl 'CPU' /proc/$(sudo cat " \
 478             f"{self._temp.get(u'pidfile')})/task/*/comm "
 479         command += r"| xargs dirname | sed -e 's/\/.*\///g' | uniq"
 480
 481         stdout, _ = exec_cmd_no_error(self._node, command)
 482         return stdout.splitlines()
 483
 484     def qemu_set_affinity(self, *host_cpus):
 485         """Set qemu affinity by getting thread PIDs via QMP and taskset to list
 486         of CPU cores. Function tries to execute 3 times to avoid race condition
 487         in getting thread PIDs.
 488
 489         :param host_cpus: List of CPU cores.
 490         :type host_cpus: list
 491         """
 492         for _ in range(3):
 493             try:
 494                 qemu_cpus = self.get_qemu_pids()
 495
 496                 if len(qemu_cpus) != len(host_cpus):
 497                     sleep(1)
 498                     continue
 499                 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
 500                     command = f"taskset -pc {host_cpu} {qemu_cpu}"
 501                     message = f"QEMU: Set affinity failed " \
 502                         f"on {self._node[u'host']}!"
 503                     exec_cmd_no_error(
 504                         self._node, command, sudo=True, message=message
 505                     )
 506                 break
 507             except (RuntimeError, ValueError):
 508                 self.qemu_kill_all()
 509                 raise
 510         else:
 511             self.qemu_kill_all()
 512             raise RuntimeError(u"Failed to set Qemu threads affinity!")
 513
 514     def qemu_set_scheduler_policy(self):
 515         """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
 516         processes.
 517
 518         :raises RuntimeError: Set scheduler policy failed.
 519         """
 520         try:
 521             qemu_cpus = self.get_qemu_pids()
 522
 523             for qemu_cpu in qemu_cpus:
 524                 command = f"chrt -r -p 1 {qemu_cpu}"
 525                 message = f"QEMU: Set SCHED_RR failed on {self._node[u'host']}"
 526                 exec_cmd_no_error(
 527                     self._node, command, sudo=True, message=message
 528                 )
 529         except (RuntimeError, ValueError):
 530             self.qemu_kill_all()
 531             raise
 532
 533     def _qemu_qmp_exec(self, cmd):
 534         """Execute QMP command.
 535
 536         QMP is JSON based protocol which allows to control QEMU instance.
 537
 538         :param cmd: QMP command to execute.
 539         :type cmd: str
 540         :returns: Command output in python representation of JSON format. The
 541             { "return": {} } response is QMP's success response. An error
 542             response will contain the "error" keyword instead of "return".
 543         """
 544         # To enter command mode, the qmp_capabilities command must be issued.
 545         command = f"echo \"{{{{ \\\"execute\\\": " \
 546             f"\\\"qmp_capabilities\\\" }}}}" \
 547             f"{{{{ \\\"execute\\\": \\\"{cmd}\\\" }}}}\" | " \
 548             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qmp')}"
 549         message = f"QMP execute '{cmd}' failed on {self._node[u'host']}"
 550
 551         stdout, _ = exec_cmd_no_error(
 552             self._node, command, sudo=False, message=message
 553         )
 554
 555         # Skip capabilities negotiation messages.
 556         out_list = stdout.splitlines()
 557         if len(out_list) < 3:
 558             raise RuntimeError(f"Invalid QMP output on {self._node[u'host']}")
 559         return json.loads(out_list[2])
 560
 561     def _qemu_qga_flush(self):
 562         """Flush the QGA parser state."""
 563         command = f"(printf \"\xFF\"; sleep 1) | sudo -S socat " \
 564             f"- UNIX-CONNECT:{self._temp.get(u'qga')}"
 565         message = f"QGA flush failed on {self._node[u'host']}"
 566         stdout, _ = exec_cmd_no_error(
 567             self._node, command, sudo=False, message=message
 568         )
 569
 570         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 571
 572     def _qemu_qga_exec(self, cmd):
 573         """Execute QGA command.
 574
 575         QGA provide access to a system-level agent via standard QMP commands.
 576
 577         :param cmd: QGA command to execute.
 578         :type cmd: str
 579         """
 580         command = f"(echo \"{{{{ \\\"execute\\\": " \
 581             f"\\\"{cmd}\\\" }}}}\"; sleep 1) | " \
 582             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qga')}"
 583         message = f"QGA execute '{cmd}' failed on {self._node[u'host']}"
 584         stdout, _ = exec_cmd_no_error(
 585             self._node, command, sudo=False, message=message
 586         )
 587
 588         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 589
 590     def _wait_until_vm_boot(self):
 591         """Wait until QEMU VM is booted."""
 592         try:
 593             getattr(self, f'_wait_{self._opt["vnf"]}')()
 594         except AttributeError:
 595             self._wait_default()
 596
 597     def _wait_default(self, retries=60):
 598         """Wait until QEMU with VPP is booted.
 599
 600         :param retries: Number of retries.
 601         :type retries: int
 602         """
 603         for _ in range(retries):
 604             command = f"tail -1 {self._temp.get(u'log')}"
 605             stdout = None
 606             try:
 607                 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 608                 sleep(1)
 609             except RuntimeError:
 610                 pass
 611             if "vpp " in stdout and "built by" in stdout:
 612                 break
 613             if u"Press enter to exit" in stdout:
 614                 break
 615             if u"reboot: Power down" in stdout:
 616                 raise RuntimeError(
 617                     f"QEMU: NF failed to run on {self._node[u'host']}!"
 618                 )
 619         else:
 620             raise RuntimeError(
 621                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 622             )
 623
 624     def _wait_nestedvm(self, retries=12):
 625         """Wait until QEMU with NestedVM is booted.
 626
 627         First try to flush qga until there is output.
 628         Then ping QEMU guest agent each 5s until VM booted or timeout.
 629
 630         :param retries: Number of retries with 5s between trials.
 631         :type retries: int
 632         """
 633         for _ in range(retries):
 634             out = None
 635             try:
 636                 out = self._qemu_qga_flush()
 637             except ValueError:
 638                 logger.trace(f"QGA qga flush unexpected output {out}")
 639             # Empty output - VM not booted yet
 640             if not out:
 641                 sleep(5)
 642             else:
 643                 break
 644         else:
 645             raise RuntimeError(
 646                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 647             )
 648         for _ in range(retries):
 649             out = None
 650             try:
 651                 out = self._qemu_qga_exec(u"guest-ping")
 652             except ValueError:
 653                 logger.trace(f"QGA guest-ping unexpected output {out}")
 654             # Empty output - VM not booted yet.
 655             if not out:
 656                 sleep(5)
 657             # Non-error return - VM booted.
 658             elif out.get(u"return") is not None:
 659                 break
 660             # Skip error and wait.
 661             elif out.get(u"error") is not None:
 662                 sleep(5)
 663             else:
 664                 # If there is an unexpected output from QGA guest-info, try
 665                 # again until timeout.
 666                 logger.trace(f"QGA guest-ping unexpected output {out}")
 667         else:
 668             raise RuntimeError(
 669                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 670             )
 671
 672     def _wait_iperf3(self, retries=60):
 673         """Wait until QEMU with iPerf3 is booted.
 674
 675         :param retries: Number of retries.
 676         :type retries: int
 677         """
 678         grep = u"Server listening on 0.0.0.0 port 22."
 679         cmd = f"fgrep '{grep}' {self._temp.get(u'log')}"
 680         message = f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 681         exec_cmd_no_error(
 682             self._node, cmd=cmd, sudo=True, message=message, retries=retries,
 683             include_reason=True
 684         )
 685
 686     def _update_vm_interfaces(self):
 687         """Update interface names in VM node dict."""
 688         # Send guest-network-get-interfaces command via QGA, output example:
 689         # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
 690         # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
 691         out = self._qemu_qga_exec(u"guest-network-get-interfaces")
 692         interfaces = out.get(u"return")
 693         mac_name = {}
 694         if not interfaces:
 695             raise RuntimeError(
 696                 f"Get VM interface list failed on {self._node[u'host']}"
 697             )
 698         # Create MAC-name dict.
 699         for interface in interfaces:
 700             if u"hardware-address" not in interface:
 701                 continue
 702             mac_name[interface[u"hardware-address"]] = interface[u"name"]
 703         # Match interface by MAC and save interface name.
 704         for interface in self._vm_info[u"interfaces"].values():
 705             mac = interface.get(u"mac_address")
 706             if_name = mac_name.get(mac)
 707             if if_name is None:
 708                 logger.trace(f"Interface name for MAC {mac} not found")
 709             else:
 710                 interface[u"name"] = if_name
 711
 712     def qemu_start(self):
 713         """Start QEMU and wait until VM boot.
 714
 715         :returns: VM node info.
 716         :rtype: dict
 717         """
 718         cmd_opts = OptionString()
 719         cmd_opts.add(f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch}")
 720         cmd_opts.extend(self._params)
 721         message = f"QEMU: Start failed on {self._node[u'host']}!"
 722         try:
 723             DUTSetup.check_huge_page(
 724                 self._node, u"/dev/hugepages", int(self._opt.get(u"mem")))
 725
 726             exec_cmd_no_error(
 727                 self._node, cmd_opts, timeout=300, sudo=True, message=message
 728             )
 729             self._wait_until_vm_boot()
 730         except RuntimeError:
 731             self.qemu_kill_all()
 732             raise
 733         return self._vm_info
 734
 735     def qemu_kill(self):
 736         """Kill qemu process."""
 737         exec_cmd(
 738             self._node, f"chmod +r {self._temp.get(u'pidfile')}", sudo=True
 739         )
 740         exec_cmd(
 741             self._node, f"kill -SIGKILL $(cat {self._temp.get(u'pidfile')})",
 742             sudo=True
 743         )
 744
 745         for value in self._temp.values():
 746             exec_cmd(self._node, f"cat {value}", sudo=True)
 747             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 748
 749     def qemu_kill_all(self):
 750         """Kill all qemu processes on DUT node if specified."""
 751         exec_cmd(self._node, u"pkill -SIGKILL qemu", sudo=True)
 752
 753         for value in self._temp.values():
 754             exec_cmd(self._node, f"cat {value}", sudo=True)
 755             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 756
 757     def qemu_version(self):
 758         """Return Qemu version.
 759
 760         :returns: Qemu version.
 761         :rtype: str
 762         """
 763         command = f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch} " \
 764             f"--version"
 765         try:
 766             stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 767             return match(r"QEMU emulator version ([\d.]*)", stdout).group(1)
 768         except RuntimeError:
 769             self.qemu_kill_all()
 770             raise