resources/libraries/python/QemuUtils.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """QEMU utilities library."""
  15
  16 import json
  17
  18 from re import match
  19 from string import Template
  20 from time import sleep
  21
  22 from robot.api import logger
  23
  24 from resources.libraries.python.Constants import Constants
  25 from resources.libraries.python.DpdkUtil import DpdkUtil
  26 from resources.libraries.python.DUTSetup import DUTSetup
  27 from resources.libraries.python.OptionString import OptionString
  28 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
  29 from resources.libraries.python.topology import NodeType, Topology
  30 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
  31
  32 __all__ = [u"QemuUtils"]
  33
  34
  35 class QemuUtils:
  36     """QEMU utilities."""
  37
  38     # Use one instance of class per tests.
  39     ROBOT_LIBRARY_SCOPE = u"TEST CASE"
  40
  41     def __init__(
  42             self, node, qemu_id=1, smp=1, mem=512, vnf=None,
  43             img=Constants.QEMU_VM_IMAGE):
  44         """Initialize QemuUtil class.
  45
  46         :param node: Node to run QEMU on.
  47         :param qemu_id: QEMU identifier.
  48         :param smp: Number of virtual SMP units (cores).
  49         :param mem: Amount of memory.
  50         :param vnf: Network function workload.
  51         :param img: QEMU disk image or kernel image path.
  52         :type node: dict
  53         :type qemu_id: int
  54         :type smp: int
  55         :type mem: int
  56         :type vnf: str
  57         :type img: str
  58         """
  59         self._nic_id = 0
  60         self._node = node
  61         self._arch = Topology.get_node_arch(self._node)
  62         self._opt = dict()
  63
  64         # Architecture specific options
  65         if self._arch == u"aarch64":
  66             dpdk_target = u"arm64-armv8a"
  67             self._opt[u"machine_args"] = \
  68                 u"virt,accel=kvm,usb=off,mem-merge=off,gic-version=3"
  69             self._opt[u"console"] = u"ttyAMA0"
  70         else:
  71             dpdk_target = u"x86_64-native"
  72             self._opt[u"machine_args"] = u"pc,accel=kvm,usb=off,mem-merge=off"
  73             self._opt[u"console"] = u"ttyS0"
  74         self._testpmd_path = f"{Constants.QEMU_VM_DPDK}/" \
  75             f"{dpdk_target}-linux-gcc/app"
  76         self._vm_info = {
  77             u"host": node[u"host"],
  78             u"type": NodeType.VM,
  79             u"port": 10021 + qemu_id,
  80             u"serial": 4555 + qemu_id,
  81             u"username": 'cisco',
  82             u"password": 'cisco',
  83             u"interfaces": {},
  84         }
  85         if node[u"port"] != 22:
  86             self._vm_info[u"host_port"] = node[u"port"]
  87             self._vm_info[u"host_username"] = node[u"username"]
  88             self._vm_info[u"host_password"] = node[u"password"]
  89         # Input Options.
  90         self._opt[u"qemu_id"] = qemu_id
  91         self._opt[u"mem"] = int(mem)
  92         self._opt[u"smp"] = int(smp)
  93         self._opt[u"img"] = img
  94         self._opt[u"vnf"] = vnf
  95         # Temporary files.
  96         self._temp = dict()
  97         self._temp[u"log"] = f"/tmp/serial_{qemu_id}.log"
  98         self._temp[u"pidfile"] = f"/run/qemu_{qemu_id}.pid"
  99         if img == Constants.QEMU_VM_IMAGE:
 100             self._temp[u"qmp"] = f"/run/qmp_{qemu_id}.sock"
 101             self._temp[u"qga"] = f"/run/qga_{qemu_id}.sock"
 102         elif img == Constants.QEMU_VM_KERNEL:
 103             self._opt[u"img"], _ = exec_cmd_no_error(
 104                 node, f"ls -1 {Constants.QEMU_VM_KERNEL}* | tail -1",
 105                 message=u"Qemu Kernel VM image not found!"
 106             )
 107             self._temp[u"ini"] = f"/etc/vm_init_{qemu_id}.conf"
 108             self._opt[u"initrd"], _ = exec_cmd_no_error(
 109                 node, f"ls -1 {Constants.QEMU_VM_KERNEL_INITRD}* | tail -1",
 110                 message=u"Qemu Kernel initrd image not found!"
 111             )
 112         else:
 113             raise RuntimeError(f"QEMU: Unknown VM image option: {img}")
 114         # Computed parameters for QEMU command line.
 115         self._params = OptionString(prefix=u"-")
 116
 117     def add_default_params(self):
 118         """Set default QEMU command line parameters."""
 119         self._params.add(u"daemonize")
 120         self._params.add(u"nodefaults")
 121         self._params.add_with_value(
 122             u"name", f"vnf{self._opt.get(u'qemu_id')},debug-threads=on"
 123         )
 124         self._params.add(u"no-user-config")
 125         self._params.add(u"nographic")
 126         self._params.add(u"enable-kvm")
 127         self._params.add_with_value(u"pidfile", self._temp.get(u"pidfile"))
 128         self._params.add_with_value(u"cpu", u"host")
 129
 130         self._params.add_with_value(u"machine", self._opt.get(u"machine_args"))
 131         self._params.add_with_value(
 132             u"smp", f"{self._opt.get(u'smp')},sockets=1,"
 133             f"cores={self._opt.get(u'smp')},threads=1"
 134         )
 135         self._params.add_with_value(
 136             u"object", f"memory-backend-file,id=mem,"
 137             f"size={self._opt.get(u'mem')}M,mem-path=/dev/hugepages,share=on"
 138         )
 139         self._params.add_with_value(u"m", f"{self._opt.get(u'mem')}M")
 140         self._params.add_with_value(u"numa", u"node,memdev=mem")
 141         self._params.add_with_value(u"balloon", u"none")
 142
 143     def add_net_user(self):
 144         """Set managment port forwarding."""
 145         self._params.add_with_value(
 146             u"netdev", f"user,id=mgmt,net=172.16.255.0/24,"
 147             f"hostfwd=tcp::{self._vm_info[u'port']}-:22"
 148         )
 149         self._params.add_with_value(
 150             u"device", f"virtio-net,netdev=mgmt"
 151         )
 152
 153     def add_qmp_qga(self):
 154         """Set QMP, QGA management."""
 155         self._params.add_with_value(
 156             u"chardev", f"socket,path={self._temp.get(u'qga')},"
 157             f"server,nowait,id=qga0"
 158         )
 159         self._params.add_with_value(u"device", u"isa-serial,chardev=qga0")
 160         self._params.add_with_value(
 161             u"qmp", f"unix:{self._temp.get(u'qmp')},server,nowait"
 162         )
 163
 164     def add_serial(self):
 165         """Set serial to file redirect."""
 166         self._params.add_with_value(
 167             u"chardev", f"socket,host=127.0.0.1,"
 168             f"port={self._vm_info[u'serial']},id=gnc0,server,nowait")
 169         self._params.add_with_value(u"device", u"isa-serial,chardev=gnc0")
 170         self._params.add_with_value(
 171             u"serial", f"file:{self._temp.get(u'log')}"
 172         )
 173
 174     def add_drive_cdrom(self, drive_file, index=None):
 175         """Set CD-ROM drive.
 176
 177         :param drive_file: Path to drive image.
 178         :param index: Drive index.
 179         :type drive_file: str
 180         :type index: int
 181         """
 182         index = f"index={index}," if index else u""
 183         self._params.add_with_value(
 184             u"drive", f"file={drive_file},{index}media=cdrom"
 185         )
 186
 187     def add_drive(self, drive_file, drive_format):
 188         """Set drive with custom format.
 189
 190         :param drive_file: Path to drive image.
 191         :param drive_format: Drive image format.
 192         :type drive_file: str
 193         :type drive_format: str
 194         """
 195         self._params.add_with_value(
 196             u"drive", f"file={drive_file},format={drive_format},"
 197             u"cache=none,if=virtio,file.locking=off"
 198         )
 199
 200     def add_kernelvm_params(self):
 201         """Set KernelVM QEMU parameters."""
 202         self._params.add_with_value(
 203             u"serial", f"file:{self._temp.get(u'log')}"
 204         )
 205         self._params.add_with_value(
 206             u"fsdev", u"local,id=root9p,path=/,security_model=none"
 207         )
 208         self._params.add_with_value(
 209             u"device", u"virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot"
 210         )
 211         self._params.add_with_value(u"kernel", f"{self._opt.get(u'img')}")
 212         self._params.add_with_value(u"initrd", f"{self._opt.get(u'initrd')}")
 213         self._params.add_with_value(
 214             u"append", f"'ro rootfstype=9p rootflags=trans=virtio "
 215             f"root=virtioroot console={self._opt.get(u'console')} "
 216             f"tsc=reliable hugepages=256 "
 217             f"init={self._temp.get(u'ini')} fastboot'"
 218         )
 219
 220     def add_vhost_user_if(
 221             self, socket, server=True, jumbo_frames=False, queue_size=None,
 222             queues=1, csum=False, gso=False):
 223         """Add Vhost-user interface.
 224
 225         :param socket: Path of the unix socket.
 226         :param server: If True the socket shall be a listening socket.
 227         :param jumbo_frames: Set True if jumbo frames are used in the test.
 228         :param queue_size: Vring queue size.
 229         :param queues: Number of queues.
 230         :param csum: Checksum offloading.
 231         :param gso: Generic segmentation offloading.
 232         :type socket: str
 233         :type server: bool
 234         :type jumbo_frames: bool
 235         :type queue_size: int
 236         :type queues: int
 237         :type csum: bool
 238         :type gso: bool
 239         """
 240         self._nic_id += 1
 241         self._params.add_with_value(
 242             u"chardev", f"socket,id=char{self._nic_id},"
 243             f"path={socket}{u',server' if server is True else u''}"
 244         )
 245         self._params.add_with_value(
 246             u"netdev", f"vhost-user,id=vhost{self._nic_id},"
 247             f"chardev=char{self._nic_id},queues={queues}"
 248         )
 249         mac = f"52:54:00:00:{self._opt.get(u'qemu_id'):02x}:" \
 250             f"{self._nic_id:02x}"
 251         queue_size = f"rx_queue_size={queue_size},tx_queue_size={queue_size}" \
 252             if queue_size else u""
 253         self._params.add_with_value(
 254             u"device", f"virtio-net-pci,netdev=vhost{self._nic_id},mac={mac},"
 255             f"addr={self._nic_id+5}.0,mq=on,vectors={2 * queues + 2},"
 256             f"csum={u'on' if csum else u'off'},gso={u'on' if gso else u'off'},"
 257             f"guest_tso4=off,guest_tso6=off,guest_ecn=off,"
 258             f"{queue_size}"
 259         )
 260
 261         # Add interface MAC and socket to the node dict.
 262         if_data = {u"mac_address": mac, u"socket": socket}
 263         if_name = f"vhost{self._nic_id}"
 264         self._vm_info[u"interfaces"][if_name] = if_data
 265         # Add socket to temporary file list.
 266         self._temp[if_name] = socket
 267
 268     def add_vfio_pci_if(self, pci):
 269         """Add VFIO PCI interface.
 270
 271         :param pci: PCI address of interface.
 272         :type pci: str
 273         """
 274         self._nic_id += 1
 275         self._params.add_with_value(
 276             u"device", f"vfio-pci,host={pci},addr={self._nic_id+5}.0"
 277         )
 278
 279     def create_kernelvm_config_vpp(self, **kwargs):
 280         """Create QEMU VPP config files.
 281
 282         :param kwargs: Key-value pairs to replace content of VPP configuration
 283             file.
 284         :type kwargs: dict
 285         """
 286         startup = f"/etc/vpp/vm_startup_{self._opt.get(u'qemu_id')}.conf"
 287         running = f"/etc/vpp/vm_running_{self._opt.get(u'qemu_id')}.exec"
 288
 289         self._temp[u"startup"] = startup
 290         self._temp[u"running"] = running
 291         self._opt[u"vnf_bin"] = f"/usr/bin/vpp -c {startup}"
 292
 293         # Create VPP startup configuration.
 294         vpp_config = VppConfigGenerator()
 295         vpp_config.set_node(self._node)
 296         vpp_config.add_unix_nodaemon()
 297         vpp_config.add_unix_cli_listen()
 298         vpp_config.add_unix_exec(running)
 299         vpp_config.add_socksvr()
 300         vpp_config.add_statseg_per_node_counters(value=u"on")
 301         vpp_config.add_buffers_per_numa(107520)
 302         vpp_config.add_heapsize(u"1G")
 303         vpp_config.add_ip_heap_size(u"1G")
 304         vpp_config.add_statseg_size(u"1G")
 305         vpp_config.add_cpu_main_core(u"0")
 306         if self._opt.get(u"smp") > 1:
 307             vpp_config.add_cpu_corelist_workers(f"1-{self._opt.get(u'smp')-1}")
 308         vpp_config.add_plugin(u"disable", u"default")
 309         vpp_config.add_plugin(u"enable", u"ping_plugin.so")
 310         if "2vfpt" in self._opt.get(u'vnf'):
 311             vpp_config.add_plugin(u"enable", u"avf_plugin.so")
 312         if "vhost" in self._opt.get(u'vnf'):
 313             vpp_config.add_plugin(u"enable", u"dpdk_plugin.so")
 314             vpp_config.add_dpdk_dev(u"0000:00:06.0", u"0000:00:07.0")
 315             vpp_config.add_dpdk_dev_default_rxq(kwargs[u"queues"])
 316             vpp_config.add_dpdk_log_level(u"debug")
 317             if not kwargs[u"jumbo_frames"]:
 318                 vpp_config.add_dpdk_no_multi_seg()
 319                 vpp_config.add_dpdk_no_tx_checksum_offload()
 320         if "ipsec" in self._opt.get(u'vnf'):
 321             vpp_config.add_plugin(u"enable", u"crypto_native_plugin.so")
 322             vpp_config.add_plugin(u"enable", u"crypto_ipsecmb_plugin.so")
 323             vpp_config.add_plugin(u"enable", u"crypto_openssl_plugin.so")
 324         if "nat" in self._opt.get(u'vnf'):
 325             vpp_config.add_nat(value=u"endpoint-dependent")
 326             vpp_config.add_nat_max_translations_per_thread(value=655360)
 327             vpp_config.add_plugin(u"enable", u"nat_plugin.so")
 328         vpp_config.write_config(startup)
 329
 330         # Create VPP running configuration.
 331         template = f"{Constants.RESOURCES_TPL_VM}/{self._opt.get(u'vnf')}.exec"
 332         exec_cmd_no_error(self._node, f"rm -f {running}", sudo=True)
 333
 334         with open(template, u"rt") as src_file:
 335             src = Template(src_file.read())
 336             exec_cmd_no_error(
 337                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 338                 f"sudo tee {running}"
 339             )
 340
 341     def create_kernelvm_config_testpmd_io(self, **kwargs):
 342         """Create QEMU testpmd-io command line.
 343
 344         :param kwargs: Key-value pairs to construct command line parameters.
 345         :type kwargs: dict
 346         """
 347         pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
 348         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 349             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 350             eal_driver=False,
 351             eal_pci_whitelist0=u"0000:00:06.0",
 352             eal_pci_whitelist1=u"0000:00:07.0",
 353             eal_in_memory=True,
 354             pmd_num_mbufs=16384,
 355             pmd_fwd_mode=u"io",
 356             pmd_nb_ports=u"2",
 357             pmd_portmask=u"0x3",
 358             pmd_max_pkt_len=pmd_max_pkt_len,
 359             pmd_mbuf_size=u"16384",
 360             pmd_rxq=kwargs[u"queues"],
 361             pmd_txq=kwargs[u"queues"],
 362             pmd_tx_offloads='0x0',
 363             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 364         )
 365
 366         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 367
 368     def create_kernelvm_config_testpmd_mac(self, **kwargs):
 369         """Create QEMU testpmd-mac command line.
 370
 371         :param kwargs: Key-value pairs to construct command line parameters.
 372         :type kwargs: dict
 373         """
 374         pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
 375         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 376             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 377             eal_driver=False,
 378             eal_pci_whitelist0=u"0000:00:06.0",
 379             eal_pci_whitelist1=u"0000:00:07.0",
 380             eal_in_memory=True,
 381             pmd_num_mbufs=16384,
 382             pmd_fwd_mode=u"mac",
 383             pmd_nb_ports=u"2",
 384             pmd_portmask=u"0x3",
 385             pmd_max_pkt_len=pmd_max_pkt_len,
 386             pmd_mbuf_size=u"16384",
 387             pmd_eth_peer_0=f"0,{kwargs[u'vif1_mac']}",
 388             pmd_eth_peer_1=f"1,{kwargs[u'vif2_mac']}",
 389             pmd_rxq=kwargs[u"queues"],
 390             pmd_txq=kwargs[u"queues"],
 391             pmd_tx_offloads=u"0x0",
 392             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 393         )
 394
 395         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 396
 397     def create_kernelvm_init(self, **kwargs):
 398         """Create QEMU init script.
 399
 400         :param kwargs: Key-value pairs to replace content of init startup file.
 401         :type kwargs: dict
 402         """
 403         template = f"{Constants.RESOURCES_TPL_VM}/init.sh"
 404         init = self._temp.get(u"ini")
 405         exec_cmd_no_error(self._node, f"rm -f {init}", sudo=True)
 406
 407         with open(template, u"rt") as src_file:
 408             src = Template(src_file.read())
 409             exec_cmd_no_error(
 410                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 411                 f"sudo tee {init}"
 412             )
 413             exec_cmd_no_error(self._node, f"chmod +x {init}", sudo=True)
 414
 415     def configure_kernelvm_vnf(self, **kwargs):
 416         """Create KernelVM VNF configurations.
 417
 418         :param kwargs: Key-value pairs for templating configs.
 419         :type kwargs: dict
 420         """
 421         if u"vpp" in self._opt.get(u"vnf"):
 422             self.create_kernelvm_config_vpp(**kwargs)
 423             self.create_kernelvm_init(vnf_bin=self._opt.get(u"vnf_bin"))
 424         elif u"testpmd_io" in self._opt.get(u"vnf"):
 425             self.create_kernelvm_config_testpmd_io(**kwargs)
 426             self.create_kernelvm_init(vnf_bin=self._opt.get(u"vnf_bin"))
 427         elif u"testpmd_mac" in self._opt.get(u"vnf"):
 428             self.create_kernelvm_config_testpmd_mac(**kwargs)
 429             self.create_kernelvm_init(vnf_bin=self._opt.get(u"vnf_bin"))
 430         else:
 431             raise RuntimeError(u"QEMU: Unsupported VNF!")
 432
 433     def get_qemu_pids(self):
 434         """Get QEMU CPU pids.
 435
 436         :returns: List of QEMU CPU pids.
 437         :rtype: list of str
 438         """
 439         command = f"grep -rwl 'CPU' /proc/$(sudo cat " \
 440             f"{self._temp.get(u'pidfile')})/task/*/comm "
 441         command += r"| xargs dirname | sed -e 's/\/.*\///g' | uniq"
 442
 443         stdout, _ = exec_cmd_no_error(self._node, command)
 444         return stdout.splitlines()
 445
 446     def qemu_set_affinity(self, *host_cpus):
 447         """Set qemu affinity by getting thread PIDs via QMP and taskset to list
 448         of CPU cores. Function tries to execute 3 times to avoid race condition
 449         in getting thread PIDs.
 450
 451         :param host_cpus: List of CPU cores.
 452         :type host_cpus: list
 453         """
 454         for _ in range(3):
 455             try:
 456                 qemu_cpus = self.get_qemu_pids()
 457
 458                 if len(qemu_cpus) != len(host_cpus):
 459                     sleep(1)
 460                     continue
 461                 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
 462                     command = f"taskset -pc {host_cpu} {qemu_cpu}"
 463                     message = f"QEMU: Set affinity failed " \
 464                         f"on {self._node[u'host']}!"
 465                     exec_cmd_no_error(
 466                         self._node, command, sudo=True, message=message
 467                     )
 468                 break
 469             except (RuntimeError, ValueError):
 470                 self.qemu_kill_all()
 471                 raise
 472         else:
 473             self.qemu_kill_all()
 474             raise RuntimeError(u"Failed to set Qemu threads affinity!")
 475
 476     def qemu_set_scheduler_policy(self):
 477         """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
 478         processes.
 479
 480         :raises RuntimeError: Set scheduler policy failed.
 481         """
 482         try:
 483             qemu_cpus = self.get_qemu_pids()
 484
 485             for qemu_cpu in qemu_cpus:
 486                 command = f"chrt -r -p 1 {qemu_cpu}"
 487                 message = f"QEMU: Set SCHED_RR failed on {self._node[u'host']}"
 488                 exec_cmd_no_error(
 489                     self._node, command, sudo=True, message=message
 490                 )
 491         except (RuntimeError, ValueError):
 492             self.qemu_kill_all()
 493             raise
 494
 495     def _qemu_qmp_exec(self, cmd):
 496         """Execute QMP command.
 497
 498         QMP is JSON based protocol which allows to control QEMU instance.
 499
 500         :param cmd: QMP command to execute.
 501         :type cmd: str
 502         :returns: Command output in python representation of JSON format. The
 503             { "return": {} } response is QMP's success response. An error
 504             response will contain the "error" keyword instead of "return".
 505         """
 506         # To enter command mode, the qmp_capabilities command must be issued.
 507         command = f"echo \"{{{{ \\\"execute\\\": " \
 508             f"\\\"qmp_capabilities\\\" }}}}" \
 509             f"{{{{ \\\"execute\\\": \\\"{cmd}\\\" }}}}\" | " \
 510             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qmp')}"
 511         message = f"QMP execute '{cmd}' failed on {self._node[u'host']}"
 512
 513         stdout, _ = exec_cmd_no_error(
 514             self._node, command, sudo=False, message=message
 515         )
 516
 517         # Skip capabilities negotiation messages.
 518         out_list = stdout.splitlines()
 519         if len(out_list) < 3:
 520             raise RuntimeError(f"Invalid QMP output on {self._node[u'host']}")
 521         return json.loads(out_list[2])
 522
 523     def _qemu_qga_flush(self):
 524         """Flush the QGA parser state."""
 525         command = f"(printf \"\xFF\"; sleep 1) | sudo -S socat " \
 526             f"- UNIX-CONNECT:{self._temp.get(u'qga')}"
 527         message = f"QGA flush failed on {self._node[u'host']}"
 528         stdout, _ = exec_cmd_no_error(
 529             self._node, command, sudo=False, message=message
 530         )
 531
 532         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 533
 534     def _qemu_qga_exec(self, cmd):
 535         """Execute QGA command.
 536
 537         QGA provide access to a system-level agent via standard QMP commands.
 538
 539         :param cmd: QGA command to execute.
 540         :type cmd: str
 541         """
 542         command = f"(echo \"{{{{ \\\"execute\\\": " \
 543             f"\\\"{cmd}\\\" }}}}\"; sleep 1) | " \
 544             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qga')}"
 545         message = f"QGA execute '{cmd}' failed on {self._node[u'host']}"
 546         stdout, _ = exec_cmd_no_error(
 547             self._node, command, sudo=False, message=message
 548         )
 549
 550         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 551
 552     def _wait_until_vm_boot(self):
 553         """Wait until QEMU VM is booted."""
 554         try:
 555             getattr(self, f'_wait_{self._opt["vnf"]}')()
 556         except AttributeError:
 557             self._wait_default()
 558
 559     def _wait_default(self, retries=60):
 560         """Wait until QEMU with VPP is booted.
 561
 562         :param retries: Number of retries.
 563         :type retries: int
 564         """
 565         for _ in range(retries):
 566             command = f"tail -1 {self._temp.get(u'log')}"
 567             stdout = None
 568             try:
 569                 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 570                 sleep(1)
 571             except RuntimeError:
 572                 pass
 573             if "vpp " in stdout and "built by" in stdout:
 574                 break
 575             if u"Press enter to exit" in stdout:
 576                 break
 577             if u"reboot: Power down" in stdout:
 578                 raise RuntimeError(
 579                     f"QEMU: NF failed to run on {self._node[u'host']}!"
 580                 )
 581         else:
 582             raise RuntimeError(
 583                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 584             )
 585
 586     def _wait_nestedvm(self, retries=12):
 587         """Wait until QEMU with NestedVM is booted.
 588
 589         First try to flush qga until there is output.
 590         Then ping QEMU guest agent each 5s until VM booted or timeout.
 591
 592         :param retries: Number of retries with 5s between trials.
 593         :type retries: int
 594         """
 595         for _ in range(retries):
 596             out = None
 597             try:
 598                 out = self._qemu_qga_flush()
 599             except ValueError:
 600                 logger.trace(f"QGA qga flush unexpected output {out}")
 601             # Empty output - VM not booted yet
 602             if not out:
 603                 sleep(5)
 604             else:
 605                 break
 606         else:
 607             raise RuntimeError(
 608                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 609             )
 610         for _ in range(retries):
 611             out = None
 612             try:
 613                 out = self._qemu_qga_exec(u"guest-ping")
 614             except ValueError:
 615                 logger.trace(f"QGA guest-ping unexpected output {out}")
 616             # Empty output - VM not booted yet.
 617             if not out:
 618                 sleep(5)
 619             # Non-error return - VM booted.
 620             elif out.get(u"return") is not None:
 621                 break
 622             # Skip error and wait.
 623             elif out.get(u"error") is not None:
 624                 sleep(5)
 625             else:
 626                 # If there is an unexpected output from QGA guest-info, try
 627                 # again until timeout.
 628                 logger.trace(f"QGA guest-ping unexpected output {out}")
 629         else:
 630             raise RuntimeError(
 631                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 632             )
 633
 634     def _update_vm_interfaces(self):
 635         """Update interface names in VM node dict."""
 636         # Send guest-network-get-interfaces command via QGA, output example:
 637         # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
 638         # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
 639         out = self._qemu_qga_exec(u"guest-network-get-interfaces")
 640         interfaces = out.get(u"return")
 641         mac_name = {}
 642         if not interfaces:
 643             raise RuntimeError(
 644                 f"Get VM interface list failed on {self._node[u'host']}"
 645             )
 646         # Create MAC-name dict.
 647         for interface in interfaces:
 648             if u"hardware-address" not in interface:
 649                 continue
 650             mac_name[interface[u"hardware-address"]] = interface[u"name"]
 651         # Match interface by MAC and save interface name.
 652         for interface in self._vm_info[u"interfaces"].values():
 653             mac = interface.get(u"mac_address")
 654             if_name = mac_name.get(mac)
 655             if if_name is None:
 656                 logger.trace(f"Interface name for MAC {mac} not found")
 657             else:
 658                 interface[u"name"] = if_name
 659
 660     def qemu_start(self):
 661         """Start QEMU and wait until VM boot.
 662
 663         :returns: VM node info.
 664         :rtype: dict
 665         """
 666         cmd_opts = OptionString()
 667         cmd_opts.add(f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch}")
 668         cmd_opts.extend(self._params)
 669         message = f"QEMU: Start failed on {self._node[u'host']}!"
 670         try:
 671             DUTSetup.check_huge_page(
 672                 self._node, u"/dev/hugepages", int(self._opt.get(u"mem")))
 673
 674             exec_cmd_no_error(
 675                 self._node, cmd_opts, timeout=300, sudo=True, message=message
 676             )
 677             self._wait_until_vm_boot()
 678         except RuntimeError:
 679             self.qemu_kill_all()
 680             raise
 681         return self._vm_info
 682
 683     def qemu_kill(self):
 684         """Kill qemu process."""
 685         exec_cmd(
 686             self._node, f"chmod +r {self._temp.get(u'pidfile')}", sudo=True
 687         )
 688         exec_cmd(
 689             self._node, f"kill -SIGKILL $(cat {self._temp.get(u'pidfile')})",
 690             sudo=True
 691         )
 692
 693         for value in self._temp.values():
 694             exec_cmd(self._node, f"cat {value}", sudo=True)
 695             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 696
 697     def qemu_kill_all(self):
 698         """Kill all qemu processes on DUT node if specified."""
 699         exec_cmd(self._node, u"pkill -SIGKILL qemu", sudo=True)
 700
 701         for value in self._temp.values():
 702             exec_cmd(self._node, f"cat {value}", sudo=True)
 703             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 704
 705     def qemu_version(self):
 706         """Return Qemu version.
 707
 708         :returns: Qemu version.
 709         :rtype: str
 710         """
 711         command = f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch} " \
 712             f"--version"
 713         try:
 714             stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 715             return match(r"QEMU emulator version ([\d.]*)", stdout).group(1)
 716         except RuntimeError:
 717             self.qemu_kill_all()
 718             raise