resources/libraries/python/QemuUtils.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """QEMU utilities library."""
  15
  16 import json
  17
  18 from re import match
  19 from string import Template
  20 from time import sleep
  21
  22 from robot.api import logger
  23
  24 from resources.libraries.python.Constants import Constants
  25 from resources.libraries.python.DpdkUtil import DpdkUtil
  26 from resources.libraries.python.DUTSetup import DUTSetup
  27 from resources.libraries.python.OptionString import OptionString
  28 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
  29 from resources.libraries.python.topology import NodeType, Topology
  30 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
  31
  32 __all__ = [u"QemuUtils"]
  33
  34
  35 class QemuUtils:
  36     """QEMU utilities."""
  37
  38     # Use one instance of class per tests.
  39     ROBOT_LIBRARY_SCOPE = u"TEST CASE"
  40
  41     def __init__(
  42             self, node, qemu_id=1, smp=1, mem=512, vnf=None,
  43             img=Constants.QEMU_VM_IMAGE):
  44         """Initialize QemuUtil class.
  45
  46         :param node: Node to run QEMU on.
  47         :param qemu_id: QEMU identifier.
  48         :param smp: Number of virtual SMP units (cores).
  49         :param mem: Amount of memory.
  50         :param vnf: Network function workload.
  51         :param img: QEMU disk image or kernel image path.
  52         :type node: dict
  53         :type qemu_id: int
  54         :type smp: int
  55         :type mem: int
  56         :type vnf: str
  57         :type img: str
  58         """
  59         self._nic_id = 0
  60         self._node = node
  61         self._arch = Topology.get_node_arch(self._node)
  62         self._opt = dict()
  63
  64         # Architecture specific options
  65         if self._arch == u"aarch64":
  66             dpdk_target = u"arm64-armv8a"
  67             self._opt[u"machine_args"] = \
  68                 u"virt,accel=kvm,usb=off,mem-merge=off,gic-version=3"
  69             self._opt[u"console"] = u"ttyAMA0"
  70         else:
  71             dpdk_target = u"x86_64-native"
  72             self._opt[u"machine_args"] = u"pc,accel=kvm,usb=off,mem-merge=off"
  73             self._opt[u"console"] = u"ttyS0"
  74         self._testpmd_path = f"{Constants.QEMU_VM_DPDK}/" \
  75             f"{dpdk_target}-linux-gcc/app"
  76         self._vm_info = {
  77             u"host": node[u"host"],
  78             u"type": NodeType.VM,
  79             u"port": 10021 + qemu_id,
  80             u"serial": 4555 + qemu_id,
  81             u"username": 'cisco',
  82             u"password": 'cisco',
  83             u"interfaces": {},
  84         }
  85         if node[u"port"] != 22:
  86             self._vm_info[u"host_port"] = node[u"port"]
  87             self._vm_info[u"host_username"] = node[u"username"]
  88             self._vm_info[u"host_password"] = node[u"password"]
  89         # Input Options.
  90         self._opt[u"qemu_id"] = qemu_id
  91         self._opt[u"mem"] = int(mem)
  92         self._opt[u"smp"] = int(smp)
  93         self._opt[u"img"] = img
  94         self._opt[u"vnf"] = vnf
  95         # Temporary files.
  96         self._temp = dict()
  97         self._temp[u"log"] = f"/tmp/serial_{qemu_id}.log"
  98         self._temp[u"pidfile"] = f"/run/qemu_{qemu_id}.pid"
  99         if img == Constants.QEMU_VM_IMAGE:
 100             self._temp[u"qmp"] = f"/run/qmp_{qemu_id}.sock"
 101             self._temp[u"qga"] = f"/run/qga_{qemu_id}.sock"
 102         elif img == Constants.QEMU_VM_KERNEL:
 103             self._opt[u"img"], _ = exec_cmd_no_error(
 104                 node, f"ls -1 {Constants.QEMU_VM_KERNEL}* | tail -1",
 105                 message=u"Qemu Kernel VM image not found!"
 106             )
 107             self._temp[u"ini"] = f"/etc/vm_init_{qemu_id}.conf"
 108             self._opt[u"initrd"], _ = exec_cmd_no_error(
 109                 node, f"ls -1 {Constants.QEMU_VM_KERNEL_INITRD}* | tail -1",
 110                 message=u"Qemu Kernel initrd image not found!"
 111             )
 112         else:
 113             raise RuntimeError(f"QEMU: Unknown VM image option: {img}")
 114         # Computed parameters for QEMU command line.
 115         self._params = OptionString(prefix=u"-")
 116
 117     def add_default_params(self):
 118         """Set default QEMU command line parameters."""
 119         self._params.add(u"daemonize")
 120         self._params.add(u"nodefaults")
 121         self._params.add_with_value(
 122             u"name", f"vnf{self._opt.get(u'qemu_id')},debug-threads=on"
 123         )
 124         self._params.add(u"no-user-config")
 125         self._params.add(u"nographic")
 126         self._params.add(u"enable-kvm")
 127         self._params.add_with_value(u"pidfile", self._temp.get(u"pidfile"))
 128         self._params.add_with_value(u"cpu", u"host")
 129
 130         self._params.add_with_value(u"machine", self._opt.get(u"machine_args"))
 131         self._params.add_with_value(
 132             u"smp", f"{self._opt.get(u'smp')},sockets=1,"
 133             f"cores={self._opt.get(u'smp')},threads=1"
 134         )
 135         self._params.add_with_value(
 136             u"object", f"memory-backend-file,id=mem,"
 137             f"size={self._opt.get(u'mem')}M,mem-path=/dev/hugepages,share=on"
 138         )
 139         self._params.add_with_value(u"m", f"{self._opt.get(u'mem')}M")
 140         self._params.add_with_value(u"numa", u"node,memdev=mem")
 141
 142     def add_net_user(self):
 143         """Set managment port forwarding."""
 144         self._params.add_with_value(
 145             u"netdev", f"user,id=mgmt,net=172.16.255.0/24,"
 146             f"hostfwd=tcp::{self._vm_info[u'port']}-:22"
 147         )
 148         self._params.add_with_value(
 149             u"device", f"virtio-net,netdev=mgmt"
 150         )
 151
 152     def add_qmp_qga(self):
 153         """Set QMP, QGA management."""
 154         self._params.add_with_value(
 155             u"chardev", f"socket,path={self._temp.get(u'qga')},"
 156             f"server,nowait,id=qga0"
 157         )
 158         self._params.add_with_value(u"device", u"isa-serial,chardev=qga0")
 159         self._params.add_with_value(
 160             u"qmp", f"unix:{self._temp.get(u'qmp')},server,nowait"
 161         )
 162
 163     def add_serial(self):
 164         """Set serial to file redirect."""
 165         self._params.add_with_value(
 166             u"chardev", f"socket,host=127.0.0.1,"
 167             f"port={self._vm_info[u'serial']},id=gnc0,server,nowait")
 168         self._params.add_with_value(u"device", u"isa-serial,chardev=gnc0")
 169         self._params.add_with_value(
 170             u"serial", f"file:{self._temp.get(u'log')}"
 171         )
 172
 173     def add_drive_cdrom(self, drive_file, index=None):
 174         """Set CD-ROM drive.
 175
 176         :param drive_file: Path to drive image.
 177         :param index: Drive index.
 178         :type drive_file: str
 179         :type index: int
 180         """
 181         index = f"index={index}," if index else u""
 182         self._params.add_with_value(
 183             u"drive", f"file={drive_file},{index}media=cdrom"
 184         )
 185
 186     def add_drive(self, drive_file, drive_format):
 187         """Set drive with custom format.
 188
 189         :param drive_file: Path to drive image.
 190         :param drive_format: Drive image format.
 191         :type drive_file: str
 192         :type drive_format: str
 193         """
 194         self._params.add_with_value(
 195             u"drive", f"file={drive_file},format={drive_format},"
 196             u"cache=none,if=virtio,file.locking=off"
 197         )
 198
 199     def add_kernelvm_params(self):
 200         """Set KernelVM QEMU parameters."""
 201         self._params.add_with_value(
 202             u"serial", f"file:{self._temp.get(u'log')}"
 203         )
 204         self._params.add_with_value(
 205             u"fsdev", u"local,id=root9p,path=/,security_model=none"
 206         )
 207         self._params.add_with_value(
 208             u"device", u"virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot"
 209         )
 210         self._params.add_with_value(u"kernel", f"{self._opt.get(u'img')}")
 211         self._params.add_with_value(u"initrd", f"{self._opt.get(u'initrd')}")
 212         self._params.add_with_value(
 213             u"append", f"'ro rootfstype=9p rootflags=trans=virtio "
 214             f"root=virtioroot console={self._opt.get(u'console')} "
 215             f"tsc=reliable hugepages=256 "
 216             f"init={self._temp.get(u'ini')} fastboot'"
 217         )
 218
 219     def add_vhost_user_if(
 220             self, socket, server=True, jumbo_frames=False, queue_size=None,
 221             queues=1, csum=False, gso=False):
 222         """Add Vhost-user interface.
 223
 224         :param socket: Path of the unix socket.
 225         :param server: If True the socket shall be a listening socket.
 226         :param jumbo_frames: Set True if jumbo frames are used in the test.
 227         :param queue_size: Vring queue size.
 228         :param queues: Number of queues.
 229         :param csum: Checksum offloading.
 230         :param gso: Generic segmentation offloading.
 231         :type socket: str
 232         :type server: bool
 233         :type jumbo_frames: bool
 234         :type queue_size: int
 235         :type queues: int
 236         :type csum: bool
 237         :type gso: bool
 238         """
 239         self._nic_id += 1
 240         self._params.add_with_value(
 241             u"chardev", f"socket,id=char{self._nic_id},"
 242             f"path={socket}{u',server' if server is True else u''}"
 243         )
 244         self._params.add_with_value(
 245             u"netdev", f"vhost-user,id=vhost{self._nic_id},"
 246             f"chardev=char{self._nic_id},queues={queues}"
 247         )
 248         mac = f"52:54:00:00:{self._opt.get(u'qemu_id'):02x}:" \
 249             f"{self._nic_id:02x}"
 250         queue_size = f"rx_queue_size={queue_size},tx_queue_size={queue_size}" \
 251             if queue_size else u""
 252         self._params.add_with_value(
 253             u"device", f"virtio-net-pci,netdev=vhost{self._nic_id},mac={mac},"
 254             f"addr={self._nic_id+5}.0,mq=on,vectors={2 * queues + 2},"
 255             f"csum={u'on' if csum else u'off'},gso={u'on' if gso else u'off'},"
 256             f"guest_tso4=off,guest_tso6=off,guest_ecn=off,"
 257             f"{queue_size}"
 258         )
 259
 260         # Add interface MAC and socket to the node dict.
 261         if_data = {u"mac_address": mac, u"socket": socket}
 262         if_name = f"vhost{self._nic_id}"
 263         self._vm_info[u"interfaces"][if_name] = if_data
 264         # Add socket to temporary file list.
 265         self._temp[if_name] = socket
 266
 267     def add_vfio_pci_if(self, pci):
 268         """Add VFIO PCI interface.
 269
 270         :param pci: PCI address of interface.
 271         :type pci: str
 272         """
 273         self._nic_id += 1
 274         self._params.add_with_value(
 275             u"device", f"vfio-pci,host={pci},addr={self._nic_id+5}.0"
 276         )
 277
 278     def create_kernelvm_config_vpp(self, **kwargs):
 279         """Create QEMU VPP config files.
 280
 281         :param kwargs: Key-value pairs to replace content of VPP configuration
 282             file.
 283         :type kwargs: dict
 284         """
 285         startup = f"/etc/vpp/vm_startup_{self._opt.get(u'qemu_id')}.conf"
 286         running = f"/etc/vpp/vm_running_{self._opt.get(u'qemu_id')}.exec"
 287
 288         self._temp[u"startup"] = startup
 289         self._temp[u"running"] = running
 290         self._opt[u"vnf_bin"] = f"/usr/bin/vpp -c {startup}"
 291
 292         # Create VPP startup configuration.
 293         vpp_config = VppConfigGenerator()
 294         vpp_config.set_node(self._node)
 295         vpp_config.add_unix_nodaemon()
 296         vpp_config.add_unix_cli_listen()
 297         vpp_config.add_unix_exec(running)
 298         vpp_config.add_socksvr()
 299         vpp_config.add_statseg_per_node_counters(value=u"on")
 300         vpp_config.add_buffers_per_numa(107520)
 301         vpp_config.add_heapsize(u"1G")
 302         vpp_config.add_ip_heap_size(u"1G")
 303         vpp_config.add_statseg_size(u"1G")
 304         vpp_config.add_cpu_main_core(u"0")
 305         if self._opt.get(u"smp") > 1:
 306             vpp_config.add_cpu_corelist_workers(f"1-{self._opt.get(u'smp')-1}")
 307         vpp_config.add_plugin(u"disable", u"default")
 308         vpp_config.add_plugin(u"enable", u"ping_plugin.so")
 309         if "2vfpt" in self._opt.get(u'vnf'):
 310             vpp_config.add_plugin(u"enable", u"avf_plugin.so")
 311         if "vhost" in self._opt.get(u'vnf'):
 312             vpp_config.add_plugin(u"enable", u"dpdk_plugin.so")
 313             vpp_config.add_dpdk_dev(u"0000:00:06.0", u"0000:00:07.0")
 314             vpp_config.add_dpdk_dev_default_rxq(kwargs[u"queues"])
 315             vpp_config.add_dpdk_log_level(u"debug")
 316             if not kwargs[u"jumbo_frames"]:
 317                 vpp_config.add_dpdk_no_multi_seg()
 318                 vpp_config.add_dpdk_no_tx_checksum_offload()
 319         if "ipsec" in self._opt.get(u'vnf'):
 320             vpp_config.add_plugin(u"enable", u"crypto_native_plugin.so")
 321             vpp_config.add_plugin(u"enable", u"crypto_ipsecmb_plugin.so")
 322             vpp_config.add_plugin(u"enable", u"crypto_openssl_plugin.so")
 323         if "nat" in self._opt.get(u'vnf'):
 324             vpp_config.add_nat(value=u"endpoint-dependent")
 325             #vpp_config.add_nat_max_translations_per_thread(value=655360)
 326             vpp_config.add_plugin(u"enable", u"nat_plugin.so")
 327         vpp_config.write_config(startup)
 328
 329         # Create VPP running configuration.
 330         template = f"{Constants.RESOURCES_TPL}/vm/{self._opt.get(u'vnf')}.exec"
 331         exec_cmd_no_error(self._node, f"rm -f {running}", sudo=True)
 332
 333         with open(template, u"rt") as src_file:
 334             src = Template(src_file.read())
 335             exec_cmd_no_error(
 336                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 337                 f"sudo tee {running}"
 338             )
 339
 340     def create_kernelvm_config_testpmd_io(self, **kwargs):
 341         """Create QEMU testpmd-io command line.
 342
 343         :param kwargs: Key-value pairs to construct command line parameters.
 344         :type kwargs: dict
 345         """
 346         pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
 347         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 348             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 349             eal_driver=False,
 350             eal_pci_whitelist0=u"0000:00:06.0",
 351             eal_pci_whitelist1=u"0000:00:07.0",
 352             eal_in_memory=True,
 353             pmd_num_mbufs=16384,
 354             pmd_fwd_mode=u"io",
 355             pmd_nb_ports=u"2",
 356             pmd_portmask=u"0x3",
 357             pmd_max_pkt_len=pmd_max_pkt_len,
 358             pmd_mbuf_size=u"16384",
 359             pmd_rxq=kwargs[u"queues"],
 360             pmd_txq=kwargs[u"queues"],
 361             pmd_tx_offloads='0x0',
 362             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 363         )
 364
 365         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 366
 367     def create_kernelvm_config_testpmd_mac(self, **kwargs):
 368         """Create QEMU testpmd-mac command line.
 369
 370         :param kwargs: Key-value pairs to construct command line parameters.
 371         :type kwargs: dict
 372         """
 373         pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
 374         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 375             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 376             eal_driver=False,
 377             eal_pci_whitelist0=u"0000:00:06.0",
 378             eal_pci_whitelist1=u"0000:00:07.0",
 379             eal_in_memory=True,
 380             pmd_num_mbufs=16384,
 381             pmd_fwd_mode=u"mac",
 382             pmd_nb_ports=u"2",
 383             pmd_portmask=u"0x3",
 384             pmd_max_pkt_len=pmd_max_pkt_len,
 385             pmd_mbuf_size=u"16384",
 386             pmd_eth_peer_0=f"0,{kwargs[u'vif1_mac']}",
 387             pmd_eth_peer_1=f"1,{kwargs[u'vif2_mac']}",
 388             pmd_rxq=kwargs[u"queues"],
 389             pmd_txq=kwargs[u"queues"],
 390             pmd_tx_offloads=u"0x0",
 391             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 392         )
 393
 394         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 395
 396     def create_kernelvm_init(self, **kwargs):
 397         """Create QEMU init script.
 398
 399         :param kwargs: Key-value pairs to replace content of init startup file.
 400         :type kwargs: dict
 401         """
 402         template = f"{Constants.RESOURCES_TPL}/vm/init.sh"
 403         init = self._temp.get(u"ini")
 404         exec_cmd_no_error(self._node, f"rm -f {init}", sudo=True)
 405
 406         with open(template, u"rt") as src_file:
 407             src = Template(src_file.read())
 408             exec_cmd_no_error(
 409                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 410                 f"sudo tee {init}"
 411             )
 412             exec_cmd_no_error(self._node, f"chmod +x {init}", sudo=True)
 413
 414     def configure_kernelvm_vnf(self, **kwargs):
 415         """Create KernelVM VNF configurations.
 416
 417         :param kwargs: Key-value pairs for templating configs.
 418         :type kwargs: dict
 419         """
 420         if u"vpp" in self._opt.get(u"vnf"):
 421             self.create_kernelvm_config_vpp(**kwargs)
 422             self.create_kernelvm_init(vnf_bin=self._opt.get(u"vnf_bin"))
 423         elif u"testpmd_io" in self._opt.get(u"vnf"):
 424             self.create_kernelvm_config_testpmd_io(**kwargs)
 425             self.create_kernelvm_init(vnf_bin=self._opt.get(u"vnf_bin"))
 426         elif u"testpmd_mac" in self._opt.get(u"vnf"):
 427             self.create_kernelvm_config_testpmd_mac(**kwargs)
 428             self.create_kernelvm_init(vnf_bin=self._opt.get(u"vnf_bin"))
 429         else:
 430             raise RuntimeError(u"QEMU: Unsupported VNF!")
 431
 432     def get_qemu_pids(self):
 433         """Get QEMU CPU pids.
 434
 435         :returns: List of QEMU CPU pids.
 436         :rtype: list of str
 437         """
 438         command = f"grep -rwl 'CPU' /proc/$(sudo cat " \
 439             f"{self._temp.get(u'pidfile')})/task/*/comm "
 440         command += r"| xargs dirname | sed -e 's/\/.*\///g' | uniq"
 441
 442         stdout, _ = exec_cmd_no_error(self._node, command)
 443         return stdout.splitlines()
 444
 445     def qemu_set_affinity(self, *host_cpus):
 446         """Set qemu affinity by getting thread PIDs via QMP and taskset to list
 447         of CPU cores. Function tries to execute 3 times to avoid race condition
 448         in getting thread PIDs.
 449
 450         :param host_cpus: List of CPU cores.
 451         :type host_cpus: list
 452         """
 453         for _ in range(3):
 454             try:
 455                 qemu_cpus = self.get_qemu_pids()
 456
 457                 if len(qemu_cpus) != len(host_cpus):
 458                     sleep(1)
 459                     continue
 460                 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
 461                     command = f"taskset -pc {host_cpu} {qemu_cpu}"
 462                     message = f"QEMU: Set affinity failed " \
 463                         f"on {self._node[u'host']}!"
 464                     exec_cmd_no_error(
 465                         self._node, command, sudo=True, message=message
 466                     )
 467                 break
 468             except (RuntimeError, ValueError):
 469                 self.qemu_kill_all()
 470                 raise
 471         else:
 472             self.qemu_kill_all()
 473             raise RuntimeError(u"Failed to set Qemu threads affinity!")
 474
 475     def qemu_set_scheduler_policy(self):
 476         """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
 477         processes.
 478
 479         :raises RuntimeError: Set scheduler policy failed.
 480         """
 481         try:
 482             qemu_cpus = self.get_qemu_pids()
 483
 484             for qemu_cpu in qemu_cpus:
 485                 command = f"chrt -r -p 1 {qemu_cpu}"
 486                 message = f"QEMU: Set SCHED_RR failed on {self._node[u'host']}"
 487                 exec_cmd_no_error(
 488                     self._node, command, sudo=True, message=message
 489                 )
 490         except (RuntimeError, ValueError):
 491             self.qemu_kill_all()
 492             raise
 493
 494     def _qemu_qmp_exec(self, cmd):
 495         """Execute QMP command.
 496
 497         QMP is JSON based protocol which allows to control QEMU instance.
 498
 499         :param cmd: QMP command to execute.
 500         :type cmd: str
 501         :returns: Command output in python representation of JSON format. The
 502             { "return": {} } response is QMP's success response. An error
 503             response will contain the "error" keyword instead of "return".
 504         """
 505         # To enter command mode, the qmp_capabilities command must be issued.
 506         command = f"echo \"{{{{ \\\"execute\\\": " \
 507             f"\\\"qmp_capabilities\\\" }}}}" \
 508             f"{{{{ \\\"execute\\\": \\\"{cmd}\\\" }}}}\" | " \
 509             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qmp')}"
 510         message = f"QMP execute '{cmd}' failed on {self._node[u'host']}"
 511
 512         stdout, _ = exec_cmd_no_error(
 513             self._node, command, sudo=False, message=message
 514         )
 515
 516         # Skip capabilities negotiation messages.
 517         out_list = stdout.splitlines()
 518         if len(out_list) < 3:
 519             raise RuntimeError(f"Invalid QMP output on {self._node[u'host']}")
 520         return json.loads(out_list[2])
 521
 522     def _qemu_qga_flush(self):
 523         """Flush the QGA parser state."""
 524         command = f"(printf \"\xFF\"; sleep 1) | sudo -S socat " \
 525             f"- UNIX-CONNECT:{self._temp.get(u'qga')}"
 526         message = f"QGA flush failed on {self._node[u'host']}"
 527         stdout, _ = exec_cmd_no_error(
 528             self._node, command, sudo=False, message=message
 529         )
 530
 531         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 532
 533     def _qemu_qga_exec(self, cmd):
 534         """Execute QGA command.
 535
 536         QGA provide access to a system-level agent via standard QMP commands.
 537
 538         :param cmd: QGA command to execute.
 539         :type cmd: str
 540         """
 541         command = f"(echo \"{{{{ \\\"execute\\\": " \
 542             f"\\\"{cmd}\\\" }}}}\"; sleep 1) | " \
 543             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qga')}"
 544         message = f"QGA execute '{cmd}' failed on {self._node[u'host']}"
 545         stdout, _ = exec_cmd_no_error(
 546             self._node, command, sudo=False, message=message
 547         )
 548
 549         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 550
 551     def _wait_until_vm_boot(self):
 552         """Wait until QEMU VM is booted."""
 553         try:
 554             getattr(self, f'_wait_{self._opt["vnf"]}')()
 555         except AttributeError:
 556             self._wait_default()
 557
 558     def _wait_default(self, retries=60):
 559         """Wait until QEMU with VPP is booted.
 560
 561         :param retries: Number of retries.
 562         :type retries: int
 563         """
 564         for _ in range(retries):
 565             command = f"tail -1 {self._temp.get(u'log')}"
 566             stdout = None
 567             try:
 568                 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 569                 sleep(1)
 570             except RuntimeError:
 571                 pass
 572             if "vpp " in stdout and "built by" in stdout:
 573                 break
 574             if u"Press enter to exit" in stdout:
 575                 break
 576             if u"reboot: Power down" in stdout:
 577                 raise RuntimeError(
 578                     f"QEMU: NF failed to run on {self._node[u'host']}!"
 579                 )
 580         else:
 581             raise RuntimeError(
 582                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 583             )
 584
 585     def _wait_nestedvm(self, retries=12):
 586         """Wait until QEMU with NestedVM is booted.
 587
 588         First try to flush qga until there is output.
 589         Then ping QEMU guest agent each 5s until VM booted or timeout.
 590
 591         :param retries: Number of retries with 5s between trials.
 592         :type retries: int
 593         """
 594         for _ in range(retries):
 595             out = None
 596             try:
 597                 out = self._qemu_qga_flush()
 598             except ValueError:
 599                 logger.trace(f"QGA qga flush unexpected output {out}")
 600             # Empty output - VM not booted yet
 601             if not out:
 602                 sleep(5)
 603             else:
 604                 break
 605         else:
 606             raise RuntimeError(
 607                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 608             )
 609         for _ in range(retries):
 610             out = None
 611             try:
 612                 out = self._qemu_qga_exec(u"guest-ping")
 613             except ValueError:
 614                 logger.trace(f"QGA guest-ping unexpected output {out}")
 615             # Empty output - VM not booted yet.
 616             if not out:
 617                 sleep(5)
 618             # Non-error return - VM booted.
 619             elif out.get(u"return") is not None:
 620                 break
 621             # Skip error and wait.
 622             elif out.get(u"error") is not None:
 623                 sleep(5)
 624             else:
 625                 # If there is an unexpected output from QGA guest-info, try
 626                 # again until timeout.
 627                 logger.trace(f"QGA guest-ping unexpected output {out}")
 628         else:
 629             raise RuntimeError(
 630                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 631             )
 632
 633     def _update_vm_interfaces(self):
 634         """Update interface names in VM node dict."""
 635         # Send guest-network-get-interfaces command via QGA, output example:
 636         # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
 637         # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
 638         out = self._qemu_qga_exec(u"guest-network-get-interfaces")
 639         interfaces = out.get(u"return")
 640         mac_name = {}
 641         if not interfaces:
 642             raise RuntimeError(
 643                 f"Get VM interface list failed on {self._node[u'host']}"
 644             )
 645         # Create MAC-name dict.
 646         for interface in interfaces:
 647             if u"hardware-address" not in interface:
 648                 continue
 649             mac_name[interface[u"hardware-address"]] = interface[u"name"]
 650         # Match interface by MAC and save interface name.
 651         for interface in self._vm_info[u"interfaces"].values():
 652             mac = interface.get(u"mac_address")
 653             if_name = mac_name.get(mac)
 654             if if_name is None:
 655                 logger.trace(f"Interface name for MAC {mac} not found")
 656             else:
 657                 interface[u"name"] = if_name
 658
 659     def qemu_start(self):
 660         """Start QEMU and wait until VM boot.
 661
 662         :returns: VM node info.
 663         :rtype: dict
 664         """
 665         cmd_opts = OptionString()
 666         cmd_opts.add(f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch}")
 667         cmd_opts.extend(self._params)
 668         message = f"QEMU: Start failed on {self._node[u'host']}!"
 669         try:
 670             DUTSetup.check_huge_page(
 671                 self._node, u"/dev/hugepages", int(self._opt.get(u"mem")))
 672
 673             exec_cmd_no_error(
 674                 self._node, cmd_opts, timeout=300, sudo=True, message=message
 675             )
 676             self._wait_until_vm_boot()
 677         except RuntimeError:
 678             self.qemu_kill_all()
 679             raise
 680         return self._vm_info
 681
 682     def qemu_kill(self):
 683         """Kill qemu process."""
 684         exec_cmd(
 685             self._node, f"chmod +r {self._temp.get(u'pidfile')}", sudo=True
 686         )
 687         exec_cmd(
 688             self._node, f"kill -SIGKILL $(cat {self._temp.get(u'pidfile')})",
 689             sudo=True
 690         )
 691
 692         for value in self._temp.values():
 693             exec_cmd(self._node, f"cat {value}", sudo=True)
 694             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 695
 696     def qemu_kill_all(self):
 697         """Kill all qemu processes on DUT node if specified."""
 698         exec_cmd(self._node, u"pkill -SIGKILL qemu", sudo=True)
 699
 700         for value in self._temp.values():
 701             exec_cmd(self._node, f"cat {value}", sudo=True)
 702             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 703
 704     def qemu_version(self):
 705         """Return Qemu version.
 706
 707         :returns: Qemu version.
 708         :rtype: str
 709         """
 710         command = f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch} " \
 711             f"--version"
 712         try:
 713             stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 714             return match(r"QEMU emulator version ([\d.]*)", stdout).group(1)
 715         except RuntimeError:
 716             self.qemu_kill_all()
 717             raise