resources/libraries/python/QemuUtils.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """QEMU utilities library."""
  15
  16 import json
  17
  18 from re import match
  19 from string import Template
  20 from time import sleep
  21
  22 from robot.api import logger
  23
  24 from resources.libraries.python.Constants import Constants
  25 from resources.libraries.python.DpdkUtil import DpdkUtil
  26 from resources.libraries.python.DUTSetup import DUTSetup
  27 from resources.libraries.python.OptionString import OptionString
  28 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
  29 from resources.libraries.python.topology import NodeType, Topology
  30 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
  31 from resources.libraries.python.VPPUtil import VPPUtil
  32
  33 __all__ = [u"QemuUtils"]
  34
  35
  36 class QemuUtils:
  37     """QEMU utilities."""
  38
  39     # Use one instance of class per tests.
  40     ROBOT_LIBRARY_SCOPE = u"TEST CASE"
  41
  42     def __init__(
  43             self, node, qemu_id=1, smp=1, mem=512, vnf=None,
  44             img=Constants.QEMU_VM_IMAGE):
  45         """Initialize QemuUtil class.
  46
  47         :param node: Node to run QEMU on.
  48         :param qemu_id: QEMU identifier.
  49         :param smp: Number of virtual SMP units (cores).
  50         :param mem: Amount of memory.
  51         :param vnf: Network function workload.
  52         :param img: QEMU disk image or kernel image path.
  53         :type node: dict
  54         :type qemu_id: int
  55         :type smp: int
  56         :type mem: int
  57         :type vnf: str
  58         :type img: str
  59         """
  60         self._vhost_id = 0
  61         self._node = node
  62         self._arch = Topology.get_node_arch(self._node)
  63         self._opt = dict()
  64
  65         # Architecture specific options
  66         if self._arch == u"aarch64":
  67             dpdk_target = u"arm64-armv8a"
  68             self._opt[u"machine_args"] = \
  69                 u"virt,accel=kvm,usb=off,mem-merge=off,gic-version=3"
  70             self._opt[u"console"] = u"ttyAMA0"
  71         else:
  72             dpdk_target = u"x86_64-native"
  73             self._opt[u"machine_args"] = u"pc,accel=kvm,usb=off,mem-merge=off"
  74             self._opt[u"console"] = u"ttyS0"
  75         self._testpmd_path = f"{Constants.QEMU_VM_DPDK}/" \
  76             f"{dpdk_target}-linux-gcc/app"
  77         self._vm_info = {
  78             u"host": node[u"host"],
  79             u"type": NodeType.VM,
  80             u"port": 10021 + qemu_id,
  81             u"serial": 4555 + qemu_id,
  82             u"username": 'cisco',
  83             u"password": 'cisco',
  84             u"interfaces": {},
  85         }
  86         if node[u"port"] != 22:
  87             self._vm_info[u"host_port"] = node[u"port"]
  88             self._vm_info[u"host_username"] = node[u"username"]
  89             self._vm_info[u"host_password"] = node[u"password"]
  90         # Input Options.
  91         self._opt[u"qemu_id"] = qemu_id
  92         self._opt[u"mem"] = int(mem)
  93         self._opt[u"smp"] = int(smp)
  94         self._opt[u"img"] = img
  95         self._opt[u"vnf"] = vnf
  96         # Temporary files.
  97         self._temp = dict()
  98         self._temp[u"pidfile"] = f"/run/qemu_{qemu_id}.pid"
  99         if img == Constants.QEMU_VM_IMAGE:
 100             self._opt[u"vm_type"] = u"nestedvm"
 101             self._temp[u"qmp"] = f"/run/qmp_{qemu_id}.sock"
 102             self._temp[u"qga"] = f"/run/qga_{qemu_id}.sock"
 103         elif img == Constants.QEMU_VM_KERNEL:
 104             self._opt[u"img"], _ = exec_cmd_no_error(
 105                 node, f"ls -1 {Constants.QEMU_VM_KERNEL}* | tail -1",
 106                 message=u"Qemu Kernel VM image not found!"
 107             )
 108             self._opt[u"vm_type"] = u"kernelvm"
 109             self._temp[u"log"] = f"/tmp/serial_{qemu_id}.log"
 110             self._temp[u"ini"] = f"/etc/vm_init_{qemu_id}.conf"
 111             self._opt[u"initrd"], _ = exec_cmd_no_error(
 112                 node, f"ls -1 {Constants.QEMU_VM_KERNEL_INITRD}* | tail -1",
 113                 message=u"Qemu Kernel initrd image not found!"
 114             )
 115         else:
 116             raise RuntimeError(f"QEMU: Unknown VM image option: {img}")
 117         # Computed parameters for QEMU command line.
 118         self._params = OptionString(prefix=u"-")
 119         self.add_params()
 120
 121     def add_params(self):
 122         """Set QEMU command line parameters."""
 123         self.add_default_params()
 124         if self._opt.get(u"vm_type", u"") == u"nestedvm":
 125             self.add_nestedvm_params()
 126         elif self._opt.get(u"vm_type", u"") == u"kernelvm":
 127             self.add_kernelvm_params()
 128         else:
 129             raise RuntimeError(u"QEMU: Unsupported VM type!")
 130
 131     def add_default_params(self):
 132         """Set default QEMU command line parameters."""
 133         self._params.add(u"daemonize")
 134         self._params.add(u"nodefaults")
 135         self._params.add_with_value(
 136             u"name", f"vnf{self._opt.get(u'qemu_id')},debug-threads=on"
 137         )
 138         self._params.add(u"no-user-config")
 139         self._params.add_with_value(u"monitor", u"none")
 140         self._params.add_with_value(u"display", u"none")
 141         self._params.add_with_value(u"vga", u"none")
 142         self._params.add(u"enable-kvm")
 143         self._params.add_with_value(u"pidfile", self._temp.get(u"pidfile"))
 144         self._params.add_with_value(u"cpu", u"host")
 145
 146         self._params.add_with_value(u"machine", self._opt.get(u"machine_args"))
 147         self._params.add_with_value(
 148             u"smp", f"{self._opt.get(u'smp')},sockets=1,"
 149             f"cores={self._opt.get(u'smp')},threads=1"
 150         )
 151         self._params.add_with_value(
 152             u"object", f"memory-backend-file,id=mem,"
 153             f"size={self._opt.get(u'mem')}M,mem-path=/dev/hugepages,share=on"
 154         )
 155         self._params.add_with_value(u"m", f"{self._opt.get(u'mem')}M")
 156         self._params.add_with_value(u"numa", u"node,memdev=mem")
 157         self._params.add_with_value(u"balloon", u"none")
 158
 159     def add_nestedvm_params(self):
 160         """Set NestedVM QEMU parameters."""
 161         self._params.add_with_value(
 162             u"net",
 163             f"nic,macaddr=52:54:00:00:{self._opt.get(u'qemu_id'):02x}:ff"
 164         )
 165         self._params.add_with_value(
 166             u"net", f"user,hostfwd=tcp::{self._vm_info[u'port']}-:22"
 167         )
 168         locking = u",file.locking=off"
 169         self._params.add_with_value(
 170             u"drive", f"file={self._opt.get(u'img')},"
 171             f"format=raw,cache=none,if=virtio{locking}"
 172         )
 173         self._params.add_with_value(
 174             u"qmp", f"unix:{self._temp.get(u'qmp')},server,nowait"
 175         )
 176         self._params.add_with_value(
 177             u"chardev", f"socket,host=127.0.0.1,"
 178             f"port={self._vm_info[u'serial']},id=gnc0,server,nowait")
 179         self._params.add_with_value(u"device", u"isa-serial,chardev=gnc0")
 180         self._params.add_with_value(
 181             u"chardev", f"socket,path={self._temp.get(u'qga')},"
 182             f"server,nowait,id=qga0"
 183         )
 184         self._params.add_with_value(u"device", u"isa-serial,chardev=qga0")
 185
 186     def add_kernelvm_params(self):
 187         """Set KernelVM QEMU parameters."""
 188         self._params.add_with_value(
 189             u"serial", f"file:{self._temp.get(u'log')}"
 190         )
 191         self._params.add_with_value(
 192             u"fsdev", u"local,id=root9p,path=/,security_model=none"
 193         )
 194         self._params.add_with_value(
 195             u"device", u"virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot"
 196         )
 197         self._params.add_with_value(u"kernel", f"{self._opt.get(u'img')}")
 198         self._params.add_with_value(u"initrd", f"{self._opt.get(u'initrd')}")
 199         self._params.add_with_value(
 200             u"append", f"'ro rootfstype=9p rootflags=trans=virtio "
 201             f"root=virtioroot console={self._opt.get(u'console')} "
 202             f"tsc=reliable hugepages=256 "
 203             f"init={self._temp.get(u'ini')} fastboot'"
 204         )
 205
 206     def create_kernelvm_config_vpp(self, **kwargs):
 207         """Create QEMU VPP config files.
 208
 209         :param kwargs: Key-value pairs to replace content of VPP configuration
 210             file.
 211         :type kwargs: dict
 212         """
 213         startup = f"/etc/vpp/vm_startup_{self._opt.get(u'qemu_id')}.conf"
 214         running = f"/etc/vpp/vm_running_{self._opt.get(u'qemu_id')}.exec"
 215
 216         self._temp[u"startup"] = startup
 217         self._temp[u"running"] = running
 218         self._opt[u"vnf_bin"] = f"/usr/bin/vpp -c {startup}"
 219
 220         # Create VPP startup configuration.
 221         vpp_config = VppConfigGenerator()
 222         vpp_config.set_node(self._node)
 223         vpp_config.add_unix_nodaemon()
 224         vpp_config.add_unix_cli_listen()
 225         vpp_config.add_unix_exec(running)
 226         vpp_config.add_socksvr()
 227         vpp_config.add_cpu_main_core(u"0")
 228         if self._opt.get(u"smp") > 1:
 229             vpp_config.add_cpu_corelist_workers(f"1-{self._opt.get(u'smp')-1}")
 230         vpp_config.add_plugin(u"disable", u"default")
 231         if "virtio" not in self._opt.get(u'vnf'):
 232             vpp_config.add_plugin(u"enable", u"dpdk_plugin.so")
 233             vpp_config.add_dpdk_dev(u"0000:00:06.0", u"0000:00:07.0")
 234             vpp_config.add_dpdk_dev_default_rxq(kwargs[u"queues"])
 235             vpp_config.add_dpdk_log_level(u"debug")
 236             if not kwargs[u"jumbo_frames"]:
 237                 vpp_config.add_dpdk_no_multi_seg()
 238                 vpp_config.add_dpdk_no_tx_checksum_offload()
 239         vpp_config.write_config(startup)
 240
 241         # Create VPP running configuration.
 242         template = f"{Constants.RESOURCES_TPL_VM}/{self._opt.get(u'vnf')}.exec"
 243         exec_cmd_no_error(self._node, f"rm -f {running}", sudo=True)
 244
 245         with open(template, u"rt") as src_file:
 246             src = Template(src_file.read())
 247             exec_cmd_no_error(
 248                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 249                 f"sudo tee {running}"
 250             )
 251
 252     def create_kernelvm_config_testpmd_io(self, **kwargs):
 253         """Create QEMU testpmd-io command line.
 254
 255         :param kwargs: Key-value pairs to construct command line parameters.
 256         :type kwargs: dict
 257         """
 258         pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
 259         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 260             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 261             eal_driver=False,
 262             eal_pci_whitelist0=u"0000:00:06.0",
 263             eal_pci_whitelist1=u"0000:00:07.0",
 264             eal_in_memory=True,
 265             pmd_num_mbufs=16384,
 266             pmd_fwd_mode=u"io",
 267             pmd_nb_ports=u"2",
 268             pmd_portmask=u"0x3",
 269             pmd_max_pkt_len=pmd_max_pkt_len,
 270             pmd_mbuf_size=u"16384",
 271             pmd_rxq=kwargs[u"queues"],
 272             pmd_txq=kwargs[u"queues"],
 273             pmd_tx_offloads='0x0',
 274             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 275         )
 276
 277         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 278
 279     def create_kernelvm_config_testpmd_mac(self, **kwargs):
 280         """Create QEMU testpmd-mac command line.
 281
 282         :param kwargs: Key-value pairs to construct command line parameters.
 283         :type kwargs: dict
 284         """
 285         pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
 286         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 287             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 288             eal_driver=False,
 289             eal_pci_whitelist0=u"0000:00:06.0",
 290             eal_pci_whitelist1=u"0000:00:07.0",
 291             eal_in_memory=True,
 292             pmd_num_mbufs=16384,
 293             pmd_fwd_mode=u"mac",
 294             pmd_nb_ports=u"2",
 295             pmd_portmask=u"0x3",
 296             pmd_max_pkt_len=pmd_max_pkt_len,
 297             pmd_mbuf_size=u"16384",
 298             pmd_eth_peer_0=f"0,{kwargs[u'vif1_mac']}",
 299             pmd_eth_peer_1=f"1,{kwargs[u'vif2_mac']}",
 300             pmd_rxq=kwargs[u"queues"],
 301             pmd_txq=kwargs[u"queues"],
 302             pmd_tx_offloads=u"0x0",
 303             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 304         )
 305
 306         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 307
 308     def create_kernelvm_init(self, **kwargs):
 309         """Create QEMU init script.
 310
 311         :param kwargs: Key-value pairs to replace content of init startup file.
 312         :type kwargs: dict
 313         """
 314         template = f"{Constants.RESOURCES_TPL_VM}/init.sh"
 315         init = self._temp.get(u"ini")
 316         exec_cmd_no_error(self._node, f"rm -f {init}", sudo=True)
 317
 318         with open(template, u"rt") as src_file:
 319             src = Template(src_file.read())
 320             exec_cmd_no_error(
 321                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 322                 f"sudo tee {init}"
 323             )
 324             exec_cmd_no_error(self._node, f"chmod +x {init}", sudo=True)
 325
 326     def configure_kernelvm_vnf(self, **kwargs):
 327         """Create KernelVM VNF configurations.
 328
 329         :param kwargs: Key-value pairs for templating configs.
 330         :type kwargs: dict
 331         """
 332         if u"vpp" in self._opt.get(u"vnf"):
 333             self.create_kernelvm_config_vpp(**kwargs)
 334         elif u"testpmd_io" in self._opt.get(u"vnf"):
 335             self.create_kernelvm_config_testpmd_io(**kwargs)
 336         elif u"testpmd_mac" in self._opt.get(u"vnf"):
 337             self.create_kernelvm_config_testpmd_mac(**kwargs)
 338         else:
 339             raise RuntimeError(u"QEMU: Unsupported VNF!")
 340         self.create_kernelvm_init(vnf_bin=self._opt.get(u"vnf_bin"))
 341
 342     def get_qemu_pids(self):
 343         """Get QEMU CPU pids.
 344
 345         :returns: List of QEMU CPU pids.
 346         :rtype: list of str
 347         """
 348         command = f"grep -rwl 'CPU' /proc/$(sudo cat " \
 349             f"{self._temp.get(u'pidfile')})/task/*/comm "
 350         command += r"| xargs dirname | sed -e 's/\/.*\///g' | uniq"
 351
 352         stdout, _ = exec_cmd_no_error(self._node, command)
 353         return stdout.splitlines()
 354
 355     def qemu_set_affinity(self, *host_cpus):
 356         """Set qemu affinity by getting thread PIDs via QMP and taskset to list
 357         of CPU cores. Function tries to execute 3 times to avoid race condition
 358         in getting thread PIDs.
 359
 360         :param host_cpus: List of CPU cores.
 361         :type host_cpus: list
 362         """
 363         for _ in range(3):
 364             try:
 365                 qemu_cpus = self.get_qemu_pids()
 366
 367                 if len(qemu_cpus) != len(host_cpus):
 368                     sleep(1)
 369                     continue
 370                 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
 371                     command = f"taskset -pc {host_cpu} {qemu_cpu}"
 372                     message = f"QEMU: Set affinity failed " \
 373                         f"on {self._node[u'host']}!"
 374                     exec_cmd_no_error(
 375                         self._node, command, sudo=True, message=message
 376                     )
 377                 break
 378             except (RuntimeError, ValueError):
 379                 self.qemu_kill_all()
 380                 raise
 381         else:
 382             self.qemu_kill_all()
 383             raise RuntimeError(u"Failed to set Qemu threads affinity!")
 384
 385     def qemu_set_scheduler_policy(self):
 386         """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
 387         processes.
 388
 389         :raises RuntimeError: Set scheduler policy failed.
 390         """
 391         try:
 392             qemu_cpus = self.get_qemu_pids()
 393
 394             for qemu_cpu in qemu_cpus:
 395                 command = f"chrt -r -p 1 {qemu_cpu}"
 396                 message = f"QEMU: Set SCHED_RR failed on {self._node[u'host']}"
 397                 exec_cmd_no_error(
 398                     self._node, command, sudo=True, message=message
 399                 )
 400         except (RuntimeError, ValueError):
 401             self.qemu_kill_all()
 402             raise
 403
 404     def qemu_add_vhost_user_if(
 405             self, socket, server=True, jumbo_frames=False, queue_size=None,
 406             queues=1, csum=False, gso=False):
 407         """Add Vhost-user interface.
 408
 409         :param socket: Path of the unix socket.
 410         :param server: If True the socket shall be a listening socket.
 411         :param jumbo_frames: Set True if jumbo frames are used in the test.
 412         :param queue_size: Vring queue size.
 413         :param queues: Number of queues.
 414         :param csum: Checksum offloading.
 415         :param gso: Generic segmentation offloading.
 416         :type socket: str
 417         :type server: bool
 418         :type jumbo_frames: bool
 419         :type queue_size: int
 420         :type queues: int
 421         :type csum: bool
 422         :type gso: bool
 423         """
 424         self._vhost_id += 1
 425         self._params.add_with_value(
 426             u"chardev", f"socket,id=char{self._vhost_id},"
 427             f"path={socket}{u',server' if server is True else u''}"
 428         )
 429         self._params.add_with_value(
 430             u"netdev", f"vhost-user,id=vhost{self._vhost_id},"
 431             f"chardev=char{self._vhost_id},queues={queues}"
 432         )
 433         mac = f"52:54:00:00:{self._opt.get(u'qemu_id'):02x}:" \
 434             f"{self._vhost_id:02x}"
 435         queue_size = f"rx_queue_size={queue_size},tx_queue_size={queue_size}" \
 436             if queue_size else u""
 437         self._params.add_with_value(
 438             u"device", f"virtio-net-pci,netdev=vhost{self._vhost_id},mac={mac},"
 439             f"addr={self._vhost_id+5}.0,mq=on,vectors={2 * queues + 2},"
 440             f"csum={u'on' if csum else u'off'},gso={u'on' if gso else u'off'},"
 441             f"guest_tso4=off,guest_tso6=off,guest_ecn=off,"
 442             f"{queue_size}"
 443         )
 444
 445         # Add interface MAC and socket to the node dict.
 446         if_data = {u"mac_address": mac, u"socket": socket}
 447         if_name = f"vhost{self._vhost_id}"
 448         self._vm_info[u"interfaces"][if_name] = if_data
 449         # Add socket to temporary file list.
 450         self._temp[if_name] = socket
 451
 452     def _qemu_qmp_exec(self, cmd):
 453         """Execute QMP command.
 454
 455         QMP is JSON based protocol which allows to control QEMU instance.
 456
 457         :param cmd: QMP command to execute.
 458         :type cmd: str
 459         :returns: Command output in python representation of JSON format. The
 460             { "return": {} } response is QMP's success response. An error
 461             response will contain the "error" keyword instead of "return".
 462         """
 463         # To enter command mode, the qmp_capabilities command must be issued.
 464         command = f"echo \"{{{{ \\\"execute\\\": " \
 465             f"\\\"qmp_capabilities\\\" }}}}" \
 466             f"{{{{ \\\"execute\\\": \\\"{cmd}\\\" }}}}\" | " \
 467             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qmp')}"
 468         message = f"QMP execute '{cmd}' failed on {self._node[u'host']}"
 469
 470         stdout, _ = exec_cmd_no_error(
 471             self._node, command, sudo=False, message=message
 472         )
 473
 474         # Skip capabilities negotiation messages.
 475         out_list = stdout.splitlines()
 476         if len(out_list) < 3:
 477             raise RuntimeError(f"Invalid QMP output on {self._node[u'host']}")
 478         return json.loads(out_list[2])
 479
 480     def _qemu_qga_flush(self):
 481         """Flush the QGA parser state."""
 482         command = f"(printf \"\xFF\"; sleep 1) | sudo -S socat " \
 483             f"- UNIX-CONNECT:{self._temp.get(u'qga')}"
 484         message = f"QGA flush failed on {self._node[u'host']}"
 485         stdout, _ = exec_cmd_no_error(
 486             self._node, command, sudo=False, message=message
 487         )
 488
 489         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 490
 491     def _qemu_qga_exec(self, cmd):
 492         """Execute QGA command.
 493
 494         QGA provide access to a system-level agent via standard QMP commands.
 495
 496         :param cmd: QGA command to execute.
 497         :type cmd: str
 498         """
 499         command = f"(echo \"{{{{ \\\"execute\\\": " \
 500             f"\\\"{cmd}\\\" }}}}\"; sleep 1) | " \
 501             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qga')}"
 502         message = f"QGA execute '{cmd}' failed on {self._node[u'host']}"
 503         stdout, _ = exec_cmd_no_error(
 504             self._node, command, sudo=False, message=message
 505         )
 506
 507         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 508
 509     def _wait_until_vm_boot(self):
 510         """Wait until QEMU with NestedVM is booted."""
 511         if self._opt.get(u"vm_type") == u"nestedvm":
 512             self._wait_until_nestedvm_boot()
 513             self._update_vm_interfaces()
 514         elif self._opt.get(u"vm_type") == u"kernelvm":
 515             self._wait_until_kernelvm_boot()
 516         else:
 517             raise RuntimeError(u"QEMU: Unsupported VM type!")
 518
 519     def _wait_until_nestedvm_boot(self, retries=12):
 520         """Wait until QEMU with NestedVM is booted.
 521
 522         First try to flush qga until there is output.
 523         Then ping QEMU guest agent each 5s until VM booted or timeout.
 524
 525         :param retries: Number of retries with 5s between trials.
 526         :type retries: int
 527         """
 528         for _ in range(retries):
 529             out = None
 530             try:
 531                 out = self._qemu_qga_flush()
 532             except ValueError:
 533                 logger.trace(f"QGA qga flush unexpected output {out}")
 534             # Empty output - VM not booted yet
 535             if not out:
 536                 sleep(5)
 537             else:
 538                 break
 539         else:
 540             raise RuntimeError(
 541                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 542             )
 543         for _ in range(retries):
 544             out = None
 545             try:
 546                 out = self._qemu_qga_exec(u"guest-ping")
 547             except ValueError:
 548                 logger.trace(f"QGA guest-ping unexpected output {out}")
 549             # Empty output - VM not booted yet.
 550             if not out:
 551                 sleep(5)
 552             # Non-error return - VM booted.
 553             elif out.get(u"return") is not None:
 554                 break
 555             # Skip error and wait.
 556             elif out.get(u"error") is not None:
 557                 sleep(5)
 558             else:
 559                 # If there is an unexpected output from QGA guest-info, try
 560                 # again until timeout.
 561                 logger.trace(f"QGA guest-ping unexpected output {out}")
 562         else:
 563             raise RuntimeError(
 564                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 565             )
 566
 567     def _wait_until_kernelvm_boot(self, retries=60):
 568         """Wait until QEMU KernelVM is booted.
 569
 570         :param retries: Number of retries.
 571         :type retries: int
 572         """
 573         vpp_ver = VPPUtil.vpp_show_version(self._node)
 574
 575         for _ in range(retries):
 576             command = f"tail -1 {self._temp.get(u'log')}"
 577             stdout = None
 578             try:
 579                 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 580                 sleep(1)
 581             except RuntimeError:
 582                 pass
 583             if vpp_ver in stdout or u"Press enter to exit" in stdout:
 584                 break
 585             if u"reboot: Power down" in stdout:
 586                 raise RuntimeError(
 587                     f"QEMU: NF failed to run on {self._node[u'host']}!"
 588                 )
 589         else:
 590             raise RuntimeError(
 591                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 592             )
 593
 594     def _update_vm_interfaces(self):
 595         """Update interface names in VM node dict."""
 596         # Send guest-network-get-interfaces command via QGA, output example:
 597         # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
 598         # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
 599         out = self._qemu_qga_exec(u"guest-network-get-interfaces")
 600         interfaces = out.get(u"return")
 601         mac_name = {}
 602         if not interfaces:
 603             raise RuntimeError(
 604                 f"Get VM interface list failed on {self._node[u'host']}"
 605             )
 606         # Create MAC-name dict.
 607         for interface in interfaces:
 608             if u"hardware-address" not in interface:
 609                 continue
 610             mac_name[interface[u"hardware-address"]] = interface[u"name"]
 611         # Match interface by MAC and save interface name.
 612         for interface in self._vm_info[u"interfaces"].values():
 613             mac = interface.get(u"mac_address")
 614             if_name = mac_name.get(mac)
 615             if if_name is None:
 616                 logger.trace(f"Interface name for MAC {mac} not found")
 617             else:
 618                 interface[u"name"] = if_name
 619
 620     def qemu_start(self):
 621         """Start QEMU and wait until VM boot.
 622
 623         :returns: VM node info.
 624         :rtype: dict
 625         """
 626         cmd_opts = OptionString()
 627         cmd_opts.add(f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch}")
 628         cmd_opts.extend(self._params)
 629         message = f"QEMU: Start failed on {self._node[u'host']}!"
 630         try:
 631             DUTSetup.check_huge_page(
 632                 self._node, u"/dev/hugepages", int(self._opt.get(u"mem")))
 633
 634             exec_cmd_no_error(
 635                 self._node, cmd_opts, timeout=300, sudo=True, message=message
 636             )
 637             self._wait_until_vm_boot()
 638         except RuntimeError:
 639             self.qemu_kill_all()
 640             raise
 641         return self._vm_info
 642
 643     def qemu_kill(self):
 644         """Kill qemu process."""
 645         exec_cmd(
 646             self._node, f"chmod +r {self._temp.get(u'pidfile')}", sudo=True
 647         )
 648         exec_cmd(
 649             self._node, f"kill -SIGKILL $(cat {self._temp.get(u'pidfile')})",
 650             sudo=True
 651         )
 652
 653         for value in self._temp.values():
 654             exec_cmd(self._node, f"cat {value}", sudo=True)
 655             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 656
 657     def qemu_kill_all(self):
 658         """Kill all qemu processes on DUT node if specified."""
 659         exec_cmd(self._node, u"pkill -SIGKILL qemu", sudo=True)
 660
 661         for value in self._temp.values():
 662             exec_cmd(self._node, f"cat {value}", sudo=True)
 663             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 664
 665     def qemu_version(self):
 666         """Return Qemu version.
 667
 668         :returns: Qemu version.
 669         :rtype: str
 670         """
 671         command = f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch} " \
 672             f"--version"
 673         try:
 674             stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 675             return match(r"QEMU emulator version ([\d.]*)", stdout).group(1)
 676         except RuntimeError:
 677             self.qemu_kill_all()
 678             raise