resources/libraries/python/QemuUtils.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """QEMU utilities library."""
  15
  16 import json
  17
  18 from re import match
  19 from string import Template
  20 from time import sleep
  21
  22 from robot.api import logger
  23
  24 from resources.libraries.python.Constants import Constants
  25 from resources.libraries.python.DpdkUtil import DpdkUtil
  26 from resources.libraries.python.DUTSetup import DUTSetup
  27 from resources.libraries.python.OptionString import OptionString
  28 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
  29 from resources.libraries.python.topology import NodeType, Topology
  30 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
  31 from resources.libraries.python.VPPUtil import VPPUtil
  32
  33 __all__ = [u"QemuUtils"]
  34
  35
  36 class QemuUtils:
  37     """QEMU utilities."""
  38
  39     # Use one instance of class per tests.
  40     ROBOT_LIBRARY_SCOPE = u"TEST CASE"
  41
  42     def __init__(
  43             self, node, qemu_id=1, smp=1, mem=512, vnf=None,
  44             img=Constants.QEMU_VM_IMAGE):
  45         """Initialize QemuUtil class.
  46
  47         :param node: Node to run QEMU on.
  48         :param qemu_id: QEMU identifier.
  49         :param smp: Number of virtual SMP units (cores).
  50         :param mem: Amount of memory.
  51         :param vnf: Network function workload.
  52         :param img: QEMU disk image or kernel image path.
  53         :type node: dict
  54         :type qemu_id: int
  55         :type smp: int
  56         :type mem: int
  57         :type vnf: str
  58         :type img: str
  59         """
  60         self._vhost_id = 0
  61         self._node = node
  62         self._arch = Topology.get_node_arch(self._node)
  63         self._opt = dict()
  64
  65         # Architecture specific options
  66         if self._arch == u"aarch64":
  67             dpdk_target = u"arm64-armv8a"
  68             self._opt[u"machine_args"] = \
  69                 u"virt,accel=kvm,usb=off,mem-merge=off,gic-version=3"
  70             self._opt[u"console"] = u"ttyAMA0"
  71             self._opt[u"unsafe_iommu"] = u"echo Y > /sys/module/vfio/para" \
  72                                          u"meters/enable_unsafe_noiommu_mode"
  73         else:
  74             dpdk_target = u"x86_64-native"
  75             self._opt[u"machine_args"] = u"pc,accel=kvm,usb=off,mem-merge=off"
  76             self._opt[u"console"] = u"ttyS0"
  77             self._opt[u"unsafe_iommu"] = u""
  78
  79         self._testpmd_path = f"{Constants.QEMU_VM_DPDK}/" \
  80             f"{dpdk_target}-linux-gcc/app"
  81         self._vm_info = {
  82             u"host": node[u"host"],
  83             u"type": NodeType.VM,
  84             u"port": 10021 + qemu_id,
  85             u"serial": 4555 + qemu_id,
  86             u"username": 'cisco',
  87             u"password": 'cisco',
  88             u"interfaces": {},
  89         }
  90         if node[u"port"] != 22:
  91             self._vm_info[u"host_port"] = node[u"port"]
  92             self._vm_info[u"host_username"] = node[u"username"]
  93             self._vm_info[u"host_password"] = node[u"password"]
  94         # Input Options.
  95         self._opt[u"qemu_id"] = qemu_id
  96         self._opt[u"mem"] = int(mem)
  97         self._opt[u"smp"] = int(smp)
  98         self._opt[u"img"] = img
  99         self._opt[u"vnf"] = vnf
 100         # Temporary files.
 101         self._temp = dict()
 102         self._temp[u"pidfile"] = f"/var/run/qemu_{qemu_id}.pid"
 103         if img == Constants.QEMU_VM_IMAGE:
 104             self._opt[u"vm_type"] = u"nestedvm"
 105             self._temp[u"qmp"] = f"/var/run/qmp_{qemu_id}.sock"
 106             self._temp[u"qga"] = f"/var/run/qga_{qemu_id}.sock"
 107         elif img == Constants.QEMU_VM_KERNEL:
 108             self._opt[u"img"], _ = exec_cmd_no_error(
 109                 node, f"ls -1 {Constants.QEMU_VM_KERNEL}* | tail -1",
 110                 message=u"Qemu Kernel VM image not found!"
 111             )
 112             self._opt[u"vm_type"] = u"kernelvm"
 113             self._temp[u"log"] = f"/tmp/serial_{qemu_id}.log"
 114             self._temp[u"ini"] = f"/etc/vm_init_{qemu_id}.conf"
 115             self._opt[u"initrd"], _ = exec_cmd_no_error(
 116                 node, f"ls -1 {Constants.QEMU_VM_KERNEL_INITRD}* | tail -1",
 117                 message=u"Qemu Kernel initrd image not found!"
 118             )
 119         else:
 120             raise RuntimeError(f"QEMU: Unknown VM image option: {img}")
 121         # Computed parameters for QEMU command line.
 122         self._params = OptionString(prefix=u"-")
 123         self.add_params()
 124
 125     def add_params(self):
 126         """Set QEMU command line parameters."""
 127         self.add_default_params()
 128         if self._opt.get(u"vm_type", u"") == u"nestedvm":
 129             self.add_nestedvm_params()
 130         elif self._opt.get(u"vm_type", u"") == u"kernelvm":
 131             self.add_kernelvm_params()
 132         else:
 133             raise RuntimeError(u"QEMU: Unsupported VM type!")
 134
 135     def add_default_params(self):
 136         """Set default QEMU command line parameters."""
 137         self._params.add(u"daemonize")
 138         self._params.add(u"nodefaults")
 139         self._params.add_with_value(
 140             u"name", f"vnf{self._opt.get(u'qemu_id')},debug-threads=on"
 141         )
 142         self._params.add(u"no-user-config")
 143         self._params.add_with_value(u"monitor", u"none")
 144         self._params.add_with_value(u"display", u"none")
 145         self._params.add_with_value(u"vga", u"none")
 146         self._params.add(u"enable-kvm")
 147         self._params.add_with_value(u"pidfile", self._temp.get(u"pidfile"))
 148         self._params.add_with_value(u"cpu", u"host")
 149
 150         self._params.add_with_value(u"machine", self._opt.get(u"machine_args"))
 151         self._params.add_with_value(
 152             u"smp", f"{self._opt.get(u'smp')},sockets=1,"
 153             f"cores={self._opt.get(u'smp')},threads=1"
 154         )
 155         self._params.add_with_value(
 156             u"object", f"memory-backend-file,id=mem,"
 157             f"size={self._opt.get(u'mem')}M,mem-path=/dev/hugepages,share=on"
 158         )
 159         self._params.add_with_value(u"m", f"{self._opt.get(u'mem')}M")
 160         self._params.add_with_value(u"numa", u"node,memdev=mem")
 161         self._params.add_with_value(u"balloon", u"none")
 162
 163     def add_nestedvm_params(self):
 164         """Set NestedVM QEMU parameters."""
 165         self._params.add_with_value(
 166             u"net",
 167             f"nic,macaddr=52:54:00:00:{self._opt.get(u'qemu_id'):02x}:ff"
 168         )
 169         self._params.add_with_value(
 170             u"net", f"user,hostfwd=tcp::{self._vm_info[u'port']}-:22"
 171         )
 172         locking = u",file.locking=off"
 173         self._params.add_with_value(
 174             u"drive", f"file={self._opt.get(u'img')},"
 175             f"format=raw,cache=none,if=virtio{locking}"
 176         )
 177         self._params.add_with_value(
 178             u"qmp", f"unix:{self._temp.get(u'qmp')},server,nowait"
 179         )
 180         self._params.add_with_value(
 181             u"chardev", f"socket,host=127.0.0.1,"
 182             f"port={self._vm_info[u'serial']},id=gnc0,server,nowait")
 183         self._params.add_with_value(u"device", u"isa-serial,chardev=gnc0")
 184         self._params.add_with_value(
 185             u"chardev", f"socket,path={self._temp.get(u'qga')},"
 186             f"server,nowait,id=qga0"
 187         )
 188         self._params.add_with_value(u"device", u"isa-serial,chardev=qga0")
 189
 190     def add_kernelvm_params(self):
 191         """Set KernelVM QEMU parameters."""
 192         self._params.add_with_value(
 193             u"serial", f"file:{self._temp.get(u'log')}"
 194         )
 195         self._params.add_with_value(
 196             u"fsdev", u"local,id=root9p,path=/,security_model=none"
 197         )
 198         self._params.add_with_value(
 199             u"device", u"virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot"
 200         )
 201         self._params.add_with_value(u"kernel", f"{self._opt.get(u'img')}")
 202         self._params.add_with_value(u"initrd", f"{self._opt.get(u'initrd')}")
 203         self._params.add_with_value(
 204             u"append", f"'ro rootfstype=9p rootflags=trans=virtio "
 205             f"root=virtioroot console={self._opt.get(u'console')} "
 206             f"tsc=reliable hugepages=256 "
 207             f"init={self._temp.get(u'ini')} fastboot'"
 208         )
 209
 210     def create_kernelvm_config_vpp(self, **kwargs):
 211         """Create QEMU VPP config files.
 212
 213         :param kwargs: Key-value pairs to replace content of VPP configuration
 214             file.
 215         :type kwargs: dict
 216         """
 217         startup = f"/etc/vpp/vm_startup_{self._opt.get(u'qemu_id')}.conf"
 218         running = f"/etc/vpp/vm_running_{self._opt.get(u'qemu_id')}.exec"
 219
 220         self._temp[u"startup"] = startup
 221         self._temp[u"running"] = running
 222         self._opt[u"vnf_bin"] = f"/usr/bin/vpp -c {startup}"
 223
 224         # Create VPP startup configuration.
 225         vpp_config = VppConfigGenerator()
 226         vpp_config.set_node(self._node)
 227         vpp_config.add_unix_nodaemon()
 228         vpp_config.add_unix_cli_listen()
 229         vpp_config.add_unix_exec(running)
 230         vpp_config.add_socksvr()
 231         vpp_config.add_cpu_main_core(u"0")
 232         if self._opt.get(u"smp") > 1:
 233             vpp_config.add_cpu_corelist_workers(f"1-{self._opt.get(u'smp')-1}")
 234         vpp_config.add_dpdk_dev(u"0000:00:06.0", u"0000:00:07.0")
 235         vpp_config.add_dpdk_dev_default_rxq(kwargs[u"queues"])
 236         vpp_config.add_dpdk_log_level(u"debug")
 237         if not kwargs[u"jumbo_frames"]:
 238             vpp_config.add_dpdk_no_multi_seg()
 239             vpp_config.add_dpdk_no_tx_checksum_offload()
 240         vpp_config.add_plugin(u"disable", [u"default"])
 241         vpp_config.add_plugin(u"enable", [u"dpdk_plugin.so"])
 242         vpp_config.write_config(startup)
 243
 244         # Create VPP running configuration.
 245         template = f"{Constants.RESOURCES_TPL_VM}/{self._opt.get(u'vnf')}.exec"
 246         exec_cmd_no_error(self._node, f"rm -f {running}", sudo=True)
 247
 248         with open(template, u"rt") as src_file:
 249             src = Template(src_file.read())
 250             exec_cmd_no_error(
 251                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 252                 f"sudo tee {running}"
 253             )
 254
 255     def create_kernelvm_config_testpmd_io(self, **kwargs):
 256         """Create QEMU testpmd-io command line.
 257
 258         :param kwargs: Key-value pairs to construct command line parameters.
 259         :type kwargs: dict
 260         """
 261         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 262             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 263             eal_driver=False,
 264             eal_in_memory=True,
 265             pmd_num_mbufs=16384,
 266             pmd_rxq=kwargs[u"queues"],
 267             pmd_txq=kwargs[u"queues"],
 268             pmd_tx_offloads='0x0',
 269             pmd_disable_hw_vlan=False,
 270             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 271         )
 272
 273         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 274
 275     def create_kernelvm_config_testpmd_mac(self, **kwargs):
 276         """Create QEMU testpmd-mac command line.
 277
 278         :param kwargs: Key-value pairs to construct command line parameters.
 279         :type kwargs: dict
 280         """
 281         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 282             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 283             eal_driver=False,
 284             eal_in_memory=True,
 285             pmd_num_mbufs=16384,
 286             pmd_fwd_mode=u"mac",
 287             pmd_eth_peer_0=f"0,{kwargs[u'vif1_mac']}",
 288             pmd_eth_peer_1=f"1,{kwargs[u'vif2_mac']}",
 289             pmd_rxq=kwargs[u"queues"],
 290             pmd_txq=kwargs[u"queues"],
 291             pmd_tx_offloads=u"0x0",
 292             pmd_disable_hw_vlan=False,
 293             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 294         )
 295
 296         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 297
 298     def create_kernelvm_init(self, **kwargs):
 299         """Create QEMU init script.
 300
 301         :param kwargs: Key-value pairs to replace content of init startup file.
 302         :type kwargs: dict
 303         """
 304         template = f"{Constants.RESOURCES_TPL_VM}/init.sh"
 305         init = self._temp.get(u"ini")
 306         exec_cmd_no_error(self._node, f"rm -f {init}", sudo=True)
 307
 308         with open(template, u"rt") as src_file:
 309             src = Template(src_file.read())
 310             exec_cmd_no_error(
 311                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 312                 f"sudo tee {init}"
 313             )
 314             exec_cmd_no_error(self._node, f"chmod +x {init}", sudo=True)
 315
 316     def configure_kernelvm_vnf(self, **kwargs):
 317         """Create KernelVM VNF configurations.
 318
 319         :param kwargs: Key-value pairs for templating configs.
 320         :type kwargs: dict
 321         """
 322         if u"vpp" in self._opt.get(u"vnf"):
 323             self.create_kernelvm_config_vpp(**kwargs)
 324         elif u"testpmd_io" in self._opt.get(u"vnf"):
 325             self.create_kernelvm_config_testpmd_io(**kwargs)
 326         elif u"testpmd_mac" in self._opt.get(u"vnf"):
 327             self.create_kernelvm_config_testpmd_mac(**kwargs)
 328         else:
 329             raise RuntimeError(u"QEMU: Unsupported VNF!")
 330         self.create_kernelvm_init(vnf_bin=self._opt.get(u"vnf_bin"),
 331                                   unsafe_iommu=self._opt.get(u"unsafe_iommu"))
 332
 333     def get_qemu_pids(self):
 334         """Get QEMU CPU pids.
 335
 336         :returns: List of QEMU CPU pids.
 337         :rtype: list of str
 338         """
 339         command = f"grep -rwl 'CPU' /proc/$(sudo cat " \
 340             f"{self._temp.get(u'pidfile')})/task/*/comm "
 341         command += r"| xargs dirname | sed -e 's/\/.*\///g' | uniq"
 342
 343         stdout, _ = exec_cmd_no_error(self._node, command)
 344         return stdout.splitlines()
 345
 346     def qemu_set_affinity(self, *host_cpus):
 347         """Set qemu affinity by getting thread PIDs via QMP and taskset to list
 348         of CPU cores. Function tries to execute 3 times to avoid race condition
 349         in getting thread PIDs.
 350
 351         :param host_cpus: List of CPU cores.
 352         :type host_cpus: list
 353         """
 354         for _ in range(3):
 355             try:
 356                 qemu_cpus = self.get_qemu_pids()
 357
 358                 if len(qemu_cpus) != len(host_cpus):
 359                     sleep(1)
 360                     continue
 361                 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
 362                     command = f"taskset -pc {host_cpu} {qemu_cpu}"
 363                     message = f"QEMU: Set affinity failed " \
 364                         f"on {self._node[u'host']}!"
 365                     exec_cmd_no_error(
 366                         self._node, command, sudo=True, message=message
 367                     )
 368                 break
 369             except (RuntimeError, ValueError):
 370                 self.qemu_kill_all()
 371                 raise
 372         else:
 373             self.qemu_kill_all()
 374             raise RuntimeError(u"Failed to set Qemu threads affinity!")
 375
 376     def qemu_set_scheduler_policy(self):
 377         """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
 378         processes.
 379
 380         :raises RuntimeError: Set scheduler policy failed.
 381         """
 382         try:
 383             qemu_cpus = self.get_qemu_pids()
 384
 385             for qemu_cpu in qemu_cpus:
 386                 command = f"chrt -r -p 1 {qemu_cpu}"
 387                 message = f"QEMU: Set SCHED_RR failed on {self._node[u'host']}"
 388                 exec_cmd_no_error(
 389                     self._node, command, sudo=True, message=message
 390                 )
 391         except (RuntimeError, ValueError):
 392             self.qemu_kill_all()
 393             raise
 394
 395     def qemu_add_vhost_user_if(
 396             self, socket, server=True, jumbo_frames=False, queue_size=None,
 397             queues=1, csum=False, gso=False):
 398         """Add Vhost-user interface.
 399
 400         :param socket: Path of the unix socket.
 401         :param server: If True the socket shall be a listening socket.
 402         :param jumbo_frames: Set True if jumbo frames are used in the test.
 403         :param queue_size: Vring queue size.
 404         :param queues: Number of queues.
 405         :param csum: Checksum offloading.
 406         :param gso: Generic segmentation offloading.
 407         :type socket: str
 408         :type server: bool
 409         :type jumbo_frames: bool
 410         :type queue_size: int
 411         :type queues: int
 412         :type csum: bool
 413         :type gso: bool
 414         """
 415         self._vhost_id += 1
 416         self._params.add_with_value(
 417             u"chardev", f"socket,id=char{self._vhost_id},"
 418             f"path={socket}{u',server' if server is True else u''}"
 419         )
 420         self._params.add_with_value(
 421             u"netdev", f"vhost-user,id=vhost{self._vhost_id},"
 422             f"chardev=char{self._vhost_id},queues={queues}"
 423         )
 424         mac = f"52:54:00:00:{self._opt.get(u'qemu_id'):02x}:" \
 425             f"{self._vhost_id:02x}"
 426         queue_size = f"rx_queue_size={queue_size},tx_queue_size={queue_size}" \
 427             if queue_size else u""
 428         self._params.add_with_value(
 429             u"device", f"virtio-net-pci,netdev=vhost{self._vhost_id},mac={mac},"
 430             f"addr={self._vhost_id+5}.0,mq=on,vectors={2 * queues + 2},"
 431             f"csum={u'on' if csum else u'off'},gso={u'on' if gso else u'off'},"
 432             f"guest_tso4=off,guest_tso6=off,guest_ecn=off,"
 433             f"mrg_rxbuf={u'on,host_mtu=9200' if jumbo_frames else u'off'},"
 434             f"{queue_size}"
 435         )
 436
 437         # Add interface MAC and socket to the node dict.
 438         if_data = {u"mac_address": mac, u"socket": socket}
 439         if_name = f"vhost{self._vhost_id}"
 440         self._vm_info[u"interfaces"][if_name] = if_data
 441         # Add socket to temporary file list.
 442         self._temp[if_name] = socket
 443
 444     def _qemu_qmp_exec(self, cmd):
 445         """Execute QMP command.
 446
 447         QMP is JSON based protocol which allows to control QEMU instance.
 448
 449         :param cmd: QMP command to execute.
 450         :type cmd: str
 451         :returns: Command output in python representation of JSON format. The
 452             { "return": {} } response is QMP's success response. An error
 453             response will contain the "error" keyword instead of "return".
 454         """
 455         # To enter command mode, the qmp_capabilities command must be issued.
 456         command = f"echo \"{{{{ \\\"execute\\\": " \
 457             f"\\\"qmp_capabilities\\\" }}}}" \
 458             f"{{{{ \\\"execute\\\": \\\"{cmd}\\\" }}}}\" | " \
 459             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qmp')}"
 460         message = f"QMP execute '{cmd}' failed on {self._node[u'host']}"
 461
 462         stdout, _ = exec_cmd_no_error(
 463             self._node, command, sudo=False, message=message
 464         )
 465
 466         # Skip capabilities negotiation messages.
 467         out_list = stdout.splitlines()
 468         if len(out_list) < 3:
 469             raise RuntimeError(f"Invalid QMP output on {self._node[u'host']}")
 470         return json.loads(out_list[2])
 471
 472     def _qemu_qga_flush(self):
 473         """Flush the QGA parser state."""
 474         command = f"(printf \"\xFF\"; sleep 1) | sudo -S socat " \
 475             f"- UNIX-CONNECT:{self._temp.get(u'qga')}"
 476         message = f"QGA flush failed on {self._node[u'host']}"
 477         stdout, _ = exec_cmd_no_error(
 478             self._node, command, sudo=False, message=message
 479         )
 480
 481         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 482
 483     def _qemu_qga_exec(self, cmd):
 484         """Execute QGA command.
 485
 486         QGA provide access to a system-level agent via standard QMP commands.
 487
 488         :param cmd: QGA command to execute.
 489         :type cmd: str
 490         """
 491         command = f"(echo \"{{{{ \\\"execute\\\": " \
 492             f"\\\"{cmd}\\\" }}}}\"; sleep 1) | " \
 493             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qga')}"
 494         message = f"QGA execute '{cmd}' failed on {self._node[u'host']}"
 495         stdout, _ = exec_cmd_no_error(
 496             self._node, command, sudo=False, message=message
 497         )
 498
 499         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 500
 501     def _wait_until_vm_boot(self):
 502         """Wait until QEMU with NestedVM is booted."""
 503         if self._opt.get(u"vm_type") == u"nestedvm":
 504             self._wait_until_nestedvm_boot()
 505             self._update_vm_interfaces()
 506         elif self._opt.get(u"vm_type") == u"kernelvm":
 507             self._wait_until_kernelvm_boot()
 508         else:
 509             raise RuntimeError(u"QEMU: Unsupported VM type!")
 510
 511     def _wait_until_nestedvm_boot(self, retries=12):
 512         """Wait until QEMU with NestedVM is booted.
 513
 514         First try to flush qga until there is output.
 515         Then ping QEMU guest agent each 5s until VM booted or timeout.
 516
 517         :param retries: Number of retries with 5s between trials.
 518         :type retries: int
 519         """
 520         for _ in range(retries):
 521             out = None
 522             try:
 523                 out = self._qemu_qga_flush()
 524             except ValueError:
 525                 logger.trace(f"QGA qga flush unexpected output {out}")
 526             # Empty output - VM not booted yet
 527             if not out:
 528                 sleep(5)
 529             else:
 530                 break
 531         else:
 532             raise RuntimeError(
 533                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 534             )
 535         for _ in range(retries):
 536             out = None
 537             try:
 538                 out = self._qemu_qga_exec(u"guest-ping")
 539             except ValueError:
 540                 logger.trace(f"QGA guest-ping unexpected output {out}")
 541             # Empty output - VM not booted yet.
 542             if not out:
 543                 sleep(5)
 544             # Non-error return - VM booted.
 545             elif out.get(u"return") is not None:
 546                 break
 547             # Skip error and wait.
 548             elif out.get(u"error") is not None:
 549                 sleep(5)
 550             else:
 551                 # If there is an unexpected output from QGA guest-info, try
 552                 # again until timeout.
 553                 logger.trace(f"QGA guest-ping unexpected output {out}")
 554         else:
 555             raise RuntimeError(
 556                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 557             )
 558
 559     def _wait_until_kernelvm_boot(self, retries=60):
 560         """Wait until QEMU KernelVM is booted.
 561
 562         :param retries: Number of retries.
 563         :type retries: int
 564         """
 565         vpp_ver = VPPUtil.vpp_show_version(self._node)
 566
 567         for _ in range(retries):
 568             command = f"tail -1 {self._temp.get(u'log')}"
 569             stdout = None
 570             try:
 571                 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 572                 sleep(1)
 573             except RuntimeError:
 574                 pass
 575             if vpp_ver in stdout or u"Press enter to exit" in stdout:
 576                 break
 577             if u"reboot: Power down" in stdout:
 578                 raise RuntimeError(
 579                     f"QEMU: NF failed to run on {self._node[u'host']}!"
 580                 )
 581         else:
 582             raise RuntimeError(
 583                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 584             )
 585
 586     def _update_vm_interfaces(self):
 587         """Update interface names in VM node dict."""
 588         # Send guest-network-get-interfaces command via QGA, output example:
 589         # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
 590         # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
 591         out = self._qemu_qga_exec(u"guest-network-get-interfaces")
 592         interfaces = out.get(u"return")
 593         mac_name = {}
 594         if not interfaces:
 595             raise RuntimeError(
 596                 f"Get VM interface list failed on {self._node[u'host']}"
 597             )
 598         # Create MAC-name dict.
 599         for interface in interfaces:
 600             if u"hardware-address" not in interface:
 601                 continue
 602             mac_name[interface[u"hardware-address"]] = interface[u"name"]
 603         # Match interface by MAC and save interface name.
 604         for interface in self._vm_info[u"interfaces"].values():
 605             mac = interface.get(u"mac_address")
 606             if_name = mac_name.get(mac)
 607             if if_name is None:
 608                 logger.trace(f"Interface name for MAC {mac} not found")
 609             else:
 610                 interface[u"name"] = if_name
 611
 612     def qemu_start(self):
 613         """Start QEMU and wait until VM boot.
 614
 615         :returns: VM node info.
 616         :rtype: dict
 617         """
 618         cmd_opts = OptionString()
 619         cmd_opts.add(f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch}")
 620         cmd_opts.extend(self._params)
 621         message = f"QEMU: Start failed on {self._node[u'host']}!"
 622         try:
 623             DUTSetup.check_huge_page(
 624                 self._node, u"/dev/hugepages", int(self._opt.get(u"mem")))
 625
 626             exec_cmd_no_error(
 627                 self._node, cmd_opts, timeout=300, sudo=True, message=message
 628             )
 629             self._wait_until_vm_boot()
 630         except RuntimeError:
 631             self.qemu_kill_all()
 632             raise
 633         return self._vm_info
 634
 635     def qemu_kill(self):
 636         """Kill qemu process."""
 637         exec_cmd(
 638             self._node, f"chmod +r {self._temp.get(u'pidfile')}", sudo=True
 639         )
 640         exec_cmd(
 641             self._node, f"kill -SIGKILL $(cat {self._temp.get(u'pidfile')})",
 642             sudo=True
 643         )
 644
 645         for value in self._temp.values():
 646             exec_cmd(self._node, f"cat {value}", sudo=True)
 647             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 648
 649     def qemu_kill_all(self):
 650         """Kill all qemu processes on DUT node if specified."""
 651         exec_cmd(self._node, u"pkill -SIGKILL qemu", sudo=True)
 652
 653         for value in self._temp.values():
 654             exec_cmd(self._node, f"cat {value}", sudo=True)
 655             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 656
 657     def qemu_version(self):
 658         """Return Qemu version.
 659
 660         :returns: Qemu version.
 661         :rtype: str
 662         """
 663         command = f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch} " \
 664             f"--version"
 665         try:
 666             stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 667             return match(r"QEMU emulator version ([\d.]*)", stdout).group(1)
 668         except RuntimeError:
 669             self.qemu_kill_all()
 670             raise