resources/libraries/python/QemuUtils.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """QEMU utilities library."""
  15
  16 import json
  17
  18 from re import match
  19 from string import Template
  20 from time import sleep
  21
  22 from robot.api import logger
  23
  24 from resources.libraries.python.Constants import Constants
  25 from resources.libraries.python.DpdkUtil import DpdkUtil
  26 from resources.libraries.python.DUTSetup import DUTSetup
  27 from resources.libraries.python.OptionString import OptionString
  28 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
  29 from resources.libraries.python.topology import NodeType, Topology
  30 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
  31 from resources.libraries.python.VPPUtil import VPPUtil
  32
  33 __all__ = [u"QemuUtils"]
  34
  35
  36 class QemuUtils:
  37     """QEMU utilities."""
  38
  39     # Use one instance of class per tests.
  40     ROBOT_LIBRARY_SCOPE = u"TEST CASE"
  41
  42     def __init__(
  43             self, node, qemu_id=1, smp=1, mem=512, vnf=None,
  44             img=Constants.QEMU_VM_IMAGE):
  45         """Initialize QemuUtil class.
  46
  47         :param node: Node to run QEMU on.
  48         :param qemu_id: QEMU identifier.
  49         :param smp: Number of virtual SMP units (cores).
  50         :param mem: Amount of memory.
  51         :param vnf: Network function workload.
  52         :param img: QEMU disk image or kernel image path.
  53         :type node: dict
  54         :type qemu_id: int
  55         :type smp: int
  56         :type mem: int
  57         :type vnf: str
  58         :type img: str
  59         """
  60         self._vhost_id = 0
  61         self._node = node
  62         self._arch = Topology.get_node_arch(self._node)
  63         dpdk_target = u"arm64-armv8a" if self._arch == u"aarch64" \
  64             else u"x86_64-native"
  65         self._testpmd_path = f"{Constants.QEMU_VM_DPDK}/" \
  66             f"{dpdk_target}-linuxapp-gcc/app"
  67         self._vm_info = {
  68             u"host": node[u"host"],
  69             u"type": NodeType.VM,
  70             u"port": 10021 + qemu_id,
  71             u"serial": 4555 + qemu_id,
  72             u"username": 'cisco',
  73             u"password": 'cisco',
  74             u"interfaces": {},
  75         }
  76         if node[u"port"] != 22:
  77             self._vm_info[u"host_port"] = node[u"port"]
  78             self._vm_info[u"host_username"] = node[u"username"]
  79             self._vm_info[u"host_password"] = node[u"password"]
  80         # Input Options.
  81         self._opt = dict()
  82         self._opt[u"qemu_id"] = qemu_id
  83         self._opt[u"mem"] = int(mem)
  84         self._opt[u"smp"] = int(smp)
  85         self._opt[u"img"] = img
  86         self._opt[u"vnf"] = vnf
  87         # Temporary files.
  88         self._temp = dict()
  89         self._temp[u"pidfile"] = f"/var/run/qemu_{qemu_id}.pid"
  90         if img == Constants.QEMU_VM_IMAGE:
  91             self._opt[u"vm_type"] = u"nestedvm"
  92             self._temp[u"qmp"] = f"/var/run/qmp_{qemu_id}.sock"
  93             self._temp[u"qga"] = f"/var/run/qga_{qemu_id}.sock"
  94         elif img == Constants.QEMU_VM_KERNEL:
  95             self._opt[u"img"], _ = exec_cmd_no_error(
  96                 node, f"ls -1 {Constants.QEMU_VM_KERNEL}* | tail -1",
  97                 message=u"Qemu Kernel VM image not found!"
  98             )
  99             self._opt[u"vm_type"] = u"kernelvm"
 100             self._temp[u"log"] = f"/tmp/serial_{qemu_id}.log"
 101             self._temp[u"ini"] = f"/etc/vm_init_{qemu_id}.conf"
 102             self._opt[u"initrd"], _ = exec_cmd_no_error(
 103                 node, f"ls -1 {Constants.QEMU_VM_KERNEL_INITRD}* | tail -1",
 104                 message=u"Qemu Kernel initrd image not found!"
 105             )
 106         else:
 107             raise RuntimeError(f"QEMU: Unknown VM image option: {img}")
 108         # Computed parameters for QEMU command line.
 109         self._params = OptionString(prefix=u"-")
 110         self.add_params()
 111
 112     def add_params(self):
 113         """Set QEMU command line parameters."""
 114         self.add_default_params()
 115         if self._opt.get(u"vm_type", u"") == u"nestedvm":
 116             self.add_nestedvm_params()
 117         elif self._opt.get(u"vm_type", u"") == u"kernelvm":
 118             self.add_kernelvm_params()
 119         else:
 120             raise RuntimeError(u"QEMU: Unsupported VM type!")
 121
 122     def add_default_params(self):
 123         """Set default QEMU command line parameters."""
 124         self._params.add(u"daemonize")
 125         self._params.add(u"nodefaults")
 126         self._params.add_with_value(
 127             u"name", f"vnf{self._opt.get(u'qemu_id')},debug-threads=on"
 128         )
 129         self._params.add(u"no-user-config")
 130         self._params.add_with_value(u"monitor", u"none")
 131         self._params.add_with_value(u"display", u"none")
 132         self._params.add_with_value(u"vga", u"none")
 133         self._params.add(u"enable-kvm")
 134         self._params.add_with_value(u"pidfile", self._temp.get(u"pidfile"))
 135         self._params.add_with_value(u"cpu", u"host")
 136
 137         if self._arch == u"aarch64":
 138             machine_args = u"virt,accel=kvm,usb=off,mem-merge=off,gic-version=3"
 139         else:
 140             machine_args = u"pc,accel=kvm,usb=off,mem-merge=off"
 141         self._params.add_with_value(u"machine", machine_args)
 142         self._params.add_with_value(
 143             u"smp", f"{self._opt.get(u'smp')},sockets=1,"
 144             f"cores={self._opt.get(u'smp')},threads=1"
 145         )
 146         self._params.add_with_value(
 147             u"object", f"memory-backend-file,id=mem,"
 148             f"size={self._opt.get(u'mem')}M,mem-path=/dev/hugepages,share=on"
 149         )
 150         self._params.add_with_value(u"m", f"{self._opt.get(u'mem')}M")
 151         self._params.add_with_value(u"numa", u"node,memdev=mem")
 152         self._params.add_with_value(u"balloon", u"none")
 153
 154     def add_nestedvm_params(self):
 155         """Set NestedVM QEMU parameters."""
 156         self._params.add_with_value(
 157             u"net",
 158             f"nic,macaddr=52:54:00:00:{self._opt.get(u'qemu_id'):02x}:ff"
 159         )
 160         self._params.add_with_value(
 161             u"net", f"user,hostfwd=tcp::{self._vm_info[u'port']}-:22"
 162         )
 163         locking = u",file.locking=off"
 164         self._params.add_with_value(
 165             u"drive", f"file={self._opt.get(u'img')},"
 166             f"format=raw,cache=none,if=virtio{locking}"
 167         )
 168         self._params.add_with_value(
 169             u"qmp", f"unix:{self._temp.get(u'qmp')},server,nowait"
 170         )
 171         self._params.add_with_value(
 172             u"chardev", f"socket,host=127.0.0.1,"
 173             f"port={self._vm_info[u'serial']},id=gnc0,server,nowait")
 174         self._params.add_with_value(u"device", u"isa-serial,chardev=gnc0")
 175         self._params.add_with_value(
 176             u"chardev", f"socket,path={self._temp.get(u'qga')},"
 177             f"server,nowait,id=qga0"
 178         )
 179         self._params.add_with_value(u"device", u"isa-serial,chardev=qga0")
 180
 181     def add_kernelvm_params(self):
 182         """Set KernelVM QEMU parameters."""
 183         console = u"ttyAMA0" if self._arch == u"aarch64" else u"ttyS0"
 184         self._params.add_with_value(
 185             u"serial", f"file:{self._temp.get(u'log')}"
 186         )
 187         self._params.add_with_value(
 188             u"fsdev", u"local,id=root9p,path=/,security_model=none"
 189         )
 190         self._params.add_with_value(
 191             u"device", u"virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot"
 192         )
 193         self._params.add_with_value(u"kernel", f"{self._opt.get(u'img')}")
 194         self._params.add_with_value(u"initrd", f"{self._opt.get(u'initrd')}")
 195         self._params.add_with_value(
 196             u"append", f"'ro rootfstype=9p rootflags=trans=virtio "
 197             f"root=virtioroot console={console} tsc=reliable hugepages=256 "
 198             f"init={self._temp.get(u'ini')} fastboot'"
 199         )
 200
 201     def create_kernelvm_config_vpp(self, **kwargs):
 202         """Create QEMU VPP config files.
 203
 204         :param kwargs: Key-value pairs to replace content of VPP configuration
 205             file.
 206         :type kwargs: dict
 207         """
 208         startup = f"/etc/vpp/vm_startup_{self._opt.get(u'qemu_id')}.conf"
 209         running = f"/etc/vpp/vm_running_{self._opt.get(u'qemu_id')}.exec"
 210
 211         self._temp[u"startup"] = startup
 212         self._temp[u"running"] = running
 213         self._opt[u"vnf_bin"] = f"/usr/bin/vpp -c {startup}"
 214
 215         # Create VPP startup configuration.
 216         vpp_config = VppConfigGenerator()
 217         vpp_config.set_node(self._node)
 218         vpp_config.add_unix_nodaemon()
 219         vpp_config.add_unix_cli_listen()
 220         vpp_config.add_unix_exec(running)
 221         vpp_config.add_socksvr()
 222         vpp_config.add_cpu_main_core(u"0")
 223         if self._opt.get(u"smp") > 1:
 224             vpp_config.add_cpu_corelist_workers(f"1-{self._opt.get(u'smp')-1}")
 225         vpp_config.add_dpdk_dev(u"0000:00:06.0", u"0000:00:07.0")
 226         vpp_config.add_dpdk_dev_default_rxq(kwargs[u"queues"])
 227         vpp_config.add_dpdk_log_level(u"debug")
 228         if not kwargs[u"jumbo_frames"]:
 229             vpp_config.add_dpdk_no_multi_seg()
 230             vpp_config.add_dpdk_no_tx_checksum_offload()
 231         vpp_config.add_plugin(u"disable", [u"default"])
 232         vpp_config.add_plugin(u"enable", [u"dpdk_plugin.so"])
 233         vpp_config.write_config(startup)
 234
 235         # Create VPP running configuration.
 236         template = f"{Constants.RESOURCES_TPL_VM}/{self._opt.get(u'vnf')}.exec"
 237         exec_cmd_no_error(self._node, f"rm -f {running}", sudo=True)
 238
 239         with open(template, "r") as src_file:
 240             src = Template(src_file.read())
 241             exec_cmd_no_error(
 242                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 243                 f"sudo tee {running}"
 244             )
 245
 246     def create_kernelvm_config_testpmd_io(self, **kwargs):
 247         """Create QEMU testpmd-io command line.
 248
 249         :param kwargs: Key-value pairs to construct command line parameters.
 250         :type kwargs: dict
 251         """
 252         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 253             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 254             eal_driver=False,
 255             eal_in_memory=True,
 256             pmd_num_mbufs=16384,
 257             pmd_rxq=kwargs[u"queues"],
 258             pmd_txq=kwargs[u"queues"],
 259             pmd_tx_offloads='0x0',
 260             pmd_disable_hw_vlan=False,
 261             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 262         )
 263
 264         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 265
 266     def create_kernelvm_config_testpmd_mac(self, **kwargs):
 267         """Create QEMU testpmd-mac command line.
 268
 269         :param kwargs: Key-value pairs to construct command line parameters.
 270         :type kwargs: dict
 271         """
 272         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 273             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 274             eal_driver=False,
 275             eal_in_memory=True,
 276             pmd_num_mbufs=16384,
 277             pmd_fwd_mode=u"mac",
 278             pmd_eth_peer_0=f"0,{kwargs[u'vif1_mac']}",
 279             pmd_eth_peer_1=f"1,{kwargs[u'vif2_mac']}",
 280             pmd_rxq=kwargs[u"queues"],
 281             pmd_txq=kwargs[u"queues"],
 282             pmd_tx_offloads=u"0x0",
 283             pmd_disable_hw_vlan=False,
 284             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 285         )
 286
 287         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 288
 289     def create_kernelvm_init(self, **kwargs):
 290         """Create QEMU init script.
 291
 292         :param kwargs: Key-value pairs to replace content of init startup file.
 293         :type kwargs: dict
 294         """
 295         template = f"{Constants.RESOURCES_TPL_VM}/init.sh"
 296         init = self._temp.get(u"ini")
 297         exec_cmd_no_error(self._node, f"rm -f {init}", sudo=True)
 298
 299         with open(template, "r") as src_file:
 300             src = Template(src_file.read())
 301             exec_cmd_no_error(
 302                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 303                 f"sudo tee {init}"
 304             )
 305             exec_cmd_no_error(self._node, f"chmod +x {init}", sudo=True)
 306
 307     def configure_kernelvm_vnf(self, **kwargs):
 308         """Create KernelVM VNF configurations.
 309
 310         :param kwargs: Key-value pairs for templating configs.
 311         :type kwargs: dict
 312         """
 313         if u"vpp" in self._opt.get(u"vnf"):
 314             self.create_kernelvm_config_vpp(**kwargs)
 315         elif u"testpmd_io" in self._opt.get(u"vnf"):
 316             self.create_kernelvm_config_testpmd_io(**kwargs)
 317         elif u"testpmd_mac" in self._opt.get(u"vnf"):
 318             self.create_kernelvm_config_testpmd_mac(**kwargs)
 319         else:
 320             raise RuntimeError(u"QEMU: Unsupported VNF!")
 321         self.create_kernelvm_init(vnf_bin=self._opt[u"vnf_bin"])
 322
 323     def get_qemu_pids(self):
 324         """Get QEMU CPU pids.
 325
 326         :returns: List of QEMU CPU pids.
 327         :rtype: list of str
 328         """
 329         command = f"grep -rwl 'CPU' /proc/$(sudo cat " \
 330             f"{self._temp.get(u'pidfile')})/task/*/comm "
 331         command += r"| xargs dirname | sed -e 's/\/.*\///g' | uniq"
 332
 333         stdout, _ = exec_cmd_no_error(self._node, command)
 334         return stdout.splitlines()
 335
 336     def qemu_set_affinity(self, *host_cpus):
 337         """Set qemu affinity by getting thread PIDs via QMP and taskset to list
 338         of CPU cores. Function tries to execute 3 times to avoid race condition
 339         in getting thread PIDs.
 340
 341         :param host_cpus: List of CPU cores.
 342         :type host_cpus: list
 343         """
 344         for _ in range(3):
 345             try:
 346                 qemu_cpus = self.get_qemu_pids()
 347
 348                 if len(qemu_cpus) != len(host_cpus):
 349                     sleep(1)
 350                     continue
 351                 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
 352                     command = f"taskset -pc {host_cpu} {qemu_cpu}"
 353                     message = f"QEMU: Set affinity failed " \
 354                         f"on {self._node[u'host']}!"
 355                     exec_cmd_no_error(
 356                         self._node, command, sudo=True, message=message
 357                     )
 358                 break
 359             except (RuntimeError, ValueError):
 360                 self.qemu_kill_all()
 361                 raise
 362         else:
 363             self.qemu_kill_all()
 364             raise RuntimeError(u"Failed to set Qemu threads affinity!")
 365
 366     def qemu_set_scheduler_policy(self):
 367         """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
 368         processes.
 369
 370         :raises RuntimeError: Set scheduler policy failed.
 371         """
 372         try:
 373             qemu_cpus = self.get_qemu_pids()
 374
 375             for qemu_cpu in qemu_cpus:
 376                 command = f"chrt -r -p 1 {qemu_cpu}"
 377                 message = f"QEMU: Set SCHED_RR failed on {self._node[u'host']}"
 378                 exec_cmd_no_error(
 379                     self._node, command, sudo=True, message=message
 380                 )
 381         except (RuntimeError, ValueError):
 382             self.qemu_kill_all()
 383             raise
 384
 385     def qemu_add_vhost_user_if(
 386             self, socket, server=True, jumbo_frames=False, queue_size=None,
 387             queues=1, csum=False, gso=False):
 388         """Add Vhost-user interface.
 389
 390         :param socket: Path of the unix socket.
 391         :param server: If True the socket shall be a listening socket.
 392         :param jumbo_frames: Set True if jumbo frames are used in the test.
 393         :param queue_size: Vring queue size.
 394         :param queues: Number of queues.
 395         :param csum: Checksum offloading.
 396         :param gso: Generic segmentation offloading.
 397         :type socket: str
 398         :type server: bool
 399         :type jumbo_frames: bool
 400         :type queue_size: int
 401         :type queues: int
 402         :type csum: bool
 403         :type gso: bool
 404         """
 405         self._vhost_id += 1
 406         self._params.add_with_value(
 407             u"chardev", f"socket,id=char{self._vhost_id},"
 408             f"path={socket}{u',server' if server is True else u''}"
 409         )
 410         self._params.add_with_value(
 411             u"netdev", f"vhost-user,id=vhost{self._vhost_id},"
 412             f"chardev=char{self._vhost_id},queues={queues}"
 413         )
 414         mac = f"52:54:00:00:{self._opt.get(u'qemu_id'):02x}:" \
 415             f"{self._vhost_id:02x}"
 416         queue_size = f"rx_queue_size={queue_size},tx_queue_size={queue_size}" \
 417             if queue_size else u""
 418         self._params.add_with_value(
 419             u"device", f"virtio-net-pci,netdev=vhost{self._vhost_id},mac={mac},"
 420             f"addr={self._vhost_id+5}.0,mq=on,vectors={2 * queues + 2},"
 421             f"csum={u'on' if csum else u'off'},gso={u'on' if gso else u'off'},"
 422             f"guest_tso4=off,guest_tso6=off,guest_ecn=off,"
 423             f"mrg_rxbuf={u'on,host_mtu=9200' if jumbo_frames else u'off'},"
 424             f"{queue_size}"
 425         )
 426
 427         # Add interface MAC and socket to the node dict.
 428         if_data = {u"mac_address": mac, u"socket": socket}
 429         if_name = f"vhost{self._vhost_id}"
 430         self._vm_info[u"interfaces"][if_name] = if_data
 431         # Add socket to temporary file list.
 432         self._temp[if_name] = socket
 433
 434     def _qemu_qmp_exec(self, cmd):
 435         """Execute QMP command.
 436
 437         QMP is JSON based protocol which allows to control QEMU instance.
 438
 439         :param cmd: QMP command to execute.
 440         :type cmd: str
 441         :returns: Command output in python representation of JSON format. The
 442             { "return": {} } response is QMP's success response. An error
 443             response will contain the "error" keyword instead of "return".
 444         """
 445         # To enter command mode, the qmp_capabilities command must be issued.
 446         command = f"echo \"{{{{ \\\"execute\\\": " \
 447             f"\\\"qmp_capabilities\\\" }}}}" \
 448             f"{{{{ \\\"execute\\\": \\\"{cmd}\\\" }}}}\" | " \
 449             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qmp')}"
 450         message = f"QMP execute '{cmd}' failed on {self._node[u'host']}"
 451
 452         stdout, _ = exec_cmd_no_error(
 453             self._node, command, sudo=False, message=message
 454         )
 455
 456         # Skip capabilities negotiation messages.
 457         out_list = stdout.splitlines()
 458         if len(out_list) < 3:
 459             raise RuntimeError(f"Invalid QMP output on {self._node[u'host']}")
 460         return json.loads(out_list[2])
 461
 462     def _qemu_qga_flush(self):
 463         """Flush the QGA parser state."""
 464         command = f"(printf \"\xFF\"; sleep 1) | sudo -S socat " \
 465             f"- UNIX-CONNECT:{self._temp.get(u'qga')}"
 466         message = f"QGA flush failed on {self._node[u'host']}"
 467         stdout, _ = exec_cmd_no_error(
 468             self._node, command, sudo=False, message=message
 469         )
 470
 471         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 472
 473     def _qemu_qga_exec(self, cmd):
 474         """Execute QGA command.
 475
 476         QGA provide access to a system-level agent via standard QMP commands.
 477
 478         :param cmd: QGA command to execute.
 479         :type cmd: str
 480         """
 481         command = f"(echo \"{{{{ \\\"execute\\\": " \
 482             f"\\\"{cmd}\\\" }}}}\"; sleep 1) | " \
 483             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qga')}"
 484         message = f"QGA execute '{cmd}' failed on {self._node[u'host']}"
 485         stdout, _ = exec_cmd_no_error(
 486             self._node, command, sudo=False, message=message
 487         )
 488
 489         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 490
 491     def _wait_until_vm_boot(self):
 492         """Wait until QEMU with NestedVM is booted."""
 493         if self._opt.get(u"vm_type") == u"nestedvm":
 494             self._wait_until_nestedvm_boot()
 495             self._update_vm_interfaces()
 496         elif self._opt.get(u"vm_type") == u"kernelvm":
 497             self._wait_until_kernelvm_boot()
 498         else:
 499             raise RuntimeError(u"QEMU: Unsupported VM type!")
 500
 501     def _wait_until_nestedvm_boot(self, retries=12):
 502         """Wait until QEMU with NestedVM is booted.
 503
 504         First try to flush qga until there is output.
 505         Then ping QEMU guest agent each 5s until VM booted or timeout.
 506
 507         :param retries: Number of retries with 5s between trials.
 508         :type retries: int
 509         """
 510         for _ in range(retries):
 511             out = None
 512             try:
 513                 out = self._qemu_qga_flush()
 514             except ValueError:
 515                 logger.trace(f"QGA qga flush unexpected output {out}")
 516             # Empty output - VM not booted yet
 517             if not out:
 518                 sleep(5)
 519             else:
 520                 break
 521         else:
 522             raise RuntimeError(
 523                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 524             )
 525         for _ in range(retries):
 526             out = None
 527             try:
 528                 out = self._qemu_qga_exec(u"guest-ping")
 529             except ValueError:
 530                 logger.trace(f"QGA guest-ping unexpected output {out}")
 531             # Empty output - VM not booted yet.
 532             if not out:
 533                 sleep(5)
 534             # Non-error return - VM booted.
 535             elif out.get(u"return") is not None:
 536                 break
 537             # Skip error and wait.
 538             elif out.get(u"error") is not None:
 539                 sleep(5)
 540             else:
 541                 # If there is an unexpected output from QGA guest-info, try
 542                 # again until timeout.
 543                 logger.trace(f"QGA guest-ping unexpected output {out}")
 544         else:
 545             raise RuntimeError(
 546                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 547             )
 548
 549     def _wait_until_kernelvm_boot(self, retries=60):
 550         """Wait until QEMU KernelVM is booted.
 551
 552         :param retries: Number of retries.
 553         :type retries: int
 554         """
 555         vpp_ver = VPPUtil.vpp_show_version(self._node)
 556
 557         for _ in range(retries):
 558             command = f"tail -1 {self._temp.get(u'log')}"
 559             stdout = None
 560             try:
 561                 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 562                 sleep(1)
 563             except RuntimeError:
 564                 pass
 565             if vpp_ver in stdout or u"Press enter to exit" in stdout:
 566                 break
 567             if u"reboot: Power down" in stdout:
 568                 raise RuntimeError(
 569                     f"QEMU: NF failed to run on {self._node[u'host']}!"
 570                 )
 571         else:
 572             raise RuntimeError(
 573                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 574             )
 575
 576     def _update_vm_interfaces(self):
 577         """Update interface names in VM node dict."""
 578         # Send guest-network-get-interfaces command via QGA, output example:
 579         # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
 580         # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
 581         out = self._qemu_qga_exec(u"guest-network-get-interfaces")
 582         interfaces = out.get(u"return")
 583         mac_name = {}
 584         if not interfaces:
 585             raise RuntimeError(
 586                 f"Get VM interface list failed on {self._node[u'host']}"
 587             )
 588         # Create MAC-name dict.
 589         for interface in interfaces:
 590             if u"hardware-address" not in interface:
 591                 continue
 592             mac_name[interface[u"hardware-address"]] = interface[u"name"]
 593         # Match interface by MAC and save interface name.
 594         for interface in self._vm_info[u"interfaces"].values():
 595             mac = interface.get(u"mac_address")
 596             if_name = mac_name.get(mac)
 597             if if_name is None:
 598                 logger.trace(f"Interface name for MAC {mac} not found")
 599             else:
 600                 interface[u"name"] = if_name
 601
 602     def qemu_start(self):
 603         """Start QEMU and wait until VM boot.
 604
 605         :returns: VM node info.
 606         :rtype: dict
 607         """
 608         cmd_opts = OptionString()
 609         cmd_opts.add(f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch}")
 610         cmd_opts.extend(self._params)
 611         message = f"QEMU: Start failed on {self._node[u'host']}!"
 612         try:
 613             DUTSetup.check_huge_page(
 614                 self._node, u"/dev/hugepages", self._opt.get(u"mem"))
 615
 616             exec_cmd_no_error(
 617                 self._node, cmd_opts, timeout=300, sudo=True, message=message
 618             )
 619             self._wait_until_vm_boot()
 620         except RuntimeError:
 621             self.qemu_kill_all()
 622             raise
 623         return self._vm_info
 624
 625     def qemu_kill(self):
 626         """Kill qemu process."""
 627         exec_cmd(
 628             self._node, f"chmod +r {self._temp.get(u'pidfile')}", sudo=True
 629         )
 630         exec_cmd(
 631             self._node, f"kill -SIGKILL $(cat {self._temp.get(u'pidfile')})",
 632             sudo=True
 633         )
 634
 635         for value in self._temp.values():
 636             exec_cmd(self._node, f"cat {value}", sudo=True)
 637             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 638
 639     def qemu_kill_all(self):
 640         """Kill all qemu processes on DUT node if specified."""
 641         exec_cmd(self._node, u"pkill -SIGKILL qemu", sudo=True)
 642
 643         for value in self._temp.values():
 644             exec_cmd(self._node, f"cat {value}", sudo=True)
 645             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 646
 647     def qemu_version(self):
 648         """Return Qemu version.
 649
 650         :returns: Qemu version.
 651         :rtype: str
 652         """
 653         command = f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch} " \
 654             f"--version"
 655         try:
 656             stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 657             return match(r"QEMU emulator version ([\d.]*)", stdout).group(1)
 658         except RuntimeError:
 659             self.qemu_kill_all()
 660             raise