resources/libraries/python/QemuUtils.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """QEMU utilities library."""
  15
  16 import json
  17
  18 from re import match
  19 from string import Template
  20 from time import sleep
  21
  22 from robot.api import logger
  23
  24 from resources.libraries.python.Constants import Constants
  25 from resources.libraries.python.DpdkUtil import DpdkUtil
  26 from resources.libraries.python.DUTSetup import DUTSetup
  27 from resources.libraries.python.OptionString import OptionString
  28 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
  29 from resources.libraries.python.topology import NodeType, Topology
  30 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
  31 from resources.libraries.python.VPPUtil import VPPUtil
  32
  33 __all__ = [u"QemuUtils"]
  34
  35
  36 class QemuUtils:
  37     """QEMU utilities."""
  38
  39     # Use one instance of class per tests.
  40     ROBOT_LIBRARY_SCOPE = u"TEST CASE"
  41
  42     def __init__(
  43             self, node, qemu_id=1, smp=1, mem=512, vnf=None,
  44             img=Constants.QEMU_VM_IMAGE):
  45         """Initialize QemuUtil class.
  46
  47         :param node: Node to run QEMU on.
  48         :param qemu_id: QEMU identifier.
  49         :param smp: Number of virtual SMP units (cores).
  50         :param mem: Amount of memory.
  51         :param vnf: Network function workload.
  52         :param img: QEMU disk image or kernel image path.
  53         :type node: dict
  54         :type qemu_id: int
  55         :type smp: int
  56         :type mem: int
  57         :type vnf: str
  58         :type img: str
  59         """
  60         self._vhost_id = 0
  61         self._node = node
  62         self._arch = Topology.get_node_arch(self._node)
  63         dpdk_target = u"arm64-armv8a" if self._arch == u"aarch64" \
  64             else u"x86_64-native"
  65         self._testpmd_path = f"{Constants.QEMU_VM_DPDK}/" \
  66             f"{dpdk_target}-linuxapp-gcc/app"
  67         self._vm_info = {
  68             u"host": node[u"host"],
  69             u"type": NodeType.VM,
  70             u"port": 10021 + qemu_id,
  71             u"serial": 4555 + qemu_id,
  72             u"username": 'cisco',
  73             u"password": 'cisco',
  74             u"interfaces": {},
  75         }
  76         if node[u"port"] != 22:
  77             self._vm_info[u"host_port"] = node[u"port"]
  78             self._vm_info[u"host_username"] = node[u"username"]
  79             self._vm_info[u"host_password"] = node[u"password"]
  80         # Input Options.
  81         self._opt = dict()
  82         self._opt[u"qemu_id"] = qemu_id
  83         self._opt[u"mem"] = int(mem)
  84         self._opt[u"smp"] = int(smp)
  85         self._opt[u"img"] = img
  86         self._opt[u"vnf"] = vnf
  87         # Temporary files.
  88         self._temp = dict()
  89         self._temp[u"pidfile"] = f"/var/run/qemu_{qemu_id}.pid"
  90         if img == Constants.QEMU_VM_IMAGE:
  91             self._opt[u"vm_type"] = u"nestedvm"
  92             self._temp[u"qmp"] = f"/var/run/qmp_{qemu_id}.sock"
  93             self._temp[u"qga"] = f"/var/run/qga_{qemu_id}.sock"
  94         elif img == Constants.QEMU_VM_KERNEL:
  95             self._opt[u"img"], _ = exec_cmd_no_error(
  96                 node, f"ls -1 {Constants.QEMU_VM_KERNEL}* | tail -1",
  97                 message=u"Qemu Kernel VM image not found!"
  98             )
  99             self._opt[u"vm_type"] = u"kernelvm"
 100             self._temp[u"log"] = f"/tmp/serial_{qemu_id}.log"
 101             self._temp[u"ini"] = f"/etc/vm_init_{qemu_id}.conf"
 102             self._opt[u"initrd"], _ = exec_cmd_no_error(
 103                 node, f"ls -1 {Constants.QEMU_VM_KERNEL_INITRD}* | tail -1",
 104                 message=u"Qemu Kernel initrd image not found!"
 105             )
 106         else:
 107             raise RuntimeError(f"QEMU: Unknown VM image option: {img}")
 108         # Computed parameters for QEMU command line.
 109         self._params = OptionString(prefix=u"-")
 110         self.add_params()
 111
 112     def add_params(self):
 113         """Set QEMU command line parameters."""
 114         self.add_default_params()
 115         if self._opt.get(u"vm_type", u"") == u"nestedvm":
 116             self.add_nestedvm_params()
 117         elif self._opt.get(u"vm_type", u"") == u"kernelvm":
 118             self.add_kernelvm_params()
 119         else:
 120             raise RuntimeError(u"QEMU: Unsupported VM type!")
 121
 122     def add_default_params(self):
 123         """Set default QEMU command line parameters."""
 124         self._params.add(u"daemonize")
 125         self._params.add(u"nodefaults")
 126         self._params.add_with_value(
 127             u"name", f"vnf{self._opt.get(u'qemu_id')},debug-threads=on"
 128         )
 129         self._params.add(u"no-user-config")
 130         self._params.add_with_value(u"monitor", u"none")
 131         self._params.add_with_value(u"display", u"none")
 132         self._params.add_with_value(u"vga", u"none")
 133         self._params.add(u"enable-kvm")
 134         self._params.add_with_value(u"pidfile", self._temp.get(u"pidfile"))
 135         self._params.add_with_value(u"cpu", u"host")
 136
 137         if self._arch == u"aarch64":
 138             machine_args = u"virt,accel=kvm,usb=off,mem-merge=off,gic-version=3"
 139         else:
 140             machine_args = u"pc,accel=kvm,usb=off,mem-merge=off"
 141         self._params.add_with_value(u"machine", machine_args)
 142         self._params.add_with_value(
 143             u"smp", f"{self._opt.get(u'smp')},sockets=1,"
 144             f"cores={self._opt.get(u'smp')},threads=1"
 145         )
 146         self._params.add_with_value(
 147             u"object", f"memory-backend-file,id=mem,"
 148             f"size={self._opt.get(u'mem')}M,mem-path=/dev/hugepages,share=on"
 149         )
 150         self._params.add_with_value(u"m", f"{self._opt.get(u'mem')}M")
 151         self._params.add_with_value(u"numa", u"node,memdev=mem")
 152         self._params.add_with_value(u"balloon", u"none")
 153
 154     def add_nestedvm_params(self):
 155         """Set NestedVM QEMU parameters."""
 156         self._params.add_with_value(
 157             u"net",
 158             f"nic,macaddr=52:54:00:00:{self._opt.get(u'qemu_id'):02x}:ff"
 159         )
 160         self._params.add_with_value(
 161             u"net", f"user,hostfwd=tcp::{self._vm_info[u'port']}-:22"
 162         )
 163         locking = u",file.locking=off"
 164         self._params.add_with_value(
 165             u"drive", f"file={self._opt.get(u'img')},"
 166             f"format=raw,cache=none,if=virtio{locking}"
 167         )
 168         self._params.add_with_value(
 169             u"qmp", f"unix:{self._temp.get(u'qmp')},server,nowait"
 170         )
 171         self._params.add_with_value(
 172             u"chardev", f"socket,host=127.0.0.1,"
 173             f"port={self._vm_info[u'serial']},id=gnc0,server,nowait")
 174         self._params.add_with_value(u"device", u"isa-serial,chardev=gnc0")
 175         self._params.add_with_value(
 176             u"chardev", f"socket,path={self._temp.get(u'qga')},"
 177             f"server,nowait,id=qga0"
 178         )
 179         self._params.add_with_value(u"device", u"isa-serial,chardev=qga0")
 180
 181     def add_kernelvm_params(self):
 182         """Set KernelVM QEMU parameters."""
 183         console = u"ttyAMA0" if self._arch == u"aarch64" else u"ttyS0"
 184         self._params.add_with_value(
 185             u"serial", f"file:{self._temp.get(u'log')}"
 186         )
 187         self._params.add_with_value(
 188             u"fsdev", u"local,id=root9p,path=/,security_model=none"
 189         )
 190         self._params.add_with_value(
 191             u"device", u"virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot"
 192         )
 193         self._params.add_with_value(u"kernel", f"{self._opt.get(u'img')}")
 194         self._params.add_with_value(u"initrd", f"{self._opt.get(u'initrd')}")
 195         self._params.add_with_value(
 196             u"append", f"'ro rootfstype=9p rootflags=trans=virtio "
 197             f"root=virtioroot console={console} tsc=reliable hugepages=256 "
 198             f"init={self._temp.get(u'ini')} fastboot'"
 199         )
 200
 201     def create_kernelvm_config_vpp(self, **kwargs):
 202         """Create QEMU VPP config files.
 203
 204         :param kwargs: Key-value pairs to replace content of VPP configuration
 205             file.
 206         :type kwargs: dict
 207         """
 208         startup = f"/etc/vpp/vm_startup_{self._opt.get(u'qemu_id')}.conf"
 209         running = f"/etc/vpp/vm_running_{self._opt.get(u'qemu_id')}.exec"
 210
 211         self._temp[u"startup"] = startup
 212         self._temp[u"running"] = running
 213         self._opt[u"vnf_bin"] = f"/usr/bin/vpp -c {startup}"
 214
 215         # Create VPP startup configuration.
 216         vpp_config = VppConfigGenerator()
 217         vpp_config.set_node(self._node)
 218         vpp_config.add_unix_nodaemon()
 219         vpp_config.add_unix_cli_listen()
 220         vpp_config.add_unix_exec(running)
 221         vpp_config.add_socksvr()
 222         vpp_config.add_cpu_main_core(u"0")
 223         if self._opt.get(u"smp") > 1:
 224             vpp_config.add_cpu_corelist_workers(f"1-{self._opt.get(u'smp')-1}")
 225         vpp_config.add_dpdk_dev(u"0000:00:06.0", u"0000:00:07.0")
 226         vpp_config.add_dpdk_dev_default_rxq(kwargs[u"queues"])
 227         vpp_config.add_dpdk_log_level(u"debug")
 228         if not kwargs[u"jumbo_frames"]:
 229             vpp_config.add_dpdk_no_multi_seg()
 230             vpp_config.add_dpdk_no_tx_checksum_offload()
 231         vpp_config.add_plugin(u"disable", [u"default"])
 232         vpp_config.add_plugin(u"enable", [u"dpdk_plugin.so"])
 233         vpp_config.write_config(startup)
 234
 235         # Create VPP running configuration.
 236         template = f"{Constants.RESOURCES_TPL_VM}/{self._opt.get(u'vnf')}.exec"
 237         exec_cmd_no_error(self._node, f"rm -f {running}", sudo=True)
 238
 239         with open(template, "r") as src_file:
 240             src = Template(src_file.read())
 241             exec_cmd_no_error(
 242                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 243                 f"sudo tee {running}"
 244             )
 245
 246     def create_kernelvm_config_testpmd_io(self, **kwargs):
 247         """Create QEMU testpmd-io command line.
 248
 249         :param kwargs: Key-value pairs to construct command line parameters.
 250         :type kwargs: dict
 251         """
 252         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 253             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 254             eal_driver=False,
 255             eal_in_memory=True,
 256             pmd_num_mbufs=16384,
 257             pmd_rxq=kwargs[u"queues"],
 258             pmd_txq=kwargs[u"queues"],
 259             pmd_tx_offloads='0x0',
 260             pmd_disable_hw_vlan=False,
 261             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 262         )
 263
 264         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 265
 266     def create_kernelvm_config_testpmd_mac(self, **kwargs):
 267         """Create QEMU testpmd-mac command line.
 268
 269         :param kwargs: Key-value pairs to construct command line parameters.
 270         :type kwargs: dict
 271         """
 272         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 273             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 274             eal_driver=False,
 275             eal_in_memory=True,
 276             pmd_num_mbufs=16384,
 277             pmd_fwd_mode=u"mac",
 278             pmd_eth_peer_0=f"0,{kwargs[u'vif1_mac']}",
 279             pmd_eth_peer_1=f"1,{kwargs[u'vif2_mac']}",
 280             pmd_rxq=kwargs[u"queues"],
 281             pmd_txq=kwargs[u"queues"],
 282             pmd_tx_offloads=u"0x0",
 283             pmd_disable_hw_vlan=False,
 284             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 285         )
 286
 287         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 288
 289     def create_kernelvm_init(self, **kwargs):
 290         """Create QEMU init script.
 291
 292         :param kwargs: Key-value pairs to replace content of init startup file.
 293         :type kwargs: dict
 294         """
 295         template = f"{Constants.RESOURCES_TPL_VM}/init.sh"
 296         init = self._temp.get(u"ini")
 297         exec_cmd_no_error(self._node, f"rm -f {init}", sudo=True)
 298
 299         with open(template, "r") as src_file:
 300             src = Template(src_file.read())
 301             exec_cmd_no_error(
 302                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 303                 f"sudo tee {init}"
 304             )
 305             exec_cmd_no_error(self._node, f"chmod +x {init}", sudo=True)
 306
 307     def configure_kernelvm_vnf(self, **kwargs):
 308         """Create KernelVM VNF configurations.
 309
 310         :param kwargs: Key-value pairs for templating configs.
 311         :type kwargs: dict
 312         """
 313         if u"vpp" in self._opt.get(u"vnf"):
 314             self.create_kernelvm_config_vpp(**kwargs)
 315         elif u"testpmd_io" in self._opt.get(u"vnf"):
 316             self.create_kernelvm_config_testpmd_io(**kwargs)
 317         elif u"testpmd_mac" in self._opt.get(u"vnf"):
 318             self.create_kernelvm_config_testpmd_mac(**kwargs)
 319         else:
 320             raise RuntimeError(u"QEMU: Unsupported VNF!")
 321         self.create_kernelvm_init(vnf_bin=self._opt[u"vnf_bin"])
 322
 323     def get_qemu_pids(self):
 324         """Get QEMU CPU pids.
 325
 326         :returns: List of QEMU CPU pids.
 327         :rtype: list of str
 328         """
 329         command = f"grep -rwl 'CPU' /proc/$(sudo cat " \
 330             f"{self._temp.get(u'pidfile')})/task/*/comm "
 331         command += r"| xargs dirname | sed -e 's/\/.*\///g' | uniq"
 332
 333         stdout, _ = exec_cmd_no_error(self._node, command)
 334         return stdout.splitlines()
 335
 336     def qemu_set_affinity(self, *host_cpus):
 337         """Set qemu affinity by getting thread PIDs via QMP and taskset to list
 338         of CPU cores. Function tries to execute 3 times to avoid race condition
 339         in getting thread PIDs.
 340
 341         :param host_cpus: List of CPU cores.
 342         :type host_cpus: list
 343         """
 344         for _ in range(3):
 345             try:
 346                 qemu_cpus = self.get_qemu_pids()
 347
 348                 if len(qemu_cpus) != len(host_cpus):
 349                     sleep(1)
 350                     continue
 351                 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
 352                     command = f"taskset -pc {host_cpu} {qemu_cpu}"
 353                     message = f"QEMU: Set affinity failed " \
 354                         f"on {self._node[u'host']}!"
 355                     exec_cmd_no_error(
 356                         self._node, command, sudo=True, message=message
 357                     )
 358                 break
 359             except (RuntimeError, ValueError):
 360                 self.qemu_kill_all()
 361                 raise
 362         else:
 363             self.qemu_kill_all()
 364             raise RuntimeError(u"Failed to set Qemu threads affinity!")
 365
 366     def qemu_set_scheduler_policy(self):
 367         """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
 368         processes.
 369
 370         :raises RuntimeError: Set scheduler policy failed.
 371         """
 372         try:
 373             qemu_cpus = self.get_qemu_pids()
 374
 375             for qemu_cpu in qemu_cpus:
 376                 command = f"chrt -r -p 1 {qemu_cpu}"
 377                 message = f"QEMU: Set SCHED_RR failed on {self._node[u'host']}"
 378                 exec_cmd_no_error(
 379                     self._node, command, sudo=True, message=message
 380                 )
 381         except (RuntimeError, ValueError):
 382             self.qemu_kill_all()
 383             raise
 384
 385     def qemu_add_vhost_user_if(
 386             self, socket, server=True, jumbo_frames=False, queue_size=None,
 387             queues=1):
 388         """Add Vhost-user interface.
 389
 390         :param socket: Path of the unix socket.
 391         :param server: If True the socket shall be a listening socket.
 392         :param jumbo_frames: Set True if jumbo frames are used in the test.
 393         :param queue_size: Vring queue size.
 394         :param queues: Number of queues.
 395         :type socket: str
 396         :type server: bool
 397         :type jumbo_frames: bool
 398         :type queue_size: int
 399         :type queues: int
 400         """
 401         self._vhost_id += 1
 402         self._params.add_with_value(
 403             u"chardev", f"socket,id=char{self._vhost_id},"
 404             f"path={socket}{u',server' if server is True else u''}"
 405         )
 406         self._params.add_with_value(
 407             u"netdev", f"vhost-user,id=vhost{self._vhost_id},"
 408             f"chardev=char{self._vhost_id},queues={queues}"
 409         )
 410         mac = f"52:54:00:00:{self._opt.get(u'qemu_id'):02x}:" \
 411             f"{self._vhost_id:02x}"
 412         queue_size = f"rx_queue_size={queue_size},tx_queue_size={queue_size}" \
 413             if queue_size else u""
 414         mbuf = u"on,host_mtu=9200"
 415         self._params.add_with_value(
 416             u"device", f"virtio-net-pci,netdev=vhost{self._vhost_id},mac={mac},"
 417             f"addr={self._vhost_id+5}.0,mq=on,vectors={2 * queues + 2},"
 418             f"csum=off,gso=off,guest_tso4=off,guest_tso6=off,guest_ecn=off,"
 419             f"mrg_rxbuf={mbuf if jumbo_frames else u'off'},{queue_size}"
 420         )
 421
 422         # Add interface MAC and socket to the node dict.
 423         if_data = {u"mac_address": mac, u"socket": socket}
 424         if_name = f"vhost{self._vhost_id}"
 425         self._vm_info[u"interfaces"][if_name] = if_data
 426         # Add socket to temporary file list.
 427         self._temp[if_name] = socket
 428
 429     def _qemu_qmp_exec(self, cmd):
 430         """Execute QMP command.
 431
 432         QMP is JSON based protocol which allows to control QEMU instance.
 433
 434         :param cmd: QMP command to execute.
 435         :type cmd: str
 436         :returns: Command output in python representation of JSON format. The
 437             { "return": {} } response is QMP's success response. An error
 438             response will contain the "error" keyword instead of "return".
 439         """
 440         # To enter command mode, the qmp_capabilities command must be issued.
 441         command = f"echo \"{{{{ \\\"execute\\\": " \
 442             f"\\\"qmp_capabilities\\\" }}}}" \
 443             f"{{{{ \\\"execute\\\": \\\"{cmd}\\\" }}}}\" | " \
 444             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qmp')}"
 445         message = f"QMP execute '{cmd}' failed on {self._node[u'host']}"
 446
 447         stdout, _ = exec_cmd_no_error(
 448             self._node, command, sudo=False, message=message
 449         )
 450
 451         # Skip capabilities negotiation messages.
 452         out_list = stdout.splitlines()
 453         if len(out_list) < 3:
 454             raise RuntimeError(f"Invalid QMP output on {self._node[u'host']}")
 455         return json.loads(out_list[2])
 456
 457     def _qemu_qga_flush(self):
 458         """Flush the QGA parser state."""
 459         command = f"(printf \"\xFF\"; sleep 1) | sudo -S socat " \
 460             f"- UNIX-CONNECT:{self._temp.get(u'qga')}"
 461         message = f"QGA flush failed on {self._node[u'host']}"
 462         stdout, _ = exec_cmd_no_error(
 463             self._node, command, sudo=False, message=message
 464         )
 465
 466         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 467
 468     def _qemu_qga_exec(self, cmd):
 469         """Execute QGA command.
 470
 471         QGA provide access to a system-level agent via standard QMP commands.
 472
 473         :param cmd: QGA command to execute.
 474         :type cmd: str
 475         """
 476         command = f"(echo \"{{{{ \\\"execute\\\": " \
 477             f"\\\"{cmd}\\\" }}}}\"; sleep 1) | " \
 478             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qga')}"
 479         message = f"QGA execute '{cmd}' failed on {self._node[u'host']}"
 480         stdout, _ = exec_cmd_no_error(
 481             self._node, command, sudo=False, message=message
 482         )
 483
 484         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 485
 486     def _wait_until_vm_boot(self):
 487         """Wait until QEMU with NestedVM is booted."""
 488         if self._opt.get(u"vm_type") == u"nestedvm":
 489             self._wait_until_nestedvm_boot()
 490             self._update_vm_interfaces()
 491         elif self._opt.get(u"vm_type") == u"kernelvm":
 492             self._wait_until_kernelvm_boot()
 493         else:
 494             raise RuntimeError(u"QEMU: Unsupported VM type!")
 495
 496     def _wait_until_nestedvm_boot(self, retries=12):
 497         """Wait until QEMU with NestedVM is booted.
 498
 499         First try to flush qga until there is output.
 500         Then ping QEMU guest agent each 5s until VM booted or timeout.
 501
 502         :param retries: Number of retries with 5s between trials.
 503         :type retries: int
 504         """
 505         for _ in range(retries):
 506             out = None
 507             try:
 508                 out = self._qemu_qga_flush()
 509             except ValueError:
 510                 logger.trace(f"QGA qga flush unexpected output {out}")
 511             # Empty output - VM not booted yet
 512             if not out:
 513                 sleep(5)
 514             else:
 515                 break
 516         else:
 517             raise RuntimeError(
 518                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 519             )
 520         for _ in range(retries):
 521             out = None
 522             try:
 523                 out = self._qemu_qga_exec(u"guest-ping")
 524             except ValueError:
 525                 logger.trace(f"QGA guest-ping unexpected output {out}")
 526             # Empty output - VM not booted yet.
 527             if not out:
 528                 sleep(5)
 529             # Non-error return - VM booted.
 530             elif out.get(u"return") is not None:
 531                 break
 532             # Skip error and wait.
 533             elif out.get(u"error") is not None:
 534                 sleep(5)
 535             else:
 536                 # If there is an unexpected output from QGA guest-info, try
 537                 # again until timeout.
 538                 logger.trace(f"QGA guest-ping unexpected output {out}")
 539         else:
 540             raise RuntimeError(
 541                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 542             )
 543
 544     def _wait_until_kernelvm_boot(self, retries=60):
 545         """Wait until QEMU KernelVM is booted.
 546
 547         :param retries: Number of retries.
 548         :type retries: int
 549         """
 550         vpp_ver = VPPUtil.vpp_show_version(self._node)
 551
 552         for _ in range(retries):
 553             command = f"tail -1 {self._temp.get(u'log')}"
 554             stdout = None
 555             try:
 556                 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 557                 sleep(1)
 558             except RuntimeError:
 559                 pass
 560             if vpp_ver in stdout or u"Press enter to exit" in stdout:
 561                 break
 562             if u"reboot: Power down" in stdout:
 563                 raise RuntimeError(
 564                     f"QEMU: NF failed to run on {self._node[u'host']}!"
 565                 )
 566         else:
 567             raise RuntimeError(
 568                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 569             )
 570
 571     def _update_vm_interfaces(self):
 572         """Update interface names in VM node dict."""
 573         # Send guest-network-get-interfaces command via QGA, output example:
 574         # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
 575         # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
 576         out = self._qemu_qga_exec(u"guest-network-get-interfaces")
 577         interfaces = out.get(u"return")
 578         mac_name = {}
 579         if not interfaces:
 580             raise RuntimeError(
 581                 f"Get VM interface list failed on {self._node[u'host']}"
 582             )
 583         # Create MAC-name dict.
 584         for interface in interfaces:
 585             if u"hardware-address" not in interface:
 586                 continue
 587             mac_name[interface[u"hardware-address"]] = interface[u"name"]
 588         # Match interface by MAC and save interface name.
 589         for interface in self._vm_info[u"interfaces"].values():
 590             mac = interface.get(u"mac_address")
 591             if_name = mac_name.get(mac)
 592             if if_name is None:
 593                 logger.trace(f"Interface name for MAC {mac} not found")
 594             else:
 595                 interface[u"name"] = if_name
 596
 597     def qemu_start(self):
 598         """Start QEMU and wait until VM boot.
 599
 600         :returns: VM node info.
 601         :rtype: dict
 602         """
 603         cmd_opts = OptionString()
 604         cmd_opts.add(f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch}")
 605         cmd_opts.extend(self._params)
 606         message = f"QEMU: Start failed on {self._node[u'host']}!"
 607         try:
 608             DUTSetup.check_huge_page(
 609                 self._node, u"/dev/hugepages", self._opt.get(u"mem"))
 610
 611             exec_cmd_no_error(
 612                 self._node, cmd_opts, timeout=300, sudo=True, message=message
 613             )
 614             self._wait_until_vm_boot()
 615         except RuntimeError:
 616             self.qemu_kill_all()
 617             raise
 618         return self._vm_info
 619
 620     def qemu_kill(self):
 621         """Kill qemu process."""
 622         exec_cmd(
 623             self._node, f"chmod +r {self._temp.get(u'pidfile')}", sudo=True
 624         )
 625         exec_cmd(
 626             self._node, f"kill -SIGKILL $(cat {self._temp.get(u'pidfile')})",
 627             sudo=True
 628         )
 629
 630         for value in self._temp.values():
 631             exec_cmd(self._node, f"cat {value}", sudo=True)
 632             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 633
 634     def qemu_kill_all(self):
 635         """Kill all qemu processes on DUT node if specified."""
 636         exec_cmd(self._node, u"pkill -SIGKILL qemu", sudo=True)
 637
 638         for value in self._temp.values():
 639             exec_cmd(self._node, f"cat {value}", sudo=True)
 640             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 641
 642     def qemu_version(self):
 643         """Return Qemu version.
 644
 645         :returns: Qemu version.
 646         :rtype: str
 647         """
 648         command = f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch} " \
 649             f"--version"
 650         try:
 651             stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 652             return match(r"QEMU emulator version ([\d.]*)", stdout).group(1)
 653         except RuntimeError:
 654             self.qemu_kill_all()
 655             raise