resources/libraries/python/QemuUtils.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """QEMU utilities library."""
  15
  16 import json
  17
  18 from re import match
  19 from string import Template
  20 from time import sleep
  21
  22 from robot.api import logger
  23
  24 from resources.libraries.python.Constants import Constants
  25 from resources.libraries.python.DpdkUtil import DpdkUtil
  26 from resources.libraries.python.DUTSetup import DUTSetup
  27 from resources.libraries.python.OptionString import OptionString
  28 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
  29 from resources.libraries.python.topology import NodeType, Topology
  30 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
  31 from resources.libraries.python.VPPUtil import VPPUtil
  32
  33 __all__ = [u"QemuUtils"]
  34
  35
  36 class QemuUtils:
  37     """QEMU utilities."""
  38
  39     # Use one instance of class per tests.
  40     ROBOT_LIBRARY_SCOPE = u"TEST CASE"
  41
  42     def __init__(
  43             self, node, qemu_id=1, smp=1, mem=512, vnf=None,
  44             img=Constants.QEMU_VM_IMAGE):
  45         """Initialize QemuUtil class.
  46
  47         :param node: Node to run QEMU on.
  48         :param qemu_id: QEMU identifier.
  49         :param smp: Number of virtual SMP units (cores).
  50         :param mem: Amount of memory.
  51         :param vnf: Network function workload.
  52         :param img: QEMU disk image or kernel image path.
  53         :type node: dict
  54         :type qemu_id: int
  55         :type smp: int
  56         :type mem: int
  57         :type vnf: str
  58         :type img: str
  59         """
  60         self._vhost_id = 0
  61         self._node = node
  62         self._arch = Topology.get_node_arch(self._node)
  63         self._opt = dict()
  64
  65         # Architecture specific options
  66         if self._arch == u"aarch64":
  67             dpdk_target = u"arm64-armv8a"
  68             self._opt[u"machine_args"] = \
  69                 u"virt,accel=kvm,usb=off,mem-merge=off,gic-version=3"
  70             self._opt[u"console"] = u"ttyAMA0"
  71         else:
  72             dpdk_target = u"x86_64-native"
  73             self._opt[u"machine_args"] = u"pc,accel=kvm,usb=off,mem-merge=off"
  74             self._opt[u"console"] = u"ttyS0"
  75         self._testpmd_path = f"{Constants.QEMU_VM_DPDK}/" \
  76             f"{dpdk_target}-linux-gcc/app"
  77         self._vm_info = {
  78             u"host": node[u"host"],
  79             u"type": NodeType.VM,
  80             u"port": 10021 + qemu_id,
  81             u"serial": 4555 + qemu_id,
  82             u"username": 'cisco',
  83             u"password": 'cisco',
  84             u"interfaces": {},
  85         }
  86         if node[u"port"] != 22:
  87             self._vm_info[u"host_port"] = node[u"port"]
  88             self._vm_info[u"host_username"] = node[u"username"]
  89             self._vm_info[u"host_password"] = node[u"password"]
  90         # Input Options.
  91         self._opt[u"qemu_id"] = qemu_id
  92         self._opt[u"mem"] = int(mem)
  93         self._opt[u"smp"] = int(smp)
  94         self._opt[u"img"] = img
  95         self._opt[u"vnf"] = vnf
  96         # Temporary files.
  97         self._temp = dict()
  98         self._temp[u"pidfile"] = f"/run/qemu_{qemu_id}.pid"
  99         if img == Constants.QEMU_VM_IMAGE:
 100             self._opt[u"vm_type"] = u"nestedvm"
 101             self._temp[u"qmp"] = f"/run/qmp_{qemu_id}.sock"
 102             self._temp[u"qga"] = f"/run/qga_{qemu_id}.sock"
 103         elif img == Constants.QEMU_VM_KERNEL:
 104             self._opt[u"img"], _ = exec_cmd_no_error(
 105                 node, f"ls -1 {Constants.QEMU_VM_KERNEL}* | tail -1",
 106                 message=u"Qemu Kernel VM image not found!"
 107             )
 108             self._opt[u"vm_type"] = u"kernelvm"
 109             self._temp[u"log"] = f"/tmp/serial_{qemu_id}.log"
 110             self._temp[u"ini"] = f"/etc/vm_init_{qemu_id}.conf"
 111             self._opt[u"initrd"], _ = exec_cmd_no_error(
 112                 node, f"ls -1 {Constants.QEMU_VM_KERNEL_INITRD}* | tail -1",
 113                 message=u"Qemu Kernel initrd image not found!"
 114             )
 115         else:
 116             raise RuntimeError(f"QEMU: Unknown VM image option: {img}")
 117         # Computed parameters for QEMU command line.
 118         self._params = OptionString(prefix=u"-")
 119         self.add_params()
 120
 121     def add_params(self):
 122         """Set QEMU command line parameters."""
 123         self.add_default_params()
 124         if self._opt.get(u"vm_type", u"") == u"nestedvm":
 125             self.add_nestedvm_params()
 126         elif self._opt.get(u"vm_type", u"") == u"kernelvm":
 127             self.add_kernelvm_params()
 128         else:
 129             raise RuntimeError(u"QEMU: Unsupported VM type!")
 130
 131     def add_default_params(self):
 132         """Set default QEMU command line parameters."""
 133         self._params.add(u"daemonize")
 134         self._params.add(u"nodefaults")
 135         self._params.add_with_value(
 136             u"name", f"vnf{self._opt.get(u'qemu_id')},debug-threads=on"
 137         )
 138         self._params.add(u"no-user-config")
 139         self._params.add_with_value(u"monitor", u"none")
 140         self._params.add_with_value(u"display", u"none")
 141         self._params.add_with_value(u"vga", u"none")
 142         self._params.add(u"enable-kvm")
 143         self._params.add_with_value(u"pidfile", self._temp.get(u"pidfile"))
 144         self._params.add_with_value(u"cpu", u"host")
 145
 146         self._params.add_with_value(u"machine", self._opt.get(u"machine_args"))
 147         self._params.add_with_value(
 148             u"smp", f"{self._opt.get(u'smp')},sockets=1,"
 149             f"cores={self._opt.get(u'smp')},threads=1"
 150         )
 151         self._params.add_with_value(
 152             u"object", f"memory-backend-file,id=mem,"
 153             f"size={self._opt.get(u'mem')}M,mem-path=/dev/hugepages,share=on"
 154         )
 155         self._params.add_with_value(u"m", f"{self._opt.get(u'mem')}M")
 156         self._params.add_with_value(u"numa", u"node,memdev=mem")
 157         self._params.add_with_value(u"balloon", u"none")
 158
 159     def add_nestedvm_params(self):
 160         """Set NestedVM QEMU parameters."""
 161         self._params.add_with_value(
 162             u"net",
 163             f"nic,macaddr=52:54:00:00:{self._opt.get(u'qemu_id'):02x}:ff"
 164         )
 165         self._params.add_with_value(
 166             u"net", f"user,hostfwd=tcp::{self._vm_info[u'port']}-:22"
 167         )
 168         locking = u",file.locking=off"
 169         self._params.add_with_value(
 170             u"drive", f"file={self._opt.get(u'img')},"
 171             f"format=raw,cache=none,if=virtio{locking}"
 172         )
 173         self._params.add_with_value(
 174             u"qmp", f"unix:{self._temp.get(u'qmp')},server,nowait"
 175         )
 176         self._params.add_with_value(
 177             u"chardev", f"socket,host=127.0.0.1,"
 178             f"port={self._vm_info[u'serial']},id=gnc0,server,nowait")
 179         self._params.add_with_value(u"device", u"isa-serial,chardev=gnc0")
 180         self._params.add_with_value(
 181             u"chardev", f"socket,path={self._temp.get(u'qga')},"
 182             f"server,nowait,id=qga0"
 183         )
 184         self._params.add_with_value(u"device", u"isa-serial,chardev=qga0")
 185
 186     def add_kernelvm_params(self):
 187         """Set KernelVM QEMU parameters."""
 188         self._params.add_with_value(
 189             u"serial", f"file:{self._temp.get(u'log')}"
 190         )
 191         self._params.add_with_value(
 192             u"fsdev", u"local,id=root9p,path=/,security_model=none"
 193         )
 194         self._params.add_with_value(
 195             u"device", u"virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot"
 196         )
 197         self._params.add_with_value(u"kernel", f"{self._opt.get(u'img')}")
 198         self._params.add_with_value(u"initrd", f"{self._opt.get(u'initrd')}")
 199         self._params.add_with_value(
 200             u"append", f"'ro rootfstype=9p rootflags=trans=virtio "
 201             f"root=virtioroot console={self._opt.get(u'console')} "
 202             f"tsc=reliable hugepages=256 "
 203             f"init={self._temp.get(u'ini')} fastboot'"
 204         )
 205
 206     def create_kernelvm_config_vpp(self, **kwargs):
 207         """Create QEMU VPP config files.
 208
 209         :param kwargs: Key-value pairs to replace content of VPP configuration
 210             file.
 211         :type kwargs: dict
 212         """
 213         startup = f"/etc/vpp/vm_startup_{self._opt.get(u'qemu_id')}.conf"
 214         running = f"/etc/vpp/vm_running_{self._opt.get(u'qemu_id')}.exec"
 215
 216         self._temp[u"startup"] = startup
 217         self._temp[u"running"] = running
 218         self._opt[u"vnf_bin"] = f"/usr/bin/vpp -c {startup}"
 219
 220         # Create VPP startup configuration.
 221         vpp_config = VppConfigGenerator()
 222         vpp_config.set_node(self._node)
 223         vpp_config.add_unix_nodaemon()
 224         vpp_config.add_unix_cli_listen()
 225         vpp_config.add_unix_exec(running)
 226         vpp_config.add_socksvr()
 227         vpp_config.add_cpu_main_core(u"0")
 228         if self._opt.get(u"smp") > 1:
 229             vpp_config.add_cpu_corelist_workers(f"1-{self._opt.get(u'smp')-1}")
 230         vpp_config.add_dpdk_dev(u"0000:00:06.0", u"0000:00:07.0")
 231         vpp_config.add_dpdk_dev_default_rxq(kwargs[u"queues"])
 232         vpp_config.add_dpdk_log_level(u"debug")
 233         if not kwargs[u"jumbo_frames"]:
 234             vpp_config.add_dpdk_no_multi_seg()
 235             vpp_config.add_dpdk_no_tx_checksum_offload()
 236         vpp_config.add_plugin(u"disable", u"default")
 237         vpp_config.add_plugin(u"enable", u"dpdk_plugin.so")
 238         vpp_config.write_config(startup)
 239
 240         # Create VPP running configuration.
 241         template = f"{Constants.RESOURCES_TPL_VM}/{self._opt.get(u'vnf')}.exec"
 242         exec_cmd_no_error(self._node, f"rm -f {running}", sudo=True)
 243
 244         with open(template, u"rt") as src_file:
 245             src = Template(src_file.read())
 246             exec_cmd_no_error(
 247                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 248                 f"sudo tee {running}"
 249             )
 250
 251     def create_kernelvm_config_testpmd_io(self, **kwargs):
 252         """Create QEMU testpmd-io command line.
 253
 254         :param kwargs: Key-value pairs to construct command line parameters.
 255         :type kwargs: dict
 256         """
 257         pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
 258         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 259             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 260             eal_driver=False,
 261             eal_pci_whitelist0=u"0000:00:06.0",
 262             eal_pci_whitelist1=u"0000:00:07.0",
 263             eal_in_memory=True,
 264             pmd_num_mbufs=16384,
 265             pmd_fwd_mode=u"io",
 266             pmd_nb_ports=u"2",
 267             pmd_portmask=u"0x3",
 268             pmd_max_pkt_len=pmd_max_pkt_len,
 269             pmd_mbuf_size=u"16384",
 270             pmd_rxq=kwargs[u"queues"],
 271             pmd_txq=kwargs[u"queues"],
 272             pmd_tx_offloads='0x0',
 273             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 274         )
 275
 276         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 277
 278     def create_kernelvm_config_testpmd_mac(self, **kwargs):
 279         """Create QEMU testpmd-mac command line.
 280
 281         :param kwargs: Key-value pairs to construct command line parameters.
 282         :type kwargs: dict
 283         """
 284         pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
 285         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 286             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 287             eal_driver=False,
 288             eal_pci_whitelist0=u"0000:00:06.0",
 289             eal_pci_whitelist1=u"0000:00:07.0",
 290             eal_in_memory=True,
 291             pmd_num_mbufs=16384,
 292             pmd_fwd_mode=u"mac",
 293             pmd_nb_ports=u"2",
 294             pmd_portmask=u"0x3",
 295             pmd_max_pkt_len=pmd_max_pkt_len,
 296             pmd_mbuf_size=u"16384",
 297             pmd_eth_peer_0=f"0,{kwargs[u'vif1_mac']}",
 298             pmd_eth_peer_1=f"1,{kwargs[u'vif2_mac']}",
 299             pmd_rxq=kwargs[u"queues"],
 300             pmd_txq=kwargs[u"queues"],
 301             pmd_tx_offloads=u"0x0",
 302             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 303         )
 304
 305         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 306
 307     def create_kernelvm_init(self, **kwargs):
 308         """Create QEMU init script.
 309
 310         :param kwargs: Key-value pairs to replace content of init startup file.
 311         :type kwargs: dict
 312         """
 313         template = f"{Constants.RESOURCES_TPL_VM}/init.sh"
 314         init = self._temp.get(u"ini")
 315         exec_cmd_no_error(self._node, f"rm -f {init}", sudo=True)
 316
 317         with open(template, u"rt") as src_file:
 318             src = Template(src_file.read())
 319             exec_cmd_no_error(
 320                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 321                 f"sudo tee {init}"
 322             )
 323             exec_cmd_no_error(self._node, f"chmod +x {init}", sudo=True)
 324
 325     def configure_kernelvm_vnf(self, **kwargs):
 326         """Create KernelVM VNF configurations.
 327
 328         :param kwargs: Key-value pairs for templating configs.
 329         :type kwargs: dict
 330         """
 331         if u"vpp" in self._opt.get(u"vnf"):
 332             self.create_kernelvm_config_vpp(**kwargs)
 333         elif u"testpmd_io" in self._opt.get(u"vnf"):
 334             self.create_kernelvm_config_testpmd_io(**kwargs)
 335         elif u"testpmd_mac" in self._opt.get(u"vnf"):
 336             self.create_kernelvm_config_testpmd_mac(**kwargs)
 337         else:
 338             raise RuntimeError(u"QEMU: Unsupported VNF!")
 339         self.create_kernelvm_init(vnf_bin=self._opt.get(u"vnf_bin"))
 340
 341     def get_qemu_pids(self):
 342         """Get QEMU CPU pids.
 343
 344         :returns: List of QEMU CPU pids.
 345         :rtype: list of str
 346         """
 347         command = f"grep -rwl 'CPU' /proc/$(sudo cat " \
 348             f"{self._temp.get(u'pidfile')})/task/*/comm "
 349         command += r"| xargs dirname | sed -e 's/\/.*\///g' | uniq"
 350
 351         stdout, _ = exec_cmd_no_error(self._node, command)
 352         return stdout.splitlines()
 353
 354     def qemu_set_affinity(self, *host_cpus):
 355         """Set qemu affinity by getting thread PIDs via QMP and taskset to list
 356         of CPU cores. Function tries to execute 3 times to avoid race condition
 357         in getting thread PIDs.
 358
 359         :param host_cpus: List of CPU cores.
 360         :type host_cpus: list
 361         """
 362         for _ in range(3):
 363             try:
 364                 qemu_cpus = self.get_qemu_pids()
 365
 366                 if len(qemu_cpus) != len(host_cpus):
 367                     sleep(1)
 368                     continue
 369                 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
 370                     command = f"taskset -pc {host_cpu} {qemu_cpu}"
 371                     message = f"QEMU: Set affinity failed " \
 372                         f"on {self._node[u'host']}!"
 373                     exec_cmd_no_error(
 374                         self._node, command, sudo=True, message=message
 375                     )
 376                 break
 377             except (RuntimeError, ValueError):
 378                 self.qemu_kill_all()
 379                 raise
 380         else:
 381             self.qemu_kill_all()
 382             raise RuntimeError(u"Failed to set Qemu threads affinity!")
 383
 384     def qemu_set_scheduler_policy(self):
 385         """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
 386         processes.
 387
 388         :raises RuntimeError: Set scheduler policy failed.
 389         """
 390         try:
 391             qemu_cpus = self.get_qemu_pids()
 392
 393             for qemu_cpu in qemu_cpus:
 394                 command = f"chrt -r -p 1 {qemu_cpu}"
 395                 message = f"QEMU: Set SCHED_RR failed on {self._node[u'host']}"
 396                 exec_cmd_no_error(
 397                     self._node, command, sudo=True, message=message
 398                 )
 399         except (RuntimeError, ValueError):
 400             self.qemu_kill_all()
 401             raise
 402
 403     def qemu_add_vhost_user_if(
 404             self, socket, server=True, jumbo_frames=False, queue_size=None,
 405             queues=1, csum=False, gso=False):
 406         """Add Vhost-user interface.
 407
 408         :param socket: Path of the unix socket.
 409         :param server: If True the socket shall be a listening socket.
 410         :param jumbo_frames: Set True if jumbo frames are used in the test.
 411         :param queue_size: Vring queue size.
 412         :param queues: Number of queues.
 413         :param csum: Checksum offloading.
 414         :param gso: Generic segmentation offloading.
 415         :type socket: str
 416         :type server: bool
 417         :type jumbo_frames: bool
 418         :type queue_size: int
 419         :type queues: int
 420         :type csum: bool
 421         :type gso: bool
 422         """
 423         self._vhost_id += 1
 424         self._params.add_with_value(
 425             u"chardev", f"socket,id=char{self._vhost_id},"
 426             f"path={socket}{u',server' if server is True else u''}"
 427         )
 428         self._params.add_with_value(
 429             u"netdev", f"vhost-user,id=vhost{self._vhost_id},"
 430             f"chardev=char{self._vhost_id},queues={queues}"
 431         )
 432         mac = f"52:54:00:00:{self._opt.get(u'qemu_id'):02x}:" \
 433             f"{self._vhost_id:02x}"
 434         queue_size = f"rx_queue_size={queue_size},tx_queue_size={queue_size}" \
 435             if queue_size else u""
 436         self._params.add_with_value(
 437             u"device", f"virtio-net-pci,netdev=vhost{self._vhost_id},mac={mac},"
 438             f"addr={self._vhost_id+5}.0,mq=on,vectors={2 * queues + 2},"
 439             f"csum={u'on' if csum else u'off'},gso={u'on' if gso else u'off'},"
 440             f"guest_tso4=off,guest_tso6=off,guest_ecn=off,"
 441             f"{queue_size}"
 442         )
 443
 444         # Add interface MAC and socket to the node dict.
 445         if_data = {u"mac_address": mac, u"socket": socket}
 446         if_name = f"vhost{self._vhost_id}"
 447         self._vm_info[u"interfaces"][if_name] = if_data
 448         # Add socket to temporary file list.
 449         self._temp[if_name] = socket
 450
 451     def _qemu_qmp_exec(self, cmd):
 452         """Execute QMP command.
 453
 454         QMP is JSON based protocol which allows to control QEMU instance.
 455
 456         :param cmd: QMP command to execute.
 457         :type cmd: str
 458         :returns: Command output in python representation of JSON format. The
 459             { "return": {} } response is QMP's success response. An error
 460             response will contain the "error" keyword instead of "return".
 461         """
 462         # To enter command mode, the qmp_capabilities command must be issued.
 463         command = f"echo \"{{{{ \\\"execute\\\": " \
 464             f"\\\"qmp_capabilities\\\" }}}}" \
 465             f"{{{{ \\\"execute\\\": \\\"{cmd}\\\" }}}}\" | " \
 466             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qmp')}"
 467         message = f"QMP execute '{cmd}' failed on {self._node[u'host']}"
 468
 469         stdout, _ = exec_cmd_no_error(
 470             self._node, command, sudo=False, message=message
 471         )
 472
 473         # Skip capabilities negotiation messages.
 474         out_list = stdout.splitlines()
 475         if len(out_list) < 3:
 476             raise RuntimeError(f"Invalid QMP output on {self._node[u'host']}")
 477         return json.loads(out_list[2])
 478
 479     def _qemu_qga_flush(self):
 480         """Flush the QGA parser state."""
 481         command = f"(printf \"\xFF\"; sleep 1) | sudo -S socat " \
 482             f"- UNIX-CONNECT:{self._temp.get(u'qga')}"
 483         message = f"QGA flush failed on {self._node[u'host']}"
 484         stdout, _ = exec_cmd_no_error(
 485             self._node, command, sudo=False, message=message
 486         )
 487
 488         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 489
 490     def _qemu_qga_exec(self, cmd):
 491         """Execute QGA command.
 492
 493         QGA provide access to a system-level agent via standard QMP commands.
 494
 495         :param cmd: QGA command to execute.
 496         :type cmd: str
 497         """
 498         command = f"(echo \"{{{{ \\\"execute\\\": " \
 499             f"\\\"{cmd}\\\" }}}}\"; sleep 1) | " \
 500             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qga')}"
 501         message = f"QGA execute '{cmd}' failed on {self._node[u'host']}"
 502         stdout, _ = exec_cmd_no_error(
 503             self._node, command, sudo=False, message=message
 504         )
 505
 506         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 507
 508     def _wait_until_vm_boot(self):
 509         """Wait until QEMU with NestedVM is booted."""
 510         if self._opt.get(u"vm_type") == u"nestedvm":
 511             self._wait_until_nestedvm_boot()
 512             self._update_vm_interfaces()
 513         elif self._opt.get(u"vm_type") == u"kernelvm":
 514             self._wait_until_kernelvm_boot()
 515         else:
 516             raise RuntimeError(u"QEMU: Unsupported VM type!")
 517
 518     def _wait_until_nestedvm_boot(self, retries=12):
 519         """Wait until QEMU with NestedVM is booted.
 520
 521         First try to flush qga until there is output.
 522         Then ping QEMU guest agent each 5s until VM booted or timeout.
 523
 524         :param retries: Number of retries with 5s between trials.
 525         :type retries: int
 526         """
 527         for _ in range(retries):
 528             out = None
 529             try:
 530                 out = self._qemu_qga_flush()
 531             except ValueError:
 532                 logger.trace(f"QGA qga flush unexpected output {out}")
 533             # Empty output - VM not booted yet
 534             if not out:
 535                 sleep(5)
 536             else:
 537                 break
 538         else:
 539             raise RuntimeError(
 540                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 541             )
 542         for _ in range(retries):
 543             out = None
 544             try:
 545                 out = self._qemu_qga_exec(u"guest-ping")
 546             except ValueError:
 547                 logger.trace(f"QGA guest-ping unexpected output {out}")
 548             # Empty output - VM not booted yet.
 549             if not out:
 550                 sleep(5)
 551             # Non-error return - VM booted.
 552             elif out.get(u"return") is not None:
 553                 break
 554             # Skip error and wait.
 555             elif out.get(u"error") is not None:
 556                 sleep(5)
 557             else:
 558                 # If there is an unexpected output from QGA guest-info, try
 559                 # again until timeout.
 560                 logger.trace(f"QGA guest-ping unexpected output {out}")
 561         else:
 562             raise RuntimeError(
 563                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 564             )
 565
 566     def _wait_until_kernelvm_boot(self, retries=60):
 567         """Wait until QEMU KernelVM is booted.
 568
 569         :param retries: Number of retries.
 570         :type retries: int
 571         """
 572         vpp_ver = VPPUtil.vpp_show_version(self._node)
 573
 574         for _ in range(retries):
 575             command = f"tail -1 {self._temp.get(u'log')}"
 576             stdout = None
 577             try:
 578                 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 579                 sleep(1)
 580             except RuntimeError:
 581                 pass
 582             if vpp_ver in stdout or u"Press enter to exit" in stdout:
 583                 break
 584             if u"reboot: Power down" in stdout:
 585                 raise RuntimeError(
 586                     f"QEMU: NF failed to run on {self._node[u'host']}!"
 587                 )
 588         else:
 589             raise RuntimeError(
 590                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 591             )
 592
 593     def _update_vm_interfaces(self):
 594         """Update interface names in VM node dict."""
 595         # Send guest-network-get-interfaces command via QGA, output example:
 596         # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
 597         # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
 598         out = self._qemu_qga_exec(u"guest-network-get-interfaces")
 599         interfaces = out.get(u"return")
 600         mac_name = {}
 601         if not interfaces:
 602             raise RuntimeError(
 603                 f"Get VM interface list failed on {self._node[u'host']}"
 604             )
 605         # Create MAC-name dict.
 606         for interface in interfaces:
 607             if u"hardware-address" not in interface:
 608                 continue
 609             mac_name[interface[u"hardware-address"]] = interface[u"name"]
 610         # Match interface by MAC and save interface name.
 611         for interface in self._vm_info[u"interfaces"].values():
 612             mac = interface.get(u"mac_address")
 613             if_name = mac_name.get(mac)
 614             if if_name is None:
 615                 logger.trace(f"Interface name for MAC {mac} not found")
 616             else:
 617                 interface[u"name"] = if_name
 618
 619     def qemu_start(self):
 620         """Start QEMU and wait until VM boot.
 621
 622         :returns: VM node info.
 623         :rtype: dict
 624         """
 625         cmd_opts = OptionString()
 626         cmd_opts.add(f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch}")
 627         cmd_opts.extend(self._params)
 628         message = f"QEMU: Start failed on {self._node[u'host']}!"
 629         try:
 630             DUTSetup.check_huge_page(
 631                 self._node, u"/dev/hugepages", int(self._opt.get(u"mem")))
 632
 633             exec_cmd_no_error(
 634                 self._node, cmd_opts, timeout=300, sudo=True, message=message
 635             )
 636             self._wait_until_vm_boot()
 637         except RuntimeError:
 638             self.qemu_kill_all()
 639             raise
 640         return self._vm_info
 641
 642     def qemu_kill(self):
 643         """Kill qemu process."""
 644         exec_cmd(
 645             self._node, f"chmod +r {self._temp.get(u'pidfile')}", sudo=True
 646         )
 647         exec_cmd(
 648             self._node, f"kill -SIGKILL $(cat {self._temp.get(u'pidfile')})",
 649             sudo=True
 650         )
 651
 652         for value in self._temp.values():
 653             exec_cmd(self._node, f"cat {value}", sudo=True)
 654             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 655
 656     def qemu_kill_all(self):
 657         """Kill all qemu processes on DUT node if specified."""
 658         exec_cmd(self._node, u"pkill -SIGKILL qemu", sudo=True)
 659
 660         for value in self._temp.values():
 661             exec_cmd(self._node, f"cat {value}", sudo=True)
 662             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 663
 664     def qemu_version(self):
 665         """Return Qemu version.
 666
 667         :returns: Qemu version.
 668         :rtype: str
 669         """
 670         command = f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch} " \
 671             f"--version"
 672         try:
 673             stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 674             return match(r"QEMU emulator version ([\d.]*)", stdout).group(1)
 675         except RuntimeError:
 676             self.qemu_kill_all()
 677             raise