resources/libraries/python/QemuUtils.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """QEMU utilities library."""
  15
  16 import json
  17
  18 from re import match
  19 from string import Template
  20 from time import sleep
  21
  22 from robot.api import logger
  23
  24 from resources.libraries.python.Constants import Constants
  25 from resources.libraries.python.DpdkUtil import DpdkUtil
  26 from resources.libraries.python.DUTSetup import DUTSetup
  27 from resources.libraries.python.OptionString import OptionString
  28 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
  29 from resources.libraries.python.topology import NodeType, Topology
  30 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
  31 from resources.libraries.python.VPPUtil import VPPUtil
  32
  33 __all__ = [u"QemuUtils"]
  34
  35
  36 class QemuUtils:
  37     """QEMU utilities."""
  38
  39     # Use one instance of class per tests.
  40     ROBOT_LIBRARY_SCOPE = u"TEST CASE"
  41
  42     def __init__(
  43             self, node, qemu_id=1, smp=1, mem=512, vnf=None,
  44             img=Constants.QEMU_VM_IMAGE):
  45         """Initialize QemuUtil class.
  46
  47         :param node: Node to run QEMU on.
  48         :param qemu_id: QEMU identifier.
  49         :param smp: Number of virtual SMP units (cores).
  50         :param mem: Amount of memory.
  51         :param vnf: Network function workload.
  52         :param img: QEMU disk image or kernel image path.
  53         :type node: dict
  54         :type qemu_id: int
  55         :type smp: int
  56         :type mem: int
  57         :type vnf: str
  58         :type img: str
  59         """
  60         self._vhost_id = 0
  61         self._node = node
  62         self._arch = Topology.get_node_arch(self._node)
  63         self._opt = dict()
  64
  65         # Architecture specific options
  66         if self._arch == u"aarch64":
  67             dpdk_target = u"arm64-armv8a"
  68             self._opt[u"machine_args"] = \
  69                 u"virt,accel=kvm,usb=off,mem-merge=off,gic-version=3"
  70             self._opt[u"console"] = u"ttyAMA0"
  71         else:
  72             dpdk_target = u"x86_64-native"
  73             self._opt[u"machine_args"] = u"pc,accel=kvm,usb=off,mem-merge=off"
  74             self._opt[u"console"] = u"ttyS0"
  75         self._testpmd_path = f"{Constants.QEMU_VM_DPDK}/" \
  76             f"{dpdk_target}-linux-gcc/app"
  77         self._vm_info = {
  78             u"host": node[u"host"],
  79             u"type": NodeType.VM,
  80             u"port": 10021 + qemu_id,
  81             u"serial": 4555 + qemu_id,
  82             u"username": 'cisco',
  83             u"password": 'cisco',
  84             u"interfaces": {},
  85         }
  86         if node[u"port"] != 22:
  87             self._vm_info[u"host_port"] = node[u"port"]
  88             self._vm_info[u"host_username"] = node[u"username"]
  89             self._vm_info[u"host_password"] = node[u"password"]
  90         # Input Options.
  91         self._opt[u"qemu_id"] = qemu_id
  92         self._opt[u"mem"] = int(mem)
  93         self._opt[u"smp"] = int(smp)
  94         self._opt[u"img"] = img
  95         self._opt[u"vnf"] = vnf
  96         # Temporary files.
  97         self._temp = dict()
  98         self._temp[u"pidfile"] = f"/run/qemu_{qemu_id}.pid"
  99         if img == Constants.QEMU_VM_IMAGE:
 100             self._opt[u"vm_type"] = u"nestedvm"
 101             self._temp[u"qmp"] = f"/run/qmp_{qemu_id}.sock"
 102             self._temp[u"qga"] = f"/run/qga_{qemu_id}.sock"
 103         elif img == Constants.QEMU_VM_KERNEL:
 104             self._opt[u"img"], _ = exec_cmd_no_error(
 105                 node, f"ls -1 {Constants.QEMU_VM_KERNEL}* | tail -1",
 106                 message=u"Qemu Kernel VM image not found!"
 107             )
 108             self._opt[u"vm_type"] = u"kernelvm"
 109             self._temp[u"log"] = f"/tmp/serial_{qemu_id}.log"
 110             self._temp[u"ini"] = f"/etc/vm_init_{qemu_id}.conf"
 111             self._opt[u"initrd"], _ = exec_cmd_no_error(
 112                 node, f"ls -1 {Constants.QEMU_VM_KERNEL_INITRD}* | tail -1",
 113                 message=u"Qemu Kernel initrd image not found!"
 114             )
 115         else:
 116             raise RuntimeError(f"QEMU: Unknown VM image option: {img}")
 117         # Computed parameters for QEMU command line.
 118         self._params = OptionString(prefix=u"-")
 119         self.add_params()
 120
 121     def add_params(self):
 122         """Set QEMU command line parameters."""
 123         self.add_default_params()
 124         if self._opt.get(u"vm_type", u"") == u"nestedvm":
 125             self.add_nestedvm_params()
 126         elif self._opt.get(u"vm_type", u"") == u"kernelvm":
 127             self.add_kernelvm_params()
 128         else:
 129             raise RuntimeError(u"QEMU: Unsupported VM type!")
 130
 131     def add_default_params(self):
 132         """Set default QEMU command line parameters."""
 133         self._params.add(u"daemonize")
 134         self._params.add(u"nodefaults")
 135         self._params.add_with_value(
 136             u"name", f"vnf{self._opt.get(u'qemu_id')},debug-threads=on"
 137         )
 138         self._params.add(u"no-user-config")
 139         self._params.add_with_value(u"monitor", u"none")
 140         self._params.add_with_value(u"display", u"none")
 141         self._params.add_with_value(u"vga", u"none")
 142         self._params.add(u"enable-kvm")
 143         self._params.add_with_value(u"pidfile", self._temp.get(u"pidfile"))
 144         self._params.add_with_value(u"cpu", u"host")
 145
 146         self._params.add_with_value(u"machine", self._opt.get(u"machine_args"))
 147         self._params.add_with_value(
 148             u"smp", f"{self._opt.get(u'smp')},sockets=1,"
 149             f"cores={self._opt.get(u'smp')},threads=1"
 150         )
 151         self._params.add_with_value(
 152             u"object", f"memory-backend-file,id=mem,"
 153             f"size={self._opt.get(u'mem')}M,mem-path=/dev/hugepages,share=on"
 154         )
 155         self._params.add_with_value(u"m", f"{self._opt.get(u'mem')}M")
 156         self._params.add_with_value(u"numa", u"node,memdev=mem")
 157         self._params.add_with_value(u"balloon", u"none")
 158
 159     def add_nestedvm_params(self):
 160         """Set NestedVM QEMU parameters."""
 161         self._params.add_with_value(
 162             u"net",
 163             f"nic,macaddr=52:54:00:00:{self._opt.get(u'qemu_id'):02x}:ff"
 164         )
 165         self._params.add_with_value(
 166             u"net", f"user,hostfwd=tcp::{self._vm_info[u'port']}-:22"
 167         )
 168         locking = u",file.locking=off"
 169         self._params.add_with_value(
 170             u"drive", f"file={self._opt.get(u'img')},"
 171             f"format=raw,cache=none,if=virtio{locking}"
 172         )
 173         self._params.add_with_value(
 174             u"qmp", f"unix:{self._temp.get(u'qmp')},server,nowait"
 175         )
 176         self._params.add_with_value(
 177             u"chardev", f"socket,host=127.0.0.1,"
 178             f"port={self._vm_info[u'serial']},id=gnc0,server,nowait")
 179         self._params.add_with_value(u"device", u"isa-serial,chardev=gnc0")
 180         self._params.add_with_value(
 181             u"chardev", f"socket,path={self._temp.get(u'qga')},"
 182             f"server,nowait,id=qga0"
 183         )
 184         self._params.add_with_value(u"device", u"isa-serial,chardev=qga0")
 185
 186     def add_kernelvm_params(self):
 187         """Set KernelVM QEMU parameters."""
 188         self._params.add_with_value(
 189             u"serial", f"file:{self._temp.get(u'log')}"
 190         )
 191         self._params.add_with_value(
 192             u"fsdev", u"local,id=root9p,path=/,security_model=none"
 193         )
 194         self._params.add_with_value(
 195             u"device", u"virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot"
 196         )
 197         self._params.add_with_value(u"kernel", f"{self._opt.get(u'img')}")
 198         self._params.add_with_value(u"initrd", f"{self._opt.get(u'initrd')}")
 199         self._params.add_with_value(
 200             u"append", f"'ro rootfstype=9p rootflags=trans=virtio "
 201             f"root=virtioroot console={self._opt.get(u'console')} "
 202             f"tsc=reliable hugepages=256 "
 203             f"init={self._temp.get(u'ini')} fastboot'"
 204         )
 205
 206     def create_kernelvm_config_vpp(self, **kwargs):
 207         """Create QEMU VPP config files.
 208
 209         :param kwargs: Key-value pairs to replace content of VPP configuration
 210             file.
 211         :type kwargs: dict
 212         """
 213         startup = f"/etc/vpp/vm_startup_{self._opt.get(u'qemu_id')}.conf"
 214         running = f"/etc/vpp/vm_running_{self._opt.get(u'qemu_id')}.exec"
 215
 216         self._temp[u"startup"] = startup
 217         self._temp[u"running"] = running
 218         self._opt[u"vnf_bin"] = f"/usr/bin/vpp -c {startup}"
 219
 220         # Create VPP startup configuration.
 221         vpp_config = VppConfigGenerator()
 222         vpp_config.set_node(self._node)
 223         vpp_config.add_unix_nodaemon()
 224         vpp_config.add_unix_cli_listen()
 225         vpp_config.add_unix_exec(running)
 226         vpp_config.add_socksvr()
 227         vpp_config.add_cpu_main_core(u"0")
 228         if self._opt.get(u"smp") > 1:
 229             vpp_config.add_cpu_corelist_workers(f"1-{self._opt.get(u'smp')-1}")
 230         vpp_config.add_dpdk_dev(u"0000:00:06.0", u"0000:00:07.0")
 231         vpp_config.add_dpdk_dev_default_rxq(kwargs[u"queues"])
 232         vpp_config.add_dpdk_log_level(u"debug")
 233         if not kwargs[u"jumbo_frames"]:
 234             vpp_config.add_dpdk_no_multi_seg()
 235             vpp_config.add_dpdk_no_tx_checksum_offload()
 236         vpp_config.add_plugin(u"disable", u"default")
 237         vpp_config.add_plugin(u"enable", u"dpdk_plugin.so")
 238         vpp_config.write_config(startup)
 239
 240         # Create VPP running configuration.
 241         template = f"{Constants.RESOURCES_TPL_VM}/{self._opt.get(u'vnf')}.exec"
 242         exec_cmd_no_error(self._node, f"rm -f {running}", sudo=True)
 243
 244         with open(template, u"rt") as src_file:
 245             src = Template(src_file.read())
 246             exec_cmd_no_error(
 247                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 248                 f"sudo tee {running}"
 249             )
 250
 251     def create_kernelvm_config_testpmd_io(self, **kwargs):
 252         """Create QEMU testpmd-io command line.
 253
 254         :param kwargs: Key-value pairs to construct command line parameters.
 255         :type kwargs: dict
 256         """
 257         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 258             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 259             eal_driver=False,
 260             eal_in_memory=True,
 261             pmd_num_mbufs=16384,
 262             pmd_rxq=kwargs[u"queues"],
 263             pmd_txq=kwargs[u"queues"],
 264             pmd_tx_offloads='0x0',
 265             pmd_disable_hw_vlan=False,
 266             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 267         )
 268
 269         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 270
 271     def create_kernelvm_config_testpmd_mac(self, **kwargs):
 272         """Create QEMU testpmd-mac command line.
 273
 274         :param kwargs: Key-value pairs to construct command line parameters.
 275         :type kwargs: dict
 276         """
 277         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 278             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 279             eal_driver=False,
 280             eal_in_memory=True,
 281             pmd_num_mbufs=16384,
 282             pmd_fwd_mode=u"mac",
 283             pmd_eth_peer_0=f"0,{kwargs[u'vif1_mac']}",
 284             pmd_eth_peer_1=f"1,{kwargs[u'vif2_mac']}",
 285             pmd_rxq=kwargs[u"queues"],
 286             pmd_txq=kwargs[u"queues"],
 287             pmd_tx_offloads=u"0x0",
 288             pmd_disable_hw_vlan=False,
 289             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 290         )
 291
 292         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 293
 294     def create_kernelvm_init(self, **kwargs):
 295         """Create QEMU init script.
 296
 297         :param kwargs: Key-value pairs to replace content of init startup file.
 298         :type kwargs: dict
 299         """
 300         template = f"{Constants.RESOURCES_TPL_VM}/init.sh"
 301         init = self._temp.get(u"ini")
 302         exec_cmd_no_error(self._node, f"rm -f {init}", sudo=True)
 303
 304         with open(template, u"rt") as src_file:
 305             src = Template(src_file.read())
 306             exec_cmd_no_error(
 307                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 308                 f"sudo tee {init}"
 309             )
 310             exec_cmd_no_error(self._node, f"chmod +x {init}", sudo=True)
 311
 312     def configure_kernelvm_vnf(self, **kwargs):
 313         """Create KernelVM VNF configurations.
 314
 315         :param kwargs: Key-value pairs for templating configs.
 316         :type kwargs: dict
 317         """
 318         if u"vpp" in self._opt.get(u"vnf"):
 319             self.create_kernelvm_config_vpp(**kwargs)
 320         elif u"testpmd_io" in self._opt.get(u"vnf"):
 321             self.create_kernelvm_config_testpmd_io(**kwargs)
 322         elif u"testpmd_mac" in self._opt.get(u"vnf"):
 323             self.create_kernelvm_config_testpmd_mac(**kwargs)
 324         else:
 325             raise RuntimeError(u"QEMU: Unsupported VNF!")
 326         self.create_kernelvm_init(vnf_bin=self._opt.get(u"vnf_bin"))
 327
 328     def get_qemu_pids(self):
 329         """Get QEMU CPU pids.
 330
 331         :returns: List of QEMU CPU pids.
 332         :rtype: list of str
 333         """
 334         command = f"grep -rwl 'CPU' /proc/$(sudo cat " \
 335             f"{self._temp.get(u'pidfile')})/task/*/comm "
 336         command += r"| xargs dirname | sed -e 's/\/.*\///g' | uniq"
 337
 338         stdout, _ = exec_cmd_no_error(self._node, command)
 339         return stdout.splitlines()
 340
 341     def qemu_set_affinity(self, *host_cpus):
 342         """Set qemu affinity by getting thread PIDs via QMP and taskset to list
 343         of CPU cores. Function tries to execute 3 times to avoid race condition
 344         in getting thread PIDs.
 345
 346         :param host_cpus: List of CPU cores.
 347         :type host_cpus: list
 348         """
 349         for _ in range(3):
 350             try:
 351                 qemu_cpus = self.get_qemu_pids()
 352
 353                 if len(qemu_cpus) != len(host_cpus):
 354                     sleep(1)
 355                     continue
 356                 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
 357                     command = f"taskset -pc {host_cpu} {qemu_cpu}"
 358                     message = f"QEMU: Set affinity failed " \
 359                         f"on {self._node[u'host']}!"
 360                     exec_cmd_no_error(
 361                         self._node, command, sudo=True, message=message
 362                     )
 363                 break
 364             except (RuntimeError, ValueError):
 365                 self.qemu_kill_all()
 366                 raise
 367         else:
 368             self.qemu_kill_all()
 369             raise RuntimeError(u"Failed to set Qemu threads affinity!")
 370
 371     def qemu_set_scheduler_policy(self):
 372         """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
 373         processes.
 374
 375         :raises RuntimeError: Set scheduler policy failed.
 376         """
 377         try:
 378             qemu_cpus = self.get_qemu_pids()
 379
 380             for qemu_cpu in qemu_cpus:
 381                 command = f"chrt -r -p 1 {qemu_cpu}"
 382                 message = f"QEMU: Set SCHED_RR failed on {self._node[u'host']}"
 383                 exec_cmd_no_error(
 384                     self._node, command, sudo=True, message=message
 385                 )
 386         except (RuntimeError, ValueError):
 387             self.qemu_kill_all()
 388             raise
 389
 390     def qemu_add_vhost_user_if(
 391             self, socket, server=True, jumbo_frames=False, queue_size=None,
 392             queues=1, csum=False, gso=False):
 393         """Add Vhost-user interface.
 394
 395         :param socket: Path of the unix socket.
 396         :param server: If True the socket shall be a listening socket.
 397         :param jumbo_frames: Set True if jumbo frames are used in the test.
 398         :param queue_size: Vring queue size.
 399         :param queues: Number of queues.
 400         :param csum: Checksum offloading.
 401         :param gso: Generic segmentation offloading.
 402         :type socket: str
 403         :type server: bool
 404         :type jumbo_frames: bool
 405         :type queue_size: int
 406         :type queues: int
 407         :type csum: bool
 408         :type gso: bool
 409         """
 410         self._vhost_id += 1
 411         self._params.add_with_value(
 412             u"chardev", f"socket,id=char{self._vhost_id},"
 413             f"path={socket}{u',server' if server is True else u''}"
 414         )
 415         self._params.add_with_value(
 416             u"netdev", f"vhost-user,id=vhost{self._vhost_id},"
 417             f"chardev=char{self._vhost_id},queues={queues}"
 418         )
 419         mac = f"52:54:00:00:{self._opt.get(u'qemu_id'):02x}:" \
 420             f"{self._vhost_id:02x}"
 421         queue_size = f"rx_queue_size={queue_size},tx_queue_size={queue_size}" \
 422             if queue_size else u""
 423         self._params.add_with_value(
 424             u"device", f"virtio-net-pci,netdev=vhost{self._vhost_id},mac={mac},"
 425             f"addr={self._vhost_id+5}.0,mq=on,vectors={2 * queues + 2},"
 426             f"csum={u'on' if csum else u'off'},gso={u'on' if gso else u'off'},"
 427             f"guest_tso4=off,guest_tso6=off,guest_ecn=off,"
 428             f"{queue_size}"
 429         )
 430
 431         # Add interface MAC and socket to the node dict.
 432         if_data = {u"mac_address": mac, u"socket": socket}
 433         if_name = f"vhost{self._vhost_id}"
 434         self._vm_info[u"interfaces"][if_name] = if_data
 435         # Add socket to temporary file list.
 436         self._temp[if_name] = socket
 437
 438     def _qemu_qmp_exec(self, cmd):
 439         """Execute QMP command.
 440
 441         QMP is JSON based protocol which allows to control QEMU instance.
 442
 443         :param cmd: QMP command to execute.
 444         :type cmd: str
 445         :returns: Command output in python representation of JSON format. The
 446             { "return": {} } response is QMP's success response. An error
 447             response will contain the "error" keyword instead of "return".
 448         """
 449         # To enter command mode, the qmp_capabilities command must be issued.
 450         command = f"echo \"{{{{ \\\"execute\\\": " \
 451             f"\\\"qmp_capabilities\\\" }}}}" \
 452             f"{{{{ \\\"execute\\\": \\\"{cmd}\\\" }}}}\" | " \
 453             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qmp')}"
 454         message = f"QMP execute '{cmd}' failed on {self._node[u'host']}"
 455
 456         stdout, _ = exec_cmd_no_error(
 457             self._node, command, sudo=False, message=message
 458         )
 459
 460         # Skip capabilities negotiation messages.
 461         out_list = stdout.splitlines()
 462         if len(out_list) < 3:
 463             raise RuntimeError(f"Invalid QMP output on {self._node[u'host']}")
 464         return json.loads(out_list[2])
 465
 466     def _qemu_qga_flush(self):
 467         """Flush the QGA parser state."""
 468         command = f"(printf \"\xFF\"; sleep 1) | sudo -S socat " \
 469             f"- UNIX-CONNECT:{self._temp.get(u'qga')}"
 470         message = f"QGA flush failed on {self._node[u'host']}"
 471         stdout, _ = exec_cmd_no_error(
 472             self._node, command, sudo=False, message=message
 473         )
 474
 475         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 476
 477     def _qemu_qga_exec(self, cmd):
 478         """Execute QGA command.
 479
 480         QGA provide access to a system-level agent via standard QMP commands.
 481
 482         :param cmd: QGA command to execute.
 483         :type cmd: str
 484         """
 485         command = f"(echo \"{{{{ \\\"execute\\\": " \
 486             f"\\\"{cmd}\\\" }}}}\"; sleep 1) | " \
 487             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qga')}"
 488         message = f"QGA execute '{cmd}' failed on {self._node[u'host']}"
 489         stdout, _ = exec_cmd_no_error(
 490             self._node, command, sudo=False, message=message
 491         )
 492
 493         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 494
 495     def _wait_until_vm_boot(self):
 496         """Wait until QEMU with NestedVM is booted."""
 497         if self._opt.get(u"vm_type") == u"nestedvm":
 498             self._wait_until_nestedvm_boot()
 499             self._update_vm_interfaces()
 500         elif self._opt.get(u"vm_type") == u"kernelvm":
 501             self._wait_until_kernelvm_boot()
 502         else:
 503             raise RuntimeError(u"QEMU: Unsupported VM type!")
 504
 505     def _wait_until_nestedvm_boot(self, retries=12):
 506         """Wait until QEMU with NestedVM is booted.
 507
 508         First try to flush qga until there is output.
 509         Then ping QEMU guest agent each 5s until VM booted or timeout.
 510
 511         :param retries: Number of retries with 5s between trials.
 512         :type retries: int
 513         """
 514         for _ in range(retries):
 515             out = None
 516             try:
 517                 out = self._qemu_qga_flush()
 518             except ValueError:
 519                 logger.trace(f"QGA qga flush unexpected output {out}")
 520             # Empty output - VM not booted yet
 521             if not out:
 522                 sleep(5)
 523             else:
 524                 break
 525         else:
 526             raise RuntimeError(
 527                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 528             )
 529         for _ in range(retries):
 530             out = None
 531             try:
 532                 out = self._qemu_qga_exec(u"guest-ping")
 533             except ValueError:
 534                 logger.trace(f"QGA guest-ping unexpected output {out}")
 535             # Empty output - VM not booted yet.
 536             if not out:
 537                 sleep(5)
 538             # Non-error return - VM booted.
 539             elif out.get(u"return") is not None:
 540                 break
 541             # Skip error and wait.
 542             elif out.get(u"error") is not None:
 543                 sleep(5)
 544             else:
 545                 # If there is an unexpected output from QGA guest-info, try
 546                 # again until timeout.
 547                 logger.trace(f"QGA guest-ping unexpected output {out}")
 548         else:
 549             raise RuntimeError(
 550                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 551             )
 552
 553     def _wait_until_kernelvm_boot(self, retries=60):
 554         """Wait until QEMU KernelVM is booted.
 555
 556         :param retries: Number of retries.
 557         :type retries: int
 558         """
 559         vpp_ver = VPPUtil.vpp_show_version(self._node)
 560
 561         for _ in range(retries):
 562             command = f"tail -1 {self._temp.get(u'log')}"
 563             stdout = None
 564             try:
 565                 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 566                 sleep(1)
 567             except RuntimeError:
 568                 pass
 569             if vpp_ver in stdout or u"Press enter to exit" in stdout:
 570                 break
 571             if u"reboot: Power down" in stdout:
 572                 raise RuntimeError(
 573                     f"QEMU: NF failed to run on {self._node[u'host']}!"
 574                 )
 575         else:
 576             raise RuntimeError(
 577                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 578             )
 579
 580     def _update_vm_interfaces(self):
 581         """Update interface names in VM node dict."""
 582         # Send guest-network-get-interfaces command via QGA, output example:
 583         # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
 584         # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
 585         out = self._qemu_qga_exec(u"guest-network-get-interfaces")
 586         interfaces = out.get(u"return")
 587         mac_name = {}
 588         if not interfaces:
 589             raise RuntimeError(
 590                 f"Get VM interface list failed on {self._node[u'host']}"
 591             )
 592         # Create MAC-name dict.
 593         for interface in interfaces:
 594             if u"hardware-address" not in interface:
 595                 continue
 596             mac_name[interface[u"hardware-address"]] = interface[u"name"]
 597         # Match interface by MAC and save interface name.
 598         for interface in self._vm_info[u"interfaces"].values():
 599             mac = interface.get(u"mac_address")
 600             if_name = mac_name.get(mac)
 601             if if_name is None:
 602                 logger.trace(f"Interface name for MAC {mac} not found")
 603             else:
 604                 interface[u"name"] = if_name
 605
 606     def qemu_start(self):
 607         """Start QEMU and wait until VM boot.
 608
 609         :returns: VM node info.
 610         :rtype: dict
 611         """
 612         cmd_opts = OptionString()
 613         cmd_opts.add(f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch}")
 614         cmd_opts.extend(self._params)
 615         message = f"QEMU: Start failed on {self._node[u'host']}!"
 616         try:
 617             DUTSetup.check_huge_page(
 618                 self._node, u"/dev/hugepages", int(self._opt.get(u"mem")))
 619
 620             exec_cmd_no_error(
 621                 self._node, cmd_opts, timeout=300, sudo=True, message=message
 622             )
 623             self._wait_until_vm_boot()
 624         except RuntimeError:
 625             self.qemu_kill_all()
 626             raise
 627         return self._vm_info
 628
 629     def qemu_kill(self):
 630         """Kill qemu process."""
 631         exec_cmd(
 632             self._node, f"chmod +r {self._temp.get(u'pidfile')}", sudo=True
 633         )
 634         exec_cmd(
 635             self._node, f"kill -SIGKILL $(cat {self._temp.get(u'pidfile')})",
 636             sudo=True
 637         )
 638
 639         for value in self._temp.values():
 640             exec_cmd(self._node, f"cat {value}", sudo=True)
 641             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 642
 643     def qemu_kill_all(self):
 644         """Kill all qemu processes on DUT node if specified."""
 645         exec_cmd(self._node, u"pkill -SIGKILL qemu", sudo=True)
 646
 647         for value in self._temp.values():
 648             exec_cmd(self._node, f"cat {value}", sudo=True)
 649             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 650
 651     def qemu_version(self):
 652         """Return Qemu version.
 653
 654         :returns: Qemu version.
 655         :rtype: str
 656         """
 657         command = f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch} " \
 658             f"--version"
 659         try:
 660             stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 661             return match(r"QEMU emulator version ([\d.]*)", stdout).group(1)
 662         except RuntimeError:
 663             self.qemu_kill_all()
 664             raise