resources/libraries/python/QemuUtils.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """QEMU utilities library."""
  15
  16 import json
  17
  18 from re import match
  19 from string import Template
  20 from time import sleep
  21
  22 from robot.api import logger
  23
  24 from resources.libraries.python.Constants import Constants
  25 from resources.libraries.python.DpdkUtil import DpdkUtil
  26 from resources.libraries.python.DUTSetup import DUTSetup
  27 from resources.libraries.python.OptionString import OptionString
  28 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
  29 from resources.libraries.python.topology import NodeType, Topology
  30 from resources.libraries.python.VhostUser import VirtioFeaturesFlags
  31 from resources.libraries.python.VhostUser import VirtioFeatureMask
  32 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
  33
  34 __all__ = [u"QemuUtils"]
  35
  36
  37 class QemuUtils:
  38     """QEMU utilities."""
  39
  40     # Use one instance of class per tests.
  41     ROBOT_LIBRARY_SCOPE = u"TEST CASE"
  42
  43     def __init__(
  44             self, node, qemu_id=1, smp=1, mem=512, vnf=None,
  45             img=Constants.QEMU_VM_IMAGE):
  46         """Initialize QemuUtil class.
  47
  48         :param node: Node to run QEMU on.
  49         :param qemu_id: QEMU identifier.
  50         :param smp: Number of virtual SMP units (cores).
  51         :param mem: Amount of memory.
  52         :param vnf: Network function workload.
  53         :param img: QEMU disk image or kernel image path.
  54         :type node: dict
  55         :type qemu_id: int
  56         :type smp: int
  57         :type mem: int
  58         :type vnf: str
  59         :type img: str
  60         """
  61         self._nic_id = 0
  62         self._node = node
  63         self._arch = Topology.get_node_arch(self._node)
  64         self._opt = dict()
  65
  66         # Architecture specific options
  67         if self._arch == u"aarch64":
  68             dpdk_target = u"arm64-armv8a"
  69             self._opt[u"machine_args"] = \
  70                 u"virt,accel=kvm,usb=off,mem-merge=off,gic-version=3"
  71             self._opt[u"console"] = u"ttyAMA0"
  72         else:
  73             dpdk_target = u"x86_64-native"
  74             self._opt[u"machine_args"] = u"pc,accel=kvm,usb=off,mem-merge=off"
  75             self._opt[u"console"] = u"ttyS0"
  76         self._testpmd_path = f"{Constants.QEMU_VM_DPDK}/" \
  77             f"{dpdk_target}-linux-gcc/app"
  78         self._vm_info = {
  79             u"host": node[u"host"],
  80             u"type": NodeType.VM,
  81             u"port": 10021 + qemu_id,
  82             u"serial": 4555 + qemu_id,
  83             u"username": 'cisco',
  84             u"password": 'cisco',
  85             u"interfaces": {},
  86         }
  87         if node[u"port"] != 22:
  88             self._vm_info[u"host_port"] = node[u"port"]
  89             self._vm_info[u"host_username"] = node[u"username"]
  90             self._vm_info[u"host_password"] = node[u"password"]
  91         # Input Options.
  92         self._opt[u"qemu_id"] = qemu_id
  93         self._opt[u"mem"] = int(mem)
  94         self._opt[u"smp"] = int(smp)
  95         self._opt[u"img"] = img
  96         self._opt[u"vnf"] = vnf
  97         # Temporary files.
  98         self._temp = dict()
  99         self._temp[u"log"] = f"/tmp/serial_{qemu_id}.log"
 100         self._temp[u"pidfile"] = f"/run/qemu_{qemu_id}.pid"
 101         if img == Constants.QEMU_VM_IMAGE:
 102             self._temp[u"qmp"] = f"/run/qmp_{qemu_id}.sock"
 103             self._temp[u"qga"] = f"/run/qga_{qemu_id}.sock"
 104         elif img == Constants.QEMU_VM_KERNEL:
 105             self._opt[u"img"], _ = exec_cmd_no_error(
 106                 node, f"ls -1 {Constants.QEMU_VM_KERNEL}* | tail -1",
 107                 message=u"Qemu Kernel VM image not found!"
 108             )
 109             self._temp[u"ini"] = f"/etc/vm_init_{qemu_id}.conf"
 110             self._opt[u"initrd"], _ = exec_cmd_no_error(
 111                 node, f"ls -1 {Constants.QEMU_VM_KERNEL_INITRD}* | tail -1",
 112                 message=u"Qemu Kernel initrd image not found!"
 113             )
 114         else:
 115             raise RuntimeError(f"QEMU: Unknown VM image option: {img}")
 116         # Computed parameters for QEMU command line.
 117         self._params = OptionString(prefix=u"-")
 118
 119     def add_default_params(self):
 120         """Set default QEMU command line parameters."""
 121         self._params.add(u"daemonize")
 122         self._params.add(u"nodefaults")
 123         self._params.add_with_value(
 124             u"name", f"vnf{self._opt.get(u'qemu_id')},debug-threads=on"
 125         )
 126         self._params.add(u"no-user-config")
 127         self._params.add(u"nographic")
 128         self._params.add(u"enable-kvm")
 129         self._params.add_with_value(u"pidfile", self._temp.get(u"pidfile"))
 130         self._params.add_with_value(u"cpu", u"host")
 131
 132         self._params.add_with_value(u"machine", self._opt.get(u"machine_args"))
 133         self._params.add_with_value(
 134             u"smp", f"{self._opt.get(u'smp')},sockets=1,"
 135             f"cores={self._opt.get(u'smp')},threads=1"
 136         )
 137         self._params.add_with_value(
 138             u"object", f"memory-backend-file,id=mem,"
 139             f"size={self._opt.get(u'mem')}M,mem-path=/dev/hugepages,share=on"
 140         )
 141         self._params.add_with_value(u"m", f"{self._opt.get(u'mem')}M")
 142         self._params.add_with_value(u"numa", u"node,memdev=mem")
 143         self._params.add_with_value(u"balloon", u"none")
 144
 145     def add_net_user(self):
 146         """Set managment port forwarding."""
 147         self._params.add_with_value(
 148             u"netdev", f"user,id=mgmt,net=172.16.255.0/24,"
 149             f"hostfwd=tcp::{self._vm_info[u'port']}-:22"
 150         )
 151         self._params.add_with_value(
 152             u"device", f"virtio-net,netdev=mgmt"
 153         )
 154
 155     def add_qmp_qga(self):
 156         """Set QMP, QGA management."""
 157         self._params.add_with_value(
 158             u"chardev", f"socket,path={self._temp.get(u'qga')},"
 159             f"server,nowait,id=qga0"
 160         )
 161         self._params.add_with_value(u"device", u"isa-serial,chardev=qga0")
 162         self._params.add_with_value(
 163             u"qmp", f"unix:{self._temp.get(u'qmp')},server,nowait"
 164         )
 165
 166     def add_serial(self):
 167         """Set serial to file redirect."""
 168         self._params.add_with_value(
 169             u"chardev", f"socket,host=127.0.0.1,"
 170             f"port={self._vm_info[u'serial']},id=gnc0,server,nowait")
 171         self._params.add_with_value(u"device", u"isa-serial,chardev=gnc0")
 172         self._params.add_with_value(
 173             u"serial", f"file:{self._temp.get(u'log')}"
 174         )
 175
 176     def add_drive_cdrom(self, drive_file, index=None):
 177         """Set CD-ROM drive.
 178
 179         :param drive_file: Path to drive image.
 180         :param index: Drive index.
 181         :type drive_file: str
 182         :type index: int
 183         """
 184         index = f"index={index}," if index else u""
 185         self._params.add_with_value(
 186             u"drive", f"file={drive_file},{index}media=cdrom"
 187         )
 188
 189     def add_drive(self, drive_file, drive_format):
 190         """Set drive with custom format.
 191
 192         :param drive_file: Path to drive image.
 193         :param drive_format: Drive image format.
 194         :type drive_file: str
 195         :type drive_format: str
 196         """
 197         self._params.add_with_value(
 198             u"drive", f"file={drive_file},format={drive_format},"
 199             u"cache=none,if=virtio,file.locking=off"
 200         )
 201
 202     def add_kernelvm_params(self):
 203         """Set KernelVM QEMU parameters."""
 204         self._params.add_with_value(
 205             u"serial", f"file:{self._temp.get(u'log')}"
 206         )
 207         self._params.add_with_value(
 208             u"fsdev", u"local,id=root9p,path=/,security_model=none"
 209         )
 210         self._params.add_with_value(
 211             u"device", u"virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot"
 212         )
 213         self._params.add_with_value(u"kernel", f"{self._opt.get(u'img')}")
 214         self._params.add_with_value(u"initrd", f"{self._opt.get(u'initrd')}")
 215         self._params.add_with_value(
 216             u"append", f"'ro rootfstype=9p rootflags=trans=virtio "
 217             f"root=virtioroot console={self._opt.get(u'console')} "
 218             f"tsc=reliable hugepages=512 "
 219             f"init={self._temp.get(u'ini')} fastboot'"
 220         )
 221
 222     def add_vhost_user_if(
 223             self, socket, server=True, jumbo_frames=False, queue_size=None,
 224             queues=1, virtio_feature_mask=None):
 225         """Add Vhost-user interface.
 226
 227         :param socket: Path of the unix socket.
 228         :param server: If True the socket shall be a listening socket.
 229         :param jumbo_frames: Set True if jumbo frames are used in the test.
 230         :param queue_size: Vring queue size.
 231         :param queues: Number of queues.
 232         :param virtio_feature_mask: Mask of virtio features to be enabled.
 233         :type socket: str
 234         :type server: bool
 235         :type jumbo_frames: bool
 236         :type queue_size: int
 237         :type queues: int
 238         :type virtio_feature_mask: int
 239         """
 240         self._nic_id += 1
 241         self._params.add_with_value(
 242             u"chardev", f"socket,id=char{self._nic_id},"
 243             f"path={socket}{u',server' if server is True else u''}"
 244         )
 245         self._params.add_with_value(
 246             u"netdev", f"vhost-user,id=vhost{self._nic_id},"
 247             f"chardev=char{self._nic_id},queues={queues}"
 248         )
 249         mac = f"52:54:00:00:{self._opt.get(u'qemu_id'):02x}:" \
 250             f"{self._nic_id:02x}"
 251         queue_size = f"rx_queue_size={queue_size},tx_queue_size={queue_size}" \
 252             if queue_size else u""
 253         if virtio_feature_mask is None:
 254             gso = False
 255             csum = False
 256         else:
 257             gso = VirtioFeatureMask.is_feature_enabled(
 258                 virtio_feature_mask, VirtioFeaturesFlags.VIRTIO_NET_F_API_GSO)
 259             csum = VirtioFeatureMask.is_feature_enabled(
 260                 virtio_feature_mask, VirtioFeaturesFlags.VIRTIO_NET_F_API_CSUM)
 261         self._params.add_with_value(
 262             u"device", f"virtio-net-pci,netdev=vhost{self._nic_id},mac={mac},"
 263             f"addr={self._nic_id+5}.0,mq=on,vectors={2 * queues + 2},"
 264             f"csum={u'on' if csum else u'off'},gso={u'on' if gso else u'off'},"
 265             f"guest_tso4=off,guest_tso6=off,guest_ecn=off,"
 266             f"{queue_size}"
 267         )
 268
 269         # Add interface MAC and socket to the node dict.
 270         if_data = {u"mac_address": mac, u"socket": socket}
 271         if_name = f"vhost{self._nic_id}"
 272         self._vm_info[u"interfaces"][if_name] = if_data
 273         # Add socket to temporary file list.
 274         self._temp[if_name] = socket
 275
 276     def add_vfio_pci_if(self, pci):
 277         """Add VFIO PCI interface.
 278
 279         :param pci: PCI address of interface.
 280         :type pci: str
 281         """
 282         self._nic_id += 1
 283         self._params.add_with_value(
 284             u"device", f"vfio-pci,host={pci},addr={self._nic_id+5}.0"
 285         )
 286
 287     def create_kernelvm_config_vpp(self, **kwargs):
 288         """Create QEMU VPP config files.
 289
 290         :param kwargs: Key-value pairs to replace content of VPP configuration
 291             file.
 292         :type kwargs: dict
 293         """
 294         startup = f"/etc/vpp/vm_startup_{self._opt.get(u'qemu_id')}.conf"
 295         running = f"/etc/vpp/vm_running_{self._opt.get(u'qemu_id')}.exec"
 296
 297         self._temp[u"startup"] = startup
 298         self._temp[u"running"] = running
 299         self._opt[u"vnf_bin"] = f"/usr/bin/vpp -c {startup}"
 300
 301         # Create VPP startup configuration.
 302         vpp_config = VppConfigGenerator()
 303         vpp_config.set_node(self._node)
 304         vpp_config.add_unix_nodaemon()
 305         vpp_config.add_unix_cli_listen()
 306         vpp_config.add_unix_exec(running)
 307         vpp_config.add_socksvr()
 308         vpp_config.add_main_heap_size(u"512M")
 309         vpp_config.add_main_heap_page_size(u"2M")
 310         vpp_config.add_statseg_size(u"512M")
 311         vpp_config.add_statseg_page_size(u"2M")
 312         vpp_config.add_statseg_per_node_counters(u"on")
 313         vpp_config.add_buffers_per_numa(107520)
 314         vpp_config.add_cpu_main_core(u"0")
 315         if self._opt.get(u"smp") > 1:
 316             vpp_config.add_cpu_corelist_workers(f"1-{self._opt.get(u'smp')-1}")
 317         vpp_config.add_plugin(u"disable", u"default")
 318         vpp_config.add_plugin(u"enable", u"ping_plugin.so")
 319         if "2vfpt" in self._opt.get(u'vnf'):
 320             vpp_config.add_plugin(u"enable", u"avf_plugin.so")
 321         if "vhost" in self._opt.get(u'vnf'):
 322             vpp_config.add_plugin(u"enable", u"dpdk_plugin.so")
 323             vpp_config.add_dpdk_dev(u"0000:00:06.0", u"0000:00:07.0")
 324             vpp_config.add_dpdk_dev_default_rxq(kwargs[u"queues"])
 325             vpp_config.add_dpdk_log_level(u"debug")
 326             if not kwargs[u"jumbo_frames"]:
 327                 vpp_config.add_dpdk_no_multi_seg()
 328                 vpp_config.add_dpdk_no_tx_checksum_offload()
 329         if "ipsec" in self._opt.get(u'vnf'):
 330             vpp_config.add_plugin(u"enable", u"crypto_native_plugin.so")
 331             vpp_config.add_plugin(u"enable", u"crypto_ipsecmb_plugin.so")
 332             vpp_config.add_plugin(u"enable", u"crypto_openssl_plugin.so")
 333         if "nat" in self._opt.get(u'vnf'):
 334             vpp_config.add_nat(value=u"endpoint-dependent")
 335             vpp_config.add_plugin(u"enable", u"nat_plugin.so")
 336         vpp_config.write_config(startup)
 337
 338         # Create VPP running configuration.
 339         template = f"{Constants.RESOURCES_TPL}/vm/{self._opt.get(u'vnf')}.exec"
 340         exec_cmd_no_error(self._node, f"rm -f {running}", sudo=True)
 341
 342         with open(template, u"rt") as src_file:
 343             src = Template(src_file.read())
 344             exec_cmd_no_error(
 345                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 346                 f"sudo tee {running}"
 347             )
 348
 349     def create_kernelvm_config_testpmd_io(self, **kwargs):
 350         """Create QEMU testpmd-io command line.
 351
 352         :param kwargs: Key-value pairs to construct command line parameters.
 353         :type kwargs: dict
 354         """
 355         pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
 356         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 357             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 358             eal_driver=False,
 359             eal_pci_whitelist0=u"0000:00:06.0",
 360             eal_pci_whitelist1=u"0000:00:07.0",
 361             eal_in_memory=True,
 362             pmd_num_mbufs=16384,
 363             pmd_fwd_mode=u"io",
 364             pmd_nb_ports=u"2",
 365             pmd_portmask=u"0x3",
 366             pmd_max_pkt_len=pmd_max_pkt_len,
 367             pmd_mbuf_size=u"16384",
 368             pmd_rxq=kwargs[u"queues"],
 369             pmd_txq=kwargs[u"queues"],
 370             pmd_tx_offloads='0x0',
 371             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 372         )
 373
 374         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 375
 376     def create_kernelvm_config_testpmd_mac(self, **kwargs):
 377         """Create QEMU testpmd-mac command line.
 378
 379         :param kwargs: Key-value pairs to construct command line parameters.
 380         :type kwargs: dict
 381         """
 382         pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
 383         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 384             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
 385             eal_driver=False,
 386             eal_pci_whitelist0=u"0000:00:06.0",
 387             eal_pci_whitelist1=u"0000:00:07.0",
 388             eal_in_memory=True,
 389             pmd_num_mbufs=16384,
 390             pmd_fwd_mode=u"mac",
 391             pmd_nb_ports=u"2",
 392             pmd_portmask=u"0x3",
 393             pmd_max_pkt_len=pmd_max_pkt_len,
 394             pmd_mbuf_size=u"16384",
 395             pmd_eth_peer_0=f"0,{kwargs[u'vif1_mac']}",
 396             pmd_eth_peer_1=f"1,{kwargs[u'vif2_mac']}",
 397             pmd_rxq=kwargs[u"queues"],
 398             pmd_txq=kwargs[u"queues"],
 399             pmd_tx_offloads=u"0x0",
 400             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
 401         )
 402
 403         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
 404
 405     def create_kernelvm_init(self, **kwargs):
 406         """Create QEMU init script.
 407
 408         :param kwargs: Key-value pairs to replace content of init startup file.
 409         :type kwargs: dict
 410         """
 411         template = f"{Constants.RESOURCES_TPL}/vm/init.sh"
 412         init = self._temp.get(u"ini")
 413         exec_cmd_no_error(self._node, f"rm -f {init}", sudo=True)
 414
 415         with open(template, u"rt") as src_file:
 416             src = Template(src_file.read())
 417             exec_cmd_no_error(
 418                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
 419                 f"sudo tee {init}"
 420             )
 421             exec_cmd_no_error(self._node, f"chmod +x {init}", sudo=True)
 422
 423     def configure_kernelvm_vnf(self, **kwargs):
 424         """Create KernelVM VNF configurations.
 425
 426         :param kwargs: Key-value pairs for templating configs.
 427         :type kwargs: dict
 428         """
 429         if u"vpp" in self._opt.get(u"vnf"):
 430             self.create_kernelvm_config_vpp(**kwargs)
 431             self.create_kernelvm_init(vnf_bin=self._opt.get(u"vnf_bin"))
 432         elif u"testpmd_io" in self._opt.get(u"vnf"):
 433             self.create_kernelvm_config_testpmd_io(**kwargs)
 434             self.create_kernelvm_init(vnf_bin=self._opt.get(u"vnf_bin"))
 435         elif u"testpmd_mac" in self._opt.get(u"vnf"):
 436             self.create_kernelvm_config_testpmd_mac(**kwargs)
 437             self.create_kernelvm_init(vnf_bin=self._opt.get(u"vnf_bin"))
 438         else:
 439             raise RuntimeError(u"QEMU: Unsupported VNF!")
 440
 441     def get_qemu_pids(self):
 442         """Get QEMU CPU pids.
 443
 444         :returns: List of QEMU CPU pids.
 445         :rtype: list of str
 446         """
 447         command = f"grep -rwl 'CPU' /proc/$(sudo cat " \
 448             f"{self._temp.get(u'pidfile')})/task/*/comm "
 449         command += r"| xargs dirname | sed -e 's/\/.*\///g' | uniq"
 450
 451         stdout, _ = exec_cmd_no_error(self._node, command)
 452         return stdout.splitlines()
 453
 454     def qemu_set_affinity(self, *host_cpus):
 455         """Set qemu affinity by getting thread PIDs via QMP and taskset to list
 456         of CPU cores. Function tries to execute 3 times to avoid race condition
 457         in getting thread PIDs.
 458
 459         :param host_cpus: List of CPU cores.
 460         :type host_cpus: list
 461         """
 462         for _ in range(3):
 463             try:
 464                 qemu_cpus = self.get_qemu_pids()
 465
 466                 if len(qemu_cpus) != len(host_cpus):
 467                     sleep(1)
 468                     continue
 469                 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
 470                     command = f"taskset -pc {host_cpu} {qemu_cpu}"
 471                     message = f"QEMU: Set affinity failed " \
 472                         f"on {self._node[u'host']}!"
 473                     exec_cmd_no_error(
 474                         self._node, command, sudo=True, message=message
 475                     )
 476                 break
 477             except (RuntimeError, ValueError):
 478                 self.qemu_kill_all()
 479                 raise
 480         else:
 481             self.qemu_kill_all()
 482             raise RuntimeError(u"Failed to set Qemu threads affinity!")
 483
 484     def qemu_set_scheduler_policy(self):
 485         """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
 486         processes.
 487
 488         :raises RuntimeError: Set scheduler policy failed.
 489         """
 490         try:
 491             qemu_cpus = self.get_qemu_pids()
 492
 493             for qemu_cpu in qemu_cpus:
 494                 command = f"chrt -r -p 1 {qemu_cpu}"
 495                 message = f"QEMU: Set SCHED_RR failed on {self._node[u'host']}"
 496                 exec_cmd_no_error(
 497                     self._node, command, sudo=True, message=message
 498                 )
 499         except (RuntimeError, ValueError):
 500             self.qemu_kill_all()
 501             raise
 502
 503     def _qemu_qmp_exec(self, cmd):
 504         """Execute QMP command.
 505
 506         QMP is JSON based protocol which allows to control QEMU instance.
 507
 508         :param cmd: QMP command to execute.
 509         :type cmd: str
 510         :returns: Command output in python representation of JSON format. The
 511             { "return": {} } response is QMP's success response. An error
 512             response will contain the "error" keyword instead of "return".
 513         """
 514         # To enter command mode, the qmp_capabilities command must be issued.
 515         command = f"echo \"{{{{ \\\"execute\\\": " \
 516             f"\\\"qmp_capabilities\\\" }}}}" \
 517             f"{{{{ \\\"execute\\\": \\\"{cmd}\\\" }}}}\" | " \
 518             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qmp')}"
 519         message = f"QMP execute '{cmd}' failed on {self._node[u'host']}"
 520
 521         stdout, _ = exec_cmd_no_error(
 522             self._node, command, sudo=False, message=message
 523         )
 524
 525         # Skip capabilities negotiation messages.
 526         out_list = stdout.splitlines()
 527         if len(out_list) < 3:
 528             raise RuntimeError(f"Invalid QMP output on {self._node[u'host']}")
 529         return json.loads(out_list[2])
 530
 531     def _qemu_qga_flush(self):
 532         """Flush the QGA parser state."""
 533         command = f"(printf \"\xFF\"; sleep 1) | sudo -S socat " \
 534             f"- UNIX-CONNECT:{self._temp.get(u'qga')}"
 535         message = f"QGA flush failed on {self._node[u'host']}"
 536         stdout, _ = exec_cmd_no_error(
 537             self._node, command, sudo=False, message=message
 538         )
 539
 540         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 541
 542     def _qemu_qga_exec(self, cmd):
 543         """Execute QGA command.
 544
 545         QGA provide access to a system-level agent via standard QMP commands.
 546
 547         :param cmd: QGA command to execute.
 548         :type cmd: str
 549         """
 550         command = f"(echo \"{{{{ \\\"execute\\\": " \
 551             f"\\\"{cmd}\\\" }}}}\"; sleep 1) | " \
 552             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qga')}"
 553         message = f"QGA execute '{cmd}' failed on {self._node[u'host']}"
 554         stdout, _ = exec_cmd_no_error(
 555             self._node, command, sudo=False, message=message
 556         )
 557
 558         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
 559
 560     def _wait_until_vm_boot(self):
 561         """Wait until QEMU VM is booted."""
 562         try:
 563             getattr(self, f'_wait_{self._opt["vnf"]}')()
 564         except AttributeError:
 565             self._wait_default()
 566
 567     def _wait_default(self, retries=60):
 568         """Wait until QEMU with VPP is booted.
 569
 570         :param retries: Number of retries.
 571         :type retries: int
 572         """
 573         for _ in range(retries):
 574             command = f"tail -1 {self._temp.get(u'log')}"
 575             stdout = None
 576             try:
 577                 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 578                 sleep(1)
 579             except RuntimeError:
 580                 pass
 581             if "vpp " in stdout and "built by" in stdout:
 582                 break
 583             if u"Press enter to exit" in stdout:
 584                 break
 585             if u"reboot: Power down" in stdout:
 586                 raise RuntimeError(
 587                     f"QEMU: NF failed to run on {self._node[u'host']}!"
 588                 )
 589         else:
 590             raise RuntimeError(
 591                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 592             )
 593
 594     def _wait_nestedvm(self, retries=12):
 595         """Wait until QEMU with NestedVM is booted.
 596
 597         First try to flush qga until there is output.
 598         Then ping QEMU guest agent each 5s until VM booted or timeout.
 599
 600         :param retries: Number of retries with 5s between trials.
 601         :type retries: int
 602         """
 603         for _ in range(retries):
 604             out = None
 605             try:
 606                 out = self._qemu_qga_flush()
 607             except ValueError:
 608                 logger.trace(f"QGA qga flush unexpected output {out}")
 609             # Empty output - VM not booted yet
 610             if not out:
 611                 sleep(5)
 612             else:
 613                 break
 614         else:
 615             raise RuntimeError(
 616                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 617             )
 618         for _ in range(retries):
 619             out = None
 620             try:
 621                 out = self._qemu_qga_exec(u"guest-ping")
 622             except ValueError:
 623                 logger.trace(f"QGA guest-ping unexpected output {out}")
 624             # Empty output - VM not booted yet.
 625             if not out:
 626                 sleep(5)
 627             # Non-error return - VM booted.
 628             elif out.get(u"return") is not None:
 629                 break
 630             # Skip error and wait.
 631             elif out.get(u"error") is not None:
 632                 sleep(5)
 633             else:
 634                 # If there is an unexpected output from QGA guest-info, try
 635                 # again until timeout.
 636                 logger.trace(f"QGA guest-ping unexpected output {out}")
 637         else:
 638             raise RuntimeError(
 639                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
 640             )
 641
 642     def _update_vm_interfaces(self):
 643         """Update interface names in VM node dict."""
 644         # Send guest-network-get-interfaces command via QGA, output example:
 645         # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
 646         # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
 647         out = self._qemu_qga_exec(u"guest-network-get-interfaces")
 648         interfaces = out.get(u"return")
 649         mac_name = {}
 650         if not interfaces:
 651             raise RuntimeError(
 652                 f"Get VM interface list failed on {self._node[u'host']}"
 653             )
 654         # Create MAC-name dict.
 655         for interface in interfaces:
 656             if u"hardware-address" not in interface:
 657                 continue
 658             mac_name[interface[u"hardware-address"]] = interface[u"name"]
 659         # Match interface by MAC and save interface name.
 660         for interface in self._vm_info[u"interfaces"].values():
 661             mac = interface.get(u"mac_address")
 662             if_name = mac_name.get(mac)
 663             if if_name is None:
 664                 logger.trace(f"Interface name for MAC {mac} not found")
 665             else:
 666                 interface[u"name"] = if_name
 667
 668     def qemu_start(self):
 669         """Start QEMU and wait until VM boot.
 670
 671         :returns: VM node info.
 672         :rtype: dict
 673         """
 674         cmd_opts = OptionString()
 675         cmd_opts.add(f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch}")
 676         cmd_opts.extend(self._params)
 677         message = f"QEMU: Start failed on {self._node[u'host']}!"
 678         try:
 679             DUTSetup.check_huge_page(
 680                 self._node, u"/dev/hugepages", int(self._opt.get(u"mem")))
 681
 682             exec_cmd_no_error(
 683                 self._node, cmd_opts, timeout=300, sudo=True, message=message
 684             )
 685             self._wait_until_vm_boot()
 686         except RuntimeError:
 687             self.qemu_kill_all()
 688             raise
 689         return self._vm_info
 690
 691     def qemu_kill(self):
 692         """Kill qemu process."""
 693         exec_cmd(
 694             self._node, f"chmod +r {self._temp.get(u'pidfile')}", sudo=True
 695         )
 696         exec_cmd(
 697             self._node, f"kill -SIGKILL $(cat {self._temp.get(u'pidfile')})",
 698             sudo=True
 699         )
 700
 701         for value in self._temp.values():
 702             exec_cmd(self._node, f"cat {value}", sudo=True)
 703             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 704
 705     def qemu_kill_all(self):
 706         """Kill all qemu processes on DUT node if specified."""
 707         exec_cmd(self._node, u"pkill -SIGKILL qemu", sudo=True)
 708
 709         for value in self._temp.values():
 710             exec_cmd(self._node, f"cat {value}", sudo=True)
 711             exec_cmd(self._node, f"rm -f {value}", sudo=True)
 712
 713     def qemu_version(self):
 714         """Return Qemu version.
 715
 716         :returns: Qemu version.
 717         :rtype: str
 718         """
 719         command = f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch} " \
 720             f"--version"
 721         try:
 722             stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 723             return match(r"QEMU emulator version ([\d.]*)", stdout).group(1)
 724         except RuntimeError:
 725             self.qemu_kill_all()
 726             raise