resources/libraries/python/QemuUtils.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """QEMU utilities library."""
  15
  16 # Disable due to pylint bug
  17 # pylint: disable=no-name-in-module,import-error
  18 from distutils.version import StrictVersion
  19 import json
  20 from re import match
  21 from string import Template
  22 from time import sleep
  23
  24 from robot.api import logger
  25 from resources.libraries.python.Constants import Constants
  26 from resources.libraries.python.DpdkUtil import DpdkUtil
  27 from resources.libraries.python.DUTSetup import DUTSetup
  28 from resources.libraries.python.OptionString import OptionString
  29 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
  30 from resources.libraries.python.VPPUtil import VPPUtil
  31 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
  32 from resources.libraries.python.topology import NodeType, Topology
  33
  34 __all__ = ["QemuUtils"]
  35
  36
  37 class QemuUtils(object):
  38     """QEMU utilities."""
  39
  40     # Use one instance of class per tests.
  41     ROBOT_LIBRARY_SCOPE = 'TEST CASE'
  42
  43     def __init__(self, node, qemu_id=1, smp=1, mem=512, vnf=None,
  44                  img=Constants.QEMU_VM_IMAGE):
  45         """Initialize QemuUtil class.
  46
  47         :param node: Node to run QEMU on.
  48         :param qemu_id: QEMU identifier.
  49         :param smp: Number of virtual SMP units (cores).
  50         :param mem: Amount of memory.
  51         :param vnf: Network function workload.
  52         :param img: QEMU disk image or kernel image path.
  53         :type node: dict
  54         :type qemu_id: int
  55         :type smp: int
  56         :type mem: int
  57         :type vnf: str
  58         :type img: str
  59         """
  60         self._vhost_id = 0
  61         self._node = node
  62         self._arch = Topology.get_node_arch(self._node)
  63         dpdk_target = 'arm64-armv8a' if self._arch == 'aarch64' \
  64             else 'x86_64-native'
  65         self._testpmd_path = '{path}/{dpdk_target}-linuxapp-gcc/app'\
  66             .format(path=Constants.QEMU_VM_DPDK, dpdk_target=dpdk_target)
  67         self._vm_info = {
  68             'host': node['host'],
  69             'type': NodeType.VM,
  70             'port': 10021 + qemu_id,
  71             'serial': 4555 + qemu_id,
  72             'username': 'cisco',
  73             'password': 'cisco',
  74             'interfaces': {},
  75         }
  76         if node['port'] != 22:
  77             self._vm_info['host_port'] = node['port']
  78             self._vm_info['host_username'] = node['username']
  79             self._vm_info['host_password'] = node['password']
  80         # Input Options.
  81         self._opt = dict()
  82         self._opt['qemu_id'] = qemu_id
  83         self._opt['mem'] = int(mem)
  84         self._opt['smp'] = int(smp)
  85         self._opt['img'] = img
  86         self._opt['vnf'] = vnf
  87         # Temporary files.
  88         self._temp = dict()
  89         self._temp['pidfile'] = '/var/run/qemu_{id}.pid'.format(id=qemu_id)
  90         if img == Constants.QEMU_VM_IMAGE:
  91             self._opt['vm_type'] = 'nestedvm'
  92             self._temp['qmp'] = '/var/run/qmp_{id}.sock'.format(id=qemu_id)
  93             self._temp['qga'] = '/var/run/qga_{id}.sock'.format(id=qemu_id)
  94         elif img == Constants.QEMU_VM_KERNEL:
  95             self._opt['img'], _ = exec_cmd_no_error(
  96                 node,
  97                 'ls -1 {img}* | tail -1'.format(img=Constants.QEMU_VM_KERNEL),
  98                 message='Qemu Kernel VM image not found!')
  99             self._opt['vm_type'] = 'kernelvm'
 100             self._temp['log'] = '/tmp/serial_{id}.log'.format(id=qemu_id)
 101             self._temp['ini'] = '/etc/vm_init_{id}.conf'.format(id=qemu_id)
 102             self._opt['initrd'], _ = exec_cmd_no_error(
 103                 node,
 104                 'ls -1 {initrd}* | tail -1'.format(
 105                     initrd=Constants.QEMU_VM_KERNEL_INITRD),
 106                 message='Qemu Kernel initrd image not found!')
 107         else:
 108             raise RuntimeError('QEMU: Unknown VM image option: {}'.format(img))
 109         # Computed parameters for QEMU command line.
 110         self._params = OptionString(prefix='-')
 111         self.add_params()
 112
 113     def add_params(self):
 114         """Set QEMU command line parameters."""
 115         self.add_default_params()
 116         if self._opt.get('vm_type', '') == 'nestedvm':
 117             self.add_nestedvm_params()
 118         elif self._opt.get('vm_type', '') == 'kernelvm':
 119             self.add_kernelvm_params()
 120         else:
 121             raise RuntimeError('QEMU: Unsupported VM type!')
 122
 123     def add_default_params(self):
 124         """Set default QEMU command line parameters."""
 125         self._params.add('daemonize')
 126         self._params.add('nodefaults')
 127         self._params.add_with_value('name', 'vnf{qemu},debug-threads=on'.format(
 128             qemu=self._opt.get('qemu_id')))
 129         self._params.add('no-user-config')
 130         self._params.add_with_value('monitor', 'none')
 131         self._params.add_with_value('display', 'none')
 132         self._params.add_with_value('vga', 'none')
 133         self._params.add('enable-kvm')
 134         self._params.add_with_value('pidfile', self._temp.get('pidfile'))
 135         self._params.add_with_value('cpu', 'host')
 136
 137         if self._arch == 'aarch64':
 138             machine_args = 'virt,accel=kvm,usb=off,mem-merge=off,gic-version=3'
 139         else:
 140             machine_args = 'pc,accel=kvm,usb=off,mem-merge=off'
 141         self._params.add_with_value(
 142             'machine', machine_args)
 143         self._params.add_with_value(
 144             'smp', '{smp},sockets=1,cores={smp},threads=1'.format(
 145                 smp=self._opt.get('smp')))
 146         self._params.add_with_value(
 147             'object', 'memory-backend-file,id=mem,size={mem}M,'
 148             'mem-path=/dev/hugepages,share=on'.format(mem=self._opt.get('mem')))
 149         self._params.add_with_value(
 150             'm', '{mem}M'.format(mem=self._opt.get('mem')))
 151         self._params.add_with_value('numa', 'node,memdev=mem')
 152         self._params.add_with_value('balloon', 'none')
 153
 154     def add_nestedvm_params(self):
 155         """Set NestedVM QEMU parameters."""
 156         self._params.add_with_value(
 157             'net', 'nic,macaddr=52:54:00:00:{qemu:02x}:ff'.format(
 158                 qemu=self._opt.get('qemu_id')))
 159         self._params.add_with_value(
 160             'net', 'user,hostfwd=tcp::{info[port]}-:22'.format(
 161                 info=self._vm_info))
 162         # TODO: Remove try except after fully migrated to Bionic or
 163         # qemu_set_node is removed.
 164         try:
 165             locking = ',file.locking=off'\
 166                 if self.qemu_version(version='2.10') else ''
 167         except AttributeError:
 168             locking = ''
 169         self._params.add_with_value(
 170             'drive', 'file={img},format=raw,cache=none,if=virtio{locking}'.
 171             format(img=self._opt.get('img'), locking=locking))
 172         self._params.add_with_value(
 173             'qmp', 'unix:{qmp},server,nowait'.format(qmp=self._temp.get('qmp')))
 174         self._params.add_with_value(
 175             'chardev', 'socket,host=127.0.0.1,port={info[serial]},'
 176             'id=gnc0,server,nowait'.format(info=self._vm_info))
 177         self._params.add_with_value('device', 'isa-serial,chardev=gnc0')
 178         self._params.add_with_value(
 179             'chardev', 'socket,path={qga},server,nowait,id=qga0'.format(
 180                 qga=self._temp.get('qga')))
 181         self._params.add_with_value('device', 'isa-serial,chardev=qga0')
 182
 183     def add_kernelvm_params(self):
 184         """Set KernelVM QEMU parameters."""
 185         console = 'ttyAMA0' if self._arch == 'aarch64' else 'ttyS0'
 186         self._params.add_with_value('serial', 'file:{log}'.format(
 187             log=self._temp.get('log')))
 188         self._params.add_with_value(
 189             'fsdev', 'local,id=root9p,path=/,security_model=none')
 190         self._params.add_with_value(
 191             'device', 'virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot')
 192         self._params.add_with_value(
 193             'kernel', '{img}'.format(img=self._opt.get('img')))
 194         self._params.add_with_value(
 195             'initrd', '{initrd}'.format(initrd=self._opt.get('initrd')))
 196         self._params.add_with_value(
 197             'append', '"ro rootfstype=9p rootflags=trans=virtio '
 198                       'root=virtioroot console={console} tsc=reliable '
 199                       'hugepages=256 init={init} fastboot"'.format(
 200                           console=console, init=self._temp.get('ini')))
 201
 202     def create_kernelvm_config_vpp(self, **kwargs):
 203         """Create QEMU VPP config files.
 204
 205         :param kwargs: Key-value pairs to replace content of VPP configuration
 206             file.
 207         :type kwargs: dict
 208         """
 209         startup = ('/etc/vpp/vm_startup_{id}.conf'.
 210                    format(id=self._opt.get('qemu_id')))
 211         running = ('/etc/vpp/vm_running_{id}.exec'.
 212                    format(id=self._opt.get('qemu_id')))
 213
 214         self._temp['startup'] = startup
 215         self._temp['running'] = running
 216         self._opt['vnf_bin'] = ('/usr/bin/vpp -c {startup}'.
 217                                 format(startup=startup))
 218
 219         # Create VPP startup configuration.
 220         vpp_config = VppConfigGenerator()
 221         vpp_config.set_node(self._node)
 222         vpp_config.add_unix_nodaemon()
 223         vpp_config.add_unix_cli_listen()
 224         vpp_config.add_unix_exec(running)
 225         vpp_config.add_socksvr()
 226         vpp_config.add_cpu_main_core('0')
 227         if self._opt.get('smp') > 1:
 228             vpp_config.add_cpu_corelist_workers('1-{smp}'.format(
 229                 smp=self._opt.get('smp')-1))
 230         vpp_config.add_dpdk_dev('0000:00:06.0', '0000:00:07.0')
 231         vpp_config.add_dpdk_dev_default_rxq(kwargs['queues'])
 232         vpp_config.add_dpdk_log_level('debug')
 233         if not kwargs['jumbo_frames']:
 234             vpp_config.add_dpdk_no_multi_seg()
 235             vpp_config.add_dpdk_no_tx_checksum_offload()
 236         vpp_config.add_plugin('disable', 'default')
 237         vpp_config.add_plugin('enable', 'dpdk_plugin.so')
 238         vpp_config.write_config(startup)
 239
 240         # Create VPP running configuration.
 241         template = '{res}/{tpl}.exec'.format(res=Constants.RESOURCES_TPL_VM,
 242                                              tpl=self._opt.get('vnf'))
 243         exec_cmd_no_error(self._node, 'rm -f {running}'.format(running=running),
 244                           sudo=True)
 245
 246         with open(template, 'r') as src_file:
 247             src = Template(src_file.read())
 248             exec_cmd_no_error(
 249                 self._node, "echo '{out}' | sudo tee {running}".format(
 250                     out=src.safe_substitute(**kwargs), running=running))
 251
 252     def create_kernelvm_config_testpmd_io(self, **kwargs):
 253         """Create QEMU testpmd-io command line.
 254
 255         :param kwargs: Key-value pairs to construct command line parameters.
 256         :type kwargs: dict
 257         """
 258         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 259             eal_corelist='0-{smp}'.format(smp=self._opt.get('smp') - 1),
 260             eal_driver=False,
 261             eal_in_memory=True,
 262             pmd_num_mbufs=16384,
 263             pmd_rxq=kwargs['queues'],
 264             pmd_txq=kwargs['queues'],
 265             pmd_tx_offloads='0x0',
 266             pmd_disable_hw_vlan=False,
 267             pmd_nb_cores=str(self._opt.get('smp') - 1))
 268
 269         self._opt['vnf_bin'] = ('{testpmd_path}/{testpmd_cmd}'.
 270                                 format(testpmd_path=self._testpmd_path,
 271                                        testpmd_cmd=testpmd_cmd))
 272
 273     def create_kernelvm_config_testpmd_mac(self, **kwargs):
 274         """Create QEMU testpmd-mac command line.
 275
 276         :param kwargs: Key-value pairs to construct command line parameters.
 277         :type kwargs: dict
 278         """
 279         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 280             eal_corelist='0-{smp}'.format(smp=self._opt.get('smp') - 1),
 281             eal_driver=False,
 282             eal_in_memory=True,
 283             pmd_num_mbufs=16384,
 284             pmd_fwd_mode='mac',
 285             pmd_eth_peer_0='0,{mac}'.format(mac=kwargs['vif1_mac']),
 286             pmd_eth_peer_1='1,{mac}'.format(mac=kwargs['vif2_mac']),
 287             pmd_rxq=kwargs['queues'],
 288             pmd_txq=kwargs['queues'],
 289             pmd_tx_offloads='0x0',
 290             pmd_disable_hw_vlan=False,
 291             pmd_nb_cores=str(self._opt.get('smp') - 1))
 292
 293         self._opt['vnf_bin'] = ('{testpmd_path}/{testpmd_cmd}'.
 294                                 format(testpmd_path=self._testpmd_path,
 295                                        testpmd_cmd=testpmd_cmd))
 296
 297     def create_kernelvm_init(self, **kwargs):
 298         """Create QEMU init script.
 299
 300         :param kwargs: Key-value pairs to replace content of init startup file.
 301         :type kwargs: dict
 302         """
 303         template = '{res}/init.sh'.format(res=Constants.RESOURCES_TPL_VM)
 304         init = self._temp.get('ini')
 305         exec_cmd_no_error(
 306             self._node, 'rm -f {init}'.format(init=init), sudo=True)
 307
 308         with open(template, 'r') as src_file:
 309             src = Template(src_file.read())
 310             exec_cmd_no_error(
 311                 self._node, "echo '{out}' | sudo tee {init}".format(
 312                     out=src.safe_substitute(**kwargs), init=init))
 313             exec_cmd_no_error(
 314                 self._node, "chmod +x {init}".format(init=init), sudo=True)
 315
 316     def configure_kernelvm_vnf(self, **kwargs):
 317         """Create KernelVM VNF configurations.
 318
 319         :param kwargs: Key-value pairs for templating configs.
 320         :type kwargs: dict
 321         """
 322         if 'vpp' in self._opt.get('vnf'):
 323             self.create_kernelvm_config_vpp(**kwargs)
 324         elif 'testpmd_io' in self._opt.get('vnf'):
 325             self.create_kernelvm_config_testpmd_io(**kwargs)
 326         elif 'testpmd_mac' in self._opt.get('vnf'):
 327             self.create_kernelvm_config_testpmd_mac(**kwargs)
 328         else:
 329             raise RuntimeError('QEMU: Unsupported VNF!')
 330         self.create_kernelvm_init(vnf_bin=self._opt['vnf_bin'])
 331
 332     def get_qemu_pids(self):
 333         """Get QEMU CPU pids.
 334
 335         :returns: List of QEMU CPU pids.
 336         :rtype: list of str
 337         """
 338         command = ("grep -rwl 'CPU' /proc/$(sudo cat {pidfile})/task/*/comm ".
 339                    format(pidfile=self._temp.get('pidfile')))
 340         command += (r"| xargs dirname | sed -e 's/\/.*\///g' | uniq")
 341
 342         stdout, _ = exec_cmd_no_error(self._node, command)
 343         return stdout.splitlines()
 344
 345     def qemu_set_affinity(self, *host_cpus):
 346         """Set qemu affinity by getting thread PIDs via QMP and taskset to list
 347         of CPU cores. Function tries to execute 3 times to avoid race condition
 348         in getting thread PIDs.
 349
 350         :param host_cpus: List of CPU cores.
 351         :type host_cpus: list
 352         """
 353         for _ in range(3):
 354             try:
 355                 qemu_cpus = self.get_qemu_pids()
 356
 357                 if len(qemu_cpus) != len(host_cpus):
 358                     sleep(1)
 359                     continue
 360                 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
 361                     command = ('taskset -pc {host_cpu} {thread}'.
 362                                format(host_cpu=host_cpu, thread=qemu_cpu))
 363                     message = ('QEMU: Set affinity failed on {host}!'.
 364                                format(host=self._node['host']))
 365                     exec_cmd_no_error(self._node, command, sudo=True,
 366                                       message=message)
 367                 break
 368             except (RuntimeError, ValueError):
 369                 self.qemu_kill_all()
 370                 raise
 371         else:
 372             self.qemu_kill_all()
 373             raise RuntimeError('Failed to set Qemu threads affinity!')
 374
 375     def qemu_set_scheduler_policy(self):
 376         """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
 377         processes.
 378
 379         :raises RuntimeError: Set scheduler policy failed.
 380         """
 381         try:
 382             qemu_cpus = self.get_qemu_pids()
 383
 384             for qemu_cpu in qemu_cpus:
 385                 command = ('chrt -r -p 1 {thread}'.
 386                            format(thread=qemu_cpu))
 387                 message = ('QEMU: Set SCHED_RR failed on {host}'.
 388                            format(host=self._node['host']))
 389                 exec_cmd_no_error(self._node, command, sudo=True,
 390                                   message=message)
 391         except (RuntimeError, ValueError):
 392             self.qemu_kill_all()
 393             raise
 394
 395     def qemu_add_vhost_user_if(self, socket, server=True, jumbo_frames=False,
 396                                queue_size=None, queues=1):
 397         """Add Vhost-user interface.
 398
 399         :param socket: Path of the unix socket.
 400         :param server: If True the socket shall be a listening socket.
 401         :param jumbo_frames: Set True if jumbo frames are used in the test.
 402         :param queue_size: Vring queue size.
 403         :param queues: Number of queues.
 404         :type socket: str
 405         :type server: bool
 406         :type jumbo_frames: bool
 407         :type queue_size: int
 408         :type queues: int
 409         """
 410         self._vhost_id += 1
 411         self._params.add_with_value(
 412             'chardev', 'socket,id=char{vhost},path={socket}{server}'.format(
 413                 vhost=self._vhost_id, socket=socket,
 414                 server=',server' if server is True else ''))
 415         self._params.add_with_value(
 416             'netdev', 'vhost-user,id=vhost{vhost},chardev=char{vhost},'
 417             'queues={queues}'.format(vhost=self._vhost_id, queues=queues))
 418         mac = ('52:54:00:00:{qemu:02x}:{vhost:02x}'.
 419                format(qemu=self._opt.get('qemu_id'), vhost=self._vhost_id))
 420         queue_size = ('rx_queue_size={queue_size},tx_queue_size={queue_size}'.
 421                       format(queue_size=queue_size)) if queue_size else ''
 422         mbuf = 'on,host_mtu=9200'
 423         self._params.add_with_value(
 424             'device', 'virtio-net-pci,netdev=vhost{vhost},mac={mac},'
 425             'addr={addr}.0,mq=on,vectors={vectors},csum=off,gso=off,'
 426             'guest_tso4=off,guest_tso6=off,guest_ecn=off,mrg_rxbuf={mbuf},'
 427             '{queue_size}'.format(
 428                 addr=self._vhost_id+5, vhost=self._vhost_id, mac=mac,
 429                 mbuf=mbuf if jumbo_frames else 'off', queue_size=queue_size,
 430                 vectors=(2 * queues + 2)))
 431
 432         # Add interface MAC and socket to the node dict.
 433         if_data = {'mac_address': mac, 'socket': socket}
 434         if_name = 'vhost{vhost}'.format(vhost=self._vhost_id)
 435         self._vm_info['interfaces'][if_name] = if_data
 436         # Add socket to temporary file list.
 437         self._temp[if_name] = socket
 438
 439     def _qemu_qmp_exec(self, cmd):
 440         """Execute QMP command.
 441
 442         QMP is JSON based protocol which allows to control QEMU instance.
 443
 444         :param cmd: QMP command to execute.
 445         :type cmd: str
 446         :returns: Command output in python representation of JSON format. The
 447             { "return": {} } response is QMP's success response. An error
 448             response will contain the "error" keyword instead of "return".
 449         """
 450         # To enter command mode, the qmp_capabilities command must be issued.
 451         command = ('echo "{{ \\"execute\\": \\"qmp_capabilities\\" }}'
 452                    '{{ \\"execute\\": \\"{cmd}\\" }}" | '
 453                    'sudo -S socat - UNIX-CONNECT:{qmp}'.
 454                    format(cmd=cmd, qmp=self._temp.get('qmp')))
 455         message = ('QMP execute "{cmd}" failed on {host}'.
 456                    format(cmd=cmd, host=self._node['host']))
 457         stdout, _ = exec_cmd_no_error(
 458             self._node, command, sudo=False, message=message)
 459
 460         # Skip capabilities negotiation messages.
 461         out_list = stdout.splitlines()
 462         if len(out_list) < 3:
 463             raise RuntimeError(
 464                 'Invalid QMP output on {host}'.format(host=self._node['host']))
 465         return json.loads(out_list[2])
 466
 467     def _qemu_qga_flush(self):
 468         """Flush the QGA parser state."""
 469         command = ('(printf "\xFF"; sleep 1) | '
 470                    'sudo -S socat - UNIX-CONNECT:{qga}'.
 471                    format(qga=self._temp.get('qga')))
 472         message = ('QGA flush failed on {host}'.format(host=self._node['host']))
 473         stdout, _ = exec_cmd_no_error(
 474             self._node, command, sudo=False, message=message)
 475
 476         return json.loads(stdout.split('\n', 1)[0]) if stdout else dict()
 477
 478     def _qemu_qga_exec(self, cmd):
 479         """Execute QGA command.
 480
 481         QGA provide access to a system-level agent via standard QMP commands.
 482
 483         :param cmd: QGA command to execute.
 484         :type cmd: str
 485         """
 486         command = ('(echo "{{ \\"execute\\": \\"{cmd}\\" }}"; sleep 1) | '
 487                    'sudo -S socat - UNIX-CONNECT:{qga}'.
 488                    format(cmd=cmd, qga=self._temp.get('qga')))
 489         message = ('QGA execute "{cmd}" failed on {host}'.
 490                    format(cmd=cmd, host=self._node['host']))
 491         stdout, _ = exec_cmd_no_error(
 492             self._node, command, sudo=False, message=message)
 493
 494         return json.loads(stdout.split('\n', 1)[0]) if stdout else dict()
 495
 496     def _wait_until_vm_boot(self):
 497         """Wait until QEMU with NestedVM is booted."""
 498         if self._opt.get('vm_type') == 'nestedvm':
 499             self._wait_until_nestedvm_boot()
 500             self._update_vm_interfaces()
 501         elif self._opt.get('vm_type') == 'kernelvm':
 502             self._wait_until_kernelvm_boot()
 503         else:
 504             raise RuntimeError('QEMU: Unsupported VM type!')
 505
 506     def _wait_until_nestedvm_boot(self, retries=12):
 507         """Wait until QEMU with NestedVM is booted.
 508
 509         First try to flush qga until there is output.
 510         Then ping QEMU guest agent each 5s until VM booted or timeout.
 511
 512         :param retries: Number of retries with 5s between trials.
 513         :type retries: int
 514         """
 515         for _ in range(retries):
 516             out = None
 517             try:
 518                 out = self._qemu_qga_flush()
 519             except ValueError:
 520                 logger.trace('QGA qga flush unexpected output {out}'.
 521                              format(out=out))
 522             # Empty output - VM not booted yet
 523             if not out:
 524                 sleep(5)
 525             else:
 526                 break
 527         else:
 528             raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
 529                                format(host=self._node['host']))
 530         for _ in range(retries):
 531             out = None
 532             try:
 533                 out = self._qemu_qga_exec('guest-ping')
 534             except ValueError:
 535                 logger.trace('QGA guest-ping unexpected output {out}'.
 536                              format(out=out))
 537             # Empty output - VM not booted yet.
 538             if not out:
 539                 sleep(5)
 540             # Non-error return - VM booted.
 541             elif out.get('return') is not None:
 542                 break
 543             # Skip error and wait.
 544             elif out.get('error') is not None:
 545                 sleep(5)
 546             else:
 547                 # If there is an unexpected output from QGA guest-info, try
 548                 # again until timeout.
 549                 logger.trace('QGA guest-ping unexpected output {out}'.
 550                              format(out=out))
 551         else:
 552             raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
 553                                format(host=self._node['host']))
 554
 555     def _wait_until_kernelvm_boot(self, retries=60):
 556         """Wait until QEMU KernelVM is booted.
 557
 558         :param retries: Number of retries.
 559         :type retries: int
 560         """
 561         vpp_ver = VPPUtil.vpp_show_version(self._node)
 562
 563         for _ in range(retries):
 564             command = ('tail -1 {log}'.format(log=self._temp.get('log')))
 565             stdout = None
 566             try:
 567                 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 568                 sleep(1)
 569             except RuntimeError:
 570                 pass
 571             if vpp_ver in stdout or 'Press enter to exit' in stdout:
 572                 break
 573             if 'reboot: Power down' in stdout:
 574                 raise RuntimeError('QEMU: NF failed to run on {host}!'.
 575                                    format(host=self._node['host']))
 576         else:
 577             raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
 578                                format(host=self._node['host']))
 579
 580     def _update_vm_interfaces(self):
 581         """Update interface names in VM node dict."""
 582         # Send guest-network-get-interfaces command via QGA, output example:
 583         # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
 584         # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
 585         out = self._qemu_qga_exec('guest-network-get-interfaces')
 586         interfaces = out.get('return')
 587         mac_name = {}
 588         if not interfaces:
 589             raise RuntimeError('Get VM interface list failed on {host}'.
 590                                format(host=self._node['host']))
 591         # Create MAC-name dict.
 592         for interface in interfaces:
 593             if 'hardware-address' not in interface:
 594                 continue
 595             mac_name[interface['hardware-address']] = interface['name']
 596         # Match interface by MAC and save interface name.
 597         for interface in self._vm_info['interfaces'].values():
 598             mac = interface.get('mac_address')
 599             if_name = mac_name.get(mac)
 600             if if_name is None:
 601                 logger.trace(
 602                     'Interface name for MAC {mac} not found'.format(mac=mac))
 603             else:
 604                 interface['name'] = if_name
 605
 606     def qemu_start(self):
 607         """Start QEMU and wait until VM boot.
 608
 609         :returns: VM node info.
 610         :rtype: dict
 611         """
 612         cmd_opts = OptionString()
 613         cmd_opts.add('{bin_path}/qemu-system-{arch}'.format(
 614             bin_path=Constants.QEMU_BIN_PATH, arch=self._arch))
 615         cmd_opts.extend(self._params)
 616         message = ('QEMU: Start failed on {host}!'.
 617                    format(host=self._node['host']))
 618         try:
 619             DUTSetup.check_huge_page(
 620                 self._node, '/dev/hugepages', self._opt.get('mem'))
 621
 622             exec_cmd_no_error(
 623                 self._node, cmd_opts, timeout=300, sudo=True, message=message)
 624             self._wait_until_vm_boot()
 625         except RuntimeError:
 626             self.qemu_kill_all()
 627             raise
 628         return self._vm_info
 629
 630     def qemu_kill(self):
 631         """Kill qemu process."""
 632         exec_cmd(self._node, 'chmod +r {pidfile}'.
 633                  format(pidfile=self._temp.get('pidfile')), sudo=True)
 634         exec_cmd(self._node, 'kill -SIGKILL $(cat {pidfile})'.
 635                  format(pidfile=self._temp.get('pidfile')), sudo=True)
 636
 637         for value in self._temp.values():
 638             exec_cmd(self._node, 'cat {value}'.format(value=value), sudo=True)
 639             exec_cmd(self._node, 'rm -f {value}'.format(value=value), sudo=True)
 640
 641     def qemu_kill_all(self):
 642         """Kill all qemu processes on DUT node if specified."""
 643         exec_cmd(self._node, 'pkill -SIGKILL qemu', sudo=True)
 644
 645         for value in self._temp.values():
 646             exec_cmd(self._node, 'cat {value}'.format(value=value), sudo=True)
 647             exec_cmd(self._node, 'rm -f {value}'.format(value=value), sudo=True)
 648
 649     def qemu_version(self, version=None):
 650         """Return Qemu version or compare if version is higher than parameter.
 651
 652         :param version: Version to compare.
 653         :type version: str
 654         :returns: Qemu version or Boolean if version is higher than parameter.
 655         :rtype: str or bool
 656         """
 657         command = ('{bin_path}/qemu-system-{arch} --version'.format(
 658             bin_path=Constants.QEMU_BIN_PATH,
 659             arch=self._arch))
 660         try:
 661             stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 662             ver = match(r'QEMU emulator version ([\d.]*)', stdout).group(1)
 663             return StrictVersion(ver) > StrictVersion(version) \
 664                 if version else ver
 665         except RuntimeError:
 666             self.qemu_kill_all()
 667             raise