resources/libraries/python/QemuUtils.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """QEMU utilities library."""
  15
  16 # Disable due to pylint bug
  17 # pylint: disable=no-name-in-module,import-error
  18 from distutils.version import StrictVersion
  19 import json
  20 from re import match
  21 from string import Template
  22 from time import sleep
  23
  24 from robot.api import logger
  25 from resources.libraries.python.Constants import Constants
  26 from resources.libraries.python.DpdkUtil import DpdkUtil
  27 from resources.libraries.python.DUTSetup import DUTSetup
  28 from resources.libraries.python.OptionString import OptionString
  29 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
  30 from resources.libraries.python.VPPUtil import VPPUtil
  31 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
  32 from resources.libraries.python.topology import NodeType, Topology
  33
  34 __all__ = ["QemuUtils"]
  35
  36
  37 class QemuUtils(object):
  38     """QEMU utilities."""
  39
  40     # Use one instance of class per tests.
  41     ROBOT_LIBRARY_SCOPE = 'TEST CASE'
  42
  43     def __init__(self, node, qemu_id=1, smp=1, mem=512, vnf=None,
  44                  img=Constants.QEMU_VM_IMAGE):
  45         """Initialize QemuUtil class.
  46
  47         :param node: Node to run QEMU on.
  48         :param qemu_id: QEMU identifier.
  49         :param smp: Number of virtual SMP units (cores).
  50         :param mem: Amount of memory.
  51         :param vnf: Network function workload.
  52         :param img: QEMU disk image or kernel image path.
  53         :type node: dict
  54         :type qemu_id: int
  55         :type smp: int
  56         :type mem: int
  57         :type vnf: str
  58         :type img: str
  59         """
  60         self._vhost_id = 0
  61         self._node = node
  62         self._arch = Topology.get_node_arch(self._node)
  63         dpdk_target = 'arm64-armv8a' if self._arch == 'aarch64' \
  64             else 'x86_64-native'
  65         self._testpmd_path = '{path}/{dpdk_target}-linuxapp-gcc/app'\
  66             .format(path=Constants.QEMU_VM_DPDK, dpdk_target=dpdk_target)
  67         self._vm_info = {
  68             'host': node['host'],
  69             'type': NodeType.VM,
  70             'port': 10021 + qemu_id,
  71             'serial': 4555 + qemu_id,
  72             'username': 'cisco',
  73             'password': 'cisco',
  74             'interfaces': {},
  75         }
  76         if node['port'] != 22:
  77             self._vm_info['host_port'] = node['port']
  78             self._vm_info['host_username'] = node['username']
  79             self._vm_info['host_password'] = node['password']
  80         # Input Options.
  81         self._opt = dict()
  82         self._opt['qemu_id'] = qemu_id
  83         self._opt['mem'] = int(mem)
  84         self._opt['smp'] = int(smp)
  85         self._opt['img'] = img
  86         self._opt['vnf'] = vnf
  87         # Temporary files.
  88         self._temp = dict()
  89         self._temp['pidfile'] = '/var/run/qemu_{id}.pid'.format(id=qemu_id)
  90         if img == Constants.QEMU_VM_IMAGE:
  91             self._opt['vm_type'] = 'nestedvm'
  92             self._temp['qmp'] = '/var/run/qmp_{id}.sock'.format(id=qemu_id)
  93             self._temp['qga'] = '/var/run/qga_{id}.sock'.format(id=qemu_id)
  94         elif img == Constants.QEMU_VM_KERNEL:
  95             self._opt['img'], _ = exec_cmd_no_error(
  96                 node,
  97                 'ls -1 {img}* | tail -1'.format(img=Constants.QEMU_VM_KERNEL),
  98                 message='Qemu Kernel VM image not found!')
  99             self._opt['vm_type'] = 'kernelvm'
 100             self._temp['log'] = '/tmp/serial_{id}.log'.format(id=qemu_id)
 101             self._temp['ini'] = '/etc/vm_init_{id}.conf'.format(id=qemu_id)
 102             self._opt['initrd'], _ = exec_cmd_no_error(
 103                 node,
 104                 'ls -1 {initrd}* | tail -1'.format(
 105                     initrd=Constants.QEMU_VM_KERNEL_INITRD),
 106                 message='Qemu Kernel initrd image not found!')
 107         else:
 108             raise RuntimeError('QEMU: Unknown VM image option: {}'.format(img))
 109         # Computed parameters for QEMU command line.
 110         self._params = OptionString(prefix='-')
 111         self.add_params()
 112
 113     def add_params(self):
 114         """Set QEMU command line parameters."""
 115         self.add_default_params()
 116         if self._opt.get('vm_type', '') == 'nestedvm':
 117             self.add_nestedvm_params()
 118         elif self._opt.get('vm_type', '') == 'kernelvm':
 119             self.add_kernelvm_params()
 120         else:
 121             raise RuntimeError('QEMU: Unsupported VM type!')
 122
 123     def add_default_params(self):
 124         """Set default QEMU command line parameters."""
 125         self._params.add('daemonize')
 126         self._params.add('nodefaults')
 127         self._params.add_with_value('name', 'vnf{qemu},debug-threads=on'.format(
 128             qemu=self._opt.get('qemu_id')))
 129         self._params.add('no-user-config')
 130         self._params.add_with_value('monitor', 'none')
 131         self._params.add_with_value('display', 'none')
 132         self._params.add_with_value('vga', 'none')
 133         self._params.add('enable-kvm')
 134         self._params.add_with_value('pidfile', self._temp.get('pidfile'))
 135         self._params.add_with_value('cpu', 'host')
 136
 137         if self._arch == 'aarch64':
 138             machine_args = 'virt,accel=kvm,usb=off,mem-merge=off,gic-version=3'
 139         else:
 140             machine_args = 'pc,accel=kvm,usb=off,mem-merge=off'
 141         self._params.add_with_value(
 142             'machine', machine_args)
 143         self._params.add_with_value(
 144             'smp', '{smp},sockets=1,cores={smp},threads=1'.format(
 145                 smp=self._opt.get('smp')))
 146         self._params.add_with_value(
 147             'object', 'memory-backend-file,id=mem,size={mem}M,'
 148             'mem-path=/dev/hugepages,share=on'.format(mem=self._opt.get('mem')))
 149         self._params.add_with_value(
 150             'm', '{mem}M'.format(mem=self._opt.get('mem')))
 151         self._params.add_with_value('numa', 'node,memdev=mem')
 152         self._params.add_with_value('balloon', 'none')
 153
 154     def add_nestedvm_params(self):
 155         """Set NestedVM QEMU parameters."""
 156         self._params.add_with_value(
 157             'net', 'nic,macaddr=52:54:00:00:{qemu:02x}:ff'.format(
 158                 qemu=self._opt.get('qemu_id')))
 159         self._params.add_with_value(
 160             'net', 'user,hostfwd=tcp::{info[port]}-:22'.format(
 161                 info=self._vm_info))
 162         # TODO: Remove try except after fully migrated to Bionic or
 163         # qemu_set_node is removed.
 164         try:
 165             locking = ',file.locking=off'\
 166                 if self.qemu_version(version='2.10') else ''
 167         except AttributeError:
 168             locking = ''
 169         self._params.add_with_value(
 170             'drive', 'file={img},format=raw,cache=none,if=virtio{locking}'.
 171             format(img=self._opt.get('img'), locking=locking))
 172         self._params.add_with_value(
 173             'qmp', 'unix:{qmp},server,nowait'.format(qmp=self._temp.get('qmp')))
 174         self._params.add_with_value(
 175             'chardev', 'socket,host=127.0.0.1,port={info[serial]},'
 176             'id=gnc0,server,nowait'.format(info=self._vm_info))
 177         self._params.add_with_value('device', 'isa-serial,chardev=gnc0')
 178         self._params.add_with_value(
 179             'chardev', 'socket,path={qga},server,nowait,id=qga0'.format(
 180                 qga=self._temp.get('qga')))
 181         self._params.add_with_value('device', 'isa-serial,chardev=qga0')
 182
 183     def add_kernelvm_params(self):
 184         """Set KernelVM QEMU parameters."""
 185         console = 'ttyAMA0' if self._arch == 'aarch64' else 'ttyS0'
 186         self._params.add_with_value('serial', 'file:{log}'.format(
 187             log=self._temp.get('log')))
 188         self._params.add_with_value(
 189             'fsdev', 'local,id=root9p,path=/,security_model=none')
 190         self._params.add_with_value(
 191             'device', 'virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot')
 192         self._params.add_with_value(
 193             'kernel', '{img}'.format(img=self._opt.get('img')))
 194         self._params.add_with_value(
 195             'initrd', '{initrd}'.format(initrd=self._opt.get('initrd')))
 196         self._params.add_with_value(
 197             'append', '"ro rootfstype=9p rootflags=trans=virtio '
 198                       'root=virtioroot console={console} tsc=reliable '
 199                       'hugepages=256 init={init}"'.format(
 200                           console=console, init=self._temp.get('ini')))
 201
 202     def create_kernelvm_config_vpp(self, **kwargs):
 203         """Create QEMU VPP config files.
 204
 205         :param kwargs: Key-value pairs to replace content of VPP configuration
 206             file.
 207         :type kwargs: dict
 208         """
 209         startup = ('/etc/vpp/vm_startup_{id}.conf'.
 210                    format(id=self._opt.get('qemu_id')))
 211         running = ('/etc/vpp/vm_running_{id}.exec'.
 212                    format(id=self._opt.get('qemu_id')))
 213
 214         self._temp['startup'] = startup
 215         self._temp['running'] = running
 216         self._opt['vnf_bin'] = ('/usr/bin/vpp -c {startup}'.
 217                                 format(startup=startup))
 218
 219         # Create VPP startup configuration.
 220         vpp_config = VppConfigGenerator()
 221         vpp_config.set_node(self._node)
 222         vpp_config.add_unix_nodaemon()
 223         vpp_config.add_unix_cli_listen()
 224         vpp_config.add_unix_exec(running)
 225         vpp_config.add_cpu_main_core('0')
 226         if self._opt.get('smp') > 1:
 227             vpp_config.add_cpu_corelist_workers('1-{smp}'.format(
 228                 smp=self._opt.get('smp')-1))
 229         vpp_config.add_dpdk_dev('0000:00:06.0', '0000:00:07.0')
 230         vpp_config.add_dpdk_dev_default_rxq(kwargs['queues'])
 231         vpp_config.add_dpdk_log_level('debug')
 232         if not kwargs['jumbo_frames']:
 233             vpp_config.add_dpdk_no_multi_seg()
 234             vpp_config.add_dpdk_no_tx_checksum_offload()
 235         vpp_config.add_plugin('disable', 'default')
 236         vpp_config.add_plugin('enable', 'dpdk_plugin.so')
 237         vpp_config.write_config(startup)
 238
 239         # Create VPP running configuration.
 240         template = '{res}/{tpl}.exec'.format(res=Constants.RESOURCES_TPL_VM,
 241                                              tpl=self._opt.get('vnf'))
 242         exec_cmd_no_error(self._node, 'rm -f {running}'.format(running=running),
 243                           sudo=True)
 244
 245         with open(template, 'r') as src_file:
 246             src = Template(src_file.read())
 247             exec_cmd_no_error(
 248                 self._node, "echo '{out}' | sudo tee {running}".format(
 249                     out=src.safe_substitute(**kwargs), running=running))
 250
 251     def create_kernelvm_config_testpmd_io(self, **kwargs):
 252         """Create QEMU testpmd-io command line.
 253
 254         :param kwargs: Key-value pairs to construct command line parameters.
 255         :type kwargs: dict
 256         """
 257         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 258             eal_corelist='0-{smp}'.format(smp=self._opt.get('smp') - 1),
 259             eal_driver=False,
 260             eal_in_memory=True,
 261             pmd_num_mbufs=16384,
 262             pmd_rxq=kwargs['queues'],
 263             pmd_txq=kwargs['queues'],
 264             pmd_tx_offloads=False,
 265             pmd_disable_hw_vlan=False,
 266             pmd_max_pkt_len=9200 if kwargs['jumbo_frames'] else None,
 267             pmd_nb_cores=str(self._opt.get('smp') - 1))
 268
 269         self._opt['vnf_bin'] = ('{testpmd_path}/{testpmd_cmd}'.
 270                                 format(testpmd_path=self._testpmd_path,
 271                                        testpmd_cmd=testpmd_cmd))
 272
 273     def create_kernelvm_config_testpmd_mac(self, **kwargs):
 274         """Create QEMU testpmd-mac command line.
 275
 276         :param kwargs: Key-value pairs to construct command line parameters.
 277         :type kwargs: dict
 278         """
 279         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 280             eal_corelist='0-{smp}'.format(smp=self._opt.get('smp') - 1),
 281             eal_driver=False,
 282             eal_in_memory=True,
 283             pmd_num_mbufs=16384,
 284             pmd_fwd_mode='mac',
 285             pmd_eth_peer_0='0,{mac}'.format(mac=kwargs['vif1_mac']),
 286             pmd_eth_peer_1='1,{mac}'.format(mac=kwargs['vif2_mac']),
 287             pmd_rxq=kwargs['queues'],
 288             pmd_txq=kwargs['queues'],
 289             pmd_tx_offloads=False,
 290             pmd_disable_hw_vlan=False,
 291             pmd_max_pkt_len=9200 if kwargs['jumbo_frames'] else None,
 292             pmd_nb_cores=str(self._opt.get('smp') - 1))
 293
 294         self._opt['vnf_bin'] = ('{testpmd_path}/{testpmd_cmd}'.
 295                                 format(testpmd_path=self._testpmd_path,
 296                                        testpmd_cmd=testpmd_cmd))
 297
 298     def create_kernelvm_init(self, **kwargs):
 299         """Create QEMU init script.
 300
 301         :param kwargs: Key-value pairs to replace content of init startup file.
 302         :type kwargs: dict
 303         """
 304         template = '{res}/init.sh'.format(res=Constants.RESOURCES_TPL_VM)
 305         init = self._temp.get('ini')
 306         exec_cmd_no_error(
 307             self._node, 'rm -f {init}'.format(init=init), sudo=True)
 308
 309         with open(template, 'r') as src_file:
 310             src = Template(src_file.read())
 311             exec_cmd_no_error(
 312                 self._node, "echo '{out}' | sudo tee {init}".format(
 313                     out=src.safe_substitute(**kwargs), init=init))
 314             exec_cmd_no_error(
 315                 self._node, "chmod +x {init}".format(init=init), sudo=True)
 316
 317     def configure_kernelvm_vnf(self, **kwargs):
 318         """Create KernelVM VNF configurations.
 319
 320         :param kwargs: Key-value pairs for templating configs.
 321         :type kwargs: dict
 322         """
 323         if 'vpp' in self._opt.get('vnf'):
 324             self.create_kernelvm_config_vpp(**kwargs)
 325         elif 'testpmd_io' in self._opt.get('vnf'):
 326             self.create_kernelvm_config_testpmd_io(**kwargs)
 327         elif 'testpmd_mac' in self._opt.get('vnf'):
 328             self.create_kernelvm_config_testpmd_mac(**kwargs)
 329         else:
 330             raise RuntimeError('QEMU: Unsupported VNF!')
 331         self.create_kernelvm_init(vnf_bin=self._opt['vnf_bin'])
 332
 333     def get_qemu_pids(self):
 334         """Get QEMU CPU pids.
 335
 336         :returns: List of QEMU CPU pids.
 337         :rtype: list of str
 338         """
 339         command = ("grep -rwl 'CPU' /proc/$(sudo cat {pidfile})/task/*/comm ".
 340                    format(pidfile=self._temp.get('pidfile')))
 341         command += (r"| xargs dirname | sed -e 's/\/.*\///g' | uniq")
 342
 343         stdout, _ = exec_cmd_no_error(self._node, command)
 344         return stdout.splitlines()
 345
 346     def qemu_set_affinity(self, *host_cpus):
 347         """Set qemu affinity by getting thread PIDs via QMP and taskset to list
 348         of CPU cores. Function tries to execute 3 times to avoid race condition
 349         in getting thread PIDs.
 350
 351         :param host_cpus: List of CPU cores.
 352         :type host_cpus: list
 353         """
 354         for _ in range(3):
 355             try:
 356                 qemu_cpus = self.get_qemu_pids()
 357
 358                 if len(qemu_cpus) != len(host_cpus):
 359                     sleep(1)
 360                     continue
 361                 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
 362                     command = ('taskset -pc {host_cpu} {thread}'.
 363                                format(host_cpu=host_cpu, thread=qemu_cpu))
 364                     message = ('QEMU: Set affinity failed on {host}!'.
 365                                format(host=self._node['host']))
 366                     exec_cmd_no_error(self._node, command, sudo=True,
 367                                       message=message)
 368                 break
 369             except (RuntimeError, ValueError):
 370                 self.qemu_kill_all()
 371                 raise
 372         else:
 373             self.qemu_kill_all()
 374             raise RuntimeError('Failed to set Qemu threads affinity!')
 375
 376     def qemu_set_scheduler_policy(self):
 377         """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
 378         processes.
 379
 380         :raises RuntimeError: Set scheduler policy failed.
 381         """
 382         try:
 383             qemu_cpus = self.get_qemu_pids()
 384
 385             for qemu_cpu in qemu_cpus:
 386                 command = ('chrt -r -p 1 {thread}'.
 387                            format(thread=qemu_cpu))
 388                 message = ('QEMU: Set SCHED_RR failed on {host}'.
 389                            format(host=self._node['host']))
 390                 exec_cmd_no_error(self._node, command, sudo=True,
 391                                   message=message)
 392         except (RuntimeError, ValueError):
 393             self.qemu_kill_all()
 394             raise
 395
 396     def qemu_add_vhost_user_if(self, socket, server=True, jumbo_frames=False,
 397                                queue_size=None, queues=1):
 398         """Add Vhost-user interface.
 399
 400         :param socket: Path of the unix socket.
 401         :param server: If True the socket shall be a listening socket.
 402         :param jumbo_frames: Set True if jumbo frames are used in the test.
 403         :param queue_size: Vring queue size.
 404         :param queues: Number of queues.
 405         :type socket: str
 406         :type server: bool
 407         :type jumbo_frames: bool
 408         :type queue_size: int
 409         :type queues: int
 410         """
 411         self._vhost_id += 1
 412         self._params.add_with_value(
 413             'chardev', 'socket,id=char{vhost},path={socket}{server}'.format(
 414                 vhost=self._vhost_id, socket=socket,
 415                 server=',server' if server is True else ''))
 416         self._params.add_with_value(
 417             'netdev', 'vhost-user,id=vhost{vhost},chardev=char{vhost},'
 418             'queues={queues}'.format(vhost=self._vhost_id, queues=queues))
 419         mac = ('52:54:00:00:{qemu:02x}:{vhost:02x}'.
 420                format(qemu=self._opt.get('qemu_id'), vhost=self._vhost_id))
 421         queue_size = ('rx_queue_size={queue_size},tx_queue_size={queue_size}'.
 422                       format(queue_size=queue_size)) if queue_size else ''
 423         mbuf = 'on,host_mtu=9200'
 424         self._params.add_with_value(
 425             'device', 'virtio-net-pci,netdev=vhost{vhost},mac={mac},'
 426             'addr={addr}.0,mq=on,vectors={vectors},csum=off,gso=off,'
 427             'guest_tso4=off,guest_tso6=off,guest_ecn=off,mrg_rxbuf={mbuf},'
 428             '{queue_size}'.format(
 429                 addr=self._vhost_id+5, vhost=self._vhost_id, mac=mac,
 430                 mbuf=mbuf if jumbo_frames else 'off', queue_size=queue_size,
 431                 vectors=(2 * queues + 2)))
 432
 433         # Add interface MAC and socket to the node dict.
 434         if_data = {'mac_address': mac, 'socket': socket}
 435         if_name = 'vhost{vhost}'.format(vhost=self._vhost_id)
 436         self._vm_info['interfaces'][if_name] = if_data
 437         # Add socket to temporary file list.
 438         self._temp[if_name] = socket
 439
 440     def _qemu_qmp_exec(self, cmd):
 441         """Execute QMP command.
 442
 443         QMP is JSON based protocol which allows to control QEMU instance.
 444
 445         :param cmd: QMP command to execute.
 446         :type cmd: str
 447         :returns: Command output in python representation of JSON format. The
 448             { "return": {} } response is QMP's success response. An error
 449             response will contain the "error" keyword instead of "return".
 450         """
 451         # To enter command mode, the qmp_capabilities command must be issued.
 452         command = ('echo "{{ \\"execute\\": \\"qmp_capabilities\\" }}'
 453                    '{{ \\"execute\\": \\"{cmd}\\" }}" | '
 454                    'sudo -S socat - UNIX-CONNECT:{qmp}'.
 455                    format(cmd=cmd, qmp=self._temp.get('qmp')))
 456         message = ('QMP execute "{cmd}" failed on {host}'.
 457                    format(cmd=cmd, host=self._node['host']))
 458         stdout, _ = exec_cmd_no_error(
 459             self._node, command, sudo=False, message=message)
 460
 461         # Skip capabilities negotiation messages.
 462         out_list = stdout.splitlines()
 463         if len(out_list) < 3:
 464             raise RuntimeError(
 465                 'Invalid QMP output on {host}'.format(host=self._node['host']))
 466         return json.loads(out_list[2])
 467
 468     def _qemu_qga_flush(self):
 469         """Flush the QGA parser state."""
 470         command = ('(printf "\xFF"; sleep 1) | '
 471                    'sudo -S socat - UNIX-CONNECT:{qga}'.
 472                    format(qga=self._temp.get('qga')))
 473         message = ('QGA flush failed on {host}'.format(host=self._node['host']))
 474         stdout, _ = exec_cmd_no_error(
 475             self._node, command, sudo=False, message=message)
 476
 477         return json.loads(stdout.split('\n', 1)[0]) if stdout else dict()
 478
 479     def _qemu_qga_exec(self, cmd):
 480         """Execute QGA command.
 481
 482         QGA provide access to a system-level agent via standard QMP commands.
 483
 484         :param cmd: QGA command to execute.
 485         :type cmd: str
 486         """
 487         command = ('(echo "{{ \\"execute\\": \\"{cmd}\\" }}"; sleep 1) | '
 488                    'sudo -S socat - UNIX-CONNECT:{qga}'.
 489                    format(cmd=cmd, qga=self._temp.get('qga')))
 490         message = ('QGA execute "{cmd}" failed on {host}'.
 491                    format(cmd=cmd, host=self._node['host']))
 492         stdout, _ = exec_cmd_no_error(
 493             self._node, command, sudo=False, message=message)
 494
 495         return json.loads(stdout.split('\n', 1)[0]) if stdout else dict()
 496
 497     def _wait_until_vm_boot(self):
 498         """Wait until QEMU with NestedVM is booted."""
 499         if self._opt.get('vm_type') == 'nestedvm':
 500             self._wait_until_nestedvm_boot()
 501             self._update_vm_interfaces()
 502         elif self._opt.get('vm_type') == 'kernelvm':
 503             self._wait_until_kernelvm_boot()
 504         else:
 505             raise RuntimeError('QEMU: Unsupported VM type!')
 506
 507     def _wait_until_nestedvm_boot(self, retries=12):
 508         """Wait until QEMU with NestedVM is booted.
 509
 510         First try to flush qga until there is output.
 511         Then ping QEMU guest agent each 5s until VM booted or timeout.
 512
 513         :param retries: Number of retries with 5s between trials.
 514         :type retries: int
 515         """
 516         for _ in range(retries):
 517             out = None
 518             try:
 519                 out = self._qemu_qga_flush()
 520             except ValueError:
 521                 logger.trace('QGA qga flush unexpected output {out}'.
 522                              format(out=out))
 523             # Empty output - VM not booted yet
 524             if not out:
 525                 sleep(5)
 526             else:
 527                 break
 528         else:
 529             raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
 530                                format(host=self._node['host']))
 531         for _ in range(retries):
 532             out = None
 533             try:
 534                 out = self._qemu_qga_exec('guest-ping')
 535             except ValueError:
 536                 logger.trace('QGA guest-ping unexpected output {out}'.
 537                              format(out=out))
 538             # Empty output - VM not booted yet.
 539             if not out:
 540                 sleep(5)
 541             # Non-error return - VM booted.
 542             elif out.get('return') is not None:
 543                 break
 544             # Skip error and wait.
 545             elif out.get('error') is not None:
 546                 sleep(5)
 547             else:
 548                 # If there is an unexpected output from QGA guest-info, try
 549                 # again until timeout.
 550                 logger.trace('QGA guest-ping unexpected output {out}'.
 551                              format(out=out))
 552         else:
 553             raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
 554                                format(host=self._node['host']))
 555
 556     def _wait_until_kernelvm_boot(self, retries=60):
 557         """Wait until QEMU KernelVM is booted.
 558
 559         :param retries: Number of retries.
 560         :type retries: int
 561         """
 562         vpp_ver = VPPUtil.vpp_show_version(self._node)
 563
 564         for _ in range(retries):
 565             command = ('tail -1 {log}'.format(log=self._temp.get('log')))
 566             stdout = None
 567             try:
 568                 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 569                 sleep(1)
 570             except RuntimeError:
 571                 pass
 572             if vpp_ver in stdout or 'Press enter to exit' in stdout:
 573                 break
 574             if 'reboot: Power down' in stdout:
 575                 raise RuntimeError('QEMU: NF failed to run on {host}!'.
 576                                    format(host=self._node['host']))
 577         else:
 578             raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
 579                                format(host=self._node['host']))
 580
 581     def _update_vm_interfaces(self):
 582         """Update interface names in VM node dict."""
 583         # Send guest-network-get-interfaces command via QGA, output example:
 584         # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
 585         # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
 586         out = self._qemu_qga_exec('guest-network-get-interfaces')
 587         interfaces = out.get('return')
 588         mac_name = {}
 589         if not interfaces:
 590             raise RuntimeError('Get VM interface list failed on {host}'.
 591                                format(host=self._node['host']))
 592         # Create MAC-name dict.
 593         for interface in interfaces:
 594             if 'hardware-address' not in interface:
 595                 continue
 596             mac_name[interface['hardware-address']] = interface['name']
 597         # Match interface by MAC and save interface name.
 598         for interface in self._vm_info['interfaces'].values():
 599             mac = interface.get('mac_address')
 600             if_name = mac_name.get(mac)
 601             if if_name is None:
 602                 logger.trace(
 603                     'Interface name for MAC {mac} not found'.format(mac=mac))
 604             else:
 605                 interface['name'] = if_name
 606
 607     def qemu_start(self):
 608         """Start QEMU and wait until VM boot.
 609
 610         :returns: VM node info.
 611         :rtype: dict
 612         """
 613         cmd_opts = OptionString()
 614         cmd_opts.add('{bin_path}/qemu-system-{arch}'.format(
 615             bin_path=Constants.QEMU_BIN_PATH, arch=self._arch))
 616         cmd_opts.extend(self._params)
 617         message = ('QEMU: Start failed on {host}!'.
 618                    format(host=self._node['host']))
 619         try:
 620             DUTSetup.check_huge_page(
 621                 self._node, '/dev/hugepages', self._opt.get('mem'))
 622
 623             exec_cmd_no_error(
 624                 self._node, cmd_opts, timeout=300, sudo=True, message=message)
 625             self._wait_until_vm_boot()
 626         except RuntimeError:
 627             self.qemu_kill_all()
 628             raise
 629         return self._vm_info
 630
 631     def qemu_kill(self):
 632         """Kill qemu process."""
 633         exec_cmd(self._node, 'chmod +r {pidfile}'.
 634                  format(pidfile=self._temp.get('pidfile')), sudo=True)
 635         exec_cmd(self._node, 'kill -SIGKILL $(cat {pidfile})'.
 636                  format(pidfile=self._temp.get('pidfile')), sudo=True)
 637
 638         for value in self._temp.values():
 639             exec_cmd(self._node, 'cat {value}'.format(value=value), sudo=True)
 640             exec_cmd(self._node, 'rm -f {value}'.format(value=value), sudo=True)
 641
 642     def qemu_kill_all(self):
 643         """Kill all qemu processes on DUT node if specified."""
 644         exec_cmd(self._node, 'pkill -SIGKILL qemu', sudo=True)
 645
 646         for value in self._temp.values():
 647             exec_cmd(self._node, 'cat {value}'.format(value=value), sudo=True)
 648             exec_cmd(self._node, 'rm -f {value}'.format(value=value), sudo=True)
 649
 650     def qemu_version(self, version=None):
 651         """Return Qemu version or compare if version is higher than parameter.
 652
 653         :param version: Version to compare.
 654         :type version: str
 655         :returns: Qemu version or Boolean if version is higher than parameter.
 656         :rtype: str or bool
 657         """
 658         command = ('{bin_path}/qemu-system-{arch} --version'.format(
 659             bin_path=Constants.QEMU_BIN_PATH,
 660             arch=self._arch))
 661         try:
 662             stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 663             ver = match(r'QEMU emulator version ([\d.]*)', stdout).group(1)
 664             return StrictVersion(ver) > StrictVersion(version) \
 665                 if version else ver
 666         except RuntimeError:
 667             self.qemu_kill_all()
 668             raise