resources/libraries/python/QemuUtils.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """QEMU utilities library."""
  15
  16 from time import sleep
  17 from string import Template
  18 import json
  19 from re import match
  20 # Disable due to pylint bug
  21 # pylint: disable=no-name-in-module,import-error
  22 from distutils.version import StrictVersion
  23
  24 from robot.api import logger
  25 from resources.libraries.python.Constants import Constants
  26 from resources.libraries.python.DpdkUtil import DpdkUtil
  27 from resources.libraries.python.DUTSetup import DUTSetup
  28 from resources.libraries.python.OptionString import OptionString
  29 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
  30 from resources.libraries.python.VPPUtil import VPPUtil
  31 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
  32 from resources.libraries.python.topology import NodeType, Topology
  33
  34 __all__ = ["QemuUtils"]
  35
  36
  37 class QemuUtils(object):
  38     """QEMU utilities."""
  39
  40     # Use one instance of class per tests.
  41     ROBOT_LIBRARY_SCOPE = 'TEST CASE'
  42
  43     def __init__(self, node, qemu_id=1, smp=1, mem=512, vnf=None,
  44                  img=Constants.QEMU_VM_IMAGE):
  45         """Initialize QemuUtil class.
  46
  47         :param node: Node to run QEMU on.
  48         :param qemu_id: QEMU identifier.
  49         :param smp: Number of virtual SMP units (cores).
  50         :param mem: Amount of memory.
  51         :param vnf: Network function workload.
  52         :param img: QEMU disk image or kernel image path.
  53         :type node: dict
  54         :type qemu_id: int
  55         :type smp: int
  56         :type mem: int
  57         :type vnf: str
  58         :type img: str
  59         """
  60         self._vhost_id = 0
  61         self._node = node
  62         self._vm_info = {
  63             'host': node['host'],
  64             'type': NodeType.VM,
  65             'port': 10021 + qemu_id,
  66             'serial': 4555 + qemu_id,
  67             'username': 'cisco',
  68             'password': 'cisco',
  69             'interfaces': {},
  70         }
  71         if node['port'] != 22:
  72             self._vm_info['host_port'] = node['port']
  73             self._vm_info['host_username'] = node['username']
  74             self._vm_info['host_password'] = node['password']
  75         # Input Options.
  76         self._opt = dict()
  77         self._opt['qemu_id'] = qemu_id
  78         self._opt['mem'] = int(mem)
  79         self._opt['smp'] = int(smp)
  80         self._opt['img'] = img
  81         self._opt['vnf'] = vnf
  82         # Temporary files.
  83         self._temp = dict()
  84         self._temp['pidfile'] = '/var/run/qemu_{id}.pid'.format(id=qemu_id)
  85         if '/var/lib/vm/' in img:
  86             self._opt['vm_type'] = 'nestedvm'
  87             self._temp['qmp'] = '/var/run/qmp_{id}.sock'.format(id=qemu_id)
  88             self._temp['qga'] = '/var/run/qga_{id}.sock'.format(id=qemu_id)
  89         elif '/opt/boot/vmlinuz' in img:
  90             self._opt['vm_type'] = 'kernelvm'
  91             self._temp['log'] = '/tmp/serial_{id}.log'.format(id=qemu_id)
  92             self._temp['ini'] = '/etc/vm_init_{id}.conf'.format(id=qemu_id)
  93         else:
  94             raise RuntimeError('QEMU: Unknown VM image option!')
  95         # Computed parameters for QEMU command line.
  96         self._params = OptionString(prefix='-')
  97         self.add_params()
  98
  99     def add_params(self):
 100         """Set QEMU command line parameters."""
 101         self.add_default_params()
 102         if self._opt.get('vm_type', '') == 'nestedvm':
 103             self.add_nestedvm_params()
 104         elif self._opt.get('vm_type', '') == 'kernelvm':
 105             self.add_kernelvm_params()
 106         else:
 107             raise RuntimeError('QEMU: Unsupported VM type!')
 108
 109     def add_default_params(self):
 110         """Set default QEMU command line parameters."""
 111         self._params.add('daemonize')
 112         self._params.add('nodefaults')
 113         self._params.add_with_value('name', 'vnf{qemu},debug-threads=on'.format(
 114             qemu=self._opt.get('qemu_id')))
 115         self._params.add('no-user-config')
 116         self._params.add_with_value('monitor', 'none')
 117         self._params.add_with_value('display', 'none')
 118         self._params.add_with_value('vga', 'none')
 119         self._params.add('enable-kvm')
 120         self._params.add_with_value('pidfile', self._temp.get('pidfile'))
 121         self._params.add_with_value('cpu', 'host')
 122         self._params.add_with_value(
 123             'machine', 'pc,accel=kvm,usb=off,mem-merge=off')
 124         self._params.add_with_value(
 125             'smp', '{smp},sockets=1,cores={smp},threads=1'.format(
 126                 smp=self._opt.get('smp')))
 127         self._params.add_with_value(
 128             'object', 'memory-backend-file,id=mem,size={mem}M,'
 129             'mem-path=/dev/hugepages,share=on'.format(mem=self._opt.get('mem')))
 130         self._params.add_with_value(
 131             'm', '{mem}M'.format(mem=self._opt.get('mem')))
 132         self._params.add_with_value('numa', 'node,memdev=mem')
 133         self._params.add_with_value('balloon', 'none')
 134
 135     def add_nestedvm_params(self):
 136         """Set NestedVM QEMU parameters."""
 137         self._params.add_with_value(
 138             'net', 'nic,macaddr=52:54:00:00:{qemu:02x}:ff'.format(
 139                 qemu=self._opt.get('qemu_id')))
 140         self._params.add_with_value(
 141             'net', 'user,hostfwd=tcp::{info[port]}-:22'.format(
 142                 info=self._vm_info))
 143         # TODO: Remove try except after fully migrated to Bionic or
 144         # qemu_set_node is removed.
 145         try:
 146             locking = ',file.locking=off'\
 147                 if self.qemu_version(version='2.10') else ''
 148         except AttributeError:
 149             locking = ''
 150         self._params.add_with_value(
 151             'drive', 'file={img},format=raw,cache=none,if=virtio{locking}'.
 152             format(img=self._opt.get('img'), locking=locking))
 153         self._params.add_with_value(
 154             'qmp', 'unix:{qmp},server,nowait'.format(qmp=self._temp.get('qmp')))
 155         self._params.add_with_value(
 156             'chardev', 'socket,host=127.0.0.1,port={info[serial]},'
 157             'id=gnc0,server,nowait'.format(info=self._vm_info))
 158         self._params.add_with_value('device', 'isa-serial,chardev=gnc0')
 159         self._params.add_with_value(
 160             'chardev', 'socket,path={qga},server,nowait,id=qga0'.format(
 161                 qga=self._temp.get('qga')))
 162         self._params.add_with_value('device', 'isa-serial,chardev=qga0')
 163
 164     def add_kernelvm_params(self):
 165         """Set KernelVM QEMU parameters."""
 166         self._params.add_with_value(
 167             'chardev', 'file,id=char0,path={log}'.format(
 168                 log=self._temp.get('log')))
 169         self._params.add_with_value('device', 'isa-serial,chardev=char0')
 170         self._params.add_with_value(
 171             'fsdev', 'local,id=root9p,path=/,security_model=none')
 172         self._params.add_with_value(
 173             'device', 'virtio-9p-pci,fsdev=root9p,mount_tag=/dev/root')
 174         self._params.add_with_value(
 175             'kernel', '$(readlink -m {img}* | tail -1)'.format(
 176                 img=self._opt.get('img')))
 177         self._params.add_with_value(
 178             'append', '"ro rootfstype=9p rootflags=trans=virtio console=ttyS0'
 179             ' tsc=reliable hugepages=256 init={init}"'.format(
 180                 init=self._temp.get('ini')))
 181
 182     def create_kernelvm_config_vpp(self, **kwargs):
 183         """Create QEMU VPP config files.
 184
 185         :param kwargs: Key-value pairs to replace content of VPP configuration
 186             file.
 187         :type kwargs: dict
 188         """
 189         startup = ('/etc/vpp/vm_startup_{id}.conf'.
 190                    format(id=self._opt.get('qemu_id')))
 191         running = ('/etc/vpp/vm_running_{id}.exec'.
 192                    format(id=self._opt.get('qemu_id')))
 193
 194         self._temp['startup'] = startup
 195         self._temp['running'] = running
 196         self._opt['vnf_bin'] = ('/usr/bin/vpp -c {startup}'.
 197                                 format(startup=startup))
 198
 199         # Create VPP startup configuration.
 200         vpp_config = VppConfigGenerator()
 201         vpp_config.set_node(self._node)
 202         vpp_config.add_unix_nodaemon()
 203         vpp_config.add_unix_cli_listen()
 204         vpp_config.add_unix_exec(running)
 205         vpp_config.add_cpu_main_core('0')
 206         vpp_config.add_cpu_corelist_workers('1-{smp}'.
 207                                             format(smp=self._opt.get('smp')-1))
 208         vpp_config.add_dpdk_dev('0000:00:06.0', '0000:00:07.0')
 209         vpp_config.add_dpdk_log_level('debug')
 210         if not kwargs['jumbo_frames']:
 211             vpp_config.add_dpdk_no_multi_seg()
 212             vpp_config.add_dpdk_no_tx_checksum_offload()
 213         vpp_config.add_plugin('disable', 'default')
 214         vpp_config.add_plugin('enable', 'dpdk_plugin.so')
 215         vpp_config.write_config(startup)
 216
 217         # Create VPP running configuration.
 218         template = '{res}/{tpl}.exec'.format(res=Constants.RESOURCES_TPL_VM,
 219                                              tpl=self._opt.get('vnf'))
 220         exec_cmd_no_error(self._node, 'rm -f {running}'.format(running=running),
 221                           sudo=True)
 222
 223         with open(template, 'r') as src_file:
 224             src = Template(src_file.read())
 225             exec_cmd_no_error(
 226                 self._node, "echo '{out}' | sudo tee {running}".format(
 227                     out=src.safe_substitute(**kwargs), running=running))
 228
 229     def create_kernelvm_config_testpmd_io(self, **kwargs):
 230         """Create QEMU testpmd-io command line.
 231
 232         :param kwargs: Key-value pairs to construct command line parameters.
 233         :type kwargs: dict
 234         """
 235         testpmd_path = ('{path}/{arch}-native-linuxapp-gcc/app'.
 236                         format(path=Constants.QEMU_VM_DPDK,
 237                                arch=Topology.get_node_arch(self._node)))
 238         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 239             eal_corelist='0-{smp}'.format(smp=self._opt.get('smp') - 1),
 240             eal_driver=False,
 241             eal_in_memory=True,
 242             pmd_num_mbufs=16384,
 243             pmd_rxq=kwargs['queues'],
 244             pmd_txq=kwargs['queues'],
 245             pmd_tx_offloads=False,
 246             pmd_disable_hw_vlan=False,
 247             pmd_max_pkt_len=9200 if kwargs['jumbo_frames'] else None,
 248             pmd_nb_cores=str(self._opt.get('smp') - 1))
 249
 250         self._opt['vnf_bin'] = ('{testpmd_path}/{testpmd_cmd}'.
 251                                 format(testpmd_path=testpmd_path,
 252                                        testpmd_cmd=testpmd_cmd))
 253
 254     def create_kernelvm_config_testpmd_mac(self, **kwargs):
 255         """Create QEMU testpmd-mac command line.
 256
 257         :param kwargs: Key-value pairs to construct command line parameters.
 258         :type kwargs: dict
 259         """
 260         testpmd_path = ('{path}/{arch}-native-linuxapp-gcc/app'.
 261                         format(path=Constants.QEMU_VM_DPDK,
 262                                arch=Topology.get_node_arch(self._node)))
 263         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 264             eal_corelist='0-{smp}'.format(smp=self._opt.get('smp') - 1),
 265             eal_driver=False,
 266             eal_in_memory=True,
 267             pmd_num_mbufs=16384,
 268             pmd_fwd_mode='mac',
 269             pmd_eth_peer_0='0,{mac}'.format(mac=kwargs['vif1_mac']),
 270             pmd_eth_peer_1='1,{mac}'.format(mac=kwargs['vif2_mac']),
 271             pmd_rxq=kwargs['queues'],
 272             pmd_txq=kwargs['queues'],
 273             pmd_tx_offloads=False,
 274             pmd_disable_hw_vlan=False,
 275             pmd_max_pkt_len=9200 if kwargs['jumbo_frames'] else None,
 276             pmd_nb_cores=str(self._opt.get('smp') - 1))
 277
 278         self._opt['vnf_bin'] = ('{testpmd_path}/{testpmd_cmd}'.
 279                                 format(testpmd_path=testpmd_path,
 280                                        testpmd_cmd=testpmd_cmd))
 281
 282     def create_kernelvm_init(self, **kwargs):
 283         """Create QEMU init script.
 284
 285         :param kwargs: Key-value pairs to replace content of init startup file.
 286         :type kwargs: dict
 287         """
 288         template = '{res}/init.sh'.format(res=Constants.RESOURCES_TPL_VM)
 289         init = self._temp.get('ini')
 290         exec_cmd_no_error(
 291             self._node, 'rm -f {init}'.format(init=init), sudo=True)
 292
 293         with open(template, 'r') as src_file:
 294             src = Template(src_file.read())
 295             exec_cmd_no_error(
 296                 self._node, "echo '{out}' | sudo tee {init}".format(
 297                     out=src.safe_substitute(**kwargs), init=init))
 298             exec_cmd_no_error(
 299                 self._node, "chmod +x {init}".format(init=init), sudo=True)
 300
 301     def configure_kernelvm_vnf(self, **kwargs):
 302         """Create KernelVM VNF configurations.
 303
 304         :param kwargs: Key-value pairs for templating configs.
 305         :type kwargs: dict
 306         """
 307         if 'vpp' in self._opt.get('vnf'):
 308             self.create_kernelvm_config_vpp(**kwargs)
 309         elif 'testpmd_io' in self._opt.get('vnf'):
 310             self.create_kernelvm_config_testpmd_io(**kwargs)
 311         elif 'testpmd_mac' in self._opt.get('vnf'):
 312             self.create_kernelvm_config_testpmd_mac(**kwargs)
 313         else:
 314             raise RuntimeError('QEMU: Unsupported VNF!')
 315         self.create_kernelvm_init(vnf_bin=self._opt['vnf_bin'])
 316
 317     def get_qemu_pids(self):
 318         """Get QEMU CPU pids.
 319
 320         :returns: List of QEMU CPU pids.
 321         :rtype: list of str
 322         """
 323         command = ("grep -rwl 'CPU' /proc/$(sudo cat {pidfile})/task/*/comm ".
 324                    format(pidfile=self._temp.get('pidfile')))
 325         command += (r"| xargs dirname | sed -e 's/\/.*\///g' | uniq")
 326
 327         stdout, _ = exec_cmd_no_error(self._node, command)
 328         return stdout.splitlines()
 329
 330     def qemu_set_affinity(self, *host_cpus):
 331         """Set qemu affinity by getting thread PIDs via QMP and taskset to list
 332         of CPU cores. Function tries to execute 3 times to avoid race condition
 333         in getting thread PIDs.
 334
 335         :param host_cpus: List of CPU cores.
 336         :type host_cpus: list
 337         """
 338         for _ in range(3):
 339             try:
 340                 qemu_cpus = self.get_qemu_pids()
 341
 342                 if len(qemu_cpus) != len(host_cpus):
 343                     sleep(1)
 344                     continue
 345                 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
 346                     command = ('taskset -pc {host_cpu} {thread}'.
 347                                format(host_cpu=host_cpu, thread=qemu_cpu))
 348                     message = ('QEMU: Set affinity failed on {host}!'.
 349                                format(host=self._node['host']))
 350                     exec_cmd_no_error(self._node, command, sudo=True,
 351                                       message=message)
 352                 break
 353             except (RuntimeError, ValueError):
 354                 self.qemu_kill_all()
 355                 raise
 356         else:
 357             self.qemu_kill_all()
 358             raise RuntimeError('Failed to set Qemu threads affinity!')
 359
 360     def qemu_set_scheduler_policy(self):
 361         """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
 362         processes.
 363
 364         :raises RuntimeError: Set scheduler policy failed.
 365         """
 366         try:
 367             qemu_cpus = self.get_qemu_pids()
 368
 369             for qemu_cpu in qemu_cpus:
 370                 command = ('chrt -r -p 1 {thread}'.
 371                            format(thread=qemu_cpu))
 372                 message = ('QEMU: Set SCHED_RR failed on {host}'.
 373                            format(host=self._node['host']))
 374                 exec_cmd_no_error(self._node, command, sudo=True,
 375                                   message=message)
 376         except (RuntimeError, ValueError):
 377             self.qemu_kill_all()
 378             raise
 379
 380     def qemu_add_vhost_user_if(self, socket, server=True, jumbo_frames=False,
 381                                queue_size=None, queues=1):
 382         """Add Vhost-user interface.
 383
 384         :param socket: Path of the unix socket.
 385         :param server: If True the socket shall be a listening socket.
 386         :param jumbo_frames: Set True if jumbo frames are used in the test.
 387         :param queue_size: Vring queue size.
 388         :param queues: Number of queues.
 389         :type socket: str
 390         :type server: bool
 391         :type jumbo_frames: bool
 392         :type queue_size: int
 393         :type queues: int
 394         """
 395         self._vhost_id += 1
 396         self._params.add_with_value(
 397             'chardev', 'socket,id=char{vhost},path={socket}{server}'.format(
 398                 vhost=self._vhost_id, socket=socket,
 399                 server=',server' if server is True else ''))
 400         self._params.add_with_value(
 401             'netdev', 'vhost-user,id=vhost{vhost},chardev=char{vhost},'
 402             'queues={queues}'.format(vhost=self._vhost_id, queues=queues))
 403         mac = ('52:54:00:00:{qemu:02x}:{vhost:02x}'.
 404                format(qemu=self._opt.get('qemu_id'), vhost=self._vhost_id))
 405         queue_size = ('rx_queue_size={queue_size},tx_queue_size={queue_size}'.
 406                       format(queue_size=queue_size)) if queue_size else ''
 407         mbuf = 'on,host_mtu=9200'
 408         self._params.add_with_value(
 409             'device', 'virtio-net-pci,netdev=vhost{vhost},mac={mac},bus=pci.0,'
 410             'addr={addr}.0,mq=on,vectors={vectors},csum=off,gso=off,'
 411             'guest_tso4=off,guest_tso6=off,guest_ecn=off,mrg_rxbuf={mbuf},'
 412             '{queue_size}'.format(
 413                 addr=self._vhost_id+5, vhost=self._vhost_id, mac=mac,
 414                 mbuf=mbuf if jumbo_frames else 'off', queue_size=queue_size,
 415                 vectors=(2 * queues + 2)))
 416
 417         # Add interface MAC and socket to the node dict.
 418         if_data = {'mac_address': mac, 'socket': socket}
 419         if_name = 'vhost{vhost}'.format(vhost=self._vhost_id)
 420         self._vm_info['interfaces'][if_name] = if_data
 421         # Add socket to temporary file list.
 422         self._temp[if_name] = socket
 423
 424     def _qemu_qmp_exec(self, cmd):
 425         """Execute QMP command.
 426
 427         QMP is JSON based protocol which allows to control QEMU instance.
 428
 429         :param cmd: QMP command to execute.
 430         :type cmd: str
 431         :returns: Command output in python representation of JSON format. The
 432             { "return": {} } response is QMP's success response. An error
 433             response will contain the "error" keyword instead of "return".
 434         """
 435         # To enter command mode, the qmp_capabilities command must be issued.
 436         command = ('echo "{{ \\"execute\\": \\"qmp_capabilities\\" }}'
 437                    '{{ \\"execute\\": \\"{cmd}\\" }}" | '
 438                    'sudo -S socat - UNIX-CONNECT:{qmp}'.
 439                    format(cmd=cmd, qmp=self._temp.get('qmp')))
 440         message = ('QMP execute "{cmd}" failed on {host}'.
 441                    format(cmd=cmd, host=self._node['host']))
 442         stdout, _ = exec_cmd_no_error(
 443             self._node, command, sudo=False, message=message)
 444
 445         # Skip capabilities negotiation messages.
 446         out_list = stdout.splitlines()
 447         if len(out_list) < 3:
 448             raise RuntimeError(
 449                 'Invalid QMP output on {host}'.format(host=self._node['host']))
 450         return json.loads(out_list[2])
 451
 452     def _qemu_qga_flush(self):
 453         """Flush the QGA parser state."""
 454         command = ('(printf "\xFF"; sleep 1) | '
 455                    'sudo -S socat - UNIX-CONNECT:{qga}'.
 456                    format(qga=self._temp.get('qga')))
 457         message = ('QGA flush failed on {host}'.format(host=self._node['host']))
 458         stdout, _ = exec_cmd_no_error(
 459             self._node, command, sudo=False, message=message)
 460
 461         return json.loads(stdout.split('\n', 1)[0]) if stdout else dict()
 462
 463     def _qemu_qga_exec(self, cmd):
 464         """Execute QGA command.
 465
 466         QGA provide access to a system-level agent via standard QMP commands.
 467
 468         :param cmd: QGA command to execute.
 469         :type cmd: str
 470         """
 471         command = ('(echo "{{ \\"execute\\": \\"{cmd}\\" }}"; sleep 1) | '
 472                    'sudo -S socat - UNIX-CONNECT:{qga}'.
 473                    format(cmd=cmd, qga=self._temp.get('qga')))
 474         message = ('QGA execute "{cmd}" failed on {host}'.
 475                    format(cmd=cmd, host=self._node['host']))
 476         stdout, _ = exec_cmd_no_error(
 477             self._node, command, sudo=False, message=message)
 478
 479         return json.loads(stdout.split('\n', 1)[0]) if stdout else dict()
 480
 481     def _wait_until_vm_boot(self):
 482         """Wait until QEMU with NestedVM is booted."""
 483         if self._opt.get('vm_type') == 'nestedvm':
 484             self._wait_until_nestedvm_boot()
 485             self._update_vm_interfaces()
 486         elif self._opt.get('vm_type') == 'kernelvm':
 487             self._wait_until_kernelvm_boot()
 488         else:
 489             raise RuntimeError('QEMU: Unsupported VM type!')
 490
 491     def _wait_until_nestedvm_boot(self, retries=12):
 492         """Wait until QEMU with NestedVM is booted.
 493
 494         First try to flush qga until there is output.
 495         Then ping QEMU guest agent each 5s until VM booted or timeout.
 496
 497         :param retries: Number of retries with 5s between trials.
 498         :type retries: int
 499         """
 500         for _ in range(retries):
 501             out = None
 502             try:
 503                 out = self._qemu_qga_flush()
 504             except ValueError:
 505                 logger.trace('QGA qga flush unexpected output {out}'.
 506                              format(out=out))
 507             # Empty output - VM not booted yet
 508             if not out:
 509                 sleep(5)
 510             else:
 511                 break
 512         else:
 513             raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
 514                                format(host=self._node['host']))
 515         for _ in range(retries):
 516             out = None
 517             try:
 518                 out = self._qemu_qga_exec('guest-ping')
 519             except ValueError:
 520                 logger.trace('QGA guest-ping unexpected output {out}'.
 521                              format(out=out))
 522             # Empty output - VM not booted yet.
 523             if not out:
 524                 sleep(5)
 525             # Non-error return - VM booted.
 526             elif out.get('return') is not None:
 527                 break
 528             # Skip error and wait.
 529             elif out.get('error') is not None:
 530                 sleep(5)
 531             else:
 532                 # If there is an unexpected output from QGA guest-info, try
 533                 # again until timeout.
 534                 logger.trace('QGA guest-ping unexpected output {out}'.
 535                              format(out=out))
 536         else:
 537             raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
 538                                format(host=self._node['host']))
 539
 540     def _wait_until_kernelvm_boot(self, retries=60):
 541         """Wait until QEMU KernelVM is booted.
 542
 543         :param retries: Number of retries.
 544         :type retries: int
 545         """
 546         vpp_ver = VPPUtil.vpp_show_version(self._node)
 547
 548         for _ in range(retries):
 549             command = ('tail -1 {log}'.format(log=self._temp.get('log')))
 550             stdout = None
 551             try:
 552                 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 553                 sleep(1)
 554             except RuntimeError:
 555                 pass
 556             if vpp_ver in stdout or 'Press enter to exit' in stdout:
 557                 break
 558             if 'reboot: Power down' in stdout:
 559                 raise RuntimeError('QEMU: NF failed to run on {host}!'.
 560                                    format(host=self._node['host']))
 561         else:
 562             raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
 563                                format(host=self._node['host']))
 564
 565     def _update_vm_interfaces(self):
 566         """Update interface names in VM node dict."""
 567         # Send guest-network-get-interfaces command via QGA, output example:
 568         # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
 569         # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
 570         out = self._qemu_qga_exec('guest-network-get-interfaces')
 571         interfaces = out.get('return')
 572         mac_name = {}
 573         if not interfaces:
 574             raise RuntimeError('Get VM interface list failed on {host}'.
 575                                format(host=self._node['host']))
 576         # Create MAC-name dict.
 577         for interface in interfaces:
 578             if 'hardware-address' not in interface:
 579                 continue
 580             mac_name[interface['hardware-address']] = interface['name']
 581         # Match interface by MAC and save interface name.
 582         for interface in self._vm_info['interfaces'].values():
 583             mac = interface.get('mac_address')
 584             if_name = mac_name.get(mac)
 585             if if_name is None:
 586                 logger.trace(
 587                     'Interface name for MAC {mac} not found'.format(mac=mac))
 588             else:
 589                 interface['name'] = if_name
 590
 591     def qemu_start(self):
 592         """Start QEMU and wait until VM boot.
 593
 594         :returns: VM node info.
 595         :rtype: dict
 596         """
 597         cmd_opts = OptionString()
 598         cmd_opts.add('{bin_path}/qemu-system-{arch}'.format(
 599             bin_path=Constants.QEMU_BIN_PATH,
 600             arch=Topology.get_node_arch(self._node)))
 601         cmd_opts.extend(self._params)
 602         message = ('QEMU: Start failed on {host}!'.
 603                    format(host=self._node['host']))
 604         try:
 605             DUTSetup.check_huge_page(
 606                 self._node, '/dev/hugepages', self._opt.get('mem'))
 607
 608             exec_cmd_no_error(
 609                 self._node, cmd_opts, timeout=300, sudo=True, message=message)
 610             self._wait_until_vm_boot()
 611         except RuntimeError:
 612             self.qemu_kill_all()
 613             raise
 614         return self._vm_info
 615
 616     def qemu_kill(self):
 617         """Kill qemu process."""
 618         exec_cmd(self._node, 'chmod +r {pidfile}'.
 619                  format(pidfile=self._temp.get('pidfile')), sudo=True)
 620         exec_cmd(self._node, 'kill -SIGKILL $(cat {pidfile})'.
 621                  format(pidfile=self._temp.get('pidfile')), sudo=True)
 622
 623         for value in self._temp.values():
 624             exec_cmd(self._node, 'cat {value}'.format(value=value), sudo=True)
 625             exec_cmd(self._node, 'rm -f {value}'.format(value=value), sudo=True)
 626
 627     def qemu_kill_all(self):
 628         """Kill all qemu processes on DUT node if specified."""
 629         exec_cmd(self._node, 'pkill -SIGKILL qemu', sudo=True)
 630
 631         for value in self._temp.values():
 632             exec_cmd(self._node, 'cat {value}'.format(value=value), sudo=True)
 633             exec_cmd(self._node, 'rm -f {value}'.format(value=value), sudo=True)
 634
 635     def qemu_version(self, version=None):
 636         """Return Qemu version or compare if version is higher than parameter.
 637
 638         :param version: Version to compare.
 639         :type version: str
 640         :returns: Qemu version or Boolean if version is higher than parameter.
 641         :rtype: str or bool
 642         """
 643         command = ('{bin_path}/qemu-system-{arch} --version'.format(
 644             bin_path=Constants.QEMU_BIN_PATH,
 645             arch=Topology.get_node_arch(self._node)))
 646         try:
 647             stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 648             ver = match(r'QEMU emulator version ([\d.]*)', stdout).group(1)
 649             return StrictVersion(ver) > StrictVersion(version) \
 650                 if version else ver
 651         except RuntimeError:
 652             self.qemu_kill_all()
 653             raise