resources/libraries/python/QemuUtils.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """QEMU utilities library."""
  15
  16 # Disable due to pylint bug
  17 # pylint: disable=no-name-in-module,import-error
  18 from distutils.version import StrictVersion
  19 import json
  20 from re import match
  21 from string import Template
  22 from time import sleep
  23
  24 from robot.api import logger
  25 from resources.libraries.python.Constants import Constants
  26 from resources.libraries.python.DpdkUtil import DpdkUtil
  27 from resources.libraries.python.DUTSetup import DUTSetup
  28 from resources.libraries.python.OptionString import OptionString
  29 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
  30 from resources.libraries.python.VPPUtil import VPPUtil
  31 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
  32 from resources.libraries.python.topology import NodeType, Topology
  33
  34 __all__ = ["QemuUtils"]
  35
  36
  37 class QemuUtils(object):
  38     """QEMU utilities."""
  39
  40     # Use one instance of class per tests.
  41     ROBOT_LIBRARY_SCOPE = 'TEST CASE'
  42
  43     def __init__(self, node, qemu_id=1, smp=1, mem=512, vnf=None,
  44                  img=Constants.QEMU_VM_IMAGE):
  45         """Initialize QemuUtil class.
  46
  47         :param node: Node to run QEMU on.
  48         :param qemu_id: QEMU identifier.
  49         :param smp: Number of virtual SMP units (cores).
  50         :param mem: Amount of memory.
  51         :param vnf: Network function workload.
  52         :param img: QEMU disk image or kernel image path.
  53         :type node: dict
  54         :type qemu_id: int
  55         :type smp: int
  56         :type mem: int
  57         :type vnf: str
  58         :type img: str
  59         """
  60         self._vhost_id = 0
  61         self._node = node
  62         self._arch = Topology.get_node_arch(self._node)
  63         dpdk_target = 'arm64-armv8a' if self._arch == 'aarch64' \
  64             else 'x86_64-native'
  65         self._testpmd_path = '{path}/{dpdk_target}-linuxapp-gcc/app'\
  66             .format(path=Constants.QEMU_VM_DPDK, dpdk_target=dpdk_target)
  67         self._vm_info = {
  68             'host': node['host'],
  69             'type': NodeType.VM,
  70             'port': 10021 + qemu_id,
  71             'serial': 4555 + qemu_id,
  72             'username': 'cisco',
  73             'password': 'cisco',
  74             'interfaces': {},
  75         }
  76         if node['port'] != 22:
  77             self._vm_info['host_port'] = node['port']
  78             self._vm_info['host_username'] = node['username']
  79             self._vm_info['host_password'] = node['password']
  80         # Input Options.
  81         self._opt = dict()
  82         self._opt['qemu_id'] = qemu_id
  83         self._opt['mem'] = int(mem)
  84         self._opt['smp'] = int(smp)
  85         self._opt['img'] = img
  86         self._opt['vnf'] = vnf
  87         # Temporary files.
  88         self._temp = dict()
  89         self._temp['pidfile'] = '/var/run/qemu_{id}.pid'.format(id=qemu_id)
  90         if img == Constants.QEMU_VM_IMAGE:
  91             self._opt['vm_type'] = 'nestedvm'
  92             self._temp['qmp'] = '/var/run/qmp_{id}.sock'.format(id=qemu_id)
  93             self._temp['qga'] = '/var/run/qga_{id}.sock'.format(id=qemu_id)
  94         elif img == Constants.QEMU_VM_KERNEL:
  95             self._opt['img'], _ = exec_cmd_no_error(
  96                 node,
  97                 'ls -1 {img}* | tail -1'.format(img=Constants.QEMU_VM_KERNEL),
  98                 message='Qemu Kernel VM image not found!')
  99             self._opt['vm_type'] = 'kernelvm'
 100             self._temp['log'] = '/tmp/serial_{id}.log'.format(id=qemu_id)
 101             self._temp['ini'] = '/etc/vm_init_{id}.conf'.format(id=qemu_id)
 102             self._opt['initrd'], _ = exec_cmd_no_error(
 103                 node,
 104                 'ls -1 {initrd}* | tail -1'.format(
 105                     initrd=Constants.QEMU_VM_KERNEL_INITRD),
 106                 message='Qemu Kernel initrd image not found!')
 107         else:
 108             raise RuntimeError('QEMU: Unknown VM image option: {}'.format(img))
 109         # Computed parameters for QEMU command line.
 110         self._params = OptionString(prefix='-')
 111         self.add_params()
 112
 113     def add_params(self):
 114         """Set QEMU command line parameters."""
 115         self.add_default_params()
 116         if self._opt.get('vm_type', '') == 'nestedvm':
 117             self.add_nestedvm_params()
 118         elif self._opt.get('vm_type', '') == 'kernelvm':
 119             self.add_kernelvm_params()
 120         else:
 121             raise RuntimeError('QEMU: Unsupported VM type!')
 122
 123     def add_default_params(self):
 124         """Set default QEMU command line parameters."""
 125         self._params.add('daemonize')
 126         self._params.add('nodefaults')
 127         self._params.add_with_value('name', 'vnf{qemu},debug-threads=on'.format(
 128             qemu=self._opt.get('qemu_id')))
 129         self._params.add('no-user-config')
 130         self._params.add_with_value('monitor', 'none')
 131         self._params.add_with_value('display', 'none')
 132         self._params.add_with_value('vga', 'none')
 133         self._params.add('enable-kvm')
 134         self._params.add_with_value('pidfile', self._temp.get('pidfile'))
 135         self._params.add_with_value('cpu', 'host')
 136
 137         if self._arch == 'aarch64':
 138             machine_args = 'virt,accel=kvm,usb=off,mem-merge=off,gic-version=3'
 139         else:
 140             machine_args = 'pc,accel=kvm,usb=off,mem-merge=off'
 141         self._params.add_with_value(
 142             'machine', machine_args)
 143         self._params.add_with_value(
 144             'smp', '{smp},sockets=1,cores={smp},threads=1'.format(
 145                 smp=self._opt.get('smp')))
 146         self._params.add_with_value(
 147             'object', 'memory-backend-file,id=mem,size={mem}M,'
 148             'mem-path=/dev/hugepages,share=on'.format(mem=self._opt.get('mem')))
 149         self._params.add_with_value(
 150             'm', '{mem}M'.format(mem=self._opt.get('mem')))
 151         self._params.add_with_value('numa', 'node,memdev=mem')
 152         self._params.add_with_value('balloon', 'none')
 153
 154     def add_nestedvm_params(self):
 155         """Set NestedVM QEMU parameters."""
 156         self._params.add_with_value(
 157             'net', 'nic,macaddr=52:54:00:00:{qemu:02x}:ff'.format(
 158                 qemu=self._opt.get('qemu_id')))
 159         self._params.add_with_value(
 160             'net', 'user,hostfwd=tcp::{info[port]}-:22'.format(
 161                 info=self._vm_info))
 162         # TODO: Remove try except after fully migrated to Bionic or
 163         # qemu_set_node is removed.
 164         try:
 165             locking = ',file.locking=off'\
 166                 if self.qemu_version(version='2.10') else ''
 167         except AttributeError:
 168             locking = ''
 169         self._params.add_with_value(
 170             'drive', 'file={img},format=raw,cache=none,if=virtio{locking}'.
 171             format(img=self._opt.get('img'), locking=locking))
 172         self._params.add_with_value(
 173             'qmp', 'unix:{qmp},server,nowait'.format(qmp=self._temp.get('qmp')))
 174         self._params.add_with_value(
 175             'chardev', 'socket,host=127.0.0.1,port={info[serial]},'
 176             'id=gnc0,server,nowait'.format(info=self._vm_info))
 177         self._params.add_with_value('device', 'isa-serial,chardev=gnc0')
 178         self._params.add_with_value(
 179             'chardev', 'socket,path={qga},server,nowait,id=qga0'.format(
 180                 qga=self._temp.get('qga')))
 181         self._params.add_with_value('device', 'isa-serial,chardev=qga0')
 182
 183     def add_kernelvm_params(self):
 184         """Set KernelVM QEMU parameters."""
 185         console = 'ttyAMA0' if self._arch == 'aarch64' else 'ttyS0'
 186         self._params.add_with_value('serial', 'file:{log}'.format(
 187             log=self._temp.get('log')))
 188         self._params.add_with_value(
 189             'fsdev', 'local,id=root9p,path=/,security_model=none')
 190         self._params.add_with_value(
 191             'device', 'virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot')
 192         self._params.add_with_value(
 193             'kernel', '{img}'.format(img=self._opt.get('img')))
 194         self._params.add_with_value(
 195             'initrd', '{initrd}'.format(initrd=self._opt.get('initrd')))
 196         self._params.add_with_value(
 197             'append', '"ro rootfstype=9p rootflags=trans=virtio '
 198                       'root=virtioroot console={console} tsc=reliable '
 199                       'hugepages=256 init={init}"'.format(
 200                           console=console, init=self._temp.get('ini')))
 201
 202     def create_kernelvm_config_vpp(self, **kwargs):
 203         """Create QEMU VPP config files.
 204
 205         :param kwargs: Key-value pairs to replace content of VPP configuration
 206             file.
 207         :type kwargs: dict
 208         """
 209         startup = ('/etc/vpp/vm_startup_{id}.conf'.
 210                    format(id=self._opt.get('qemu_id')))
 211         running = ('/etc/vpp/vm_running_{id}.exec'.
 212                    format(id=self._opt.get('qemu_id')))
 213
 214         self._temp['startup'] = startup
 215         self._temp['running'] = running
 216         self._opt['vnf_bin'] = ('/usr/bin/vpp -c {startup}'.
 217                                 format(startup=startup))
 218
 219         # Create VPP startup configuration.
 220         vpp_config = VppConfigGenerator()
 221         vpp_config.set_node(self._node)
 222         vpp_config.add_unix_nodaemon()
 223         vpp_config.add_unix_cli_listen()
 224         vpp_config.add_unix_exec(running)
 225         vpp_config.add_socksvr()
 226         vpp_config.add_cpu_main_core('0')
 227         if self._opt.get('smp') > 1:
 228             vpp_config.add_cpu_corelist_workers('1-{smp}'.format(
 229                 smp=self._opt.get('smp')-1))
 230         vpp_config.add_dpdk_dev('0000:00:06.0', '0000:00:07.0')
 231         vpp_config.add_dpdk_dev_default_rxq(kwargs['queues'])
 232         vpp_config.add_dpdk_log_level('debug')
 233         if not kwargs['jumbo_frames']:
 234             vpp_config.add_dpdk_no_multi_seg()
 235             vpp_config.add_dpdk_no_tx_checksum_offload()
 236         vpp_config.add_plugin('disable', 'default')
 237         vpp_config.add_plugin('enable', 'dpdk_plugin.so')
 238         vpp_config.write_config(startup)
 239
 240         # Create VPP running configuration.
 241         template = '{res}/{tpl}.exec'.format(res=Constants.RESOURCES_TPL_VM,
 242                                              tpl=self._opt.get('vnf'))
 243         exec_cmd_no_error(self._node, 'rm -f {running}'.format(running=running),
 244                           sudo=True)
 245
 246         with open(template, 'r') as src_file:
 247             src = Template(src_file.read())
 248             exec_cmd_no_error(
 249                 self._node, "echo '{out}' | sudo tee {running}".format(
 250                     out=src.safe_substitute(**kwargs), running=running))
 251
 252     def create_kernelvm_config_testpmd_io(self, **kwargs):
 253         """Create QEMU testpmd-io command line.
 254
 255         :param kwargs: Key-value pairs to construct command line parameters.
 256         :type kwargs: dict
 257         """
 258         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 259             eal_corelist='0-{smp}'.format(smp=self._opt.get('smp') - 1),
 260             eal_driver=False,
 261             eal_in_memory=True,
 262             pmd_num_mbufs=16384,
 263             pmd_rxq=kwargs['queues'],
 264             pmd_txq=kwargs['queues'],
 265             pmd_tx_offloads=False,
 266             pmd_disable_hw_vlan=False,
 267             pmd_max_pkt_len=9200 if kwargs['jumbo_frames'] else None,
 268             pmd_nb_cores=str(self._opt.get('smp') - 1))
 269
 270         self._opt['vnf_bin'] = ('{testpmd_path}/{testpmd_cmd}'.
 271                                 format(testpmd_path=self._testpmd_path,
 272                                        testpmd_cmd=testpmd_cmd))
 273
 274     def create_kernelvm_config_testpmd_mac(self, **kwargs):
 275         """Create QEMU testpmd-mac command line.
 276
 277         :param kwargs: Key-value pairs to construct command line parameters.
 278         :type kwargs: dict
 279         """
 280         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 281             eal_corelist='0-{smp}'.format(smp=self._opt.get('smp') - 1),
 282             eal_driver=False,
 283             eal_in_memory=True,
 284             pmd_num_mbufs=16384,
 285             pmd_fwd_mode='mac',
 286             pmd_eth_peer_0='0,{mac}'.format(mac=kwargs['vif1_mac']),
 287             pmd_eth_peer_1='1,{mac}'.format(mac=kwargs['vif2_mac']),
 288             pmd_rxq=kwargs['queues'],
 289             pmd_txq=kwargs['queues'],
 290             pmd_tx_offloads=False,
 291             pmd_disable_hw_vlan=False,
 292             pmd_max_pkt_len=9200 if kwargs['jumbo_frames'] else None,
 293             pmd_nb_cores=str(self._opt.get('smp') - 1))
 294
 295         self._opt['vnf_bin'] = ('{testpmd_path}/{testpmd_cmd}'.
 296                                 format(testpmd_path=self._testpmd_path,
 297                                        testpmd_cmd=testpmd_cmd))
 298
 299     def create_kernelvm_init(self, **kwargs):
 300         """Create QEMU init script.
 301
 302         :param kwargs: Key-value pairs to replace content of init startup file.
 303         :type kwargs: dict
 304         """
 305         template = '{res}/init.sh'.format(res=Constants.RESOURCES_TPL_VM)
 306         init = self._temp.get('ini')
 307         exec_cmd_no_error(
 308             self._node, 'rm -f {init}'.format(init=init), sudo=True)
 309
 310         with open(template, 'r') as src_file:
 311             src = Template(src_file.read())
 312             exec_cmd_no_error(
 313                 self._node, "echo '{out}' | sudo tee {init}".format(
 314                     out=src.safe_substitute(**kwargs), init=init))
 315             exec_cmd_no_error(
 316                 self._node, "chmod +x {init}".format(init=init), sudo=True)
 317
 318     def configure_kernelvm_vnf(self, **kwargs):
 319         """Create KernelVM VNF configurations.
 320
 321         :param kwargs: Key-value pairs for templating configs.
 322         :type kwargs: dict
 323         """
 324         if 'vpp' in self._opt.get('vnf'):
 325             self.create_kernelvm_config_vpp(**kwargs)
 326         elif 'testpmd_io' in self._opt.get('vnf'):
 327             self.create_kernelvm_config_testpmd_io(**kwargs)
 328         elif 'testpmd_mac' in self._opt.get('vnf'):
 329             self.create_kernelvm_config_testpmd_mac(**kwargs)
 330         else:
 331             raise RuntimeError('QEMU: Unsupported VNF!')
 332         self.create_kernelvm_init(vnf_bin=self._opt['vnf_bin'])
 333
 334     def get_qemu_pids(self):
 335         """Get QEMU CPU pids.
 336
 337         :returns: List of QEMU CPU pids.
 338         :rtype: list of str
 339         """
 340         command = ("grep -rwl 'CPU' /proc/$(sudo cat {pidfile})/task/*/comm ".
 341                    format(pidfile=self._temp.get('pidfile')))
 342         command += (r"| xargs dirname | sed -e 's/\/.*\///g' | uniq")
 343
 344         stdout, _ = exec_cmd_no_error(self._node, command)
 345         return stdout.splitlines()
 346
 347     def qemu_set_affinity(self, *host_cpus):
 348         """Set qemu affinity by getting thread PIDs via QMP and taskset to list
 349         of CPU cores. Function tries to execute 3 times to avoid race condition
 350         in getting thread PIDs.
 351
 352         :param host_cpus: List of CPU cores.
 353         :type host_cpus: list
 354         """
 355         for _ in range(3):
 356             try:
 357                 qemu_cpus = self.get_qemu_pids()
 358
 359                 if len(qemu_cpus) != len(host_cpus):
 360                     sleep(1)
 361                     continue
 362                 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
 363                     command = ('taskset -pc {host_cpu} {thread}'.
 364                                format(host_cpu=host_cpu, thread=qemu_cpu))
 365                     message = ('QEMU: Set affinity failed on {host}!'.
 366                                format(host=self._node['host']))
 367                     exec_cmd_no_error(self._node, command, sudo=True,
 368                                       message=message)
 369                 break
 370             except (RuntimeError, ValueError):
 371                 self.qemu_kill_all()
 372                 raise
 373         else:
 374             self.qemu_kill_all()
 375             raise RuntimeError('Failed to set Qemu threads affinity!')
 376
 377     def qemu_set_scheduler_policy(self):
 378         """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
 379         processes.
 380
 381         :raises RuntimeError: Set scheduler policy failed.
 382         """
 383         try:
 384             qemu_cpus = self.get_qemu_pids()
 385
 386             for qemu_cpu in qemu_cpus:
 387                 command = ('chrt -r -p 1 {thread}'.
 388                            format(thread=qemu_cpu))
 389                 message = ('QEMU: Set SCHED_RR failed on {host}'.
 390                            format(host=self._node['host']))
 391                 exec_cmd_no_error(self._node, command, sudo=True,
 392                                   message=message)
 393         except (RuntimeError, ValueError):
 394             self.qemu_kill_all()
 395             raise
 396
 397     def qemu_add_vhost_user_if(self, socket, server=True, jumbo_frames=False,
 398                                queue_size=None, queues=1):
 399         """Add Vhost-user interface.
 400
 401         :param socket: Path of the unix socket.
 402         :param server: If True the socket shall be a listening socket.
 403         :param jumbo_frames: Set True if jumbo frames are used in the test.
 404         :param queue_size: Vring queue size.
 405         :param queues: Number of queues.
 406         :type socket: str
 407         :type server: bool
 408         :type jumbo_frames: bool
 409         :type queue_size: int
 410         :type queues: int
 411         """
 412         self._vhost_id += 1
 413         self._params.add_with_value(
 414             'chardev', 'socket,id=char{vhost},path={socket}{server}'.format(
 415                 vhost=self._vhost_id, socket=socket,
 416                 server=',server' if server is True else ''))
 417         self._params.add_with_value(
 418             'netdev', 'vhost-user,id=vhost{vhost},chardev=char{vhost},'
 419             'queues={queues}'.format(vhost=self._vhost_id, queues=queues))
 420         mac = ('52:54:00:00:{qemu:02x}:{vhost:02x}'.
 421                format(qemu=self._opt.get('qemu_id'), vhost=self._vhost_id))
 422         queue_size = ('rx_queue_size={queue_size},tx_queue_size={queue_size}'.
 423                       format(queue_size=queue_size)) if queue_size else ''
 424         mbuf = 'on,host_mtu=9200'
 425         self._params.add_with_value(
 426             'device', 'virtio-net-pci,netdev=vhost{vhost},mac={mac},'
 427             'addr={addr}.0,mq=on,vectors={vectors},csum=off,gso=off,'
 428             'guest_tso4=off,guest_tso6=off,guest_ecn=off,mrg_rxbuf={mbuf},'
 429             '{queue_size}'.format(
 430                 addr=self._vhost_id+5, vhost=self._vhost_id, mac=mac,
 431                 mbuf=mbuf if jumbo_frames else 'off', queue_size=queue_size,
 432                 vectors=(2 * queues + 2)))
 433
 434         # Add interface MAC and socket to the node dict.
 435         if_data = {'mac_address': mac, 'socket': socket}
 436         if_name = 'vhost{vhost}'.format(vhost=self._vhost_id)
 437         self._vm_info['interfaces'][if_name] = if_data
 438         # Add socket to temporary file list.
 439         self._temp[if_name] = socket
 440
 441     def _qemu_qmp_exec(self, cmd):
 442         """Execute QMP command.
 443
 444         QMP is JSON based protocol which allows to control QEMU instance.
 445
 446         :param cmd: QMP command to execute.
 447         :type cmd: str
 448         :returns: Command output in python representation of JSON format. The
 449             { "return": {} } response is QMP's success response. An error
 450             response will contain the "error" keyword instead of "return".
 451         """
 452         # To enter command mode, the qmp_capabilities command must be issued.
 453         command = ('echo "{{ \\"execute\\": \\"qmp_capabilities\\" }}'
 454                    '{{ \\"execute\\": \\"{cmd}\\" }}" | '
 455                    'sudo -S socat - UNIX-CONNECT:{qmp}'.
 456                    format(cmd=cmd, qmp=self._temp.get('qmp')))
 457         message = ('QMP execute "{cmd}" failed on {host}'.
 458                    format(cmd=cmd, host=self._node['host']))
 459         stdout, _ = exec_cmd_no_error(
 460             self._node, command, sudo=False, message=message)
 461
 462         # Skip capabilities negotiation messages.
 463         out_list = stdout.splitlines()
 464         if len(out_list) < 3:
 465             raise RuntimeError(
 466                 'Invalid QMP output on {host}'.format(host=self._node['host']))
 467         return json.loads(out_list[2])
 468
 469     def _qemu_qga_flush(self):
 470         """Flush the QGA parser state."""
 471         command = ('(printf "\xFF"; sleep 1) | '
 472                    'sudo -S socat - UNIX-CONNECT:{qga}'.
 473                    format(qga=self._temp.get('qga')))
 474         message = ('QGA flush failed on {host}'.format(host=self._node['host']))
 475         stdout, _ = exec_cmd_no_error(
 476             self._node, command, sudo=False, message=message)
 477
 478         return json.loads(stdout.split('\n', 1)[0]) if stdout else dict()
 479
 480     def _qemu_qga_exec(self, cmd):
 481         """Execute QGA command.
 482
 483         QGA provide access to a system-level agent via standard QMP commands.
 484
 485         :param cmd: QGA command to execute.
 486         :type cmd: str
 487         """
 488         command = ('(echo "{{ \\"execute\\": \\"{cmd}\\" }}"; sleep 1) | '
 489                    'sudo -S socat - UNIX-CONNECT:{qga}'.
 490                    format(cmd=cmd, qga=self._temp.get('qga')))
 491         message = ('QGA execute "{cmd}" failed on {host}'.
 492                    format(cmd=cmd, host=self._node['host']))
 493         stdout, _ = exec_cmd_no_error(
 494             self._node, command, sudo=False, message=message)
 495
 496         return json.loads(stdout.split('\n', 1)[0]) if stdout else dict()
 497
 498     def _wait_until_vm_boot(self):
 499         """Wait until QEMU with NestedVM is booted."""
 500         if self._opt.get('vm_type') == 'nestedvm':
 501             self._wait_until_nestedvm_boot()
 502             self._update_vm_interfaces()
 503         elif self._opt.get('vm_type') == 'kernelvm':
 504             self._wait_until_kernelvm_boot()
 505         else:
 506             raise RuntimeError('QEMU: Unsupported VM type!')
 507
 508     def _wait_until_nestedvm_boot(self, retries=12):
 509         """Wait until QEMU with NestedVM is booted.
 510
 511         First try to flush qga until there is output.
 512         Then ping QEMU guest agent each 5s until VM booted or timeout.
 513
 514         :param retries: Number of retries with 5s between trials.
 515         :type retries: int
 516         """
 517         for _ in range(retries):
 518             out = None
 519             try:
 520                 out = self._qemu_qga_flush()
 521             except ValueError:
 522                 logger.trace('QGA qga flush unexpected output {out}'.
 523                              format(out=out))
 524             # Empty output - VM not booted yet
 525             if not out:
 526                 sleep(5)
 527             else:
 528                 break
 529         else:
 530             raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
 531                                format(host=self._node['host']))
 532         for _ in range(retries):
 533             out = None
 534             try:
 535                 out = self._qemu_qga_exec('guest-ping')
 536             except ValueError:
 537                 logger.trace('QGA guest-ping unexpected output {out}'.
 538                              format(out=out))
 539             # Empty output - VM not booted yet.
 540             if not out:
 541                 sleep(5)
 542             # Non-error return - VM booted.
 543             elif out.get('return') is not None:
 544                 break
 545             # Skip error and wait.
 546             elif out.get('error') is not None:
 547                 sleep(5)
 548             else:
 549                 # If there is an unexpected output from QGA guest-info, try
 550                 # again until timeout.
 551                 logger.trace('QGA guest-ping unexpected output {out}'.
 552                              format(out=out))
 553         else:
 554             raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
 555                                format(host=self._node['host']))
 556
 557     def _wait_until_kernelvm_boot(self, retries=60):
 558         """Wait until QEMU KernelVM is booted.
 559
 560         :param retries: Number of retries.
 561         :type retries: int
 562         """
 563         vpp_ver = VPPUtil.vpp_show_version(self._node)
 564
 565         for _ in range(retries):
 566             command = ('tail -1 {log}'.format(log=self._temp.get('log')))
 567             stdout = None
 568             try:
 569                 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 570                 sleep(1)
 571             except RuntimeError:
 572                 pass
 573             if vpp_ver in stdout or 'Press enter to exit' in stdout:
 574                 break
 575             if 'reboot: Power down' in stdout:
 576                 raise RuntimeError('QEMU: NF failed to run on {host}!'.
 577                                    format(host=self._node['host']))
 578         else:
 579             raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
 580                                format(host=self._node['host']))
 581
 582     def _update_vm_interfaces(self):
 583         """Update interface names in VM node dict."""
 584         # Send guest-network-get-interfaces command via QGA, output example:
 585         # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
 586         # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
 587         out = self._qemu_qga_exec('guest-network-get-interfaces')
 588         interfaces = out.get('return')
 589         mac_name = {}
 590         if not interfaces:
 591             raise RuntimeError('Get VM interface list failed on {host}'.
 592                                format(host=self._node['host']))
 593         # Create MAC-name dict.
 594         for interface in interfaces:
 595             if 'hardware-address' not in interface:
 596                 continue
 597             mac_name[interface['hardware-address']] = interface['name']
 598         # Match interface by MAC and save interface name.
 599         for interface in self._vm_info['interfaces'].values():
 600             mac = interface.get('mac_address')
 601             if_name = mac_name.get(mac)
 602             if if_name is None:
 603                 logger.trace(
 604                     'Interface name for MAC {mac} not found'.format(mac=mac))
 605             else:
 606                 interface['name'] = if_name
 607
 608     def qemu_start(self):
 609         """Start QEMU and wait until VM boot.
 610
 611         :returns: VM node info.
 612         :rtype: dict
 613         """
 614         cmd_opts = OptionString()
 615         cmd_opts.add('{bin_path}/qemu-system-{arch}'.format(
 616             bin_path=Constants.QEMU_BIN_PATH, arch=self._arch))
 617         cmd_opts.extend(self._params)
 618         message = ('QEMU: Start failed on {host}!'.
 619                    format(host=self._node['host']))
 620         try:
 621             DUTSetup.check_huge_page(
 622                 self._node, '/dev/hugepages', self._opt.get('mem'))
 623
 624             exec_cmd_no_error(
 625                 self._node, cmd_opts, timeout=300, sudo=True, message=message)
 626             self._wait_until_vm_boot()
 627         except RuntimeError:
 628             self.qemu_kill_all()
 629             raise
 630         return self._vm_info
 631
 632     def qemu_kill(self):
 633         """Kill qemu process."""
 634         exec_cmd(self._node, 'chmod +r {pidfile}'.
 635                  format(pidfile=self._temp.get('pidfile')), sudo=True)
 636         exec_cmd(self._node, 'kill -SIGKILL $(cat {pidfile})'.
 637                  format(pidfile=self._temp.get('pidfile')), sudo=True)
 638
 639         for value in self._temp.values():
 640             exec_cmd(self._node, 'cat {value}'.format(value=value), sudo=True)
 641             exec_cmd(self._node, 'rm -f {value}'.format(value=value), sudo=True)
 642
 643     def qemu_kill_all(self):
 644         """Kill all qemu processes on DUT node if specified."""
 645         exec_cmd(self._node, 'pkill -SIGKILL qemu', sudo=True)
 646
 647         for value in self._temp.values():
 648             exec_cmd(self._node, 'cat {value}'.format(value=value), sudo=True)
 649             exec_cmd(self._node, 'rm -f {value}'.format(value=value), sudo=True)
 650
 651     def qemu_version(self, version=None):
 652         """Return Qemu version or compare if version is higher than parameter.
 653
 654         :param version: Version to compare.
 655         :type version: str
 656         :returns: Qemu version or Boolean if version is higher than parameter.
 657         :rtype: str or bool
 658         """
 659         command = ('{bin_path}/qemu-system-{arch} --version'.format(
 660             bin_path=Constants.QEMU_BIN_PATH,
 661             arch=self._arch))
 662         try:
 663             stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 664             ver = match(r'QEMU emulator version ([\d.]*)', stdout).group(1)
 665             return StrictVersion(ver) > StrictVersion(version) \
 666                 if version else ver
 667         except RuntimeError:
 668             self.qemu_kill_all()
 669             raise