resources/libraries/python/QemuUtils.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """QEMU utilities library."""
  15
  16 import json
  17 from re import match
  18 from string import Template
  19 from time import sleep
  20
  21 from robot.api import logger
  22 from resources.libraries.python.Constants import Constants
  23 from resources.libraries.python.DpdkUtil import DpdkUtil
  24 from resources.libraries.python.DUTSetup import DUTSetup
  25 from resources.libraries.python.OptionString import OptionString
  26 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
  27 from resources.libraries.python.VPPUtil import VPPUtil
  28 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
  29 from resources.libraries.python.topology import NodeType, Topology
  30
  31 __all__ = ["QemuUtils"]
  32
  33
  34 class QemuUtils(object):
  35     """QEMU utilities."""
  36
  37     # Use one instance of class per tests.
  38     ROBOT_LIBRARY_SCOPE = 'TEST CASE'
  39
  40     def __init__(self, node, qemu_id=1, smp=1, mem=512, vnf=None,
  41                  img=Constants.QEMU_VM_IMAGE):
  42         """Initialize QemuUtil class.
  43
  44         :param node: Node to run QEMU on.
  45         :param qemu_id: QEMU identifier.
  46         :param smp: Number of virtual SMP units (cores).
  47         :param mem: Amount of memory.
  48         :param vnf: Network function workload.
  49         :param img: QEMU disk image or kernel image path.
  50         :type node: dict
  51         :type qemu_id: int
  52         :type smp: int
  53         :type mem: int
  54         :type vnf: str
  55         :type img: str
  56         """
  57         self._vhost_id = 0
  58         self._node = node
  59         self._arch = Topology.get_node_arch(self._node)
  60         dpdk_target = 'arm64-armv8a' if self._arch == 'aarch64' \
  61             else 'x86_64-native'
  62         self._testpmd_path = '{path}/{dpdk_target}-linuxapp-gcc/app'\
  63             .format(path=Constants.QEMU_VM_DPDK, dpdk_target=dpdk_target)
  64         self._vm_info = {
  65             'host': node['host'],
  66             'type': NodeType.VM,
  67             'port': 10021 + qemu_id,
  68             'serial': 4555 + qemu_id,
  69             'username': 'cisco',
  70             'password': 'cisco',
  71             'interfaces': {},
  72         }
  73         if node['port'] != 22:
  74             self._vm_info['host_port'] = node['port']
  75             self._vm_info['host_username'] = node['username']
  76             self._vm_info['host_password'] = node['password']
  77         # Input Options.
  78         self._opt = dict()
  79         self._opt['qemu_id'] = qemu_id
  80         self._opt['mem'] = int(mem)
  81         self._opt['smp'] = int(smp)
  82         self._opt['img'] = img
  83         self._opt['vnf'] = vnf
  84         # Temporary files.
  85         self._temp = dict()
  86         self._temp['pidfile'] = '/var/run/qemu_{id}.pid'.format(id=qemu_id)
  87         if img == Constants.QEMU_VM_IMAGE:
  88             self._opt['vm_type'] = 'nestedvm'
  89             self._temp['qmp'] = '/var/run/qmp_{id}.sock'.format(id=qemu_id)
  90             self._temp['qga'] = '/var/run/qga_{id}.sock'.format(id=qemu_id)
  91         elif img == Constants.QEMU_VM_KERNEL:
  92             self._opt['img'], _ = exec_cmd_no_error(
  93                 node,
  94                 'ls -1 {img}* | tail -1'.format(img=Constants.QEMU_VM_KERNEL),
  95                 message='Qemu Kernel VM image not found!')
  96             self._opt['vm_type'] = 'kernelvm'
  97             self._temp['log'] = '/tmp/serial_{id}.log'.format(id=qemu_id)
  98             self._temp['ini'] = '/etc/vm_init_{id}.conf'.format(id=qemu_id)
  99             self._opt['initrd'], _ = exec_cmd_no_error(
 100                 node,
 101                 'ls -1 {initrd}* | tail -1'.format(
 102                     initrd=Constants.QEMU_VM_KERNEL_INITRD),
 103                 message='Qemu Kernel initrd image not found!')
 104         else:
 105             raise RuntimeError('QEMU: Unknown VM image option: {}'.format(img))
 106         # Computed parameters for QEMU command line.
 107         self._params = OptionString(prefix='-')
 108         self.add_params()
 109
 110     def add_params(self):
 111         """Set QEMU command line parameters."""
 112         self.add_default_params()
 113         if self._opt.get('vm_type', '') == 'nestedvm':
 114             self.add_nestedvm_params()
 115         elif self._opt.get('vm_type', '') == 'kernelvm':
 116             self.add_kernelvm_params()
 117         else:
 118             raise RuntimeError('QEMU: Unsupported VM type!')
 119
 120     def add_default_params(self):
 121         """Set default QEMU command line parameters."""
 122         self._params.add('daemonize')
 123         self._params.add('nodefaults')
 124         self._params.add_with_value('name', 'vnf{qemu},debug-threads=on'.format(
 125             qemu=self._opt.get('qemu_id')))
 126         self._params.add('no-user-config')
 127         self._params.add_with_value('monitor', 'none')
 128         self._params.add_with_value('display', 'none')
 129         self._params.add_with_value('vga', 'none')
 130         self._params.add('enable-kvm')
 131         self._params.add_with_value('pidfile', self._temp.get('pidfile'))
 132         self._params.add_with_value('cpu', 'host')
 133
 134         if self._arch == 'aarch64':
 135             machine_args = 'virt,accel=kvm,usb=off,mem-merge=off,gic-version=3'
 136         else:
 137             machine_args = 'pc,accel=kvm,usb=off,mem-merge=off'
 138         self._params.add_with_value(
 139             'machine', machine_args)
 140         self._params.add_with_value(
 141             'smp', '{smp},sockets=1,cores={smp},threads=1'.format(
 142                 smp=self._opt.get('smp')))
 143         self._params.add_with_value(
 144             'object', 'memory-backend-file,id=mem,size={mem}M,'
 145             'mem-path=/dev/hugepages,share=on'.format(mem=self._opt.get('mem')))
 146         self._params.add_with_value(
 147             'm', '{mem}M'.format(mem=self._opt.get('mem')))
 148         self._params.add_with_value('numa', 'node,memdev=mem')
 149         self._params.add_with_value('balloon', 'none')
 150
 151     def add_nestedvm_params(self):
 152         """Set NestedVM QEMU parameters."""
 153         self._params.add_with_value(
 154             'net', 'nic,macaddr=52:54:00:00:{qemu:02x}:ff'.format(
 155                 qemu=self._opt.get('qemu_id')))
 156         self._params.add_with_value(
 157             'net', 'user,hostfwd=tcp::{info[port]}-:22'.format(
 158                 info=self._vm_info))
 159         locking = ',file.locking=off'
 160         self._params.add_with_value(
 161             'drive', 'file={img},format=raw,cache=none,if=virtio{locking}'.
 162             format(img=self._opt.get('img'), locking=locking))
 163         self._params.add_with_value(
 164             'qmp', 'unix:{qmp},server,nowait'.format(qmp=self._temp.get('qmp')))
 165         self._params.add_with_value(
 166             'chardev', 'socket,host=127.0.0.1,port={info[serial]},'
 167             'id=gnc0,server,nowait'.format(info=self._vm_info))
 168         self._params.add_with_value('device', 'isa-serial,chardev=gnc0')
 169         self._params.add_with_value(
 170             'chardev', 'socket,path={qga},server,nowait,id=qga0'.format(
 171                 qga=self._temp.get('qga')))
 172         self._params.add_with_value('device', 'isa-serial,chardev=qga0')
 173
 174     def add_kernelvm_params(self):
 175         """Set KernelVM QEMU parameters."""
 176         console = 'ttyAMA0' if self._arch == 'aarch64' else 'ttyS0'
 177         self._params.add_with_value('serial', 'file:{log}'.format(
 178             log=self._temp.get('log')))
 179         self._params.add_with_value(
 180             'fsdev', 'local,id=root9p,path=/,security_model=none')
 181         self._params.add_with_value(
 182             'device', 'virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot')
 183         self._params.add_with_value(
 184             'kernel', '{img}'.format(img=self._opt.get('img')))
 185         self._params.add_with_value(
 186             'initrd', '{initrd}'.format(initrd=self._opt.get('initrd')))
 187         self._params.add_with_value(
 188             'append', '"ro rootfstype=9p rootflags=trans=virtio '
 189                       'root=virtioroot console={console} tsc=reliable '
 190                       'hugepages=256 init={init} fastboot"'.format(
 191                           console=console, init=self._temp.get('ini')))
 192
 193     def create_kernelvm_config_vpp(self, **kwargs):
 194         """Create QEMU VPP config files.
 195
 196         :param kwargs: Key-value pairs to replace content of VPP configuration
 197             file.
 198         :type kwargs: dict
 199         """
 200         startup = ('/etc/vpp/vm_startup_{id}.conf'.
 201                    format(id=self._opt.get('qemu_id')))
 202         running = ('/etc/vpp/vm_running_{id}.exec'.
 203                    format(id=self._opt.get('qemu_id')))
 204
 205         self._temp['startup'] = startup
 206         self._temp['running'] = running
 207         self._opt['vnf_bin'] = ('/usr/bin/vpp -c {startup}'.
 208                                 format(startup=startup))
 209
 210         # Create VPP startup configuration.
 211         vpp_config = VppConfigGenerator()
 212         vpp_config.set_node(self._node)
 213         vpp_config.add_unix_nodaemon()
 214         vpp_config.add_unix_cli_listen()
 215         vpp_config.add_unix_exec(running)
 216         vpp_config.add_socksvr()
 217         vpp_config.add_cpu_main_core('0')
 218         if self._opt.get('smp') > 1:
 219             vpp_config.add_cpu_corelist_workers('1-{smp}'.format(
 220                 smp=self._opt.get('smp')-1))
 221         vpp_config.add_dpdk_dev('0000:00:06.0', '0000:00:07.0')
 222         vpp_config.add_dpdk_dev_default_rxq(kwargs['queues'])
 223         vpp_config.add_dpdk_log_level('debug')
 224         if not kwargs['jumbo_frames']:
 225             vpp_config.add_dpdk_no_multi_seg()
 226             vpp_config.add_dpdk_no_tx_checksum_offload()
 227         vpp_config.add_plugin('disable', 'default')
 228         vpp_config.add_plugin('enable', 'dpdk_plugin.so')
 229         vpp_config.write_config(startup)
 230
 231         # Create VPP running configuration.
 232         template = '{res}/{tpl}.exec'.format(res=Constants.RESOURCES_TPL_VM,
 233                                              tpl=self._opt.get('vnf'))
 234         exec_cmd_no_error(self._node, 'rm -f {running}'.format(running=running),
 235                           sudo=True)
 236
 237         with open(template, 'r') as src_file:
 238             src = Template(src_file.read())
 239             exec_cmd_no_error(
 240                 self._node, "echo '{out}' | sudo tee {running}".format(
 241                     out=src.safe_substitute(**kwargs), running=running))
 242
 243     def create_kernelvm_config_testpmd_io(self, **kwargs):
 244         """Create QEMU testpmd-io command line.
 245
 246         :param kwargs: Key-value pairs to construct command line parameters.
 247         :type kwargs: dict
 248         """
 249         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 250             eal_corelist='0-{smp}'.format(smp=self._opt.get('smp') - 1),
 251             eal_driver=False,
 252             eal_in_memory=True,
 253             pmd_num_mbufs=16384,
 254             pmd_rxq=kwargs['queues'],
 255             pmd_txq=kwargs['queues'],
 256             pmd_tx_offloads='0x0',
 257             pmd_disable_hw_vlan=False,
 258             pmd_nb_cores=str(self._opt.get('smp') - 1))
 259
 260         self._opt['vnf_bin'] = ('{testpmd_path}/{testpmd_cmd}'.
 261                                 format(testpmd_path=self._testpmd_path,
 262                                        testpmd_cmd=testpmd_cmd))
 263
 264     def create_kernelvm_config_testpmd_mac(self, **kwargs):
 265         """Create QEMU testpmd-mac command line.
 266
 267         :param kwargs: Key-value pairs to construct command line parameters.
 268         :type kwargs: dict
 269         """
 270         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
 271             eal_corelist='0-{smp}'.format(smp=self._opt.get('smp') - 1),
 272             eal_driver=False,
 273             eal_in_memory=True,
 274             pmd_num_mbufs=16384,
 275             pmd_fwd_mode='mac',
 276             pmd_eth_peer_0='0,{mac}'.format(mac=kwargs['vif1_mac']),
 277             pmd_eth_peer_1='1,{mac}'.format(mac=kwargs['vif2_mac']),
 278             pmd_rxq=kwargs['queues'],
 279             pmd_txq=kwargs['queues'],
 280             pmd_tx_offloads='0x0',
 281             pmd_disable_hw_vlan=False,
 282             pmd_nb_cores=str(self._opt.get('smp') - 1))
 283
 284         self._opt['vnf_bin'] = ('{testpmd_path}/{testpmd_cmd}'.
 285                                 format(testpmd_path=self._testpmd_path,
 286                                        testpmd_cmd=testpmd_cmd))
 287
 288     def create_kernelvm_init(self, **kwargs):
 289         """Create QEMU init script.
 290
 291         :param kwargs: Key-value pairs to replace content of init startup file.
 292         :type kwargs: dict
 293         """
 294         template = '{res}/init.sh'.format(res=Constants.RESOURCES_TPL_VM)
 295         init = self._temp.get('ini')
 296         exec_cmd_no_error(
 297             self._node, 'rm -f {init}'.format(init=init), sudo=True)
 298
 299         with open(template, 'r') as src_file:
 300             src = Template(src_file.read())
 301             exec_cmd_no_error(
 302                 self._node, "echo '{out}' | sudo tee {init}".format(
 303                     out=src.safe_substitute(**kwargs), init=init))
 304             exec_cmd_no_error(
 305                 self._node, "chmod +x {init}".format(init=init), sudo=True)
 306
 307     def configure_kernelvm_vnf(self, **kwargs):
 308         """Create KernelVM VNF configurations.
 309
 310         :param kwargs: Key-value pairs for templating configs.
 311         :type kwargs: dict
 312         """
 313         if 'vpp' in self._opt.get('vnf'):
 314             self.create_kernelvm_config_vpp(**kwargs)
 315         elif 'testpmd_io' in self._opt.get('vnf'):
 316             self.create_kernelvm_config_testpmd_io(**kwargs)
 317         elif 'testpmd_mac' in self._opt.get('vnf'):
 318             self.create_kernelvm_config_testpmd_mac(**kwargs)
 319         else:
 320             raise RuntimeError('QEMU: Unsupported VNF!')
 321         self.create_kernelvm_init(vnf_bin=self._opt['vnf_bin'])
 322
 323     def get_qemu_pids(self):
 324         """Get QEMU CPU pids.
 325
 326         :returns: List of QEMU CPU pids.
 327         :rtype: list of str
 328         """
 329         command = ("grep -rwl 'CPU' /proc/$(sudo cat {pidfile})/task/*/comm ".
 330                    format(pidfile=self._temp.get('pidfile')))
 331         command += (r"| xargs dirname | sed -e 's/\/.*\///g' | uniq")
 332
 333         stdout, _ = exec_cmd_no_error(self._node, command)
 334         return stdout.splitlines()
 335
 336     def qemu_set_affinity(self, *host_cpus):
 337         """Set qemu affinity by getting thread PIDs via QMP and taskset to list
 338         of CPU cores. Function tries to execute 3 times to avoid race condition
 339         in getting thread PIDs.
 340
 341         :param host_cpus: List of CPU cores.
 342         :type host_cpus: list
 343         """
 344         for _ in range(3):
 345             try:
 346                 qemu_cpus = self.get_qemu_pids()
 347
 348                 if len(qemu_cpus) != len(host_cpus):
 349                     sleep(1)
 350                     continue
 351                 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
 352                     command = ('taskset -pc {host_cpu} {thread}'.
 353                                format(host_cpu=host_cpu, thread=qemu_cpu))
 354                     message = ('QEMU: Set affinity failed on {host}!'.
 355                                format(host=self._node['host']))
 356                     exec_cmd_no_error(self._node, command, sudo=True,
 357                                       message=message)
 358                 break
 359             except (RuntimeError, ValueError):
 360                 self.qemu_kill_all()
 361                 raise
 362         else:
 363             self.qemu_kill_all()
 364             raise RuntimeError('Failed to set Qemu threads affinity!')
 365
 366     def qemu_set_scheduler_policy(self):
 367         """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
 368         processes.
 369
 370         :raises RuntimeError: Set scheduler policy failed.
 371         """
 372         try:
 373             qemu_cpus = self.get_qemu_pids()
 374
 375             for qemu_cpu in qemu_cpus:
 376                 command = ('chrt -r -p 1 {thread}'.
 377                            format(thread=qemu_cpu))
 378                 message = ('QEMU: Set SCHED_RR failed on {host}'.
 379                            format(host=self._node['host']))
 380                 exec_cmd_no_error(self._node, command, sudo=True,
 381                                   message=message)
 382         except (RuntimeError, ValueError):
 383             self.qemu_kill_all()
 384             raise
 385
 386     def qemu_add_vhost_user_if(self, socket, server=True, jumbo_frames=False,
 387                                queue_size=None, queues=1):
 388         """Add Vhost-user interface.
 389
 390         :param socket: Path of the unix socket.
 391         :param server: If True the socket shall be a listening socket.
 392         :param jumbo_frames: Set True if jumbo frames are used in the test.
 393         :param queue_size: Vring queue size.
 394         :param queues: Number of queues.
 395         :type socket: str
 396         :type server: bool
 397         :type jumbo_frames: bool
 398         :type queue_size: int
 399         :type queues: int
 400         """
 401         self._vhost_id += 1
 402         self._params.add_with_value(
 403             'chardev', 'socket,id=char{vhost},path={socket}{server}'.format(
 404                 vhost=self._vhost_id, socket=socket,
 405                 server=',server' if server is True else ''))
 406         self._params.add_with_value(
 407             'netdev', 'vhost-user,id=vhost{vhost},chardev=char{vhost},'
 408             'queues={queues}'.format(vhost=self._vhost_id, queues=queues))
 409         mac = ('52:54:00:00:{qemu:02x}:{vhost:02x}'.
 410                format(qemu=self._opt.get('qemu_id'), vhost=self._vhost_id))
 411         queue_size = ('rx_queue_size={queue_size},tx_queue_size={queue_size}'.
 412                       format(queue_size=queue_size)) if queue_size else ''
 413         mbuf = 'on,host_mtu=9200'
 414         self._params.add_with_value(
 415             'device', 'virtio-net-pci,netdev=vhost{vhost},mac={mac},'
 416             'addr={addr}.0,mq=on,vectors={vectors},csum=off,gso=off,'
 417             'guest_tso4=off,guest_tso6=off,guest_ecn=off,mrg_rxbuf={mbuf},'
 418             '{queue_size}'.format(
 419                 addr=self._vhost_id+5, vhost=self._vhost_id, mac=mac,
 420                 mbuf=mbuf if jumbo_frames else 'off', queue_size=queue_size,
 421                 vectors=(2 * queues + 2)))
 422
 423         # Add interface MAC and socket to the node dict.
 424         if_data = {'mac_address': mac, 'socket': socket}
 425         if_name = 'vhost{vhost}'.format(vhost=self._vhost_id)
 426         self._vm_info['interfaces'][if_name] = if_data
 427         # Add socket to temporary file list.
 428         self._temp[if_name] = socket
 429
 430     def _qemu_qmp_exec(self, cmd):
 431         """Execute QMP command.
 432
 433         QMP is JSON based protocol which allows to control QEMU instance.
 434
 435         :param cmd: QMP command to execute.
 436         :type cmd: str
 437         :returns: Command output in python representation of JSON format. The
 438             { "return": {} } response is QMP's success response. An error
 439             response will contain the "error" keyword instead of "return".
 440         """
 441         # To enter command mode, the qmp_capabilities command must be issued.
 442         command = ('echo "{{ \\"execute\\": \\"qmp_capabilities\\" }}'
 443                    '{{ \\"execute\\": \\"{cmd}\\" }}" | '
 444                    'sudo -S socat - UNIX-CONNECT:{qmp}'.
 445                    format(cmd=cmd, qmp=self._temp.get('qmp')))
 446         message = ('QMP execute "{cmd}" failed on {host}'.
 447                    format(cmd=cmd, host=self._node['host']))
 448         stdout, _ = exec_cmd_no_error(
 449             self._node, command, sudo=False, message=message)
 450
 451         # Skip capabilities negotiation messages.
 452         out_list = stdout.splitlines()
 453         if len(out_list) < 3:
 454             raise RuntimeError(
 455                 'Invalid QMP output on {host}'.format(host=self._node['host']))
 456         return json.loads(out_list[2])
 457
 458     def _qemu_qga_flush(self):
 459         """Flush the QGA parser state."""
 460         command = ('(printf "\xFF"; sleep 1) | '
 461                    'sudo -S socat - UNIX-CONNECT:{qga}'.
 462                    format(qga=self._temp.get('qga')))
 463         message = ('QGA flush failed on {host}'.format(host=self._node['host']))
 464         stdout, _ = exec_cmd_no_error(
 465             self._node, command, sudo=False, message=message)
 466
 467         return json.loads(stdout.split('\n', 1)[0]) if stdout else dict()
 468
 469     def _qemu_qga_exec(self, cmd):
 470         """Execute QGA command.
 471
 472         QGA provide access to a system-level agent via standard QMP commands.
 473
 474         :param cmd: QGA command to execute.
 475         :type cmd: str
 476         """
 477         command = ('(echo "{{ \\"execute\\": \\"{cmd}\\" }}"; sleep 1) | '
 478                    'sudo -S socat - UNIX-CONNECT:{qga}'.
 479                    format(cmd=cmd, qga=self._temp.get('qga')))
 480         message = ('QGA execute "{cmd}" failed on {host}'.
 481                    format(cmd=cmd, host=self._node['host']))
 482         stdout, _ = exec_cmd_no_error(
 483             self._node, command, sudo=False, message=message)
 484
 485         return json.loads(stdout.split('\n', 1)[0]) if stdout else dict()
 486
 487     def _wait_until_vm_boot(self):
 488         """Wait until QEMU with NestedVM is booted."""
 489         if self._opt.get('vm_type') == 'nestedvm':
 490             self._wait_until_nestedvm_boot()
 491             self._update_vm_interfaces()
 492         elif self._opt.get('vm_type') == 'kernelvm':
 493             self._wait_until_kernelvm_boot()
 494         else:
 495             raise RuntimeError('QEMU: Unsupported VM type!')
 496
 497     def _wait_until_nestedvm_boot(self, retries=12):
 498         """Wait until QEMU with NestedVM is booted.
 499
 500         First try to flush qga until there is output.
 501         Then ping QEMU guest agent each 5s until VM booted or timeout.
 502
 503         :param retries: Number of retries with 5s between trials.
 504         :type retries: int
 505         """
 506         for _ in range(retries):
 507             out = None
 508             try:
 509                 out = self._qemu_qga_flush()
 510             except ValueError:
 511                 logger.trace('QGA qga flush unexpected output {out}'.
 512                              format(out=out))
 513             # Empty output - VM not booted yet
 514             if not out:
 515                 sleep(5)
 516             else:
 517                 break
 518         else:
 519             raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
 520                                format(host=self._node['host']))
 521         for _ in range(retries):
 522             out = None
 523             try:
 524                 out = self._qemu_qga_exec('guest-ping')
 525             except ValueError:
 526                 logger.trace('QGA guest-ping unexpected output {out}'.
 527                              format(out=out))
 528             # Empty output - VM not booted yet.
 529             if not out:
 530                 sleep(5)
 531             # Non-error return - VM booted.
 532             elif out.get('return') is not None:
 533                 break
 534             # Skip error and wait.
 535             elif out.get('error') is not None:
 536                 sleep(5)
 537             else:
 538                 # If there is an unexpected output from QGA guest-info, try
 539                 # again until timeout.
 540                 logger.trace('QGA guest-ping unexpected output {out}'.
 541                              format(out=out))
 542         else:
 543             raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
 544                                format(host=self._node['host']))
 545
 546     def _wait_until_kernelvm_boot(self, retries=60):
 547         """Wait until QEMU KernelVM is booted.
 548
 549         :param retries: Number of retries.
 550         :type retries: int
 551         """
 552         vpp_ver = VPPUtil.vpp_show_version(self._node)
 553
 554         for _ in range(retries):
 555             command = ('tail -1 {log}'.format(log=self._temp.get('log')))
 556             stdout = None
 557             try:
 558                 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 559                 sleep(1)
 560             except RuntimeError:
 561                 pass
 562             if vpp_ver in stdout or 'Press enter to exit' in stdout:
 563                 break
 564             if 'reboot: Power down' in stdout:
 565                 raise RuntimeError('QEMU: NF failed to run on {host}!'.
 566                                    format(host=self._node['host']))
 567         else:
 568             raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
 569                                format(host=self._node['host']))
 570
 571     def _update_vm_interfaces(self):
 572         """Update interface names in VM node dict."""
 573         # Send guest-network-get-interfaces command via QGA, output example:
 574         # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
 575         # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
 576         out = self._qemu_qga_exec('guest-network-get-interfaces')
 577         interfaces = out.get('return')
 578         mac_name = {}
 579         if not interfaces:
 580             raise RuntimeError('Get VM interface list failed on {host}'.
 581                                format(host=self._node['host']))
 582         # Create MAC-name dict.
 583         for interface in interfaces:
 584             if 'hardware-address' not in interface:
 585                 continue
 586             mac_name[interface['hardware-address']] = interface['name']
 587         # Match interface by MAC and save interface name.
 588         for interface in self._vm_info['interfaces'].values():
 589             mac = interface.get('mac_address')
 590             if_name = mac_name.get(mac)
 591             if if_name is None:
 592                 logger.trace(
 593                     'Interface name for MAC {mac} not found'.format(mac=mac))
 594             else:
 595                 interface['name'] = if_name
 596
 597     def qemu_start(self):
 598         """Start QEMU and wait until VM boot.
 599
 600         :returns: VM node info.
 601         :rtype: dict
 602         """
 603         cmd_opts = OptionString()
 604         cmd_opts.add('{bin_path}/qemu-system-{arch}'.format(
 605             bin_path=Constants.QEMU_BIN_PATH, arch=self._arch))
 606         cmd_opts.extend(self._params)
 607         message = ('QEMU: Start failed on {host}!'.
 608                    format(host=self._node['host']))
 609         try:
 610             DUTSetup.check_huge_page(
 611                 self._node, '/dev/hugepages', self._opt.get('mem'))
 612
 613             exec_cmd_no_error(
 614                 self._node, cmd_opts, timeout=300, sudo=True, message=message)
 615             self._wait_until_vm_boot()
 616         except RuntimeError:
 617             self.qemu_kill_all()
 618             raise
 619         return self._vm_info
 620
 621     def qemu_kill(self):
 622         """Kill qemu process."""
 623         exec_cmd(self._node, 'chmod +r {pidfile}'.
 624                  format(pidfile=self._temp.get('pidfile')), sudo=True)
 625         exec_cmd(self._node, 'kill -SIGKILL $(cat {pidfile})'.
 626                  format(pidfile=self._temp.get('pidfile')), sudo=True)
 627
 628         for value in self._temp.values():
 629             exec_cmd(self._node, 'cat {value}'.format(value=value), sudo=True)
 630             exec_cmd(self._node, 'rm -f {value}'.format(value=value), sudo=True)
 631
 632     def qemu_kill_all(self):
 633         """Kill all qemu processes on DUT node if specified."""
 634         exec_cmd(self._node, 'pkill -SIGKILL qemu', sudo=True)
 635
 636         for value in self._temp.values():
 637             exec_cmd(self._node, 'cat {value}'.format(value=value), sudo=True)
 638             exec_cmd(self._node, 'rm -f {value}'.format(value=value), sudo=True)
 639
 640     def qemu_version(self):
 641         """Return Qemu version.
 642
 643         :returns: Qemu version.
 644         :rtype: str
 645         """
 646         command = ('{bin_path}/qemu-system-{arch} --version'.format(
 647             bin_path=Constants.QEMU_BIN_PATH,
 648             arch=self._arch))
 649         try:
 650             stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
 651             return match(r'QEMU emulator version ([\d.]*)', stdout).group(1)
 652         except RuntimeError:
 653             self.qemu_kill_all()
 654             raise