1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """QEMU utilities library."""
16 from time import sleep
17 from string import Template
20 # Disable due to pylint bug
21 # pylint: disable=no-name-in-module,import-error
22 from distutils.version import StrictVersion
24 from robot.api import logger
25 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
26 from resources.libraries.python.Constants import Constants
27 from resources.libraries.python.DUTSetup import DUTSetup
28 from resources.libraries.python.topology import NodeType, Topology
29 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
30 from resources.libraries.python.VPPUtil import VPPUtil
32 __all__ = ["QemuOptions", "QemuUtils"]
35 class QemuOptions(object):
38 The class can handle input parameters that acts as QEMU command line
39 parameters. The only variable is a list of dictionaries where dictionaries
40 can be added multiple times. This emulates the QEMU behavior where one
41 command line parameter can be used multiple times (1..N). Example can be
42 device or object (so it is not an issue to have one memory
43 block of 2G and and second memory block of 512M but from other numa).
45 Class does support get value or string representation that will return
46 space separated, dash prefixed string of key value pairs used for command
50 # Use one instance of class per tests.
51 ROBOT_LIBRARY_SCOPE = 'TEST CASE'
54 self.variables = list()
56 def add(self, variable, value):
57 """Add parameter to the list.
59 :param variable: QEMU parameter name (without dash).
60 :param value: Paired value.
62 :type value: str or int
64 self.variables.append({str(variable): value})
67 """Return space separated string of key value pairs.
69 The format is suitable to be pasted to qemu command line.
71 :returns: Space separated string of key value pairs.
74 return " ".join(["-{k} {v}".format(k=d.keys()[0], v=d.values()[0])
75 for d in self.variables])
78 class QemuUtils(object):
81 # Use one instance of class per tests.
82 ROBOT_LIBRARY_SCOPE = 'TEST CASE'
84 def __init__(self, node=None, qemu_id=1, smp=1, mem=512, vnf=None,
85 img='/var/lib/vm/vhost-nested.img', bin_path='/usr/bin'):
86 """Initialize QemuUtil class.
88 :param node: Node to run QEMU on.
89 :param qemu_id: QEMU identifier.
90 :param smp: Number of virtual SMP units (cores).
91 :param mem: Amount of memory.
92 :param vnf: Network function workload.
93 :param img: QEMU disk image or kernel image path.
94 :param bin_path: QEMU binary path.
106 'port': 10021 + qemu_id,
107 'serial': 4555 + qemu_id,
113 self.qemu_set_node(node)
116 self._opt['qemu_id'] = qemu_id
117 self._opt['bin_path'] = bin_path
118 self._opt['mem'] = int(mem)
119 self._opt['smp'] = int(smp)
120 self._opt['img'] = img
121 self._opt['vnf'] = vnf
124 self._temp['pidfile'] = '/var/run/qemu_{id}.pid'.format(id=qemu_id)
125 if '/var/lib/vm/' in img:
126 self._opt['vm_type'] = 'nestedvm'
127 self._temp['qmp'] = '/var/run/qmp_{id}.sock'.format(id=qemu_id)
128 self._temp['qga'] = '/var/run/qga_{id}.sock'.format(id=qemu_id)
129 elif '/opt/boot/vmlinuz' in img:
130 self._opt['vm_type'] = 'kernelvm'
131 self._temp['log'] = '/tmp/serial_{id}.log'.format(id=qemu_id)
132 self._temp['ini'] = '/etc/vm_init_{id}.conf'.format(id=qemu_id)
134 raise RuntimeError('QEMU: Unknown VM image option!')
135 # Computed parameters for QEMU command line.
136 self._params = QemuOptions()
139 def add_params(self):
140 """Set QEMU command line parameters."""
141 self.add_default_params()
142 if self._opt.get('vm_type', '') == 'nestedvm':
143 self.add_nestedvm_params()
144 elif self._opt.get('vm_type', '') == 'kernelvm':
145 self.add_kernelvm_params()
147 raise RuntimeError('QEMU: Unsupported VM type!')
149 def add_default_params(self):
150 """Set default QEMU command line parameters."""
151 self._params.add('daemonize', '')
152 self._params.add('nodefaults', '')
153 self._params.add('name', 'vnf{qemu},debug-threads=on'.
154 format(qemu=self._opt.get('qemu_id')))
155 self._params.add('no-user-config', '')
156 self._params.add('monitor', 'none')
157 self._params.add('display', 'none')
158 self._params.add('vga', 'none')
159 self._params.add('enable-kvm', '')
160 self._params.add('pidfile', '{pidfile}'.
161 format(pidfile=self._temp.get('pidfile')))
162 self._params.add('cpu', 'host')
163 self._params.add('machine', 'pc,accel=kvm,usb=off,mem-merge=off')
164 self._params.add('smp', '{smp},sockets=1,cores={smp},threads=1'.
165 format(smp=self._opt.get('smp')))
166 self._params.add('object',
167 'memory-backend-file,id=mem,size={mem}M,'
168 'mem-path=/dev/hugepages,share=on'.
169 format(mem=self._opt.get('mem')))
170 self._params.add('m', '{mem}M'.
171 format(mem=self._opt.get('mem')))
172 self._params.add('numa', 'node,memdev=mem')
173 self._params.add('balloon', 'none')
175 def add_nestedvm_params(self):
176 """Set NestedVM QEMU parameters."""
177 self._params.add('net', 'nic,macaddr=52:54:00:00:{qemu:02x}:ff'.
178 format(qemu=self._opt.get('qemu_id')))
179 self._params.add('net', 'user,hostfwd=tcp::{info[port]}-:22'.
180 format(info=self._vm_info))
181 # TODO: Remove try except after fully migrated to Bionic or
182 # qemu_set_node is removed.
184 locking = ',file.locking=off'\
185 if self.qemu_version(version='2.10') else ''
186 except AttributeError:
188 self._params.add('drive',
189 'file={img},format=raw,cache=none,if=virtio{locking}'.
190 format(img=self._opt.get('img'), locking=locking))
191 self._params.add('qmp', 'unix:{qmp},server,nowait'.
192 format(qmp=self._temp.get('qmp')))
193 self._params.add('chardev', 'socket,host=127.0.0.1,port={info[serial]},'
194 'id=gnc0,server,nowait'.format(info=self._vm_info))
195 self._params.add('device', 'isa-serial,chardev=gnc0')
196 self._params.add('chardev',
197 'socket,path={qga},server,nowait,id=qga0'.
198 format(qga=self._temp.get('qga')))
199 self._params.add('device', 'isa-serial,chardev=qga0')
201 def add_kernelvm_params(self):
202 """Set KernelVM QEMU parameters."""
203 self._params.add('chardev', 'file,id=char0,path={log}'.
204 format(log=self._temp.get('log')))
205 self._params.add('device', 'isa-serial,chardev=char0')
206 self._params.add('fsdev', 'local,id=root9p,path=/,security_model=none')
207 self._params.add('device',
208 'virtio-9p-pci,fsdev=root9p,mount_tag=/dev/root')
209 self._params.add('kernel', '$(readlink -m {img}* | tail -1)'.
210 format(img=self._opt.get('img')))
211 self._params.add('append',
212 '"ro rootfstype=9p rootflags=trans=virtio '
213 'console=ttyS0 tsc=reliable hugepages=256 '
214 'init={init}"'.format(init=self._temp.get('ini')))
216 def create_kernelvm_config_vpp(self, **kwargs):
217 """Create QEMU VPP config files.
219 :param kwargs: Key-value pairs to replace content of VPP configuration
223 startup = ('/etc/vpp/vm_startup_{id}.conf'.
224 format(id=self._opt.get('qemu_id')))
225 running = ('/etc/vpp/vm_running_{id}.exec'.
226 format(id=self._opt.get('qemu_id')))
228 self._temp['startup'] = startup
229 self._temp['running'] = running
230 self._opt['vnf_bin'] = ('/usr/bin/vpp -c {startup}'.
231 format(startup=startup))
233 # Create VPP startup configuration.
234 vpp_config = VppConfigGenerator()
235 vpp_config.set_node(self._node)
236 vpp_config.add_unix_nodaemon()
237 vpp_config.add_unix_cli_listen()
238 vpp_config.add_unix_exec(running)
239 vpp_config.add_cpu_main_core('0')
240 vpp_config.add_cpu_corelist_workers('1-{smp}'.
241 format(smp=self._opt.get('smp')-1))
242 vpp_config.add_dpdk_dev('0000:00:06.0', '0000:00:07.0')
243 vpp_config.add_dpdk_log_level('debug')
244 vpp_config.add_dpdk_no_tx_checksum_offload()
245 vpp_config.add_dpdk_no_multi_seg()
246 vpp_config.add_plugin('disable', 'default')
247 vpp_config.add_plugin('enable', 'dpdk_plugin.so')
248 vpp_config.apply_config(startup, restart_vpp=False)
250 # Create VPP running configuration.
251 template = '{res}/{tpl}.exec'.format(res=Constants.RESOURCES_TPL_VM,
252 tpl=self._opt.get('vnf'))
253 exec_cmd_no_error(self._node, 'rm -f {running}'.format(running=running),
256 with open(template, 'r') as src_file:
257 src = Template(src_file.read())
258 exec_cmd_no_error(self._node, "echo '{out}' | sudo tee {running}".
259 format(out=src.safe_substitute(**kwargs),
262 def create_kernelvm_init(self, **kwargs):
263 """Create QEMU init script.
265 :param kwargs: Key-value pairs to replace content of init startup file.
268 template = '{res}/init.sh'.format(res=Constants.RESOURCES_TPL_VM)
269 init = self._temp.get('ini')
270 exec_cmd_no_error(self._node, 'rm -f {init}'.format(init=init),
273 with open(template, 'r') as src_file:
274 src = Template(src_file.read())
275 exec_cmd_no_error(self._node, "echo '{out}' | sudo tee {init}".
276 format(out=src.safe_substitute(**kwargs),
278 exec_cmd_no_error(self._node, "chmod +x {init}".
279 format(init=init), sudo=True)
281 def configure_kernelvm_vnf(self, **kwargs):
282 """Create KernelVM VNF configurations.
284 :param kwargs: Key-value pairs for templating configs.
287 if 'vpp' in self._opt.get('vnf'):
288 self.create_kernelvm_config_vpp(**kwargs)
290 raise RuntimeError('QEMU: Unsupported VNF!')
291 self.create_kernelvm_init(vnf_bin=self._opt.get('vnf_bin'))
293 def qemu_set_node(self, node):
294 """Set node to run QEMU on.
296 :param node: Node to run QEMU on.
300 self._vm_info['host'] = node['host']
301 if node['port'] != 22:
302 self._vm_info['host_port'] = node['port']
303 self._vm_info['host_username'] = node['username']
304 self._vm_info['host_password'] = node['password']
306 def get_qemu_pids(self):
307 """Get QEMU CPU pids.
309 :returns: List of QEMU CPU pids.
312 command = ("grep -rwl 'CPU' /proc/$(sudo cat {pidfile})/task/*/comm ".
313 format(pidfile=self._temp.get('pidfile')))
314 command += (r"| xargs dirname | sed -e 's/\/.*\///g'")
316 stdout, _ = exec_cmd_no_error(self._node, command)
317 return stdout.splitlines()
319 def qemu_set_affinity(self, *host_cpus):
320 """Set qemu affinity by getting thread PIDs via QMP and taskset to list
323 :param host_cpus: List of CPU cores.
324 :type host_cpus: list
327 qemu_cpus = self.get_qemu_pids()
329 if len(qemu_cpus) != len(host_cpus):
330 raise ValueError('Host CPU count must match Qemu Thread count!')
332 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
333 command = ('taskset -pc {host_cpu} {thread}'.
334 format(host_cpu=host_cpu, thread=qemu_cpu))
335 message = ('QEMU: Set affinity failed on {host}!'.
336 format(host=self._node['host']))
337 exec_cmd_no_error(self._node, command, sudo=True,
339 except (RuntimeError, ValueError):
343 def qemu_set_scheduler_policy(self):
344 """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
347 :raises RuntimeError: Set scheduler policy failed.
350 qemu_cpus = self.get_qemu_pids()
352 for qemu_cpu in qemu_cpus:
353 command = ('chrt -r -p 1 {thread}'.
354 format(thread=qemu_cpu))
355 message = ('QEMU: Set SCHED_RR failed on {host}'.
356 format(host=self._node['host']))
357 exec_cmd_no_error(self._node, command, sudo=True,
359 except (RuntimeError, ValueError):
363 def qemu_add_vhost_user_if(self, socket, server=True, jumbo_frames=False,
364 queue_size=None, queues=1):
365 """Add Vhost-user interface.
367 :param socket: Path of the unix socket.
368 :param server: If True the socket shall be a listening socket.
369 :param jumbo_frames: Set True if jumbo frames are used in the test.
370 :param queue_size: Vring queue size.
371 :param queues: Number of queues.
374 :type jumbo_frames: bool
375 :type queue_size: int
379 self._params.add('chardev',
380 'socket,id=char{vhost},path={socket}{server}'.
381 format(vhost=self._vhost_id, socket=socket,
382 server=',server' if server is True else ''))
383 self._params.add('netdev',
384 'vhost-user,id=vhost{vhost},'
385 'chardev=char{vhost},queues={queues}'.
386 format(vhost=self._vhost_id, queues=queues))
387 mac = ('52:54:00:00:{qemu:02x}:{vhost:02x}'.
388 format(qemu=self._opt.get('qemu_id'), vhost=self._vhost_id))
389 queue_size = ('rx_queue_size={queue_size},tx_queue_size={queue_size}'.
390 format(queue_size=queue_size)) if queue_size else ''
391 mbuf = 'on,host_mtu=9200'
392 self._params.add('device',
393 'virtio-net-pci,netdev=vhost{vhost},'
394 'mac={mac},bus=pci.0,addr={addr}.0,mq=on,'
395 'vectors={vectors},csum=off,gso=off,'
396 'guest_tso4=off,guest_tso6=off,guest_ecn=off,'
397 'mrg_rxbuf={mbuf},{queue_size}'.
398 format(addr=self._vhost_id+5,
399 vhost=self._vhost_id, mac=mac,
400 mbuf=mbuf if jumbo_frames else 'off',
401 queue_size=queue_size,
402 vectors=(2 * queues + 2)))
404 # Add interface MAC and socket to the node dict.
405 if_data = {'mac_address': mac, 'socket': socket}
406 if_name = 'vhost{vhost}'.format(vhost=self._vhost_id)
407 self._vm_info['interfaces'][if_name] = if_data
408 # Add socket to temporary file list.
409 self._temp[if_name] = socket
411 def _qemu_qmp_exec(self, cmd):
412 """Execute QMP command.
414 QMP is JSON based protocol which allows to control QEMU instance.
416 :param cmd: QMP command to execute.
418 :returns: Command output in python representation of JSON format. The
419 { "return": {} } response is QMP's success response. An error
420 response will contain the "error" keyword instead of "return".
422 # To enter command mode, the qmp_capabilities command must be issued.
423 command = ('echo "{{ \\"execute\\": \\"qmp_capabilities\\" }}'
424 '{{ \\"execute\\": \\"{cmd}\\" }}" | '
425 'sudo -S socat - UNIX-CONNECT:{qmp}'.
426 format(cmd=cmd, qmp=self._temp.get('qmp')))
427 message = ('QMP execute "{cmd}" failed on {host}'.
428 format(cmd=cmd, host=self._node['host']))
429 stdout, _ = exec_cmd_no_error(self._node, command, sudo=False,
432 # Skip capabilities negotiation messages.
433 out_list = stdout.splitlines()
434 if len(out_list) < 3:
435 raise RuntimeError('Invalid QMP output on {host}'.
436 format(host=self._node['host']))
437 return json.loads(out_list[2])
439 def _qemu_qga_flush(self):
440 """Flush the QGA parser state."""
441 command = ('(printf "\xFF"; sleep 1) | '
442 'sudo -S socat - UNIX-CONNECT:{qga}'.
443 format(qga=self._temp.get('qga')))
444 message = ('QGA flush failed on {host}'.format(host=self._node['host']))
445 stdout, _ = exec_cmd_no_error(self._node, command, sudo=False,
448 return json.loads(stdout.split('\n', 1)[0]) if stdout else dict()
450 def _qemu_qga_exec(self, cmd):
451 """Execute QGA command.
453 QGA provide access to a system-level agent via standard QMP commands.
455 :param cmd: QGA command to execute.
458 command = ('(echo "{{ \\"execute\\": \\"{cmd}\\" }}"; sleep 1) | '
459 'sudo -S socat - UNIX-CONNECT:{qga}'.
460 format(cmd=cmd, qga=self._temp.get('qga')))
461 message = ('QGA execute "{cmd}" failed on {host}'.
462 format(cmd=cmd, host=self._node['host']))
463 stdout, _ = exec_cmd_no_error(self._node, command, sudo=False,
466 return json.loads(stdout.split('\n', 1)[0]) if stdout else dict()
468 def _wait_until_vm_boot(self):
469 """Wait until QEMU with NestedVM is booted."""
470 if self._opt.get('vm_type') == 'nestedvm':
471 self._wait_until_nestedvm_boot()
472 self._update_vm_interfaces()
473 elif self._opt.get('vm_type') == 'kernelvm':
474 self._wait_until_kernelvm_boot()
476 raise RuntimeError('QEMU: Unsupported VM type!')
478 def _wait_until_nestedvm_boot(self, retries=12):
479 """Wait until QEMU with NestedVM is booted.
481 First try to flush qga until there is output.
482 Then ping QEMU guest agent each 5s until VM booted or timeout.
484 :param retries: Number of retries with 5s between trials.
487 for _ in range(retries):
490 out = self._qemu_qga_flush()
492 logger.trace('QGA qga flush unexpected output {out}'.
494 # Empty output - VM not booted yet
500 raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
501 format(host=self._node['host']))
502 for _ in range(retries):
505 out = self._qemu_qga_exec('guest-ping')
507 logger.trace('QGA guest-ping unexpected output {out}'.
509 # Empty output - VM not booted yet.
512 # Non-error return - VM booted.
513 elif out.get('return') is not None:
515 # Skip error and wait.
516 elif out.get('error') is not None:
519 # If there is an unexpected output from QGA guest-info, try
520 # again until timeout.
521 logger.trace('QGA guest-ping unexpected output {out}'.
524 raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
525 format(host=self._node['host']))
527 def _wait_until_kernelvm_boot(self, retries=60):
528 """Wait until QEMU KernelVM is booted.
530 :param retries: Number of retries.
533 for _ in range(retries):
534 command = ('tail -1 {log}'.format(log=self._temp.get('log')))
537 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
541 if VPPUtil.vpp_show_version(self._node) in stdout:
544 raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
545 format(host=self._node['host']))
547 def _update_vm_interfaces(self):
548 """Update interface names in VM node dict."""
549 # Send guest-network-get-interfaces command via QGA, output example:
550 # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
551 # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
552 out = self._qemu_qga_exec('guest-network-get-interfaces')
553 interfaces = out.get('return')
556 raise RuntimeError('Get VM interface list failed on {host}'.
557 format(host=self._node['host']))
558 # Create MAC-name dict.
559 for interface in interfaces:
560 if 'hardware-address' not in interface:
562 mac_name[interface['hardware-address']] = interface['name']
563 # Match interface by MAC and save interface name.
564 for interface in self._vm_info['interfaces'].values():
565 mac = interface.get('mac_address')
566 if_name = mac_name.get(mac)
568 logger.trace('Interface name for MAC {mac} not found'.
571 interface['name'] = if_name
573 def qemu_start(self):
574 """Start QEMU and wait until VM boot.
576 :returns: VM node info.
579 command = ('{bin_path}/qemu-system-{arch} {params}'.
580 format(bin_path=self._opt.get('bin_path'),
581 arch=Topology.get_node_arch(self._node),
582 params=self._params))
583 message = ('QEMU: Start failed on {host}!'.
584 format(host=self._node['host']))
586 DUTSetup.check_huge_page(self._node, '/dev/hugepages',
587 self._opt.get('mem'))
589 exec_cmd_no_error(self._node, command, timeout=300, sudo=True,
591 self._wait_until_vm_boot()
598 """Kill qemu process."""
599 exec_cmd(self._node, 'chmod +r {pidfile}'.
600 format(pidfile=self._temp.get('pidfile')), sudo=True)
601 exec_cmd(self._node, 'kill -SIGKILL $(cat {pidfile})'.
602 format(pidfile=self._temp.get('pidfile')), sudo=True)
604 for value in self._temp.values():
605 exec_cmd(self._node, 'rm -f {value}'.format(value=value), sudo=True)
607 def qemu_kill_all(self, node=None):
608 """Kill all qemu processes on DUT node if specified.
610 :param node: Node to kill all QEMU processes on.
614 self.qemu_set_node(node)
615 exec_cmd(self._node, 'pkill -SIGKILL qemu', sudo=True)
617 for value in self._temp.values():
618 exec_cmd(self._node, 'rm -f {value}'.format(value=value), sudo=True)
620 def qemu_version(self, version=None):
621 """Return Qemu version or compare if version is higher than parameter.
623 :param version: Version to compare.
625 :returns: Qemu version or Boolean if version is higher than parameter.
628 command = ('{bin_path}/qemu-system-{arch} --version'.
629 format(bin_path=self._opt.get('bin_path'),
630 arch=Topology.get_node_arch(self._node)))
632 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
633 ver = match(r'QEMU emulator version ([\d.]*)', stdout).group(1)
634 return StrictVersion(ver) > StrictVersion(version) \
641 def build_qemu(node, force_install=False, apply_patch=False):
642 """Build QEMU from sources.
644 :param node: Node to build QEMU on.
645 :param force_install: If True, then remove previous build.
646 :param apply_patch: If True, then apply patches from qemu_patches dir.
648 :type force_install: bool
649 :type apply_patch: bool
650 :raises RuntimeError: If building QEMU failed.
652 directory = (' --directory={install_dir}{patch}'.
653 format(install_dir=Constants.QEMU_INSTALL_DIR,
654 patch='-patch' if apply_patch else '-base'))
655 version = (' --version={install_version}'.
656 format(install_version=Constants.QEMU_INSTALL_VERSION))
657 force = ' --force' if force_install else ''
658 patch = ' --patch' if apply_patch else ''
659 target_list = (' --target-list={arch}-softmmu'.
660 format(arch=Topology.get_node_arch(node)))
662 command = ("sudo -E sh -c "
663 "'{fw_dir}/{lib_sh}/qemu_build.sh{version}{directory}"
664 "{force}{patch}{target_list}'".
665 format(fw_dir=Constants.REMOTE_FW_DIR,
666 lib_sh=Constants.RESOURCES_LIB_SH,
667 version=version, directory=directory, force=force,
668 patch=patch, target_list=target_list))
669 message = ('QEMU: Build failed on {host}!'.format(host=node['host']))
670 exec_cmd_no_error(node, command, sudo=False, message=message,