1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """QEMU utilities library."""
16 from time import sleep
17 from string import Template
20 # Disable due to pylint bug
21 # pylint: disable=no-name-in-module,import-error
22 from distutils.version import StrictVersion
24 from robot.api import logger
25 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
26 from resources.libraries.python.Constants import Constants
27 from resources.libraries.python.DUTSetup import DUTSetup
28 from resources.libraries.python.topology import NodeType, Topology
29 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
30 from resources.libraries.python.VPPUtil import VPPUtil
32 __all__ = ["QemuOptions", "QemuUtils"]
35 class QemuOptions(object):
38 The class can handle input parameters that acts as QEMU command line
39 parameters. The only variable is a list of dictionaries where dictionaries
40 can be added multiple times. This emulates the QEMU behavior where one
41 command line parameter can be used multiple times (1..N). Example can be
42 device or object (so it is not an issue to have one memory
43 block of 2G and and second memory block of 512M but from other numa).
45 Class does support get value or string representation that will return
46 space separated, dash prefixed string of key value pairs used for command
50 # Use one instance of class per tests.
51 ROBOT_LIBRARY_SCOPE = 'TEST CASE'
54 self.variables = list()
56 def add(self, variable, value):
57 """Add parameter to the list.
59 :param variable: QEMU parameter name (without dash).
60 :param value: Paired value.
62 :type value: str or int
64 self.variables.append({str(variable): value})
67 """Return space separated string of key value pairs.
69 The format is suitable to be pasted to qemu command line.
71 :returns: Space separated string of key value pairs.
74 return " ".join(["-{k} {v}".format(k=d.keys()[0], v=d.values()[0])
75 for d in self.variables])
78 class QemuUtils(object):
81 # Use one instance of class per tests.
82 ROBOT_LIBRARY_SCOPE = 'TEST CASE'
84 def __init__(self, node=None, qemu_id=1, smp=1, mem=512, vnf=None,
85 img='/var/lib/vm/vhost-nested.img', bin_path='/usr/bin'):
86 """Initialize QemuUtil class.
88 :param node: Node to run QEMU on.
89 :param qemu_id: QEMU identifier.
90 :param smp: Number of virtual SMP units (cores).
91 :param mem: Amount of memory.
92 :param vnf: Network function workload.
93 :param img: QEMU disk image or kernel image path.
94 :param bin_path: QEMU binary path.
106 'host': node['host'],
108 'port': 10021 + qemu_id,
109 'serial': 4555 + qemu_id,
114 if node['port'] != 22:
115 self._vm_info['host_port'] = node['port']
116 self._vm_info['host_username'] = node['username']
117 self._vm_info['host_password'] = node['password']
120 self._opt['qemu_id'] = qemu_id
121 self._opt['bin_path'] = bin_path
122 self._opt['mem'] = int(mem)
123 self._opt['smp'] = int(smp)
124 self._opt['img'] = img
125 self._opt['vnf'] = vnf
128 self._temp['pidfile'] = '/var/run/qemu_{id}.pid'.format(id=qemu_id)
129 if '/var/lib/vm/' in img:
130 self._opt['vm_type'] = 'nestedvm'
131 self._temp['qmp'] = '/var/run/qmp_{id}.sock'.format(id=qemu_id)
132 self._temp['qga'] = '/var/run/qga_{id}.sock'.format(id=qemu_id)
133 elif '/opt/boot/vmlinuz' in img:
134 self._opt['vm_type'] = 'kernelvm'
135 self._temp['log'] = '/tmp/serial_{id}.log'.format(id=qemu_id)
136 self._temp['ini'] = '/etc/vm_init_{id}.conf'.format(id=qemu_id)
138 raise RuntimeError('QEMU: Unknown VM image option!')
139 # Computed parameters for QEMU command line.
140 self._params = QemuOptions()
143 def add_params(self):
144 """Set QEMU command line parameters."""
145 self.add_default_params()
146 if self._opt.get('vm_type', '') == 'nestedvm':
147 self.add_nestedvm_params()
148 elif self._opt.get('vm_type', '') == 'kernelvm':
149 self.add_kernelvm_params()
151 raise RuntimeError('QEMU: Unsupported VM type!')
153 def add_default_params(self):
154 """Set default QEMU command line parameters."""
155 self._params.add('daemonize', '')
156 self._params.add('nodefaults', '')
157 self._params.add('name', 'vnf{qemu},debug-threads=on'.
158 format(qemu=self._opt.get('qemu_id')))
159 self._params.add('no-user-config', '')
160 self._params.add('monitor', 'none')
161 self._params.add('display', 'none')
162 self._params.add('vga', 'none')
163 self._params.add('enable-kvm', '')
164 self._params.add('pidfile', '{pidfile}'.
165 format(pidfile=self._temp.get('pidfile')))
166 self._params.add('cpu', 'host')
167 self._params.add('machine', 'pc,accel=kvm,usb=off,mem-merge=off')
168 self._params.add('smp', '{smp},sockets=1,cores={smp},threads=1'.
169 format(smp=self._opt.get('smp')))
170 self._params.add('object',
171 'memory-backend-file,id=mem,size={mem}M,'
172 'mem-path=/dev/hugepages,share=on'.
173 format(mem=self._opt.get('mem')))
174 self._params.add('m', '{mem}M'.
175 format(mem=self._opt.get('mem')))
176 self._params.add('numa', 'node,memdev=mem')
177 self._params.add('balloon', 'none')
179 def add_nestedvm_params(self):
180 """Set NestedVM QEMU parameters."""
181 self._params.add('net', 'nic,macaddr=52:54:00:00:{qemu:02x}:ff'.
182 format(qemu=self._opt.get('qemu_id')))
183 self._params.add('net', 'user,hostfwd=tcp::{info[port]}-:22'.
184 format(info=self._vm_info))
185 # TODO: Remove try except after fully migrated to Bionic or
186 # qemu_set_node is removed.
188 locking = ',file.locking=off'\
189 if self.qemu_version(version='2.10') else ''
190 except AttributeError:
192 self._params.add('drive',
193 'file={img},format=raw,cache=none,if=virtio{locking}'.
194 format(img=self._opt.get('img'), locking=locking))
195 self._params.add('qmp', 'unix:{qmp},server,nowait'.
196 format(qmp=self._temp.get('qmp')))
197 self._params.add('chardev', 'socket,host=127.0.0.1,port={info[serial]},'
198 'id=gnc0,server,nowait'.format(info=self._vm_info))
199 self._params.add('device', 'isa-serial,chardev=gnc0')
200 self._params.add('chardev',
201 'socket,path={qga},server,nowait,id=qga0'.
202 format(qga=self._temp.get('qga')))
203 self._params.add('device', 'isa-serial,chardev=qga0')
205 def add_kernelvm_params(self):
206 """Set KernelVM QEMU parameters."""
207 self._params.add('chardev', 'file,id=char0,path={log}'.
208 format(log=self._temp.get('log')))
209 self._params.add('device', 'isa-serial,chardev=char0')
210 self._params.add('fsdev', 'local,id=root9p,path=/,security_model=none')
211 self._params.add('device',
212 'virtio-9p-pci,fsdev=root9p,mount_tag=/dev/root')
213 self._params.add('kernel', '$(readlink -m {img}* | tail -1)'.
214 format(img=self._opt.get('img')))
215 self._params.add('append',
216 '"ro rootfstype=9p rootflags=trans=virtio '
217 'console=ttyS0 tsc=reliable hugepages=256 '
218 'init={init}"'.format(init=self._temp.get('ini')))
220 def create_kernelvm_config_vpp(self, **kwargs):
221 """Create QEMU VPP config files.
223 :param kwargs: Key-value pairs to replace content of VPP configuration
227 startup = ('/etc/vpp/vm_startup_{id}.conf'.
228 format(id=self._opt.get('qemu_id')))
229 running = ('/etc/vpp/vm_running_{id}.exec'.
230 format(id=self._opt.get('qemu_id')))
232 self._temp['startup'] = startup
233 self._temp['running'] = running
234 self._opt['vnf_bin'] = ('/usr/bin/vpp -c {startup}'.
235 format(startup=startup))
237 # Create VPP startup configuration.
238 vpp_config = VppConfigGenerator()
239 vpp_config.set_node(self._node)
240 vpp_config.add_unix_nodaemon()
241 vpp_config.add_unix_cli_listen()
242 vpp_config.add_unix_exec(running)
243 vpp_config.add_cpu_main_core('0')
244 vpp_config.add_cpu_corelist_workers('1-{smp}'.
245 format(smp=self._opt.get('smp')-1))
246 vpp_config.add_dpdk_dev('0000:00:06.0', '0000:00:07.0')
247 vpp_config.add_dpdk_log_level('debug')
248 vpp_config.add_dpdk_no_tx_checksum_offload()
249 vpp_config.add_dpdk_no_multi_seg()
250 vpp_config.add_plugin('disable', 'default')
251 vpp_config.add_plugin('enable', 'dpdk_plugin.so')
252 vpp_config.apply_config(startup, restart_vpp=False)
254 # Create VPP running configuration.
255 template = '{res}/{tpl}.exec'.format(res=Constants.RESOURCES_TPL_VM,
256 tpl=self._opt.get('vnf'))
257 exec_cmd_no_error(self._node, 'rm -f {running}'.format(running=running),
260 with open(template, 'r') as src_file:
261 src = Template(src_file.read())
262 exec_cmd_no_error(self._node, "echo '{out}' | sudo tee {running}".
263 format(out=src.safe_substitute(**kwargs),
266 def create_kernelvm_init(self, **kwargs):
267 """Create QEMU init script.
269 :param kwargs: Key-value pairs to replace content of init startup file.
272 template = '{res}/init.sh'.format(res=Constants.RESOURCES_TPL_VM)
273 init = self._temp.get('ini')
274 exec_cmd_no_error(self._node, 'rm -f {init}'.format(init=init),
277 with open(template, 'r') as src_file:
278 src = Template(src_file.read())
279 exec_cmd_no_error(self._node, "echo '{out}' | sudo tee {init}".
280 format(out=src.safe_substitute(**kwargs),
282 exec_cmd_no_error(self._node, "chmod +x {init}".
283 format(init=init), sudo=True)
285 def configure_kernelvm_vnf(self, **kwargs):
286 """Create KernelVM VNF configurations.
288 :param kwargs: Key-value pairs for templating configs.
291 if 'vpp' in self._opt.get('vnf'):
292 self.create_kernelvm_config_vpp(**kwargs)
294 raise RuntimeError('QEMU: Unsupported VNF!')
295 self.create_kernelvm_init(vnf_bin=self._opt.get('vnf_bin'))
297 def get_qemu_pids(self):
298 """Get QEMU CPU pids.
300 :returns: List of QEMU CPU pids.
303 command = ("grep -rwl 'CPU' /proc/$(sudo cat {pidfile})/task/*/comm ".
304 format(pidfile=self._temp.get('pidfile')))
305 command += (r"| xargs dirname | sed -e 's/\/.*\///g'")
307 stdout, _ = exec_cmd_no_error(self._node, command)
308 return stdout.splitlines()
310 def qemu_set_affinity(self, *host_cpus):
311 """Set qemu affinity by getting thread PIDs via QMP and taskset to list
312 of CPU cores. Function tries to execute 3 times to avoid race condition
313 in getting thread PIDs.
315 :param host_cpus: List of CPU cores.
316 :type host_cpus: list
320 qemu_cpus = self.get_qemu_pids()
322 if len(qemu_cpus) != len(host_cpus):
325 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
326 command = ('taskset -pc {host_cpu} {thread}'.
327 format(host_cpu=host_cpu, thread=qemu_cpu))
328 message = ('QEMU: Set affinity failed on {host}!'.
329 format(host=self._node['host']))
330 exec_cmd_no_error(self._node, command, sudo=True,
333 except (RuntimeError, ValueError):
338 raise RuntimeError('Failed to set Qemu threads affinity!')
340 def qemu_set_scheduler_policy(self):
341 """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
344 :raises RuntimeError: Set scheduler policy failed.
347 qemu_cpus = self.get_qemu_pids()
349 for qemu_cpu in qemu_cpus:
350 command = ('chrt -r -p 1 {thread}'.
351 format(thread=qemu_cpu))
352 message = ('QEMU: Set SCHED_RR failed on {host}'.
353 format(host=self._node['host']))
354 exec_cmd_no_error(self._node, command, sudo=True,
356 except (RuntimeError, ValueError):
360 def qemu_add_vhost_user_if(self, socket, server=True, jumbo_frames=False,
361 queue_size=None, queues=1):
362 """Add Vhost-user interface.
364 :param socket: Path of the unix socket.
365 :param server: If True the socket shall be a listening socket.
366 :param jumbo_frames: Set True if jumbo frames are used in the test.
367 :param queue_size: Vring queue size.
368 :param queues: Number of queues.
371 :type jumbo_frames: bool
372 :type queue_size: int
376 self._params.add('chardev',
377 'socket,id=char{vhost},path={socket}{server}'.
378 format(vhost=self._vhost_id, socket=socket,
379 server=',server' if server is True else ''))
380 self._params.add('netdev',
381 'vhost-user,id=vhost{vhost},'
382 'chardev=char{vhost},queues={queues}'.
383 format(vhost=self._vhost_id, queues=queues))
384 mac = ('52:54:00:00:{qemu:02x}:{vhost:02x}'.
385 format(qemu=self._opt.get('qemu_id'), vhost=self._vhost_id))
386 queue_size = ('rx_queue_size={queue_size},tx_queue_size={queue_size}'.
387 format(queue_size=queue_size)) if queue_size else ''
388 mbuf = 'on,host_mtu=9200'
389 self._params.add('device',
390 'virtio-net-pci,netdev=vhost{vhost},'
391 'mac={mac},bus=pci.0,addr={addr}.0,mq=on,'
392 'vectors={vectors},csum=off,gso=off,'
393 'guest_tso4=off,guest_tso6=off,guest_ecn=off,'
394 'mrg_rxbuf={mbuf},{queue_size}'.
395 format(addr=self._vhost_id+5,
396 vhost=self._vhost_id, mac=mac,
397 mbuf=mbuf if jumbo_frames else 'off',
398 queue_size=queue_size,
399 vectors=(2 * queues + 2)))
401 # Add interface MAC and socket to the node dict.
402 if_data = {'mac_address': mac, 'socket': socket}
403 if_name = 'vhost{vhost}'.format(vhost=self._vhost_id)
404 self._vm_info['interfaces'][if_name] = if_data
405 # Add socket to temporary file list.
406 self._temp[if_name] = socket
408 def _qemu_qmp_exec(self, cmd):
409 """Execute QMP command.
411 QMP is JSON based protocol which allows to control QEMU instance.
413 :param cmd: QMP command to execute.
415 :returns: Command output in python representation of JSON format. The
416 { "return": {} } response is QMP's success response. An error
417 response will contain the "error" keyword instead of "return".
419 # To enter command mode, the qmp_capabilities command must be issued.
420 command = ('echo "{{ \\"execute\\": \\"qmp_capabilities\\" }}'
421 '{{ \\"execute\\": \\"{cmd}\\" }}" | '
422 'sudo -S socat - UNIX-CONNECT:{qmp}'.
423 format(cmd=cmd, qmp=self._temp.get('qmp')))
424 message = ('QMP execute "{cmd}" failed on {host}'.
425 format(cmd=cmd, host=self._node['host']))
426 stdout, _ = exec_cmd_no_error(self._node, command, sudo=False,
429 # Skip capabilities negotiation messages.
430 out_list = stdout.splitlines()
431 if len(out_list) < 3:
432 raise RuntimeError('Invalid QMP output on {host}'.
433 format(host=self._node['host']))
434 return json.loads(out_list[2])
436 def _qemu_qga_flush(self):
437 """Flush the QGA parser state."""
438 command = ('(printf "\xFF"; sleep 1) | '
439 'sudo -S socat - UNIX-CONNECT:{qga}'.
440 format(qga=self._temp.get('qga')))
441 message = ('QGA flush failed on {host}'.format(host=self._node['host']))
442 stdout, _ = exec_cmd_no_error(self._node, command, sudo=False,
445 return json.loads(stdout.split('\n', 1)[0]) if stdout else dict()
447 def _qemu_qga_exec(self, cmd):
448 """Execute QGA command.
450 QGA provide access to a system-level agent via standard QMP commands.
452 :param cmd: QGA command to execute.
455 command = ('(echo "{{ \\"execute\\": \\"{cmd}\\" }}"; sleep 1) | '
456 'sudo -S socat - UNIX-CONNECT:{qga}'.
457 format(cmd=cmd, qga=self._temp.get('qga')))
458 message = ('QGA execute "{cmd}" failed on {host}'.
459 format(cmd=cmd, host=self._node['host']))
460 stdout, _ = exec_cmd_no_error(self._node, command, sudo=False,
463 return json.loads(stdout.split('\n', 1)[0]) if stdout else dict()
465 def _wait_until_vm_boot(self):
466 """Wait until QEMU with NestedVM is booted."""
467 if self._opt.get('vm_type') == 'nestedvm':
468 self._wait_until_nestedvm_boot()
469 self._update_vm_interfaces()
470 elif self._opt.get('vm_type') == 'kernelvm':
471 self._wait_until_kernelvm_boot()
473 raise RuntimeError('QEMU: Unsupported VM type!')
475 def _wait_until_nestedvm_boot(self, retries=12):
476 """Wait until QEMU with NestedVM is booted.
478 First try to flush qga until there is output.
479 Then ping QEMU guest agent each 5s until VM booted or timeout.
481 :param retries: Number of retries with 5s between trials.
484 for _ in range(retries):
487 out = self._qemu_qga_flush()
489 logger.trace('QGA qga flush unexpected output {out}'.
491 # Empty output - VM not booted yet
497 raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
498 format(host=self._node['host']))
499 for _ in range(retries):
502 out = self._qemu_qga_exec('guest-ping')
504 logger.trace('QGA guest-ping unexpected output {out}'.
506 # Empty output - VM not booted yet.
509 # Non-error return - VM booted.
510 elif out.get('return') is not None:
512 # Skip error and wait.
513 elif out.get('error') is not None:
516 # If there is an unexpected output from QGA guest-info, try
517 # again until timeout.
518 logger.trace('QGA guest-ping unexpected output {out}'.
521 raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
522 format(host=self._node['host']))
524 def _wait_until_kernelvm_boot(self, retries=60):
525 """Wait until QEMU KernelVM is booted.
527 :param retries: Number of retries.
530 for _ in range(retries):
531 command = ('tail -1 {log}'.format(log=self._temp.get('log')))
534 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
538 if VPPUtil.vpp_show_version(self._node) in stdout:
541 raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
542 format(host=self._node['host']))
544 def _update_vm_interfaces(self):
545 """Update interface names in VM node dict."""
546 # Send guest-network-get-interfaces command via QGA, output example:
547 # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
548 # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
549 out = self._qemu_qga_exec('guest-network-get-interfaces')
550 interfaces = out.get('return')
553 raise RuntimeError('Get VM interface list failed on {host}'.
554 format(host=self._node['host']))
555 # Create MAC-name dict.
556 for interface in interfaces:
557 if 'hardware-address' not in interface:
559 mac_name[interface['hardware-address']] = interface['name']
560 # Match interface by MAC and save interface name.
561 for interface in self._vm_info['interfaces'].values():
562 mac = interface.get('mac_address')
563 if_name = mac_name.get(mac)
565 logger.trace('Interface name for MAC {mac} not found'.
568 interface['name'] = if_name
570 def qemu_start(self):
571 """Start QEMU and wait until VM boot.
573 :returns: VM node info.
576 command = ('{bin_path}/qemu-system-{arch} {params}'.
577 format(bin_path=self._opt.get('bin_path'),
578 arch=Topology.get_node_arch(self._node),
579 params=self._params))
580 message = ('QEMU: Start failed on {host}!'.
581 format(host=self._node['host']))
583 DUTSetup.check_huge_page(self._node, '/dev/hugepages',
584 self._opt.get('mem'))
586 exec_cmd_no_error(self._node, command, timeout=300, sudo=True,
588 self._wait_until_vm_boot()
595 """Kill qemu process."""
596 exec_cmd(self._node, 'chmod +r {pidfile}'.
597 format(pidfile=self._temp.get('pidfile')), sudo=True)
598 exec_cmd(self._node, 'kill -SIGKILL $(cat {pidfile})'.
599 format(pidfile=self._temp.get('pidfile')), sudo=True)
601 for value in self._temp.values():
602 exec_cmd(self._node, 'rm -f {value}'.format(value=value), sudo=True)
604 def qemu_kill_all(self):
605 """Kill all qemu processes on DUT node if specified."""
606 exec_cmd(self._node, 'pkill -SIGKILL qemu', sudo=True)
608 for value in self._temp.values():
609 exec_cmd(self._node, 'rm -f {value}'.format(value=value), sudo=True)
611 def qemu_version(self, version=None):
612 """Return Qemu version or compare if version is higher than parameter.
614 :param version: Version to compare.
616 :returns: Qemu version or Boolean if version is higher than parameter.
619 command = ('{bin_path}/qemu-system-{arch} --version'.
620 format(bin_path=self._opt.get('bin_path'),
621 arch=Topology.get_node_arch(self._node)))
623 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
624 ver = match(r'QEMU emulator version ([\d.]*)', stdout).group(1)
625 return StrictVersion(ver) > StrictVersion(version) \
632 def build_qemu(node, force_install=False, apply_patch=False):
633 """Build QEMU from sources.
635 :param node: Node to build QEMU on.
636 :param force_install: If True, then remove previous build.
637 :param apply_patch: If True, then apply patches from qemu_patches dir.
639 :type force_install: bool
640 :type apply_patch: bool
641 :raises RuntimeError: If building QEMU failed.
643 directory = (' --directory={install_dir}{patch}'.
644 format(install_dir=Constants.QEMU_INSTALL_DIR,
645 patch='-patch' if apply_patch else '-base'))
646 version = (' --version={install_version}'.
647 format(install_version=Constants.QEMU_INSTALL_VERSION))
648 force = ' --force' if force_install else ''
649 patch = ' --patch' if apply_patch else ''
650 target_list = (' --target-list={arch}-softmmu'.
651 format(arch=Topology.get_node_arch(node)))
653 command = ("sudo -E sh -c "
654 "'{fw_dir}/{lib_sh}/qemu_build.sh{version}{directory}"
655 "{force}{patch}{target_list}'".
656 format(fw_dir=Constants.REMOTE_FW_DIR,
657 lib_sh=Constants.RESOURCES_LIB_SH,
658 version=version, directory=directory, force=force,
659 patch=patch, target_list=target_list))
660 message = ('QEMU: Build failed on {host}!'.format(host=node['host']))
661 exec_cmd_no_error(node, command, sudo=False, message=message,