1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """QEMU utilities library."""
16 from time import time, sleep
19 # Disable due to pylint bug
20 # pylint: disable=no-name-in-module,import-error
21 from distutils.version import StrictVersion
23 from robot.api import logger
24 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
25 from resources.libraries.python.Constants import Constants
26 from resources.libraries.python.DUTSetup import DUTSetup
27 from resources.libraries.python.topology import NodeType, Topology
29 __all__ = ["QemuOptions", "QemuUtils"]
32 class QemuOptions(object):
35 The class can handle input parameters or parameters that acts as QEMU
36 command line parameters. The only variable is a list of dictionaries
37 where dictionaries can be added multiple times. This emulates the QEMU
38 behavior where one command line parameter can be used multiple times (1..N).
39 Example can be device or object (so it is not an issue to have one memory
40 block of 2G and and second memory block of 512M but from other numa).
42 Class does support get value or string representation that will return
43 space separated, dash prefixed string of key value pairs used for command
47 # Use one instance of class per tests.
48 ROBOT_LIBRARY_SCOPE = 'TEST CASE'
51 self.variables = list()
53 def add(self, variable, value):
54 """Add parameter to the list.
56 :param variable: QEMU parameter.
57 :param value: Parameter value.
59 :type value: str or int
61 self.variables.append({str(variable): value})
63 def get(self, variable):
64 """Get parameter of variable(s) from list that matches input value.
66 :param variable: QEMU parameter to get.
68 :returns: List of values or value that matches input parameter.
71 selected = [d[variable] for d in self.variables if variable in d]
72 return selected if len(selected) > 1 else selected[0]
75 """Get all values from dict items in list.
77 :returns: List of all dictionary values.
80 return [d.values()[0] for d in self.variables]
83 """Return space separated string of key value pairs.
85 :returns: Space separated string of key value pairs.
88 return " ".join(["-{k} {v}".format(k=d.keys()[0], v=d.values()[0])
89 for d in self.variables])
91 class QemuUtils(object):
94 # Use one instance of class per tests.
95 ROBOT_LIBRARY_SCOPE = 'TEST CASE'
97 def __init__(self, node=None, qemu_id=1, smp=1, mem=512, vnf=None,
98 img='/var/lib/vm/vhost-nested.img', bin_path='/usr/bin'):
99 """Initialize QemuUtil class.
101 :param node: Node to run QEMU on.
102 :param qemu_id: QEMU identifier.
103 :param smp: Number of virtual SMP units (cores).
104 :param mem: Amount of memory.
105 :param vnf: Network function workload.
106 :param img: QEMU disk image or kernel image path.
107 :param bin_path: QEMU binary path.
119 'port': 10021 + qemu_id,
120 'serial': 4555 + qemu_id,
126 self.qemu_set_node(node)
128 self._opt = QemuOptions()
129 self._opt.add('qemu_id', qemu_id)
130 self._opt.add('bin_path', bin_path)
131 self._opt.add('mem', int(mem))
132 self._opt.add('smp', int(smp))
133 self._opt.add('img', img)
134 self._opt.add('vnf', vnf)
136 self._temp = QemuOptions()
137 self._temp.add('pid', '/var/run/qemu_{id}.pid'.format(id=qemu_id))
138 # Computed parameters for QEMU command line.
139 if '/var/lib/vm/' in img:
140 self._opt.add('vm_type', 'nestedvm')
141 self._temp.add('qmp', '/var/run/qmp_{id}.sock'.format(id=qemu_id))
142 self._temp.add('qga', '/var/run/qga_{id}.sock'.format(id=qemu_id))
144 raise RuntimeError('QEMU: Unknown VM image option!')
145 self._params = QemuOptions()
148 def add_params(self):
149 """Set QEMU command line parameters."""
150 self.add_default_params()
151 if self._opt.get('vm_type') == 'nestedvm':
152 self.add_nestedvm_params()
154 raise RuntimeError('QEMU: Unsupported VM type!')
156 def add_default_params(self):
157 """Set default QEMU command line parameters."""
158 self._params.add('daemonize', '')
159 self._params.add('nodefaults', '')
160 self._params.add('name', 'vnf{qemu},debug-threads=on'.
161 format(qemu=self._opt.get('qemu_id')))
162 self._params.add('no-user-config', '')
163 self._params.add('monitor', 'none')
164 self._params.add('display', 'none')
165 self._params.add('vga', 'none')
166 self._params.add('enable-kvm', '')
167 self._params.add('pidfile', '{pid}'.
168 format(pid=self._temp.get('pid')))
169 self._params.add('cpu', 'host')
170 self._params.add('machine', 'pc,accel=kvm,usb=off,mem-merge=off')
171 self._params.add('smp', '{smp},sockets=1,cores={smp},threads=1'.
172 format(smp=self._opt.get('smp')))
173 self._params.add('object',
174 'memory-backend-file,id=mem,size={mem}M,'
175 'mem-path=/mnt/huge,share=on'.
176 format(mem=self._opt.get('mem')))
177 self._params.add('m', '{mem}M'.
178 format(mem=self._opt.get('mem')))
179 self._params.add('numa', 'node,memdev=mem')
180 self._params.add('balloon', 'none')
182 def add_nestedvm_params(self):
183 """Set NestedVM QEMU parameters."""
184 self._params.add('net', 'nic,macaddr=52:54:00:00:{qemu:02x}:ff'.
185 format(qemu=self._opt.get('qemu_id')))
186 self._params.add('net', 'user,hostfwd=tcp::{info[port]}-:22'.
187 format(info=self._vm_info))
188 # TODO: Remove try except after fully migrated to Bionic or
189 # qemu_set_node is removed.
191 locking = ',file.locking=off'\
192 if self.qemu_version(version='2.10') else ''
193 except AttributeError:
195 self._params.add('drive',
196 'file={img},format=raw,cache=none,if=virtio{locking}'.
197 format(img=self._opt.get('img'), locking=locking))
198 self._params.add('qmp', 'unix:{qmp},server,nowait'.
199 format(qmp=self._temp.get('qmp')))
200 self._params.add('chardev', 'socket,host=127.0.0.1,port={info[serial]},'
201 'id=gnc0,server,nowait'.format(info=self._vm_info))
202 self._params.add('device', 'isa-serial,chardev=gnc0')
203 self._params.add('chardev',
204 'socket,path={qga},server,nowait,id=qga0'.
205 format(qga=self._temp.get('qga')))
206 self._params.add('device', 'isa-serial,chardev=qga0')
208 def qemu_set_node(self, node):
209 """Set node to run QEMU on.
211 :param node: Node to run QEMU on.
215 self._vm_info['host'] = node['host']
216 if node['port'] != 22:
217 self._vm_info['host_port'] = node['port']
218 self._vm_info['host_username'] = node['username']
219 self._vm_info['host_password'] = node['password']
221 def get_qemu_pids(self):
222 """Get QEMU CPU pids.
224 :returns: List of QEMU CPU pids.
227 command = ("grep -rwl 'CPU' /proc/$(sudo cat {pid})/task/*/comm ".
228 format(pid=self._temp.get('pid')))
229 command += (r"| xargs dirname | sed -e 's/\/.*\///g'")
231 stdout, _ = exec_cmd_no_error(self._node, command)
232 return stdout.splitlines()
234 def qemu_set_affinity(self, *host_cpus):
235 """Set qemu affinity by getting thread PIDs via QMP and taskset to list
238 :param host_cpus: List of CPU cores.
239 :type host_cpus: list
242 qemu_cpus = self.get_qemu_pids()
244 if len(qemu_cpus) != len(host_cpus):
245 raise ValueError('Host CPU count must match Qemu Thread count!')
247 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
248 command = ('taskset -pc {host_cpu} {thread}'.
249 format(host_cpu=host_cpu, thread=qemu_cpu))
250 message = ('QEMU: Set affinity failed on {host}!'.
251 format(host=self._node['host']))
252 exec_cmd_no_error(self._node, command, sudo=True,
254 except (RuntimeError, ValueError):
258 def qemu_set_scheduler_policy(self):
259 """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
262 :raises RuntimeError: Set scheduler policy failed.
265 qemu_cpus = self.get_qemu_pids()
267 for qemu_cpu in qemu_cpus:
268 command = ('chrt -r -p 1 {thread}'.
269 format(thread=qemu_cpu))
270 message = ('QEMU: Set SCHED_RR failed on {host}'.
271 format(host=self._node['host']))
272 exec_cmd_no_error(self._node, command, sudo=True,
274 except (RuntimeError, ValueError):
278 def qemu_add_vhost_user_if(self, socket, server=True, jumbo_frames=False,
279 queue_size=None, queues=1):
280 """Add Vhost-user interface.
282 :param socket: Path of the unix socket.
283 :param server: If True the socket shall be a listening socket.
284 :param jumbo_frames: Set True if jumbo frames are used in the test.
285 :param queue_size: Vring queue size.
286 :param queues: Number of queues.
289 :type jumbo_frames: bool
290 :type queue_size: int
294 self._params.add('chardev',
295 'socket,id=char{vhost},path={socket}{server}'.
296 format(vhost=self._vhost_id, socket=socket,
297 server=',server' if server is True else ''))
298 self._params.add('netdev',
299 'vhost-user,id=vhost{vhost},'
300 'chardev=char{vhost},queues={queues}'.
301 format(vhost=self._vhost_id, queues=queues))
302 mac = ('52:54:00:00:{qemu:02x}:{vhost:02x}'.
303 format(qemu=self._opt.get('qemu_id'), vhost=self._vhost_id))
304 queue_size = ('rx_queue_size={queue_size},tx_queue_size={queue_size}'.
305 format(queue_size=queue_size)) if queue_size else ''
306 mbuf = 'on,host_mtu=9200'
307 self._params.add('device',
308 'virtio-net-pci,netdev=vhost{vhost},'
309 'mac={mac},mq=on,vectors={vectors},csum=off,gso=off,'
310 'guest_tso4=off,guest_tso6=off,guest_ecn=off,'
311 'mrg_rxbuf={mbuf},{queue_size}'.
312 format(vhost=self._vhost_id, mac=mac,
313 mbuf=mbuf if jumbo_frames else 'off',
314 queue_size=queue_size,
315 vectors=(2 * queues + 2)))
317 # Add interface MAC and socket to the node dict.
318 if_data = {'mac_address': mac, 'socket': socket}
319 if_name = 'vhost{vhost}'.format(vhost=self._vhost_id)
320 self._vm_info['interfaces'][if_name] = if_data
321 # Add socket to temporary file list.
322 self._temp.add(if_name, socket)
324 def _qemu_qmp_exec(self, cmd):
325 """Execute QMP command.
327 QMP is JSON based protocol which allows to control QEMU instance.
329 :param cmd: QMP command to execute.
331 :returns: Command output in python representation of JSON format. The
332 { "return": {} } response is QMP's success response. An error
333 response will contain the "error" keyword instead of "return".
335 # To enter command mode, the qmp_capabilities command must be issued.
336 command = ('echo "{{ \\"execute\\": \\"qmp_capabilities\\" }}'
337 '{{ \\"execute\\": \\"{cmd}\\" }}" | '
338 'sudo -S socat - UNIX-CONNECT:{qmp}'.
339 format(cmd=cmd, qmp=self._temp.get('qmp')))
340 message = ('QMP execute "{cmd}" failed on {host}'.
341 format(cmd=cmd, host=self._node['host']))
342 stdout, _ = exec_cmd_no_error(self._node, command, sudo=False,
345 # Skip capabilities negotiation messages.
346 out_list = stdout.splitlines()
347 if len(out_list) < 3:
348 raise RuntimeError('Invalid QMP output on {host}'.
349 format(host=self._node['host']))
350 return json.loads(out_list[2])
352 def _qemu_qga_flush(self):
353 """Flush the QGA parser state."""
354 command = ('(printf "\xFF"; sleep 1) | '
355 'sudo -S socat - UNIX-CONNECT:{qga}'.
356 format(qga=self._temp.get('qga')))
357 message = ('QGA flush failed on {host}'.format(host=self._node['host']))
358 stdout, _ = exec_cmd_no_error(self._node, command, sudo=False,
361 return json.loads(stdout.split('\n', 1)[0]) if stdout else dict()
363 def _qemu_qga_exec(self, cmd):
364 """Execute QGA command.
366 QGA provide access to a system-level agent via standard QMP commands.
368 :param cmd: QGA command to execute.
371 command = ('(echo "{{ \\"execute\\": \\"{cmd}\\" }}"; sleep 1) | '
372 'sudo -S socat - UNIX-CONNECT:{qga}'.
373 format(cmd=cmd, qga=self._temp.get('qga')))
374 message = ('QGA execute "{cmd}" failed on {host}'.
375 format(cmd=cmd, host=self._node['host']))
376 stdout, _ = exec_cmd_no_error(self._node, command, sudo=False,
379 return json.loads(stdout.split('\n', 1)[0]) if stdout else dict()
381 def _wait_until_vm_boot(self, timeout=60):
382 """Wait until QEMU VM is booted.
384 First try to flush qga until there is output.
385 Then ping QEMU guest agent each 5s until VM booted or timeout.
387 :param timeout: Waiting timeout in seconds (optional, default 60s).
392 if time() - start > timeout:
393 raise RuntimeError('timeout, VM not booted on {host}'.
394 format(host=self._node['host']))
397 out = self._qemu_qga_flush()
399 logger.trace('QGA qga flush unexpected output {out}'.
401 # Empty output - VM not booted yet
407 if time() - start > timeout:
408 raise RuntimeError('timeout, VM not booted on {host}'.
409 format(host=self._node['host']))
412 out = self._qemu_qga_exec('guest-ping')
414 logger.trace('QGA guest-ping unexpected output {out}'.
416 # Empty output - VM not booted yet.
419 # Non-error return - VM booted.
420 elif out.get('return') is not None:
422 # Skip error and wait.
423 elif out.get('error') is not None:
426 # If there is an unexpected output from QGA guest-info, try
427 # again until timeout.
428 logger.trace('QGA guest-ping unexpected output {out}'.
431 logger.trace('VM booted on {host}'.format(host=self._node['host']))
433 def _update_vm_interfaces(self):
434 """Update interface names in VM node dict."""
435 # Send guest-network-get-interfaces command via QGA, output example:
436 # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
437 # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
438 out = self._qemu_qga_exec('guest-network-get-interfaces')
439 interfaces = out.get('return')
442 raise RuntimeError('Get VM interface list failed on {host}'.
443 format(host=self._node['host']))
444 # Create MAC-name dict.
445 for interface in interfaces:
446 if 'hardware-address' not in interface:
448 mac_name[interface['hardware-address']] = interface['name']
449 # Match interface by MAC and save interface name.
450 for interface in self._vm_info['interfaces'].values():
451 mac = interface.get('mac_address')
452 if_name = mac_name.get(mac)
454 logger.trace('Interface name for MAC {mac} not found'.
457 interface['name'] = if_name
459 def qemu_start(self):
460 """Start QEMU and wait until VM boot.
462 :returns: VM node info.
465 DUTSetup.check_huge_page(self._node, '/mnt/huge', self._opt.get('mem'))
467 command = ('{bin_path}/qemu-system-{arch} {params}'.
468 format(bin_path=self._opt.get('bin_path'),
469 arch=Topology.get_node_arch(self._node),
470 params=self._params))
471 message = ('QEMU: Start failed on {host}!'.
472 format(host=self._node['host']))
475 exec_cmd_no_error(self._node, command, timeout=300, sudo=True,
477 self._wait_until_vm_boot()
478 # Update interface names in VM node dict.
479 self._update_vm_interfaces()
486 """Kill qemu process."""
487 exec_cmd(self._node, 'chmod +r {pid}'.
488 format(pid=self._temp.get('pid')), sudo=True)
489 exec_cmd(self._node, 'kill -SIGKILL $(cat {pid})'.
490 format(pid=self._temp.get('pid')), sudo=True)
492 for value in self._temp.get_values():
493 exec_cmd(self._node, 'rm -f {value}'.format(value=value), sudo=True)
495 def qemu_kill_all(self, node=None):
496 """Kill all qemu processes on DUT node if specified.
498 :param node: Node to kill all QEMU processes on.
502 self.qemu_set_node(node)
503 exec_cmd(self._node, 'pkill -SIGKILL qemu', sudo=True)
505 for value in self._temp.get_values():
506 exec_cmd(self._node, 'rm -f {value}'.format(value=value), sudo=True)
508 def qemu_version(self, version=None):
509 """Return Qemu version or compare if version is higher than parameter.
511 :param version: Version to compare.
513 :returns: Qemu version or Boolean if version is higher than parameter.
516 command = ('{bin_path}/qemu-system-{arch} --version'.
517 format(bin_path=self._opt.get('bin_path'),
518 arch=Topology.get_node_arch(self._node)))
520 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
521 ver = match(r'QEMU emulator version ([\d.]*)', stdout).group(1)
522 return StrictVersion(ver) > StrictVersion(version) \
529 def build_qemu(node, force_install=False, apply_patch=False):
530 """Build QEMU from sources.
532 :param node: Node to build QEMU on.
533 :param force_install: If True, then remove previous build.
534 :param apply_patch: If True, then apply patches from qemu_patches dir.
536 :type force_install: bool
537 :type apply_patch: bool
538 :raises RuntimeError: If building QEMU failed.
540 directory = (' --directory={install_dir}{patch}'.
541 format(install_dir=Constants.QEMU_INSTALL_DIR,
542 patch='-patch' if apply_patch else '-base'))
543 version = (' --version={install_version}'.
544 format(install_version=Constants.QEMU_INSTALL_VERSION))
545 force = ' --force' if force_install else ''
546 patch = ' --patch' if apply_patch else ''
547 target_list = (' --target-list={arch}-softmmu'.
548 format(arch=Topology.get_node_arch(node)))
550 command = ("sudo -E sh -c "
551 "'{fw_dir}/{lib_sh}/qemu_build.sh{version}{directory}"
552 "{force}{patch}{target_list}'".
553 format(fw_dir=Constants.REMOTE_FW_DIR,
554 lib_sh=Constants.RESOURCES_LIB_SH,
555 version=version, directory=directory, force=force,
556 patch=patch, target_list=target_list))
557 message = ('QEMU: Build failed on {host}!'.format(host=node['host']))
558 exec_cmd_no_error(node, command, sudo=False, message=message,