1 # Copyright (c) 2016 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """QEMU utilities library."""
16 from time import time, sleep
19 from robot.api import logger
21 from resources.libraries.python.ssh import SSH, SSHTimeout
22 from resources.libraries.python.constants import Constants
23 from resources.libraries.python.topology import NodeType, Topology
26 class QemuUtils(object):
29 def __init__(self, qemu_id=1):
30 self._qemu_id = qemu_id
31 # Path to QEMU binary. Use x86_64 by default
32 self._qemu_path = '/usr/bin/'
33 self._qemu_bin = 'qemu-system-x86_64'
34 # QEMU Machine Protocol socket
35 self._qmp_sock = '/tmp/qmp{0}.sock'.format(self._qemu_id)
36 # QEMU Guest Agent socket
37 self._qga_sock = '/tmp/qga{0}.sock'.format(self._qemu_id)
39 self._pid_file = '/tmp/qemu{0}.pid'.format(self._qemu_id)
42 self._qemu_opt['smp'] = '-smp 1,sockets=1,cores=1,threads=1'
43 # Daemonize the QEMU process after initialization. Default one
44 # management interface.
45 self._qemu_opt['options'] = '-cpu host -daemonize -enable-kvm ' \
46 '-machine pc,accel=kvm,usb=off,mem-merge=off ' \
47 '-net nic,macaddr=52:54:00:00:{0:02x}:ff -balloon none'\
48 .format(self._qemu_id)
49 self._qemu_opt['ssh_fwd_port'] = 10021 + qemu_id
50 # Default serial console port
51 self._qemu_opt['serial_port'] = 4555 + qemu_id
52 # Default 512MB virtual RAM
53 self._qemu_opt['mem_size'] = 512
54 # Default huge page mount point, required for Vhost-user interfaces.
55 self._qemu_opt['huge_mnt'] = '/mnt/huge'
56 # Default do not allocate huge pages.
57 self._qemu_opt['huge_allocate'] = False
58 # Default image for CSIT virl setup
59 self._qemu_opt['disk_image'] = '/var/lib/vm/vhost-nested.img'
63 'port': self._qemu_opt['ssh_fwd_port'],
69 self._qemu_opt['queues'] = 1
73 self._socks = [self._qmp_sock, self._qga_sock]
75 def qemu_set_path(self, path):
76 """Set binary path for QEMU.
78 :param path: Absolute path in filesystem.
81 self._qemu_path = path
83 def qemu_set_smp(self, cpus, cores, threads, sockets):
84 """Set SMP option for QEMU.
86 :param cpus: Number of CPUs.
87 :param cores: Number of CPU cores on one socket.
88 :param threads: Number of threads on one CPU core.
89 :param sockets: Number of discrete sockets in the system.
95 self._qemu_opt['smp'] = '-smp {},cores={},threads={},sockets={}'.format(
96 cpus, cores, threads, sockets)
98 def qemu_set_ssh_fwd_port(self, fwd_port):
99 """Set host port for guest SSH forwarding.
101 :param fwd_port: Port number on host for guest SSH forwarding.
104 self._qemu_opt['ssh_fwd_port'] = fwd_port
105 self._vm_info['port'] = fwd_port
107 def qemu_set_serial_port(self, port):
108 """Set serial console port.
110 :param port: Serial console port.
113 self._qemu_opt['serial_port'] = port
115 def qemu_set_mem_size(self, mem_size):
116 """Set virtual RAM size.
118 :param mem_size: RAM size in Mega Bytes.
121 self._qemu_opt['mem_size'] = int(mem_size)
123 def qemu_set_huge_mnt(self, huge_mnt):
124 """Set hugefile mount point.
126 :param huge_mnt: System hugefile mount point.
129 self._qemu_opt['huge_mnt'] = huge_mnt
131 def qemu_set_huge_allocate(self):
132 """Set flag to allocate more huge pages if needed."""
133 self._qemu_opt['huge_allocate'] = True
135 def qemu_set_disk_image(self, disk_image):
138 :param disk_image: Path of the disk image.
139 :type disk_image: str
141 self._qemu_opt['disk_image'] = disk_image
143 def qemu_set_affinity(self, *host_cpus):
144 """Set qemu affinity by getting thread PIDs via QMP and taskset to list
147 :param host_cpus: List of CPU cores.
148 :type host_cpus: list
150 qemu_cpus = self._qemu_qmp_exec('query-cpus')['return']
152 if len(qemu_cpus) != len(host_cpus):
153 logger.debug('Host CPU count {0}, Qemu Thread count {1}'.format(
154 len(host_cpus), len(qemu_cpus)))
155 raise ValueError('Host CPU count must match Qemu Thread count')
157 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
158 cmd = 'taskset -pc {0} {1}'.format(host_cpu, qemu_cpu['thread_id'])
159 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
160 if int(ret_code) != 0:
161 logger.debug('Set affinity failed {0}'.format(stderr))
162 raise RuntimeError('Set affinity failed on {0}'.format(
165 def qemu_set_scheduler_policy(self):
166 """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
169 :raises RuntimeError: Set scheduler policy failed.
171 qemu_cpus = self._qemu_qmp_exec('query-cpus')['return']
173 for qemu_cpu in qemu_cpus:
174 cmd = 'chrt -r -p 1 {0}'.format(qemu_cpu['thread_id'])
175 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
176 if int(ret_code) != 0:
177 logger.debug('Set SCHED_RR failed {0}'.format(stderr))
178 raise RuntimeError('Set SCHED_RR failed on {0}'.format(
181 def qemu_set_node(self, node):
182 """Set node to run QEMU on.
184 :param node: Node to run QEMU on.
189 self._ssh.connect(node)
190 self._vm_info['host'] = node['host']
192 arch = Topology.get_node_arch(node)
193 self._qemu_bin = 'qemu-system-{arch}'.format(arch=arch)
195 def qemu_add_vhost_user_if(self, socket, server=True, mac=None,
197 """Add Vhost-user interface.
199 :param socket: Path of the unix socket.
200 :param server: If True the socket shall be a listening socket.
201 :param mac: Vhost-user interface MAC address (optional, otherwise is
202 used auto-generated MAC 52:54:00:00:xx:yy).
203 :param jumbo_frames: Set True if jumbo frames are used in the test.
207 :type jumbo_frames: bool
210 # Create unix socket character device.
211 chardev = ' -chardev socket,id=char{0},path={1}'.format(self._vhost_id,
215 self._qemu_opt['options'] += chardev
216 # Create Vhost-user network backend.
217 netdev = (' -netdev vhost-user,id=vhost{0},chardev=char{0},queues={1}'
218 .format(self._vhost_id, self._qemu_opt['queues']))
219 self._qemu_opt['options'] += netdev
220 # If MAC is not specified use auto-generated MAC address based on
221 # template 52:54:00:00:<qemu_id>:<vhost_id>, e.g. vhost1 MAC of QEMU
222 # with ID 1 is 52:54:00:00:01:01
224 mac = '52:54:00:00:{0:02x}:{1:02x}'.\
225 format(self._qemu_id, self._vhost_id)
226 extend_options = 'mq=on,csum=off,gso=off,guest_tso4=off,'\
227 'guest_tso6=off,guest_ecn=off'
229 extend_options += ",mrg_rxbuf=on"
231 extend_options += ",mrg_rxbuf=off"
232 # Create Virtio network device.
233 device = ' -device virtio-net-pci,netdev=vhost{0},mac={1},{2}'.format(
234 self._vhost_id, mac, extend_options)
235 self._qemu_opt['options'] += device
236 # Add interface MAC and socket to the node dict
237 if_data = {'mac_address': mac, 'socket': socket}
238 if_name = 'vhost{}'.format(self._vhost_id)
239 self._vm_info['interfaces'][if_name] = if_data
240 # Add socket to the socket list
241 self._socks.append(socket)
243 def _qemu_qmp_exec(self, cmd):
244 """Execute QMP command.
246 QMP is JSON based protocol which allows to control QEMU instance.
248 :param cmd: QMP command to execute.
250 :return: Command output in python representation of JSON format. The
251 { "return": {} } response is QMP's success response. An error
252 response will contain the "error" keyword instead of "return".
254 # To enter command mode, the qmp_capabilities command must be issued.
255 qmp_cmd = 'echo "{ \\"execute\\": \\"qmp_capabilities\\" }' \
256 '{ \\"execute\\": \\"' + cmd + \
257 '\\" }" | sudo -S socat - UNIX-CONNECT:' + self._qmp_sock
259 (ret_code, stdout, stderr) = self._ssh.exec_command(qmp_cmd)
260 if int(ret_code) != 0:
261 logger.debug('QMP execute failed {0}'.format(stderr))
262 raise RuntimeError('QMP execute "{0}"'
263 ' failed on {1}'.format(cmd, self._node['host']))
265 # Skip capabilities negotiation messages.
266 out_list = stdout.splitlines()
267 if len(out_list) < 3:
268 raise RuntimeError('Invalid QMP output on {0}'.format(
270 return json.loads(out_list[2])
272 def _qemu_qga_flush(self):
273 """Flush the QGA parser state
275 qga_cmd = '(printf "\xFF"; sleep 1) | sudo -S socat - UNIX-CONNECT:' + \
277 #TODO: probably need something else
278 (ret_code, stdout, stderr) = self._ssh.exec_command(qga_cmd)
279 if int(ret_code) != 0:
280 logger.debug('QGA execute failed {0}'.format(stderr))
281 raise RuntimeError('QGA execute "{0}" '
282 'failed on {1}'.format(qga_cmd,
287 return json.loads(stdout.split('\n', 1)[0])
289 def _qemu_qga_exec(self, cmd):
290 """Execute QGA command.
292 QGA provide access to a system-level agent via standard QMP commands.
294 :param cmd: QGA command to execute.
297 qga_cmd = '(echo "{ \\"execute\\": \\"' + \
299 '\\" }"; sleep 1) | sudo -S socat - UNIX-CONNECT:' + \
301 (ret_code, stdout, stderr) = self._ssh.exec_command(qga_cmd)
302 if int(ret_code) != 0:
303 logger.debug('QGA execute failed {0}'.format(stderr))
304 raise RuntimeError('QGA execute "{0}"'
305 ' failed on {1}'.format(cmd, self._node['host']))
309 return json.loads(stdout.split('\n', 1)[0])
311 def _wait_until_vm_boot(self, timeout=60):
312 """Wait until QEMU VM is booted.
314 First try to flush qga until there is output.
315 Then ping QEMU guest agent each 5s until VM booted or timeout.
317 :param timeout: Waiting timeout in seconds (optional, default 60s).
322 if time() - start > timeout:
323 raise RuntimeError('timeout, VM {0} not booted on {1}'.format(
324 self._qemu_opt['disk_image'], self._node['host']))
327 out = self._qemu_qga_flush()
329 logger.trace('QGA qga flush unexpected output {}'.format(out))
330 # Empty output - VM not booted yet
336 if time() - start > timeout:
337 raise RuntimeError('timeout, VM {0} not booted on {1}'.format(
338 self._qemu_opt['disk_image'], self._node['host']))
341 out = self._qemu_qga_exec('guest-ping')
343 logger.trace('QGA guest-ping unexpected output {}'.format(out))
344 # Empty output - VM not booted yet
347 # Non-error return - VM booted
348 elif out.get('return') is not None:
350 # Skip error and wait
351 elif out.get('error') is not None:
354 # If there is an unexpected output from QGA guest-info, try
355 # again until timeout.
356 logger.trace('QGA guest-ping unexpected output {}'.format(out))
358 logger.trace('VM {0} booted on {1}'.format(self._qemu_opt['disk_image'],
361 def _update_vm_interfaces(self):
362 """Update interface names in VM node dict."""
363 # Send guest-network-get-interfaces command via QGA, output example:
364 # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
365 # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}
366 out = self._qemu_qga_exec('guest-network-get-interfaces')
367 interfaces = out.get('return')
370 raise RuntimeError('Get VM {0} interface list failed on {1}'.format(
371 self._qemu_opt['disk_image'], self._node['host']))
372 # Create MAC-name dict
373 for interface in interfaces:
374 if 'hardware-address' not in interface:
376 mac_name[interface['hardware-address']] = interface['name']
377 # Match interface by MAC and save interface name
378 for interface in self._vm_info['interfaces'].values():
379 mac = interface.get('mac_address')
380 if_name = mac_name.get(mac)
382 logger.trace('Interface name for MAC {} not found'.format(mac))
384 interface['name'] = if_name
386 def _huge_page_check(self, allocate=False):
387 """Huge page check."""
388 huge_mnt = self._qemu_opt.get('huge_mnt')
389 mem_size = self._qemu_opt.get('mem_size')
391 # Get huge pages information
392 huge_size = self._get_huge_page_size()
393 huge_free = self._get_huge_page_free(huge_size)
394 huge_total = self._get_huge_page_total(huge_size)
396 # Check if memory reqested by qemu is available on host
397 if (mem_size * 1024) > (huge_free * huge_size):
398 # If we want to allocate hugepage dynamically
400 mem_needed = abs((huge_free * huge_size) - (mem_size * 1024))
401 huge_to_allocate = ((mem_needed / huge_size) * 2) + huge_total
402 max_map_count = huge_to_allocate*4
403 # Increase maximum number of memory map areas a process may have
404 cmd = 'echo "{0}" | sudo tee /proc/sys/vm/max_map_count'.format(
406 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
407 # Increase hugepage count
408 cmd = 'echo "{0}" | sudo tee /proc/sys/vm/nr_hugepages'.format(
410 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
411 if int(ret_code) != 0:
412 logger.debug('Mount huge pages failed {0}'.format(stderr))
413 raise RuntimeError('Mount huge pages failed on {0}'.format(
415 # If we do not want to allocate dynamicaly end with error
418 'Not enough free huge pages: {0}, '
419 '{1} MB'.format(huge_free, huge_free * huge_size)
421 # Check if huge pages mount point exist
423 (_, output, _) = self._ssh.exec_command('cat /proc/mounts')
424 for line in output.splitlines():
425 # Try to find something like:
426 # none /mnt/huge hugetlbfs rw,relatime,pagesize=2048k 0 0
428 if mount[2] == 'hugetlbfs' and mount[1] == huge_mnt:
431 # If huge page mount point not exist create one
433 cmd = 'mkdir -p {0}'.format(huge_mnt)
434 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
435 if int(ret_code) != 0:
436 logger.debug('Create mount dir failed: {0}'.format(stderr))
437 raise RuntimeError('Create mount dir failed on {0}'.format(
439 cmd = 'mount -t hugetlbfs -o pagesize=2048k none {0}'.format(
441 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
442 if int(ret_code) != 0:
443 logger.debug('Mount huge pages failed {0}'.format(stderr))
444 raise RuntimeError('Mount huge pages failed on {0}'.format(
447 def _get_huge_page_size(self):
448 """Get default size of huge pages in system.
450 :returns: Default size of free huge pages in system.
452 :raises: RuntimeError if reading failed for three times.
454 # TODO: remove to dedicated library
455 cmd_huge_size = "grep Hugepagesize /proc/meminfo | awk '{ print $2 }'"
457 (ret, out, _) = self._ssh.exec_command_sudo(cmd_huge_size)
462 logger.trace('Reading huge page size information failed')
466 raise RuntimeError('Getting huge page size information failed.')
469 def _get_huge_page_free(self, huge_size):
470 """Get total number of huge pages in system.
472 :param huge_size: Size of hugepages.
474 :returns: Number of free huge pages in system.
476 :raises: RuntimeError if reading failed for three times.
478 # TODO: add numa aware option
479 # TODO: remove to dedicated library
480 cmd_huge_free = 'cat /sys/kernel/mm/hugepages/hugepages-{0}kB/'\
481 'free_hugepages'.format(huge_size)
483 (ret, out, _) = self._ssh.exec_command_sudo(cmd_huge_free)
488 logger.trace('Reading free huge pages information failed')
492 raise RuntimeError('Getting free huge pages information failed.')
495 def _get_huge_page_total(self, huge_size):
496 """Get total number of huge pages in system.
498 :param huge_size: Size of hugepages.
500 :returns: Total number of huge pages in system.
502 :raises: RuntimeError if reading failed for three times.
504 # TODO: add numa aware option
505 # TODO: remove to dedicated library
506 cmd_huge_total = 'cat /sys/kernel/mm/hugepages/hugepages-{0}kB/'\
507 'nr_hugepages'.format(huge_size)
509 (ret, out, _) = self._ssh.exec_command_sudo(cmd_huge_total)
512 huge_total = int(out)
514 logger.trace('Reading total huge pages information failed')
518 raise RuntimeError('Getting total huge pages information failed.')
521 def qemu_start(self):
522 """Start QEMU and wait until VM boot.
524 :return: VM node info.
526 .. note:: First set at least node to run QEMU on.
527 .. warning:: Starts only one VM on the node.
530 bin_path = '{0}{1}'.format(self._qemu_path, self._qemu_bin)
533 ssh_fwd = '-net user,hostfwd=tcp::{0}-:22'.format(
534 self._qemu_opt.get('ssh_fwd_port'))
535 # Memory and huge pages
536 mem = '-object memory-backend-file,id=mem,size={0}M,mem-path={1},' \
537 'share=on -m {0} -numa node,memdev=mem'.format(
538 self._qemu_opt.get('mem_size'), self._qemu_opt.get('huge_mnt'))
540 # By default check only if hugepages are available.
541 # If 'huge_allocate' is set to true try to allocate as well.
542 self._huge_page_check(allocate=self._qemu_opt.get('huge_allocate'))
545 drive = '-drive file={0},format=raw,cache=none,if=virtio'.format(
546 self._qemu_opt.get('disk_image'))
547 # Setup QMP via unix socket
548 qmp = '-qmp unix:{0},server,nowait'.format(self._qmp_sock)
549 # Setup serial console
550 serial = '-chardev socket,host=127.0.0.1,port={0},id=gnc0,server,' \
551 'nowait -device isa-serial,chardev=gnc0'.format(
552 self._qemu_opt.get('serial_port'))
553 # Setup QGA via chardev (unix socket) and isa-serial channel
554 qga = '-chardev socket,path={0},server,nowait,id=qga0 ' \
555 '-device isa-serial,chardev=qga0'.format(self._qga_sock)
557 graphic = '-monitor none -display none -vga none'
559 pid = '-pidfile {}'.format(self._pid_file)
562 cmd = '{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}'.format(bin_path,
563 self._qemu_opt.get('smp'), mem, ssh_fwd,
564 self._qemu_opt.get('options'), drive, qmp, serial, qga, graphic,
567 (ret_code, _, _) = self._ssh.exec_command_sudo(cmd, timeout=300)
568 if int(ret_code) != 0:
569 raise RuntimeError('QEMU start failed on {0}'.format(
572 self._wait_until_vm_boot()
573 except (RuntimeError, SSHTimeout):
575 self.qemu_clear_socks()
577 logger.trace('QEMU started successfully.')
578 # Update interface names in VM node dict
579 self._update_vm_interfaces()
580 # Return VM node dict
584 """Quit the QEMU emulator."""
585 out = self._qemu_qmp_exec('quit')
586 err = out.get('error')
588 raise RuntimeError('QEMU quit failed on {0}, error: {1}'.format(
589 self._node['host'], json.dumps(err)))
591 def qemu_system_powerdown(self):
592 """Power down the system (if supported)."""
593 out = self._qemu_qmp_exec('system_powerdown')
594 err = out.get('error')
597 'QEMU system powerdown failed on {0}, '
598 'error: {1}'.format(self._node['host'], json.dumps(err))
601 def qemu_system_reset(self):
602 """Reset the system."""
603 out = self._qemu_qmp_exec('system_reset')
604 err = out.get('error')
607 'QEMU system reset failed on {0}, '
608 'error: {1}'.format(self._node['host'], json.dumps(err)))
611 """Kill qemu process."""
612 # Note: in QEMU start phase there are 3 QEMU processes because we
614 self._ssh.exec_command_sudo('chmod +r {}'.format(self._pid_file))
615 self._ssh.exec_command_sudo('kill -SIGKILL $(cat {})'
616 .format(self._pid_file))
618 cmd = 'rm -f {}'.format(self._pid_file)
619 self._ssh.exec_command_sudo(cmd)
621 def qemu_kill_all(self, node=None):
622 """Kill all qemu processes on DUT node if specified.
624 :param node: Node to kill all QEMU processes on.
628 self.qemu_set_node(node)
629 self._ssh.exec_command_sudo('pkill -SIGKILL qemu')
631 def qemu_clear_socks(self):
632 """Remove all sockets created by QEMU."""
633 # If serial console port still open kill process
634 cmd = 'fuser -k {}/tcp'.format(self._qemu_opt.get('serial_port'))
635 self._ssh.exec_command_sudo(cmd)
636 # Delete all created sockets
637 for sock in self._socks:
638 cmd = 'rm -f {}'.format(sock)
639 self._ssh.exec_command_sudo(cmd)
641 def qemu_system_status(self):
642 """Return current VM status.
644 VM should be in following status:
646 - debug: QEMU running on a debugger
647 - finish-migrate: paused to finish the migration process
648 - inmigrate: waiting for an incoming migration
649 - internal-error: internal error has occurred
650 - io-error: the last IOP has failed
652 - postmigrate: paused following a successful migrate
653 - prelaunch: QEMU was started with -S and guest has not started
654 - restore-vm: paused to restore VM state
655 - running: actively running
656 - save-vm: paused to save the VM state
657 - shutdown: shut down (and -no-shutdown is in use)
658 - suspended: suspended (ACPI S3)
659 - watchdog: watchdog action has been triggered
660 - guest-panicked: panicked as a result of guest OS panic
665 out = self._qemu_qmp_exec('query-status')
666 ret = out.get('return')
668 return ret.get('status')
670 err = out.get('error')
672 'QEMU query-status failed on {0}, '
673 'error: {1}'.format(self._node['host'], json.dumps(err)))
676 def build_qemu(node, force_install=False, apply_patch=False):
677 """Build QEMU from sources.
679 :param node: Node to build QEMU on.
680 :param force_install: If True, then remove previous build.
681 :param apply_patch: If True, then apply patches from qemu_patches dir.
683 :type force_install: bool
684 :type apply_patch: bool
685 :raises: RuntimeError if building QEMU failed.
690 directory = ' --directory={0}'.format(Constants.QEMU_INSTALL_DIR)
692 directory += '-patch'
695 version = ' --version={0}'.format(Constants.QEMU_INSTALL_VERSION)
696 force = ' --force' if force_install else ''
697 patch = ' --patch' if apply_patch else ''
698 arch = Topology.get_node_arch(node)
699 target_list = ' --target-list={0}-softmmu'.format(arch)
701 (ret_code, stdout, stderr) = \
703 "sudo -E sh -c '{0}/{1}/qemu_build.sh{2}{3}{4}{5}{6}'"\
704 .format(Constants.REMOTE_FW_DIR, Constants.RESOURCES_LIB_SH,
705 version, directory, force, patch, target_list), 1000)
707 if int(ret_code) != 0:
708 logger.debug('QEMU build failed {0}'.format(stdout + stderr))
709 raise RuntimeError('QEMU build failed on {0}'.format(node['host']))