1 # Copyright (c) 2016 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """QEMU utilities library."""
16 from time import time, sleep
19 from robot.api import logger
21 from resources.libraries.python.ssh import SSH, SSHTimeout
22 from resources.libraries.python.constants import Constants
23 from resources.libraries.python.topology import NodeType, Topology
26 class QemuUtils(object):
29 def __init__(self, qemu_id=1):
30 self._qemu_id = qemu_id
31 # Path to QEMU binary. Use x86_64 by default
32 self._qemu_path = '/usr/bin/'
33 self._qemu_bin = 'qemu-system-x86_64'
34 # QEMU Machine Protocol socket
35 self._qmp_sock = '/tmp/qmp{0}.sock'.format(self._qemu_id)
36 # QEMU Guest Agent socket
37 self._qga_sock = '/tmp/qga{0}.sock'.format(self._qemu_id)
39 self._pid_file = '/tmp/qemu{0}.pid'.format(self._qemu_id)
42 self._qemu_opt['smp'] = '-smp 1,sockets=1,cores=1,threads=1'
43 # Daemonize the QEMU process after initialization. Default one
44 # management interface.
45 self._qemu_opt['options'] = '-cpu host -daemonize -enable-kvm ' \
46 '-machine pc,accel=kvm,usb=off,mem-merge=off ' \
47 '-net nic,macaddr=52:54:00:00:{0:02x}:ff -balloon none'\
48 .format(self._qemu_id)
49 self._qemu_opt['ssh_fwd_port'] = 10021 + qemu_id
50 # Default serial console port
51 self._qemu_opt['serial_port'] = 4555 + qemu_id
52 # Default 512MB virtual RAM
53 self._qemu_opt['mem_size'] = 512
54 # Default huge page mount point, required for Vhost-user interfaces.
55 self._qemu_opt['huge_mnt'] = '/mnt/huge'
56 # Default do not allocate huge pages.
57 self._qemu_opt['huge_allocate'] = False
58 # Default image for CSIT virl setup
59 self._qemu_opt['disk_image'] = '/var/lib/vm/vhost-nested.img'
63 'port': self._qemu_opt['ssh_fwd_port'],
69 self._qemu_opt['queues'] = 1
73 self._socks = [self._qmp_sock, self._qga_sock]
75 def qemu_set_path(self, path):
76 """Set binary path for QEMU.
78 :param path: Absolute path in filesystem.
81 self._qemu_path = path
83 def qemu_set_smp(self, cpus, cores, threads, sockets):
84 """Set SMP option for QEMU.
86 :param cpus: Number of CPUs.
87 :param cores: Number of CPU cores on one socket.
88 :param threads: Number of threads on one CPU core.
89 :param sockets: Number of discrete sockets in the system.
95 self._qemu_opt['smp'] = '-smp {},cores={},threads={},sockets={}'.format(
96 cpus, cores, threads, sockets)
98 def qemu_set_ssh_fwd_port(self, fwd_port):
99 """Set host port for guest SSH forwarding.
101 :param fwd_port: Port number on host for guest SSH forwarding.
104 self._qemu_opt['ssh_fwd_port'] = fwd_port
105 self._vm_info['port'] = fwd_port
107 def qemu_set_serial_port(self, port):
108 """Set serial console port.
110 :param port: Serial console port.
113 self._qemu_opt['serial_port'] = port
115 def qemu_set_mem_size(self, mem_size):
116 """Set virtual RAM size.
118 :param mem_size: RAM size in Mega Bytes.
121 self._qemu_opt['mem_size'] = int(mem_size)
123 def qemu_set_huge_mnt(self, huge_mnt):
124 """Set hugefile mount point.
126 :param huge_mnt: System hugefile mount point.
129 self._qemu_opt['huge_mnt'] = huge_mnt
131 def qemu_set_huge_allocate(self):
132 """Set flag to allocate more huge pages if needed."""
133 self._qemu_opt['huge_allocate'] = True
135 def qemu_set_disk_image(self, disk_image):
138 :param disk_image: Path of the disk image.
139 :type disk_image: str
141 self._qemu_opt['disk_image'] = disk_image
143 def qemu_set_affinity(self, *host_cpus):
144 """Set qemu affinity by getting thread PIDs via QMP and taskset to list
147 :param host_cpus: List of CPU cores.
148 :type host_cpus: list
150 qemu_cpus = self._qemu_qmp_exec('query-cpus')['return']
152 if len(qemu_cpus) != len(host_cpus):
153 logger.debug('Host CPU count {0}, Qemu Thread count {1}'.format(
154 len(host_cpus), len(qemu_cpus)))
155 raise ValueError('Host CPU count must match Qemu Thread count')
157 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
158 cmd = 'taskset -pc {0} {1}'.format(host_cpu, qemu_cpu['thread_id'])
159 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
160 if int(ret_code) != 0:
161 logger.debug('Set affinity failed {0}'.format(stderr))
162 raise RuntimeError('Set affinity failed on {0}'.format(
165 def qemu_set_scheduler_policy(self):
166 """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
169 :raises RuntimeError: Set scheduler policy failed.
171 qemu_cpus = self._qemu_qmp_exec('query-cpus')['return']
173 for qemu_cpu in qemu_cpus:
174 cmd = 'chrt -r -p 1 {0}'.format(qemu_cpu['thread_id'])
175 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
176 if int(ret_code) != 0:
177 logger.debug('Set SCHED_RR failed {0}'.format(stderr))
178 raise RuntimeError('Set SCHED_RR failed on {0}'.format(
181 def qemu_set_node(self, node):
182 """Set node to run QEMU on.
184 :param node: Node to run QEMU on.
189 self._ssh.connect(node)
190 self._vm_info['host'] = node['host']
192 arch = Topology.get_node_arch(node)
193 self._qemu_bin = 'qemu-system-{arch}'.format(arch=arch)
195 def qemu_add_vhost_user_if(self, socket, server=True, mac=None):
196 """Add Vhost-user interface.
198 :param socket: Path of the unix socket.
199 :param server: If True the socket shall be a listening socket.
200 :param mac: Vhost-user interface MAC address (optional, otherwise is
201 used auto-generated MAC 52:54:00:00:xx:yy).
207 # Create unix socket character device.
208 chardev = ' -chardev socket,id=char{0},path={1}'.format(self._vhost_id,
212 self._qemu_opt['options'] += chardev
213 # Create Vhost-user network backend.
214 netdev = (' -netdev vhost-user,id=vhost{0},chardev=char{0},queues={1}'
215 .format(self._vhost_id, self._qemu_opt['queues']))
216 self._qemu_opt['options'] += netdev
217 # If MAC is not specified use auto-generated MAC address based on
218 # template 52:54:00:00:<qemu_id>:<vhost_id>, e.g. vhost1 MAC of QEMU
219 # with ID 1 is 52:54:00:00:01:01
221 mac = '52:54:00:00:{0:02x}:{1:02x}'.\
222 format(self._qemu_id, self._vhost_id)
223 extend_options = 'mq=on,csum=off,gso=off,guest_tso4=off,'\
224 'guest_tso6=off,guest_ecn=off,mrg_rxbuf=off'
225 # Create Virtio network device.
226 device = ' -device virtio-net-pci,netdev=vhost{0},mac={1},{2}'.format(
227 self._vhost_id, mac, extend_options)
228 self._qemu_opt['options'] += device
229 # Add interface MAC and socket to the node dict
230 if_data = {'mac_address': mac, 'socket': socket}
231 if_name = 'vhost{}'.format(self._vhost_id)
232 self._vm_info['interfaces'][if_name] = if_data
233 # Add socket to the socket list
234 self._socks.append(socket)
236 def _qemu_qmp_exec(self, cmd):
237 """Execute QMP command.
239 QMP is JSON based protocol which allows to control QEMU instance.
241 :param cmd: QMP command to execute.
243 :return: Command output in python representation of JSON format. The
244 { "return": {} } response is QMP's success response. An error
245 response will contain the "error" keyword instead of "return".
247 # To enter command mode, the qmp_capabilities command must be issued.
248 qmp_cmd = 'echo "{ \\"execute\\": \\"qmp_capabilities\\" }' \
249 '{ \\"execute\\": \\"' + cmd + \
250 '\\" }" | sudo -S socat - UNIX-CONNECT:' + self._qmp_sock
252 (ret_code, stdout, stderr) = self._ssh.exec_command(qmp_cmd)
253 if int(ret_code) != 0:
254 logger.debug('QMP execute failed {0}'.format(stderr))
255 raise RuntimeError('QMP execute "{0}"'
256 ' failed on {1}'.format(cmd, self._node['host']))
258 # Skip capabilities negotiation messages.
259 out_list = stdout.splitlines()
260 if len(out_list) < 3:
261 raise RuntimeError('Invalid QMP output on {0}'.format(
263 return json.loads(out_list[2])
265 def _qemu_qga_flush(self):
266 """Flush the QGA parser state
268 qga_cmd = '(printf "\xFF"; sleep 1) | sudo -S socat - UNIX-CONNECT:' + \
270 #TODO: probably need something else
271 (ret_code, stdout, stderr) = self._ssh.exec_command(qga_cmd)
272 if int(ret_code) != 0:
273 logger.debug('QGA execute failed {0}'.format(stderr))
274 raise RuntimeError('QGA execute "{0}" '
275 'failed on {1}'.format(qga_cmd,
280 return json.loads(stdout.split('\n', 1)[0])
282 def _qemu_qga_exec(self, cmd):
283 """Execute QGA command.
285 QGA provide access to a system-level agent via standard QMP commands.
287 :param cmd: QGA command to execute.
290 qga_cmd = '(echo "{ \\"execute\\": \\"' + \
292 '\\" }"; sleep 1) | sudo -S socat - UNIX-CONNECT:' + \
294 (ret_code, stdout, stderr) = self._ssh.exec_command(qga_cmd)
295 if int(ret_code) != 0:
296 logger.debug('QGA execute failed {0}'.format(stderr))
297 raise RuntimeError('QGA execute "{0}"'
298 ' failed on {1}'.format(cmd, self._node['host']))
302 return json.loads(stdout.split('\n', 1)[0])
304 def _wait_until_vm_boot(self, timeout=60):
305 """Wait until QEMU VM is booted.
307 First try to flush qga until there is output.
308 Then ping QEMU guest agent each 5s until VM booted or timeout.
310 :param timeout: Waiting timeout in seconds (optional, default 60s).
315 if time() - start > timeout:
316 raise RuntimeError('timeout, VM {0} not booted on {1}'.format(
317 self._qemu_opt['disk_image'], self._node['host']))
320 out = self._qemu_qga_flush()
322 logger.trace('QGA qga flush unexpected output {}'.format(out))
323 # Empty output - VM not booted yet
329 if time() - start > timeout:
330 raise RuntimeError('timeout, VM {0} not booted on {1}'.format(
331 self._qemu_opt['disk_image'], self._node['host']))
334 out = self._qemu_qga_exec('guest-ping')
336 logger.trace('QGA guest-ping unexpected output {}'.format(out))
337 # Empty output - VM not booted yet
340 # Non-error return - VM booted
341 elif out.get('return') is not None:
343 # Skip error and wait
344 elif out.get('error') is not None:
347 # If there is an unexpected output from QGA guest-info, try
348 # again until timeout.
349 logger.trace('QGA guest-ping unexpected output {}'.format(out))
351 logger.trace('VM {0} booted on {1}'.format(self._qemu_opt['disk_image'],
354 def _update_vm_interfaces(self):
355 """Update interface names in VM node dict."""
356 # Send guest-network-get-interfaces command via QGA, output example:
357 # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
358 # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}
359 out = self._qemu_qga_exec('guest-network-get-interfaces')
360 interfaces = out.get('return')
363 raise RuntimeError('Get VM {0} interface list failed on {1}'.format(
364 self._qemu_opt['disk_image'], self._node['host']))
365 # Create MAC-name dict
366 for interface in interfaces:
367 if 'hardware-address' not in interface:
369 mac_name[interface['hardware-address']] = interface['name']
370 # Match interface by MAC and save interface name
371 for interface in self._vm_info['interfaces'].values():
372 mac = interface.get('mac_address')
373 if_name = mac_name.get(mac)
375 logger.trace('Interface name for MAC {} not found'.format(mac))
377 interface['name'] = if_name
379 def _huge_page_check(self, allocate=False):
380 """Huge page check."""
381 huge_mnt = self._qemu_opt.get('huge_mnt')
382 mem_size = self._qemu_opt.get('mem_size')
384 # Get huge pages information
385 huge_size = self._get_huge_page_size()
386 huge_free = self._get_huge_page_free(huge_size)
387 huge_total = self._get_huge_page_total(huge_size)
389 # Check if memory reqested by qemu is available on host
390 if (mem_size * 1024) > (huge_free * huge_size):
391 # If we want to allocate hugepage dynamically
393 mem_needed = abs((huge_free * huge_size) - (mem_size * 1024))
394 huge_to_allocate = ((mem_needed / huge_size) * 2) + huge_total
395 max_map_count = huge_to_allocate*4
396 # Increase maximum number of memory map areas a process may have
397 cmd = 'echo "{0}" | sudo tee /proc/sys/vm/max_map_count'.format(
399 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
400 # Increase hugepage count
401 cmd = 'echo "{0}" | sudo tee /proc/sys/vm/nr_hugepages'.format(
403 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
404 if int(ret_code) != 0:
405 logger.debug('Mount huge pages failed {0}'.format(stderr))
406 raise RuntimeError('Mount huge pages failed on {0}'.format(
408 # If we do not want to allocate dynamicaly end with error
411 'Not enough free huge pages: {0}, '
412 '{1} MB'.format(huge_free, huge_free * huge_size)
414 # Check if huge pages mount point exist
416 (_, output, _) = self._ssh.exec_command('cat /proc/mounts')
417 for line in output.splitlines():
418 # Try to find something like:
419 # none /mnt/huge hugetlbfs rw,relatime,pagesize=2048k 0 0
421 if mount[2] == 'hugetlbfs' and mount[1] == huge_mnt:
424 # If huge page mount point not exist create one
426 cmd = 'mkdir -p {0}'.format(huge_mnt)
427 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
428 if int(ret_code) != 0:
429 logger.debug('Create mount dir failed: {0}'.format(stderr))
430 raise RuntimeError('Create mount dir failed on {0}'.format(
432 cmd = 'mount -t hugetlbfs -o pagesize=2048k none {0}'.format(
434 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
435 if int(ret_code) != 0:
436 logger.debug('Mount huge pages failed {0}'.format(stderr))
437 raise RuntimeError('Mount huge pages failed on {0}'.format(
440 def _get_huge_page_size(self):
441 """Get default size of huge pages in system.
443 :returns: Default size of free huge pages in system.
445 :raises: RuntimeError if reading failed for three times.
447 # TODO: remove to dedicated library
448 cmd_huge_size = "grep Hugepagesize /proc/meminfo | awk '{ print $2 }'"
450 (ret, out, _) = self._ssh.exec_command_sudo(cmd_huge_size)
455 logger.trace('Reading huge page size information failed')
459 raise RuntimeError('Getting huge page size information failed.')
462 def _get_huge_page_free(self, huge_size):
463 """Get total number of huge pages in system.
465 :param huge_size: Size of hugepages.
467 :returns: Number of free huge pages in system.
469 :raises: RuntimeError if reading failed for three times.
471 # TODO: add numa aware option
472 # TODO: remove to dedicated library
473 cmd_huge_free = 'cat /sys/kernel/mm/hugepages/hugepages-{0}kB/'\
474 'free_hugepages'.format(huge_size)
476 (ret, out, _) = self._ssh.exec_command_sudo(cmd_huge_free)
481 logger.trace('Reading free huge pages information failed')
485 raise RuntimeError('Getting free huge pages information failed.')
488 def _get_huge_page_total(self, huge_size):
489 """Get total number of huge pages in system.
491 :param huge_size: Size of hugepages.
493 :returns: Total number of huge pages in system.
495 :raises: RuntimeError if reading failed for three times.
497 # TODO: add numa aware option
498 # TODO: remove to dedicated library
499 cmd_huge_total = 'cat /sys/kernel/mm/hugepages/hugepages-{0}kB/'\
500 'nr_hugepages'.format(huge_size)
502 (ret, out, _) = self._ssh.exec_command_sudo(cmd_huge_total)
505 huge_total = int(out)
507 logger.trace('Reading total huge pages information failed')
511 raise RuntimeError('Getting total huge pages information failed.')
514 def qemu_start(self):
515 """Start QEMU and wait until VM boot.
517 :return: VM node info.
519 .. note:: First set at least node to run QEMU on.
520 .. warning:: Starts only one VM on the node.
523 bin_path = '{0}{1}'.format(self._qemu_path, self._qemu_bin)
526 ssh_fwd = '-net user,hostfwd=tcp::{0}-:22'.format(
527 self._qemu_opt.get('ssh_fwd_port'))
528 # Memory and huge pages
529 mem = '-object memory-backend-file,id=mem,size={0}M,mem-path={1},' \
530 'share=on -m {0} -numa node,memdev=mem'.format(
531 self._qemu_opt.get('mem_size'), self._qemu_opt.get('huge_mnt'))
533 # By default check only if hugepages are available.
534 # If 'huge_allocate' is set to true try to allocate as well.
535 self._huge_page_check(allocate=self._qemu_opt.get('huge_allocate'))
538 drive = '-drive file={0},format=raw,cache=none,if=virtio'.format(
539 self._qemu_opt.get('disk_image'))
540 # Setup QMP via unix socket
541 qmp = '-qmp unix:{0},server,nowait'.format(self._qmp_sock)
542 # Setup serial console
543 serial = '-chardev socket,host=127.0.0.1,port={0},id=gnc0,server,' \
544 'nowait -device isa-serial,chardev=gnc0'.format(
545 self._qemu_opt.get('serial_port'))
546 # Setup QGA via chardev (unix socket) and isa-serial channel
547 qga = '-chardev socket,path={0},server,nowait,id=qga0 ' \
548 '-device isa-serial,chardev=qga0'.format(self._qga_sock)
550 graphic = '-monitor none -display none -vga none'
552 pid = '-pidfile {}'.format(self._pid_file)
555 cmd = '{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}'.format(bin_path,
556 self._qemu_opt.get('smp'), mem, ssh_fwd,
557 self._qemu_opt.get('options'), drive, qmp, serial, qga, graphic,
560 (ret_code, _, _) = self._ssh.exec_command_sudo(cmd, timeout=300)
561 if int(ret_code) != 0:
562 raise RuntimeError('QEMU start failed on {0}'.format(
565 self._wait_until_vm_boot()
566 except (RuntimeError, SSHTimeout):
568 self.qemu_clear_socks()
570 logger.trace('QEMU started successfully.')
571 # Update interface names in VM node dict
572 self._update_vm_interfaces()
573 # Return VM node dict
577 """Quit the QEMU emulator."""
578 out = self._qemu_qmp_exec('quit')
579 err = out.get('error')
581 raise RuntimeError('QEMU quit failed on {0}, error: {1}'.format(
582 self._node['host'], json.dumps(err)))
584 def qemu_system_powerdown(self):
585 """Power down the system (if supported)."""
586 out = self._qemu_qmp_exec('system_powerdown')
587 err = out.get('error')
590 'QEMU system powerdown failed on {0}, '
591 'error: {1}'.format(self._node['host'], json.dumps(err))
594 def qemu_system_reset(self):
595 """Reset the system."""
596 out = self._qemu_qmp_exec('system_reset')
597 err = out.get('error')
600 'QEMU system reset failed on {0}, '
601 'error: {1}'.format(self._node['host'], json.dumps(err)))
604 """Kill qemu process."""
605 # Note: in QEMU start phase there are 3 QEMU processes because we
607 self._ssh.exec_command_sudo('chmod +r {}'.format(self._pid_file))
608 self._ssh.exec_command_sudo('kill -SIGKILL $(cat {})'
609 .format(self._pid_file))
611 cmd = 'rm -f {}'.format(self._pid_file)
612 self._ssh.exec_command_sudo(cmd)
614 def qemu_kill_all(self, node=None):
615 """Kill all qemu processes on DUT node if specified.
617 :param node: Node to kill all QEMU processes on.
621 self.qemu_set_node(node)
622 self._ssh.exec_command_sudo('pkill -SIGKILL qemu')
624 def qemu_clear_socks(self):
625 """Remove all sockets created by QEMU."""
626 # If serial console port still open kill process
627 cmd = 'fuser -k {}/tcp'.format(self._qemu_opt.get('serial_port'))
628 self._ssh.exec_command_sudo(cmd)
629 # Delete all created sockets
630 for sock in self._socks:
631 cmd = 'rm -f {}'.format(sock)
632 self._ssh.exec_command_sudo(cmd)
634 def qemu_system_status(self):
635 """Return current VM status.
637 VM should be in following status:
639 - debug: QEMU running on a debugger
640 - finish-migrate: paused to finish the migration process
641 - inmigrate: waiting for an incoming migration
642 - internal-error: internal error has occurred
643 - io-error: the last IOP has failed
645 - postmigrate: paused following a successful migrate
646 - prelaunch: QEMU was started with -S and guest has not started
647 - restore-vm: paused to restore VM state
648 - running: actively running
649 - save-vm: paused to save the VM state
650 - shutdown: shut down (and -no-shutdown is in use)
651 - suspended: suspended (ACPI S3)
652 - watchdog: watchdog action has been triggered
653 - guest-panicked: panicked as a result of guest OS panic
658 out = self._qemu_qmp_exec('query-status')
659 ret = out.get('return')
661 return ret.get('status')
663 err = out.get('error')
665 'QEMU query-status failed on {0}, '
666 'error: {1}'.format(self._node['host'], json.dumps(err)))
669 def build_qemu(node, force_install=False, apply_patch=False):
670 """Build QEMU from sources.
672 :param node: Node to build QEMU on.
673 :param force_install: If True, then remove previous build.
674 :param apply_patch: If True, then apply patches from qemu_patches dir.
676 :type force_install: bool
677 :type apply_patch: bool
678 :raises: RuntimeError if building QEMU failed.
683 directory = ' --directory={0}'.format(Constants.QEMU_INSTALL_DIR)
684 version = ' --version={0}'.format(Constants.QEMU_INSTALL_VERSION)
685 force = ' --force' if force_install else ''
686 patch = ' --patch' if apply_patch else ''
687 arch = Topology.get_node_arch(node)
688 target_list = ' --target-list={0}-softmmu'.format(arch)
690 (ret_code, stdout, stderr) = \
692 "sudo -E sh -c '{0}/{1}/qemu_build.sh{2}{3}{4}{5}{6}'"\
693 .format(Constants.REMOTE_FW_DIR, Constants.RESOURCES_LIB_SH,
694 version, directory, force, patch, target_list), 1000)
696 if int(ret_code) != 0:
697 logger.debug('QEMU build failed {0}'.format(stdout + stderr))
698 raise RuntimeError('QEMU build failed on {0}'.format(node['host']))