1 # Copyright (c) 2018 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """QEMU utilities library."""
16 from time import time, sleep
19 from robot.api import logger
21 from resources.libraries.python.ssh import SSH, SSHTimeout
22 from resources.libraries.python.constants import Constants
23 from resources.libraries.python.topology import NodeType, Topology
26 class QemuUtils(object):
29 def __init__(self, qemu_id=1):
30 self._qemu_id = qemu_id
31 # Path to QEMU binary. Use x86_64 by default
32 self._qemu_path = '/usr/bin/'
33 self._qemu_bin = 'qemu-system-x86_64'
34 # QEMU Machine Protocol socket
35 self._qmp_sock = '/tmp/qmp{0}.sock'.format(self._qemu_id)
36 # QEMU Guest Agent socket
37 self._qga_sock = '/tmp/qga{0}.sock'.format(self._qemu_id)
39 self._pid_file = '/tmp/qemu{0}.pid'.format(self._qemu_id)
42 self._qemu_opt['smp'] = '-smp 1,sockets=1,cores=1,threads=1'
43 # Daemonize the QEMU process after initialization. Default one
44 # management interface.
45 self._qemu_opt['options'] = '-cpu host -daemonize -enable-kvm ' \
46 '-machine pc,accel=kvm,usb=off,mem-merge=off ' \
47 '-net nic,macaddr=52:54:00:00:{0:02x}:ff -balloon none'\
48 .format(self._qemu_id)
49 self._qemu_opt['ssh_fwd_port'] = 10021 + qemu_id
50 # Default serial console port
51 self._qemu_opt['serial_port'] = 4555 + qemu_id
52 # Default 512MB virtual RAM
53 self._qemu_opt['mem_size'] = 512
54 # Default huge page mount point, required for Vhost-user interfaces.
55 self._qemu_opt['huge_mnt'] = '/mnt/huge'
56 # Default do not allocate huge pages.
57 self._qemu_opt['huge_allocate'] = False
58 # Default image for CSIT virl setup
59 self._qemu_opt['disk_image'] = '/var/lib/vm/vhost-nested.img'
63 'port': self._qemu_opt['ssh_fwd_port'],
69 self._qemu_opt['queues'] = 1
73 self._socks = [self._qmp_sock, self._qga_sock]
75 def qemu_set_path(self, path):
76 """Set binary path for QEMU.
78 :param path: Absolute path in filesystem.
81 self._qemu_path = path
83 def qemu_set_smp(self, cpus, cores, threads, sockets):
84 """Set SMP option for QEMU.
86 :param cpus: Number of CPUs.
87 :param cores: Number of CPU cores on one socket.
88 :param threads: Number of threads on one CPU core.
89 :param sockets: Number of discrete sockets in the system.
95 self._qemu_opt['smp'] = '-smp {},cores={},threads={},sockets={}'.format(
96 cpus, cores, threads, sockets)
98 def qemu_set_ssh_fwd_port(self, fwd_port):
99 """Set host port for guest SSH forwarding.
101 :param fwd_port: Port number on host for guest SSH forwarding.
104 self._qemu_opt['ssh_fwd_port'] = fwd_port
105 self._vm_info['port'] = fwd_port
107 def qemu_set_serial_port(self, port):
108 """Set serial console port.
110 :param port: Serial console port.
113 self._qemu_opt['serial_port'] = port
115 def qemu_set_mem_size(self, mem_size):
116 """Set virtual RAM size.
118 :param mem_size: RAM size in Mega Bytes.
121 self._qemu_opt['mem_size'] = int(mem_size)
123 def qemu_set_huge_mnt(self, huge_mnt):
124 """Set hugefile mount point.
126 :param huge_mnt: System hugefile mount point.
129 self._qemu_opt['huge_mnt'] = huge_mnt
131 def qemu_set_huge_allocate(self):
132 """Set flag to allocate more huge pages if needed."""
133 self._qemu_opt['huge_allocate'] = True
135 def qemu_set_disk_image(self, disk_image):
138 :param disk_image: Path of the disk image.
139 :type disk_image: str
141 self._qemu_opt['disk_image'] = disk_image
143 def qemu_set_affinity(self, *host_cpus):
144 """Set qemu affinity by getting thread PIDs via QMP and taskset to list
147 :param host_cpus: List of CPU cores.
148 :type host_cpus: list
150 qemu_cpus = self._qemu_qmp_exec('query-cpus')['return']
152 if len(qemu_cpus) != len(host_cpus):
153 logger.debug('Host CPU count {0}, Qemu Thread count {1}'.format(
154 len(host_cpus), len(qemu_cpus)))
155 raise ValueError('Host CPU count must match Qemu Thread count')
157 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
158 cmd = 'taskset -pc {0} {1}'.format(host_cpu, qemu_cpu['thread_id'])
159 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
160 if int(ret_code) != 0:
161 logger.debug('Set affinity failed {0}'.format(stderr))
162 raise RuntimeError('Set affinity failed on {0}'.format(
165 def qemu_set_scheduler_policy(self):
166 """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
169 :raises RuntimeError: Set scheduler policy failed.
171 qemu_cpus = self._qemu_qmp_exec('query-cpus')['return']
173 for qemu_cpu in qemu_cpus:
174 cmd = 'chrt -r -p 1 {0}'.format(qemu_cpu['thread_id'])
175 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
176 if int(ret_code) != 0:
177 logger.debug('Set SCHED_RR failed {0}'.format(stderr))
178 raise RuntimeError('Set SCHED_RR failed on {0}'.format(
181 def qemu_set_node(self, node):
182 """Set node to run QEMU on.
184 :param node: Node to run QEMU on.
189 self._ssh.connect(node)
190 self._vm_info['host'] = node['host']
192 arch = Topology.get_node_arch(node)
193 self._qemu_bin = 'qemu-system-{arch}'.format(arch=arch)
195 def qemu_add_vhost_user_if(self, socket, server=True, mac=None,
197 """Add Vhost-user interface.
199 :param socket: Path of the unix socket.
200 :param server: If True the socket shall be a listening socket.
201 :param mac: Vhost-user interface MAC address (optional, otherwise is
202 used auto-generated MAC 52:54:00:00:xx:yy).
203 :param jumbo_frames: Set True if jumbo frames are used in the test.
207 :type jumbo_frames: bool
210 # Create unix socket character device.
211 chardev = ' -chardev socket,id=char{0},path={1}'.format(self._vhost_id,
215 self._qemu_opt['options'] += chardev
216 # Create Vhost-user network backend.
217 netdev = (' -netdev vhost-user,id=vhost{0},chardev=char{0},queues={1}'
218 .format(self._vhost_id, self._qemu_opt['queues']))
219 self._qemu_opt['options'] += netdev
220 # If MAC is not specified use auto-generated MAC address based on
221 # template 52:54:00:00:<qemu_id>:<vhost_id>, e.g. vhost1 MAC of QEMU
222 # with ID 1 is 52:54:00:00:01:01
224 mac = '52:54:00:00:{0:02x}:{1:02x}'.\
225 format(self._qemu_id, self._vhost_id)
226 extend_options = 'mq=on,csum=off,gso=off,guest_tso4=off,'\
227 'guest_tso6=off,guest_ecn=off'
229 extend_options += ",mrg_rxbuf=on"
231 extend_options += ",mrg_rxbuf=off"
232 # Create Virtio network device.
233 device = ' -device virtio-net-pci,netdev=vhost{0},mac={1},{2}'.format(
234 self._vhost_id, mac, extend_options)
235 self._qemu_opt['options'] += device
236 # Add interface MAC and socket to the node dict
237 if_data = {'mac_address': mac, 'socket': socket}
238 if_name = 'vhost{}'.format(self._vhost_id)
239 self._vm_info['interfaces'][if_name] = if_data
240 # Add socket to the socket list
241 self._socks.append(socket)
243 def _qemu_qmp_exec(self, cmd):
244 """Execute QMP command.
246 QMP is JSON based protocol which allows to control QEMU instance.
248 :param cmd: QMP command to execute.
250 :returns: Command output in python representation of JSON format. The
251 { "return": {} } response is QMP's success response. An error
252 response will contain the "error" keyword instead of "return".
254 # To enter command mode, the qmp_capabilities command must be issued.
255 qmp_cmd = 'echo "{ \\"execute\\": \\"qmp_capabilities\\" }' \
256 '{ \\"execute\\": \\"' + cmd + \
257 '\\" }" | sudo -S socat - UNIX-CONNECT:' + self._qmp_sock
259 (ret_code, stdout, stderr) = self._ssh.exec_command(qmp_cmd)
260 if int(ret_code) != 0:
261 logger.debug('QMP execute failed {0}'.format(stderr))
262 raise RuntimeError('QMP execute "{0}"'
263 ' failed on {1}'.format(cmd, self._node['host']))
265 # Skip capabilities negotiation messages.
266 out_list = stdout.splitlines()
267 if len(out_list) < 3:
268 raise RuntimeError('Invalid QMP output on {0}'.format(
270 return json.loads(out_list[2])
272 def _qemu_qga_flush(self):
273 """Flush the QGA parser state
275 qga_cmd = '(printf "\xFF"; sleep 1) | sudo -S socat - UNIX-CONNECT:' + \
277 #TODO: probably need something else
278 (ret_code, stdout, stderr) = self._ssh.exec_command(qga_cmd)
279 if int(ret_code) != 0:
280 logger.debug('QGA execute failed {0}'.format(stderr))
281 raise RuntimeError('QGA execute "{0}" '
282 'failed on {1}'.format(qga_cmd,
287 return json.loads(stdout.split('\n', 1)[0])
289 def _qemu_qga_exec(self, cmd):
290 """Execute QGA command.
292 QGA provide access to a system-level agent via standard QMP commands.
294 :param cmd: QGA command to execute.
297 qga_cmd = '(echo "{ \\"execute\\": \\"' + \
299 '\\" }"; sleep 1) | sudo -S socat - UNIX-CONNECT:' + \
301 (ret_code, stdout, stderr) = self._ssh.exec_command(qga_cmd)
302 if int(ret_code) != 0:
303 logger.debug('QGA execute failed {0}'.format(stderr))
304 raise RuntimeError('QGA execute "{0}"'
305 ' failed on {1}'.format(cmd, self._node['host']))
309 return json.loads(stdout.split('\n', 1)[0])
311 def _wait_until_vm_boot(self, timeout=60):
312 """Wait until QEMU VM is booted.
314 First try to flush qga until there is output.
315 Then ping QEMU guest agent each 5s until VM booted or timeout.
317 :param timeout: Waiting timeout in seconds (optional, default 60s).
322 if time() - start > timeout:
323 raise RuntimeError('timeout, VM {0} not booted on {1}'.format(
324 self._qemu_opt['disk_image'], self._node['host']))
327 out = self._qemu_qga_flush()
329 logger.trace('QGA qga flush unexpected output {}'.format(out))
330 # Empty output - VM not booted yet
336 if time() - start > timeout:
337 raise RuntimeError('timeout, VM {0} not booted on {1}'.format(
338 self._qemu_opt['disk_image'], self._node['host']))
341 out = self._qemu_qga_exec('guest-ping')
343 logger.trace('QGA guest-ping unexpected output {}'.format(out))
344 # Empty output - VM not booted yet
347 # Non-error return - VM booted
348 elif out.get('return') is not None:
350 # Skip error and wait
351 elif out.get('error') is not None:
354 # If there is an unexpected output from QGA guest-info, try
355 # again until timeout.
356 logger.trace('QGA guest-ping unexpected output {}'.format(out))
358 logger.trace('VM {0} booted on {1}'.format(self._qemu_opt['disk_image'],
361 def _update_vm_interfaces(self):
362 """Update interface names in VM node dict."""
363 # Send guest-network-get-interfaces command via QGA, output example:
364 # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
365 # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}
366 out = self._qemu_qga_exec('guest-network-get-interfaces')
367 interfaces = out.get('return')
370 raise RuntimeError('Get VM {0} interface list failed on {1}'.format(
371 self._qemu_opt['disk_image'], self._node['host']))
372 # Create MAC-name dict
373 for interface in interfaces:
374 if 'hardware-address' not in interface:
376 mac_name[interface['hardware-address']] = interface['name']
377 # Match interface by MAC and save interface name
378 for interface in self._vm_info['interfaces'].values():
379 mac = interface.get('mac_address')
380 if_name = mac_name.get(mac)
382 logger.trace('Interface name for MAC {} not found'.format(mac))
384 interface['name'] = if_name
386 def _huge_page_check(self, allocate=False):
387 """Huge page check."""
388 huge_mnt = self._qemu_opt.get('huge_mnt')
389 mem_size = self._qemu_opt.get('mem_size')
391 # Get huge pages information
392 huge_size = self._get_huge_page_size()
393 huge_free = self._get_huge_page_free(huge_size)
394 huge_total = self._get_huge_page_total(huge_size)
396 # Check if memory reqested by qemu is available on host
397 if (mem_size * 1024) > (huge_free * huge_size):
398 # If we want to allocate hugepage dynamically
400 mem_needed = abs((huge_free * huge_size) - (mem_size * 1024))
401 huge_to_allocate = ((mem_needed / huge_size) * 2) + huge_total
402 max_map_count = huge_to_allocate*4
403 # Increase maximum number of memory map areas a process may have
404 cmd = 'echo "{0}" | sudo tee /proc/sys/vm/max_map_count'.format(
406 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
407 # Increase hugepage count
408 cmd = 'echo "{0}" | sudo tee /proc/sys/vm/nr_hugepages'.format(
410 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
411 if int(ret_code) != 0:
412 logger.debug('Mount huge pages failed {0}'.format(stderr))
413 raise RuntimeError('Mount huge pages failed on {0}'.format(
415 # If we do not want to allocate dynamicaly end with error
418 'Not enough free huge pages: {0}, '
419 '{1} MB'.format(huge_free, huge_free * huge_size)
421 # Check if huge pages mount point exist
423 (_, output, _) = self._ssh.exec_command('cat /proc/mounts')
424 for line in output.splitlines():
425 # Try to find something like:
426 # none /mnt/huge hugetlbfs rw,relatime,pagesize=2048k 0 0
428 if mount[2] == 'hugetlbfs' and mount[1] == huge_mnt:
431 # If huge page mount point not exist create one
433 cmd = 'mkdir -p {0}'.format(huge_mnt)
434 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
435 if int(ret_code) != 0:
436 logger.debug('Create mount dir failed: {0}'.format(stderr))
437 raise RuntimeError('Create mount dir failed on {0}'.format(
439 cmd = 'mount -t hugetlbfs -o pagesize=2048k none {0}'.format(
441 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
442 if int(ret_code) != 0:
443 logger.debug('Mount huge pages failed {0}'.format(stderr))
444 raise RuntimeError('Mount huge pages failed on {0}'.format(
447 def _get_huge_page_size(self):
448 """Get default size of huge pages in system.
450 :returns: Default size of free huge pages in system.
452 :raises RuntimeError: If reading failed for three times.
454 # TODO: remove to dedicated library
455 cmd_huge_size = "grep Hugepagesize /proc/meminfo | awk '{ print $2 }'"
457 (ret, out, _) = self._ssh.exec_command_sudo(cmd_huge_size)
462 logger.trace('Reading huge page size information failed')
466 raise RuntimeError('Getting huge page size information failed.')
469 def _get_huge_page_free(self, huge_size):
470 """Get total number of huge pages in system.
472 :param huge_size: Size of hugepages.
474 :returns: Number of free huge pages in system.
476 :raises RuntimeError: If reading failed for three times.
478 # TODO: add numa aware option
479 # TODO: remove to dedicated library
480 cmd_huge_free = 'cat /sys/kernel/mm/hugepages/hugepages-{0}kB/'\
481 'free_hugepages'.format(huge_size)
483 (ret, out, _) = self._ssh.exec_command_sudo(cmd_huge_free)
488 logger.trace('Reading free huge pages information failed')
492 raise RuntimeError('Getting free huge pages information failed.')
495 def _get_huge_page_total(self, huge_size):
496 """Get total number of huge pages in system.
498 :param huge_size: Size of hugepages.
500 :returns: Total number of huge pages in system.
502 :raises RuntimeError: If reading failed for three times.
504 # TODO: add numa aware option
505 # TODO: remove to dedicated library
506 cmd_huge_total = 'cat /sys/kernel/mm/hugepages/hugepages-{0}kB/'\
507 'nr_hugepages'.format(huge_size)
509 (ret, out, _) = self._ssh.exec_command_sudo(cmd_huge_total)
512 huge_total = int(out)
514 logger.trace('Reading total huge pages information failed')
518 raise RuntimeError('Getting total huge pages information failed.')
521 def qemu_start(self):
522 """Start QEMU and wait until VM boot.
524 .. note:: First set at least node to run QEMU on.
525 .. warning:: Starts only one VM on the node.
527 :returns: VM node info.
531 bin_path = '{0}{1}'.format(self._qemu_path, self._qemu_bin)
534 ssh_fwd = '-net user,hostfwd=tcp::{0}-:22'.format(
535 self._qemu_opt.get('ssh_fwd_port'))
536 # Memory and huge pages
537 mem = '-object memory-backend-file,id=mem,size={0}M,mem-path={1},' \
538 'share=on -m {0} -numa node,memdev=mem'.format(
539 self._qemu_opt.get('mem_size'), self._qemu_opt.get('huge_mnt'))
541 # By default check only if hugepages are available.
542 # If 'huge_allocate' is set to true try to allocate as well.
543 self._huge_page_check(allocate=self._qemu_opt.get('huge_allocate'))
546 drive = '-drive file={0},format=raw,cache=none,if=virtio'.format(
547 self._qemu_opt.get('disk_image'))
548 # Setup QMP via unix socket
549 qmp = '-qmp unix:{0},server,nowait'.format(self._qmp_sock)
550 # Setup serial console
551 serial = '-chardev socket,host=127.0.0.1,port={0},id=gnc0,server,' \
552 'nowait -device isa-serial,chardev=gnc0'.format(
553 self._qemu_opt.get('serial_port'))
554 # Setup QGA via chardev (unix socket) and isa-serial channel
555 qga = '-chardev socket,path={0},server,nowait,id=qga0 ' \
556 '-device isa-serial,chardev=qga0'.format(self._qga_sock)
558 graphic = '-monitor none -display none -vga none'
560 pid = '-pidfile {}'.format(self._pid_file)
563 cmd = '{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}'.format(
564 bin_path, self._qemu_opt.get('smp'), mem, ssh_fwd,
565 self._qemu_opt.get('options'), drive, qmp, serial, qga, graphic,
568 (ret_code, _, _) = self._ssh.exec_command_sudo(cmd, timeout=300)
569 if int(ret_code) != 0:
570 raise RuntimeError('QEMU start failed on {0}'.format(
573 self._wait_until_vm_boot()
574 except (RuntimeError, SSHTimeout):
576 self.qemu_clear_socks()
578 logger.trace('QEMU started successfully.')
579 # Update interface names in VM node dict
580 self._update_vm_interfaces()
581 # Return VM node dict
585 """Quit the QEMU emulator."""
586 out = self._qemu_qmp_exec('quit')
587 err = out.get('error')
589 raise RuntimeError('QEMU quit failed on {0}, error: {1}'.format(
590 self._node['host'], json.dumps(err)))
592 def qemu_system_powerdown(self):
593 """Power down the system (if supported)."""
594 out = self._qemu_qmp_exec('system_powerdown')
595 err = out.get('error')
598 'QEMU system powerdown failed on {0}, '
599 'error: {1}'.format(self._node['host'], json.dumps(err))
602 def qemu_system_reset(self):
603 """Reset the system."""
604 out = self._qemu_qmp_exec('system_reset')
605 err = out.get('error')
608 'QEMU system reset failed on {0}, '
609 'error: {1}'.format(self._node['host'], json.dumps(err)))
612 """Kill qemu process."""
613 # Note: in QEMU start phase there are 3 QEMU processes because we
615 self._ssh.exec_command_sudo('chmod +r {}'.format(self._pid_file))
616 self._ssh.exec_command_sudo('kill -SIGKILL $(cat {})'
617 .format(self._pid_file))
619 cmd = 'rm -f {}'.format(self._pid_file)
620 self._ssh.exec_command_sudo(cmd)
622 def qemu_kill_all(self, node=None):
623 """Kill all qemu processes on DUT node if specified.
625 :param node: Node to kill all QEMU processes on.
629 self.qemu_set_node(node)
630 self._ssh.exec_command_sudo('pkill -SIGKILL qemu')
632 def qemu_clear_socks(self):
633 """Remove all sockets created by QEMU."""
634 # If serial console port still open kill process
635 cmd = 'fuser -k {}/tcp'.format(self._qemu_opt.get('serial_port'))
636 self._ssh.exec_command_sudo(cmd)
637 # Delete all created sockets
638 for sock in self._socks:
639 cmd = 'rm -f {}'.format(sock)
640 self._ssh.exec_command_sudo(cmd)
642 def qemu_system_status(self):
643 """Return current VM status.
645 VM should be in following status:
647 - debug: QEMU running on a debugger
648 - finish-migrate: paused to finish the migration process
649 - inmigrate: waiting for an incoming migration
650 - internal-error: internal error has occurred
651 - io-error: the last IOP has failed
653 - postmigrate: paused following a successful migrate
654 - prelaunch: QEMU was started with -S and guest has not started
655 - restore-vm: paused to restore VM state
656 - running: actively running
657 - save-vm: paused to save the VM state
658 - shutdown: shut down (and -no-shutdown is in use)
659 - suspended: suspended (ACPI S3)
660 - watchdog: watchdog action has been triggered
661 - guest-panicked: panicked as a result of guest OS panic
666 out = self._qemu_qmp_exec('query-status')
667 ret = out.get('return')
669 return ret.get('status')
671 err = out.get('error')
673 'QEMU query-status failed on {0}, '
674 'error: {1}'.format(self._node['host'], json.dumps(err)))
677 def build_qemu(node, force_install=False, apply_patch=False):
678 """Build QEMU from sources.
680 :param node: Node to build QEMU on.
681 :param force_install: If True, then remove previous build.
682 :param apply_patch: If True, then apply patches from qemu_patches dir.
684 :type force_install: bool
685 :type apply_patch: bool
686 :raises RuntimeError: If building QEMU failed.
691 directory = ' --directory={0}'.format(Constants.QEMU_INSTALL_DIR)
693 directory += '-patch'
696 version = ' --version={0}'.format(Constants.QEMU_INSTALL_VERSION)
697 force = ' --force' if force_install else ''
698 patch = ' --patch' if apply_patch else ''
699 arch = Topology.get_node_arch(node)
700 target_list = ' --target-list={0}-softmmu'.format(arch)
702 (ret_code, stdout, stderr) = \
704 "sudo -E sh -c '{0}/{1}/qemu_build.sh{2}{3}{4}{5}{6}'"\
705 .format(Constants.REMOTE_FW_DIR, Constants.RESOURCES_LIB_SH,
706 version, directory, force, patch, target_list), 1000)
708 if int(ret_code) != 0:
709 logger.debug('QEMU build failed {0}'.format(stdout + stderr))
710 raise RuntimeError('QEMU build failed on {0}'.format(node['host']))