1 # Copyright (c) 2016 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """QEMU utilities library."""
16 from time import time, sleep
19 from robot.api import logger
21 from resources.libraries.python.ssh import SSH, SSHTimeout
22 from resources.libraries.python.constants import Constants
23 from resources.libraries.python.topology import NodeType
26 class QemuUtils(object):
29 def __init__(self, qemu_id=1):
30 self._qemu_id = qemu_id
32 self._qemu_bin = '/usr/bin/qemu-system-x86_64'
33 # QEMU Machine Protocol socket
34 self._qmp_sock = '/tmp/qmp{0}.sock'.format(self._qemu_id)
35 # QEMU Guest Agent socket
36 self._qga_sock = '/tmp/qga{0}.sock'.format(self._qemu_id)
39 self._qemu_opt['smp'] = '-smp 1,sockets=1,cores=1,threads=1'
40 # Daemonize the QEMU process after initialization. Default one
41 # management interface.
42 self._qemu_opt['options'] = '-cpu host -daemonize -enable-kvm ' \
43 '-machine pc,accel=kvm,usb=off,mem-merge=off ' \
44 '-net nic,macaddr=52:54:00:00:00:{0:02x} -balloon none'\
45 .format(self._qemu_id)
46 self._qemu_opt['ssh_fwd_port'] = 10022
47 # Default serial console port
48 self._qemu_opt['serial_port'] = 4556
49 # Default 512MB virtual RAM
50 self._qemu_opt['mem_size'] = 512
51 # Default huge page mount point, required for Vhost-user interfaces.
52 self._qemu_opt['huge_mnt'] = '/mnt/huge'
53 # Default do not allocate huge pages.
54 self._qemu_opt['huge_allocate'] = False
55 # Default image for CSIT virl setup
56 self._qemu_opt['disk_image'] = '/var/lib/vm/vhost-nested.img'
60 'port': self._qemu_opt['ssh_fwd_port'],
66 self._qemu_opt['queues'] = 1
70 self._socks = [self._qmp_sock, self._qga_sock]
72 def qemu_set_bin(self, path):
73 """Set binary path for QEMU.
75 :param path: Absolute path in filesystem.
80 def qemu_set_smp(self, cpus, cores, threads, sockets):
81 """Set SMP option for QEMU.
83 :param cpus: Number of CPUs.
84 :param cores: Number of CPU cores on one socket.
85 :param threads: Number of threads on one CPU core.
86 :param sockets: Number of discrete sockets in the system.
92 self._qemu_opt['smp'] = '-smp {},cores={},threads={},sockets={}'.format(
93 cpus, cores, threads, sockets)
95 def qemu_set_ssh_fwd_port(self, fwd_port):
96 """Set host port for guest SSH forwarding.
98 :param fwd_port: Port number on host for guest SSH forwarding.
101 self._qemu_opt['ssh_fwd_port'] = fwd_port
102 self._vm_info['port'] = fwd_port
104 def qemu_set_serial_port(self, port):
105 """Set serial console port.
107 :param port: Serial console port.
110 self._qemu_opt['serial_port'] = port
112 def qemu_set_mem_size(self, mem_size):
113 """Set virtual RAM size.
115 :param mem_size: RAM size in Mega Bytes.
118 self._qemu_opt['mem_size'] = int(mem_size)
120 def qemu_set_huge_mnt(self, huge_mnt):
121 """Set hugefile mount point.
123 :param huge_mnt: System hugefile mount point.
126 self._qemu_opt['huge_mnt'] = huge_mnt
128 def qemu_set_huge_allocate(self):
129 """Set flag to allocate more huge pages if needed."""
130 self._qemu_opt['huge_allocate'] = True
132 def qemu_set_disk_image(self, disk_image):
135 :param disk_image: Path of the disk image.
136 :type disk_image: str
138 self._qemu_opt['disk_image'] = disk_image
140 def qemu_set_affinity(self, *host_cpus):
141 """Set qemu affinity by getting thread PIDs via QMP and taskset to list
144 :param host_cpus: List of CPU cores.
145 :type host_cpus: list
147 qemu_cpus = self._qemu_qmp_exec('query-cpus')['return']
149 if len(qemu_cpus) != len(host_cpus):
150 logger.debug('Host CPU count {0}, Qemu Thread count {1}'.format(
151 len(host_cpus), len(qemu_cpus)))
152 raise ValueError('Host CPU count must match Qemu Thread count')
154 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
155 cmd = 'taskset -pc {0} {1}'.format(host_cpu, qemu_cpu['thread_id'])
156 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
157 if int(ret_code) != 0:
158 logger.debug('Set affinity failed {0}'.format(stderr))
159 raise RuntimeError('Set affinity failed on {0}'.format(
162 def qemu_set_scheduler_policy(self):
163 """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
166 :raises RuntimeError: Set scheduler policy failed.
168 qemu_cpus = self._qemu_qmp_exec('query-cpus')['return']
170 for qemu_cpu in qemu_cpus:
171 cmd = 'chrt -r -p 1 {0}'.format(qemu_cpu['thread_id'])
172 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
173 if int(ret_code) != 0:
174 logger.debug('Set SCHED_RR failed {0}'.format(stderr))
175 raise RuntimeError('Set SCHED_RR failed on {0}'.format(
178 def qemu_set_node(self, node):
179 """Set node to run QEMU on.
181 :param node: Node to run QEMU on.
186 self._ssh.connect(node)
187 self._vm_info['host'] = node['host']
189 def qemu_add_vhost_user_if(self, socket, server=True, mac=None):
190 """Add Vhost-user interface.
192 :param socket: Path of the unix socket.
193 :param server: If True the socket shall be a listening socket.
194 :param mac: Vhost-user interface MAC address (optional, otherwise is
195 used auto-generated MAC 52:54:00:00:xx:yy).
201 # Create unix socket character device.
202 chardev = ' -chardev socket,id=char{0},path={1}'.format(self._vhost_id,
206 self._qemu_opt['options'] += chardev
207 # Create Vhost-user network backend.
208 netdev = ' -netdev vhost-user,id=vhost{0},chardev=char{0},'\
209 'queues={1}'.format(self._vhost_id, self._qemu_opt['queues'])
210 self._qemu_opt['options'] += netdev
211 # If MAC is not specified use auto-generated MAC address based on
212 # template 52:54:00:00:<qemu_id>:<vhost_id>, e.g. vhost1 MAC of QEMU
213 # with ID 1 is 52:54:00:00:01:01
215 mac = '52:54:00:00:{0:02x}:{1:02x}'.\
216 format(self._qemu_id, self._vhost_id)
217 extend_options = 'mq=on,csum=off,gso=off,guest_tso4=off,'\
218 'guest_tso6=off,guest_ecn=off,mrg_rxbuf=off'
219 # Create Virtio network device.
220 device = ' -device virtio-net-pci,netdev=vhost{0},mac={1},{2}'.format(
221 self._vhost_id, mac, extend_options)
222 self._qemu_opt['options'] += device
223 # Add interface MAC and socket to the node dict
224 if_data = {'mac_address': mac, 'socket': socket}
225 if_name = 'vhost{}'.format(self._vhost_id)
226 self._vm_info['interfaces'][if_name] = if_data
227 # Add socket to the socket list
228 self._socks.append(socket)
230 def _qemu_qmp_exec(self, cmd):
231 """Execute QMP command.
233 QMP is JSON based protocol which allows to control QEMU instance.
235 :param cmd: QMP command to execute.
237 :return: Command output in python representation of JSON format. The
238 { "return": {} } response is QMP's success response. An error
239 response will contain the "error" keyword instead of "return".
241 # To enter command mode, the qmp_capabilities command must be issued.
242 qmp_cmd = 'echo "{ \\"execute\\": \\"qmp_capabilities\\" }' \
243 '{ \\"execute\\": \\"' + cmd + \
244 '\\" }" | sudo -S socat - UNIX-CONNECT:' + self._qmp_sock
246 (ret_code, stdout, stderr) = self._ssh.exec_command(qmp_cmd)
247 if int(ret_code) != 0:
248 logger.debug('QMP execute failed {0}'.format(stderr))
249 raise RuntimeError('QMP execute "{0}"'
250 ' failed on {1}'.format(cmd, self._node['host']))
252 # Skip capabilities negotiation messages.
253 out_list = stdout.splitlines()
254 if len(out_list) < 3:
255 raise RuntimeError('Invalid QMP output on {0}'.format(
257 return json.loads(out_list[2])
259 def _qemu_qga_flush(self):
260 """Flush the QGA parser state
262 qga_cmd = '(printf "\xFF"; sleep 1) | sudo -S socat - UNIX-CONNECT:' + \
264 #TODO: probably need something else
265 (ret_code, stdout, stderr) = self._ssh.exec_command(qga_cmd)
266 if int(ret_code) != 0:
267 logger.debug('QGA execute failed {0}'.format(stderr))
268 raise RuntimeError('QGA execute "{0}" '
269 'failed on {1}'.format(qga_cmd,
274 return json.loads(stdout.split('\n', 1)[0])
276 def _qemu_qga_exec(self, cmd):
277 """Execute QGA command.
279 QGA provide access to a system-level agent via standard QMP commands.
281 :param cmd: QGA command to execute.
284 qga_cmd = '(echo "{ \\"execute\\": \\"' + \
286 '\\" }"; sleep 1) | sudo -S socat - UNIX-CONNECT:' + \
288 (ret_code, stdout, stderr) = self._ssh.exec_command(qga_cmd)
289 if int(ret_code) != 0:
290 logger.debug('QGA execute failed {0}'.format(stderr))
291 raise RuntimeError('QGA execute "{0}"'
292 ' failed on {1}'.format(cmd, self._node['host']))
296 return json.loads(stdout.split('\n', 1)[0])
298 def _wait_until_vm_boot(self, timeout=60):
299 """Wait until QEMU VM is booted.
301 Ping QEMU guest agent each 5s until VM booted or timeout.
303 :param timeout: Waiting timeout in seconds (optional, default 60s).
308 if time() - start > timeout:
309 raise RuntimeError('timeout, VM {0} not booted on {1}'.format(
310 self._qemu_opt['disk_image'], self._node['host']))
313 self._qemu_qga_flush()
314 out = self._qemu_qga_exec('guest-ping')
316 logger.trace('QGA guest-ping unexpected output {}'.format(out))
317 # Empty output - VM not booted yet
320 # Non-error return - VM booted
321 elif out.get('return') is not None:
323 # Skip error and wait
324 elif out.get('error') is not None:
327 # If there is an unexpected output from QGA guest-info, try
328 # again until timeout.
329 logger.trace('QGA guest-ping unexpected output {}'.format(out))
331 logger.trace('VM {0} booted on {1}'.format(self._qemu_opt['disk_image'],
334 def _update_vm_interfaces(self):
335 """Update interface names in VM node dict."""
336 # Send guest-network-get-interfaces command via QGA, output example:
337 # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
338 # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}
339 out = self._qemu_qga_exec('guest-network-get-interfaces')
340 interfaces = out.get('return')
343 raise RuntimeError('Get VM {0} interface list failed on {1}'.format(
344 self._qemu_opt['disk_image'], self._node['host']))
345 # Create MAC-name dict
346 for interface in interfaces:
347 if 'hardware-address' not in interface:
349 mac_name[interface['hardware-address']] = interface['name']
350 # Match interface by MAC and save interface name
351 for interface in self._vm_info['interfaces'].values():
352 mac = interface.get('mac_address')
353 if_name = mac_name.get(mac)
355 logger.trace('Interface name for MAC {} not found'.format(mac))
357 interface['name'] = if_name
359 def _huge_page_check(self, allocate=False):
360 """Huge page check."""
361 huge_mnt = self._qemu_opt.get('huge_mnt')
362 mem_size = self._qemu_opt.get('mem_size')
364 # Get huge pages information
365 huge_size = self._get_huge_page_size()
366 huge_free = self._get_huge_page_free(huge_size)
367 huge_total = self._get_huge_page_total(huge_size)
369 # Check if memory reqested by qemu is available on host
370 if (mem_size * 1024) > (huge_free * huge_size):
371 # If we want to allocate hugepage dynamically
373 mem_needed = abs((huge_free * huge_size) - (mem_size * 1024))
374 huge_to_allocate = ((mem_needed / huge_size) * 2) + huge_total
375 max_map_count = huge_to_allocate*4
376 # Increase maximum number of memory map areas a process may have
377 cmd = 'echo "{0}" | sudo tee /proc/sys/vm/max_map_count'.format(
379 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
380 # Increase hugepage count
381 cmd = 'echo "{0}" | sudo tee /proc/sys/vm/nr_hugepages'.format(
383 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
384 if int(ret_code) != 0:
385 logger.debug('Mount huge pages failed {0}'.format(stderr))
386 raise RuntimeError('Mount huge pages failed on {0}'.format(
388 # If we do not want to allocate dynamicaly end with error
391 'Not enough free huge pages: {0}, '
392 '{1} MB'.format(huge_free, huge_free * huge_size)
394 # Check if huge pages mount point exist
396 (_, output, _) = self._ssh.exec_command('cat /proc/mounts')
397 for line in output.splitlines():
398 # Try to find something like:
399 # none /mnt/huge hugetlbfs rw,relatime,pagesize=2048k 0 0
401 if mount[2] == 'hugetlbfs' and mount[1] == huge_mnt:
404 # If huge page mount point not exist create one
406 cmd = 'mkdir -p {0}'.format(huge_mnt)
407 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
408 if int(ret_code) != 0:
409 logger.debug('Create mount dir failed: {0}'.format(stderr))
410 raise RuntimeError('Create mount dir failed on {0}'.format(
412 cmd = 'mount -t hugetlbfs -o pagesize=2048k none {0}'.format(
414 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
415 if int(ret_code) != 0:
416 logger.debug('Mount huge pages failed {0}'.format(stderr))
417 raise RuntimeError('Mount huge pages failed on {0}'.format(
420 def _get_huge_page_size(self):
421 """Get default size of huge pages in system.
423 :returns: Default size of free huge pages in system.
425 :raises: RuntimeError if reading failed for three times.
427 # TODO: remove to dedicated library
428 cmd_huge_size = "grep Hugepagesize /proc/meminfo | awk '{ print $2 }'"
430 (ret, out, _) = self._ssh.exec_command_sudo(cmd_huge_size)
435 logger.trace('Reading huge page size information failed')
439 raise RuntimeError('Getting huge page size information failed.')
442 def _get_huge_page_free(self, huge_size):
443 """Get total number of huge pages in system.
445 :param huge_size: Size of hugepages.
447 :returns: Number of free huge pages in system.
449 :raises: RuntimeError if reading failed for three times.
451 # TODO: add numa aware option
452 # TODO: remove to dedicated library
453 cmd_huge_free = 'cat /sys/kernel/mm/hugepages/hugepages-{0}kB/'\
454 'free_hugepages'.format(huge_size)
456 (ret, out, _) = self._ssh.exec_command_sudo(cmd_huge_free)
461 logger.trace('Reading free huge pages information failed')
465 raise RuntimeError('Getting free huge pages information failed.')
468 def _get_huge_page_total(self, huge_size):
469 """Get total number of huge pages in system.
471 :param huge_size: Size of hugepages.
473 :returns: Total number of huge pages in system.
475 :raises: RuntimeError if reading failed for three times.
477 # TODO: add numa aware option
478 # TODO: remove to dedicated library
479 cmd_huge_total = 'cat /sys/kernel/mm/hugepages/hugepages-{0}kB/'\
480 'nr_hugepages'.format(huge_size)
482 (ret, out, _) = self._ssh.exec_command_sudo(cmd_huge_total)
485 huge_total = int(out)
487 logger.trace('Reading total huge pages information failed')
491 raise RuntimeError('Getting total huge pages information failed.')
494 def qemu_start(self):
495 """Start QEMU and wait until VM boot.
497 :return: VM node info.
499 .. note:: First set at least node to run QEMU on.
500 .. warning:: Starts only one VM on the node.
503 ssh_fwd = '-net user,hostfwd=tcp::{0}-:22'.format(
504 self._qemu_opt.get('ssh_fwd_port'))
505 # Memory and huge pages
506 mem = '-object memory-backend-file,id=mem,size={0}M,mem-path={1},' \
507 'share=on -m {0} -numa node,memdev=mem'.format(
508 self._qemu_opt.get('mem_size'), self._qemu_opt.get('huge_mnt'))
510 # By default check only if hugepages are available.
511 # If 'huge_allocate' is set to true try to allocate as well.
512 self._huge_page_check(allocate=self._qemu_opt.get('huge_allocate'))
515 drive = '-drive file={0},format=raw,cache=none,if=virtio'.format(
516 self._qemu_opt.get('disk_image'))
517 # Setup QMP via unix socket
518 qmp = '-qmp unix:{0},server,nowait'.format(self._qmp_sock)
519 # Setup serial console
520 serial = '-chardev socket,host=127.0.0.1,port={0},id=gnc0,server,' \
521 'nowait -device isa-serial,chardev=gnc0'.format(
522 self._qemu_opt.get('serial_port'))
523 # Setup QGA via chardev (unix socket) and isa-serial channel
524 qga = '-chardev socket,path={0},server,nowait,id=qga0 ' \
525 '-device isa-serial,chardev=qga0'.format(self._qga_sock)
527 graphic = '-monitor none -display none -vga none'
530 cmd = '{0} {1} {2} {3} {4} {5} {6} {7} {8} {9}'.format(
531 self._qemu_bin, self._qemu_opt.get('smp'), mem, ssh_fwd,
532 self._qemu_opt.get('options'),
533 drive, qmp, serial, qga, graphic)
534 (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd, timeout=300)
535 if int(ret_code) != 0:
536 logger.debug('QEMU start failed {0}'.format(stderr))
537 raise RuntimeError('QEMU start failed on {0}'.format(
539 logger.trace('QEMU running')
542 self._wait_until_vm_boot()
543 except (RuntimeError, SSHTimeout):
545 self.qemu_clear_socks()
547 # Update interface names in VM node dict
548 self._update_vm_interfaces()
549 # Return VM node dict
553 """Quit the QEMU emulator."""
554 out = self._qemu_qmp_exec('quit')
555 err = out.get('error')
557 raise RuntimeError('QEMU quit failed on {0}, error: {1}'.format(
558 self._node['host'], json.dumps(err)))
560 def qemu_system_powerdown(self):
561 """Power down the system (if supported)."""
562 out = self._qemu_qmp_exec('system_powerdown')
563 err = out.get('error')
566 'QEMU system powerdown failed on {0}, '
567 'error: {1}'.format(self._node['host'], json.dumps(err))
570 def qemu_system_reset(self):
571 """Reset the system."""
572 out = self._qemu_qmp_exec('system_reset')
573 err = out.get('error')
576 'QEMU system reset failed on {0}, '
577 'error: {1}'.format(self._node['host'], json.dumps(err)))
580 """Kill qemu process."""
581 # TODO: add PID storage so that we can kill specific PID
582 # Note: in QEMU start phase there are 3 QEMU processes because we
584 self._ssh.exec_command_sudo('pkill -SIGKILL qemu')
586 def qemu_clear_socks(self):
587 """Remove all sockets created by QEMU."""
588 # If serial console port still open kill process
589 cmd = 'fuser -k {}/tcp'.format(self._qemu_opt.get('serial_port'))
590 self._ssh.exec_command_sudo(cmd)
591 # Delete all created sockets
592 for sock in self._socks:
593 cmd = 'rm -f {}'.format(sock)
594 self._ssh.exec_command_sudo(cmd)
596 def qemu_system_status(self):
597 """Return current VM status.
599 VM should be in following status:
601 - debug: QEMU running on a debugger
602 - finish-migrate: paused to finish the migration process
603 - inmigrate: waiting for an incoming migration
604 - internal-error: internal error has occurred
605 - io-error: the last IOP has failed
607 - postmigrate: paused following a successful migrate
608 - prelaunch: QEMU was started with -S and guest has not started
609 - restore-vm: paused to restore VM state
610 - running: actively running
611 - save-vm: paused to save the VM state
612 - shutdown: shut down (and -no-shutdown is in use)
613 - suspended: suspended (ACPI S3)
614 - watchdog: watchdog action has been triggered
615 - guest-panicked: panicked as a result of guest OS panic
620 out = self._qemu_qmp_exec('query-status')
621 ret = out.get('return')
623 return ret.get('status')
625 err = out.get('error')
627 'QEMU query-status failed on {0}, '
628 'error: {1}'.format(self._node['host'], json.dumps(err)))
631 def build_qemu(node, force_install=False, apply_patch=False):
632 """Build QEMU from sources.
634 :param node: Node to build QEMU on.
635 :param force_install: If True, then remove previous build.
636 :param apply_patch: If True, then apply patches from qemu_patches dir.
638 :type force_install: bool
639 :type apply_patch: bool
640 :raises: RuntimeError if building QEMU failed.
645 directory = ' --directory={0}'.format(Constants.QEMU_INSTALL_DIR)
646 version = ' --version={0}'.format(Constants.QEMU_INSTALL_VERSION)
647 force = ' --force' if force_install else ''
648 patch = ' --patch' if apply_patch else ''
650 (ret_code, stdout, stderr) = \
652 "sudo -E sh -c '{0}/{1}/qemu_build.sh{2}{3}{4}{5}'"\
653 .format(Constants.REMOTE_FW_DIR, Constants.RESOURCES_LIB_SH,
654 version, directory, force, patch), 1000)
656 if int(ret_code) != 0:
657 logger.debug('QEMU build failed {0}'.format(stdout + stderr))
658 raise RuntimeError('QEMU build failed on {0}'.format(node['host']))