1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """QEMU utilities library."""
19 from string import Template
20 from time import sleep
22 from robot.api import logger
24 from resources.libraries.python.Constants import Constants
25 from resources.libraries.python.DpdkUtil import DpdkUtil
26 from resources.libraries.python.DUTSetup import DUTSetup
27 from resources.libraries.python.OptionString import OptionString
28 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
29 from resources.libraries.python.topology import NodeType, Topology
30 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
31 from resources.libraries.python.VPPUtil import VPPUtil
33 __all__ = [u"QemuUtils"]
39 # Use one instance of class per tests.
40 ROBOT_LIBRARY_SCOPE = u"TEST CASE"
43 self, node, qemu_id=1, smp=1, mem=512, vnf=None,
44 img=Constants.QEMU_VM_IMAGE):
45 """Initialize QemuUtil class.
47 :param node: Node to run QEMU on.
48 :param qemu_id: QEMU identifier.
49 :param smp: Number of virtual SMP units (cores).
50 :param mem: Amount of memory.
51 :param vnf: Network function workload.
52 :param img: QEMU disk image or kernel image path.
62 self._arch = Topology.get_node_arch(self._node)
65 # Architecture specific options
66 if self._arch == u"aarch64":
67 dpdk_target = u"arm64-armv8a"
68 self._opt[u"machine_args"] = \
69 u"virt,accel=kvm,usb=off,mem-merge=off,gic-version=3"
70 self._opt[u"console"] = u"ttyAMA0"
71 self._opt[u"unsafe_iommu"] = u"echo Y > /sys/module/vfio/para" \
72 u"meters/enable_unsafe_noiommu_mode"
74 dpdk_target = u"x86_64-native"
75 self._opt[u"machine_args"] = u"pc,accel=kvm,usb=off,mem-merge=off"
76 self._opt[u"console"] = u"ttyS0"
77 self._opt[u"unsafe_iommu"] = u""
79 self._testpmd_path = f"{Constants.QEMU_VM_DPDK}/" \
80 f"{dpdk_target}-linux-gcc/app"
82 u"host": node[u"host"],
84 u"port": 10021 + qemu_id,
85 u"serial": 4555 + qemu_id,
90 if node[u"port"] != 22:
91 self._vm_info[u"host_port"] = node[u"port"]
92 self._vm_info[u"host_username"] = node[u"username"]
93 self._vm_info[u"host_password"] = node[u"password"]
95 self._opt[u"qemu_id"] = qemu_id
96 self._opt[u"mem"] = int(mem)
97 self._opt[u"smp"] = int(smp)
98 self._opt[u"img"] = img
99 self._opt[u"vnf"] = vnf
102 self._temp[u"pidfile"] = f"/var/run/qemu_{qemu_id}.pid"
103 if img == Constants.QEMU_VM_IMAGE:
104 self._opt[u"vm_type"] = u"nestedvm"
105 self._temp[u"qmp"] = f"/var/run/qmp_{qemu_id}.sock"
106 self._temp[u"qga"] = f"/var/run/qga_{qemu_id}.sock"
107 elif img == Constants.QEMU_VM_KERNEL:
108 self._opt[u"img"], _ = exec_cmd_no_error(
109 node, f"ls -1 {Constants.QEMU_VM_KERNEL}* | tail -1",
110 message=u"Qemu Kernel VM image not found!"
112 self._opt[u"vm_type"] = u"kernelvm"
113 self._temp[u"log"] = f"/tmp/serial_{qemu_id}.log"
114 self._temp[u"ini"] = f"/etc/vm_init_{qemu_id}.conf"
115 self._opt[u"initrd"], _ = exec_cmd_no_error(
116 node, f"ls -1 {Constants.QEMU_VM_KERNEL_INITRD}* | tail -1",
117 message=u"Qemu Kernel initrd image not found!"
120 raise RuntimeError(f"QEMU: Unknown VM image option: {img}")
121 # Computed parameters for QEMU command line.
122 self._params = OptionString(prefix=u"-")
125 def add_params(self):
126 """Set QEMU command line parameters."""
127 self.add_default_params()
128 if self._opt.get(u"vm_type", u"") == u"nestedvm":
129 self.add_nestedvm_params()
130 elif self._opt.get(u"vm_type", u"") == u"kernelvm":
131 self.add_kernelvm_params()
133 raise RuntimeError(u"QEMU: Unsupported VM type!")
135 def add_default_params(self):
136 """Set default QEMU command line parameters."""
137 self._params.add(u"daemonize")
138 self._params.add(u"nodefaults")
139 self._params.add_with_value(
140 u"name", f"vnf{self._opt.get(u'qemu_id')},debug-threads=on"
142 self._params.add(u"no-user-config")
143 self._params.add_with_value(u"monitor", u"none")
144 self._params.add_with_value(u"display", u"none")
145 self._params.add_with_value(u"vga", u"none")
146 self._params.add(u"enable-kvm")
147 self._params.add_with_value(u"pidfile", self._temp.get(u"pidfile"))
148 self._params.add_with_value(u"cpu", u"host")
150 self._params.add_with_value(u"machine", self._opt.get(u"machine_args"))
151 self._params.add_with_value(
152 u"smp", f"{self._opt.get(u'smp')},sockets=1,"
153 f"cores={self._opt.get(u'smp')},threads=1"
155 self._params.add_with_value(
156 u"object", f"memory-backend-file,id=mem,"
157 f"size={self._opt.get(u'mem')}M,mem-path=/dev/hugepages,share=on"
159 self._params.add_with_value(u"m", f"{self._opt.get(u'mem')}M")
160 self._params.add_with_value(u"numa", u"node,memdev=mem")
161 self._params.add_with_value(u"balloon", u"none")
163 def add_nestedvm_params(self):
164 """Set NestedVM QEMU parameters."""
165 self._params.add_with_value(
167 f"nic,macaddr=52:54:00:00:{self._opt.get(u'qemu_id'):02x}:ff"
169 self._params.add_with_value(
170 u"net", f"user,hostfwd=tcp::{self._vm_info[u'port']}-:22"
172 locking = u",file.locking=off"
173 self._params.add_with_value(
174 u"drive", f"file={self._opt.get(u'img')},"
175 f"format=raw,cache=none,if=virtio{locking}"
177 self._params.add_with_value(
178 u"qmp", f"unix:{self._temp.get(u'qmp')},server,nowait"
180 self._params.add_with_value(
181 u"chardev", f"socket,host=127.0.0.1,"
182 f"port={self._vm_info[u'serial']},id=gnc0,server,nowait")
183 self._params.add_with_value(u"device", u"isa-serial,chardev=gnc0")
184 self._params.add_with_value(
185 u"chardev", f"socket,path={self._temp.get(u'qga')},"
186 f"server,nowait,id=qga0"
188 self._params.add_with_value(u"device", u"isa-serial,chardev=qga0")
190 def add_kernelvm_params(self):
191 """Set KernelVM QEMU parameters."""
192 self._params.add_with_value(
193 u"serial", f"file:{self._temp.get(u'log')}"
195 self._params.add_with_value(
196 u"fsdev", u"local,id=root9p,path=/,security_model=none"
198 self._params.add_with_value(
199 u"device", u"virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot"
201 self._params.add_with_value(u"kernel", f"{self._opt.get(u'img')}")
202 self._params.add_with_value(u"initrd", f"{self._opt.get(u'initrd')}")
203 self._params.add_with_value(
204 u"append", f"'ro rootfstype=9p rootflags=trans=virtio "
205 f"root=virtioroot console={self._opt.get(u'console')} "
206 f"tsc=reliable hugepages=256 "
207 f"init={self._temp.get(u'ini')} fastboot'"
210 def create_kernelvm_config_vpp(self, **kwargs):
211 """Create QEMU VPP config files.
213 :param kwargs: Key-value pairs to replace content of VPP configuration
217 startup = f"/etc/vpp/vm_startup_{self._opt.get(u'qemu_id')}.conf"
218 running = f"/etc/vpp/vm_running_{self._opt.get(u'qemu_id')}.exec"
220 self._temp[u"startup"] = startup
221 self._temp[u"running"] = running
222 self._opt[u"vnf_bin"] = f"/usr/bin/vpp -c {startup}"
224 # Create VPP startup configuration.
225 vpp_config = VppConfigGenerator()
226 vpp_config.set_node(self._node)
227 vpp_config.add_unix_nodaemon()
228 vpp_config.add_unix_cli_listen()
229 vpp_config.add_unix_exec(running)
230 vpp_config.add_socksvr()
231 vpp_config.add_cpu_main_core(u"0")
232 if self._opt.get(u"smp") > 1:
233 vpp_config.add_cpu_corelist_workers(f"1-{self._opt.get(u'smp')-1}")
234 vpp_config.add_dpdk_dev(u"0000:00:06.0", u"0000:00:07.0")
235 vpp_config.add_dpdk_dev_default_rxq(kwargs[u"queues"])
236 vpp_config.add_dpdk_log_level(u"debug")
237 if not kwargs[u"jumbo_frames"]:
238 vpp_config.add_dpdk_no_multi_seg()
239 vpp_config.add_dpdk_no_tx_checksum_offload()
240 vpp_config.add_plugin(u"disable", [u"default"])
241 vpp_config.add_plugin(u"enable", [u"dpdk_plugin.so"])
242 vpp_config.write_config(startup)
244 # Create VPP running configuration.
245 template = f"{Constants.RESOURCES_TPL_VM}/{self._opt.get(u'vnf')}.exec"
246 exec_cmd_no_error(self._node, f"rm -f {running}", sudo=True)
248 with open(template, u"rt") as src_file:
249 src = Template(src_file.read())
251 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
252 f"sudo tee {running}"
255 def create_kernelvm_config_testpmd_io(self, **kwargs):
256 """Create QEMU testpmd-io command line.
258 :param kwargs: Key-value pairs to construct command line parameters.
261 testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
262 eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
266 pmd_rxq=kwargs[u"queues"],
267 pmd_txq=kwargs[u"queues"],
268 pmd_tx_offloads='0x0',
269 pmd_disable_hw_vlan=False,
270 pmd_nb_cores=str(self._opt.get(u"smp") - 1)
273 self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
275 def create_kernelvm_config_testpmd_mac(self, **kwargs):
276 """Create QEMU testpmd-mac command line.
278 :param kwargs: Key-value pairs to construct command line parameters.
281 testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
282 eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
287 pmd_eth_peer_0=f"0,{kwargs[u'vif1_mac']}",
288 pmd_eth_peer_1=f"1,{kwargs[u'vif2_mac']}",
289 pmd_rxq=kwargs[u"queues"],
290 pmd_txq=kwargs[u"queues"],
291 pmd_tx_offloads=u"0x0",
292 pmd_disable_hw_vlan=False,
293 pmd_nb_cores=str(self._opt.get(u"smp") - 1)
296 self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
298 def create_kernelvm_init(self, **kwargs):
299 """Create QEMU init script.
301 :param kwargs: Key-value pairs to replace content of init startup file.
304 template = f"{Constants.RESOURCES_TPL_VM}/init.sh"
305 init = self._temp.get(u"ini")
306 exec_cmd_no_error(self._node, f"rm -f {init}", sudo=True)
308 with open(template, u"rt") as src_file:
309 src = Template(src_file.read())
311 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
314 exec_cmd_no_error(self._node, f"chmod +x {init}", sudo=True)
316 def configure_kernelvm_vnf(self, **kwargs):
317 """Create KernelVM VNF configurations.
319 :param kwargs: Key-value pairs for templating configs.
322 if u"vpp" in self._opt.get(u"vnf"):
323 self.create_kernelvm_config_vpp(**kwargs)
324 elif u"testpmd_io" in self._opt.get(u"vnf"):
325 self.create_kernelvm_config_testpmd_io(**kwargs)
326 elif u"testpmd_mac" in self._opt.get(u"vnf"):
327 self.create_kernelvm_config_testpmd_mac(**kwargs)
329 raise RuntimeError(u"QEMU: Unsupported VNF!")
330 self.create_kernelvm_init(vnf_bin=self._opt.get(u"vnf_bin"),
331 unsafe_iommu=self._opt.get(u"unsafe_iommu"))
333 def get_qemu_pids(self):
334 """Get QEMU CPU pids.
336 :returns: List of QEMU CPU pids.
339 command = f"grep -rwl 'CPU' /proc/$(sudo cat " \
340 f"{self._temp.get(u'pidfile')})/task/*/comm "
341 command += r"| xargs dirname | sed -e 's/\/.*\///g' | uniq"
343 stdout, _ = exec_cmd_no_error(self._node, command)
344 return stdout.splitlines()
346 def qemu_set_affinity(self, *host_cpus):
347 """Set qemu affinity by getting thread PIDs via QMP and taskset to list
348 of CPU cores. Function tries to execute 3 times to avoid race condition
349 in getting thread PIDs.
351 :param host_cpus: List of CPU cores.
352 :type host_cpus: list
356 qemu_cpus = self.get_qemu_pids()
358 if len(qemu_cpus) != len(host_cpus):
361 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
362 command = f"taskset -pc {host_cpu} {qemu_cpu}"
363 message = f"QEMU: Set affinity failed " \
364 f"on {self._node[u'host']}!"
366 self._node, command, sudo=True, message=message
369 except (RuntimeError, ValueError):
374 raise RuntimeError(u"Failed to set Qemu threads affinity!")
376 def qemu_set_scheduler_policy(self):
377 """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
380 :raises RuntimeError: Set scheduler policy failed.
383 qemu_cpus = self.get_qemu_pids()
385 for qemu_cpu in qemu_cpus:
386 command = f"chrt -r -p 1 {qemu_cpu}"
387 message = f"QEMU: Set SCHED_RR failed on {self._node[u'host']}"
389 self._node, command, sudo=True, message=message
391 except (RuntimeError, ValueError):
395 def qemu_add_vhost_user_if(
396 self, socket, server=True, jumbo_frames=False, queue_size=None,
397 queues=1, csum=False, gso=False):
398 """Add Vhost-user interface.
400 :param socket: Path of the unix socket.
401 :param server: If True the socket shall be a listening socket.
402 :param jumbo_frames: Set True if jumbo frames are used in the test.
403 :param queue_size: Vring queue size.
404 :param queues: Number of queues.
405 :param csum: Checksum offloading.
406 :param gso: Generic segmentation offloading.
409 :type jumbo_frames: bool
410 :type queue_size: int
416 self._params.add_with_value(
417 u"chardev", f"socket,id=char{self._vhost_id},"
418 f"path={socket}{u',server' if server is True else u''}"
420 self._params.add_with_value(
421 u"netdev", f"vhost-user,id=vhost{self._vhost_id},"
422 f"chardev=char{self._vhost_id},queues={queues}"
424 mac = f"52:54:00:00:{self._opt.get(u'qemu_id'):02x}:" \
425 f"{self._vhost_id:02x}"
426 queue_size = f"rx_queue_size={queue_size},tx_queue_size={queue_size}" \
427 if queue_size else u""
428 self._params.add_with_value(
429 u"device", f"virtio-net-pci,netdev=vhost{self._vhost_id},mac={mac},"
430 f"addr={self._vhost_id+5}.0,mq=on,vectors={2 * queues + 2},"
431 f"csum={u'on' if csum else u'off'},gso={u'on' if gso else u'off'},"
432 f"guest_tso4=off,guest_tso6=off,guest_ecn=off,"
433 f"mrg_rxbuf={u'on,host_mtu=9200' if jumbo_frames else u'off'},"
437 # Add interface MAC and socket to the node dict.
438 if_data = {u"mac_address": mac, u"socket": socket}
439 if_name = f"vhost{self._vhost_id}"
440 self._vm_info[u"interfaces"][if_name] = if_data
441 # Add socket to temporary file list.
442 self._temp[if_name] = socket
444 def _qemu_qmp_exec(self, cmd):
445 """Execute QMP command.
447 QMP is JSON based protocol which allows to control QEMU instance.
449 :param cmd: QMP command to execute.
451 :returns: Command output in python representation of JSON format. The
452 { "return": {} } response is QMP's success response. An error
453 response will contain the "error" keyword instead of "return".
455 # To enter command mode, the qmp_capabilities command must be issued.
456 command = f"echo \"{{{{ \\\"execute\\\": " \
457 f"\\\"qmp_capabilities\\\" }}}}" \
458 f"{{{{ \\\"execute\\\": \\\"{cmd}\\\" }}}}\" | " \
459 f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qmp')}"
460 message = f"QMP execute '{cmd}' failed on {self._node[u'host']}"
462 stdout, _ = exec_cmd_no_error(
463 self._node, command, sudo=False, message=message
466 # Skip capabilities negotiation messages.
467 out_list = stdout.splitlines()
468 if len(out_list) < 3:
469 raise RuntimeError(f"Invalid QMP output on {self._node[u'host']}")
470 return json.loads(out_list[2])
472 def _qemu_qga_flush(self):
473 """Flush the QGA parser state."""
474 command = f"(printf \"\xFF\"; sleep 1) | sudo -S socat " \
475 f"- UNIX-CONNECT:{self._temp.get(u'qga')}"
476 message = f"QGA flush failed on {self._node[u'host']}"
477 stdout, _ = exec_cmd_no_error(
478 self._node, command, sudo=False, message=message
481 return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
483 def _qemu_qga_exec(self, cmd):
484 """Execute QGA command.
486 QGA provide access to a system-level agent via standard QMP commands.
488 :param cmd: QGA command to execute.
491 command = f"(echo \"{{{{ \\\"execute\\\": " \
492 f"\\\"{cmd}\\\" }}}}\"; sleep 1) | " \
493 f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qga')}"
494 message = f"QGA execute '{cmd}' failed on {self._node[u'host']}"
495 stdout, _ = exec_cmd_no_error(
496 self._node, command, sudo=False, message=message
499 return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
501 def _wait_until_vm_boot(self):
502 """Wait until QEMU with NestedVM is booted."""
503 if self._opt.get(u"vm_type") == u"nestedvm":
504 self._wait_until_nestedvm_boot()
505 self._update_vm_interfaces()
506 elif self._opt.get(u"vm_type") == u"kernelvm":
507 self._wait_until_kernelvm_boot()
509 raise RuntimeError(u"QEMU: Unsupported VM type!")
511 def _wait_until_nestedvm_boot(self, retries=12):
512 """Wait until QEMU with NestedVM is booted.
514 First try to flush qga until there is output.
515 Then ping QEMU guest agent each 5s until VM booted or timeout.
517 :param retries: Number of retries with 5s between trials.
520 for _ in range(retries):
523 out = self._qemu_qga_flush()
525 logger.trace(f"QGA qga flush unexpected output {out}")
526 # Empty output - VM not booted yet
533 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
535 for _ in range(retries):
538 out = self._qemu_qga_exec(u"guest-ping")
540 logger.trace(f"QGA guest-ping unexpected output {out}")
541 # Empty output - VM not booted yet.
544 # Non-error return - VM booted.
545 elif out.get(u"return") is not None:
547 # Skip error and wait.
548 elif out.get(u"error") is not None:
551 # If there is an unexpected output from QGA guest-info, try
552 # again until timeout.
553 logger.trace(f"QGA guest-ping unexpected output {out}")
556 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
559 def _wait_until_kernelvm_boot(self, retries=60):
560 """Wait until QEMU KernelVM is booted.
562 :param retries: Number of retries.
565 vpp_ver = VPPUtil.vpp_show_version(self._node)
567 for _ in range(retries):
568 command = f"tail -1 {self._temp.get(u'log')}"
571 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
575 if vpp_ver in stdout or u"Press enter to exit" in stdout:
577 if u"reboot: Power down" in stdout:
579 f"QEMU: NF failed to run on {self._node[u'host']}!"
583 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
586 def _update_vm_interfaces(self):
587 """Update interface names in VM node dict."""
588 # Send guest-network-get-interfaces command via QGA, output example:
589 # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
590 # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
591 out = self._qemu_qga_exec(u"guest-network-get-interfaces")
592 interfaces = out.get(u"return")
596 f"Get VM interface list failed on {self._node[u'host']}"
598 # Create MAC-name dict.
599 for interface in interfaces:
600 if u"hardware-address" not in interface:
602 mac_name[interface[u"hardware-address"]] = interface[u"name"]
603 # Match interface by MAC and save interface name.
604 for interface in self._vm_info[u"interfaces"].values():
605 mac = interface.get(u"mac_address")
606 if_name = mac_name.get(mac)
608 logger.trace(f"Interface name for MAC {mac} not found")
610 interface[u"name"] = if_name
612 def qemu_start(self):
613 """Start QEMU and wait until VM boot.
615 :returns: VM node info.
618 cmd_opts = OptionString()
619 cmd_opts.add(f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch}")
620 cmd_opts.extend(self._params)
621 message = f"QEMU: Start failed on {self._node[u'host']}!"
623 DUTSetup.check_huge_page(
624 self._node, u"/dev/hugepages", int(self._opt.get(u"mem")))
627 self._node, cmd_opts, timeout=300, sudo=True, message=message
629 self._wait_until_vm_boot()
636 """Kill qemu process."""
638 self._node, f"chmod +r {self._temp.get(u'pidfile')}", sudo=True
641 self._node, f"kill -SIGKILL $(cat {self._temp.get(u'pidfile')})",
645 for value in self._temp.values():
646 exec_cmd(self._node, f"cat {value}", sudo=True)
647 exec_cmd(self._node, f"rm -f {value}", sudo=True)
649 def qemu_kill_all(self):
650 """Kill all qemu processes on DUT node if specified."""
651 exec_cmd(self._node, u"pkill -SIGKILL qemu", sudo=True)
653 for value in self._temp.values():
654 exec_cmd(self._node, f"cat {value}", sudo=True)
655 exec_cmd(self._node, f"rm -f {value}", sudo=True)
657 def qemu_version(self):
658 """Return Qemu version.
660 :returns: Qemu version.
663 command = f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch} " \
666 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
667 return match(r"QEMU emulator version ([\d.]*)", stdout).group(1)