1 # Copyright (c) 2020 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """QEMU utilities library."""
19 from string import Template
20 from time import sleep
22 from robot.api import logger
24 from resources.libraries.python.Constants import Constants
25 from resources.libraries.python.DpdkUtil import DpdkUtil
26 from resources.libraries.python.DUTSetup import DUTSetup
27 from resources.libraries.python.OptionString import OptionString
28 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
29 from resources.libraries.python.topology import NodeType, Topology
30 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
31 from resources.libraries.python.VPPUtil import VPPUtil
33 __all__ = [u"QemuUtils"]
39 # Use one instance of class per tests.
40 ROBOT_LIBRARY_SCOPE = u"TEST CASE"
43 self, node, qemu_id=1, smp=1, mem=512, vnf=None,
44 img=Constants.QEMU_VM_IMAGE):
45 """Initialize QemuUtil class.
47 :param node: Node to run QEMU on.
48 :param qemu_id: QEMU identifier.
49 :param smp: Number of virtual SMP units (cores).
50 :param mem: Amount of memory.
51 :param vnf: Network function workload.
52 :param img: QEMU disk image or kernel image path.
62 self._arch = Topology.get_node_arch(self._node)
65 # Architecture specific options
66 if self._arch == u"aarch64":
67 dpdk_target = u"arm64-armv8a"
68 self._opt[u"machine_args"] = \
69 u"virt,accel=kvm,usb=off,mem-merge=off,gic-version=3"
70 self._opt[u"console"] = u"ttyAMA0"
72 dpdk_target = u"x86_64-native"
73 self._opt[u"machine_args"] = u"pc,accel=kvm,usb=off,mem-merge=off"
74 self._opt[u"console"] = u"ttyS0"
75 self._testpmd_path = f"{Constants.QEMU_VM_DPDK}/" \
76 f"{dpdk_target}-linux-gcc/app"
78 u"host": node[u"host"],
80 u"port": 10021 + qemu_id,
81 u"serial": 4555 + qemu_id,
86 if node[u"port"] != 22:
87 self._vm_info[u"host_port"] = node[u"port"]
88 self._vm_info[u"host_username"] = node[u"username"]
89 self._vm_info[u"host_password"] = node[u"password"]
91 self._opt[u"qemu_id"] = qemu_id
92 self._opt[u"mem"] = int(mem)
93 self._opt[u"smp"] = int(smp)
94 self._opt[u"img"] = img
95 self._opt[u"vnf"] = vnf
98 self._temp[u"pidfile"] = f"/run/qemu_{qemu_id}.pid"
99 if img == Constants.QEMU_VM_IMAGE:
100 self._opt[u"vm_type"] = u"nestedvm"
101 self._temp[u"qmp"] = f"/run/qmp_{qemu_id}.sock"
102 self._temp[u"qga"] = f"/run/qga_{qemu_id}.sock"
103 elif img == Constants.QEMU_VM_KERNEL:
104 self._opt[u"img"], _ = exec_cmd_no_error(
105 node, f"ls -1 {Constants.QEMU_VM_KERNEL}* | tail -1",
106 message=u"Qemu Kernel VM image not found!"
108 self._opt[u"vm_type"] = u"kernelvm"
109 self._temp[u"log"] = f"/tmp/serial_{qemu_id}.log"
110 self._temp[u"ini"] = f"/etc/vm_init_{qemu_id}.conf"
111 self._opt[u"initrd"], _ = exec_cmd_no_error(
112 node, f"ls -1 {Constants.QEMU_VM_KERNEL_INITRD}* | tail -1",
113 message=u"Qemu Kernel initrd image not found!"
116 raise RuntimeError(f"QEMU: Unknown VM image option: {img}")
117 # Computed parameters for QEMU command line.
118 self._params = OptionString(prefix=u"-")
121 def add_params(self):
122 """Set QEMU command line parameters."""
123 self.add_default_params()
124 if self._opt.get(u"vm_type", u"") == u"nestedvm":
125 self.add_nestedvm_params()
126 elif self._opt.get(u"vm_type", u"") == u"kernelvm":
127 self.add_kernelvm_params()
129 raise RuntimeError(u"QEMU: Unsupported VM type!")
131 def add_default_params(self):
132 """Set default QEMU command line parameters."""
133 self._params.add(u"daemonize")
134 self._params.add(u"nodefaults")
135 self._params.add_with_value(
136 u"name", f"vnf{self._opt.get(u'qemu_id')},debug-threads=on"
138 self._params.add(u"no-user-config")
139 self._params.add_with_value(u"monitor", u"none")
140 self._params.add_with_value(u"display", u"none")
141 self._params.add_with_value(u"vga", u"none")
142 self._params.add(u"enable-kvm")
143 self._params.add_with_value(u"pidfile", self._temp.get(u"pidfile"))
144 self._params.add_with_value(u"cpu", u"host")
146 self._params.add_with_value(u"machine", self._opt.get(u"machine_args"))
147 self._params.add_with_value(
148 u"smp", f"{self._opt.get(u'smp')},sockets=1,"
149 f"cores={self._opt.get(u'smp')},threads=1"
151 self._params.add_with_value(
152 u"object", f"memory-backend-file,id=mem,"
153 f"size={self._opt.get(u'mem')}M,mem-path=/dev/hugepages,share=on"
155 self._params.add_with_value(u"m", f"{self._opt.get(u'mem')}M")
156 self._params.add_with_value(u"numa", u"node,memdev=mem")
157 self._params.add_with_value(u"balloon", u"none")
159 def add_nestedvm_params(self):
160 """Set NestedVM QEMU parameters."""
161 self._params.add_with_value(
163 f"nic,macaddr=52:54:00:00:{self._opt.get(u'qemu_id'):02x}:ff"
165 self._params.add_with_value(
166 u"net", f"user,hostfwd=tcp::{self._vm_info[u'port']}-:22"
168 locking = u",file.locking=off"
169 self._params.add_with_value(
170 u"drive", f"file={self._opt.get(u'img')},"
171 f"format=raw,cache=none,if=virtio{locking}"
173 self._params.add_with_value(
174 u"qmp", f"unix:{self._temp.get(u'qmp')},server,nowait"
176 self._params.add_with_value(
177 u"chardev", f"socket,host=127.0.0.1,"
178 f"port={self._vm_info[u'serial']},id=gnc0,server,nowait")
179 self._params.add_with_value(u"device", u"isa-serial,chardev=gnc0")
180 self._params.add_with_value(
181 u"chardev", f"socket,path={self._temp.get(u'qga')},"
182 f"server,nowait,id=qga0"
184 self._params.add_with_value(u"device", u"isa-serial,chardev=qga0")
186 def add_kernelvm_params(self):
187 """Set KernelVM QEMU parameters."""
188 self._params.add_with_value(
189 u"serial", f"file:{self._temp.get(u'log')}"
191 self._params.add_with_value(
192 u"fsdev", u"local,id=root9p,path=/,security_model=none"
194 self._params.add_with_value(
195 u"device", u"virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot"
197 self._params.add_with_value(u"kernel", f"{self._opt.get(u'img')}")
198 self._params.add_with_value(u"initrd", f"{self._opt.get(u'initrd')}")
199 self._params.add_with_value(
200 u"append", f"'ro rootfstype=9p rootflags=trans=virtio "
201 f"root=virtioroot console={self._opt.get(u'console')} "
202 f"tsc=reliable hugepages=256 "
203 f"init={self._temp.get(u'ini')} fastboot'"
206 def create_kernelvm_config_vpp(self, **kwargs):
207 """Create QEMU VPP config files.
209 :param kwargs: Key-value pairs to replace content of VPP configuration
213 startup = f"/etc/vpp/vm_startup_{self._opt.get(u'qemu_id')}.conf"
214 running = f"/etc/vpp/vm_running_{self._opt.get(u'qemu_id')}.exec"
216 self._temp[u"startup"] = startup
217 self._temp[u"running"] = running
218 self._opt[u"vnf_bin"] = f"/usr/bin/vpp -c {startup}"
220 # Create VPP startup configuration.
221 vpp_config = VppConfigGenerator()
222 vpp_config.set_node(self._node)
223 vpp_config.add_unix_nodaemon()
224 vpp_config.add_unix_cli_listen()
225 vpp_config.add_unix_exec(running)
226 vpp_config.add_socksvr()
227 vpp_config.add_cpu_main_core(u"0")
228 if self._opt.get(u"smp") > 1:
229 vpp_config.add_cpu_corelist_workers(f"1-{self._opt.get(u'smp')-1}")
230 vpp_config.add_dpdk_dev(u"0000:00:06.0", u"0000:00:07.0")
231 vpp_config.add_dpdk_dev_default_rxq(kwargs[u"queues"])
232 vpp_config.add_dpdk_log_level(u"debug")
233 if not kwargs[u"jumbo_frames"]:
234 vpp_config.add_dpdk_no_multi_seg()
235 vpp_config.add_dpdk_no_tx_checksum_offload()
236 vpp_config.add_plugin(u"disable", u"default")
237 vpp_config.add_plugin(u"enable", u"dpdk_plugin.so")
238 vpp_config.write_config(startup)
240 # Create VPP running configuration.
241 template = f"{Constants.RESOURCES_TPL_VM}/{self._opt.get(u'vnf')}.exec"
242 exec_cmd_no_error(self._node, f"rm -f {running}", sudo=True)
244 with open(template, u"rt") as src_file:
245 src = Template(src_file.read())
247 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
248 f"sudo tee {running}"
251 def create_kernelvm_config_testpmd_io(self, **kwargs):
252 """Create QEMU testpmd-io command line.
254 :param kwargs: Key-value pairs to construct command line parameters.
257 pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
258 testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
259 eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
261 eal_pci_whitelist0=u"0000:00:06.0",
262 eal_pci_whitelist1=u"0000:00:07.0",
268 pmd_max_pkt_len=pmd_max_pkt_len,
269 pmd_mbuf_size=u"16384",
270 pmd_rxq=kwargs[u"queues"],
271 pmd_txq=kwargs[u"queues"],
272 pmd_tx_offloads='0x0',
273 pmd_nb_cores=str(self._opt.get(u"smp") - 1)
276 self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
278 def create_kernelvm_config_testpmd_mac(self, **kwargs):
279 """Create QEMU testpmd-mac command line.
281 :param kwargs: Key-value pairs to construct command line parameters.
284 pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
285 testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
286 eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
288 eal_pci_whitelist0=u"0000:00:06.0",
289 eal_pci_whitelist1=u"0000:00:07.0",
295 pmd_max_pkt_len=pmd_max_pkt_len,
296 pmd_mbuf_size=u"16384",
297 pmd_eth_peer_0=f"0,{kwargs[u'vif1_mac']}",
298 pmd_eth_peer_1=f"1,{kwargs[u'vif2_mac']}",
299 pmd_rxq=kwargs[u"queues"],
300 pmd_txq=kwargs[u"queues"],
301 pmd_tx_offloads=u"0x0",
302 pmd_nb_cores=str(self._opt.get(u"smp") - 1)
305 self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
307 def create_kernelvm_init(self, **kwargs):
308 """Create QEMU init script.
310 :param kwargs: Key-value pairs to replace content of init startup file.
313 template = f"{Constants.RESOURCES_TPL_VM}/init.sh"
314 init = self._temp.get(u"ini")
315 exec_cmd_no_error(self._node, f"rm -f {init}", sudo=True)
317 with open(template, u"rt") as src_file:
318 src = Template(src_file.read())
320 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
323 exec_cmd_no_error(self._node, f"chmod +x {init}", sudo=True)
325 def configure_kernelvm_vnf(self, **kwargs):
326 """Create KernelVM VNF configurations.
328 :param kwargs: Key-value pairs for templating configs.
331 if u"vpp" in self._opt.get(u"vnf"):
332 self.create_kernelvm_config_vpp(**kwargs)
333 elif u"testpmd_io" in self._opt.get(u"vnf"):
334 self.create_kernelvm_config_testpmd_io(**kwargs)
335 elif u"testpmd_mac" in self._opt.get(u"vnf"):
336 self.create_kernelvm_config_testpmd_mac(**kwargs)
338 raise RuntimeError(u"QEMU: Unsupported VNF!")
339 self.create_kernelvm_init(vnf_bin=self._opt.get(u"vnf_bin"))
341 def get_qemu_pids(self):
342 """Get QEMU CPU pids.
344 :returns: List of QEMU CPU pids.
347 command = f"grep -rwl 'CPU' /proc/$(sudo cat " \
348 f"{self._temp.get(u'pidfile')})/task/*/comm "
349 command += r"| xargs dirname | sed -e 's/\/.*\///g' | uniq"
351 stdout, _ = exec_cmd_no_error(self._node, command)
352 return stdout.splitlines()
354 def qemu_set_affinity(self, *host_cpus):
355 """Set qemu affinity by getting thread PIDs via QMP and taskset to list
356 of CPU cores. Function tries to execute 3 times to avoid race condition
357 in getting thread PIDs.
359 :param host_cpus: List of CPU cores.
360 :type host_cpus: list
364 qemu_cpus = self.get_qemu_pids()
366 if len(qemu_cpus) != len(host_cpus):
369 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
370 command = f"taskset -pc {host_cpu} {qemu_cpu}"
371 message = f"QEMU: Set affinity failed " \
372 f"on {self._node[u'host']}!"
374 self._node, command, sudo=True, message=message
377 except (RuntimeError, ValueError):
382 raise RuntimeError(u"Failed to set Qemu threads affinity!")
384 def qemu_set_scheduler_policy(self):
385 """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
388 :raises RuntimeError: Set scheduler policy failed.
391 qemu_cpus = self.get_qemu_pids()
393 for qemu_cpu in qemu_cpus:
394 command = f"chrt -r -p 1 {qemu_cpu}"
395 message = f"QEMU: Set SCHED_RR failed on {self._node[u'host']}"
397 self._node, command, sudo=True, message=message
399 except (RuntimeError, ValueError):
403 def qemu_add_vhost_user_if(
404 self, socket, server=True, jumbo_frames=False, queue_size=None,
405 queues=1, csum=False, gso=False):
406 """Add Vhost-user interface.
408 :param socket: Path of the unix socket.
409 :param server: If True the socket shall be a listening socket.
410 :param jumbo_frames: Set True if jumbo frames are used in the test.
411 :param queue_size: Vring queue size.
412 :param queues: Number of queues.
413 :param csum: Checksum offloading.
414 :param gso: Generic segmentation offloading.
417 :type jumbo_frames: bool
418 :type queue_size: int
424 self._params.add_with_value(
425 u"chardev", f"socket,id=char{self._vhost_id},"
426 f"path={socket}{u',server' if server is True else u''}"
428 self._params.add_with_value(
429 u"netdev", f"vhost-user,id=vhost{self._vhost_id},"
430 f"chardev=char{self._vhost_id},queues={queues}"
432 mac = f"52:54:00:00:{self._opt.get(u'qemu_id'):02x}:" \
433 f"{self._vhost_id:02x}"
434 queue_size = f"rx_queue_size={queue_size},tx_queue_size={queue_size}" \
435 if queue_size else u""
436 self._params.add_with_value(
437 u"device", f"virtio-net-pci,netdev=vhost{self._vhost_id},mac={mac},"
438 f"addr={self._vhost_id+5}.0,mq=on,vectors={2 * queues + 2},"
439 f"csum={u'on' if csum else u'off'},gso={u'on' if gso else u'off'},"
440 f"guest_tso4=off,guest_tso6=off,guest_ecn=off,"
444 # Add interface MAC and socket to the node dict.
445 if_data = {u"mac_address": mac, u"socket": socket}
446 if_name = f"vhost{self._vhost_id}"
447 self._vm_info[u"interfaces"][if_name] = if_data
448 # Add socket to temporary file list.
449 self._temp[if_name] = socket
451 def _qemu_qmp_exec(self, cmd):
452 """Execute QMP command.
454 QMP is JSON based protocol which allows to control QEMU instance.
456 :param cmd: QMP command to execute.
458 :returns: Command output in python representation of JSON format. The
459 { "return": {} } response is QMP's success response. An error
460 response will contain the "error" keyword instead of "return".
462 # To enter command mode, the qmp_capabilities command must be issued.
463 command = f"echo \"{{{{ \\\"execute\\\": " \
464 f"\\\"qmp_capabilities\\\" }}}}" \
465 f"{{{{ \\\"execute\\\": \\\"{cmd}\\\" }}}}\" | " \
466 f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qmp')}"
467 message = f"QMP execute '{cmd}' failed on {self._node[u'host']}"
469 stdout, _ = exec_cmd_no_error(
470 self._node, command, sudo=False, message=message
473 # Skip capabilities negotiation messages.
474 out_list = stdout.splitlines()
475 if len(out_list) < 3:
476 raise RuntimeError(f"Invalid QMP output on {self._node[u'host']}")
477 return json.loads(out_list[2])
479 def _qemu_qga_flush(self):
480 """Flush the QGA parser state."""
481 command = f"(printf \"\xFF\"; sleep 1) | sudo -S socat " \
482 f"- UNIX-CONNECT:{self._temp.get(u'qga')}"
483 message = f"QGA flush failed on {self._node[u'host']}"
484 stdout, _ = exec_cmd_no_error(
485 self._node, command, sudo=False, message=message
488 return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
490 def _qemu_qga_exec(self, cmd):
491 """Execute QGA command.
493 QGA provide access to a system-level agent via standard QMP commands.
495 :param cmd: QGA command to execute.
498 command = f"(echo \"{{{{ \\\"execute\\\": " \
499 f"\\\"{cmd}\\\" }}}}\"; sleep 1) | " \
500 f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qga')}"
501 message = f"QGA execute '{cmd}' failed on {self._node[u'host']}"
502 stdout, _ = exec_cmd_no_error(
503 self._node, command, sudo=False, message=message
506 return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
508 def _wait_until_vm_boot(self):
509 """Wait until QEMU with NestedVM is booted."""
510 if self._opt.get(u"vm_type") == u"nestedvm":
511 self._wait_until_nestedvm_boot()
512 self._update_vm_interfaces()
513 elif self._opt.get(u"vm_type") == u"kernelvm":
514 self._wait_until_kernelvm_boot()
516 raise RuntimeError(u"QEMU: Unsupported VM type!")
518 def _wait_until_nestedvm_boot(self, retries=12):
519 """Wait until QEMU with NestedVM is booted.
521 First try to flush qga until there is output.
522 Then ping QEMU guest agent each 5s until VM booted or timeout.
524 :param retries: Number of retries with 5s between trials.
527 for _ in range(retries):
530 out = self._qemu_qga_flush()
532 logger.trace(f"QGA qga flush unexpected output {out}")
533 # Empty output - VM not booted yet
540 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
542 for _ in range(retries):
545 out = self._qemu_qga_exec(u"guest-ping")
547 logger.trace(f"QGA guest-ping unexpected output {out}")
548 # Empty output - VM not booted yet.
551 # Non-error return - VM booted.
552 elif out.get(u"return") is not None:
554 # Skip error and wait.
555 elif out.get(u"error") is not None:
558 # If there is an unexpected output from QGA guest-info, try
559 # again until timeout.
560 logger.trace(f"QGA guest-ping unexpected output {out}")
563 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
566 def _wait_until_kernelvm_boot(self, retries=60):
567 """Wait until QEMU KernelVM is booted.
569 :param retries: Number of retries.
572 vpp_ver = VPPUtil.vpp_show_version(self._node)
574 for _ in range(retries):
575 command = f"tail -1 {self._temp.get(u'log')}"
578 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
582 if vpp_ver in stdout or u"Press enter to exit" in stdout:
584 if u"reboot: Power down" in stdout:
586 f"QEMU: NF failed to run on {self._node[u'host']}!"
590 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
593 def _update_vm_interfaces(self):
594 """Update interface names in VM node dict."""
595 # Send guest-network-get-interfaces command via QGA, output example:
596 # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
597 # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
598 out = self._qemu_qga_exec(u"guest-network-get-interfaces")
599 interfaces = out.get(u"return")
603 f"Get VM interface list failed on {self._node[u'host']}"
605 # Create MAC-name dict.
606 for interface in interfaces:
607 if u"hardware-address" not in interface:
609 mac_name[interface[u"hardware-address"]] = interface[u"name"]
610 # Match interface by MAC and save interface name.
611 for interface in self._vm_info[u"interfaces"].values():
612 mac = interface.get(u"mac_address")
613 if_name = mac_name.get(mac)
615 logger.trace(f"Interface name for MAC {mac} not found")
617 interface[u"name"] = if_name
619 def qemu_start(self):
620 """Start QEMU and wait until VM boot.
622 :returns: VM node info.
625 cmd_opts = OptionString()
626 cmd_opts.add(f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch}")
627 cmd_opts.extend(self._params)
628 message = f"QEMU: Start failed on {self._node[u'host']}!"
630 DUTSetup.check_huge_page(
631 self._node, u"/dev/hugepages", int(self._opt.get(u"mem")))
634 self._node, cmd_opts, timeout=300, sudo=True, message=message
636 self._wait_until_vm_boot()
643 """Kill qemu process."""
645 self._node, f"chmod +r {self._temp.get(u'pidfile')}", sudo=True
648 self._node, f"kill -SIGKILL $(cat {self._temp.get(u'pidfile')})",
652 for value in self._temp.values():
653 exec_cmd(self._node, f"cat {value}", sudo=True)
654 exec_cmd(self._node, f"rm -f {value}", sudo=True)
656 def qemu_kill_all(self):
657 """Kill all qemu processes on DUT node if specified."""
658 exec_cmd(self._node, u"pkill -SIGKILL qemu", sudo=True)
660 for value in self._temp.values():
661 exec_cmd(self._node, f"cat {value}", sudo=True)
662 exec_cmd(self._node, f"rm -f {value}", sudo=True)
664 def qemu_version(self):
665 """Return Qemu version.
667 :returns: Qemu version.
670 command = f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch} " \
673 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
674 return match(r"QEMU emulator version ([\d.]*)", stdout).group(1)