1 # Copyright (c) 2020 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """QEMU utilities library."""
19 from string import Template
20 from time import sleep
22 from robot.api import logger
24 from resources.libraries.python.Constants import Constants
25 from resources.libraries.python.DpdkUtil import DpdkUtil
26 from resources.libraries.python.DUTSetup import DUTSetup
27 from resources.libraries.python.OptionString import OptionString
28 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
29 from resources.libraries.python.topology import NodeType, Topology
30 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
31 from resources.libraries.python.VPPUtil import VPPUtil
33 __all__ = [u"QemuUtils"]
39 # Use one instance of class per tests.
40 ROBOT_LIBRARY_SCOPE = u"TEST CASE"
43 self, node, qemu_id=1, smp=1, mem=512, vnf=None,
44 img=Constants.QEMU_VM_IMAGE):
45 """Initialize QemuUtil class.
47 :param node: Node to run QEMU on.
48 :param qemu_id: QEMU identifier.
49 :param smp: Number of virtual SMP units (cores).
50 :param mem: Amount of memory.
51 :param vnf: Network function workload.
52 :param img: QEMU disk image or kernel image path.
62 self._arch = Topology.get_node_arch(self._node)
65 # Architecture specific options
66 if self._arch == u"aarch64":
67 dpdk_target = u"arm64-armv8a"
68 self._opt[u"machine_args"] = \
69 u"virt,accel=kvm,usb=off,mem-merge=off,gic-version=3"
70 self._opt[u"console"] = u"ttyAMA0"
72 dpdk_target = u"x86_64-native"
73 self._opt[u"machine_args"] = u"pc,accel=kvm,usb=off,mem-merge=off"
74 self._opt[u"console"] = u"ttyS0"
75 self._testpmd_path = f"{Constants.QEMU_VM_DPDK}/" \
76 f"{dpdk_target}-linux-gcc/app"
78 u"host": node[u"host"],
80 u"port": 10021 + qemu_id,
81 u"serial": 4555 + qemu_id,
86 if node[u"port"] != 22:
87 self._vm_info[u"host_port"] = node[u"port"]
88 self._vm_info[u"host_username"] = node[u"username"]
89 self._vm_info[u"host_password"] = node[u"password"]
91 self._opt[u"qemu_id"] = qemu_id
92 self._opt[u"mem"] = int(mem)
93 self._opt[u"smp"] = int(smp)
94 self._opt[u"img"] = img
95 self._opt[u"vnf"] = vnf
98 self._temp[u"pidfile"] = f"/run/qemu_{qemu_id}.pid"
99 if img == Constants.QEMU_VM_IMAGE:
100 self._opt[u"vm_type"] = u"nestedvm"
101 self._temp[u"qmp"] = f"/run/qmp_{qemu_id}.sock"
102 self._temp[u"qga"] = f"/run/qga_{qemu_id}.sock"
103 elif img == Constants.QEMU_VM_KERNEL:
104 self._opt[u"img"], _ = exec_cmd_no_error(
105 node, f"ls -1 {Constants.QEMU_VM_KERNEL}* | tail -1",
106 message=u"Qemu Kernel VM image not found!"
108 self._opt[u"vm_type"] = u"kernelvm"
109 self._temp[u"log"] = f"/tmp/serial_{qemu_id}.log"
110 self._temp[u"ini"] = f"/etc/vm_init_{qemu_id}.conf"
111 self._opt[u"initrd"], _ = exec_cmd_no_error(
112 node, f"ls -1 {Constants.QEMU_VM_KERNEL_INITRD}* | tail -1",
113 message=u"Qemu Kernel initrd image not found!"
116 raise RuntimeError(f"QEMU: Unknown VM image option: {img}")
117 # Computed parameters for QEMU command line.
118 self._params = OptionString(prefix=u"-")
121 def add_params(self):
122 """Set QEMU command line parameters."""
123 self.add_default_params()
124 if self._opt.get(u"vm_type", u"") == u"nestedvm":
125 self.add_nestedvm_params()
126 elif self._opt.get(u"vm_type", u"") == u"kernelvm":
127 self.add_kernelvm_params()
129 raise RuntimeError(u"QEMU: Unsupported VM type!")
131 def add_default_params(self):
132 """Set default QEMU command line parameters."""
133 self._params.add(u"daemonize")
134 self._params.add(u"nodefaults")
135 self._params.add_with_value(
136 u"name", f"vnf{self._opt.get(u'qemu_id')},debug-threads=on"
138 self._params.add(u"no-user-config")
139 self._params.add_with_value(u"monitor", u"none")
140 self._params.add_with_value(u"display", u"none")
141 self._params.add_with_value(u"vga", u"none")
142 self._params.add(u"enable-kvm")
143 self._params.add_with_value(u"pidfile", self._temp.get(u"pidfile"))
144 self._params.add_with_value(u"cpu", u"host")
146 self._params.add_with_value(u"machine", self._opt.get(u"machine_args"))
147 self._params.add_with_value(
148 u"smp", f"{self._opt.get(u'smp')},sockets=1,"
149 f"cores={self._opt.get(u'smp')},threads=1"
151 self._params.add_with_value(
152 u"object", f"memory-backend-file,id=mem,"
153 f"size={self._opt.get(u'mem')}M,mem-path=/dev/hugepages,share=on"
155 self._params.add_with_value(u"m", f"{self._opt.get(u'mem')}M")
156 self._params.add_with_value(u"numa", u"node,memdev=mem")
157 self._params.add_with_value(u"balloon", u"none")
159 def add_nestedvm_params(self):
160 """Set NestedVM QEMU parameters."""
161 self._params.add_with_value(
163 f"nic,macaddr=52:54:00:00:{self._opt.get(u'qemu_id'):02x}:ff"
165 self._params.add_with_value(
166 u"net", f"user,hostfwd=tcp::{self._vm_info[u'port']}-:22"
168 locking = u",file.locking=off"
169 self._params.add_with_value(
170 u"drive", f"file={self._opt.get(u'img')},"
171 f"format=raw,cache=none,if=virtio{locking}"
173 self._params.add_with_value(
174 u"qmp", f"unix:{self._temp.get(u'qmp')},server,nowait"
176 self._params.add_with_value(
177 u"chardev", f"socket,host=127.0.0.1,"
178 f"port={self._vm_info[u'serial']},id=gnc0,server,nowait")
179 self._params.add_with_value(u"device", u"isa-serial,chardev=gnc0")
180 self._params.add_with_value(
181 u"chardev", f"socket,path={self._temp.get(u'qga')},"
182 f"server,nowait,id=qga0"
184 self._params.add_with_value(u"device", u"isa-serial,chardev=qga0")
186 def add_kernelvm_params(self):
187 """Set KernelVM QEMU parameters."""
188 self._params.add_with_value(
189 u"serial", f"file:{self._temp.get(u'log')}"
191 self._params.add_with_value(
192 u"fsdev", u"local,id=root9p,path=/,security_model=none"
194 self._params.add_with_value(
195 u"device", u"virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot"
197 self._params.add_with_value(u"kernel", f"{self._opt.get(u'img')}")
198 self._params.add_with_value(u"initrd", f"{self._opt.get(u'initrd')}")
199 self._params.add_with_value(
200 u"append", f"'ro rootfstype=9p rootflags=trans=virtio "
201 f"root=virtioroot console={self._opt.get(u'console')} "
202 f"tsc=reliable hugepages=256 "
203 f"init={self._temp.get(u'ini')} fastboot'"
206 def create_kernelvm_config_vpp(self, **kwargs):
207 """Create QEMU VPP config files.
209 :param kwargs: Key-value pairs to replace content of VPP configuration
213 startup = f"/etc/vpp/vm_startup_{self._opt.get(u'qemu_id')}.conf"
214 running = f"/etc/vpp/vm_running_{self._opt.get(u'qemu_id')}.exec"
216 self._temp[u"startup"] = startup
217 self._temp[u"running"] = running
218 self._opt[u"vnf_bin"] = f"/usr/bin/vpp -c {startup}"
220 # Create VPP startup configuration.
221 vpp_config = VppConfigGenerator()
222 vpp_config.set_node(self._node)
223 vpp_config.add_unix_nodaemon()
224 vpp_config.add_unix_cli_listen()
225 vpp_config.add_unix_exec(running)
226 vpp_config.add_socksvr()
227 vpp_config.add_cpu_main_core(u"0")
228 if self._opt.get(u"smp") > 1:
229 vpp_config.add_cpu_corelist_workers(f"1-{self._opt.get(u'smp')-1}")
230 vpp_config.add_plugin(u"disable", u"default")
231 if "virtio" not in self._opt.get(u'vnf'):
232 vpp_config.add_plugin(u"enable", u"dpdk_plugin.so")
233 vpp_config.add_dpdk_dev(u"0000:00:06.0", u"0000:00:07.0")
234 vpp_config.add_dpdk_dev_default_rxq(kwargs[u"queues"])
235 vpp_config.add_dpdk_log_level(u"debug")
236 if not kwargs[u"jumbo_frames"]:
237 vpp_config.add_dpdk_no_multi_seg()
238 vpp_config.add_dpdk_no_tx_checksum_offload()
239 vpp_config.write_config(startup)
241 # Create VPP running configuration.
242 template = f"{Constants.RESOURCES_TPL_VM}/{self._opt.get(u'vnf')}.exec"
243 exec_cmd_no_error(self._node, f"rm -f {running}", sudo=True)
245 with open(template, u"rt") as src_file:
246 src = Template(src_file.read())
248 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
249 f"sudo tee {running}"
252 def create_kernelvm_config_testpmd_io(self, **kwargs):
253 """Create QEMU testpmd-io command line.
255 :param kwargs: Key-value pairs to construct command line parameters.
258 pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
259 testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
260 eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
262 eal_pci_whitelist0=u"0000:00:06.0",
263 eal_pci_whitelist1=u"0000:00:07.0",
269 pmd_max_pkt_len=pmd_max_pkt_len,
270 pmd_mbuf_size=u"16384",
271 pmd_rxq=kwargs[u"queues"],
272 pmd_txq=kwargs[u"queues"],
273 pmd_tx_offloads='0x0',
274 pmd_nb_cores=str(self._opt.get(u"smp") - 1)
277 self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
279 def create_kernelvm_config_testpmd_mac(self, **kwargs):
280 """Create QEMU testpmd-mac command line.
282 :param kwargs: Key-value pairs to construct command line parameters.
285 pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
286 testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
287 eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
289 eal_pci_whitelist0=u"0000:00:06.0",
290 eal_pci_whitelist1=u"0000:00:07.0",
296 pmd_max_pkt_len=pmd_max_pkt_len,
297 pmd_mbuf_size=u"16384",
298 pmd_eth_peer_0=f"0,{kwargs[u'vif1_mac']}",
299 pmd_eth_peer_1=f"1,{kwargs[u'vif2_mac']}",
300 pmd_rxq=kwargs[u"queues"],
301 pmd_txq=kwargs[u"queues"],
302 pmd_tx_offloads=u"0x0",
303 pmd_nb_cores=str(self._opt.get(u"smp") - 1)
306 self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
308 def create_kernelvm_init(self, **kwargs):
309 """Create QEMU init script.
311 :param kwargs: Key-value pairs to replace content of init startup file.
314 template = f"{Constants.RESOURCES_TPL_VM}/init.sh"
315 init = self._temp.get(u"ini")
316 exec_cmd_no_error(self._node, f"rm -f {init}", sudo=True)
318 with open(template, u"rt") as src_file:
319 src = Template(src_file.read())
321 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
324 exec_cmd_no_error(self._node, f"chmod +x {init}", sudo=True)
326 def configure_kernelvm_vnf(self, **kwargs):
327 """Create KernelVM VNF configurations.
329 :param kwargs: Key-value pairs for templating configs.
332 if u"vpp" in self._opt.get(u"vnf"):
333 self.create_kernelvm_config_vpp(**kwargs)
334 elif u"testpmd_io" in self._opt.get(u"vnf"):
335 self.create_kernelvm_config_testpmd_io(**kwargs)
336 elif u"testpmd_mac" in self._opt.get(u"vnf"):
337 self.create_kernelvm_config_testpmd_mac(**kwargs)
339 raise RuntimeError(u"QEMU: Unsupported VNF!")
340 self.create_kernelvm_init(vnf_bin=self._opt.get(u"vnf_bin"))
342 def get_qemu_pids(self):
343 """Get QEMU CPU pids.
345 :returns: List of QEMU CPU pids.
348 command = f"grep -rwl 'CPU' /proc/$(sudo cat " \
349 f"{self._temp.get(u'pidfile')})/task/*/comm "
350 command += r"| xargs dirname | sed -e 's/\/.*\///g' | uniq"
352 stdout, _ = exec_cmd_no_error(self._node, command)
353 return stdout.splitlines()
355 def qemu_set_affinity(self, *host_cpus):
356 """Set qemu affinity by getting thread PIDs via QMP and taskset to list
357 of CPU cores. Function tries to execute 3 times to avoid race condition
358 in getting thread PIDs.
360 :param host_cpus: List of CPU cores.
361 :type host_cpus: list
365 qemu_cpus = self.get_qemu_pids()
367 if len(qemu_cpus) != len(host_cpus):
370 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
371 command = f"taskset -pc {host_cpu} {qemu_cpu}"
372 message = f"QEMU: Set affinity failed " \
373 f"on {self._node[u'host']}!"
375 self._node, command, sudo=True, message=message
378 except (RuntimeError, ValueError):
383 raise RuntimeError(u"Failed to set Qemu threads affinity!")
385 def qemu_set_scheduler_policy(self):
386 """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
389 :raises RuntimeError: Set scheduler policy failed.
392 qemu_cpus = self.get_qemu_pids()
394 for qemu_cpu in qemu_cpus:
395 command = f"chrt -r -p 1 {qemu_cpu}"
396 message = f"QEMU: Set SCHED_RR failed on {self._node[u'host']}"
398 self._node, command, sudo=True, message=message
400 except (RuntimeError, ValueError):
404 def qemu_add_vhost_user_if(
405 self, socket, server=True, jumbo_frames=False, queue_size=None,
406 queues=1, csum=False, gso=False):
407 """Add Vhost-user interface.
409 :param socket: Path of the unix socket.
410 :param server: If True the socket shall be a listening socket.
411 :param jumbo_frames: Set True if jumbo frames are used in the test.
412 :param queue_size: Vring queue size.
413 :param queues: Number of queues.
414 :param csum: Checksum offloading.
415 :param gso: Generic segmentation offloading.
418 :type jumbo_frames: bool
419 :type queue_size: int
425 self._params.add_with_value(
426 u"chardev", f"socket,id=char{self._vhost_id},"
427 f"path={socket}{u',server' if server is True else u''}"
429 self._params.add_with_value(
430 u"netdev", f"vhost-user,id=vhost{self._vhost_id},"
431 f"chardev=char{self._vhost_id},queues={queues}"
433 mac = f"52:54:00:00:{self._opt.get(u'qemu_id'):02x}:" \
434 f"{self._vhost_id:02x}"
435 queue_size = f"rx_queue_size={queue_size},tx_queue_size={queue_size}" \
436 if queue_size else u""
437 self._params.add_with_value(
438 u"device", f"virtio-net-pci,netdev=vhost{self._vhost_id},mac={mac},"
439 f"addr={self._vhost_id+5}.0,mq=on,vectors={2 * queues + 2},"
440 f"csum={u'on' if csum else u'off'},gso={u'on' if gso else u'off'},"
441 f"guest_tso4=off,guest_tso6=off,guest_ecn=off,"
445 # Add interface MAC and socket to the node dict.
446 if_data = {u"mac_address": mac, u"socket": socket}
447 if_name = f"vhost{self._vhost_id}"
448 self._vm_info[u"interfaces"][if_name] = if_data
449 # Add socket to temporary file list.
450 self._temp[if_name] = socket
452 def _qemu_qmp_exec(self, cmd):
453 """Execute QMP command.
455 QMP is JSON based protocol which allows to control QEMU instance.
457 :param cmd: QMP command to execute.
459 :returns: Command output in python representation of JSON format. The
460 { "return": {} } response is QMP's success response. An error
461 response will contain the "error" keyword instead of "return".
463 # To enter command mode, the qmp_capabilities command must be issued.
464 command = f"echo \"{{{{ \\\"execute\\\": " \
465 f"\\\"qmp_capabilities\\\" }}}}" \
466 f"{{{{ \\\"execute\\\": \\\"{cmd}\\\" }}}}\" | " \
467 f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qmp')}"
468 message = f"QMP execute '{cmd}' failed on {self._node[u'host']}"
470 stdout, _ = exec_cmd_no_error(
471 self._node, command, sudo=False, message=message
474 # Skip capabilities negotiation messages.
475 out_list = stdout.splitlines()
476 if len(out_list) < 3:
477 raise RuntimeError(f"Invalid QMP output on {self._node[u'host']}")
478 return json.loads(out_list[2])
480 def _qemu_qga_flush(self):
481 """Flush the QGA parser state."""
482 command = f"(printf \"\xFF\"; sleep 1) | sudo -S socat " \
483 f"- UNIX-CONNECT:{self._temp.get(u'qga')}"
484 message = f"QGA flush failed on {self._node[u'host']}"
485 stdout, _ = exec_cmd_no_error(
486 self._node, command, sudo=False, message=message
489 return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
491 def _qemu_qga_exec(self, cmd):
492 """Execute QGA command.
494 QGA provide access to a system-level agent via standard QMP commands.
496 :param cmd: QGA command to execute.
499 command = f"(echo \"{{{{ \\\"execute\\\": " \
500 f"\\\"{cmd}\\\" }}}}\"; sleep 1) | " \
501 f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qga')}"
502 message = f"QGA execute '{cmd}' failed on {self._node[u'host']}"
503 stdout, _ = exec_cmd_no_error(
504 self._node, command, sudo=False, message=message
507 return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
509 def _wait_until_vm_boot(self):
510 """Wait until QEMU with NestedVM is booted."""
511 if self._opt.get(u"vm_type") == u"nestedvm":
512 self._wait_until_nestedvm_boot()
513 self._update_vm_interfaces()
514 elif self._opt.get(u"vm_type") == u"kernelvm":
515 self._wait_until_kernelvm_boot()
517 raise RuntimeError(u"QEMU: Unsupported VM type!")
519 def _wait_until_nestedvm_boot(self, retries=12):
520 """Wait until QEMU with NestedVM is booted.
522 First try to flush qga until there is output.
523 Then ping QEMU guest agent each 5s until VM booted or timeout.
525 :param retries: Number of retries with 5s between trials.
528 for _ in range(retries):
531 out = self._qemu_qga_flush()
533 logger.trace(f"QGA qga flush unexpected output {out}")
534 # Empty output - VM not booted yet
541 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
543 for _ in range(retries):
546 out = self._qemu_qga_exec(u"guest-ping")
548 logger.trace(f"QGA guest-ping unexpected output {out}")
549 # Empty output - VM not booted yet.
552 # Non-error return - VM booted.
553 elif out.get(u"return") is not None:
555 # Skip error and wait.
556 elif out.get(u"error") is not None:
559 # If there is an unexpected output from QGA guest-info, try
560 # again until timeout.
561 logger.trace(f"QGA guest-ping unexpected output {out}")
564 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
567 def _wait_until_kernelvm_boot(self, retries=60):
568 """Wait until QEMU KernelVM is booted.
570 :param retries: Number of retries.
573 vpp_ver = VPPUtil.vpp_show_version(self._node)
575 for _ in range(retries):
576 command = f"tail -1 {self._temp.get(u'log')}"
579 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
583 if vpp_ver in stdout or u"Press enter to exit" in stdout:
585 if u"reboot: Power down" in stdout:
587 f"QEMU: NF failed to run on {self._node[u'host']}!"
591 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
594 def _update_vm_interfaces(self):
595 """Update interface names in VM node dict."""
596 # Send guest-network-get-interfaces command via QGA, output example:
597 # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
598 # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
599 out = self._qemu_qga_exec(u"guest-network-get-interfaces")
600 interfaces = out.get(u"return")
604 f"Get VM interface list failed on {self._node[u'host']}"
606 # Create MAC-name dict.
607 for interface in interfaces:
608 if u"hardware-address" not in interface:
610 mac_name[interface[u"hardware-address"]] = interface[u"name"]
611 # Match interface by MAC and save interface name.
612 for interface in self._vm_info[u"interfaces"].values():
613 mac = interface.get(u"mac_address")
614 if_name = mac_name.get(mac)
616 logger.trace(f"Interface name for MAC {mac} not found")
618 interface[u"name"] = if_name
620 def qemu_start(self):
621 """Start QEMU and wait until VM boot.
623 :returns: VM node info.
626 cmd_opts = OptionString()
627 cmd_opts.add(f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch}")
628 cmd_opts.extend(self._params)
629 message = f"QEMU: Start failed on {self._node[u'host']}!"
631 DUTSetup.check_huge_page(
632 self._node, u"/dev/hugepages", int(self._opt.get(u"mem")))
635 self._node, cmd_opts, timeout=300, sudo=True, message=message
637 self._wait_until_vm_boot()
644 """Kill qemu process."""
646 self._node, f"chmod +r {self._temp.get(u'pidfile')}", sudo=True
649 self._node, f"kill -SIGKILL $(cat {self._temp.get(u'pidfile')})",
653 for value in self._temp.values():
654 exec_cmd(self._node, f"cat {value}", sudo=True)
655 exec_cmd(self._node, f"rm -f {value}", sudo=True)
657 def qemu_kill_all(self):
658 """Kill all qemu processes on DUT node if specified."""
659 exec_cmd(self._node, u"pkill -SIGKILL qemu", sudo=True)
661 for value in self._temp.values():
662 exec_cmd(self._node, f"cat {value}", sudo=True)
663 exec_cmd(self._node, f"rm -f {value}", sudo=True)
665 def qemu_version(self):
666 """Return Qemu version.
668 :returns: Qemu version.
671 command = f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch} " \
674 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
675 return match(r"QEMU emulator version ([\d.]*)", stdout).group(1)