1 # Copyright (c) 2020 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """QEMU utilities library."""
19 from string import Template
20 from time import sleep
22 from robot.api import logger
24 from resources.libraries.python.Constants import Constants
25 from resources.libraries.python.DpdkUtil import DpdkUtil
26 from resources.libraries.python.DUTSetup import DUTSetup
27 from resources.libraries.python.OptionString import OptionString
28 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
29 from resources.libraries.python.topology import NodeType, Topology
30 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
31 from resources.libraries.python.VPPUtil import VPPUtil
33 __all__ = [u"QemuUtils"]
39 # Use one instance of class per tests.
40 ROBOT_LIBRARY_SCOPE = u"TEST CASE"
43 self, node, qemu_id=1, smp=1, mem=512, vnf=None,
44 img=Constants.QEMU_VM_IMAGE):
45 """Initialize QemuUtil class.
47 :param node: Node to run QEMU on.
48 :param qemu_id: QEMU identifier.
49 :param smp: Number of virtual SMP units (cores).
50 :param mem: Amount of memory.
51 :param vnf: Network function workload.
52 :param img: QEMU disk image or kernel image path.
62 self._arch = Topology.get_node_arch(self._node)
65 # Architecture specific options
66 if self._arch == u"aarch64":
67 dpdk_target = u"arm64-armv8a"
68 self._opt[u"machine_args"] = \
69 u"virt,accel=kvm,usb=off,mem-merge=off,gic-version=3"
70 self._opt[u"console"] = u"ttyAMA0"
72 dpdk_target = u"x86_64-native"
73 self._opt[u"machine_args"] = u"pc,accel=kvm,usb=off,mem-merge=off"
74 self._opt[u"console"] = u"ttyS0"
75 self._testpmd_path = f"{Constants.QEMU_VM_DPDK}/" \
76 f"{dpdk_target}-linux-gcc/app"
78 u"host": node[u"host"],
80 u"port": 10021 + qemu_id,
81 u"serial": 4555 + qemu_id,
86 if node[u"port"] != 22:
87 self._vm_info[u"host_port"] = node[u"port"]
88 self._vm_info[u"host_username"] = node[u"username"]
89 self._vm_info[u"host_password"] = node[u"password"]
91 self._opt[u"qemu_id"] = qemu_id
92 self._opt[u"mem"] = int(mem)
93 self._opt[u"smp"] = int(smp)
94 self._opt[u"img"] = img
95 self._opt[u"vnf"] = vnf
98 self._temp[u"pidfile"] = f"/run/qemu_{qemu_id}.pid"
99 if img == Constants.QEMU_VM_IMAGE:
100 self._opt[u"vm_type"] = u"nestedvm"
101 self._temp[u"qmp"] = f"/run/qmp_{qemu_id}.sock"
102 self._temp[u"qga"] = f"/run/qga_{qemu_id}.sock"
103 elif img == Constants.QEMU_VM_KERNEL:
104 self._opt[u"img"], _ = exec_cmd_no_error(
105 node, f"ls -1 {Constants.QEMU_VM_KERNEL}* | tail -1",
106 message=u"Qemu Kernel VM image not found!"
108 self._opt[u"vm_type"] = u"kernelvm"
109 self._temp[u"log"] = f"/tmp/serial_{qemu_id}.log"
110 self._temp[u"ini"] = f"/etc/vm_init_{qemu_id}.conf"
111 self._opt[u"initrd"], _ = exec_cmd_no_error(
112 node, f"ls -1 {Constants.QEMU_VM_KERNEL_INITRD}* | tail -1",
113 message=u"Qemu Kernel initrd image not found!"
116 raise RuntimeError(f"QEMU: Unknown VM image option: {img}")
117 # Computed parameters for QEMU command line.
118 self._params = OptionString(prefix=u"-")
121 def add_params(self):
122 """Set QEMU command line parameters."""
123 self.add_default_params()
124 if self._opt.get(u"vm_type", u"") == u"nestedvm":
125 self.add_nestedvm_params()
126 elif self._opt.get(u"vm_type", u"") == u"kernelvm":
127 self.add_kernelvm_params()
129 raise RuntimeError(u"QEMU: Unsupported VM type!")
131 def add_default_params(self):
132 """Set default QEMU command line parameters."""
133 self._params.add(u"daemonize")
134 self._params.add(u"nodefaults")
135 self._params.add_with_value(
136 u"name", f"vnf{self._opt.get(u'qemu_id')},debug-threads=on"
138 self._params.add(u"no-user-config")
139 self._params.add_with_value(u"monitor", u"none")
140 self._params.add_with_value(u"display", u"none")
141 self._params.add_with_value(u"vga", u"none")
142 self._params.add(u"enable-kvm")
143 self._params.add_with_value(u"pidfile", self._temp.get(u"pidfile"))
144 self._params.add_with_value(u"cpu", u"host")
146 self._params.add_with_value(u"machine", self._opt.get(u"machine_args"))
147 self._params.add_with_value(
148 u"smp", f"{self._opt.get(u'smp')},sockets=1,"
149 f"cores={self._opt.get(u'smp')},threads=1"
151 self._params.add_with_value(
152 u"object", f"memory-backend-file,id=mem,"
153 f"size={self._opt.get(u'mem')}M,mem-path=/dev/hugepages,share=on"
155 self._params.add_with_value(u"m", f"{self._opt.get(u'mem')}M")
156 self._params.add_with_value(u"numa", u"node,memdev=mem")
157 self._params.add_with_value(u"balloon", u"none")
159 def add_nestedvm_params(self):
160 """Set NestedVM QEMU parameters."""
161 self._params.add_with_value(
163 f"nic,macaddr=52:54:00:00:{self._opt.get(u'qemu_id'):02x}:ff"
165 self._params.add_with_value(
166 u"net", f"user,hostfwd=tcp::{self._vm_info[u'port']}-:22"
168 locking = u",file.locking=off"
169 self._params.add_with_value(
170 u"drive", f"file={self._opt.get(u'img')},"
171 f"format=raw,cache=none,if=virtio{locking}"
173 self._params.add_with_value(
174 u"qmp", f"unix:{self._temp.get(u'qmp')},server,nowait"
176 self._params.add_with_value(
177 u"chardev", f"socket,host=127.0.0.1,"
178 f"port={self._vm_info[u'serial']},id=gnc0,server,nowait")
179 self._params.add_with_value(u"device", u"isa-serial,chardev=gnc0")
180 self._params.add_with_value(
181 u"chardev", f"socket,path={self._temp.get(u'qga')},"
182 f"server,nowait,id=qga0"
184 self._params.add_with_value(u"device", u"isa-serial,chardev=qga0")
186 def add_kernelvm_params(self):
187 """Set KernelVM QEMU parameters."""
188 self._params.add_with_value(
189 u"serial", f"file:{self._temp.get(u'log')}"
191 self._params.add_with_value(
192 u"fsdev", u"local,id=root9p,path=/,security_model=none"
194 self._params.add_with_value(
195 u"device", u"virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot"
197 self._params.add_with_value(u"kernel", f"{self._opt.get(u'img')}")
198 self._params.add_with_value(u"initrd", f"{self._opt.get(u'initrd')}")
199 self._params.add_with_value(
200 u"append", f"'ro rootfstype=9p rootflags=trans=virtio "
201 f"root=virtioroot console={self._opt.get(u'console')} "
202 f"tsc=reliable hugepages=256 "
203 f"init={self._temp.get(u'ini')} fastboot'"
206 def create_kernelvm_config_vpp(self, **kwargs):
207 """Create QEMU VPP config files.
209 :param kwargs: Key-value pairs to replace content of VPP configuration
213 startup = f"/etc/vpp/vm_startup_{self._opt.get(u'qemu_id')}.conf"
214 running = f"/etc/vpp/vm_running_{self._opt.get(u'qemu_id')}.exec"
216 self._temp[u"startup"] = startup
217 self._temp[u"running"] = running
218 self._opt[u"vnf_bin"] = f"/usr/bin/vpp -c {startup}"
220 # Create VPP startup configuration.
221 vpp_config = VppConfigGenerator()
222 vpp_config.set_node(self._node)
223 vpp_config.add_unix_nodaemon()
224 vpp_config.add_unix_cli_listen()
225 vpp_config.add_unix_exec(running)
226 vpp_config.add_socksvr()
227 vpp_config.add_cpu_main_core(u"0")
228 if self._opt.get(u"smp") > 1:
229 vpp_config.add_cpu_corelist_workers(f"1-{self._opt.get(u'smp')-1}")
230 vpp_config.add_dpdk_dev(u"0000:00:06.0", u"0000:00:07.0")
231 vpp_config.add_dpdk_dev_default_rxq(kwargs[u"queues"])
232 vpp_config.add_dpdk_log_level(u"debug")
233 if not kwargs[u"jumbo_frames"]:
234 vpp_config.add_dpdk_no_multi_seg()
235 vpp_config.add_dpdk_no_tx_checksum_offload()
236 vpp_config.add_plugin(u"disable", u"default")
237 vpp_config.add_plugin(u"enable", u"dpdk_plugin.so")
238 vpp_config.write_config(startup)
240 # Create VPP running configuration.
241 template = f"{Constants.RESOURCES_TPL_VM}/{self._opt.get(u'vnf')}.exec"
242 exec_cmd_no_error(self._node, f"rm -f {running}", sudo=True)
244 with open(template, u"rt") as src_file:
245 src = Template(src_file.read())
247 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
248 f"sudo tee {running}"
251 def create_kernelvm_config_testpmd_io(self, **kwargs):
252 """Create QEMU testpmd-io command line.
254 :param kwargs: Key-value pairs to construct command line parameters.
257 testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
258 eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
262 pmd_rxq=kwargs[u"queues"],
263 pmd_txq=kwargs[u"queues"],
264 pmd_tx_offloads='0x0',
265 pmd_disable_hw_vlan=False,
266 pmd_nb_cores=str(self._opt.get(u"smp") - 1)
269 self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
271 def create_kernelvm_config_testpmd_mac(self, **kwargs):
272 """Create QEMU testpmd-mac command line.
274 :param kwargs: Key-value pairs to construct command line parameters.
277 testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
278 eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
283 pmd_eth_peer_0=f"0,{kwargs[u'vif1_mac']}",
284 pmd_eth_peer_1=f"1,{kwargs[u'vif2_mac']}",
285 pmd_rxq=kwargs[u"queues"],
286 pmd_txq=kwargs[u"queues"],
287 pmd_tx_offloads=u"0x0",
288 pmd_disable_hw_vlan=False,
289 pmd_nb_cores=str(self._opt.get(u"smp") - 1)
292 self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
294 def create_kernelvm_init(self, **kwargs):
295 """Create QEMU init script.
297 :param kwargs: Key-value pairs to replace content of init startup file.
300 template = f"{Constants.RESOURCES_TPL_VM}/init.sh"
301 init = self._temp.get(u"ini")
302 exec_cmd_no_error(self._node, f"rm -f {init}", sudo=True)
304 with open(template, u"rt") as src_file:
305 src = Template(src_file.read())
307 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
310 exec_cmd_no_error(self._node, f"chmod +x {init}", sudo=True)
312 def configure_kernelvm_vnf(self, **kwargs):
313 """Create KernelVM VNF configurations.
315 :param kwargs: Key-value pairs for templating configs.
318 if u"vpp" in self._opt.get(u"vnf"):
319 self.create_kernelvm_config_vpp(**kwargs)
320 elif u"testpmd_io" in self._opt.get(u"vnf"):
321 self.create_kernelvm_config_testpmd_io(**kwargs)
322 elif u"testpmd_mac" in self._opt.get(u"vnf"):
323 self.create_kernelvm_config_testpmd_mac(**kwargs)
325 raise RuntimeError(u"QEMU: Unsupported VNF!")
326 self.create_kernelvm_init(vnf_bin=self._opt.get(u"vnf_bin"))
328 def get_qemu_pids(self):
329 """Get QEMU CPU pids.
331 :returns: List of QEMU CPU pids.
334 command = f"grep -rwl 'CPU' /proc/$(sudo cat " \
335 f"{self._temp.get(u'pidfile')})/task/*/comm "
336 command += r"| xargs dirname | sed -e 's/\/.*\///g' | uniq"
338 stdout, _ = exec_cmd_no_error(self._node, command)
339 return stdout.splitlines()
341 def qemu_set_affinity(self, *host_cpus):
342 """Set qemu affinity by getting thread PIDs via QMP and taskset to list
343 of CPU cores. Function tries to execute 3 times to avoid race condition
344 in getting thread PIDs.
346 :param host_cpus: List of CPU cores.
347 :type host_cpus: list
351 qemu_cpus = self.get_qemu_pids()
353 if len(qemu_cpus) != len(host_cpus):
356 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
357 command = f"taskset -pc {host_cpu} {qemu_cpu}"
358 message = f"QEMU: Set affinity failed " \
359 f"on {self._node[u'host']}!"
361 self._node, command, sudo=True, message=message
364 except (RuntimeError, ValueError):
369 raise RuntimeError(u"Failed to set Qemu threads affinity!")
371 def qemu_set_scheduler_policy(self):
372 """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
375 :raises RuntimeError: Set scheduler policy failed.
378 qemu_cpus = self.get_qemu_pids()
380 for qemu_cpu in qemu_cpus:
381 command = f"chrt -r -p 1 {qemu_cpu}"
382 message = f"QEMU: Set SCHED_RR failed on {self._node[u'host']}"
384 self._node, command, sudo=True, message=message
386 except (RuntimeError, ValueError):
390 def qemu_add_vhost_user_if(
391 self, socket, server=True, jumbo_frames=False, queue_size=None,
392 queues=1, csum=False, gso=False):
393 """Add Vhost-user interface.
395 :param socket: Path of the unix socket.
396 :param server: If True the socket shall be a listening socket.
397 :param jumbo_frames: Set True if jumbo frames are used in the test.
398 :param queue_size: Vring queue size.
399 :param queues: Number of queues.
400 :param csum: Checksum offloading.
401 :param gso: Generic segmentation offloading.
404 :type jumbo_frames: bool
405 :type queue_size: int
411 self._params.add_with_value(
412 u"chardev", f"socket,id=char{self._vhost_id},"
413 f"path={socket}{u',server' if server is True else u''}"
415 self._params.add_with_value(
416 u"netdev", f"vhost-user,id=vhost{self._vhost_id},"
417 f"chardev=char{self._vhost_id},queues={queues}"
419 mac = f"52:54:00:00:{self._opt.get(u'qemu_id'):02x}:" \
420 f"{self._vhost_id:02x}"
421 queue_size = f"rx_queue_size={queue_size},tx_queue_size={queue_size}" \
422 if queue_size else u""
423 self._params.add_with_value(
424 u"device", f"virtio-net-pci,netdev=vhost{self._vhost_id},mac={mac},"
425 f"addr={self._vhost_id+5}.0,mq=on,vectors={2 * queues + 2},"
426 f"csum={u'on' if csum else u'off'},gso={u'on' if gso else u'off'},"
427 f"guest_tso4=off,guest_tso6=off,guest_ecn=off,"
431 # Add interface MAC and socket to the node dict.
432 if_data = {u"mac_address": mac, u"socket": socket}
433 if_name = f"vhost{self._vhost_id}"
434 self._vm_info[u"interfaces"][if_name] = if_data
435 # Add socket to temporary file list.
436 self._temp[if_name] = socket
438 def _qemu_qmp_exec(self, cmd):
439 """Execute QMP command.
441 QMP is JSON based protocol which allows to control QEMU instance.
443 :param cmd: QMP command to execute.
445 :returns: Command output in python representation of JSON format. The
446 { "return": {} } response is QMP's success response. An error
447 response will contain the "error" keyword instead of "return".
449 # To enter command mode, the qmp_capabilities command must be issued.
450 command = f"echo \"{{{{ \\\"execute\\\": " \
451 f"\\\"qmp_capabilities\\\" }}}}" \
452 f"{{{{ \\\"execute\\\": \\\"{cmd}\\\" }}}}\" | " \
453 f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qmp')}"
454 message = f"QMP execute '{cmd}' failed on {self._node[u'host']}"
456 stdout, _ = exec_cmd_no_error(
457 self._node, command, sudo=False, message=message
460 # Skip capabilities negotiation messages.
461 out_list = stdout.splitlines()
462 if len(out_list) < 3:
463 raise RuntimeError(f"Invalid QMP output on {self._node[u'host']}")
464 return json.loads(out_list[2])
466 def _qemu_qga_flush(self):
467 """Flush the QGA parser state."""
468 command = f"(printf \"\xFF\"; sleep 1) | sudo -S socat " \
469 f"- UNIX-CONNECT:{self._temp.get(u'qga')}"
470 message = f"QGA flush failed on {self._node[u'host']}"
471 stdout, _ = exec_cmd_no_error(
472 self._node, command, sudo=False, message=message
475 return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
477 def _qemu_qga_exec(self, cmd):
478 """Execute QGA command.
480 QGA provide access to a system-level agent via standard QMP commands.
482 :param cmd: QGA command to execute.
485 command = f"(echo \"{{{{ \\\"execute\\\": " \
486 f"\\\"{cmd}\\\" }}}}\"; sleep 1) | " \
487 f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qga')}"
488 message = f"QGA execute '{cmd}' failed on {self._node[u'host']}"
489 stdout, _ = exec_cmd_no_error(
490 self._node, command, sudo=False, message=message
493 return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
495 def _wait_until_vm_boot(self):
496 """Wait until QEMU with NestedVM is booted."""
497 if self._opt.get(u"vm_type") == u"nestedvm":
498 self._wait_until_nestedvm_boot()
499 self._update_vm_interfaces()
500 elif self._opt.get(u"vm_type") == u"kernelvm":
501 self._wait_until_kernelvm_boot()
503 raise RuntimeError(u"QEMU: Unsupported VM type!")
505 def _wait_until_nestedvm_boot(self, retries=12):
506 """Wait until QEMU with NestedVM is booted.
508 First try to flush qga until there is output.
509 Then ping QEMU guest agent each 5s until VM booted or timeout.
511 :param retries: Number of retries with 5s between trials.
514 for _ in range(retries):
517 out = self._qemu_qga_flush()
519 logger.trace(f"QGA qga flush unexpected output {out}")
520 # Empty output - VM not booted yet
527 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
529 for _ in range(retries):
532 out = self._qemu_qga_exec(u"guest-ping")
534 logger.trace(f"QGA guest-ping unexpected output {out}")
535 # Empty output - VM not booted yet.
538 # Non-error return - VM booted.
539 elif out.get(u"return") is not None:
541 # Skip error and wait.
542 elif out.get(u"error") is not None:
545 # If there is an unexpected output from QGA guest-info, try
546 # again until timeout.
547 logger.trace(f"QGA guest-ping unexpected output {out}")
550 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
553 def _wait_until_kernelvm_boot(self, retries=60):
554 """Wait until QEMU KernelVM is booted.
556 :param retries: Number of retries.
559 vpp_ver = VPPUtil.vpp_show_version(self._node)
561 for _ in range(retries):
562 command = f"tail -1 {self._temp.get(u'log')}"
565 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
569 if vpp_ver in stdout or u"Press enter to exit" in stdout:
571 if u"reboot: Power down" in stdout:
573 f"QEMU: NF failed to run on {self._node[u'host']}!"
577 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
580 def _update_vm_interfaces(self):
581 """Update interface names in VM node dict."""
582 # Send guest-network-get-interfaces command via QGA, output example:
583 # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
584 # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
585 out = self._qemu_qga_exec(u"guest-network-get-interfaces")
586 interfaces = out.get(u"return")
590 f"Get VM interface list failed on {self._node[u'host']}"
592 # Create MAC-name dict.
593 for interface in interfaces:
594 if u"hardware-address" not in interface:
596 mac_name[interface[u"hardware-address"]] = interface[u"name"]
597 # Match interface by MAC and save interface name.
598 for interface in self._vm_info[u"interfaces"].values():
599 mac = interface.get(u"mac_address")
600 if_name = mac_name.get(mac)
602 logger.trace(f"Interface name for MAC {mac} not found")
604 interface[u"name"] = if_name
606 def qemu_start(self):
607 """Start QEMU and wait until VM boot.
609 :returns: VM node info.
612 cmd_opts = OptionString()
613 cmd_opts.add(f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch}")
614 cmd_opts.extend(self._params)
615 message = f"QEMU: Start failed on {self._node[u'host']}!"
617 DUTSetup.check_huge_page(
618 self._node, u"/dev/hugepages", int(self._opt.get(u"mem")))
621 self._node, cmd_opts, timeout=300, sudo=True, message=message
623 self._wait_until_vm_boot()
630 """Kill qemu process."""
632 self._node, f"chmod +r {self._temp.get(u'pidfile')}", sudo=True
635 self._node, f"kill -SIGKILL $(cat {self._temp.get(u'pidfile')})",
639 for value in self._temp.values():
640 exec_cmd(self._node, f"cat {value}", sudo=True)
641 exec_cmd(self._node, f"rm -f {value}", sudo=True)
643 def qemu_kill_all(self):
644 """Kill all qemu processes on DUT node if specified."""
645 exec_cmd(self._node, u"pkill -SIGKILL qemu", sudo=True)
647 for value in self._temp.values():
648 exec_cmd(self._node, f"cat {value}", sudo=True)
649 exec_cmd(self._node, f"rm -f {value}", sudo=True)
651 def qemu_version(self):
652 """Return Qemu version.
654 :returns: Qemu version.
657 command = f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch} " \
660 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
661 return match(r"QEMU emulator version ([\d.]*)", stdout).group(1)