1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """QEMU utilities library."""
19 from string import Template
20 from time import sleep
22 from robot.api import logger
24 from resources.libraries.python.Constants import Constants
25 from resources.libraries.python.DpdkUtil import DpdkUtil
26 from resources.libraries.python.DUTSetup import DUTSetup
27 from resources.libraries.python.OptionString import OptionString
28 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
29 from resources.libraries.python.topology import NodeType, Topology
30 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
31 from resources.libraries.python.VPPUtil import VPPUtil
33 __all__ = [u"QemuUtils"]
39 # Use one instance of class per tests.
40 ROBOT_LIBRARY_SCOPE = u"TEST CASE"
43 self, node, qemu_id=1, smp=1, mem=512, vnf=None,
44 img=Constants.QEMU_VM_IMAGE):
45 """Initialize QemuUtil class.
47 :param node: Node to run QEMU on.
48 :param qemu_id: QEMU identifier.
49 :param smp: Number of virtual SMP units (cores).
50 :param mem: Amount of memory.
51 :param vnf: Network function workload.
52 :param img: QEMU disk image or kernel image path.
62 self._arch = Topology.get_node_arch(self._node)
63 dpdk_target = u"arm64-armv8a" if self._arch == u"aarch64" \
65 self._testpmd_path = f"{Constants.QEMU_VM_DPDK}/" \
66 f"{dpdk_target}-linuxapp-gcc/app"
68 u"host": node[u"host"],
70 u"port": 10021 + qemu_id,
71 u"serial": 4555 + qemu_id,
76 if node[u"port"] != 22:
77 self._vm_info[u"host_port"] = node[u"port"]
78 self._vm_info[u"host_username"] = node[u"username"]
79 self._vm_info[u"host_password"] = node[u"password"]
82 self._opt[u"qemu_id"] = qemu_id
83 self._opt[u"mem"] = int(mem)
84 self._opt[u"smp"] = int(smp)
85 self._opt[u"img"] = img
86 self._opt[u"vnf"] = vnf
89 self._temp[u"pidfile"] = f"/var/run/qemu_{qemu_id}.pid"
90 if img == Constants.QEMU_VM_IMAGE:
91 self._opt[u"vm_type"] = u"nestedvm"
92 self._temp[u"qmp"] = f"/var/run/qmp_{qemu_id}.sock"
93 self._temp[u"qga"] = f"/var/run/qga_{qemu_id}.sock"
94 elif img == Constants.QEMU_VM_KERNEL:
95 self._opt[u"img"], _ = exec_cmd_no_error(
96 node, f"ls -1 {Constants.QEMU_VM_KERNEL}* | tail -1",
97 message=u"Qemu Kernel VM image not found!"
99 self._opt[u"vm_type"] = u"kernelvm"
100 self._temp[u"log"] = f"/tmp/serial_{qemu_id}.log"
101 self._temp[u"ini"] = f"/etc/vm_init_{qemu_id}.conf"
102 self._opt[u"initrd"], _ = exec_cmd_no_error(
103 node, f"ls -1 {Constants.QEMU_VM_KERNEL_INITRD}* | tail -1",
104 message=u"Qemu Kernel initrd image not found!"
107 raise RuntimeError(f"QEMU: Unknown VM image option: {img}")
108 # Computed parameters for QEMU command line.
109 self._params = OptionString(prefix=u"-")
112 def add_params(self):
113 """Set QEMU command line parameters."""
114 self.add_default_params()
115 if self._opt.get(u"vm_type", u"") == u"nestedvm":
116 self.add_nestedvm_params()
117 elif self._opt.get(u"vm_type", u"") == u"kernelvm":
118 self.add_kernelvm_params()
120 raise RuntimeError(u"QEMU: Unsupported VM type!")
122 def add_default_params(self):
123 """Set default QEMU command line parameters."""
124 self._params.add(u"daemonize")
125 self._params.add(u"nodefaults")
126 self._params.add_with_value(
127 u"name", f"vnf{self._opt.get(u'qemu_id')},debug-threads=on"
129 self._params.add(u"no-user-config")
130 self._params.add_with_value(u"monitor", u"none")
131 self._params.add_with_value(u"display", u"none")
132 self._params.add_with_value(u"vga", u"none")
133 self._params.add(u"enable-kvm")
134 self._params.add_with_value(u"pidfile", self._temp.get(u"pidfile"))
135 self._params.add_with_value(u"cpu", u"host")
137 if self._arch == u"aarch64":
138 machine_args = u"virt,accel=kvm,usb=off,mem-merge=off,gic-version=3"
140 machine_args = u"pc,accel=kvm,usb=off,mem-merge=off"
141 self._params.add_with_value(u"machine", machine_args)
142 self._params.add_with_value(
143 u"smp", f"{self._opt.get(u'smp')},sockets=1,"
144 f"cores={self._opt.get(u'smp')},threads=1"
146 self._params.add_with_value(
147 u"object", f"memory-backend-file,id=mem,"
148 f"size={self._opt.get(u'mem')}M,mem-path=/dev/hugepages,share=on"
150 self._params.add_with_value(u"m", f"{self._opt.get(u'mem')}M")
151 self._params.add_with_value(u"numa", u"node,memdev=mem")
152 self._params.add_with_value(u"balloon", u"none")
154 def add_nestedvm_params(self):
155 """Set NestedVM QEMU parameters."""
156 self._params.add_with_value(
158 f"nic,macaddr=52:54:00:00:{self._opt.get(u'qemu_id'):02x}:ff"
160 self._params.add_with_value(
161 u"net", f"user,hostfwd=tcp::{self._vm_info[u'port']}-:22"
163 locking = u",file.locking=off"
164 self._params.add_with_value(
165 u"drive", f"file={self._opt.get(u'img')},"
166 f"format=raw,cache=none,if=virtio{locking}"
168 self._params.add_with_value(
169 u"qmp", f"unix:{self._temp.get(u'qmp')},server,nowait"
171 self._params.add_with_value(
172 u"chardev", f"socket,host=127.0.0.1,"
173 f"port={self._vm_info[u'serial']},id=gnc0,server,nowait")
174 self._params.add_with_value(u"device", u"isa-serial,chardev=gnc0")
175 self._params.add_with_value(
176 u"chardev", f"socket,path={self._temp.get(u'qga')},"
177 f"server,nowait,id=qga0"
179 self._params.add_with_value(u"device", u"isa-serial,chardev=qga0")
181 def add_kernelvm_params(self):
182 """Set KernelVM QEMU parameters."""
183 console = u"ttyAMA0" if self._arch == u"aarch64" else u"ttyS0"
184 self._params.add_with_value(
185 u"serial", f"file:{self._temp.get(u'log')}"
187 self._params.add_with_value(
188 u"fsdev", u"local,id=root9p,path=/,security_model=none"
190 self._params.add_with_value(
191 u"device", u"virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot"
193 self._params.add_with_value(u"kernel", f"{self._opt.get(u'img')}")
194 self._params.add_with_value(u"initrd", f"{self._opt.get(u'initrd')}")
195 self._params.add_with_value(
196 u"append", f"'ro rootfstype=9p rootflags=trans=virtio "
197 f"root=virtioroot console={console} tsc=reliable hugepages=256 "
198 f"init={self._temp.get(u'ini')} fastboot'"
201 def create_kernelvm_config_vpp(self, **kwargs):
202 """Create QEMU VPP config files.
204 :param kwargs: Key-value pairs to replace content of VPP configuration
208 startup = f"/etc/vpp/vm_startup_{self._opt.get(u'qemu_id')}.conf"
209 running = f"/etc/vpp/vm_running_{self._opt.get(u'qemu_id')}.exec"
211 self._temp[u"startup"] = startup
212 self._temp[u"running"] = running
213 self._opt[u"vnf_bin"] = f"/usr/bin/vpp -c {startup}"
215 # Create VPP startup configuration.
216 vpp_config = VppConfigGenerator()
217 vpp_config.set_node(self._node)
218 vpp_config.add_unix_nodaemon()
219 vpp_config.add_unix_cli_listen()
220 vpp_config.add_unix_exec(running)
221 vpp_config.add_socksvr()
222 vpp_config.add_cpu_main_core(u"0")
223 if self._opt.get(u"smp") > 1:
224 vpp_config.add_cpu_corelist_workers(f"1-{self._opt.get(u'smp')-1}")
225 vpp_config.add_dpdk_dev(u"0000:00:06.0", u"0000:00:07.0")
226 vpp_config.add_dpdk_dev_default_rxq(kwargs[u"queues"])
227 vpp_config.add_dpdk_log_level(u"debug")
228 if not kwargs[u"jumbo_frames"]:
229 vpp_config.add_dpdk_no_multi_seg()
230 vpp_config.add_dpdk_no_tx_checksum_offload()
231 vpp_config.add_plugin(u"disable", [u"default"])
232 vpp_config.add_plugin(u"enable", [u"dpdk_plugin.so"])
233 vpp_config.write_config(startup)
235 # Create VPP running configuration.
236 template = f"{Constants.RESOURCES_TPL_VM}/{self._opt.get(u'vnf')}.exec"
237 exec_cmd_no_error(self._node, f"rm -f {running}", sudo=True)
239 with open(template, "r") as src_file:
240 src = Template(src_file.read())
242 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
243 f"sudo tee {running}"
246 def create_kernelvm_config_testpmd_io(self, **kwargs):
247 """Create QEMU testpmd-io command line.
249 :param kwargs: Key-value pairs to construct command line parameters.
252 testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
253 eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
257 pmd_rxq=kwargs[u"queues"],
258 pmd_txq=kwargs[u"queues"],
259 pmd_tx_offloads='0x0',
260 pmd_disable_hw_vlan=False,
261 pmd_nb_cores=str(self._opt.get(u"smp") - 1)
264 self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
266 def create_kernelvm_config_testpmd_mac(self, **kwargs):
267 """Create QEMU testpmd-mac command line.
269 :param kwargs: Key-value pairs to construct command line parameters.
272 testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
273 eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
278 pmd_eth_peer_0=f"0,{kwargs[u'vif1_mac']}",
279 pmd_eth_peer_1=f"1,{kwargs[u'vif2_mac']}",
280 pmd_rxq=kwargs[u"queues"],
281 pmd_txq=kwargs[u"queues"],
282 pmd_tx_offloads=u"0x0",
283 pmd_disable_hw_vlan=False,
284 pmd_nb_cores=str(self._opt.get(u"smp") - 1)
287 self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
289 def create_kernelvm_init(self, **kwargs):
290 """Create QEMU init script.
292 :param kwargs: Key-value pairs to replace content of init startup file.
295 template = f"{Constants.RESOURCES_TPL_VM}/init.sh"
296 init = self._temp.get(u"ini")
297 exec_cmd_no_error(self._node, f"rm -f {init}", sudo=True)
299 with open(template, "r") as src_file:
300 src = Template(src_file.read())
302 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
305 exec_cmd_no_error(self._node, f"chmod +x {init}", sudo=True)
307 def configure_kernelvm_vnf(self, **kwargs):
308 """Create KernelVM VNF configurations.
310 :param kwargs: Key-value pairs for templating configs.
313 if u"vpp" in self._opt.get(u"vnf"):
314 self.create_kernelvm_config_vpp(**kwargs)
315 elif u"testpmd_io" in self._opt.get(u"vnf"):
316 self.create_kernelvm_config_testpmd_io(**kwargs)
317 elif u"testpmd_mac" in self._opt.get(u"vnf"):
318 self.create_kernelvm_config_testpmd_mac(**kwargs)
320 raise RuntimeError(u"QEMU: Unsupported VNF!")
321 self.create_kernelvm_init(vnf_bin=self._opt[u"vnf_bin"])
323 def get_qemu_pids(self):
324 """Get QEMU CPU pids.
326 :returns: List of QEMU CPU pids.
329 command = f"grep -rwl 'CPU' /proc/$(sudo cat " \
330 f"{self._temp.get(u'pidfile')})/task/*/comm "
331 command += r"| xargs dirname | sed -e 's/\/.*\///g' | uniq"
333 stdout, _ = exec_cmd_no_error(self._node, command)
334 return stdout.splitlines()
336 def qemu_set_affinity(self, *host_cpus):
337 """Set qemu affinity by getting thread PIDs via QMP and taskset to list
338 of CPU cores. Function tries to execute 3 times to avoid race condition
339 in getting thread PIDs.
341 :param host_cpus: List of CPU cores.
342 :type host_cpus: list
346 qemu_cpus = self.get_qemu_pids()
348 if len(qemu_cpus) != len(host_cpus):
351 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
352 command = f"taskset -pc {host_cpu} {qemu_cpu}"
353 message = f"QEMU: Set affinity failed " \
354 f"on {self._node[u'host']}!"
356 self._node, command, sudo=True, message=message
359 except (RuntimeError, ValueError):
364 raise RuntimeError(u"Failed to set Qemu threads affinity!")
366 def qemu_set_scheduler_policy(self):
367 """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
370 :raises RuntimeError: Set scheduler policy failed.
373 qemu_cpus = self.get_qemu_pids()
375 for qemu_cpu in qemu_cpus:
376 command = f"chrt -r -p 1 {qemu_cpu}"
377 message = f"QEMU: Set SCHED_RR failed on {self._node[u'host']}"
379 self._node, command, sudo=True, message=message
381 except (RuntimeError, ValueError):
385 def qemu_add_vhost_user_if(
386 self, socket, server=True, jumbo_frames=False, queue_size=None,
387 queues=1, csum=False, gso=False):
388 """Add Vhost-user interface.
390 :param socket: Path of the unix socket.
391 :param server: If True the socket shall be a listening socket.
392 :param jumbo_frames: Set True if jumbo frames are used in the test.
393 :param queue_size: Vring queue size.
394 :param queues: Number of queues.
395 :param csum: Checksum offloading.
396 :param gso: Generic segmentation offloading.
399 :type jumbo_frames: bool
400 :type queue_size: int
406 self._params.add_with_value(
407 u"chardev", f"socket,id=char{self._vhost_id},"
408 f"path={socket}{u',server' if server is True else u''}"
410 self._params.add_with_value(
411 u"netdev", f"vhost-user,id=vhost{self._vhost_id},"
412 f"chardev=char{self._vhost_id},queues={queues}"
414 mac = f"52:54:00:00:{self._opt.get(u'qemu_id'):02x}:" \
415 f"{self._vhost_id:02x}"
416 queue_size = f"rx_queue_size={queue_size},tx_queue_size={queue_size}" \
417 if queue_size else u""
418 self._params.add_with_value(
419 u"device", f"virtio-net-pci,netdev=vhost{self._vhost_id},mac={mac},"
420 f"addr={self._vhost_id+5}.0,mq=on,vectors={2 * queues + 2},"
421 f"csum={u'on' if csum else u'off'},gso={u'on' if gso else u'off'},"
422 f"guest_tso4=off,guest_tso6=off,guest_ecn=off,"
423 f"mrg_rxbuf={u'on,host_mtu=9200' if jumbo_frames else u'off'},"
427 # Add interface MAC and socket to the node dict.
428 if_data = {u"mac_address": mac, u"socket": socket}
429 if_name = f"vhost{self._vhost_id}"
430 self._vm_info[u"interfaces"][if_name] = if_data
431 # Add socket to temporary file list.
432 self._temp[if_name] = socket
434 def _qemu_qmp_exec(self, cmd):
435 """Execute QMP command.
437 QMP is JSON based protocol which allows to control QEMU instance.
439 :param cmd: QMP command to execute.
441 :returns: Command output in python representation of JSON format. The
442 { "return": {} } response is QMP's success response. An error
443 response will contain the "error" keyword instead of "return".
445 # To enter command mode, the qmp_capabilities command must be issued.
446 command = f"echo \"{{{{ \\\"execute\\\": " \
447 f"\\\"qmp_capabilities\\\" }}}}" \
448 f"{{{{ \\\"execute\\\": \\\"{cmd}\\\" }}}}\" | " \
449 f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qmp')}"
450 message = f"QMP execute '{cmd}' failed on {self._node[u'host']}"
452 stdout, _ = exec_cmd_no_error(
453 self._node, command, sudo=False, message=message
456 # Skip capabilities negotiation messages.
457 out_list = stdout.splitlines()
458 if len(out_list) < 3:
459 raise RuntimeError(f"Invalid QMP output on {self._node[u'host']}")
460 return json.loads(out_list[2])
462 def _qemu_qga_flush(self):
463 """Flush the QGA parser state."""
464 command = f"(printf \"\xFF\"; sleep 1) | sudo -S socat " \
465 f"- UNIX-CONNECT:{self._temp.get(u'qga')}"
466 message = f"QGA flush failed on {self._node[u'host']}"
467 stdout, _ = exec_cmd_no_error(
468 self._node, command, sudo=False, message=message
471 return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
473 def _qemu_qga_exec(self, cmd):
474 """Execute QGA command.
476 QGA provide access to a system-level agent via standard QMP commands.
478 :param cmd: QGA command to execute.
481 command = f"(echo \"{{{{ \\\"execute\\\": " \
482 f"\\\"{cmd}\\\" }}}}\"; sleep 1) | " \
483 f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qga')}"
484 message = f"QGA execute '{cmd}' failed on {self._node[u'host']}"
485 stdout, _ = exec_cmd_no_error(
486 self._node, command, sudo=False, message=message
489 return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
491 def _wait_until_vm_boot(self):
492 """Wait until QEMU with NestedVM is booted."""
493 if self._opt.get(u"vm_type") == u"nestedvm":
494 self._wait_until_nestedvm_boot()
495 self._update_vm_interfaces()
496 elif self._opt.get(u"vm_type") == u"kernelvm":
497 self._wait_until_kernelvm_boot()
499 raise RuntimeError(u"QEMU: Unsupported VM type!")
501 def _wait_until_nestedvm_boot(self, retries=12):
502 """Wait until QEMU with NestedVM is booted.
504 First try to flush qga until there is output.
505 Then ping QEMU guest agent each 5s until VM booted or timeout.
507 :param retries: Number of retries with 5s between trials.
510 for _ in range(retries):
513 out = self._qemu_qga_flush()
515 logger.trace(f"QGA qga flush unexpected output {out}")
516 # Empty output - VM not booted yet
523 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
525 for _ in range(retries):
528 out = self._qemu_qga_exec(u"guest-ping")
530 logger.trace(f"QGA guest-ping unexpected output {out}")
531 # Empty output - VM not booted yet.
534 # Non-error return - VM booted.
535 elif out.get(u"return") is not None:
537 # Skip error and wait.
538 elif out.get(u"error") is not None:
541 # If there is an unexpected output from QGA guest-info, try
542 # again until timeout.
543 logger.trace(f"QGA guest-ping unexpected output {out}")
546 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
549 def _wait_until_kernelvm_boot(self, retries=60):
550 """Wait until QEMU KernelVM is booted.
552 :param retries: Number of retries.
555 vpp_ver = VPPUtil.vpp_show_version(self._node)
557 for _ in range(retries):
558 command = f"tail -1 {self._temp.get(u'log')}"
561 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
565 if vpp_ver in stdout or u"Press enter to exit" in stdout:
567 if u"reboot: Power down" in stdout:
569 f"QEMU: NF failed to run on {self._node[u'host']}!"
573 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
576 def _update_vm_interfaces(self):
577 """Update interface names in VM node dict."""
578 # Send guest-network-get-interfaces command via QGA, output example:
579 # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
580 # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
581 out = self._qemu_qga_exec(u"guest-network-get-interfaces")
582 interfaces = out.get(u"return")
586 f"Get VM interface list failed on {self._node[u'host']}"
588 # Create MAC-name dict.
589 for interface in interfaces:
590 if u"hardware-address" not in interface:
592 mac_name[interface[u"hardware-address"]] = interface[u"name"]
593 # Match interface by MAC and save interface name.
594 for interface in self._vm_info[u"interfaces"].values():
595 mac = interface.get(u"mac_address")
596 if_name = mac_name.get(mac)
598 logger.trace(f"Interface name for MAC {mac} not found")
600 interface[u"name"] = if_name
602 def qemu_start(self):
603 """Start QEMU and wait until VM boot.
605 :returns: VM node info.
608 cmd_opts = OptionString()
609 cmd_opts.add(f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch}")
610 cmd_opts.extend(self._params)
611 message = f"QEMU: Start failed on {self._node[u'host']}!"
613 DUTSetup.check_huge_page(
614 self._node, u"/dev/hugepages", self._opt.get(u"mem"))
617 self._node, cmd_opts, timeout=300, sudo=True, message=message
619 self._wait_until_vm_boot()
626 """Kill qemu process."""
628 self._node, f"chmod +r {self._temp.get(u'pidfile')}", sudo=True
631 self._node, f"kill -SIGKILL $(cat {self._temp.get(u'pidfile')})",
635 for value in self._temp.values():
636 exec_cmd(self._node, f"cat {value}", sudo=True)
637 exec_cmd(self._node, f"rm -f {value}", sudo=True)
639 def qemu_kill_all(self):
640 """Kill all qemu processes on DUT node if specified."""
641 exec_cmd(self._node, u"pkill -SIGKILL qemu", sudo=True)
643 for value in self._temp.values():
644 exec_cmd(self._node, f"cat {value}", sudo=True)
645 exec_cmd(self._node, f"rm -f {value}", sudo=True)
647 def qemu_version(self):
648 """Return Qemu version.
650 :returns: Qemu version.
653 command = f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch} " \
656 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
657 return match(r"QEMU emulator version ([\d.]*)", stdout).group(1)