1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """QEMU utilities library."""
19 from string import Template
20 from time import sleep
22 from robot.api import logger
24 from resources.libraries.python.Constants import Constants
25 from resources.libraries.python.DpdkUtil import DpdkUtil
26 from resources.libraries.python.DUTSetup import DUTSetup
27 from resources.libraries.python.OptionString import OptionString
28 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
29 from resources.libraries.python.topology import NodeType, Topology
30 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
31 from resources.libraries.python.VPPUtil import VPPUtil
33 __all__ = [u"QemuUtils"]
39 # Use one instance of class per tests.
40 ROBOT_LIBRARY_SCOPE = u"TEST CASE"
43 self, node, qemu_id=1, smp=1, mem=512, vnf=None,
44 img=Constants.QEMU_VM_IMAGE):
45 """Initialize QemuUtil class.
47 :param node: Node to run QEMU on.
48 :param qemu_id: QEMU identifier.
49 :param smp: Number of virtual SMP units (cores).
50 :param mem: Amount of memory.
51 :param vnf: Network function workload.
52 :param img: QEMU disk image or kernel image path.
62 self._arch = Topology.get_node_arch(self._node)
63 dpdk_target = u"arm64-armv8a" if self._arch == u"aarch64" \
65 self._testpmd_path = f"{Constants.QEMU_VM_DPDK}/" \
66 f"{dpdk_target}-linuxapp-gcc/app"
68 u"host": node[u"host"],
70 u"port": 10021 + qemu_id,
71 u"serial": 4555 + qemu_id,
76 if node[u"port"] != 22:
77 self._vm_info[u"host_port"] = node[u"port"]
78 self._vm_info[u"host_username"] = node[u"username"]
79 self._vm_info[u"host_password"] = node[u"password"]
82 self._opt[u"qemu_id"] = qemu_id
83 self._opt[u"mem"] = int(mem)
84 self._opt[u"smp"] = int(smp)
85 self._opt[u"img"] = img
86 self._opt[u"vnf"] = vnf
89 self._temp[u"pidfile"] = f"/var/run/qemu_{qemu_id}.pid"
90 if img == Constants.QEMU_VM_IMAGE:
91 self._opt[u"vm_type"] = u"nestedvm"
92 self._temp[u"qmp"] = f"/var/run/qmp_{qemu_id}.sock"
93 self._temp[u"qga"] = f"/var/run/qga_{qemu_id}.sock"
94 elif img == Constants.QEMU_VM_KERNEL:
95 self._opt[u"img"], _ = exec_cmd_no_error(
96 node, f"ls -1 {Constants.QEMU_VM_KERNEL}* | tail -1",
97 message=u"Qemu Kernel VM image not found!"
99 self._opt[u"vm_type"] = u"kernelvm"
100 self._temp[u"log"] = f"/tmp/serial_{qemu_id}.log"
101 self._temp[u"ini"] = f"/etc/vm_init_{qemu_id}.conf"
102 self._opt[u"initrd"], _ = exec_cmd_no_error(
103 node, f"ls -1 {Constants.QEMU_VM_KERNEL_INITRD}* | tail -1",
104 message=u"Qemu Kernel initrd image not found!"
107 raise RuntimeError(f"QEMU: Unknown VM image option: {img}")
108 # Computed parameters for QEMU command line.
109 self._params = OptionString(prefix=u"-")
112 def add_params(self):
113 """Set QEMU command line parameters."""
114 self.add_default_params()
115 if self._opt.get(u"vm_type", u"") == u"nestedvm":
116 self.add_nestedvm_params()
117 elif self._opt.get(u"vm_type", u"") == u"kernelvm":
118 self.add_kernelvm_params()
120 raise RuntimeError(u"QEMU: Unsupported VM type!")
122 def add_default_params(self):
123 """Set default QEMU command line parameters."""
124 self._params.add(u"daemonize")
125 self._params.add(u"nodefaults")
126 self._params.add_with_value(
127 u"name", f"vnf{self._opt.get(u'qemu_id')},debug-threads=on"
129 self._params.add(u"no-user-config")
130 self._params.add_with_value(u"monitor", u"none")
131 self._params.add_with_value(u"display", u"none")
132 self._params.add_with_value(u"vga", u"none")
133 self._params.add(u"enable-kvm")
134 self._params.add_with_value(u"pidfile", self._temp.get(u"pidfile"))
135 self._params.add_with_value(u"cpu", u"host")
137 if self._arch == u"aarch64":
138 machine_args = u"virt,accel=kvm,usb=off,mem-merge=off,gic-version=3"
140 machine_args = u"pc,accel=kvm,usb=off,mem-merge=off"
141 self._params.add_with_value(u"machine", machine_args)
142 self._params.add_with_value(
143 u"smp", f"{self._opt.get(u'smp')},sockets=1,"
144 f"cores={self._opt.get(u'smp')},threads=1"
146 self._params.add_with_value(
147 u"object", f"memory-backend-file,id=mem,"
148 f"size={self._opt.get(u'mem')}M,mem-path=/dev/hugepages,share=on"
150 self._params.add_with_value(u"m", f"{self._opt.get(u'mem')}M")
151 self._params.add_with_value(u"numa", u"node,memdev=mem")
152 self._params.add_with_value(u"balloon", u"none")
154 def add_nestedvm_params(self):
155 """Set NestedVM QEMU parameters."""
156 self._params.add_with_value(
158 f"nic,macaddr=52:54:00:00:{self._opt.get(u'qemu_id'):02x}:ff"
160 self._params.add_with_value(
161 u"net", f"user,hostfwd=tcp::{self._vm_info[u'port']}-:22"
163 locking = u",file.locking=off"
164 self._params.add_with_value(
165 u"drive", f"file={self._opt.get(u'img')},"
166 f"format=raw,cache=none,if=virtio{locking}"
168 self._params.add_with_value(
169 u"qmp", f"unix:{self._temp.get(u'qmp')},server,nowait"
171 self._params.add_with_value(
172 u"chardev", f"socket,host=127.0.0.1,"
173 f"port={self._vm_info[u'serial']},id=gnc0,server,nowait")
174 self._params.add_with_value(u"device", u"isa-serial,chardev=gnc0")
175 self._params.add_with_value(
176 u"chardev", f"socket,path={self._temp.get(u'qga')},"
177 f"server,nowait,id=qga0"
179 self._params.add_with_value(u"device", u"isa-serial,chardev=qga0")
181 def add_kernelvm_params(self):
182 """Set KernelVM QEMU parameters."""
183 console = u"ttyAMA0" if self._arch == u"aarch64" else u"ttyS0"
184 self._params.add_with_value(
185 u"serial", f"file:{self._temp.get(u'log')}"
187 self._params.add_with_value(
188 u"fsdev", u"local,id=root9p,path=/,security_model=none"
190 self._params.add_with_value(
191 u"device", u"virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot"
193 self._params.add_with_value(u"kernel", f"{self._opt.get(u'img')}")
194 self._params.add_with_value(u"initrd", f"{self._opt.get(u'initrd')}")
195 self._params.add_with_value(
196 u"append", f"'ro rootfstype=9p rootflags=trans=virtio "
197 f"root=virtioroot console={console} tsc=reliable hugepages=256 "
198 f"init={self._temp.get(u'ini')} fastboot'"
201 def create_kernelvm_config_vpp(self, **kwargs):
202 """Create QEMU VPP config files.
204 :param kwargs: Key-value pairs to replace content of VPP configuration
208 startup = f"/etc/vpp/vm_startup_{self._opt.get(u'qemu_id')}.conf"
209 running = f"/etc/vpp/vm_running_{self._opt.get(u'qemu_id')}.exec"
211 self._temp[u"startup"] = startup
212 self._temp[u"running"] = running
213 self._opt[u"vnf_bin"] = f"/usr/bin/vpp -c {startup}"
215 # Create VPP startup configuration.
216 vpp_config = VppConfigGenerator()
217 vpp_config.set_node(self._node)
218 vpp_config.add_unix_nodaemon()
219 vpp_config.add_unix_cli_listen()
220 vpp_config.add_unix_exec(running)
221 vpp_config.add_socksvr()
222 vpp_config.add_cpu_main_core(u"0")
223 if self._opt.get(u"smp") > 1:
224 vpp_config.add_cpu_corelist_workers(f"1-{self._opt.get(u'smp')-1}")
225 vpp_config.add_dpdk_dev(u"0000:00:06.0", u"0000:00:07.0")
226 vpp_config.add_dpdk_dev_default_rxq(kwargs[u"queues"])
227 vpp_config.add_dpdk_log_level(u"debug")
228 if not kwargs[u"jumbo_frames"]:
229 vpp_config.add_dpdk_no_multi_seg()
230 vpp_config.add_dpdk_no_tx_checksum_offload()
231 vpp_config.add_plugin(u"disable", [u"default"])
232 vpp_config.add_plugin(u"enable", [u"dpdk_plugin.so"])
233 vpp_config.write_config(startup)
235 # Create VPP running configuration.
236 template = f"{Constants.RESOURCES_TPL_VM}/{self._opt.get(u'vnf')}.exec"
237 exec_cmd_no_error(self._node, f"rm -f {running}", sudo=True)
239 with open(template, "r") as src_file:
240 src = Template(src_file.read())
242 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
243 f"sudo tee {running}"
246 def create_kernelvm_config_testpmd_io(self, **kwargs):
247 """Create QEMU testpmd-io command line.
249 :param kwargs: Key-value pairs to construct command line parameters.
252 testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
253 eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
257 pmd_rxq=kwargs[u"queues"],
258 pmd_txq=kwargs[u"queues"],
259 pmd_tx_offloads='0x0',
260 pmd_disable_hw_vlan=False,
261 pmd_nb_cores=str(self._opt.get(u"smp") - 1)
264 self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
266 def create_kernelvm_config_testpmd_mac(self, **kwargs):
267 """Create QEMU testpmd-mac command line.
269 :param kwargs: Key-value pairs to construct command line parameters.
272 testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
273 eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
278 pmd_eth_peer_0=f"0,{kwargs[u'vif1_mac']}",
279 pmd_eth_peer_1=f"1,{kwargs[u'vif2_mac']}",
280 pmd_rxq=kwargs[u"queues"],
281 pmd_txq=kwargs[u"queues"],
282 pmd_tx_offloads=u"0x0",
283 pmd_disable_hw_vlan=False,
284 pmd_nb_cores=str(self._opt.get(u"smp") - 1)
287 self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
289 def create_kernelvm_init(self, **kwargs):
290 """Create QEMU init script.
292 :param kwargs: Key-value pairs to replace content of init startup file.
295 template = f"{Constants.RESOURCES_TPL_VM}/init.sh"
296 init = self._temp.get(u"ini")
297 exec_cmd_no_error(self._node, f"rm -f {init}", sudo=True)
299 with open(template, "r") as src_file:
300 src = Template(src_file.read())
302 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
305 exec_cmd_no_error(self._node, f"chmod +x {init}", sudo=True)
307 def configure_kernelvm_vnf(self, **kwargs):
308 """Create KernelVM VNF configurations.
310 :param kwargs: Key-value pairs for templating configs.
313 if u"vpp" in self._opt.get(u"vnf"):
314 self.create_kernelvm_config_vpp(**kwargs)
315 elif u"testpmd_io" in self._opt.get(u"vnf"):
316 self.create_kernelvm_config_testpmd_io(**kwargs)
317 elif u"testpmd_mac" in self._opt.get(u"vnf"):
318 self.create_kernelvm_config_testpmd_mac(**kwargs)
320 raise RuntimeError(u"QEMU: Unsupported VNF!")
321 self.create_kernelvm_init(vnf_bin=self._opt[u"vnf_bin"])
323 def get_qemu_pids(self):
324 """Get QEMU CPU pids.
326 :returns: List of QEMU CPU pids.
329 command = f"grep -rwl 'CPU' /proc/$(sudo cat " \
330 f"{self._temp.get(u'pidfile')})/task/*/comm "
331 command += r"| xargs dirname | sed -e 's/\/.*\///g' | uniq"
333 stdout, _ = exec_cmd_no_error(self._node, command)
334 return stdout.splitlines()
336 def qemu_set_affinity(self, *host_cpus):
337 """Set qemu affinity by getting thread PIDs via QMP and taskset to list
338 of CPU cores. Function tries to execute 3 times to avoid race condition
339 in getting thread PIDs.
341 :param host_cpus: List of CPU cores.
342 :type host_cpus: list
346 qemu_cpus = self.get_qemu_pids()
348 if len(qemu_cpus) != len(host_cpus):
351 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
352 command = f"taskset -pc {host_cpu} {qemu_cpu}"
353 message = f"QEMU: Set affinity failed " \
354 f"on {self._node[u'host']}!"
356 self._node, command, sudo=True, message=message
359 except (RuntimeError, ValueError):
364 raise RuntimeError(u"Failed to set Qemu threads affinity!")
366 def qemu_set_scheduler_policy(self):
367 """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
370 :raises RuntimeError: Set scheduler policy failed.
373 qemu_cpus = self.get_qemu_pids()
375 for qemu_cpu in qemu_cpus:
376 command = f"chrt -r -p 1 {qemu_cpu}"
377 message = f"QEMU: Set SCHED_RR failed on {self._node[u'host']}"
379 self._node, command, sudo=True, message=message
381 except (RuntimeError, ValueError):
385 def qemu_add_vhost_user_if(
386 self, socket, server=True, jumbo_frames=False, queue_size=None,
388 """Add Vhost-user interface.
390 :param socket: Path of the unix socket.
391 :param server: If True the socket shall be a listening socket.
392 :param jumbo_frames: Set True if jumbo frames are used in the test.
393 :param queue_size: Vring queue size.
394 :param queues: Number of queues.
397 :type jumbo_frames: bool
398 :type queue_size: int
402 self._params.add_with_value(
403 u"chardev", f"socket,id=char{self._vhost_id},"
404 f"path={socket}{u',server' if server is True else u''}"
406 self._params.add_with_value(
407 u"netdev", f"vhost-user,id=vhost{self._vhost_id},"
408 f"chardev=char{self._vhost_id},queues={queues}"
410 mac = f"52:54:00:00:{self._opt.get(u'qemu_id'):02x}:" \
411 f"{self._vhost_id:02x}"
412 queue_size = f"rx_queue_size={queue_size},tx_queue_size={queue_size}" \
413 if queue_size else u""
414 mbuf = u"on,host_mtu=9200"
415 self._params.add_with_value(
416 u"device", f"virtio-net-pci,netdev=vhost{self._vhost_id},mac={mac},"
417 f"addr={self._vhost_id+5}.0,mq=on,vectors={2 * queues + 2},"
418 f"csum=off,gso=off,guest_tso4=off,guest_tso6=off,guest_ecn=off,"
419 f"mrg_rxbuf={mbuf if jumbo_frames else u'off'},{queue_size}"
422 # Add interface MAC and socket to the node dict.
423 if_data = {u"mac_address": mac, u"socket": socket}
424 if_name = f"vhost{self._vhost_id}"
425 self._vm_info[u"interfaces"][if_name] = if_data
426 # Add socket to temporary file list.
427 self._temp[if_name] = socket
429 def _qemu_qmp_exec(self, cmd):
430 """Execute QMP command.
432 QMP is JSON based protocol which allows to control QEMU instance.
434 :param cmd: QMP command to execute.
436 :returns: Command output in python representation of JSON format. The
437 { "return": {} } response is QMP's success response. An error
438 response will contain the "error" keyword instead of "return".
440 # To enter command mode, the qmp_capabilities command must be issued.
441 command = f"echo \"{{{{ \\\"execute\\\": " \
442 f"\\\"qmp_capabilities\\\" }}}}" \
443 f"{{{{ \\\"execute\\\": \\\"{cmd}\\\" }}}}\" | " \
444 f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qmp')}"
445 message = f"QMP execute '{cmd}' failed on {self._node[u'host']}"
447 stdout, _ = exec_cmd_no_error(
448 self._node, command, sudo=False, message=message
451 # Skip capabilities negotiation messages.
452 out_list = stdout.splitlines()
453 if len(out_list) < 3:
454 raise RuntimeError(f"Invalid QMP output on {self._node[u'host']}")
455 return json.loads(out_list[2])
457 def _qemu_qga_flush(self):
458 """Flush the QGA parser state."""
459 command = f"(printf \"\xFF\"; sleep 1) | sudo -S socat " \
460 f"- UNIX-CONNECT:{self._temp.get(u'qga')}"
461 message = f"QGA flush failed on {self._node[u'host']}"
462 stdout, _ = exec_cmd_no_error(
463 self._node, command, sudo=False, message=message
466 return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
468 def _qemu_qga_exec(self, cmd):
469 """Execute QGA command.
471 QGA provide access to a system-level agent via standard QMP commands.
473 :param cmd: QGA command to execute.
476 command = f"(echo \"{{{{ \\\"execute\\\": " \
477 f"\\\"{cmd}\\\" }}}}\"; sleep 1) | " \
478 f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qga')}"
479 message = f"QGA execute '{cmd}' failed on {self._node[u'host']}"
480 stdout, _ = exec_cmd_no_error(
481 self._node, command, sudo=False, message=message
484 return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
486 def _wait_until_vm_boot(self):
487 """Wait until QEMU with NestedVM is booted."""
488 if self._opt.get(u"vm_type") == u"nestedvm":
489 self._wait_until_nestedvm_boot()
490 self._update_vm_interfaces()
491 elif self._opt.get(u"vm_type") == u"kernelvm":
492 self._wait_until_kernelvm_boot()
494 raise RuntimeError(u"QEMU: Unsupported VM type!")
496 def _wait_until_nestedvm_boot(self, retries=12):
497 """Wait until QEMU with NestedVM is booted.
499 First try to flush qga until there is output.
500 Then ping QEMU guest agent each 5s until VM booted or timeout.
502 :param retries: Number of retries with 5s between trials.
505 for _ in range(retries):
508 out = self._qemu_qga_flush()
510 logger.trace(f"QGA qga flush unexpected output {out}")
511 # Empty output - VM not booted yet
518 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
520 for _ in range(retries):
523 out = self._qemu_qga_exec(u"guest-ping")
525 logger.trace(f"QGA guest-ping unexpected output {out}")
526 # Empty output - VM not booted yet.
529 # Non-error return - VM booted.
530 elif out.get(u"return") is not None:
532 # Skip error and wait.
533 elif out.get(u"error") is not None:
536 # If there is an unexpected output from QGA guest-info, try
537 # again until timeout.
538 logger.trace(f"QGA guest-ping unexpected output {out}")
541 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
544 def _wait_until_kernelvm_boot(self, retries=60):
545 """Wait until QEMU KernelVM is booted.
547 :param retries: Number of retries.
550 vpp_ver = VPPUtil.vpp_show_version(self._node)
552 for _ in range(retries):
553 command = f"tail -1 {self._temp.get(u'log')}"
556 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
560 if vpp_ver in stdout or u"Press enter to exit" in stdout:
562 if u"reboot: Power down" in stdout:
564 f"QEMU: NF failed to run on {self._node[u'host']}!"
568 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
571 def _update_vm_interfaces(self):
572 """Update interface names in VM node dict."""
573 # Send guest-network-get-interfaces command via QGA, output example:
574 # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
575 # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
576 out = self._qemu_qga_exec(u"guest-network-get-interfaces")
577 interfaces = out.get(u"return")
581 f"Get VM interface list failed on {self._node[u'host']}"
583 # Create MAC-name dict.
584 for interface in interfaces:
585 if u"hardware-address" not in interface:
587 mac_name[interface[u"hardware-address"]] = interface[u"name"]
588 # Match interface by MAC and save interface name.
589 for interface in self._vm_info[u"interfaces"].values():
590 mac = interface.get(u"mac_address")
591 if_name = mac_name.get(mac)
593 logger.trace(f"Interface name for MAC {mac} not found")
595 interface[u"name"] = if_name
597 def qemu_start(self):
598 """Start QEMU and wait until VM boot.
600 :returns: VM node info.
603 cmd_opts = OptionString()
604 cmd_opts.add(f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch}")
605 cmd_opts.extend(self._params)
606 message = f"QEMU: Start failed on {self._node[u'host']}!"
608 DUTSetup.check_huge_page(
609 self._node, u"/dev/hugepages", self._opt.get(u"mem"))
612 self._node, cmd_opts, timeout=300, sudo=True, message=message
614 self._wait_until_vm_boot()
621 """Kill qemu process."""
623 self._node, f"chmod +r {self._temp.get(u'pidfile')}", sudo=True
626 self._node, f"kill -SIGKILL $(cat {self._temp.get(u'pidfile')})",
630 for value in self._temp.values():
631 exec_cmd(self._node, f"cat {value}", sudo=True)
632 exec_cmd(self._node, f"rm -f {value}", sudo=True)
634 def qemu_kill_all(self):
635 """Kill all qemu processes on DUT node if specified."""
636 exec_cmd(self._node, u"pkill -SIGKILL qemu", sudo=True)
638 for value in self._temp.values():
639 exec_cmd(self._node, f"cat {value}", sudo=True)
640 exec_cmd(self._node, f"rm -f {value}", sudo=True)
642 def qemu_version(self):
643 """Return Qemu version.
645 :returns: Qemu version.
648 command = f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch} " \
651 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
652 return match(r"QEMU emulator version ([\d.]*)", stdout).group(1)