+ :param kwargs: Key-value pairs for templating configs.
+ :type kwargs: dict
+ """
+ if u"vpp" in self._opt.get(u"vnf"):
+ self.create_kernelvm_config_vpp(**kwargs)
+ self.create_kernelvm_init(
+ template=f"{Constants.RESOURCES_TPL}/vm/init.sh",
+ vnf_bin=self._opt.get(u"vnf_bin")
+ )
+ elif u"testpmd_io" in self._opt.get(u"vnf"):
+ self.create_kernelvm_config_testpmd_io(**kwargs)
+ self.create_kernelvm_init(
+ template=f"{Constants.RESOURCES_TPL}/vm/init.sh",
+ vnf_bin=self._opt.get(u"vnf_bin")
+ )
+ elif u"testpmd_mac" in self._opt.get(u"vnf"):
+ self.create_kernelvm_config_testpmd_mac(**kwargs)
+ self.create_kernelvm_init(
+ template=f"{Constants.RESOURCES_TPL}/vm/init.sh",
+ vnf_bin=self._opt.get(u"vnf_bin")
+ )
+ elif u"iperf3" in self._opt.get(u"vnf"):
+ qemu_id = self._opt.get(u'qemu_id') % 2
+ self.create_kernelvm_config_iperf3()
+ self.create_kernelvm_init(
+ template=f"{Constants.RESOURCES_TPL}/vm/init_iperf3.sh",
+ vnf_bin=self._opt.get(u"vnf_bin"),
+ ip_address_l=u"2.2.2.2/30" if qemu_id else u"1.1.1.1/30",
+ ip_address_r=u"2.2.2.1" if qemu_id else u"1.1.1.2",
+ ip_route_r=u"1.1.1.0/30" if qemu_id else u"2.2.2.0/30"
+ )
+ else:
+ raise RuntimeError(u"QEMU: Unsupported VNF!")
+
+ def get_qemu_pids(self):
+ """Get QEMU CPU pids.
+
+ :returns: List of QEMU CPU pids.
+ :rtype: list of str
+ """
+ command = f"grep -rwl 'CPU' /proc/$(sudo cat " \
+ f"{self._temp.get(u'pidfile')})/task/*/comm "
+ command += r"| xargs dirname | sed -e 's/\/.*\///g' | uniq"
+
+ stdout, _ = exec_cmd_no_error(self._node, command)
+ return stdout.splitlines()
+
+ def qemu_set_affinity(self, *host_cpus):
+ """Set qemu affinity by getting thread PIDs via QMP and taskset to list
+ of CPU cores. Function tries to execute 3 times to avoid race condition
+ in getting thread PIDs.
+
+ :param host_cpus: List of CPU cores.
+ :type host_cpus: list
+ """
+ for _ in range(3):
+ try:
+ qemu_cpus = self.get_qemu_pids()
+
+ if len(qemu_cpus) != len(host_cpus):
+ sleep(1)
+ continue
+ for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
+ command = f"taskset -pc {host_cpu} {qemu_cpu}"
+ message = f"QEMU: Set affinity failed " \
+ f"on {self._node[u'host']}!"
+ exec_cmd_no_error(
+ self._node, command, sudo=True, message=message
+ )
+ break
+ except (RuntimeError, ValueError):
+ self.qemu_kill_all()
+ raise
+ else:
+ self.qemu_kill_all()
+ raise RuntimeError(u"Failed to set Qemu threads affinity!")
+
+ def qemu_set_scheduler_policy(self):
+ """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
+ processes.
+
+ :raises RuntimeError: Set scheduler policy failed.
+ """
+ try:
+ qemu_cpus = self.get_qemu_pids()
+
+ for qemu_cpu in qemu_cpus:
+ command = f"chrt -r -p 1 {qemu_cpu}"
+ message = f"QEMU: Set SCHED_RR failed on {self._node[u'host']}"
+ exec_cmd_no_error(
+ self._node, command, sudo=True, message=message
+ )
+ except (RuntimeError, ValueError):
+ self.qemu_kill_all()
+ raise
+
+ def _qemu_qmp_exec(self, cmd):
+ """Execute QMP command.
+
+ QMP is JSON based protocol which allows to control QEMU instance.
+
+ :param cmd: QMP command to execute.
+ :type cmd: str
+ :returns: Command output in python representation of JSON format. The
+ { "return": {} } response is QMP's success response. An error
+ response will contain the "error" keyword instead of "return".
+ """
+ # To enter command mode, the qmp_capabilities command must be issued.
+ command = f"echo \"{{{{ \\\"execute\\\": " \
+ f"\\\"qmp_capabilities\\\" }}}}" \
+ f"{{{{ \\\"execute\\\": \\\"{cmd}\\\" }}}}\" | " \
+ f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qmp')}"
+ message = f"QMP execute '{cmd}' failed on {self._node[u'host']}"
+
+ stdout, _ = exec_cmd_no_error(
+ self._node, command, sudo=False, message=message
+ )
+
+ # Skip capabilities negotiation messages.
+ out_list = stdout.splitlines()
+ if len(out_list) < 3:
+ raise RuntimeError(f"Invalid QMP output on {self._node[u'host']}")
+ return json.loads(out_list[2])
+
+ def _qemu_qga_flush(self):
+ """Flush the QGA parser state."""
+ command = f"(printf \"\xFF\"; sleep 1) | sudo -S socat " \
+ f"- UNIX-CONNECT:{self._temp.get(u'qga')}"
+ message = f"QGA flush failed on {self._node[u'host']}"
+ stdout, _ = exec_cmd_no_error(
+ self._node, command, sudo=False, message=message
+ )
+
+ return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
+
+ def _qemu_qga_exec(self, cmd):
+ """Execute QGA command.
+
+ QGA provide access to a system-level agent via standard QMP commands.
+
+ :param cmd: QGA command to execute.
+ :type cmd: str
+ """
+ command = f"(echo \"{{{{ \\\"execute\\\": " \
+ f"\\\"{cmd}\\\" }}}}\"; sleep 1) | " \
+ f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qga')}"
+ message = f"QGA execute '{cmd}' failed on {self._node[u'host']}"
+ stdout, _ = exec_cmd_no_error(
+ self._node, command, sudo=False, message=message
+ )
+
+ return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
+
+ def _wait_until_vm_boot(self):
+ """Wait until QEMU VM is booted."""
+ try:
+ getattr(self, f'_wait_{self._opt["vnf"]}')()
+ except AttributeError:
+ self._wait_default()
+
+ def _wait_default(self, retries=60):
+ """Wait until QEMU with VPP is booted.
+
+ :param retries: Number of retries.
+ :type retries: int
+ """
+ for _ in range(retries):
+ command = f"tail -1 {self._temp.get(u'log')}"
+ stdout = None
+ try:
+ stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
+ sleep(1)
+ except RuntimeError:
+ pass
+ if "vpp " in stdout and "built by" in stdout:
+ break
+ if u"Press enter to exit" in stdout:
+ break
+ if u"reboot: Power down" in stdout:
+ raise RuntimeError(
+ f"QEMU: NF failed to run on {self._node[u'host']}!"
+ )
+ else:
+ raise RuntimeError(
+ f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
+ )
+
+ def _wait_nestedvm(self, retries=12):
+ """Wait until QEMU with NestedVM is booted.
+
+ First try to flush qga until there is output.
+ Then ping QEMU guest agent each 5s until VM booted or timeout.
+
+ :param retries: Number of retries with 5s between trials.
+ :type retries: int
+ """
+ for _ in range(retries):
+ out = None
+ try:
+ out = self._qemu_qga_flush()
+ except ValueError:
+ logger.trace(f"QGA qga flush unexpected output {out}")
+ # Empty output - VM not booted yet
+ if not out:
+ sleep(5)
+ else:
+ break
+ else:
+ raise RuntimeError(
+ f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
+ )
+ for _ in range(retries):
+ out = None
+ try:
+ out = self._qemu_qga_exec(u"guest-ping")
+ except ValueError:
+ logger.trace(f"QGA guest-ping unexpected output {out}")
+ # Empty output - VM not booted yet.
+ if not out:
+ sleep(5)
+ # Non-error return - VM booted.
+ elif out.get(u"return") is not None:
+ break
+ # Skip error and wait.
+ elif out.get(u"error") is not None:
+ sleep(5)
+ else:
+ # If there is an unexpected output from QGA guest-info, try
+ # again until timeout.
+ logger.trace(f"QGA guest-ping unexpected output {out}")
+ else:
+ raise RuntimeError(
+ f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
+ )
+
+ def _wait_iperf3(self, retries=60):
+ """Wait until QEMU with iPerf3 is booted.
+
+ :param retries: Number of retries.
+ :type retries: int
+ """
+ grep = u"Server listening on 0.0.0.0 port 22."
+ cmd = f"fgrep '{grep}' {self._temp.get(u'log')}"
+ message = f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
+ exec_cmd_no_error(
+ self._node, cmd=cmd, sudo=True, message=message, retries=retries,
+ include_reason=True
+ )
+
+ def _update_vm_interfaces(self):
+ """Update interface names in VM node dict."""
+ # Send guest-network-get-interfaces command via QGA, output example:
+ # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
+ # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
+ out = self._qemu_qga_exec(u"guest-network-get-interfaces")
+ interfaces = out.get(u"return")
+ mac_name = {}
+ if not interfaces:
+ raise RuntimeError(
+ f"Get VM interface list failed on {self._node[u'host']}"
+ )
+ # Create MAC-name dict.
+ for interface in interfaces:
+ if u"hardware-address" not in interface:
+ continue
+ mac_name[interface[u"hardware-address"]] = interface[u"name"]
+ # Match interface by MAC and save interface name.
+ for interface in self._vm_info[u"interfaces"].values():
+ mac = interface.get(u"mac_address")
+ if_name = mac_name.get(mac)
+ if if_name is None:
+ logger.trace(f"Interface name for MAC {mac} not found")
+ else:
+ interface[u"name"] = if_name
+
+ def qemu_start(self):
+ """Start QEMU and wait until VM boot.
+
+ :returns: VM node info.
+ :rtype: dict
+ """
+ cmd_opts = OptionString()
+ cmd_opts.add(f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch}")
+ cmd_opts.extend(self._params)
+ message = f"QEMU: Start failed on {self._node[u'host']}!"
+ try:
+ DUTSetup.check_huge_page(
+ self._node, self._opt.get(u"mem-path"),
+ int(self._opt.get(u"mem"))
+ )
+
+ exec_cmd_no_error(
+ self._node, cmd_opts, timeout=300, sudo=True, message=message
+ )
+ self._wait_until_vm_boot()
+ except RuntimeError:
+ self.qemu_kill_all()
+ raise
+ return self._vm_info
+
+ def qemu_kill(self):
+ """Kill qemu process."""
+ exec_cmd(
+ self._node, f"chmod +r {self._temp.get(u'pidfile')}", sudo=True
+ )
+ exec_cmd(
+ self._node, f"kill -SIGKILL $(cat {self._temp.get(u'pidfile')})",
+ sudo=True
+ )
+
+ for value in self._temp.values():
+ exec_cmd(self._node, f"cat {value}", sudo=True)
+ exec_cmd(self._node, f"rm -f {value}", sudo=True)
+
+ def qemu_kill_all(self):
+ """Kill all qemu processes on DUT node if specified."""
+ exec_cmd(self._node, u"pkill -SIGKILL qemu", sudo=True)
+
+ for value in self._temp.values():
+ exec_cmd(self._node, f"cat {value}", sudo=True)
+ exec_cmd(self._node, f"rm -f {value}", sudo=True)
+
+ def qemu_version(self):
+ """Return Qemu version.
+
+ :returns: Qemu version.
+ :rtype: str
+ """
+ command = f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch} " \
+ f"--version"
+ try:
+ stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
+ return match(r"QEMU emulator version ([\d.]*)", stdout).group(1)
+ except RuntimeError:
+ self.qemu_kill_all()
+ raise