Framework: Qemu alignments
[csit.git] / resources / libraries / python / QemuUtils.py
1 # Copyright (c) 2020 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """QEMU utilities library."""
15
16 import json
17
18 from re import match
19 from string import Template
20 from time import sleep
21
22 from robot.api import logger
23
24 from resources.libraries.python.Constants import Constants
25 from resources.libraries.python.DpdkUtil import DpdkUtil
26 from resources.libraries.python.DUTSetup import DUTSetup
27 from resources.libraries.python.OptionString import OptionString
28 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
29 from resources.libraries.python.topology import NodeType, Topology
30 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
31
32 __all__ = [u"QemuUtils"]
33
34
35 class QemuUtils:
36     """QEMU utilities."""
37
38     # Use one instance of class per tests.
39     ROBOT_LIBRARY_SCOPE = u"TEST CASE"
40
41     def __init__(
42             self, node, qemu_id=1, smp=1, mem=512, vnf=None,
43             img=Constants.QEMU_VM_IMAGE):
44         """Initialize QemuUtil class.
45
46         :param node: Node to run QEMU on.
47         :param qemu_id: QEMU identifier.
48         :param smp: Number of virtual SMP units (cores).
49         :param mem: Amount of memory.
50         :param vnf: Network function workload.
51         :param img: QEMU disk image or kernel image path.
52         :type node: dict
53         :type qemu_id: int
54         :type smp: int
55         :type mem: int
56         :type vnf: str
57         :type img: str
58         """
59         self._vhost_id = 0
60         self._node = node
61         self._arch = Topology.get_node_arch(self._node)
62         self._opt = dict()
63
64         # Architecture specific options
65         if self._arch == u"aarch64":
66             dpdk_target = u"arm64-armv8a"
67             self._opt[u"machine_args"] = \
68                 u"virt,accel=kvm,usb=off,mem-merge=off,gic-version=3"
69             self._opt[u"console"] = u"ttyAMA0"
70         else:
71             dpdk_target = u"x86_64-native"
72             self._opt[u"machine_args"] = u"pc,accel=kvm,usb=off,mem-merge=off"
73             self._opt[u"console"] = u"ttyS0"
74         self._testpmd_path = f"{Constants.QEMU_VM_DPDK}/" \
75             f"{dpdk_target}-linux-gcc/app"
76         self._vm_info = {
77             u"host": node[u"host"],
78             u"type": NodeType.VM,
79             u"port": 10021 + qemu_id,
80             u"serial": 4555 + qemu_id,
81             u"username": 'cisco',
82             u"password": 'cisco',
83             u"interfaces": {},
84         }
85         if node[u"port"] != 22:
86             self._vm_info[u"host_port"] = node[u"port"]
87             self._vm_info[u"host_username"] = node[u"username"]
88             self._vm_info[u"host_password"] = node[u"password"]
89         # Input Options.
90         self._opt[u"qemu_id"] = qemu_id
91         self._opt[u"mem"] = int(mem)
92         self._opt[u"smp"] = int(smp)
93         self._opt[u"img"] = img
94         self._opt[u"vnf"] = vnf
95         # Temporary files.
96         self._temp = dict()
97         self._temp[u"log"] = f"/tmp/serial_{qemu_id}.log"
98         self._temp[u"pidfile"] = f"/run/qemu_{qemu_id}.pid"
99         if img == Constants.QEMU_VM_IMAGE:
100             self._temp[u"qmp"] = f"/run/qmp_{qemu_id}.sock"
101             self._temp[u"qga"] = f"/run/qga_{qemu_id}.sock"
102         elif img == Constants.QEMU_VM_KERNEL:
103             self._opt[u"img"], _ = exec_cmd_no_error(
104                 node, f"ls -1 {Constants.QEMU_VM_KERNEL}* | tail -1",
105                 message=u"Qemu Kernel VM image not found!"
106             )
107             self._temp[u"ini"] = f"/etc/vm_init_{qemu_id}.conf"
108             self._opt[u"initrd"], _ = exec_cmd_no_error(
109                 node, f"ls -1 {Constants.QEMU_VM_KERNEL_INITRD}* | tail -1",
110                 message=u"Qemu Kernel initrd image not found!"
111             )
112         else:
113             raise RuntimeError(f"QEMU: Unknown VM image option: {img}")
114         # Computed parameters for QEMU command line.
115         self._params = OptionString(prefix=u"-")
116
117     def add_default_params(self):
118         """Set default QEMU command line parameters."""
119         self._params.add(u"daemonize")
120         self._params.add(u"nodefaults")
121         self._params.add_with_value(
122             u"name", f"vnf{self._opt.get(u'qemu_id')},debug-threads=on"
123         )
124         self._params.add(u"no-user-config")
125         self._params.add(u"nographic")
126         self._params.add(u"enable-kvm")
127         self._params.add_with_value(u"pidfile", self._temp.get(u"pidfile"))
128         self._params.add_with_value(u"cpu", u"host")
129
130         self._params.add_with_value(u"machine", self._opt.get(u"machine_args"))
131         self._params.add_with_value(
132             u"smp", f"{self._opt.get(u'smp')},sockets=1,"
133             f"cores={self._opt.get(u'smp')},threads=1"
134         )
135         self._params.add_with_value(
136             u"object", f"memory-backend-file,id=mem,"
137             f"size={self._opt.get(u'mem')}M,mem-path=/dev/hugepages,share=on"
138         )
139         self._params.add_with_value(u"m", f"{self._opt.get(u'mem')}M")
140         self._params.add_with_value(u"numa", u"node,memdev=mem")
141         self._params.add_with_value(u"balloon", u"none")
142
143     def add_net_user(self):
144         """Set managment port forwarding."""
145         self._params.add_with_value(
146             u"netdev", f"user,id=mgmt,net=172.16.255.0/24,"
147             f"hostfwd=tcp::{self._vm_info[u'port']}-:22"
148         )
149         self._params.add_with_value(
150             u"device", f"virtio-net,netdev=mgmt"
151         )
152
153     def add_qmp_qga(self):
154         """Set QMP, QGA management."""
155         self._params.add_with_value(
156             u"chardev", f"socket,path={self._temp.get(u'qga')},"
157             f"server,nowait,id=qga0"
158         )
159         self._params.add_with_value(u"device", u"isa-serial,chardev=qga0")
160         self._params.add_with_value(
161             u"qmp", f"unix:{self._temp.get(u'qmp')},server,nowait"
162         )
163
164     def add_serial(self):
165         """Set serial to file redirect."""
166         self._params.add_with_value(
167             u"chardev", f"socket,host=127.0.0.1,"
168             f"port={self._vm_info[u'serial']},id=gnc0,server,nowait")
169         self._params.add_with_value(u"device", u"isa-serial,chardev=gnc0")
170         self._params.add_with_value(
171             u"serial", f"file:{self._temp.get(u'log')}"
172         )
173
174     def add_drive_cdrom(self, drive_file, index=None):
175         """Set CD-ROM drive.
176
177         :param drive_file: Path to drive image.
178         :param index: Drive index.
179         :type drive_file: str
180         :type index: int
181         """
182         index = f"index={index}," if index else u""
183         self._params.add_with_value(
184             u"drive", f"file={drive_file},{index}media=cdrom"
185         )
186
187     def add_drive(self, drive_file, drive_format):
188         """Set drive with custom format.
189
190         :param drive_file: Path to drive image.
191         :param drive_format: Drive image format.
192         :type drive_file: str
193         :type drive_format: str
194         """
195         self._params.add_with_value(
196             u"drive", f"file={drive_file},format={drive_format},"
197             u"cache=none,if=virtio,file.locking=off"
198         )
199
200     def add_kernelvm_params(self):
201         """Set KernelVM QEMU parameters."""
202         self._params.add_with_value(
203             u"serial", f"file:{self._temp.get(u'log')}"
204         )
205         self._params.add_with_value(
206             u"fsdev", u"local,id=root9p,path=/,security_model=none"
207         )
208         self._params.add_with_value(
209             u"device", u"virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot"
210         )
211         self._params.add_with_value(u"kernel", f"{self._opt.get(u'img')}")
212         self._params.add_with_value(u"initrd", f"{self._opt.get(u'initrd')}")
213         self._params.add_with_value(
214             u"append", f"'ro rootfstype=9p rootflags=trans=virtio "
215             f"root=virtioroot console={self._opt.get(u'console')} "
216             f"tsc=reliable hugepages=256 "
217             f"init={self._temp.get(u'ini')} fastboot'"
218         )
219
220     def add_vhost_user_if(
221             self, socket, server=True, jumbo_frames=False, queue_size=None,
222             queues=1, csum=False, gso=False):
223         """Add Vhost-user interface.
224
225         :param socket: Path of the unix socket.
226         :param server: If True the socket shall be a listening socket.
227         :param jumbo_frames: Set True if jumbo frames are used in the test.
228         :param queue_size: Vring queue size.
229         :param queues: Number of queues.
230         :param csum: Checksum offloading.
231         :param gso: Generic segmentation offloading.
232         :type socket: str
233         :type server: bool
234         :type jumbo_frames: bool
235         :type queue_size: int
236         :type queues: int
237         :type csum: bool
238         :type gso: bool
239         """
240         self._vhost_id += 1
241         self._params.add_with_value(
242             u"chardev", f"socket,id=char{self._vhost_id},"
243             f"path={socket}{u',server' if server is True else u''}"
244         )
245         self._params.add_with_value(
246             u"netdev", f"vhost-user,id=vhost{self._vhost_id},"
247             f"chardev=char{self._vhost_id},queues={queues}"
248         )
249         mac = f"52:54:00:00:{self._opt.get(u'qemu_id'):02x}:" \
250             f"{self._vhost_id:02x}"
251         queue_size = f"rx_queue_size={queue_size},tx_queue_size={queue_size}" \
252             if queue_size else u""
253         self._params.add_with_value(
254             u"device", f"virtio-net-pci,netdev=vhost{self._vhost_id},mac={mac},"
255             f"addr={self._vhost_id+5}.0,mq=on,vectors={2 * queues + 2},"
256             f"csum={u'on' if csum else u'off'},gso={u'on' if gso else u'off'},"
257             f"guest_tso4=off,guest_tso6=off,guest_ecn=off,"
258             f"{queue_size}"
259         )
260
261         # Add interface MAC and socket to the node dict.
262         if_data = {u"mac_address": mac, u"socket": socket}
263         if_name = f"vhost{self._vhost_id}"
264         self._vm_info[u"interfaces"][if_name] = if_data
265         # Add socket to temporary file list.
266         self._temp[if_name] = socket
267
268     def add_vfio_pci_if(self, pci):
269         """Add VFIO PCI interface.
270
271         :param pci: PCI address of interface.
272         :type pci: str
273         """
274         self._params.add_with_value(u"device", f"vfio-pci,host={pci}")
275
276     def create_kernelvm_config_vpp(self, **kwargs):
277         """Create QEMU VPP config files.
278
279         :param kwargs: Key-value pairs to replace content of VPP configuration
280             file.
281         :type kwargs: dict
282         """
283         startup = f"/etc/vpp/vm_startup_{self._opt.get(u'qemu_id')}.conf"
284         running = f"/etc/vpp/vm_running_{self._opt.get(u'qemu_id')}.exec"
285
286         self._temp[u"startup"] = startup
287         self._temp[u"running"] = running
288         self._opt[u"vnf_bin"] = f"/usr/bin/vpp -c {startup}"
289
290         # Create VPP startup configuration.
291         vpp_config = VppConfigGenerator()
292         vpp_config.set_node(self._node)
293         vpp_config.add_unix_nodaemon()
294         vpp_config.add_unix_cli_listen()
295         vpp_config.add_unix_exec(running)
296         vpp_config.add_socksvr()
297         vpp_config.add_statseg_per_node_counters(value=u"on")
298         vpp_config.add_buffers_per_numa(107520)
299         vpp_config.add_heapsize(u"1G")
300         vpp_config.add_ip_heap_size(u"1G")
301         vpp_config.add_statseg_size(u"1G")
302         vpp_config.add_cpu_main_core(u"0")
303         if self._opt.get(u"smp") > 1:
304             vpp_config.add_cpu_corelist_workers(f"1-{self._opt.get(u'smp')-1}")
305         vpp_config.add_plugin(u"disable", u"default")
306         vpp_config.add_plugin(u"enable", u"ping_plugin.so")
307         if "2vfpt" in self._opt.get(u'vnf'):
308             vpp_config.add_plugin(u"enable", u"avf_plugin.so")
309         if "vhost" in self._opt.get(u'vnf'):
310             vpp_config.add_plugin(u"enable", u"dpdk_plugin.so")
311             vpp_config.add_dpdk_dev(u"0000:00:06.0", u"0000:00:07.0")
312             vpp_config.add_dpdk_dev_default_rxq(kwargs[u"queues"])
313             vpp_config.add_dpdk_log_level(u"debug")
314             if not kwargs[u"jumbo_frames"]:
315                 vpp_config.add_dpdk_no_multi_seg()
316                 vpp_config.add_dpdk_no_tx_checksum_offload()
317         if "ipsec" in self._opt.get(u'vnf'):
318             vpp_config.add_plugin(u"enable", u"crypto_native_plugin.so")
319             vpp_config.add_plugin(u"enable", u"crypto_ipsecmb_plugin.so")
320             vpp_config.add_plugin(u"enable", u"crypto_openssl_plugin.so")
321         if "nat" in self._opt.get(u'vnf'):
322             vpp_config.add_nat(value=u"endpoint-dependent")
323             vpp_config.add_nat_max_translations_per_thread(value=655360)
324             vpp_config.add_plugin(u"enable", u"nat_plugin.so")
325         vpp_config.write_config(startup)
326
327         # Create VPP running configuration.
328         template = f"{Constants.RESOURCES_TPL_VM}/{self._opt.get(u'vnf')}.exec"
329         exec_cmd_no_error(self._node, f"rm -f {running}", sudo=True)
330
331         with open(template, u"rt") as src_file:
332             src = Template(src_file.read())
333             exec_cmd_no_error(
334                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
335                 f"sudo tee {running}"
336             )
337
338     def create_kernelvm_config_testpmd_io(self, **kwargs):
339         """Create QEMU testpmd-io command line.
340
341         :param kwargs: Key-value pairs to construct command line parameters.
342         :type kwargs: dict
343         """
344         pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
345         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
346             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
347             eal_driver=False,
348             eal_pci_whitelist0=u"0000:00:06.0",
349             eal_pci_whitelist1=u"0000:00:07.0",
350             eal_in_memory=True,
351             pmd_num_mbufs=16384,
352             pmd_fwd_mode=u"io",
353             pmd_nb_ports=u"2",
354             pmd_portmask=u"0x3",
355             pmd_max_pkt_len=pmd_max_pkt_len,
356             pmd_mbuf_size=u"16384",
357             pmd_rxq=kwargs[u"queues"],
358             pmd_txq=kwargs[u"queues"],
359             pmd_tx_offloads='0x0',
360             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
361         )
362
363         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
364
365     def create_kernelvm_config_testpmd_mac(self, **kwargs):
366         """Create QEMU testpmd-mac command line.
367
368         :param kwargs: Key-value pairs to construct command line parameters.
369         :type kwargs: dict
370         """
371         pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
372         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
373             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
374             eal_driver=False,
375             eal_pci_whitelist0=u"0000:00:06.0",
376             eal_pci_whitelist1=u"0000:00:07.0",
377             eal_in_memory=True,
378             pmd_num_mbufs=16384,
379             pmd_fwd_mode=u"mac",
380             pmd_nb_ports=u"2",
381             pmd_portmask=u"0x3",
382             pmd_max_pkt_len=pmd_max_pkt_len,
383             pmd_mbuf_size=u"16384",
384             pmd_eth_peer_0=f"0,{kwargs[u'vif1_mac']}",
385             pmd_eth_peer_1=f"1,{kwargs[u'vif2_mac']}",
386             pmd_rxq=kwargs[u"queues"],
387             pmd_txq=kwargs[u"queues"],
388             pmd_tx_offloads=u"0x0",
389             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
390         )
391
392         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
393
394     def create_kernelvm_init(self, **kwargs):
395         """Create QEMU init script.
396
397         :param kwargs: Key-value pairs to replace content of init startup file.
398         :type kwargs: dict
399         """
400         template = f"{Constants.RESOURCES_TPL_VM}/init.sh"
401         init = self._temp.get(u"ini")
402         exec_cmd_no_error(self._node, f"rm -f {init}", sudo=True)
403
404         with open(template, u"rt") as src_file:
405             src = Template(src_file.read())
406             exec_cmd_no_error(
407                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
408                 f"sudo tee {init}"
409             )
410             exec_cmd_no_error(self._node, f"chmod +x {init}", sudo=True)
411
412     def configure_kernelvm_vnf(self, **kwargs):
413         """Create KernelVM VNF configurations.
414
415         :param kwargs: Key-value pairs for templating configs.
416         :type kwargs: dict
417         """
418         if u"vpp" in self._opt.get(u"vnf"):
419             self.create_kernelvm_config_vpp(**kwargs)
420             self.create_kernelvm_init(vnf_bin=self._opt.get(u"vnf_bin"))
421         elif u"testpmd_io" in self._opt.get(u"vnf"):
422             self.create_kernelvm_config_testpmd_io(**kwargs)
423             self.create_kernelvm_init(vnf_bin=self._opt.get(u"vnf_bin"))
424         elif u"testpmd_mac" in self._opt.get(u"vnf"):
425             self.create_kernelvm_config_testpmd_mac(**kwargs)
426             self.create_kernelvm_init(vnf_bin=self._opt.get(u"vnf_bin"))
427         else:
428             raise RuntimeError(u"QEMU: Unsupported VNF!")
429
430     def get_qemu_pids(self):
431         """Get QEMU CPU pids.
432
433         :returns: List of QEMU CPU pids.
434         :rtype: list of str
435         """
436         command = f"grep -rwl 'CPU' /proc/$(sudo cat " \
437             f"{self._temp.get(u'pidfile')})/task/*/comm "
438         command += r"| xargs dirname | sed -e 's/\/.*\///g' | uniq"
439
440         stdout, _ = exec_cmd_no_error(self._node, command)
441         return stdout.splitlines()
442
443     def qemu_set_affinity(self, *host_cpus):
444         """Set qemu affinity by getting thread PIDs via QMP and taskset to list
445         of CPU cores. Function tries to execute 3 times to avoid race condition
446         in getting thread PIDs.
447
448         :param host_cpus: List of CPU cores.
449         :type host_cpus: list
450         """
451         for _ in range(3):
452             try:
453                 qemu_cpus = self.get_qemu_pids()
454
455                 if len(qemu_cpus) != len(host_cpus):
456                     sleep(1)
457                     continue
458                 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
459                     command = f"taskset -pc {host_cpu} {qemu_cpu}"
460                     message = f"QEMU: Set affinity failed " \
461                         f"on {self._node[u'host']}!"
462                     exec_cmd_no_error(
463                         self._node, command, sudo=True, message=message
464                     )
465                 break
466             except (RuntimeError, ValueError):
467                 self.qemu_kill_all()
468                 raise
469         else:
470             self.qemu_kill_all()
471             raise RuntimeError(u"Failed to set Qemu threads affinity!")
472
473     def qemu_set_scheduler_policy(self):
474         """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
475         processes.
476
477         :raises RuntimeError: Set scheduler policy failed.
478         """
479         try:
480             qemu_cpus = self.get_qemu_pids()
481
482             for qemu_cpu in qemu_cpus:
483                 command = f"chrt -r -p 1 {qemu_cpu}"
484                 message = f"QEMU: Set SCHED_RR failed on {self._node[u'host']}"
485                 exec_cmd_no_error(
486                     self._node, command, sudo=True, message=message
487                 )
488         except (RuntimeError, ValueError):
489             self.qemu_kill_all()
490             raise
491
492     def _qemu_qmp_exec(self, cmd):
493         """Execute QMP command.
494
495         QMP is JSON based protocol which allows to control QEMU instance.
496
497         :param cmd: QMP command to execute.
498         :type cmd: str
499         :returns: Command output in python representation of JSON format. The
500             { "return": {} } response is QMP's success response. An error
501             response will contain the "error" keyword instead of "return".
502         """
503         # To enter command mode, the qmp_capabilities command must be issued.
504         command = f"echo \"{{{{ \\\"execute\\\": " \
505             f"\\\"qmp_capabilities\\\" }}}}" \
506             f"{{{{ \\\"execute\\\": \\\"{cmd}\\\" }}}}\" | " \
507             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qmp')}"
508         message = f"QMP execute '{cmd}' failed on {self._node[u'host']}"
509
510         stdout, _ = exec_cmd_no_error(
511             self._node, command, sudo=False, message=message
512         )
513
514         # Skip capabilities negotiation messages.
515         out_list = stdout.splitlines()
516         if len(out_list) < 3:
517             raise RuntimeError(f"Invalid QMP output on {self._node[u'host']}")
518         return json.loads(out_list[2])
519
520     def _qemu_qga_flush(self):
521         """Flush the QGA parser state."""
522         command = f"(printf \"\xFF\"; sleep 1) | sudo -S socat " \
523             f"- UNIX-CONNECT:{self._temp.get(u'qga')}"
524         message = f"QGA flush failed on {self._node[u'host']}"
525         stdout, _ = exec_cmd_no_error(
526             self._node, command, sudo=False, message=message
527         )
528
529         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
530
531     def _qemu_qga_exec(self, cmd):
532         """Execute QGA command.
533
534         QGA provide access to a system-level agent via standard QMP commands.
535
536         :param cmd: QGA command to execute.
537         :type cmd: str
538         """
539         command = f"(echo \"{{{{ \\\"execute\\\": " \
540             f"\\\"{cmd}\\\" }}}}\"; sleep 1) | " \
541             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qga')}"
542         message = f"QGA execute '{cmd}' failed on {self._node[u'host']}"
543         stdout, _ = exec_cmd_no_error(
544             self._node, command, sudo=False, message=message
545         )
546
547         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
548
549     def _wait_until_vm_boot(self):
550         """Wait until QEMU VM is booted."""
551         try:
552             getattr(self, f'_wait_{self._opt["vnf"]}')()
553         except AttributeError:
554             self._wait_default()
555
556     def _wait_default(self, retries=60):
557         """Wait until QEMU with VPP is booted.
558
559         :param retries: Number of retries.
560         :type retries: int
561         """
562         for _ in range(retries):
563             command = f"tail -1 {self._temp.get(u'log')}"
564             stdout = None
565             try:
566                 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
567                 sleep(1)
568             except RuntimeError:
569                 pass
570             if "vpp " in stdout and "built by" in stdout:
571                 break
572             if u"Press enter to exit" in stdout:
573                 break
574             if u"reboot: Power down" in stdout:
575                 raise RuntimeError(
576                     f"QEMU: NF failed to run on {self._node[u'host']}!"
577                 )
578         else:
579             raise RuntimeError(
580                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
581             )
582
583     def _wait_nestedvm(self, retries=12):
584         """Wait until QEMU with NestedVM is booted.
585
586         First try to flush qga until there is output.
587         Then ping QEMU guest agent each 5s until VM booted or timeout.
588
589         :param retries: Number of retries with 5s between trials.
590         :type retries: int
591         """
592         for _ in range(retries):
593             out = None
594             try:
595                 out = self._qemu_qga_flush()
596             except ValueError:
597                 logger.trace(f"QGA qga flush unexpected output {out}")
598             # Empty output - VM not booted yet
599             if not out:
600                 sleep(5)
601             else:
602                 break
603         else:
604             raise RuntimeError(
605                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
606             )
607         for _ in range(retries):
608             out = None
609             try:
610                 out = self._qemu_qga_exec(u"guest-ping")
611             except ValueError:
612                 logger.trace(f"QGA guest-ping unexpected output {out}")
613             # Empty output - VM not booted yet.
614             if not out:
615                 sleep(5)
616             # Non-error return - VM booted.
617             elif out.get(u"return") is not None:
618                 break
619             # Skip error and wait.
620             elif out.get(u"error") is not None:
621                 sleep(5)
622             else:
623                 # If there is an unexpected output from QGA guest-info, try
624                 # again until timeout.
625                 logger.trace(f"QGA guest-ping unexpected output {out}")
626         else:
627             raise RuntimeError(
628                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
629             )
630
631     def _update_vm_interfaces(self):
632         """Update interface names in VM node dict."""
633         # Send guest-network-get-interfaces command via QGA, output example:
634         # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
635         # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
636         out = self._qemu_qga_exec(u"guest-network-get-interfaces")
637         interfaces = out.get(u"return")
638         mac_name = {}
639         if not interfaces:
640             raise RuntimeError(
641                 f"Get VM interface list failed on {self._node[u'host']}"
642             )
643         # Create MAC-name dict.
644         for interface in interfaces:
645             if u"hardware-address" not in interface:
646                 continue
647             mac_name[interface[u"hardware-address"]] = interface[u"name"]
648         # Match interface by MAC and save interface name.
649         for interface in self._vm_info[u"interfaces"].values():
650             mac = interface.get(u"mac_address")
651             if_name = mac_name.get(mac)
652             if if_name is None:
653                 logger.trace(f"Interface name for MAC {mac} not found")
654             else:
655                 interface[u"name"] = if_name
656
657     def qemu_start(self):
658         """Start QEMU and wait until VM boot.
659
660         :returns: VM node info.
661         :rtype: dict
662         """
663         cmd_opts = OptionString()
664         cmd_opts.add(f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch}")
665         cmd_opts.extend(self._params)
666         message = f"QEMU: Start failed on {self._node[u'host']}!"
667         try:
668             DUTSetup.check_huge_page(
669                 self._node, u"/dev/hugepages", int(self._opt.get(u"mem")))
670
671             exec_cmd_no_error(
672                 self._node, cmd_opts, timeout=300, sudo=True, message=message
673             )
674             self._wait_until_vm_boot()
675         except RuntimeError:
676             self.qemu_kill_all()
677             raise
678         return self._vm_info
679
680     def qemu_kill(self):
681         """Kill qemu process."""
682         exec_cmd(
683             self._node, f"chmod +r {self._temp.get(u'pidfile')}", sudo=True
684         )
685         exec_cmd(
686             self._node, f"kill -SIGKILL $(cat {self._temp.get(u'pidfile')})",
687             sudo=True
688         )
689
690         for value in self._temp.values():
691             exec_cmd(self._node, f"cat {value}", sudo=True)
692             exec_cmd(self._node, f"rm -f {value}", sudo=True)
693
694     def qemu_kill_all(self):
695         """Kill all qemu processes on DUT node if specified."""
696         exec_cmd(self._node, u"pkill -SIGKILL qemu", sudo=True)
697
698         for value in self._temp.values():
699             exec_cmd(self._node, f"cat {value}", sudo=True)
700             exec_cmd(self._node, f"rm -f {value}", sudo=True)
701
702     def qemu_version(self):
703         """Return Qemu version.
704
705         :returns: Qemu version.
706         :rtype: str
707         """
708         command = f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch} " \
709             f"--version"
710         try:
711             stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
712             return match(r"QEMU emulator version ([\d.]*)", stdout).group(1)
713         except RuntimeError:
714             self.qemu_kill_all()
715             raise