JumpAvg: Fix string format
[csit.git] / resources / libraries / python / QemuUtils.py
1 # Copyright (c) 2021 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """QEMU utilities library."""
15
16 import json
17
18 from re import match
19 from string import Template
20 from time import sleep
21
22 from robot.api import logger
23
24 from resources.libraries.python.Constants import Constants
25 from resources.libraries.python.DpdkUtil import DpdkUtil
26 from resources.libraries.python.DUTSetup import DUTSetup
27 from resources.libraries.python.OptionString import OptionString
28 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
29 from resources.libraries.python.topology import NodeType, Topology
30 from resources.libraries.python.VhostUser import VirtioFeaturesFlags
31 from resources.libraries.python.VhostUser import VirtioFeatureMask
32 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
33
34 __all__ = [u"QemuUtils"]
35
36
37 class QemuUtils:
38     """QEMU utilities."""
39
40     # Use one instance of class per tests.
41     ROBOT_LIBRARY_SCOPE = u"TEST CASE"
42
43     def __init__(
44             self, node, qemu_id=1, smp=1, mem=512, vnf=None,
45             img=Constants.QEMU_VM_IMAGE, page_size=u""):
46         """Initialize QemuUtil class.
47
48         :param node: Node to run QEMU on.
49         :param qemu_id: QEMU identifier.
50         :param smp: Number of virtual SMP units (cores).
51         :param mem: Amount of memory.
52         :param vnf: Network function workload.
53         :param img: QEMU disk image or kernel image path.
54         :param page_size: Hugepage Size.
55         :type node: dict
56         :type qemu_id: int
57         :type smp: int
58         :type mem: int
59         :type vnf: str
60         :type img: str
61         :type page_size: str
62         """
63         self._nic_id = 0
64         self._node = node
65         self._arch = Topology.get_node_arch(self._node)
66         self._opt = dict()
67
68         # Architecture specific options
69         if self._arch == u"aarch64":
70             self._opt[u"machine_args"] = \
71                 u"virt,accel=kvm,usb=off,mem-merge=off,gic-version=3"
72             self._opt[u"console"] = u"ttyAMA0"
73         else:
74             self._opt[u"machine_args"] = u"pc,accel=kvm,usb=off,mem-merge=off"
75             self._opt[u"console"] = u"ttyS0"
76         self._testpmd_path = f"{Constants.QEMU_VM_DPDK}/build/app"
77         self._vm_info = {
78             u"host": node[u"host"],
79             u"type": NodeType.VM,
80             u"port": 10021 + qemu_id,
81             u"serial": 4555 + qemu_id,
82             u"username": 'testuser',
83             u"password": 'Csit1234',
84             u"interfaces": {},
85         }
86         if node[u"port"] != 22:
87             self._vm_info[u"host_port"] = node[u"port"]
88             self._vm_info[u"host_username"] = node[u"username"]
89             self._vm_info[u"host_password"] = node[u"password"]
90         # Input Options.
91         self._opt[u"qemu_id"] = qemu_id
92         self._opt[u"mem"] = int(mem)
93         self._opt[u"smp"] = int(smp)
94         self._opt[u"img"] = img
95         self._opt[u"vnf"] = vnf
96         self._opt[u"page_size"] = page_size
97
98         # Temporary files.
99         self._temp = dict()
100         self._temp[u"log"] = f"/tmp/serial_{qemu_id}.log"
101         self._temp[u"pidfile"] = f"/run/qemu_{qemu_id}.pid"
102         if img == Constants.QEMU_VM_IMAGE:
103             self._temp[u"qmp"] = f"/run/qmp_{qemu_id}.sock"
104             self._temp[u"qga"] = f"/run/qga_{qemu_id}.sock"
105         elif img == Constants.QEMU_VM_KERNEL:
106             self._opt[u"img"], _ = exec_cmd_no_error(
107                 node, f"ls -1 {Constants.QEMU_VM_KERNEL}* | tail -1",
108                 message=u"Qemu Kernel VM image not found!"
109             )
110             self._temp[u"ini"] = f"/etc/vm_init_{qemu_id}.conf"
111             self._opt[u"initrd"], _ = exec_cmd_no_error(
112                 node, f"ls -1 {Constants.QEMU_VM_KERNEL_INITRD}* | tail -1",
113                 message=u"Qemu Kernel initrd image not found!"
114             )
115         else:
116             raise RuntimeError(f"QEMU: Unknown VM image option: {img}")
117         # Computed parameters for QEMU command line.
118         self._params = OptionString(prefix=u"-")
119
120     def add_default_params(self):
121         """Set default QEMU command line parameters."""
122         mem_path = f"/dev/hugepages1G" \
123             if self._opt[u"page_size"] == u"1G" else u"/dev/hugepages"
124
125         self._params.add(u"daemonize")
126         self._params.add(u"nodefaults")
127         self._params.add_with_value(
128             u"name", f"vnf{self._opt.get(u'qemu_id')},debug-threads=on"
129         )
130         self._params.add(u"no-user-config")
131         self._params.add(u"nographic")
132         self._params.add(u"enable-kvm")
133         self._params.add_with_value(u"pidfile", self._temp.get(u"pidfile"))
134         self._params.add_with_value(u"cpu", u"host")
135
136         self._params.add_with_value(u"machine", self._opt.get(u"machine_args"))
137         self._params.add_with_value(
138             u"smp", f"{self._opt.get(u'smp')},sockets=1,"
139             f"cores={self._opt.get(u'smp')},threads=1"
140         )
141         self._params.add_with_value(
142             u"object", f"memory-backend-file,id=mem,"
143             f"size={self._opt.get(u'mem')}M,"
144             f"mem-path={mem_path},share=on"
145         )
146         self._params.add_with_value(u"m", f"{self._opt.get(u'mem')}M")
147         self._params.add_with_value(u"numa", u"node,memdev=mem")
148
149     def add_net_user(self, net="10.0.2.0/24"):
150         """Set managment port forwarding."""
151         self._params.add_with_value(
152             u"netdev", f"user,id=mgmt,net={net},"
153             f"hostfwd=tcp::{self._vm_info[u'port']}-:22"
154         )
155         self._params.add_with_value(
156             u"device", f"virtio-net,netdev=mgmt"
157         )
158
159     def add_qmp_qga(self):
160         """Set QMP, QGA management."""
161         self._params.add_with_value(
162             u"chardev", f"socket,path={self._temp.get(u'qga')},"
163             f"server,nowait,id=qga0"
164         )
165         self._params.add_with_value(
166             u"device", u"isa-serial,chardev=qga0"
167         )
168         self._params.add_with_value(
169             u"qmp", f"unix:{self._temp.get(u'qmp')},server,nowait"
170         )
171
172     def add_serial(self):
173         """Set serial to file redirect."""
174         self._params.add_with_value(
175             u"chardev", f"socket,host=127.0.0.1,"
176             f"port={self._vm_info[u'serial']},id=gnc0,server,nowait"
177         )
178         self._params.add_with_value(
179             u"device", u"isa-serial,chardev=gnc0"
180         )
181         self._params.add_with_value(
182             u"serial", f"file:{self._temp.get(u'log')}"
183         )
184
185     def add_drive_cdrom(self, drive_file, index=None):
186         """Set CD-ROM drive.
187
188         :param drive_file: Path to drive image.
189         :param index: Drive index.
190         :type drive_file: str
191         :type index: int
192         """
193         index = f"index={index}," if index else u""
194         self._params.add_with_value(
195             u"drive", f"file={drive_file},{index}media=cdrom"
196         )
197
198     def add_drive(self, drive_file, drive_format):
199         """Set drive with custom format.
200
201         :param drive_file: Path to drive image.
202         :param drive_format: Drive image format.
203         :type drive_file: str
204         :type drive_format: str
205         """
206         self._params.add_with_value(
207             u"drive", f"file={drive_file},format={drive_format},"
208             u"cache=none,if=virtio,file.locking=off"
209         )
210
211     def add_kernelvm_params(self):
212         """Set KernelVM QEMU parameters."""
213         hugepages = 3 if self._opt[u"page_size"] == u"1G" else 512
214
215         self._params.add_with_value(
216             u"serial", f"file:{self._temp.get(u'log')}"
217         )
218         self._params.add_with_value(
219             u"fsdev", u"local,id=root9p,path=/,security_model=none"
220         )
221         self._params.add_with_value(
222             u"device", u"virtio-9p-pci,fsdev=root9p,mount_tag=virtioroot"
223         )
224         self._params.add_with_value(
225             u"kernel", f"{self._opt.get(u'img')}"
226         )
227         self._params.add_with_value(
228             u"initrd", f"{self._opt.get(u'initrd')}"
229         )
230         self._params.add_with_value(
231             u"append", f"'ro rootfstype=9p rootflags=trans=virtio "
232             f"root=virtioroot console={self._opt.get(u'console')} "
233             f"tsc=reliable hugepages={hugepages} "
234             f"hugepagesz={self._opt.get(u'page_size')} "
235             f"init={self._temp.get(u'ini')} fastboot'"
236         )
237
238     def add_vhost_user_if(
239             self, socket, server=True, jumbo_frames=False, queue_size=None,
240             queues=1, virtio_feature_mask=None):
241         """Add Vhost-user interface.
242
243         :param socket: Path of the unix socket.
244         :param server: If True the socket shall be a listening socket.
245         :param jumbo_frames: Set True if jumbo frames are used in the test.
246         :param queue_size: Vring queue size.
247         :param queues: Number of queues.
248         :param virtio_feature_mask: Mask of virtio features to be enabled.
249         :type socket: str
250         :type server: bool
251         :type jumbo_frames: bool
252         :type queue_size: int
253         :type queues: int
254         :type virtio_feature_mask: int
255         """
256         self._nic_id += 1
257         if jumbo_frames:
258             logger.debug(u"Jumbo frames temporarily disabled!")
259         self._params.add_with_value(
260             u"chardev", f"socket,id=char{self._nic_id},"
261             f"path={socket}{u',server' if server is True else u''}"
262         )
263         self._params.add_with_value(
264             u"netdev", f"vhost-user,id=vhost{self._nic_id},"
265             f"chardev=char{self._nic_id},queues={queues}"
266         )
267         mac = f"52:54:00:00:{self._opt.get(u'qemu_id'):02x}:" \
268             f"{self._nic_id:02x}"
269         queue_size = f"rx_queue_size={queue_size},tx_queue_size={queue_size}" \
270             if queue_size else u""
271         gso = VirtioFeatureMask.is_feature_enabled(
272             virtio_feature_mask, VirtioFeaturesFlags.VIRTIO_NET_F_API_GSO)
273         csum = VirtioFeatureMask.is_feature_enabled(
274             virtio_feature_mask, VirtioFeaturesFlags.VIRTIO_NET_F_API_CSUM)
275
276         self._params.add_with_value(
277             u"device", f"virtio-net-pci,netdev=vhost{self._nic_id},mac={mac},"
278             f"addr={self._nic_id+5}.0,mq=on,vectors={2 * queues + 2},"
279             f"csum={u'on' if csum else u'off'},"
280             f"gso={u'on' if gso else u'off'},"
281             f"guest_tso4={u'on' if gso else u'off'},"
282             f"guest_tso6={u'on' if gso else u'off'},"
283             f"guest_ecn={u'on' if gso else u'off'},"
284             f"{queue_size}"
285         )
286
287         # Add interface MAC and socket to the node dict.
288         if_data = {u"mac_address": mac, u"socket": socket}
289         if_name = f"vhost{self._nic_id}"
290         self._vm_info[u"interfaces"][if_name] = if_data
291         # Add socket to temporary file list.
292         self._temp[if_name] = socket
293
294     def add_vfio_pci_if(self, pci):
295         """Add VFIO PCI interface.
296
297         :param pci: PCI address of interface.
298         :type pci: str
299         """
300         self._nic_id += 1
301         self._params.add_with_value(
302             u"device", f"vfio-pci,host={pci},addr={self._nic_id+5}.0"
303         )
304
305     def create_kernelvm_config_vpp(self, **kwargs):
306         """Create QEMU VPP config files.
307
308         :param kwargs: Key-value pairs to replace content of VPP configuration
309             file.
310         :type kwargs: dict
311         """
312         startup = f"/etc/vpp/vm_startup_{self._opt.get(u'qemu_id')}.conf"
313         running = f"/etc/vpp/vm_running_{self._opt.get(u'qemu_id')}.exec"
314
315         self._temp[u"startup"] = startup
316         self._temp[u"running"] = running
317         self._opt[u"vnf_bin"] = f"/usr/bin/vpp -c {startup}"
318
319         # Create VPP startup configuration.
320         vpp_config = VppConfigGenerator()
321         vpp_config.set_node(self._node)
322         vpp_config.add_unix_nodaemon()
323         vpp_config.add_unix_cli_listen()
324         vpp_config.add_unix_exec(running)
325         vpp_config.add_socksvr()
326         vpp_config.add_main_heap_size(u"512M")
327         vpp_config.add_main_heap_page_size(self._opt[u"page_size"])
328         vpp_config.add_default_hugepage_size(self._opt[u"page_size"])
329         vpp_config.add_statseg_size(u"512M")
330         vpp_config.add_statseg_page_size(self._opt[u"page_size"])
331         vpp_config.add_statseg_per_node_counters(u"on")
332         vpp_config.add_buffers_per_numa(107520)
333         vpp_config.add_cpu_main_core(u"0")
334         if self._opt.get(u"smp") > 1:
335             vpp_config.add_cpu_corelist_workers(f"1-{self._opt.get(u'smp')-1}")
336         vpp_config.add_plugin(u"disable", u"default")
337         vpp_config.add_plugin(u"enable", u"ping_plugin.so")
338         if "2vfpt" in self._opt.get(u'vnf'):
339             vpp_config.add_plugin(u"enable", u"avf_plugin.so")
340         if "vhost" in self._opt.get(u'vnf'):
341             vpp_config.add_plugin(u"enable", u"dpdk_plugin.so")
342             vpp_config.add_dpdk_dev(u"0000:00:06.0", u"0000:00:07.0")
343             vpp_config.add_dpdk_dev_default_rxq(kwargs[u"queues"])
344             vpp_config.add_dpdk_log_level(u"debug")
345             if not kwargs[u"jumbo_frames"]:
346                 vpp_config.add_dpdk_no_multi_seg()
347                 vpp_config.add_dpdk_no_tx_checksum_offload()
348         if "ipsec" in self._opt.get(u'vnf'):
349             vpp_config.add_plugin(u"enable", u"crypto_native_plugin.so")
350             vpp_config.add_plugin(u"enable", u"crypto_ipsecmb_plugin.so")
351             vpp_config.add_plugin(u"enable", u"crypto_openssl_plugin.so")
352         if "nat" in self._opt.get(u'vnf'):
353             vpp_config.add_nat(value=u"endpoint-dependent")
354             vpp_config.add_plugin(u"enable", u"nat_plugin.so")
355         vpp_config.write_config(startup)
356
357         # Create VPP running configuration.
358         template = f"{Constants.RESOURCES_TPL}/vm/{self._opt.get(u'vnf')}.exec"
359         exec_cmd_no_error(self._node, f"rm -f {running}", sudo=True)
360
361         with open(template, u"rt") as src_file:
362             src = Template(src_file.read())
363             exec_cmd_no_error(
364                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
365                 f"sudo tee {running}"
366             )
367
368     def create_kernelvm_config_testpmd_io(self, **kwargs):
369         """Create QEMU testpmd-io command line.
370
371         :param kwargs: Key-value pairs to construct command line parameters.
372         :type kwargs: dict
373         """
374         pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
375         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
376             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
377             eal_driver=False,
378             eal_pci_whitelist0=u"0000:00:06.0",
379             eal_pci_whitelist1=u"0000:00:07.0",
380             eal_in_memory=True,
381             pmd_num_mbufs=32768,
382             pmd_fwd_mode=u"io",
383             pmd_nb_ports=u"2",
384             pmd_portmask=u"0x3",
385             pmd_max_pkt_len=pmd_max_pkt_len,
386             pmd_mbuf_size=u"16384",
387             pmd_rxq=kwargs[u"queues"],
388             pmd_txq=kwargs[u"queues"],
389             pmd_tx_offloads='0x0',
390             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
391         )
392
393         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
394
395     def create_kernelvm_config_testpmd_mac(self, **kwargs):
396         """Create QEMU testpmd-mac command line.
397
398         :param kwargs: Key-value pairs to construct command line parameters.
399         :type kwargs: dict
400         """
401         pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
402         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
403             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
404             eal_driver=False,
405             eal_pci_whitelist0=u"0000:00:06.0",
406             eal_pci_whitelist1=u"0000:00:07.0",
407             eal_in_memory=True,
408             pmd_num_mbufs=32768,
409             pmd_fwd_mode=u"mac",
410             pmd_nb_ports=u"2",
411             pmd_portmask=u"0x3",
412             pmd_max_pkt_len=pmd_max_pkt_len,
413             pmd_mbuf_size=u"16384",
414             pmd_eth_peer_0=f"0,{kwargs[u'vif1_mac']}",
415             pmd_eth_peer_1=f"1,{kwargs[u'vif2_mac']}",
416             pmd_rxq=kwargs[u"queues"],
417             pmd_txq=kwargs[u"queues"],
418             pmd_tx_offloads=u"0x0",
419             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
420         )
421
422         self._opt[u"vnf_bin"] = f"{self._testpmd_path}/{testpmd_cmd}"
423
424     def create_kernelvm_config_iperf3(self):
425         """Create QEMU iperf3 command line."""
426         self._opt[u"vnf_bin"] = f"mkdir /run/sshd; /usr/sbin/sshd -D -d"
427
428     def create_kernelvm_init(self, **kwargs):
429         """Create QEMU init script.
430
431         :param kwargs: Key-value pairs to replace content of init startup file.
432         :type kwargs: dict
433         """
434         init = self._temp.get(u"ini")
435         exec_cmd_no_error(self._node, f"rm -f {init}", sudo=True)
436
437         with open(kwargs[u"template"], u"rt") as src_file:
438             src = Template(src_file.read())
439             exec_cmd_no_error(
440                 self._node, f"echo '{src.safe_substitute(**kwargs)}' | "
441                 f"sudo tee {init}"
442             )
443             exec_cmd_no_error(self._node, f"chmod +x {init}", sudo=True)
444
445     def configure_kernelvm_vnf(self, **kwargs):
446         """Create KernelVM VNF configurations.
447
448         :param kwargs: Key-value pairs for templating configs.
449         :type kwargs: dict
450         """
451         if u"vpp" in self._opt.get(u"vnf"):
452             self.create_kernelvm_config_vpp(**kwargs)
453             self.create_kernelvm_init(
454                 template=f"{Constants.RESOURCES_TPL}/vm/init.sh",
455                 vnf_bin=self._opt.get(u"vnf_bin")
456             )
457         elif u"testpmd_io" in self._opt.get(u"vnf"):
458             self.create_kernelvm_config_testpmd_io(**kwargs)
459             self.create_kernelvm_init(
460                 template=f"{Constants.RESOURCES_TPL}/vm/init.sh",
461                 vnf_bin=self._opt.get(u"vnf_bin")
462             )
463         elif u"testpmd_mac" in self._opt.get(u"vnf"):
464             self.create_kernelvm_config_testpmd_mac(**kwargs)
465             self.create_kernelvm_init(
466                 template=f"{Constants.RESOURCES_TPL}/vm/init.sh",
467                 vnf_bin=self._opt.get(u"vnf_bin")
468             )
469         elif u"iperf3" in self._opt.get(u"vnf"):
470             qemu_id = self._opt.get(u'qemu_id') % 2
471             self.create_kernelvm_config_iperf3()
472             self.create_kernelvm_init(
473                 template=f"{Constants.RESOURCES_TPL}/vm/init_iperf3.sh",
474                 vnf_bin=self._opt.get(u"vnf_bin"),
475                 ip_address_l=u"2.2.2.2/30" if qemu_id else u"1.1.1.1/30",
476                 ip_address_r=u"2.2.2.1" if qemu_id else u"1.1.1.2",
477                 ip_route_r=u"1.1.1.0/30" if qemu_id else u"2.2.2.0/30"
478             )
479         else:
480             raise RuntimeError(u"QEMU: Unsupported VNF!")
481
482     def get_qemu_pids(self):
483         """Get QEMU CPU pids.
484
485         :returns: List of QEMU CPU pids.
486         :rtype: list of str
487         """
488         command = f"grep -rwl 'CPU' /proc/$(sudo cat " \
489             f"{self._temp.get(u'pidfile')})/task/*/comm "
490         command += r"| xargs dirname | sed -e 's/\/.*\///g' | uniq"
491
492         stdout, _ = exec_cmd_no_error(self._node, command)
493         return stdout.splitlines()
494
495     def qemu_set_affinity(self, *host_cpus):
496         """Set qemu affinity by getting thread PIDs via QMP and taskset to list
497         of CPU cores. Function tries to execute 3 times to avoid race condition
498         in getting thread PIDs.
499
500         :param host_cpus: List of CPU cores.
501         :type host_cpus: list
502         """
503         for _ in range(3):
504             try:
505                 qemu_cpus = self.get_qemu_pids()
506
507                 if len(qemu_cpus) != len(host_cpus):
508                     sleep(1)
509                     continue
510                 for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus):
511                     command = f"taskset -pc {host_cpu} {qemu_cpu}"
512                     message = f"QEMU: Set affinity failed " \
513                         f"on {self._node[u'host']}!"
514                     exec_cmd_no_error(
515                         self._node, command, sudo=True, message=message
516                     )
517                 break
518             except (RuntimeError, ValueError):
519                 self.qemu_kill_all()
520                 raise
521         else:
522             self.qemu_kill_all()
523             raise RuntimeError(u"Failed to set Qemu threads affinity!")
524
525     def qemu_set_scheduler_policy(self):
526         """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU
527         processes.
528
529         :raises RuntimeError: Set scheduler policy failed.
530         """
531         try:
532             qemu_cpus = self.get_qemu_pids()
533
534             for qemu_cpu in qemu_cpus:
535                 command = f"chrt -r -p 1 {qemu_cpu}"
536                 message = f"QEMU: Set SCHED_RR failed on {self._node[u'host']}"
537                 exec_cmd_no_error(
538                     self._node, command, sudo=True, message=message
539                 )
540         except (RuntimeError, ValueError):
541             self.qemu_kill_all()
542             raise
543
544     def _qemu_qmp_exec(self, cmd):
545         """Execute QMP command.
546
547         QMP is JSON based protocol which allows to control QEMU instance.
548
549         :param cmd: QMP command to execute.
550         :type cmd: str
551         :returns: Command output in python representation of JSON format. The
552             { "return": {} } response is QMP's success response. An error
553             response will contain the "error" keyword instead of "return".
554         """
555         # To enter command mode, the qmp_capabilities command must be issued.
556         command = f"echo \"{{{{ \\\"execute\\\": " \
557             f"\\\"qmp_capabilities\\\" }}}}" \
558             f"{{{{ \\\"execute\\\": \\\"{cmd}\\\" }}}}\" | " \
559             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qmp')}"
560         message = f"QMP execute '{cmd}' failed on {self._node[u'host']}"
561
562         stdout, _ = exec_cmd_no_error(
563             self._node, command, sudo=False, message=message
564         )
565
566         # Skip capabilities negotiation messages.
567         out_list = stdout.splitlines()
568         if len(out_list) < 3:
569             raise RuntimeError(f"Invalid QMP output on {self._node[u'host']}")
570         return json.loads(out_list[2])
571
572     def _qemu_qga_flush(self):
573         """Flush the QGA parser state."""
574         command = f"(printf \"\xFF\"; sleep 1) | sudo -S socat " \
575             f"- UNIX-CONNECT:{self._temp.get(u'qga')}"
576         message = f"QGA flush failed on {self._node[u'host']}"
577         stdout, _ = exec_cmd_no_error(
578             self._node, command, sudo=False, message=message
579         )
580
581         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
582
583     def _qemu_qga_exec(self, cmd):
584         """Execute QGA command.
585
586         QGA provide access to a system-level agent via standard QMP commands.
587
588         :param cmd: QGA command to execute.
589         :type cmd: str
590         """
591         command = f"(echo \"{{{{ \\\"execute\\\": " \
592             f"\\\"{cmd}\\\" }}}}\"; sleep 1) | " \
593             f"sudo -S socat - UNIX-CONNECT:{self._temp.get(u'qga')}"
594         message = f"QGA execute '{cmd}' failed on {self._node[u'host']}"
595         stdout, _ = exec_cmd_no_error(
596             self._node, command, sudo=False, message=message
597         )
598
599         return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
600
601     def _wait_until_vm_boot(self):
602         """Wait until QEMU VM is booted."""
603         try:
604             getattr(self, f'_wait_{self._opt["vnf"]}')()
605         except AttributeError:
606             self._wait_default()
607
608     def _wait_default(self, retries=60):
609         """Wait until QEMU with VPP is booted.
610
611         :param retries: Number of retries.
612         :type retries: int
613         """
614         for _ in range(retries):
615             command = f"tail -1 {self._temp.get(u'log')}"
616             stdout = None
617             try:
618                 stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
619                 sleep(1)
620             except RuntimeError:
621                 pass
622             if "vpp " in stdout and "built by" in stdout:
623                 break
624             if u"Press enter to exit" in stdout:
625                 break
626             if u"reboot: Power down" in stdout:
627                 raise RuntimeError(
628                     f"QEMU: NF failed to run on {self._node[u'host']}!"
629                 )
630         else:
631             raise RuntimeError(
632                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
633             )
634
635     def _wait_nestedvm(self, retries=12):
636         """Wait until QEMU with NestedVM is booted.
637
638         First try to flush qga until there is output.
639         Then ping QEMU guest agent each 5s until VM booted or timeout.
640
641         :param retries: Number of retries with 5s between trials.
642         :type retries: int
643         """
644         for _ in range(retries):
645             out = None
646             try:
647                 out = self._qemu_qga_flush()
648             except ValueError:
649                 logger.trace(f"QGA qga flush unexpected output {out}")
650             # Empty output - VM not booted yet
651             if not out:
652                 sleep(5)
653             else:
654                 break
655         else:
656             raise RuntimeError(
657                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
658             )
659         for _ in range(retries):
660             out = None
661             try:
662                 out = self._qemu_qga_exec(u"guest-ping")
663             except ValueError:
664                 logger.trace(f"QGA guest-ping unexpected output {out}")
665             # Empty output - VM not booted yet.
666             if not out:
667                 sleep(5)
668             # Non-error return - VM booted.
669             elif out.get(u"return") is not None:
670                 break
671             # Skip error and wait.
672             elif out.get(u"error") is not None:
673                 sleep(5)
674             else:
675                 # If there is an unexpected output from QGA guest-info, try
676                 # again until timeout.
677                 logger.trace(f"QGA guest-ping unexpected output {out}")
678         else:
679             raise RuntimeError(
680                 f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
681             )
682
683     def _wait_iperf3(self, retries=60):
684         """Wait until QEMU with iPerf3 is booted.
685
686         :param retries: Number of retries.
687         :type retries: int
688         """
689         grep = u"Server listening on 0.0.0.0 port 22."
690         cmd = f"fgrep '{grep}' {self._temp.get(u'log')}"
691         message = f"QEMU: Timeout, VM not booted on {self._node[u'host']}!"
692         exec_cmd_no_error(
693             self._node, cmd=cmd, sudo=True, message=message, retries=retries,
694             include_reason=True
695         )
696
697     def _update_vm_interfaces(self):
698         """Update interface names in VM node dict."""
699         # Send guest-network-get-interfaces command via QGA, output example:
700         # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"},
701         # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]}.
702         out = self._qemu_qga_exec(u"guest-network-get-interfaces")
703         interfaces = out.get(u"return")
704         mac_name = {}
705         if not interfaces:
706             raise RuntimeError(
707                 f"Get VM interface list failed on {self._node[u'host']}"
708             )
709         # Create MAC-name dict.
710         for interface in interfaces:
711             if u"hardware-address" not in interface:
712                 continue
713             mac_name[interface[u"hardware-address"]] = interface[u"name"]
714         # Match interface by MAC and save interface name.
715         for interface in self._vm_info[u"interfaces"].values():
716             mac = interface.get(u"mac_address")
717             if_name = mac_name.get(mac)
718             if if_name is None:
719                 logger.trace(f"Interface name for MAC {mac} not found")
720             else:
721                 interface[u"name"] = if_name
722
723     def qemu_start(self):
724         """Start QEMU and wait until VM boot.
725
726         :returns: VM node info.
727         :rtype: dict
728         """
729         cmd_opts = OptionString()
730         cmd_opts.add(f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch}")
731         cmd_opts.extend(self._params)
732         message = f"QEMU: Start failed on {self._node[u'host']}!"
733         try:
734             DUTSetup.check_huge_page(
735                 self._node, self._opt.get(u"mem-path"),
736                 int(self._opt.get(u"mem"))
737             )
738
739             exec_cmd_no_error(
740                 self._node, cmd_opts, timeout=300, sudo=True, message=message
741             )
742             self._wait_until_vm_boot()
743         except RuntimeError:
744             self.qemu_kill_all()
745             raise
746         return self._vm_info
747
748     def qemu_kill(self):
749         """Kill qemu process."""
750         exec_cmd(
751             self._node, f"chmod +r {self._temp.get(u'pidfile')}", sudo=True
752         )
753         exec_cmd(
754             self._node, f"kill -SIGKILL $(cat {self._temp.get(u'pidfile')})",
755             sudo=True
756         )
757
758         for value in self._temp.values():
759             exec_cmd(self._node, f"cat {value}", sudo=True)
760             exec_cmd(self._node, f"rm -f {value}", sudo=True)
761
762     def qemu_kill_all(self):
763         """Kill all qemu processes on DUT node if specified."""
764         exec_cmd(self._node, u"pkill -SIGKILL qemu", sudo=True)
765
766         for value in self._temp.values():
767             exec_cmd(self._node, f"cat {value}", sudo=True)
768             exec_cmd(self._node, f"rm -f {value}", sudo=True)
769
770     def qemu_version(self):
771         """Return Qemu version.
772
773         :returns: Qemu version.
774         :rtype: str
775         """
776         command = f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch} " \
777             f"--version"
778         try:
779             stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
780             return match(r"QEMU emulator version ([\d.]*)", stdout).group(1)
781         except RuntimeError:
782             self.qemu_kill_all()
783             raise