1 # Copyright (c) 2018 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """DUT setup library."""
16 from robot.api import logger
18 from resources.libraries.python.topology import NodeType, Topology
19 from resources.libraries.python.ssh import SSH
20 from resources.libraries.python.constants import Constants
21 from resources.libraries.python.VatExecutor import VatExecutor
22 from resources.libraries.python.VPPUtil import VPPUtil
25 class DUTSetup(object):
26 """Contains methods for setting up DUTs."""
29 def get_service_logs(node, service):
30 """Get specific service unit logs by journalctl from node.
32 :param node: Node in the topology.
33 :param service: Service unit name.
40 ssh.exec_command_sudo('journalctl --no-pager --unit={name} '
41 '--since="$(echo `systemctl show -p '
42 'ActiveEnterTimestamp {name}` | '
43 'awk \'{{print $2 $3}}\')"'.
45 if int(ret_code) != 0:
46 raise RuntimeError('DUT {host} failed to get logs from unit {name}'.
47 format(host=node['host'], name=service))
50 def get_service_logs_on_all_duts(nodes, service):
51 """Get specific service unit logs by journalctl from all DUTs.
53 :param nodes: Nodes in the topology.
54 :param service: Service unit name.
58 for node in nodes.values():
59 if node['type'] == NodeType.DUT:
60 DUTSetup.get_service_logs(node, service)
63 def start_service(node, service):
64 """Start up the named service on node.
66 :param node: Node in the topology.
67 :param service: Service unit name.
73 # We are doing restart. With this we do not care if service
76 ssh.exec_command_sudo('service {name} restart'.
77 format(name=service), timeout=120)
78 if int(ret_code) != 0:
79 raise RuntimeError('DUT {host} failed to start service {name}'.
80 format(host=node['host'], name=service))
82 DUTSetup.get_service_logs(node, service)
85 def start_vpp_service_on_all_duts(nodes):
86 """Start up the VPP service on all nodes.
88 :param nodes: Nodes in the topology.
91 for node in nodes.values():
92 if node['type'] == NodeType.DUT:
93 DUTSetup.start_service(node, Constants.VPP_UNIT)
96 def vpp_show_version_verbose(node):
97 """Run "show version verbose" CLI command.
99 :param node: Node to run command on.
103 vat.execute_script("show_version_verbose.vat", node, json_out=False)
106 vat.script_should_have_passed()
107 except AssertionError:
108 raise RuntimeError('Failed to get VPP version on host: {name}'.
109 format(name=node['host']))
112 def show_vpp_version_on_all_duts(nodes):
113 """Show VPP version verbose on all DUTs.
115 :param nodes: VPP nodes
118 for node in nodes.values():
119 if node['type'] == NodeType.DUT:
120 DUTSetup.vpp_show_version_verbose(node)
123 def vpp_show_interfaces(node):
124 """Run "show interface" CLI command.
126 :param node: Node to run command on.
130 vat.execute_script("show_interface.vat", node, json_out=False)
133 vat.script_should_have_passed()
134 except AssertionError:
135 raise RuntimeError('Failed to get VPP interfaces on host: {name}'.
136 format(name=node['host']))
139 def vpp_api_trace_save(node):
140 """Run "api trace save" CLI command.
142 :param node: Node to run command on.
146 vat.execute_script("api_trace_save.vat", node, json_out=False)
149 def vpp_api_trace_dump(node):
150 """Run "api trace custom-dump" CLI command.
152 :param node: Node to run command on.
156 vat.execute_script("api_trace_dump.vat", node, json_out=False)
159 def setup_all_duts(nodes):
160 """Prepare all DUTs in given topology for test execution."""
161 for node in nodes.values():
162 if node['type'] == NodeType.DUT:
163 DUTSetup.setup_dut(node)
167 """Run script over SSH to setup the DUT node.
169 :param node: DUT node to set up.
172 :raises Exception: If the DUT setup fails.
178 ssh.exec_command('sudo -Sn bash {0}/{1}/dut_setup.sh'.
179 format(Constants.REMOTE_FW_DIR,
180 Constants.RESOURCES_LIB_SH), timeout=120)
181 if int(ret_code) != 0:
182 raise RuntimeError('DUT test setup script failed at node {name}'.
183 format(name=node['host']))
186 def get_vpp_pid(node):
187 """Get PID of running VPP process.
189 :param node: DUT node.
193 :raises RuntimeError: If it is not possible to get the PID.
200 logger.trace('Try {}: Get VPP PID'.format(i))
201 ret_code, stdout, stderr = ssh.exec_command('pidof vpp')
203 if int(ret_code) != 0:
204 raise RuntimeError('Not possible to get PID of VPP process '
205 'on node: {0}\n {1}'.
206 format(node['host'], stdout + stderr))
208 if len(stdout.splitlines()) == 1:
210 elif len(stdout.splitlines()) == 0:
211 logger.debug("No VPP PID found on node {0}".
212 format(node['host']))
215 logger.debug("More then one VPP PID found on node {0}".
216 format(node['host']))
218 for line in stdout.splitlines():
219 ret_list.append(int(line))
225 def get_vpp_pids(nodes):
226 """Get PID of running VPP process on all DUTs.
228 :param nodes: DUT nodes.
235 for node in nodes.values():
236 if node['type'] == NodeType.DUT:
237 pids[node['host']] = DUTSetup.get_vpp_pid(node)
241 def vpp_show_crypto_device_mapping(node):
242 """Run "show crypto device mapping" CLI command.
244 :param node: Node to run command on.
248 vat.execute_script("show_crypto_device_mapping.vat", node,
252 def crypto_device_verify(node, force_init=False, numvfs=32):
253 """Verify if Crypto QAT device virtual functions are initialized on all
254 DUTs. If parameter force initialization is set to True, then try to
255 initialize or disable QAT.
257 :param node: DUT node.
258 :param force_init: If True then try to initialize to specific value.
259 :param numvfs: Number of VFs to initialize, 0 - disable the VFs.
261 :type force_init: bool
264 :raises RuntimeError: If QAT is not initialized or failed to initialize.
270 cryptodev = Topology.get_cryptodev(node)
271 cmd = 'cat /sys/bus/pci/devices/{0}/sriov_numvfs'.\
272 format(cryptodev.replace(':', r'\:'))
274 # Try to read number of VFs from PCI address of QAT device
276 ret_code, stdout, _ = ssh.exec_command(cmd)
277 if int(ret_code) == 0:
279 sriov_numvfs = int(stdout)
281 logger.trace('Reading sriov_numvfs info failed on {0}'.
282 format(node['host']))
284 if sriov_numvfs != numvfs:
286 # QAT is not initialized and we want to initialize
288 DUTSetup.crypto_device_init(node, numvfs)
290 raise RuntimeError('QAT device {0} is not '
291 'initialized to {1} on host {2}'
292 .format(cryptodev, numvfs,
297 def crypto_device_init(node, numvfs):
298 """Init Crypto QAT device virtual functions on DUT.
300 :param node: DUT node.
301 :param numvfs: Number of VFs to initialize, 0 - disable the VFs.
305 :raises RuntimeError: If failed to stop VPP or QAT failed to initialize.
307 cryptodev = Topology.get_cryptodev(node)
309 # QAT device must be re-bound to kernel driver before initialization
311 kernel_module = 'qat_dh895xcc'
312 current_driver = DUTSetup.get_pci_dev_driver(
313 node, cryptodev.replace(':', r'\:'))
315 DUTSetup.kernel_module_verify(node, kernel_module, force_load=True)
317 VPPUtil.stop_vpp_service(node)
318 if current_driver is not None:
319 DUTSetup.pci_driver_unbind(node, cryptodev)
320 DUTSetup.pci_driver_bind(node, cryptodev, driver)
327 cmd = 'echo "{0}" | tee /sys/bus/pci/devices/{1}/sriov_numvfs'.\
328 format(numvfs, cryptodev.replace(':', r'\:'), timeout=180)
329 ret_code, _, _ = ssh.exec_command_sudo("sh -c '{0}'".format(cmd))
331 if int(ret_code) != 0:
332 raise RuntimeError('Failed to initialize {0} VFs on QAT device '
333 ' on host {1}'.format(numvfs, node['host']))
336 def pci_driver_unbind(node, pci_addr):
337 """Unbind PCI device from current driver on node.
339 :param node: DUT node.
340 :param pci_addr: PCI device address.
344 :raises RuntimeError: If PCI device unbind failed.
350 ret_code, _, _ = ssh.exec_command_sudo(
351 "sh -c 'echo {0} | tee /sys/bus/pci/devices/{1}/driver/unbind'"
352 .format(pci_addr, pci_addr.replace(':', r'\:')), timeout=180)
354 if int(ret_code) != 0:
355 raise RuntimeError('Failed to unbind PCI device {0} from driver on '
356 'host {1}'.format(pci_addr, node['host']))
359 def pci_driver_bind(node, pci_addr, driver):
360 """Bind PCI device to driver on node.
362 :param node: DUT node.
363 :param pci_addr: PCI device address.
364 :param driver: Driver to bind.
369 :raises RuntimeError: If PCI device bind failed.
375 ret_code, _, _ = ssh.exec_command_sudo(
376 "sh -c 'echo {0} | tee /sys/bus/pci/drivers/{1}/bind'".format(
377 pci_addr, driver), timeout=180)
379 if int(ret_code) != 0:
380 raise RuntimeError('Failed to bind PCI device {0} to {1} driver on '
381 'host {2}'.format(pci_addr, driver,
385 def get_pci_dev_driver(node, pci_addr):
386 """Get current PCI device driver on node.
389 # lspci -vmmks 0000:00:05.0
391 Class: Ethernet controller
393 Device: Virtio network device
394 SVendor: Red Hat, Inc
399 :param node: DUT node.
400 :param pci_addr: PCI device address.
403 :returns: Driver or None
404 :raises RuntimeError: If PCI rescan or lspci command execution failed.
405 :raises RuntimeError: If it is not possible to get the interface driver
406 information from the node.
412 logger.trace('Try number {0}: Get PCI device driver'.format(i))
413 cmd = 'lspci -vmmks {0}'.format(pci_addr)
414 ret_code, stdout, _ = ssh.exec_command(cmd)
415 if int(ret_code) != 0:
416 raise RuntimeError("'{0}' failed on '{1}'"
417 .format(cmd, node['host']))
419 for line in stdout.splitlines():
425 name, value = line.split("\t", 1)
427 if name == "Driver:":
429 if name == 'Driver:':
433 logger.trace('Driver for PCI device {} not found, executing '
434 'pci rescan and retrying'.format(pci_addr))
435 cmd = 'sh -c "echo 1 > /sys/bus/pci/rescan"'
436 ret_code, _, _ = ssh.exec_command_sudo(cmd)
437 if int(ret_code) != 0:
438 raise RuntimeError("'{0}' failed on '{1}'"
439 .format(cmd, node['host']))
444 def kernel_module_verify(node, module, force_load=False):
445 """Verify if kernel module is loaded on node. If parameter force
446 load is set to True, then try to load the modules.
449 :param module: Module to verify.
450 :param force_load: If True then try to load module.
453 :type force_load: bool
454 :raises RuntimeError: If module is not loaded or failed to load.
459 cmd = 'grep -w {0} /proc/modules'.format(module)
460 ret_code, _, _ = ssh.exec_command(cmd)
462 if int(ret_code) != 0:
464 # Module is not loaded and we want to load it
465 DUTSetup.kernel_module_load(node, module)
467 raise RuntimeError('Kernel module {0} is not loaded on host '
468 '{1}'.format(module, node['host']))
471 def kernel_module_verify_on_all_duts(nodes, module, force_load=False):
472 """Verify if kernel module is loaded on all DUTs. If parameter force
473 load is set to True, then try to load the modules.
475 :param node: DUT nodes.
476 :param module: Module to verify.
477 :param force_load: If True then try to load module.
480 :type force_load: bool
482 for node in nodes.values():
483 if node['type'] == NodeType.DUT:
484 DUTSetup.kernel_module_verify(node, module, force_load)
487 def verify_uio_driver_on_all_duts(nodes):
488 """Verify if uio driver kernel module is loaded on all DUTs. If module
489 is not present it will try to load it.
491 :param node: DUT nodes.
494 for node in nodes.values():
495 if node['type'] == NodeType.DUT:
496 uio_driver = Topology.get_uio_driver(node)
497 DUTSetup.kernel_module_verify(node, uio_driver, force_load=True)
500 def kernel_module_load(node, module):
501 """Load kernel module on node.
503 :param node: DUT node.
504 :param module: Module to load.
508 :raises RuntimeError: If loading failed.
514 ret_code, _, _ = ssh.exec_command_sudo("modprobe {0}".format(module))
516 if int(ret_code) != 0:
517 raise RuntimeError('Failed to load {0} kernel module on host {1}'.
518 format(module, node['host']))
521 def vpp_enable_traces_on_all_duts(nodes):
522 """Enable vpp packet traces on all DUTs in the given topology.
524 :param nodes: Nodes in the topology.
527 for node in nodes.values():
528 if node['type'] == NodeType.DUT:
529 DUTSetup.vpp_enable_traces_on_dut(node)
532 def vpp_enable_traces_on_dut(node):
533 """Enable vpp packet traces on the DUT node.
535 :param node: DUT node to set up.
540 vat.execute_script("enable_dpdk_traces.vat", node, json_out=False)
541 vat.execute_script("enable_vhost_user_traces.vat", node, json_out=False)
542 vat.execute_script("enable_memif_traces.vat", node, json_out=False)
545 def install_vpp_on_all_duts(nodes, vpp_pkg_dir, vpp_rpm_pkgs, vpp_deb_pkgs):
546 """Install VPP on all DUT nodes.
548 :param nodes: Nodes in the topology.
549 :param vpp_pkg_dir: Path to directory where VPP packages are stored.
550 :param vpp_rpm_pkgs: List of VPP rpm packages to be installed.
551 :param vpp_deb_pkgs: List of VPP deb packages to be installed.
553 :type vpp_pkg_dir: str
554 :type vpp_rpm_pkgs: list
555 :type vpp_deb_pkgs: list
556 :raises RuntimeError: If failed to remove or install VPP.
559 logger.debug("Installing VPP")
561 for node in nodes.values():
562 if node['type'] == NodeType.DUT:
563 logger.debug("Installing VPP on node {0}".format(node['host']))
568 cmd = "[[ -f /etc/redhat-release ]]"
569 return_code, _, _ = ssh.exec_command(cmd)
570 if int(return_code) == 0:
571 # workaroud - uninstall existing vpp installation until
572 # start-testcase script is updated on all virl servers
573 rpm_pkgs_remove = "vpp*"
574 cmd_u = 'yum -y remove "{0}"'.format(rpm_pkgs_remove)
575 r_rcode, _, r_err = ssh.exec_command_sudo(cmd_u, timeout=90)
576 if int(r_rcode) != 0:
577 raise RuntimeError('Failed to remove previous VPP'
578 'installation on host {0}:\n{1}'
579 .format(node['host'], r_err))
581 rpm_pkgs = "*.rpm ".join(str(vpp_pkg_dir + pkg)
582 for pkg in vpp_rpm_pkgs) + "*.rpm"
583 cmd_i = "rpm -ivh {0}".format(rpm_pkgs)
584 ret_code, _, err = ssh.exec_command_sudo(cmd_i, timeout=90)
585 if int(ret_code) != 0:
586 raise RuntimeError('Failed to install VPP on host {0}:'
587 '\n{1}'.format(node['host'], err))
589 ssh.exec_command_sudo("rpm -qai vpp*")
590 logger.info("VPP installed on node {0}".
591 format(node['host']))
593 # workaroud - uninstall existing vpp installation until
594 # start-testcase script is updated on all virl servers
595 deb_pkgs_remove = "vpp*"
596 cmd_u = 'apt-get purge -y "{0}"'.format(deb_pkgs_remove)
597 r_rcode, _, r_err = ssh.exec_command_sudo(cmd_u, timeout=90)
598 if int(r_rcode) != 0:
599 raise RuntimeError('Failed to remove previous VPP'
600 'installation on host {0}:\n{1}'
601 .format(node['host'], r_err))
602 deb_pkgs = "*.deb ".join(str(vpp_pkg_dir + pkg)
603 for pkg in vpp_deb_pkgs) + "*.deb"
604 cmd_i = "dpkg -i --force-all {0}".format(deb_pkgs)
605 ret_code, _, err = ssh.exec_command_sudo(cmd_i, timeout=90)
606 if int(ret_code) != 0:
607 raise RuntimeError('Failed to install VPP on host {0}:'
608 '\n{1}'.format(node['host'], err))
610 ssh.exec_command_sudo("dpkg -l | grep vpp")
611 logger.info("VPP installed on node {0}".
612 format(node['host']))
617 def verify_vpp_on_all_duts(nodes):
618 """Verify that VPP is installed on all DUT nodes.
620 :param nodes: Nodes in the topology.
624 logger.debug("Verify VPP on all DUTs")
626 DUTSetup.start_vpp_service_on_all_duts(nodes)
628 for node in nodes.values():
629 if node['type'] == NodeType.DUT:
630 DUTSetup.verify_vpp_on_dut(node)
633 def verify_vpp_on_dut(node):
634 """Verify that VPP is installed on DUT node.
636 :param node: DUT node.
638 :raises RuntimeError: If failed to restart VPP, get VPP version
639 or get VPP interfaces.
642 logger.debug("Verify VPP on node {0}".format(node['host']))
644 DUTSetup.vpp_show_version_verbose(node)
645 DUTSetup.vpp_show_interfaces(node)