1 # Copyright (c) 2018 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """DUT setup library."""
16 from robot.api import logger
18 from resources.libraries.python.topology import NodeType, Topology
19 from resources.libraries.python.ssh import SSH
20 from resources.libraries.python.constants import Constants
21 from resources.libraries.python.VatExecutor import VatExecutor
22 from resources.libraries.python.VPPUtil import VPPUtil
25 class DUTSetup(object):
26 """Contains methods for setting up DUTs."""
29 def get_service_logs(node, service):
30 """Get specific service unit logs by journalctl from node.
32 :param node: Node in the topology.
33 :param service: Service unit name.
40 ssh.exec_command_sudo('journalctl --no-pager --unit={name} '
41 '--since="$(echo `systemctl show -p '
42 'ActiveEnterTimestamp {name}` | '
43 'awk \'{{print $2 $3}}\')"'.
46 raise RuntimeError('DUT {host} failed to get logs from unit {name}'.
47 format(host=node['host'], name=service))
50 def get_service_logs_on_all_duts(nodes, service):
51 """Get specific service unit logs by journalctl from all DUTs.
53 :param nodes: Nodes in the topology.
54 :param service: Service unit name.
58 for node in nodes.values():
59 if node['type'] == NodeType.DUT:
60 DUTSetup.get_service_logs(node, service)
63 def start_service(node, service):
64 """Start up the named service on node.
66 :param node: Node in the topology.
67 :param service: Service unit name.
73 # We are doing restart. With this we do not care if service
76 ssh.exec_command_sudo('service {name} restart'.
77 format(name=service), timeout=120)
79 raise RuntimeError('DUT {host} failed to start service {name}'.
80 format(host=node['host'], name=service))
82 DUTSetup.get_service_logs(node, service)
85 def start_vpp_service_on_all_duts(nodes):
86 """Start up the VPP service on all nodes.
88 :param nodes: Nodes in the topology.
91 for node in nodes.values():
92 if node['type'] == NodeType.DUT:
93 DUTSetup.start_service(node, Constants.VPP_UNIT)
96 def vpp_show_version_verbose(node):
97 """Run "show version verbose" CLI command.
99 :param node: Node to run command on.
103 vat.execute_script("show_version_verbose.vat", node, json_out=False)
106 vat.script_should_have_passed()
107 except AssertionError:
108 raise RuntimeError('Failed to get VPP version on host: {name}'.
109 format(name=node['host']))
112 def show_vpp_version_on_all_duts(nodes):
113 """Show VPP version verbose on all DUTs.
115 :param nodes: VPP nodes
118 for node in nodes.values():
119 if node['type'] == NodeType.DUT:
120 DUTSetup.vpp_show_version_verbose(node)
123 def vpp_show_interfaces(node):
124 """Run "show interface" CLI command.
126 :param node: Node to run command on.
130 vat.execute_script("show_interface.vat", node, json_out=False)
133 vat.script_should_have_passed()
134 except AssertionError:
135 raise RuntimeError('Failed to get VPP interfaces on host: {name}'.
136 format(name=node['host']))
139 def vpp_api_trace_save(node):
140 """Run "api trace save" CLI command.
142 :param node: Node to run command on.
146 vat.execute_script("api_trace_save.vat", node, json_out=False)
149 def vpp_api_trace_dump(node):
150 """Run "api trace custom-dump" CLI command.
152 :param node: Node to run command on.
156 vat.execute_script("api_trace_dump.vat", node, json_out=False)
159 def setup_all_duts(nodes):
160 """Prepare all DUTs in given topology for test execution."""
161 for node in nodes.values():
162 if node['type'] == NodeType.DUT:
163 DUTSetup.setup_dut(node)
167 """Run script over SSH to setup the DUT node.
169 :param node: DUT node to set up.
172 :raises Exception: If the DUT setup fails.
178 ssh.exec_command('sudo -Sn bash {0}/{1}/dut_setup.sh'.
179 format(Constants.REMOTE_FW_DIR,
180 Constants.RESOURCES_LIB_SH), timeout=120)
182 raise RuntimeError('DUT test setup script failed at node {name}'.
183 format(name=node['host']))
186 def get_vpp_pid(node):
187 """Get PID of running VPP process.
189 :param node: DUT node.
193 :raises RuntimeError: If it is not possible to get the PID.
200 logger.trace('Try {}: Get VPP PID'.format(i))
201 ret_code, stdout, stderr = ssh.exec_command('pidof vpp')
204 raise RuntimeError('Not possible to get PID of VPP process '
205 'on node: {0}\n {1}'.
206 format(node['host'], stdout + stderr))
208 if len(stdout.splitlines()) == 1:
210 elif not stdout.splitlines():
211 logger.debug("No VPP PID found on node {0}".
212 format(node['host']))
215 logger.debug("More then one VPP PID found on node {0}".
216 format(node['host']))
218 for line in stdout.splitlines():
219 ret_list.append(int(line))
225 def get_vpp_pids(nodes):
226 """Get PID of running VPP process on all DUTs.
228 :param nodes: DUT nodes.
235 for node in nodes.values():
236 if node['type'] == NodeType.DUT:
237 pids[node['host']] = DUTSetup.get_vpp_pid(node)
241 def vpp_show_crypto_device_mapping(node):
242 """Run "show crypto device mapping" CLI command.
244 :param node: Node to run command on.
248 vat.execute_script("show_crypto_device_mapping.vat", node,
252 def crypto_device_verify(node, force_init=False, numvfs=32):
253 """Verify if Crypto QAT device virtual functions are initialized on all
254 DUTs. If parameter force initialization is set to True, then try to
255 initialize or disable QAT.
257 :param node: DUT node.
258 :param force_init: If True then try to initialize to specific value.
259 :param numvfs: Number of VFs to initialize, 0 - disable the VFs.
261 :type force_init: bool
264 :raises RuntimeError: If QAT is not initialized or failed to initialize.
270 cryptodev = Topology.get_cryptodev(node)
271 cmd = 'cat /sys/bus/pci/devices/{0}/sriov_numvfs'.\
272 format(cryptodev.replace(':', r'\:'))
274 # Try to read number of VFs from PCI address of QAT device
276 ret_code, stdout, _ = ssh.exec_command(cmd)
277 if not int(ret_code):
279 sriov_numvfs = int(stdout)
281 logger.trace('Reading sriov_numvfs info failed on {0}'.
282 format(node['host']))
284 if sriov_numvfs != numvfs:
286 # QAT is not initialized and we want to initialize
288 DUTSetup.crypto_device_init(node, numvfs)
290 raise RuntimeError('QAT device {0} is not '
291 'initialized to {1} on host {2}'
292 .format(cryptodev, numvfs,
297 def crypto_device_init(node, numvfs):
298 """Init Crypto QAT device virtual functions on DUT.
300 :param node: DUT node.
301 :param numvfs: Number of VFs to initialize, 0 - disable the VFs.
305 :raises RuntimeError: If failed to stop VPP or QAT failed to initialize.
307 cryptodev = Topology.get_cryptodev(node)
309 # QAT device must be re-bound to kernel driver before initialization
311 kernel_module = 'qat_dh895xcc'
312 current_driver = DUTSetup.get_pci_dev_driver(
313 node, cryptodev.replace(':', r'\:'))
315 DUTSetup.kernel_module_verify(node, kernel_module, force_load=True)
317 VPPUtil.stop_vpp_service(node)
318 if current_driver is not None:
319 DUTSetup.pci_driver_unbind(node, cryptodev)
320 DUTSetup.pci_driver_bind(node, cryptodev, driver)
327 cmd = 'echo "{0}" | tee /sys/bus/pci/devices/{1}/sriov_numvfs'.\
328 format(numvfs, cryptodev.replace(':', r'\:'), timeout=180)
329 ret_code, _, _ = ssh.exec_command_sudo("sh -c '{0}'".format(cmd))
332 raise RuntimeError('Failed to initialize {0} VFs on QAT device '
333 ' on host {1}'.format(numvfs, node['host']))
336 def pci_driver_unbind(node, pci_addr):
337 """Unbind PCI device from current driver on node.
339 :param node: DUT node.
340 :param pci_addr: PCI device address.
344 :raises RuntimeError: If PCI device unbind failed.
350 ret_code, _, _ = ssh.exec_command_sudo(
351 "sh -c 'echo {0} | tee /sys/bus/pci/devices/{1}/driver/unbind'"
352 .format(pci_addr, pci_addr.replace(':', r'\:')), timeout=180)
355 raise RuntimeError('Failed to unbind PCI device {0} from driver on '
356 'host {1}'.format(pci_addr, node['host']))
359 def pci_driver_bind(node, pci_addr, driver):
360 """Bind PCI device to driver on node.
362 :param node: DUT node.
363 :param pci_addr: PCI device address.
364 :param driver: Driver to bind.
369 :raises RuntimeError: If PCI device bind failed.
375 ret_code, _, _ = ssh.exec_command_sudo(
376 "sh -c 'echo {0} | tee /sys/bus/pci/drivers/{1}/bind'".format(
377 pci_addr, driver), timeout=180)
380 raise RuntimeError('Failed to bind PCI device {0} to {1} driver on '
381 'host {2}'.format(pci_addr, driver,
385 def get_pci_dev_driver(node, pci_addr):
386 """Get current PCI device driver on node.
389 # lspci -vmmks 0000:00:05.0
391 Class: Ethernet controller
393 Device: Virtio network device
394 SVendor: Red Hat, Inc
399 :param node: DUT node.
400 :param pci_addr: PCI device address.
403 :returns: Driver or None
404 :raises RuntimeError: If PCI rescan or lspci command execution failed.
405 :raises RuntimeError: If it is not possible to get the interface driver
406 information from the node.
412 logger.trace('Try number {0}: Get PCI device driver'.format(i))
414 cmd = 'lspci -vmmks {0}'.format(pci_addr)
415 ret_code, stdout, _ = ssh.exec_command(cmd)
417 raise RuntimeError("'{0}' failed on '{1}'"
418 .format(cmd, node['host']))
420 for line in stdout.splitlines():
426 name, value = line.split("\t", 1)
428 if name == "Driver:":
430 if name == 'Driver:':
434 logger.trace('Driver for PCI device {} not found, executing '
435 'pci rescan and retrying'.format(pci_addr))
436 cmd = 'sh -c "echo 1 > /sys/bus/pci/rescan"'
437 ret_code, _, _ = ssh.exec_command_sudo(cmd)
438 if int(ret_code) != 0:
439 raise RuntimeError("'{0}' failed on '{1}'"
440 .format(cmd, node['host']))
445 def kernel_module_verify(node, module, force_load=False):
446 """Verify if kernel module is loaded on node. If parameter force
447 load is set to True, then try to load the modules.
450 :param module: Module to verify.
451 :param force_load: If True then try to load module.
454 :type force_load: bool
455 :raises RuntimeError: If module is not loaded or failed to load.
460 cmd = 'grep -w {0} /proc/modules'.format(module)
461 ret_code, _, _ = ssh.exec_command(cmd)
465 # Module is not loaded and we want to load it
466 DUTSetup.kernel_module_load(node, module)
468 raise RuntimeError('Kernel module {0} is not loaded on host '
469 '{1}'.format(module, node['host']))
472 def kernel_module_verify_on_all_duts(nodes, module, force_load=False):
473 """Verify if kernel module is loaded on all DUTs. If parameter force
474 load is set to True, then try to load the modules.
476 :param node: DUT nodes.
477 :param module: Module to verify.
478 :param force_load: If True then try to load module.
481 :type force_load: bool
483 for node in nodes.values():
484 if node['type'] == NodeType.DUT:
485 DUTSetup.kernel_module_verify(node, module, force_load)
488 def verify_uio_driver_on_all_duts(nodes):
489 """Verify if uio driver kernel module is loaded on all DUTs. If module
490 is not present it will try to load it.
492 :param node: DUT nodes.
495 for node in nodes.values():
496 if node['type'] == NodeType.DUT:
497 uio_driver = Topology.get_uio_driver(node)
498 DUTSetup.kernel_module_verify(node, uio_driver, force_load=True)
501 def kernel_module_load(node, module):
502 """Load kernel module on node.
504 :param node: DUT node.
505 :param module: Module to load.
509 :raises RuntimeError: If loading failed.
515 ret_code, _, _ = ssh.exec_command_sudo("modprobe {0}".format(module))
518 raise RuntimeError('Failed to load {0} kernel module on host {1}'.
519 format(module, node['host']))
522 def vpp_enable_traces_on_all_duts(nodes):
523 """Enable vpp packet traces on all DUTs in the given topology.
525 :param nodes: Nodes in the topology.
528 for node in nodes.values():
529 if node['type'] == NodeType.DUT:
530 DUTSetup.vpp_enable_traces_on_dut(node)
533 def vpp_enable_traces_on_dut(node):
534 """Enable vpp packet traces on the DUT node.
536 :param node: DUT node to set up.
541 vat.execute_script("enable_dpdk_traces.vat", node, json_out=False)
542 vat.execute_script("enable_vhost_user_traces.vat", node, json_out=False)
543 vat.execute_script("enable_memif_traces.vat", node, json_out=False)
546 def install_vpp_on_all_duts(nodes, vpp_pkg_dir, vpp_rpm_pkgs, vpp_deb_pkgs):
547 """Install VPP on all DUT nodes.
549 :param nodes: Nodes in the topology.
550 :param vpp_pkg_dir: Path to directory where VPP packages are stored.
551 :param vpp_rpm_pkgs: List of VPP rpm packages to be installed.
552 :param vpp_deb_pkgs: List of VPP deb packages to be installed.
554 :type vpp_pkg_dir: str
555 :type vpp_rpm_pkgs: list
556 :type vpp_deb_pkgs: list
557 :raises RuntimeError: If failed to remove or install VPP.
560 logger.debug("Installing VPP")
562 for node in nodes.values():
563 if node['type'] == NodeType.DUT:
564 logger.debug("Installing VPP on node {0}".format(node['host']))
569 cmd = "[[ -f /etc/redhat-release ]]"
570 return_code, _, _ = ssh.exec_command(cmd)
571 if not int(return_code):
572 # workaroud - uninstall existing vpp installation until
573 # start-testcase script is updated on all virl servers
574 rpm_pkgs_remove = "vpp*"
575 cmd_u = 'yum -y remove "{0}"'.format(rpm_pkgs_remove)
576 r_rcode, _, r_err = ssh.exec_command_sudo(cmd_u, timeout=90)
578 raise RuntimeError('Failed to remove previous VPP'
579 'installation on host {0}:\n{1}'
580 .format(node['host'], r_err))
582 rpm_pkgs = "*.rpm ".join(str(vpp_pkg_dir + pkg)
583 for pkg in vpp_rpm_pkgs) + "*.rpm"
584 cmd_i = "rpm -ivh {0}".format(rpm_pkgs)
585 ret_code, _, err = ssh.exec_command_sudo(cmd_i, timeout=90)
587 raise RuntimeError('Failed to install VPP on host {0}:'
588 '\n{1}'.format(node['host'], err))
590 ssh.exec_command_sudo("rpm -qai vpp*")
591 logger.info("VPP installed on node {0}".
592 format(node['host']))
594 # workaroud - uninstall existing vpp installation until
595 # start-testcase script is updated on all virl servers
596 deb_pkgs_remove = "vpp*"
597 cmd_u = 'apt-get purge -y "{0}"'.format(deb_pkgs_remove)
598 r_rcode, _, r_err = ssh.exec_command_sudo(cmd_u, timeout=90)
600 raise RuntimeError('Failed to remove previous VPP'
601 'installation on host {0}:\n{1}'
602 .format(node['host'], r_err))
603 deb_pkgs = "*.deb ".join(str(vpp_pkg_dir + pkg)
604 for pkg in vpp_deb_pkgs) + "*.deb"
605 cmd_i = "dpkg -i --force-all {0}".format(deb_pkgs)
606 ret_code, _, err = ssh.exec_command_sudo(cmd_i, timeout=90)
608 raise RuntimeError('Failed to install VPP on host {0}:'
609 '\n{1}'.format(node['host'], err))
611 ssh.exec_command_sudo("dpkg -l | grep vpp")
612 logger.info("VPP installed on node {0}".
613 format(node['host']))
618 def verify_vpp_on_all_duts(nodes):
619 """Verify that VPP is installed on all DUT nodes.
621 :param nodes: Nodes in the topology.
625 logger.debug("Verify VPP on all DUTs")
627 DUTSetup.start_vpp_service_on_all_duts(nodes)
629 for node in nodes.values():
630 if node['type'] == NodeType.DUT:
631 DUTSetup.verify_vpp_on_dut(node)
634 def verify_vpp_on_dut(node):
635 """Verify that VPP is installed on DUT node.
637 :param node: DUT node.
639 :raises RuntimeError: If failed to restart VPP, get VPP version
640 or get VPP interfaces.
643 logger.debug("Verify VPP on node {0}".format(node['host']))
645 DUTSetup.vpp_show_version_verbose(node)
646 DUTSetup.vpp_show_interfaces(node)
649 def get_huge_page_size(node):
650 """Get default size of huge pages in system.
652 :param node: Node in the topology.
654 :returns: Default size of free huge pages in system.
656 :raises RuntimeError: If reading failed for three times.
662 ret_code, stdout, _ = ssh.exec_command_sudo(
663 "grep Hugepagesize /proc/meminfo | awk '{ print $2 }'")
666 huge_size = int(stdout)
668 logger.trace('Reading huge page size information failed')
672 raise RuntimeError('Getting huge page size information failed.')
676 def get_huge_page_free(node, huge_size):
677 """Get number of free huge pages in system.
679 :param node: Node in the topology.
680 :param huge_size: Size of hugepages.
683 :returns: Number of free huge pages in system.
685 :raises RuntimeError: If reading failed for three times.
687 # TODO: add numa aware option
692 ret_code, stdout, _ = ssh.exec_command_sudo(
693 'cat /sys/kernel/mm/hugepages/hugepages-{0}kB/free_hugepages'.
697 huge_free = int(stdout)
699 logger.trace('Reading free huge pages information failed')
703 raise RuntimeError('Getting free huge pages information failed.')
707 def get_huge_page_total(node, huge_size):
708 """Get total number of huge pages in system.
710 :param node: Node in the topology.
711 :param huge_size: Size of hugepages.
715 :returns: Total number of huge pages in system.
717 :raises RuntimeError: If reading failed for three times.
719 # TODO: add numa aware option
724 ret_code, stdout, _ = ssh.exec_command_sudo(
725 'cat /sys/kernel/mm/hugepages/hugepages-{0}kB/nr_hugepages'.
729 huge_total = int(stdout)
731 logger.trace('Reading total huge pages information failed')
735 raise RuntimeError('Getting total huge pages information failed.')
739 def check_huge_page(node, huge_mnt, mem_size, allocate=False):
740 """Check if there is enough HugePages in system. If allocate is set to
741 true, try to allocate more HugePages.
743 :param node: Node in the topology.
744 :param huge_mnt: HugePage mount point.
745 :param mem_size: Requested memory in MB.
746 :param allocate: Whether to allocate more memory if not enough.
752 :raises RuntimeError: Mounting hugetlbfs failed or not enough HugePages
753 or increasing map count failed.
755 # TODO: split function into smaller parts.
759 # Get huge pages information
760 huge_size = DUTSetup.get_huge_page_size(node)
761 huge_free = DUTSetup.get_huge_page_free(node, huge_size)
762 huge_total = DUTSetup.get_huge_page_total(node, huge_size)
764 # Check if memory reqested is available on host
765 if (mem_size * 1024) > (huge_free * huge_size):
766 # If we want to allocate hugepage dynamically
768 mem_needed = (mem_size * 1024) - (huge_free * huge_size)
769 huge_to_allocate = ((mem_needed / huge_size) * 2) + huge_total
770 max_map_count = huge_to_allocate*4
771 # Increase maximum number of memory map areas a process may have
772 ret_code, _, _ = ssh.exec_command_sudo(
773 'echo "{0}" | sudo tee /proc/sys/vm/max_map_count'.
774 format(max_map_count))
775 if int(ret_code) != 0:
776 raise RuntimeError('Increase map count failed on {host}'.
777 format(host=node['host']))
778 # Increase hugepage count
779 ret_code, _, _ = ssh.exec_command_sudo(
780 'echo "{0}" | sudo tee /proc/sys/vm/nr_hugepages'.
781 format(huge_to_allocate))
782 if int(ret_code) != 0:
783 raise RuntimeError('Mount huge pages failed on {host}'.
784 format(host=node['host']))
785 # If we do not want to allocate dynamicaly end with error
787 raise RuntimeError('Not enough free huge pages: {0}, {1} MB'.
788 format(huge_free, huge_free * huge_size))
789 # Check if huge pages mount point exist
791 ret_code, stdout, _ = ssh.exec_command('cat /proc/mounts')
792 if int(ret_code) == 0:
793 for line in stdout.splitlines():
794 # Try to find something like:
795 # none /mnt/huge hugetlbfs rw,relatime,pagesize=2048k 0 0
797 if mount[2] == 'hugetlbfs' and mount[1] == huge_mnt:
800 # If huge page mount point not exist create one
802 ret_code, _, _ = ssh.exec_command_sudo(
803 'mkdir -p {mnt}'.format(mnt=huge_mnt))
804 if int(ret_code) != 0:
805 raise RuntimeError('Create mount dir failed on {host}'.
806 format(host=node['host']))
807 ret_code, _, _ = ssh.exec_command_sudo(
808 'mount -t hugetlbfs -o pagesize=2048k none {mnt}'.
809 format(mnt=huge_mnt))
810 if int(ret_code) != 0:
811 raise RuntimeError('Mount huge pages failed on {host}'.
812 format(host=node['host']))