+ for i in range(3):
+ logger.trace('Try number {0}: Get PCI device driver'.format(i))
+
+ cmd = 'lspci -vmmks {0}'.format(pci_addr)
+ ret_code, stdout, _ = ssh.exec_command(cmd)
+ if int(ret_code):
+ raise RuntimeError("'{0}' failed on '{1}'"
+ .format(cmd, node['host']))
+
+ for line in stdout.splitlines():
+ if not line:
+ continue
+ name = None
+ value = None
+ try:
+ name, value = line.split("\t", 1)
+ except ValueError:
+ if name == "Driver:":
+ return None
+ if name == 'Driver:':
+ return value
+
+ if i < 2:
+ logger.trace('Driver for PCI device {} not found, executing '
+ 'pci rescan and retrying'.format(pci_addr))
+ cmd = 'sh -c "echo 1 > /sys/bus/pci/rescan"'
+ ret_code, _, _ = ssh.exec_command_sudo(cmd)
+ if int(ret_code) != 0:
+ raise RuntimeError("'{0}' failed on '{1}'"
+ .format(cmd, node['host']))
+
+ return None
+
+ @staticmethod
+ def verify_kernel_module(node, module, force_load=False):
+ """Verify if kernel module is loaded on node. If parameter force
+ load is set to True, then try to load the modules.
+
+ :param node: Node.
+ :param module: Module to verify.
+ :param force_load: If True then try to load module.
+ :type node: dict
+ :type module: str
+ :type force_load: bool
+ :raises RuntimeError: If module is not loaded or failed to load.
+ """
+ command = 'grep -w {module} /proc/modules'.format(module=module)
+ message = 'Kernel module {module} is not loaded on host {host}'.\
+ format(module=module, host=node['host'])
+
+ try:
+ exec_cmd_no_error(node, command, timeout=30, sudo=False,
+ message=message)
+ except RuntimeError:
+ if force_load:
+ # Module is not loaded and we want to load it
+ DUTSetup.load_kernel_module(node, module)
+ else:
+ raise
+
+ @staticmethod
+ def verify_kernel_module_on_all_duts(nodes, module, force_load=False):
+ """Verify if kernel module is loaded on all DUTs. If parameter force
+ load is set to True, then try to load the modules.
+
+ :param node: DUT nodes.
+ :param module: Module to verify.
+ :param force_load: If True then try to load module.
+ :type node: dict
+ :type module: str
+ :type force_load: bool
+ """
+ for node in nodes.values():
+ if node['type'] == NodeType.DUT:
+ DUTSetup.verify_kernel_module(node, module, force_load)
+
+ @staticmethod
+ def verify_uio_driver_on_all_duts(nodes):
+ """Verify if uio driver kernel module is loaded on all DUTs. If module
+ is not present it will try to load it.
+
+ :param node: DUT nodes.
+ :type node: dict
+ """
+ for node in nodes.values():
+ if node['type'] == NodeType.DUT:
+ uio_driver = Topology.get_uio_driver(node)
+ DUTSetup.verify_kernel_module(node, uio_driver, force_load=True)
+
+ @staticmethod
+ def load_kernel_module(node, module):
+ """Load kernel module on node.
+
+ :param node: DUT node.
+ :param module: Module to load.
+ :type node: dict
+ :type module: str
+ :returns: nothing
+ :raises RuntimeError: If loading failed.
+ """
+ command = 'modprobe {module}'.format(module=module)
+ message = 'Failed to load {module} on host {host}'.\
+ format(module=module, host=node['host'])
+
+ exec_cmd_no_error(node, command, timeout=30, sudo=True, message=message)
+
+ @staticmethod
+ def install_vpp_on_all_duts(nodes, vpp_pkg_dir, vpp_rpm_pkgs, vpp_deb_pkgs):
+ """Install VPP on all DUT nodes.
+
+ :param nodes: Nodes in the topology.
+ :param vpp_pkg_dir: Path to directory where VPP packages are stored.
+ :param vpp_rpm_pkgs: List of VPP rpm packages to be installed.
+ :param vpp_deb_pkgs: List of VPP deb packages to be installed.
+ :type nodes: dict
+ :type vpp_pkg_dir: str
+ :type vpp_rpm_pkgs: list
+ :type vpp_deb_pkgs: list
+ :raises RuntimeError: If failed to remove or install VPP.
+ """
+ for node in nodes.values():
+ if node['type'] == NodeType.DUT:
+ logger.debug("Installing VPP on node {0}".format(node['host']))
+
+ ssh = SSH()
+ ssh.connect(node)
+
+ cmd = "[[ -f /etc/redhat-release ]]"
+ return_code, _, _ = ssh.exec_command(cmd)
+ if not int(return_code):
+ # workaroud - uninstall existing vpp installation until
+ # start-testcase script is updated on all virl servers
+ rpm_pkgs_remove = "vpp*"
+ cmd_u = 'yum -y remove "{0}"'.format(rpm_pkgs_remove)
+ r_rcode, _, r_err = ssh.exec_command_sudo(cmd_u, timeout=90)
+ if int(r_rcode):
+ raise RuntimeError('Failed to remove previous VPP'
+ 'installation on host {0}:\n{1}'
+ .format(node['host'], r_err))
+
+ rpm_pkgs = "*.rpm ".join(str(vpp_pkg_dir + pkg)
+ for pkg in vpp_rpm_pkgs) + "*.rpm"
+ cmd_i = "rpm -ivh {0}".format(rpm_pkgs)
+ ret_code, _, err = ssh.exec_command_sudo(cmd_i, timeout=90)
+ if int(ret_code):
+ raise RuntimeError('Failed to install VPP on host {0}:'
+ '\n{1}'.format(node['host'], err))
+ else:
+ ssh.exec_command_sudo("rpm -qai vpp*")
+ logger.info("VPP installed on node {0}".
+ format(node['host']))
+ else:
+ # workaroud - uninstall existing vpp installation until
+ # start-testcase script is updated on all virl servers
+ deb_pkgs_remove = "vpp*"
+ cmd_u = 'apt-get purge -y "{0}"'.format(deb_pkgs_remove)
+ r_rcode, _, r_err = ssh.exec_command_sudo(cmd_u, timeout=90)
+ if int(r_rcode):
+ raise RuntimeError('Failed to remove previous VPP'
+ 'installation on host {0}:\n{1}'
+ .format(node['host'], r_err))
+ deb_pkgs = "*.deb ".join(str(vpp_pkg_dir + pkg)
+ for pkg in vpp_deb_pkgs) + "*.deb"
+ cmd_i = "dpkg -i --force-all {0}".format(deb_pkgs)
+ ret_code, _, err = ssh.exec_command_sudo(cmd_i, timeout=90)
+ if int(ret_code):
+ raise RuntimeError('Failed to install VPP on host {0}:'
+ '\n{1}'.format(node['host'], err))
+ else:
+ ssh.exec_command_sudo("dpkg -l | grep vpp")
+ logger.info("VPP installed on node {0}".
+ format(node['host']))
+
+ ssh.disconnect(node)
+
+ @staticmethod
+ def running_in_container(node):
+ """This method tests if topology node is running inside container.
+
+ :param node: Topology node.
+ :type node: dict
+ :returns: True if running in docker container, false if not or failed
+ to detect.
+ :rtype: bool
+ """
+ command = "fgrep docker /proc/1/cgroup"
+ message = 'Failed to get cgroup settings.'
+ try:
+ exec_cmd_no_error(node, command, timeout=30, sudo=False,
+ message=message)
+ except RuntimeError:
+ return False
+ return True
+
+ @staticmethod
+ def get_huge_page_size(node):
+ """Get default size of huge pages in system.
+
+ :param node: Node in the topology.
+ :type node: dict
+ :returns: Default size of free huge pages in system.
+ :rtype: int
+ :raises RuntimeError: If reading failed for three times.
+ """
+ ssh = SSH()
+ ssh.connect(node)
+
+ for _ in range(3):
+ ret_code, stdout, _ = ssh.exec_command_sudo(
+ "grep Hugepagesize /proc/meminfo | awk '{ print $2 }'")
+ if ret_code == 0:
+ try:
+ huge_size = int(stdout)
+ except ValueError:
+ logger.trace('Reading huge page size information failed')
+ else:
+ break
+ else:
+ raise RuntimeError('Getting huge page size information failed.')
+ return huge_size
+
+ @staticmethod
+ def get_huge_page_free(node, huge_size):
+ """Get number of free huge pages in system.
+
+ :param node: Node in the topology.
+ :param huge_size: Size of hugepages.
+ :type node: dict
+ :type huge_size: int
+ :returns: Number of free huge pages in system.
+ :rtype: int
+ :raises RuntimeError: If reading failed for three times.
+ """
+ # TODO: add numa aware option
+ ssh = SSH()
+ ssh.connect(node)
+
+ for _ in range(3):
+ ret_code, stdout, _ = ssh.exec_command_sudo(
+ 'cat /sys/kernel/mm/hugepages/hugepages-{0}kB/free_hugepages'.
+ format(huge_size))
+ if ret_code == 0:
+ try:
+ huge_free = int(stdout)
+ except ValueError:
+ logger.trace('Reading free huge pages information failed')
+ else:
+ break
+ else:
+ raise RuntimeError('Getting free huge pages information failed.')
+ return huge_free
+
+ @staticmethod
+ def get_huge_page_total(node, huge_size):
+ """Get total number of huge pages in system.
+
+ :param node: Node in the topology.
+ :param huge_size: Size of hugepages.
+ :type node: dict
+ :type huge_size: int
+
+ :returns: Total number of huge pages in system.
+ :rtype: int
+ :raises RuntimeError: If reading failed for three times.
+ """
+ # TODO: add numa aware option
+ ssh = SSH()
+ ssh.connect(node)
+
+ for _ in range(3):
+ ret_code, stdout, _ = ssh.exec_command_sudo(
+ 'cat /sys/kernel/mm/hugepages/hugepages-{0}kB/nr_hugepages'.
+ format(huge_size))
+ if ret_code == 0:
+ try:
+ huge_total = int(stdout)
+ except ValueError:
+ logger.trace('Reading total huge pages information failed')
+ else:
+ break
+ else:
+ raise RuntimeError('Getting total huge pages information failed.')
+ return huge_total
+
+ @staticmethod
+ def check_huge_page(node, huge_mnt, mem_size, allocate=False):
+ """Check if there is enough HugePages in system. If allocate is set to
+ true, try to allocate more HugePages.
+
+ :param node: Node in the topology.
+ :param huge_mnt: HugePage mount point.
+ :param mem_size: Requested memory in MB.
+ :param allocate: Whether to allocate more memory if not enough.
+ :type node: dict
+ :type huge_mnt: str
+ :type mem_size: str
+ :type allocate: bool
+
+ :raises RuntimeError: Mounting hugetlbfs failed or not enough HugePages
+ or increasing map count failed.
+ """
+ # TODO: split function into smaller parts.
+ ssh = SSH()
+ ssh.connect(node)
+
+ # Get huge pages information
+ huge_size = DUTSetup.get_huge_page_size(node)
+ huge_free = DUTSetup.get_huge_page_free(node, huge_size)
+ huge_total = DUTSetup.get_huge_page_total(node, huge_size)
+
+ # Check if memory reqested is available on host
+ if (mem_size * 1024) > (huge_free * huge_size):
+ # If we want to allocate hugepage dynamically
+ if allocate:
+ mem_needed = (mem_size * 1024) - (huge_free * huge_size)
+ huge_to_allocate = ((mem_needed / huge_size) * 2) + huge_total
+ max_map_count = huge_to_allocate*4
+ # Increase maximum number of memory map areas a process may have
+ ret_code, _, _ = ssh.exec_command_sudo(
+ 'echo "{0}" | sudo tee /proc/sys/vm/max_map_count'.
+ format(max_map_count))
+ if int(ret_code) != 0:
+ raise RuntimeError('Increase map count failed on {host}'.
+ format(host=node['host']))
+ # Increase hugepage count
+ ret_code, _, _ = ssh.exec_command_sudo(
+ 'echo "{0}" | sudo tee /proc/sys/vm/nr_hugepages'.
+ format(huge_to_allocate))
+ if int(ret_code) != 0:
+ raise RuntimeError('Mount huge pages failed on {host}'.
+ format(host=node['host']))
+ # If we do not want to allocate dynamicaly end with error
+ else:
+ raise RuntimeError('Not enough free huge pages: {0}, {1} MB'.
+ format(huge_free, huge_free * huge_size))
+ # Check if huge pages mount point exist
+ has_huge_mnt = False
+ ret_code, stdout, _ = ssh.exec_command('cat /proc/mounts')
+ if int(ret_code) == 0:
+ for line in stdout.splitlines():
+ # Try to find something like:
+ # none /mnt/huge hugetlbfs rw,relatime,pagesize=2048k 0 0
+ mount = line.split()
+ if mount[2] == 'hugetlbfs' and mount[1] == huge_mnt:
+ has_huge_mnt = True
+ break
+ # If huge page mount point not exist create one
+ if not has_huge_mnt:
+ ret_code, _, _ = ssh.exec_command_sudo(
+ 'mkdir -p {mnt}'.format(mnt=huge_mnt))
+ if int(ret_code) != 0:
+ raise RuntimeError('Create mount dir failed on {host}'.
+ format(host=node['host']))
+ ret_code, _, _ = ssh.exec_command_sudo(
+ 'mount -t hugetlbfs -o pagesize=2048k none {mnt}'.
+ format(mnt=huge_mnt))
+ if int(ret_code) != 0:
+ raise RuntimeError('Mount huge pages failed on {host}'.
+ format(host=node['host']))