X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Flibraries%2Fpython%2FDUTSetup.py;h=632e9ea073923e814d44b700c8783cd3bbc99066;hp=2939d74c1eb36166ac95ee2050e856ce6e815433;hb=92d4e47bfbca31e10c44dee7f74da4c6fd9e6e4c;hpb=501d216be3b4cf8ec0a4eea82acd5855bad4a713 diff --git a/resources/libraries/python/DUTSetup.py b/resources/libraries/python/DUTSetup.py index 2939d74c1e..632e9ea073 100644 --- a/resources/libraries/python/DUTSetup.py +++ b/resources/libraries/python/DUTSetup.py @@ -13,8 +13,6 @@ """DUT setup library.""" -import os - from robot.api import logger from resources.libraries.python.topology import NodeType, Topology @@ -26,6 +24,63 @@ from resources.libraries.python.VPPUtil import VPPUtil class DUTSetup(object): """Contains methods for setting up DUTs.""" + + @staticmethod + def get_service_logs(node, service): + """Get specific service unit logs by journalctl from node. + + :param node: Node in the topology. + :param service: Service unit name. + :type node: dict + :type service: str + """ + ssh = SSH() + ssh.connect(node) + ret_code, _, _ = \ + ssh.exec_command_sudo('journalctl --no-pager --unit={name} ' + '--since="$(echo `systemctl show -p ' + 'ActiveEnterTimestamp {name}` | ' + 'awk \'{{print $2 $3}}\')"'. + format(name=service)) + if int(ret_code) != 0: + raise RuntimeError('DUT {host} failed to get logs from unit {name}'. + format(host=node['host'], name=service)) + + @staticmethod + def get_service_logs_on_all_duts(nodes, service): + """Get specific service unit logs by journalctl from all DUTs. + + :param nodes: Nodes in the topology. + :param service: Service unit name. + :type nodes: dict + :type service: str + """ + for node in nodes.values(): + if node['type'] == NodeType.DUT: + DUTSetup.get_service_logs(node, service) + + @staticmethod + def start_service(node, service): + """Start up the named service on node. + + :param node: Node in the topology. + :param service: Service unit name. + :type node: dict + :type service: str + """ + ssh = SSH() + ssh.connect(node) + # We are doing restart. With this we do not care if service + # was running or not. + ret_code, _, _ = \ + ssh.exec_command_sudo('service {name} restart'. + format(name=service), timeout=120) + if int(ret_code) != 0: + raise RuntimeError('DUT {host} failed to start service {name}'. + format(host=node['host'], name=service)) + + DUTSetup.get_service_logs(node, service) + @staticmethod def start_vpp_service_on_all_duts(nodes): """Start up the VPP service on all nodes. @@ -33,15 +88,9 @@ class DUTSetup(object): :param nodes: Nodes in the topology. :type nodes: dict """ - ssh = SSH() for node in nodes.values(): if node['type'] == NodeType.DUT: - ssh.connect(node) - (ret_code, stdout, stderr) = \ - ssh.exec_command_sudo('service vpp restart', timeout=120) - if int(ret_code) != 0: - raise Exception('DUT {0} failed to start VPP service'. - format(node['host'])) + DUTSetup.start_service(node, Constants.VPP_UNIT) @staticmethod def vpp_show_version_verbose(node): @@ -56,8 +105,8 @@ class DUTSetup(object): try: vat.script_should_have_passed() except AssertionError: - raise RuntimeError('Failed to get VPP version on host: {}'. - format(node['host'])) + raise RuntimeError('Failed to get VPP version on host: {name}'. + format(name=node['host'])) @staticmethod def show_vpp_version_on_all_duts(nodes): @@ -83,8 +132,8 @@ class DUTSetup(object): try: vat.script_should_have_passed() except AssertionError: - raise RuntimeError('Failed to get VPP interfaces on host: {}'. - format(node['host'])) + raise RuntimeError('Failed to get VPP interfaces on host: {name}'. + format(name=node['host'])) @staticmethod def vpp_api_trace_save(node): @@ -125,15 +174,13 @@ class DUTSetup(object): ssh = SSH() ssh.connect(node) - (ret_code, stdout, stderr) = \ + ret_code, _, _ = \ ssh.exec_command('sudo -Sn bash {0}/{1}/dut_setup.sh'. format(Constants.REMOTE_FW_DIR, Constants.RESOURCES_LIB_SH), timeout=120) if int(ret_code) != 0: - logger.debug('DUT {0} setup script failed: "{1}"'. - format(node['host'], stdout + stderr)) - raise Exception('DUT test setup script failed at node {}'. - format(node['host'])) + raise RuntimeError('DUT test setup script failed at node {name}'. + format(name=node['host'])) @staticmethod def get_vpp_pid(node): @@ -143,7 +190,7 @@ class DUTSetup(object): :type node: dict :returns: PID :rtype: int - :raises RuntimeError if it is not possible to get the PID. + :raises RuntimeError: If it is not possible to get the PID. """ ssh = SSH() @@ -338,24 +385,31 @@ class DUTSetup(object): def get_pci_dev_driver(node, pci_addr): """Get current PCI device driver on node. + .. note:: + # lspci -vmmks 0000:00:05.0 + Slot: 00:05.0 + Class: Ethernet controller + Vendor: Red Hat, Inc + Device: Virtio network device + SVendor: Red Hat, Inc + SDevice: Device 0001 + PhySlot: 5 + Driver: virtio-pci + :param node: DUT node. :param pci_addr: PCI device address. :type node: dict :type pci_addr: str :returns: Driver or None :raises RuntimeError: If PCI rescan or lspci command execution failed. + :raises RuntimeError: If it is not possible to get the interface driver + information from the node. """ ssh = SSH() ssh.connect(node) for i in range(3): - logger.trace('Try {0}: Get interface driver'.format(i)) - cmd = 'sh -c "echo 1 > /sys/bus/pci/rescan"' - ret_code, _, _ = ssh.exec_command_sudo(cmd) - if int(ret_code) != 0: - raise RuntimeError("'{0}' failed on '{1}'" - .format(cmd, node['host'])) - + logger.trace('Try number {0}: Get PCI device driver'.format(i)) cmd = 'lspci -vmmks {0}'.format(pci_addr) ret_code, stdout, _ = ssh.exec_command(cmd) if int(ret_code) != 0: @@ -374,24 +428,31 @@ class DUTSetup(object): return None if name == 'Driver:': return value - else: - return None + + if i < 2: + logger.trace('Driver for PCI device {} not found, executing ' + 'pci rescan and retrying'.format(pci_addr)) + cmd = 'sh -c "echo 1 > /sys/bus/pci/rescan"' + ret_code, _, _ = ssh.exec_command_sudo(cmd) + if int(ret_code) != 0: + raise RuntimeError("'{0}' failed on '{1}'" + .format(cmd, node['host'])) + + return None @staticmethod def kernel_module_verify(node, module, force_load=False): - """Verify if kernel module is loaded on all DUTs. If parameter force + """Verify if kernel module is loaded on node. If parameter force load is set to True, then try to load the modules. - :param node: DUT node. + :param node: Node. :param module: Module to verify. :param force_load: If True then try to load module. :type node: dict :type module: str :type force_load: bool - :returns: nothing :raises RuntimeError: If module is not loaded or failed to load. """ - ssh = SSH() ssh.connect(node) @@ -406,6 +467,35 @@ class DUTSetup(object): raise RuntimeError('Kernel module {0} is not loaded on host ' '{1}'.format(module, node['host'])) + @staticmethod + def kernel_module_verify_on_all_duts(nodes, module, force_load=False): + """Verify if kernel module is loaded on all DUTs. If parameter force + load is set to True, then try to load the modules. + + :param node: DUT nodes. + :param module: Module to verify. + :param force_load: If True then try to load module. + :type node: dict + :type module: str + :type force_load: bool + """ + for node in nodes.values(): + if node['type'] == NodeType.DUT: + DUTSetup.kernel_module_verify(node, module, force_load) + + @staticmethod + def verify_uio_driver_on_all_duts(nodes): + """Verify if uio driver kernel module is loaded on all DUTs. If module + is not present it will try to load it. + + :param node: DUT nodes. + :type node: dict + """ + for node in nodes.values(): + if node['type'] == NodeType.DUT: + uio_driver = Topology.get_uio_driver(node) + DUTSetup.kernel_module_verify(node, uio_driver, force_load=True) + @staticmethod def kernel_module_load(node, module): """Load kernel module on node. @@ -449,6 +539,7 @@ class DUTSetup(object): vat = VatExecutor() vat.execute_script("enable_dpdk_traces.vat", node, json_out=False) vat.execute_script("enable_vhost_user_traces.vat", node, json_out=False) + vat.execute_script("enable_memif_traces.vat", node, json_out=False) @staticmethod def install_vpp_on_all_duts(nodes, vpp_pkg_dir, vpp_rpm_pkgs, vpp_deb_pkgs): @@ -462,7 +553,7 @@ class DUTSetup(object): :type vpp_pkg_dir: str :type vpp_rpm_pkgs: list :type vpp_deb_pkgs: list - :raises: RuntimeError if failed to remove or install VPP + :raises RuntimeError: If failed to remove or install VPP. """ logger.debug("Installing VPP") @@ -474,7 +565,9 @@ class DUTSetup(object): ssh = SSH() ssh.connect(node) - if os.path.isfile("/etc/redhat-release"): + cmd = "[[ -f /etc/redhat-release ]]" + return_code, _, _ = ssh.exec_command(cmd) + if int(return_code) == 0: # workaroud - uninstall existing vpp installation until # start-testcase script is updated on all virl servers rpm_pkgs_remove = "vpp*" @@ -483,7 +576,7 @@ class DUTSetup(object): if int(r_rcode) != 0: raise RuntimeError('Failed to remove previous VPP' 'installation on host {0}:\n{1}' - .format(node['host']), r_err) + .format(node['host'], r_err)) rpm_pkgs = "*.rpm ".join(str(vpp_pkg_dir + pkg) for pkg in vpp_rpm_pkgs) + "*.rpm" @@ -491,7 +584,7 @@ class DUTSetup(object): ret_code, _, err = ssh.exec_command_sudo(cmd_i, timeout=90) if int(ret_code) != 0: raise RuntimeError('Failed to install VPP on host {0}:' - '\n{1}'.format(node['host']), err) + '\n{1}'.format(node['host'], err)) else: ssh.exec_command_sudo("rpm -qai vpp*") logger.info("VPP installed on node {0}". @@ -505,14 +598,14 @@ class DUTSetup(object): if int(r_rcode) != 0: raise RuntimeError('Failed to remove previous VPP' 'installation on host {0}:\n{1}' - .format(node['host']), r_err) + .format(node['host'], r_err)) deb_pkgs = "*.deb ".join(str(vpp_pkg_dir + pkg) for pkg in vpp_deb_pkgs) + "*.deb" cmd_i = "dpkg -i --force-all {0}".format(deb_pkgs) ret_code, _, err = ssh.exec_command_sudo(cmd_i, timeout=90) if int(ret_code) != 0: raise RuntimeError('Failed to install VPP on host {0}:' - '\n{1}'.format(node['host']), err) + '\n{1}'.format(node['host'], err)) else: ssh.exec_command_sudo("dpkg -l | grep vpp") logger.info("VPP installed on node {0}". @@ -542,11 +635,177 @@ class DUTSetup(object): :param node: DUT node. :type node: dict - :raises: RuntimeError if failed to restart VPP, get VPP version or - get VPP interfaces + :raises RuntimeError: If failed to restart VPP, get VPP version + or get VPP interfaces. """ logger.debug("Verify VPP on node {0}".format(node['host'])) DUTSetup.vpp_show_version_verbose(node) DUTSetup.vpp_show_interfaces(node) + + @staticmethod + def get_huge_page_size(node): + """Get default size of huge pages in system. + + :param node: Node in the topology. + :type node: dict + :returns: Default size of free huge pages in system. + :rtype: int + :raises RuntimeError: If reading failed for three times. + """ + ssh = SSH() + ssh.connect(node) + + for _ in range(3): + ret_code, stdout, _ = ssh.exec_command_sudo( + "grep Hugepagesize /proc/meminfo | awk '{ print $2 }'") + if ret_code == 0: + try: + huge_size = int(stdout) + except ValueError: + logger.trace('Reading huge page size information failed') + else: + break + else: + raise RuntimeError('Getting huge page size information failed.') + return huge_size + + @staticmethod + def get_huge_page_free(node, huge_size): + """Get number of free huge pages in system. + + :param node: Node in the topology. + :param huge_size: Size of hugepages. + :type node: dict + :type huge_size: int + :returns: Number of free huge pages in system. + :rtype: int + :raises RuntimeError: If reading failed for three times. + """ + # TODO: add numa aware option + ssh = SSH() + ssh.connect(node) + + for _ in range(3): + ret_code, stdout, _ = ssh.exec_command_sudo( + 'cat /sys/kernel/mm/hugepages/hugepages-{0}kB/free_hugepages'. + format(huge_size)) + if ret_code == 0: + try: + huge_free = int(stdout) + except ValueError: + logger.trace('Reading free huge pages information failed') + else: + break + else: + raise RuntimeError('Getting free huge pages information failed.') + return huge_free + + @staticmethod + def get_huge_page_total(node, huge_size): + """Get total number of huge pages in system. + + :param node: Node in the topology. + :param huge_size: Size of hugepages. + :type node: dict + :type huge_size: int + + :returns: Total number of huge pages in system. + :rtype: int + :raises RuntimeError: If reading failed for three times. + """ + # TODO: add numa aware option + ssh = SSH() + ssh.connect(node) + + for _ in range(3): + ret_code, stdout, _ = ssh.exec_command_sudo( + 'cat /sys/kernel/mm/hugepages/hugepages-{0}kB/nr_hugepages'. + format(huge_size)) + if ret_code == 0: + try: + huge_total = int(stdout) + except ValueError: + logger.trace('Reading total huge pages information failed') + else: + break + else: + raise RuntimeError('Getting total huge pages information failed.') + return huge_total + + @staticmethod + def check_huge_page(node, huge_mnt, mem_size, allocate=False): + """Check if there is enough HugePages in system. If allocate is set to + true, try to allocate more HugePages. + + :param node: Node in the topology. + :param huge_mnt: HugePage mount point. + :param mem_size: Requested memory in MB. + :param allocate: Whether to allocate more memory if not enough. + :type node: dict + :type huge_mnt: str + :type mem_size: str + :type allocate: bool + + :raises RuntimeError: Mounting hugetlbfs failed or not enough HugePages + or increasing map count failed. + """ + # TODO: split function into smaller parts. + ssh = SSH() + ssh.connect(node) + + # Get huge pages information + huge_size = DUTSetup.get_huge_page_size(node) + huge_free = DUTSetup.get_huge_page_free(node, huge_size) + huge_total = DUTSetup.get_huge_page_total(node, huge_size) + + # Check if memory reqested is available on host + if (mem_size * 1024) > (huge_free * huge_size): + # If we want to allocate hugepage dynamically + if allocate: + mem_needed = (mem_size * 1024) - (huge_free * huge_size) + huge_to_allocate = ((mem_needed / huge_size) * 2) + huge_total + max_map_count = huge_to_allocate*4 + # Increase maximum number of memory map areas a process may have + ret_code, _, _ = ssh.exec_command_sudo( + 'echo "{0}" | sudo tee /proc/sys/vm/max_map_count'. + format(max_map_count)) + if int(ret_code) != 0: + raise RuntimeError('Increase map count failed on {host}'. + format(host=node['host'])) + # Increase hugepage count + ret_code, _, _ = ssh.exec_command_sudo( + 'echo "{0}" | sudo tee /proc/sys/vm/nr_hugepages'. + format(huge_to_allocate)) + if int(ret_code) != 0: + raise RuntimeError('Mount huge pages failed on {host}'. + format(host=node['host'])) + # If we do not want to allocate dynamicaly end with error + else: + raise RuntimeError('Not enough free huge pages: {0}, {1} MB'. + format(huge_free, huge_free * huge_size)) + # Check if huge pages mount point exist + has_huge_mnt = False + ret_code, stdout, _ = ssh.exec_command('cat /proc/mounts') + if int(ret_code) == 0: + for line in stdout.splitlines(): + # Try to find something like: + # none /mnt/huge hugetlbfs rw,relatime,pagesize=2048k 0 0 + mount = line.split() + if mount[2] == 'hugetlbfs' and mount[1] == huge_mnt: + has_huge_mnt = True + break + # If huge page mount point not exist create one + if not has_huge_mnt: + ret_code, _, _ = ssh.exec_command_sudo( + 'mkdir -p {mnt}'.format(mnt=huge_mnt)) + if int(ret_code) != 0: + raise RuntimeError('Create mount dir failed on {host}'. + format(host=node['host'])) + ret_code, _, _ = ssh.exec_command_sudo( + 'mount -t hugetlbfs -o pagesize=2048k none {mnt}'. + format(mnt=huge_mnt)) + if int(ret_code) != 0: + raise RuntimeError('Mount huge pages failed on {host}'. + format(host=node['host']))