-# Copyright (c) 2020 Cisco and/or its affiliates.
+# Copyright (c) 2023 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
from time import sleep
from robot.api import logger
-from resources.libraries.python.Constants import Constants
-from resources.libraries.python.ssh import SSH, exec_cmd, exec_cmd_no_error
+from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
from resources.libraries.python.topology import NodeType, Topology
:type node: dict
:type service: str
"""
- command = u"echo $(< /tmp/*supervisor*.log)"\
- if DUTSetup.running_in_container(node) \
- else f"journalctl --no-pager _SYSTEMD_INVOCATION_ID=$(systemctl " \
+ if DUTSetup.running_in_container(node):
+ return
+ command = (
+ f"journalctl --no-pager _SYSTEMD_INVOCATION_ID=$(systemctl "
f"show -p InvocationID --value {service})"
-
+ )
message = f"Node {node[u'host']} failed to get logs from unit {service}"
exec_cmd_no_error(
:type node: dict
:type service: str
"""
- command = f"supervisorctl restart {service}" \
- if DUTSetup.running_in_container(node) \
- else f"service {service} restart"
+ if DUTSetup.running_in_container(node):
+ command = f"supervisorctl restart {service}"
+ else:
+ command = f"systemctl restart {service}"
message = f"Node {node[u'host']} failed to restart service {service}"
exec_cmd_no_error(
:type node: dict
:type service: str
"""
- # TODO: change command to start once all parent function updated.
- command = f"supervisorctl restart {service}" \
- if DUTSetup.running_in_container(node) \
- else f"service {service} restart"
+ if DUTSetup.running_in_container(node):
+ command = f"supervisorctl restart {service}"
+ else:
+ command = f"systemctl restart {service}"
message = f"Node {node[u'host']} failed to start service {service}"
exec_cmd_no_error(
"""
DUTSetup.get_service_logs(node, service)
- command = f"supervisorctl stop {service}" \
- if DUTSetup.running_in_container(node) \
- else f"service {service} stop"
+ if DUTSetup.running_in_container(node):
+ command = f"supervisorctl stop {service}"
+ else:
+ command = f"systemctl stop {service}"
message = f"Node {node[u'host']} failed to stop service {service}"
exec_cmd_no_error(
else:
shell_cmd = f"ip netns exec {namespace} sh -c"
- pgrep_cmd = f"{shell_cmd} \'pgrep {program}\'"
- ret_code, _, _ = exec_cmd(node, pgrep_cmd, timeout=cmd_timeout,
- sudo=True)
- if ret_code == 0:
+ pgrep_cmd = f"{shell_cmd} \'pgrep -c {program}\'"
+ _, stdout, _ = exec_cmd(node, pgrep_cmd, timeout=cmd_timeout,
+ sudo=True)
+ if int(stdout) == 0:
logger.trace(f"{program} is not running on {host}")
return
- ret_code, _, _ = exec_cmd(node, f"{shell_cmd} \'pkill {program}\'",
- timeout=cmd_timeout, sudo=True)
+ exec_cmd(node, f"{shell_cmd} \'pkill {program}\'",
+ timeout=cmd_timeout, sudo=True)
for attempt in range(5):
- ret_code, _, _ = exec_cmd(node, pgrep_cmd, timeout=cmd_timeout,
- sudo=True)
- if ret_code != 0:
+ _, stdout, _ = exec_cmd(node, pgrep_cmd, timeout=cmd_timeout,
+ sudo=True)
+ if int(stdout) == 0:
logger.trace(f"Attempt {attempt}: {program} is dead on {host}")
return
sleep(1)
logger.trace(f"SIGKILLing {program} on {host}")
- ret_code, _, _ = exec_cmd(node, f"{shell_cmd} \'pkill -9 {program}\'",
- timeout=cmd_timeout, sudo=True)
+ exec_cmd(node, f"{shell_cmd} \'pkill -9 {program}\'",
+ timeout=cmd_timeout, sudo=True)
@staticmethod
def verify_program_installed(node, program):
exec_cmd_no_error(node, cmd, message=f"{program} is not installed")
@staticmethod
- def get_pid(node, process):
+ def get_pid(node, process, retries=3):
"""Get PID of running process.
:param node: DUT node.
:param process: process name.
+ :param retries: How many times to retry on failure.
:type node: dict
:type process: str
+ :type retries: int
:returns: PID
:rtype: int
:raises RuntimeError: If it is not possible to get the PID.
"""
- ssh = SSH()
- ssh.connect(node)
-
- retval = None
- for i in range(3):
- logger.trace(f"Try {i}: Get {process} PID")
- ret_code, stdout, stderr = ssh.exec_command(f"pidof {process}")
-
- if int(ret_code):
- raise RuntimeError(
- f"Not possible to get PID of {process} process on node: "
- f"{node[u'host']}\n {stdout + stderr}"
- )
-
- pid_list = stdout.split()
- if len(pid_list) == 1:
- return [int(stdout)]
- if not pid_list:
- logger.debug(f"No {process} PID found on node {node[u'host']}")
- continue
- logger.debug(f"More than one {process} PID found " \
- f"on node {node[u'host']}")
- retval = [int(pid) for pid in pid_list]
-
- return retval
+ cmd = f"pidof {process}"
+ stdout, _ = exec_cmd_no_error(
+ node, cmd, retries=retries,
+ message=f"No {process} PID found on node {node[u'host']}")
+ pid_list = stdout.split()
+ return [int(pid) for pid in pid_list]
@staticmethod
def get_vpp_pids(nodes):
pids[node[u"host"]] = DUTSetup.get_pid(node, u"vpp")
return pids
- @staticmethod
- def crypto_device_verify(node, crypto_type, numvfs, force_init=False):
- """Verify if Crypto QAT device virtual functions are initialized on all
- DUTs. If parameter force initialization is set to True, then try to
- initialize or remove VFs on QAT.
-
- :param node: DUT node.
- :crypto_type: Crypto device type - HW_DH895xcc or HW_C3xxx.
- :param numvfs: Number of VFs to initialize, 0 - disable the VFs.
- :param force_init: If True then try to initialize to specific value.
- :type node: dict
- :type crypto_type: string
- :type numvfs: int
- :type force_init: bool
- :returns: nothing
- :raises RuntimeError: If QAT VFs are not created and force init is set
- to False.
- """
- pci_addr = Topology.get_cryptodev(node)
- sriov_numvfs = DUTSetup.get_sriov_numvfs(node, pci_addr)
-
- if sriov_numvfs != numvfs:
- if force_init:
- # QAT is not initialized and we want to initialize with numvfs
- DUTSetup.crypto_device_init(node, crypto_type, numvfs)
- else:
- raise RuntimeError(
- f"QAT device failed to create VFs on {node[u'host']}"
- )
-
- @staticmethod
- def crypto_device_init(node, crypto_type, numvfs):
- """Init Crypto QAT device virtual functions on DUT.
-
- :param node: DUT node.
- :crypto_type: Crypto device type - HW_DH895xcc or HW_C3xxx.
- :param numvfs: Number of VFs to initialize, 0 - disable the VFs.
- :type node: dict
- :type crypto_type: string
- :type numvfs: int
- :returns: nothing
- :raises RuntimeError: If failed to stop VPP or QAT failed to initialize.
- """
- if crypto_type == u"HW_DH895xcc":
- kernel_mod = u"qat_dh895xcc"
- kernel_drv = u"dh895xcc"
- elif crypto_type == u"HW_C3xxx":
- kernel_mod = u"qat_c3xxx"
- kernel_drv = u"c3xxx"
- else:
- raise RuntimeError(
- f"Unsupported crypto device type on {node[u'host']}"
- )
-
- pci_addr = Topology.get_cryptodev(node)
-
- # QAT device must be re-bound to kernel driver before initialization.
- DUTSetup.verify_kernel_module(node, kernel_mod, force_load=True)
-
- # Stop VPP to prevent deadlock.
- DUTSetup.stop_service(node, Constants.VPP_UNIT)
-
- current_driver = DUTSetup.get_pci_dev_driver(
- node, pci_addr.replace(u":", r"\:")
- )
- if current_driver is not None:
- DUTSetup.pci_driver_unbind(node, pci_addr)
-
- # Bind to kernel driver.
- DUTSetup.pci_driver_bind(node, pci_addr, kernel_drv)
-
- # Initialize QAT VFs.
- if numvfs > 0:
- DUTSetup.set_sriov_numvfs(node, pci_addr, numvfs)
-
@staticmethod
def get_virtfn_pci_addr(node, pf_pci_addr, vf_id):
"""Get PCI address of Virtual Function.
return sriov_numvfs
@staticmethod
- def set_sriov_numvfs(node, pf_pci_addr, numvfs=0):
+ def set_sriov_numvfs(node, pf_pci_addr, path="devices", numvfs=0):
"""Init or reset SR-IOV virtual functions by setting its number on PCI
device on DUT. Setting to zero removes all VFs.
:param node: DUT node.
:param pf_pci_addr: Physical Function PCI device address.
+ :param path: Either device or driver.
:param numvfs: Number of VFs to initialize, 0 - removes the VFs.
:type node: dict
:type pf_pci_addr: str
+ :type path: str
:type numvfs: int
:raises RuntimeError: Failed to create VFs on PCI.
"""
+ cmd = f"test -f /sys/bus/pci/{path}/{pf_pci_addr}/sriov_numvfs"
+ sriov_unsupported, _, _ = exec_cmd(node, cmd)
+ # if sriov_numvfs doesn't exist, then sriov_unsupported != 0
+ if int(sriov_unsupported):
+ if numvfs == 0:
+ # sriov is not supported and we want 0 VFs
+ # no need to do anything
+ return
+
+ raise RuntimeError(
+ f"Can't configure {numvfs} VFs on {pf_pci_addr} device "
+ f"on {node[u'host']} since it doesn't support SR-IOV."
+ )
+
pci = pf_pci_addr.replace(u":", r"\:")
command = f"sh -c \"echo {numvfs} | " \
- f"tee /sys/bus/pci/devices/{pci}/sriov_numvfs\""
+ f"tee /sys/bus/pci/{path}/{pci}/sriov_numvfs\""
message = f"Failed to create {numvfs} VFs on {pf_pci_addr} device " \
f"on {node[u'host']}"
node, command, timeout=120, sudo=True, message=message
)
+ @staticmethod
+ def unbind_pci_devices_from_other_driver(node, driver, *pci_addrs):
+ """Unbind PCI devices from driver other than input driver on node.
+
+ :param node: DUT node.
+ :param driver: Driver to not unbind from. If None or empty string,
+ will attempt to unbind from the current driver.
+ :param pci_addrs: PCI device addresses.
+ :type node: dict
+ :type driver: str
+ :type pci_addrs: list
+ """
+ for pci_addr in pci_addrs:
+ cur_driver = DUTSetup.get_pci_dev_driver(node, pci_addr)
+ if not cur_driver:
+ return
+ if not driver or cur_driver != driver:
+ DUTSetup.pci_driver_unbind(node, pci_addr)
+
@staticmethod
def pci_driver_bind(node, pci_addr, driver):
"""Bind PCI device to driver on node.
def get_pci_dev_driver(node, pci_addr):
"""Get current PCI device driver on node.
- .. note::
- # lspci -vmmks 0000:00:05.0
- Slot: 00:05.0
- Class: Ethernet controller
- Vendor: Red Hat, Inc
- Device: Virtio network device
- SVendor: Red Hat, Inc
- SDevice: Device 0001
- PhySlot: 5
- Driver: virtio-pci
-
:param node: DUT node.
:param pci_addr: PCI device address.
:type node: dict
:type pci_addr: str
:returns: Driver or None
- :raises RuntimeError: If PCI rescan or lspci command execution failed.
:raises RuntimeError: If it is not possible to get the interface driver
information from the node.
"""
- ssh = SSH()
- ssh.connect(node)
-
- for i in range(3):
- logger.trace(f"Try number {i}: Get PCI device driver")
-
- cmd = f"lspci -vmmks {pci_addr}"
- ret_code, stdout, _ = ssh.exec_command(cmd)
- if int(ret_code):
- raise RuntimeError(f"'{cmd}' failed on '{node[u'host']}'")
-
- for line in stdout.splitlines():
- if not line:
- continue
- name = None
- value = None
- try:
- name, value = line.split(u"\t", 1)
- except ValueError:
- if name == u"Driver:":
- return None
- if name == u"Driver:":
- return value
-
- if i < 2:
- logger.trace(
- f"Driver for PCI device {pci_addr} not found, "
- f"executing pci rescan and retrying"
- )
- cmd = u"sh -c \"echo 1 > /sys/bus/pci/rescan\""
- ret_code, _, _ = ssh.exec_command_sudo(cmd)
- if int(ret_code) != 0:
- raise RuntimeError(f"'{cmd}' failed on '{node[u'host']}'")
-
- return None
+ driver_path = f"/sys/bus/pci/devices/{pci_addr}/driver"
+ cmd = f"test -d {driver_path}"
+ ret_code, ret_val, _ = exec_cmd(node, cmd)
+ if int(ret_code):
+ # the directory doesn't exist which means the device is not bound
+ # to any driver
+ return None
+ cmd = f"basename $(readlink -f {driver_path})"
+ ret_val, _ = exec_cmd_no_error(node, cmd)
+ return ret_val.strip()
@staticmethod
def verify_kernel_module(node, module, force_load=False):
exec_cmd_no_error(node, command, timeout=30, sudo=True, message=message)
- @staticmethod
- def install_vpp_on_all_duts(nodes, vpp_pkg_dir):
- """Install VPP on all DUT nodes. Start the VPP service in case of
- systemd is not available or does not support autostart.
-
- :param nodes: Nodes in the topology.
- :param vpp_pkg_dir: Path to directory where VPP packages are stored.
- :type nodes: dict
- :type vpp_pkg_dir: str
- :raises RuntimeError: If failed to remove or install VPP.
- """
- for node in nodes.values():
- message = f"Failed to install VPP on host {node[u'host']}!"
- if node[u"type"] == NodeType.DUT:
- command = u"ln -s /dev/null /etc/sysctl.d/80-vpp.conf || true"
- exec_cmd_no_error(node, command, sudo=True)
-
- command = u". /etc/lsb-release; echo \"${DISTRIB_ID}\""
- stdout, _ = exec_cmd_no_error(node, command)
-
- if stdout.strip() == u"Ubuntu":
- exec_cmd_no_error(
- node, u"apt-get purge -y '*vpp*' || true",
- timeout=120, sudo=True
- )
- # workaround to avoid installation of vpp-api-python
- exec_cmd_no_error(
- node, u"rm -f {vpp_pkg_dir}vpp-api-python.deb",
- timeout=120, sudo=True
- )
- exec_cmd_no_error(
- node, f"dpkg -i --force-all {vpp_pkg_dir}*.deb",
- timeout=120, sudo=True, message=message
- )
- exec_cmd_no_error(node, u"dpkg -l | grep vpp", sudo=True)
- if DUTSetup.running_in_container(node):
- DUTSetup.restart_service(node, Constants.VPP_UNIT)
- else:
- exec_cmd_no_error(
- node, u"yum -y remove '*vpp*' || true",
- timeout=120, sudo=True
- )
- # workaround to avoid installation of vpp-api-python
- exec_cmd_no_error(
- node, u"rm -f {vpp_pkg_dir}vpp-api-python.rpm",
- timeout=120, sudo=True
- )
- exec_cmd_no_error(
- node, f"rpm -ivh {vpp_pkg_dir}*.rpm",
- timeout=120, sudo=True, message=message
- )
- exec_cmd_no_error(node, u"rpm -qai '*vpp*'", sudo=True)
- DUTSetup.restart_service(node, Constants.VPP_UNIT)
-
@staticmethod
def running_in_container(node):
"""This method tests if topology node is running inside container.
to detect.
:rtype: bool
"""
- command = u"fgrep docker /proc/1/cgroup"
- message = u"Failed to get cgroup settings."
+ command = "cat /.dockerenv"
try:
- exec_cmd_no_error(
- node, command, timeout=30, sudo=False, message=message
- )
+ exec_cmd_no_error(node, command, timeout=30)
except RuntimeError:
return False
return True
@staticmethod
- def get_docker_mergeddir(node, uuid):
+ def get_docker_mergeddir(node, uuid=None):
"""Get Docker overlay for MergedDir diff.
:param node: DUT node.
:rtype: str
:raises RuntimeError: If getting output failed.
"""
- command = f"docker inspect " \
+ if not uuid:
+ command = 'fgrep "hostname" /proc/self/mountinfo | cut -f 4 -d" "'
+ message = "Failed to get UUID!"
+ stdout, _ = exec_cmd_no_error(node, command, message=message)
+ uuid = stdout.split(sep="/")[-2]
+ command = (
+ f"docker inspect "
f"--format='{{{{.GraphDriver.Data.MergedDir}}}}' {uuid}"
+ )
message = f"Failed to get directory of {uuid} on host {node[u'host']}"
stdout, _ = exec_cmd_no_error(node, command, sudo=True, message=message)
Code Review / csit.git / blobdiff