+ if node[u"type"] == NodeType.DUT:
+ DUTSetup.restart_service(node, service)
+
+ @staticmethod
+ def start_service(node, service):
+ """Start up the named service on node.
+
+ :param node: Node in the topology.
+ :param service: Service unit name.
+ :type node: dict
+ :type service: str
+ """
+ # TODO: change command to start once all parent function updated.
+ command = f"supervisorctl restart {service}" \
+ if DUTSetup.running_in_container(node) \
+ else f"service {service} restart"
+ message = f"Node {node[u'host']} failed to start service {service}"
+
+ exec_cmd_no_error(
+ node, command, timeout=180, sudo=True, message=message
+ )
+
+ DUTSetup.get_service_logs(node, service)
+
+ @staticmethod
+ def start_service_on_all_duts(nodes, service):
+ """Start up the named service on all DUTs.
+
+ :param nodes: Nodes in the topology.
+ :param service: Service unit name.
+ :type nodes: dict
+ :type service: str
+ """
+ for node in nodes.values():
+ if node[u"type"] == NodeType.DUT:
+ DUTSetup.start_service(node, service)
+
+ @staticmethod
+ def stop_service(node, service):
+ """Stop the named service on node.
+
+ :param node: Node in the topology.
+ :param service: Service unit name.
+ :type node: dict
+ :type service: str
+ """
+ DUTSetup.get_service_logs(node, service)
+
+ command = f"supervisorctl stop {service}" \
+ if DUTSetup.running_in_container(node) \
+ else f"service {service} stop"
+ message = f"Node {node[u'host']} failed to stop service {service}"
+
+ exec_cmd_no_error(
+ node, command, timeout=180, sudo=True, message=message
+ )
+
+ @staticmethod
+ def stop_service_on_all_duts(nodes, service):
+ """Stop the named service on all DUTs.
+
+ :param nodes: Nodes in the topology.
+ :param service: Service unit name.
+ :type nodes: dict
+ :type service: str
+ """
+ for node in nodes.values():
+ if node[u"type"] == NodeType.DUT:
+ DUTSetup.stop_service(node, service)
+
+ @staticmethod
+ def kill_program(node, program, namespace=None):
+ """Kill program on the specified topology node.
+
+ :param node: Topology node.
+ :param program: Program name.
+ :param namespace: Namespace program is running in.
+ :type node: dict
+ :type program: str
+ :type namespace: str
+ """
+ host = node[u"host"]
+ cmd_timeout = 5
+ if namespace in (None, u"default"):
+ shell_cmd = u"sh -c"
+ else:
+ shell_cmd = f"ip netns exec {namespace} sh -c"
+
+ pgrep_cmd = f"{shell_cmd} \'pgrep -c {program}\'"
+ _, stdout, _ = exec_cmd(node, pgrep_cmd, timeout=cmd_timeout,
+ sudo=True)
+ if int(stdout) == 0:
+ logger.trace(f"{program} is not running on {host}")
+ return
+ exec_cmd(node, f"{shell_cmd} \'pkill {program}\'",
+ timeout=cmd_timeout, sudo=True)
+ for attempt in range(5):
+ _, stdout, _ = exec_cmd(node, pgrep_cmd, timeout=cmd_timeout,
+ sudo=True)
+ if int(stdout) == 0:
+ logger.trace(f"Attempt {attempt}: {program} is dead on {host}")
+ return
+ sleep(1)
+ logger.trace(f"SIGKILLing {program} on {host}")
+ exec_cmd(node, f"{shell_cmd} \'pkill -9 {program}\'",
+ timeout=cmd_timeout, sudo=True)
+
+ @staticmethod
+ def verify_program_installed(node, program):
+ """Verify that program is installed on the specified topology node.
+
+ :param node: Topology node.
+ :param program: Program name.
+ :type node: dict
+ :type program: str
+ """
+ cmd = f"command -v {program}"
+ exec_cmd_no_error(node, cmd, message=f"{program} is not installed")
+
+ @staticmethod
+ def get_pid(node, process):
+ """Get PID of running process.
+
+ :param node: DUT node.
+ :param process: process name.
+ :type node: dict
+ :type process: str
+ :returns: PID
+ :rtype: int
+ :raises RuntimeError: If it is not possible to get the PID.
+ """
+ ssh = SSH()
+ ssh.connect(node)
+
+ retval = None
+ for i in range(3):
+ logger.trace(f"Try {i}: Get {process} PID")
+ ret_code, stdout, stderr = ssh.exec_command(f"pidof {process}")
+
+ if int(ret_code):
+ raise RuntimeError(
+ f"Not possible to get PID of {process} process on node: "
+ f"{node[u'host']}\n {stdout + stderr}"
+ )
+
+ pid_list = stdout.split()
+ if len(pid_list) == 1:
+ return [int(stdout)]
+ if not pid_list:
+ logger.debug(f"No {process} PID found on node {node[u'host']}")
+ continue
+ logger.debug(f"More than one {process} PID found " \
+ f"on node {node[u'host']}")
+ retval = [int(pid) for pid in pid_list]
+
+ return retval
+
+ @staticmethod
+ def get_vpp_pids(nodes):
+ """Get PID of running VPP process on all DUTs.
+
+ :param nodes: DUT nodes.
+ :type nodes: dict
+ :returns: PIDs
+ :rtype: dict
+ """
+ pids = dict()
+ for node in nodes.values():
+ if node[u"type"] == NodeType.DUT:
+ pids[node[u"host"]] = DUTSetup.get_pid(node, u"vpp")
+ return pids
+
+ @staticmethod
+ def crypto_device_verify(node, crypto_type, numvfs, force_init=False):
+ """Verify if Crypto QAT device virtual functions are initialized on all
+ DUTs. If parameter force initialization is set to True, then try to
+ initialize or remove VFs on QAT.
+
+ :param node: DUT node.
+ :crypto_type: Crypto device type - HW_DH895xcc or HW_C3xxx.
+ :param numvfs: Number of VFs to initialize, 0 - disable the VFs.
+ :param force_init: If True then try to initialize to specific value.
+ :type node: dict
+ :type crypto_type: string
+ :type numvfs: int
+ :type force_init: bool
+ :returns: nothing
+ :raises RuntimeError: If QAT VFs are not created and force init is set
+ to False.
+ """
+ pci_addr = Topology.get_cryptodev(node)
+ sriov_numvfs = DUTSetup.get_sriov_numvfs(node, pci_addr)
+
+ if sriov_numvfs != numvfs:
+ if force_init:
+ # QAT is not initialized and we want to initialize with numvfs
+ DUTSetup.crypto_device_init(node, crypto_type, numvfs)
+ else:
+ raise RuntimeError(
+ f"QAT device failed to create VFs on {node[u'host']}"
+ )
+
+ @staticmethod
+ def crypto_device_init(node, crypto_type, numvfs):
+ """Init Crypto QAT device virtual functions on DUT.
+
+ :param node: DUT node.
+ :crypto_type: Crypto device type - HW_DH895xcc or HW_C3xxx.
+ :param numvfs: Number of VFs to initialize, 0 - disable the VFs.
+ :type node: dict
+ :type crypto_type: string
+ :type numvfs: int
+ :returns: nothing
+ :raises RuntimeError: If failed to stop VPP or QAT failed to initialize.
+ """
+ if crypto_type == u"HW_DH895xcc":
+ kernel_mod = u"qat_dh895xcc"
+ kernel_drv = u"dh895xcc"
+ elif crypto_type == u"HW_C3xxx":
+ kernel_mod = u"qat_c3xxx"
+ kernel_drv = u"c3xxx"
+ else:
+ raise RuntimeError(
+ f"Unsupported crypto device type on {node[u'host']}"
+ )
+
+ pci_addr = Topology.get_cryptodev(node)
+
+ # QAT device must be re-bound to kernel driver before initialization.
+ DUTSetup.verify_kernel_module(node, kernel_mod, force_load=True)
+
+ # Stop VPP to prevent deadlock.
+ DUTSetup.stop_service(node, Constants.VPP_UNIT)
+
+ current_driver = DUTSetup.get_pci_dev_driver(
+ node, pci_addr.replace(u":", r"\:")
+ )
+ if current_driver is not None:
+ DUTSetup.pci_driver_unbind(node, pci_addr)
+
+ # Bind to kernel driver.
+ DUTSetup.pci_driver_bind(node, pci_addr, kernel_drv)
+
+ # Initialize QAT VFs.
+ if numvfs > 0:
+ DUTSetup.set_sriov_numvfs(node, pci_addr, numvfs)
+
+ @staticmethod
+ def get_virtfn_pci_addr(node, pf_pci_addr, vf_id):
+ """Get PCI address of Virtual Function.
+
+ :param node: DUT node.
+ :param pf_pci_addr: Physical Function PCI address.
+ :param vf_id: Virtual Function number.
+ :type node: dict
+ :type pf_pci_addr: str
+ :type vf_id: int
+ :returns: Virtual Function PCI address.
+ :rtype: str
+ :raises RuntimeError: If failed to get Virtual Function PCI address.
+ """
+ command = f"sh -c \"basename $(readlink " \
+ f"/sys/bus/pci/devices/{pf_pci_addr}/virtfn{vf_id})\""
+ message = u"Failed to get virtual function PCI address."
+
+ stdout, _ = exec_cmd_no_error(
+ node, command, timeout=30, sudo=True, message=message
+ )
+
+ return stdout.strip()
+
+ @staticmethod
+ def get_sriov_numvfs(node, pf_pci_addr):
+ """Get number of SR-IOV VFs.
+
+ :param node: DUT node.
+ :param pf_pci_addr: Physical Function PCI device address.
+ :type node: dict
+ :type pf_pci_addr: str
+ :returns: Number of VFs.
+ :rtype: int
+ :raises RuntimeError: If PCI device is not SR-IOV capable.
+ """
+ pci = pf_pci_addr.replace(u":", r"\:")
+ command = f"cat /sys/bus/pci/devices/{pci}/sriov_numvfs"
+ message = f"PCI device {pf_pci_addr} is not a SR-IOV device."
+
+ for _ in range(3):
+ stdout, _ = exec_cmd_no_error(
+ node, command, timeout=30, sudo=True, message=message
+ )
+ try:
+ sriov_numvfs = int(stdout)
+ except ValueError:
+ logger.trace(
+ f"Reading sriov_numvfs info failed on {node[u'host']}"
+ )
+ else:
+ return sriov_numvfs
+
+ @staticmethod
+ def set_sriov_numvfs(node, pf_pci_addr, numvfs=0):
+ """Init or reset SR-IOV virtual functions by setting its number on PCI
+ device on DUT. Setting to zero removes all VFs.
+
+ :param node: DUT node.
+ :param pf_pci_addr: Physical Function PCI device address.
+ :param numvfs: Number of VFs to initialize, 0 - removes the VFs.
+ :type node: dict
+ :type pf_pci_addr: str
+ :type numvfs: int
+ :raises RuntimeError: Failed to create VFs on PCI.
+ """
+ pci = pf_pci_addr.replace(u":", r"\:")
+ command = f"sh -c \"echo {numvfs} | " \
+ f"tee /sys/bus/pci/devices/{pci}/sriov_numvfs\""
+ message = f"Failed to create {numvfs} VFs on {pf_pci_addr} device " \
+ f"on {node[u'host']}"
+
+ exec_cmd_no_error(
+ node, command, timeout=120, sudo=True, message=message
+ )
+
+ @staticmethod
+ def pci_driver_unbind(node, pci_addr):
+ """Unbind PCI device from current driver on node.
+
+ :param node: DUT node.
+ :param pci_addr: PCI device address.
+ :type node: dict
+ :type pci_addr: str
+ :raises RuntimeError: If PCI device unbind failed.
+ """
+ pci = pci_addr.replace(u":", r"\:")
+ command = f"sh -c \"echo {pci_addr} | " \
+ f"tee /sys/bus/pci/devices/{pci}/driver/unbind\""
+ message = f"Failed to unbind PCI device {pci_addr} on {node[u'host']}"
+
+ exec_cmd_no_error(
+ node, command, timeout=120, sudo=True, message=message
+ )
+
+ @staticmethod
+ def pci_driver_unbind_list(node, *pci_addrs):
+ """Unbind PCI devices from current driver on node.
+
+ :param node: DUT node.
+ :param pci_addrs: PCI device addresses.
+ :type node: dict
+ :type pci_addrs: list
+ """
+ for pci_addr in pci_addrs:
+ DUTSetup.pci_driver_unbind(node, pci_addr)