1 # Copyright (c) 2018 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """DUT setup library."""
16 from robot.api import logger
18 from resources.libraries.python.topology import NodeType, Topology
19 from resources.libraries.python.ssh import SSH
20 from resources.libraries.python.constants import Constants
21 from resources.libraries.python.VatExecutor import VatExecutor
22 from resources.libraries.python.VPPUtil import VPPUtil
25 class DUTSetup(object):
26 """Contains methods for setting up DUTs."""
29 def get_service_logs(node, service):
30 """Get specific service unit logs by journalctl from node.
32 :param node: Node in the topology.
33 :param service: Service unit name.
40 ssh.exec_command_sudo('journalctl --no-pager --unit={name} '
41 '--since="$(echo `systemctl show -p '
42 'ActiveEnterTimestamp {name}` | '
43 'awk \'{{print $2 $3}}\')"'.
45 if int(ret_code) != 0:
46 raise RuntimeError('DUT {host} failed to get logs from unit {name}'.
47 format(host=node['host'], name=service))
50 def get_service_logs_on_all_duts(nodes, service):
51 """Get specific service unit logs by journalctl from all DUTs.
53 :param nodes: Nodes in the topology.
54 :param service: Service unit name.
58 for node in nodes.values():
59 if node['type'] == NodeType.DUT:
60 DUTSetup.get_service_logs(node, service)
63 def start_service(node, service):
64 """Start up the named service on node.
66 :param node: Node in the topology.
67 :param service: Service unit name.
73 # We are doing restart. With this we do not care if service
76 ssh.exec_command_sudo('service {name} restart'.
77 format(name=service), timeout=120)
78 if int(ret_code) != 0:
79 raise RuntimeError('DUT {host} failed to start service {name}'.
80 format(host=node['host'], name=service))
82 DUTSetup.get_service_logs(node, service)
85 def start_vpp_service_on_all_duts(nodes):
86 """Start up the VPP service on all nodes.
88 :param nodes: Nodes in the topology.
91 for node in nodes.values():
92 if node['type'] == NodeType.DUT:
93 DUTSetup.start_service(node, Constants.VPP_UNIT)
96 def vpp_show_version_verbose(node):
97 """Run "show version verbose" CLI command.
99 :param node: Node to run command on.
103 vat.execute_script("show_version_verbose.vat", node, json_out=False)
106 vat.script_should_have_passed()
107 except AssertionError:
108 raise RuntimeError('Failed to get VPP version on host: {name}'.
109 format(name=node['host']))
112 def show_vpp_version_on_all_duts(nodes):
113 """Show VPP version verbose on all DUTs.
115 :param nodes: VPP nodes
118 for node in nodes.values():
119 if node['type'] == NodeType.DUT:
120 DUTSetup.vpp_show_version_verbose(node)
123 def vpp_show_interfaces(node):
124 """Run "show interface" CLI command.
126 :param node: Node to run command on.
130 vat.execute_script("show_interface.vat", node, json_out=False)
133 vat.script_should_have_passed()
134 except AssertionError:
135 raise RuntimeError('Failed to get VPP interfaces on host: {name}'.
136 format(name=node['host']))
139 def vpp_api_trace_save(node):
140 """Run "api trace save" CLI command.
142 :param node: Node to run command on.
146 vat.execute_script("api_trace_save.vat", node, json_out=False)
149 def vpp_api_trace_dump(node):
150 """Run "api trace custom-dump" CLI command.
152 :param node: Node to run command on.
156 vat.execute_script("api_trace_dump.vat", node, json_out=False)
159 def setup_all_duts(nodes):
160 """Prepare all DUTs in given topology for test execution."""
161 for node in nodes.values():
162 if node['type'] == NodeType.DUT:
163 DUTSetup.setup_dut(node)
167 """Run script over SSH to setup the DUT node.
169 :param node: DUT node to set up.
172 :raises Exception: If the DUT setup fails.
178 ssh.exec_command('sudo -Sn bash {0}/{1}/dut_setup.sh'.
179 format(Constants.REMOTE_FW_DIR,
180 Constants.RESOURCES_LIB_SH), timeout=120)
181 if int(ret_code) != 0:
182 raise RuntimeError('DUT test setup script failed at node {name}'.
183 format(name=node['host']))
186 def get_vpp_pid(node):
187 """Get PID of running VPP process.
189 :param node: DUT node.
193 :raises RuntimeError if it is not possible to get the PID.
200 logger.trace('Try {}: Get VPP PID'.format(i))
201 ret_code, stdout, stderr = ssh.exec_command('pidof vpp')
203 if int(ret_code) != 0:
204 raise RuntimeError('Not possible to get PID of VPP process '
205 'on node: {0}\n {1}'.
206 format(node['host'], stdout + stderr))
208 if len(stdout.splitlines()) == 1:
210 elif len(stdout.splitlines()) == 0:
211 logger.debug("No VPP PID found on node {0}".
212 format(node['host']))
215 logger.debug("More then one VPP PID found on node {0}".
216 format(node['host']))
218 for line in stdout.splitlines():
219 ret_list.append(int(line))
225 def get_vpp_pids(nodes):
226 """Get PID of running VPP process on all DUTs.
228 :param nodes: DUT nodes.
235 for node in nodes.values():
236 if node['type'] == NodeType.DUT:
237 pids[node['host']] = DUTSetup.get_vpp_pid(node)
241 def vpp_show_crypto_device_mapping(node):
242 """Run "show crypto device mapping" CLI command.
244 :param node: Node to run command on.
248 vat.execute_script("show_crypto_device_mapping.vat", node,
252 def crypto_device_verify(node, force_init=False, numvfs=32):
253 """Verify if Crypto QAT device virtual functions are initialized on all
254 DUTs. If parameter force initialization is set to True, then try to
255 initialize or disable QAT.
257 :param node: DUT node.
258 :param force_init: If True then try to initialize to specific value.
259 :param numvfs: Number of VFs to initialize, 0 - disable the VFs.
261 :type force_init: bool
264 :raises RuntimeError: If QAT is not initialized or failed to initialize.
270 cryptodev = Topology.get_cryptodev(node)
271 cmd = 'cat /sys/bus/pci/devices/{0}/sriov_numvfs'.\
272 format(cryptodev.replace(':', r'\:'))
274 # Try to read number of VFs from PCI address of QAT device
276 ret_code, stdout, _ = ssh.exec_command(cmd)
277 if int(ret_code) == 0:
279 sriov_numvfs = int(stdout)
281 logger.trace('Reading sriov_numvfs info failed on {0}'.
282 format(node['host']))
284 if sriov_numvfs != numvfs:
286 # QAT is not initialized and we want to initialize
288 DUTSetup.crypto_device_init(node, numvfs)
290 raise RuntimeError('QAT device {0} is not '
291 'initialized to {1} on host {2}'
292 .format(cryptodev, numvfs,
297 def crypto_device_init(node, numvfs):
298 """Init Crypto QAT device virtual functions on DUT.
300 :param node: DUT node.
301 :param numvfs: Number of VFs to initialize, 0 - disable the VFs.
305 :raises RuntimeError: If failed to stop VPP or QAT failed to initialize.
307 cryptodev = Topology.get_cryptodev(node)
309 # QAT device must be re-bound to kernel driver before initialization
311 kernel_module = 'qat_dh895xcc'
312 current_driver = DUTSetup.get_pci_dev_driver(
313 node, cryptodev.replace(':', r'\:'))
315 DUTSetup.kernel_module_verify(node, kernel_module, force_load=True)
317 VPPUtil.stop_vpp_service(node)
318 if current_driver is not None:
319 DUTSetup.pci_driver_unbind(node, cryptodev)
320 DUTSetup.pci_driver_bind(node, cryptodev, driver)
327 cmd = 'echo "{0}" | tee /sys/bus/pci/devices/{1}/sriov_numvfs'.\
328 format(numvfs, cryptodev.replace(':', r'\:'), timeout=180)
329 ret_code, _, _ = ssh.exec_command_sudo("sh -c '{0}'".format(cmd))
331 if int(ret_code) != 0:
332 raise RuntimeError('Failed to initialize {0} VFs on QAT device '
333 ' on host {1}'.format(numvfs, node['host']))
336 def pci_driver_unbind(node, pci_addr):
337 """Unbind PCI device from current driver on node.
339 :param node: DUT node.
340 :param pci_addr: PCI device address.
344 :raises RuntimeError: If PCI device unbind failed.
350 ret_code, _, _ = ssh.exec_command_sudo(
351 "sh -c 'echo {0} | tee /sys/bus/pci/devices/{1}/driver/unbind'"
352 .format(pci_addr, pci_addr.replace(':', r'\:')), timeout=180)
354 if int(ret_code) != 0:
355 raise RuntimeError('Failed to unbind PCI device {0} from driver on '
356 'host {1}'.format(pci_addr, node['host']))
359 def pci_driver_bind(node, pci_addr, driver):
360 """Bind PCI device to driver on node.
362 :param node: DUT node.
363 :param pci_addr: PCI device address.
364 :param driver: Driver to bind.
369 :raises RuntimeError: If PCI device bind failed.
375 ret_code, _, _ = ssh.exec_command_sudo(
376 "sh -c 'echo {0} | tee /sys/bus/pci/drivers/{1}/bind'".format(
377 pci_addr, driver), timeout=180)
379 if int(ret_code) != 0:
380 raise RuntimeError('Failed to bind PCI device {0} to {1} driver on '
381 'host {2}'.format(pci_addr, driver,
385 def get_pci_dev_driver(node, pci_addr):
386 """Get current PCI device driver on node.
388 :param node: DUT node.
389 :param pci_addr: PCI device address.
392 :returns: Driver or None
393 :raises RuntimeError: If PCI rescan or lspci command execution failed.
399 logger.trace('Try {0}: Get interface driver'.format(i))
400 cmd = 'sh -c "echo 1 > /sys/bus/pci/rescan"'
401 ret_code, _, _ = ssh.exec_command_sudo(cmd)
402 if int(ret_code) != 0:
403 raise RuntimeError("'{0}' failed on '{1}'"
404 .format(cmd, node['host']))
406 cmd = 'lspci -vmmks {0}'.format(pci_addr)
407 ret_code, stdout, _ = ssh.exec_command(cmd)
408 if int(ret_code) != 0:
409 raise RuntimeError("'{0}' failed on '{1}'"
410 .format(cmd, node['host']))
412 for line in stdout.splitlines():
418 name, value = line.split("\t", 1)
420 if name == "Driver:":
422 if name == 'Driver:':
428 def kernel_module_verify(node, module, force_load=False):
429 """Verify if kernel module is loaded on all DUTs. If parameter force
430 load is set to True, then try to load the modules.
432 :param node: DUT node.
433 :param module: Module to verify.
434 :param force_load: If True then try to load module.
437 :type force_load: bool
439 :raises RuntimeError: If module is not loaded or failed to load.
445 cmd = 'grep -w {0} /proc/modules'.format(module)
446 ret_code, _, _ = ssh.exec_command(cmd)
448 if int(ret_code) != 0:
450 # Module is not loaded and we want to load it
451 DUTSetup.kernel_module_load(node, module)
453 raise RuntimeError('Kernel module {0} is not loaded on host '
454 '{1}'.format(module, node['host']))
457 def kernel_module_load(node, module):
458 """Load kernel module on node.
460 :param node: DUT node.
461 :param module: Module to load.
465 :raises RuntimeError: If loading failed.
471 ret_code, _, _ = ssh.exec_command_sudo("modprobe {0}".format(module))
473 if int(ret_code) != 0:
474 raise RuntimeError('Failed to load {0} kernel module on host {1}'.
475 format(module, node['host']))
478 def vpp_enable_traces_on_all_duts(nodes):
479 """Enable vpp packet traces on all DUTs in the given topology.
481 :param nodes: Nodes in the topology.
484 for node in nodes.values():
485 if node['type'] == NodeType.DUT:
486 DUTSetup.vpp_enable_traces_on_dut(node)
489 def vpp_enable_traces_on_dut(node):
490 """Enable vpp packet traces on the DUT node.
492 :param node: DUT node to set up.
497 vat.execute_script("enable_dpdk_traces.vat", node, json_out=False)
498 vat.execute_script("enable_vhost_user_traces.vat", node, json_out=False)
501 def install_vpp_on_all_duts(nodes, vpp_pkg_dir, vpp_rpm_pkgs, vpp_deb_pkgs):
502 """Install VPP on all DUT nodes.
504 :param nodes: Nodes in the topology.
505 :param vpp_pkg_dir: Path to directory where VPP packages are stored.
506 :param vpp_rpm_pkgs: List of VPP rpm packages to be installed.
507 :param vpp_deb_pkgs: List of VPP deb packages to be installed.
509 :type vpp_pkg_dir: str
510 :type vpp_rpm_pkgs: list
511 :type vpp_deb_pkgs: list
512 :raises: RuntimeError if failed to remove or install VPP
515 logger.debug("Installing VPP")
517 for node in nodes.values():
518 if node['type'] == NodeType.DUT:
519 logger.debug("Installing VPP on node {0}".format(node['host']))
524 cmd = "[[ -f /etc/redhat-release ]]"
525 return_code, _, _ = ssh.exec_command(cmd)
526 if int(return_code) == 0:
527 # workaroud - uninstall existing vpp installation until
528 # start-testcase script is updated on all virl servers
529 rpm_pkgs_remove = "vpp*"
530 cmd_u = 'yum -y remove "{0}"'.format(rpm_pkgs_remove)
531 r_rcode, _, r_err = ssh.exec_command_sudo(cmd_u, timeout=90)
532 if int(r_rcode) != 0:
533 raise RuntimeError('Failed to remove previous VPP'
534 'installation on host {0}:\n{1}'
535 .format(node['host'], r_err))
537 rpm_pkgs = "*.rpm ".join(str(vpp_pkg_dir + pkg)
538 for pkg in vpp_rpm_pkgs) + "*.rpm"
539 cmd_i = "rpm -ivh {0}".format(rpm_pkgs)
540 ret_code, _, err = ssh.exec_command_sudo(cmd_i, timeout=90)
541 if int(ret_code) != 0:
542 raise RuntimeError('Failed to install VPP on host {0}:'
543 '\n{1}'.format(node['host'], err))
545 ssh.exec_command_sudo("rpm -qai vpp*")
546 logger.info("VPP installed on node {0}".
547 format(node['host']))
549 # workaroud - uninstall existing vpp installation until
550 # start-testcase script is updated on all virl servers
551 deb_pkgs_remove = "vpp*"
552 cmd_u = 'apt-get purge -y "{0}"'.format(deb_pkgs_remove)
553 r_rcode, _, r_err = ssh.exec_command_sudo(cmd_u, timeout=90)
554 if int(r_rcode) != 0:
555 raise RuntimeError('Failed to remove previous VPP'
556 'installation on host {0}:\n{1}'
557 .format(node['host'], r_err))
558 deb_pkgs = "*.deb ".join(str(vpp_pkg_dir + pkg)
559 for pkg in vpp_deb_pkgs) + "*.deb"
560 cmd_i = "dpkg -i --force-all {0}".format(deb_pkgs)
561 ret_code, _, err = ssh.exec_command_sudo(cmd_i, timeout=90)
562 if int(ret_code) != 0:
563 raise RuntimeError('Failed to install VPP on host {0}:'
564 '\n{1}'.format(node['host'], err))
566 ssh.exec_command_sudo("dpkg -l | grep vpp")
567 logger.info("VPP installed on node {0}".
568 format(node['host']))
573 def verify_vpp_on_all_duts(nodes):
574 """Verify that VPP is installed on all DUT nodes.
576 :param nodes: Nodes in the topology.
580 logger.debug("Verify VPP on all DUTs")
582 DUTSetup.start_vpp_service_on_all_duts(nodes)
584 for node in nodes.values():
585 if node['type'] == NodeType.DUT:
586 DUTSetup.verify_vpp_on_dut(node)
589 def verify_vpp_on_dut(node):
590 """Verify that VPP is installed on DUT node.
592 :param node: DUT node.
594 :raises: RuntimeError if failed to restart VPP, get VPP version or
598 logger.debug("Verify VPP on node {0}".format(node['host']))
600 DUTSetup.vpp_show_version_verbose(node)
601 DUTSetup.vpp_show_interfaces(node)