1 # Copyright (c) 2018 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """DUT setup library."""
16 from robot.api import logger
18 from resources.libraries.python.topology import NodeType, Topology
19 from resources.libraries.python.ssh import SSH
20 from resources.libraries.python.constants import Constants
21 from resources.libraries.python.VatExecutor import VatExecutor
22 from resources.libraries.python.VPPUtil import VPPUtil
25 class DUTSetup(object):
26 """Contains methods for setting up DUTs."""
29 def get_service_logs(node, service):
30 """Get specific service unit logs by journalctl from node.
32 :param node: Node in the topology.
33 :param service: Service unit name.
40 ssh.exec_command_sudo('journalctl --no-pager --unit={name} '
41 '--since="$(echo `systemctl show -p '
42 'ActiveEnterTimestamp {name}` | '
43 'awk \'{{print $2 $3}}\')"'.
45 if int(ret_code) != 0:
46 raise RuntimeError('DUT {host} failed to get logs from unit {name}'.
47 format(host=node['host'], name=service))
50 def get_service_logs_on_all_duts(nodes, service):
51 """Get specific service unit logs by journalctl from all DUTs.
53 :param nodes: Nodes in the topology.
54 :param service: Service unit name.
58 for node in nodes.values():
59 if node['type'] == NodeType.DUT:
60 DUTSetup.get_service_logs(node, service)
63 def start_service(node, service):
64 """Start up the named service on node.
66 :param node: Node in the topology.
67 :param service: Service unit name.
73 # We are doing restart. With this we do not care if service
76 ssh.exec_command_sudo('service {name} restart'.
77 format(name=service), timeout=120)
78 if int(ret_code) != 0:
79 raise RuntimeError('DUT {host} failed to start service {name}'.
80 format(host=node['host'], name=service))
82 DUTSetup.get_service_logs(node, service)
85 def start_vpp_service_on_all_duts(nodes):
86 """Start up the VPP service on all nodes.
88 :param nodes: Nodes in the topology.
91 for node in nodes.values():
92 if node['type'] == NodeType.DUT:
93 DUTSetup.start_service(node, Constants.VPP_UNIT)
96 def vpp_show_version_verbose(node):
97 """Run "show version verbose" CLI command.
99 :param node: Node to run command on.
103 vat.execute_script("show_version_verbose.vat", node, json_out=False)
106 vat.script_should_have_passed()
107 except AssertionError:
108 raise RuntimeError('Failed to get VPP version on host: {name}'.
109 format(name=node['host']))
112 def show_vpp_version_on_all_duts(nodes):
113 """Show VPP version verbose on all DUTs.
115 :param nodes: VPP nodes
118 for node in nodes.values():
119 if node['type'] == NodeType.DUT:
120 DUTSetup.vpp_show_version_verbose(node)
123 def vpp_show_interfaces(node):
124 """Run "show interface" CLI command.
126 :param node: Node to run command on.
130 vat.execute_script("show_interface.vat", node, json_out=False)
133 vat.script_should_have_passed()
134 except AssertionError:
135 raise RuntimeError('Failed to get VPP interfaces on host: {name}'.
136 format(name=node['host']))
139 def vpp_api_trace_save(node):
140 """Run "api trace save" CLI command.
142 :param node: Node to run command on.
146 vat.execute_script("api_trace_save.vat", node, json_out=False)
149 def vpp_api_trace_dump(node):
150 """Run "api trace custom-dump" CLI command.
152 :param node: Node to run command on.
156 vat.execute_script("api_trace_dump.vat", node, json_out=False)
159 def setup_all_duts(nodes):
160 """Prepare all DUTs in given topology for test execution."""
161 for node in nodes.values():
162 if node['type'] == NodeType.DUT:
163 DUTSetup.setup_dut(node)
167 """Run script over SSH to setup the DUT node.
169 :param node: DUT node to set up.
172 :raises Exception: If the DUT setup fails.
178 ssh.exec_command('sudo -Sn bash {0}/{1}/dut_setup.sh'.
179 format(Constants.REMOTE_FW_DIR,
180 Constants.RESOURCES_LIB_SH), timeout=120)
181 if int(ret_code) != 0:
182 raise RuntimeError('DUT test setup script failed at node {name}'.
183 format(name=node['host']))
186 def get_vpp_pid(node):
187 """Get PID of running VPP process.
189 :param node: DUT node.
193 :raises RuntimeError: If it is not possible to get the PID.
200 logger.trace('Try {}: Get VPP PID'.format(i))
201 ret_code, stdout, stderr = ssh.exec_command('pidof vpp')
203 if int(ret_code) != 0:
204 raise RuntimeError('Not possible to get PID of VPP process '
205 'on node: {0}\n {1}'.
206 format(node['host'], stdout + stderr))
208 if len(stdout.splitlines()) == 1:
210 elif len(stdout.splitlines()) == 0:
211 logger.debug("No VPP PID found on node {0}".
212 format(node['host']))
215 logger.debug("More then one VPP PID found on node {0}".
216 format(node['host']))
218 for line in stdout.splitlines():
219 ret_list.append(int(line))
225 def get_vpp_pids(nodes):
226 """Get PID of running VPP process on all DUTs.
228 :param nodes: DUT nodes.
235 for node in nodes.values():
236 if node['type'] == NodeType.DUT:
237 pids[node['host']] = DUTSetup.get_vpp_pid(node)
241 def vpp_show_crypto_device_mapping(node):
242 """Run "show crypto device mapping" CLI command.
244 :param node: Node to run command on.
248 vat.execute_script("show_crypto_device_mapping.vat", node,
252 def crypto_device_verify(node, force_init=False, numvfs=32):
253 """Verify if Crypto QAT device virtual functions are initialized on all
254 DUTs. If parameter force initialization is set to True, then try to
255 initialize or disable QAT.
257 :param node: DUT node.
258 :param force_init: If True then try to initialize to specific value.
259 :param numvfs: Number of VFs to initialize, 0 - disable the VFs.
261 :type force_init: bool
264 :raises RuntimeError: If QAT is not initialized or failed to initialize.
270 cryptodev = Topology.get_cryptodev(node)
271 cmd = 'cat /sys/bus/pci/devices/{0}/sriov_numvfs'.\
272 format(cryptodev.replace(':', r'\:'))
274 # Try to read number of VFs from PCI address of QAT device
276 ret_code, stdout, _ = ssh.exec_command(cmd)
277 if int(ret_code) == 0:
279 sriov_numvfs = int(stdout)
281 logger.trace('Reading sriov_numvfs info failed on {0}'.
282 format(node['host']))
284 if sriov_numvfs != numvfs:
286 # QAT is not initialized and we want to initialize
288 DUTSetup.crypto_device_init(node, numvfs)
290 raise RuntimeError('QAT device {0} is not '
291 'initialized to {1} on host {2}'
292 .format(cryptodev, numvfs,
297 def crypto_device_init(node, numvfs):
298 """Init Crypto QAT device virtual functions on DUT.
300 :param node: DUT node.
301 :param numvfs: Number of VFs to initialize, 0 - disable the VFs.
305 :raises RuntimeError: If failed to stop VPP or QAT failed to initialize.
307 cryptodev = Topology.get_cryptodev(node)
309 # QAT device must be re-bound to kernel driver before initialization
311 kernel_module = 'qat_dh895xcc'
312 current_driver = DUTSetup.get_pci_dev_driver(
313 node, cryptodev.replace(':', r'\:'))
315 DUTSetup.kernel_module_verify(node, kernel_module, force_load=True)
317 VPPUtil.stop_vpp_service(node)
318 if current_driver is not None:
319 DUTSetup.pci_driver_unbind(node, cryptodev)
320 DUTSetup.pci_driver_bind(node, cryptodev, driver)
327 cmd = 'echo "{0}" | tee /sys/bus/pci/devices/{1}/sriov_numvfs'.\
328 format(numvfs, cryptodev.replace(':', r'\:'), timeout=180)
329 ret_code, _, _ = ssh.exec_command_sudo("sh -c '{0}'".format(cmd))
331 if int(ret_code) != 0:
332 raise RuntimeError('Failed to initialize {0} VFs on QAT device '
333 ' on host {1}'.format(numvfs, node['host']))
336 def pci_driver_unbind(node, pci_addr):
337 """Unbind PCI device from current driver on node.
339 :param node: DUT node.
340 :param pci_addr: PCI device address.
344 :raises RuntimeError: If PCI device unbind failed.
350 ret_code, _, _ = ssh.exec_command_sudo(
351 "sh -c 'echo {0} | tee /sys/bus/pci/devices/{1}/driver/unbind'"
352 .format(pci_addr, pci_addr.replace(':', r'\:')), timeout=180)
354 if int(ret_code) != 0:
355 raise RuntimeError('Failed to unbind PCI device {0} from driver on '
356 'host {1}'.format(pci_addr, node['host']))
359 def pci_driver_bind(node, pci_addr, driver):
360 """Bind PCI device to driver on node.
362 :param node: DUT node.
363 :param pci_addr: PCI device address.
364 :param driver: Driver to bind.
369 :raises RuntimeError: If PCI device bind failed.
375 ret_code, _, _ = ssh.exec_command_sudo(
376 "sh -c 'echo {0} | tee /sys/bus/pci/drivers/{1}/bind'".format(
377 pci_addr, driver), timeout=180)
379 if int(ret_code) != 0:
380 raise RuntimeError('Failed to bind PCI device {0} to {1} driver on '
381 'host {2}'.format(pci_addr, driver,
385 def get_pci_dev_driver(node, pci_addr):
386 """Get current PCI device driver on node.
389 # lspci -vmmks 0000:00:05.0
391 Class: Ethernet controller
393 Device: Virtio network device
394 SVendor: Red Hat, Inc
399 :param node: DUT node.
400 :param pci_addr: PCI device address.
403 :returns: Driver or None
404 :raises RuntimeError: If PCI rescan or lspci command execution failed.
405 :raises RuntimeError: If it is not possible to get the interface driver
406 information from the node.
412 logger.trace('Try number {0}: Get PCI device driver'.format(i))
413 cmd = 'lspci -vmmks {0}'.format(pci_addr)
414 ret_code, stdout, _ = ssh.exec_command(cmd)
415 if int(ret_code) != 0:
416 raise RuntimeError("'{0}' failed on '{1}'"
417 .format(cmd, node['host']))
419 for line in stdout.splitlines():
425 name, value = line.split("\t", 1)
427 if name == "Driver:":
429 if name == 'Driver:':
433 logger.trace('Driver for PCI device {} not found, executing '
434 'pci rescan and retrying'.format(pci_addr))
435 cmd = 'sh -c "echo 1 > /sys/bus/pci/rescan"'
436 ret_code, _, _ = ssh.exec_command_sudo(cmd)
437 if int(ret_code) != 0:
438 raise RuntimeError("'{0}' failed on '{1}'"
439 .format(cmd, node['host']))
444 def kernel_module_verify(node, module, force_load=False):
445 """Verify if kernel module is loaded on all DUTs. If parameter force
446 load is set to True, then try to load the modules.
448 :param node: DUT node.
449 :param module: Module to verify.
450 :param force_load: If True then try to load module.
453 :type force_load: bool
455 :raises RuntimeError: If module is not loaded or failed to load.
461 cmd = 'grep -w {0} /proc/modules'.format(module)
462 ret_code, _, _ = ssh.exec_command(cmd)
464 if int(ret_code) != 0:
466 # Module is not loaded and we want to load it
467 DUTSetup.kernel_module_load(node, module)
469 raise RuntimeError('Kernel module {0} is not loaded on host '
470 '{1}'.format(module, node['host']))
473 def kernel_module_load(node, module):
474 """Load kernel module on node.
476 :param node: DUT node.
477 :param module: Module to load.
481 :raises RuntimeError: If loading failed.
487 ret_code, _, _ = ssh.exec_command_sudo("modprobe {0}".format(module))
489 if int(ret_code) != 0:
490 raise RuntimeError('Failed to load {0} kernel module on host {1}'.
491 format(module, node['host']))
494 def vpp_enable_traces_on_all_duts(nodes):
495 """Enable vpp packet traces on all DUTs in the given topology.
497 :param nodes: Nodes in the topology.
500 for node in nodes.values():
501 if node['type'] == NodeType.DUT:
502 DUTSetup.vpp_enable_traces_on_dut(node)
505 def vpp_enable_traces_on_dut(node):
506 """Enable vpp packet traces on the DUT node.
508 :param node: DUT node to set up.
513 vat.execute_script("enable_dpdk_traces.vat", node, json_out=False)
514 vat.execute_script("enable_vhost_user_traces.vat", node, json_out=False)
515 vat.execute_script("enable_memif_traces.vat", node, json_out=False)
518 def install_vpp_on_all_duts(nodes, vpp_pkg_dir, vpp_rpm_pkgs, vpp_deb_pkgs):
519 """Install VPP on all DUT nodes.
521 :param nodes: Nodes in the topology.
522 :param vpp_pkg_dir: Path to directory where VPP packages are stored.
523 :param vpp_rpm_pkgs: List of VPP rpm packages to be installed.
524 :param vpp_deb_pkgs: List of VPP deb packages to be installed.
526 :type vpp_pkg_dir: str
527 :type vpp_rpm_pkgs: list
528 :type vpp_deb_pkgs: list
529 :raises RuntimeError: If failed to remove or install VPP.
532 logger.debug("Installing VPP")
534 for node in nodes.values():
535 if node['type'] == NodeType.DUT:
536 logger.debug("Installing VPP on node {0}".format(node['host']))
541 cmd = "[[ -f /etc/redhat-release ]]"
542 return_code, _, _ = ssh.exec_command(cmd)
543 if int(return_code) == 0:
544 # workaroud - uninstall existing vpp installation until
545 # start-testcase script is updated on all virl servers
546 rpm_pkgs_remove = "vpp*"
547 cmd_u = 'yum -y remove "{0}"'.format(rpm_pkgs_remove)
548 r_rcode, _, r_err = ssh.exec_command_sudo(cmd_u, timeout=90)
549 if int(r_rcode) != 0:
550 raise RuntimeError('Failed to remove previous VPP'
551 'installation on host {0}:\n{1}'
552 .format(node['host'], r_err))
554 rpm_pkgs = "*.rpm ".join(str(vpp_pkg_dir + pkg)
555 for pkg in vpp_rpm_pkgs) + "*.rpm"
556 cmd_i = "rpm -ivh {0}".format(rpm_pkgs)
557 ret_code, _, err = ssh.exec_command_sudo(cmd_i, timeout=90)
558 if int(ret_code) != 0:
559 raise RuntimeError('Failed to install VPP on host {0}:'
560 '\n{1}'.format(node['host'], err))
562 ssh.exec_command_sudo("rpm -qai vpp*")
563 logger.info("VPP installed on node {0}".
564 format(node['host']))
566 # workaroud - uninstall existing vpp installation until
567 # start-testcase script is updated on all virl servers
568 deb_pkgs_remove = "vpp*"
569 cmd_u = 'apt-get purge -y "{0}"'.format(deb_pkgs_remove)
570 r_rcode, _, r_err = ssh.exec_command_sudo(cmd_u, timeout=90)
571 if int(r_rcode) != 0:
572 raise RuntimeError('Failed to remove previous VPP'
573 'installation on host {0}:\n{1}'
574 .format(node['host'], r_err))
575 deb_pkgs = "*.deb ".join(str(vpp_pkg_dir + pkg)
576 for pkg in vpp_deb_pkgs) + "*.deb"
577 cmd_i = "dpkg -i --force-all {0}".format(deb_pkgs)
578 ret_code, _, err = ssh.exec_command_sudo(cmd_i, timeout=90)
579 if int(ret_code) != 0:
580 raise RuntimeError('Failed to install VPP on host {0}:'
581 '\n{1}'.format(node['host'], err))
583 ssh.exec_command_sudo("dpkg -l | grep vpp")
584 logger.info("VPP installed on node {0}".
585 format(node['host']))
590 def verify_vpp_on_all_duts(nodes):
591 """Verify that VPP is installed on all DUT nodes.
593 :param nodes: Nodes in the topology.
597 logger.debug("Verify VPP on all DUTs")
599 DUTSetup.start_vpp_service_on_all_duts(nodes)
601 for node in nodes.values():
602 if node['type'] == NodeType.DUT:
603 DUTSetup.verify_vpp_on_dut(node)
606 def verify_vpp_on_dut(node):
607 """Verify that VPP is installed on DUT node.
609 :param node: DUT node.
611 :raises RuntimeError: If failed to restart VPP, get VPP version
612 or get VPP interfaces.
615 logger.debug("Verify VPP on node {0}".format(node['host']))
617 DUTSetup.vpp_show_version_verbose(node)
618 DUTSetup.vpp_show_interfaces(node)