1 # Copyright (c) 2018 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """DUT setup library."""
18 from robot.api import logger
20 from resources.libraries.python.topology import NodeType, Topology
21 from resources.libraries.python.ssh import SSH
22 from resources.libraries.python.constants import Constants
23 from resources.libraries.python.VatExecutor import VatExecutor
24 from resources.libraries.python.VPPUtil import VPPUtil
27 class DUTSetup(object):
28 """Contains methods for setting up DUTs."""
30 def start_vpp_service_on_all_duts(nodes):
31 """Start up the VPP service on all nodes.
33 :param nodes: Nodes in the topology.
37 for node in nodes.values():
38 if node['type'] == NodeType.DUT:
40 (ret_code, stdout, stderr) = \
41 ssh.exec_command_sudo('service vpp restart', timeout=120)
42 if int(ret_code) != 0:
43 raise Exception('DUT {0} failed to start VPP service'.
47 def vpp_show_version_verbose(node):
48 """Run "show version verbose" CLI command.
50 :param node: Node to run command on.
54 vat.execute_script("show_version_verbose.vat", node, json_out=False)
57 vat.script_should_have_passed()
58 except AssertionError:
59 raise RuntimeError('Failed to get VPP version on host: {}'.
63 def show_vpp_version_on_all_duts(nodes):
64 """Show VPP version verbose on all DUTs.
66 :param nodes: VPP nodes
69 for node in nodes.values():
70 if node['type'] == NodeType.DUT:
71 DUTSetup.vpp_show_version_verbose(node)
74 def vpp_show_interfaces(node):
75 """Run "show interface" CLI command.
77 :param node: Node to run command on.
81 vat.execute_script("show_interface.vat", node, json_out=False)
84 vat.script_should_have_passed()
85 except AssertionError:
86 raise RuntimeError('Failed to get VPP interfaces on host: {}'.
90 def vpp_api_trace_save(node):
91 """Run "api trace save" CLI command.
93 :param node: Node to run command on.
97 vat.execute_script("api_trace_save.vat", node, json_out=False)
100 def vpp_api_trace_dump(node):
101 """Run "api trace custom-dump" CLI command.
103 :param node: Node to run command on.
107 vat.execute_script("api_trace_dump.vat", node, json_out=False)
110 def setup_all_duts(nodes):
111 """Prepare all DUTs in given topology for test execution."""
112 for node in nodes.values():
113 if node['type'] == NodeType.DUT:
114 DUTSetup.setup_dut(node)
118 """Run script over SSH to setup the DUT node.
120 :param node: DUT node to set up.
123 :raises Exception: If the DUT setup fails.
128 (ret_code, stdout, stderr) = \
129 ssh.exec_command('sudo -Sn bash {0}/{1}/dut_setup.sh'.
130 format(Constants.REMOTE_FW_DIR,
131 Constants.RESOURCES_LIB_SH), timeout=120)
132 if int(ret_code) != 0:
133 logger.debug('DUT {0} setup script failed: "{1}"'.
134 format(node['host'], stdout + stderr))
135 raise Exception('DUT test setup script failed at node {}'.
136 format(node['host']))
139 def get_vpp_pid(node):
140 """Get PID of running VPP process.
142 :param node: DUT node.
146 :raises RuntimeError if it is not possible to get the PID.
153 logger.trace('Try {}: Get VPP PID'.format(i))
154 ret_code, stdout, stderr = ssh.exec_command('pidof vpp')
156 if int(ret_code) != 0:
157 raise RuntimeError('Not possible to get PID of VPP process '
158 'on node: {0}\n {1}'.
159 format(node['host'], stdout + stderr))
161 if len(stdout.splitlines()) == 1:
163 elif len(stdout.splitlines()) == 0:
164 logger.debug("No VPP PID found on node {0}".
165 format(node['host']))
168 logger.debug("More then one VPP PID found on node {0}".
169 format(node['host']))
171 for line in stdout.splitlines():
172 ret_list.append(int(line))
178 def get_vpp_pids(nodes):
179 """Get PID of running VPP process on all DUTs.
181 :param nodes: DUT nodes.
188 for node in nodes.values():
189 if node['type'] == NodeType.DUT:
190 pids[node['host']] = DUTSetup.get_vpp_pid(node)
194 def vpp_show_crypto_device_mapping(node):
195 """Run "show crypto device mapping" CLI command.
197 :param node: Node to run command on.
201 vat.execute_script("show_crypto_device_mapping.vat", node,
205 def crypto_device_verify(node, force_init=False, numvfs=32):
206 """Verify if Crypto QAT device virtual functions are initialized on all
207 DUTs. If parameter force initialization is set to True, then try to
208 initialize or disable QAT.
210 :param node: DUT node.
211 :param force_init: If True then try to initialize to specific value.
212 :param numvfs: Number of VFs to initialize, 0 - disable the VFs.
214 :type force_init: bool
217 :raises RuntimeError: If QAT is not initialized or failed to initialize.
223 cryptodev = Topology.get_cryptodev(node)
224 cmd = 'cat /sys/bus/pci/devices/{0}/sriov_numvfs'.\
225 format(cryptodev.replace(':', r'\:'))
227 # Try to read number of VFs from PCI address of QAT device
229 ret_code, stdout, _ = ssh.exec_command(cmd)
230 if int(ret_code) == 0:
232 sriov_numvfs = int(stdout)
234 logger.trace('Reading sriov_numvfs info failed on {0}'.
235 format(node['host']))
237 if sriov_numvfs != numvfs:
239 # QAT is not initialized and we want to initialize
241 DUTSetup.crypto_device_init(node, numvfs)
243 raise RuntimeError('QAT device {0} is not '
244 'initialized to {1} on host {2}'
245 .format(cryptodev, numvfs,
250 def crypto_device_init(node, numvfs):
251 """Init Crypto QAT device virtual functions on DUT.
253 :param node: DUT node.
254 :param numvfs: Number of VFs to initialize, 0 - disable the VFs.
258 :raises RuntimeError: If failed to stop VPP or QAT failed to initialize.
260 cryptodev = Topology.get_cryptodev(node)
262 # QAT device must be re-bound to kernel driver before initialization
264 kernel_module = 'qat_dh895xcc'
265 current_driver = DUTSetup.get_pci_dev_driver(
266 node, cryptodev.replace(':', r'\:'))
268 DUTSetup.kernel_module_verify(node, kernel_module, force_load=True)
270 VPPUtil.stop_vpp_service(node)
271 if current_driver is not None:
272 DUTSetup.pci_driver_unbind(node, cryptodev)
273 DUTSetup.pci_driver_bind(node, cryptodev, driver)
280 cmd = 'echo "{0}" | tee /sys/bus/pci/devices/{1}/sriov_numvfs'.\
281 format(numvfs, cryptodev.replace(':', r'\:'), timeout=180)
282 ret_code, _, _ = ssh.exec_command_sudo("sh -c '{0}'".format(cmd))
284 if int(ret_code) != 0:
285 raise RuntimeError('Failed to initialize {0} VFs on QAT device '
286 ' on host {1}'.format(numvfs, node['host']))
289 def pci_driver_unbind(node, pci_addr):
290 """Unbind PCI device from current driver on node.
292 :param node: DUT node.
293 :param pci_addr: PCI device address.
297 :raises RuntimeError: If PCI device unbind failed.
303 ret_code, _, _ = ssh.exec_command_sudo(
304 "sh -c 'echo {0} | tee /sys/bus/pci/devices/{1}/driver/unbind'"
305 .format(pci_addr, pci_addr.replace(':', r'\:')), timeout=180)
307 if int(ret_code) != 0:
308 raise RuntimeError('Failed to unbind PCI device {0} from driver on '
309 'host {1}'.format(pci_addr, node['host']))
312 def pci_driver_bind(node, pci_addr, driver):
313 """Bind PCI device to driver on node.
315 :param node: DUT node.
316 :param pci_addr: PCI device address.
317 :param driver: Driver to bind.
322 :raises RuntimeError: If PCI device bind failed.
328 ret_code, _, _ = ssh.exec_command_sudo(
329 "sh -c 'echo {0} | tee /sys/bus/pci/drivers/{1}/bind'".format(
330 pci_addr, driver), timeout=180)
332 if int(ret_code) != 0:
333 raise RuntimeError('Failed to bind PCI device {0} to {1} driver on '
334 'host {2}'.format(pci_addr, driver,
338 def get_pci_dev_driver(node, pci_addr):
339 """Get current PCI device driver on node.
341 :param node: DUT node.
342 :param pci_addr: PCI device address.
345 :returns: Driver or None
346 :raises RuntimeError: If PCI rescan or lspci command execution failed.
352 logger.trace('Try {0}: Get interface driver'.format(i))
353 cmd = 'sh -c "echo 1 > /sys/bus/pci/rescan"'
354 ret_code, _, _ = ssh.exec_command_sudo(cmd)
355 if int(ret_code) != 0:
356 raise RuntimeError("'{0}' failed on '{1}'"
357 .format(cmd, node['host']))
359 cmd = 'lspci -vmmks {0}'.format(pci_addr)
360 ret_code, stdout, _ = ssh.exec_command(cmd)
361 if int(ret_code) != 0:
362 raise RuntimeError("'{0}' failed on '{1}'"
363 .format(cmd, node['host']))
365 for line in stdout.splitlines():
371 name, value = line.split("\t", 1)
373 if name == "Driver:":
375 if name == 'Driver:':
381 def kernel_module_verify(node, module, force_load=False):
382 """Verify if kernel module is loaded on all DUTs. If parameter force
383 load is set to True, then try to load the modules.
385 :param node: DUT node.
386 :param module: Module to verify.
387 :param force_load: If True then try to load module.
390 :type force_load: bool
392 :raises RuntimeError: If module is not loaded or failed to load.
398 cmd = 'grep -w {0} /proc/modules'.format(module)
399 ret_code, _, _ = ssh.exec_command(cmd)
401 if int(ret_code) != 0:
403 # Module is not loaded and we want to load it
404 DUTSetup.kernel_module_load(node, module)
406 raise RuntimeError('Kernel module {0} is not loaded on host '
407 '{1}'.format(module, node['host']))
410 def kernel_module_load(node, module):
411 """Load kernel module on node.
413 :param node: DUT node.
414 :param module: Module to load.
418 :raises RuntimeError: If loading failed.
424 ret_code, _, _ = ssh.exec_command_sudo("modprobe {0}".format(module))
426 if int(ret_code) != 0:
427 raise RuntimeError('Failed to load {0} kernel module on host {1}'.
428 format(module, node['host']))
431 def vpp_enable_traces_on_all_duts(nodes):
432 """Enable vpp packet traces on all DUTs in the given topology.
434 :param nodes: Nodes in the topology.
437 for node in nodes.values():
438 if node['type'] == NodeType.DUT:
439 DUTSetup.vpp_enable_traces_on_dut(node)
442 def vpp_enable_traces_on_dut(node):
443 """Enable vpp packet traces on the DUT node.
445 :param node: DUT node to set up.
450 vat.execute_script("enable_dpdk_traces.vat", node, json_out=False)
451 vat.execute_script("enable_vhost_user_traces.vat", node, json_out=False)
454 def install_vpp_on_all_duts(nodes, vpp_pkg_dir, vpp_rpm_pkgs, vpp_deb_pkgs):
455 """Install VPP on all DUT nodes.
457 :param nodes: Nodes in the topology.
458 :param vpp_pkg_dir: Path to directory where VPP packages are stored.
459 :param vpp_rpm_pkgs: List of VPP rpm packages to be installed.
460 :param vpp_deb_pkgs: List of VPP deb packages to be installed.
462 :type vpp_pkg_dir: str
463 :type vpp_rpm_pkgs: list
464 :type vpp_deb_pkgs: list
465 :raises: RuntimeError if failed to remove or install VPP
468 logger.debug("Installing VPP")
470 for node in nodes.values():
471 if node['type'] == NodeType.DUT:
472 logger.debug("Installing VPP on node {0}".format(node['host']))
477 cmd = "[[ -f /etc/redhat-release ]]"
478 return_code, _, _ = ssh.exec_command(cmd)
479 if int(return_code) == 0:
480 # workaroud - uninstall existing vpp installation until
481 # start-testcase script is updated on all virl servers
482 rpm_pkgs_remove = "vpp*"
483 cmd_u = 'yum -y remove "{0}"'.format(rpm_pkgs_remove)
484 r_rcode, _, r_err = ssh.exec_command_sudo(cmd_u, timeout=90)
485 if int(r_rcode) != 0:
486 raise RuntimeError('Failed to remove previous VPP'
487 'installation on host {0}:\n{1}'
488 .format(node['host']), r_err)
490 rpm_pkgs = "*.rpm ".join(str(vpp_pkg_dir + pkg)
491 for pkg in vpp_rpm_pkgs) + "*.rpm"
492 cmd_i = "rpm -ivh {0}".format(rpm_pkgs)
493 ret_code, _, err = ssh.exec_command_sudo(cmd_i, timeout=90)
494 if int(ret_code) != 0:
495 raise RuntimeError('Failed to install VPP on host {0}:'
496 '\n{1}'.format(node['host']), err)
498 ssh.exec_command_sudo("rpm -qai vpp*")
499 logger.info("VPP installed on node {0}".
500 format(node['host']))
502 # workaroud - uninstall existing vpp installation until
503 # start-testcase script is updated on all virl servers
504 deb_pkgs_remove = "vpp*"
505 cmd_u = 'apt-get purge -y "{0}"'.format(deb_pkgs_remove)
506 r_rcode, _, r_err = ssh.exec_command_sudo(cmd_u, timeout=90)
507 if int(r_rcode) != 0:
508 raise RuntimeError('Failed to remove previous VPP'
509 'installation on host {0}:\n{1}'
510 .format(node['host']), r_err)
511 deb_pkgs = "*.deb ".join(str(vpp_pkg_dir + pkg)
512 for pkg in vpp_deb_pkgs) + "*.deb"
513 cmd_i = "dpkg -i --force-all {0}".format(deb_pkgs)
514 ret_code, _, err = ssh.exec_command_sudo(cmd_i, timeout=90)
515 if int(ret_code) != 0:
516 raise RuntimeError('Failed to install VPP on host {0}:'
517 '\n{1}'.format(node['host']), err)
519 ssh.exec_command_sudo("dpkg -l | grep vpp")
520 logger.info("VPP installed on node {0}".
521 format(node['host']))
526 def verify_vpp_on_all_duts(nodes):
527 """Verify that VPP is installed on all DUT nodes.
529 :param nodes: Nodes in the topology.
533 logger.debug("Verify VPP on all DUTs")
535 DUTSetup.start_vpp_service_on_all_duts(nodes)
537 for node in nodes.values():
538 if node['type'] == NodeType.DUT:
539 DUTSetup.verify_vpp_on_dut(node)
542 def verify_vpp_on_dut(node):
543 """Verify that VPP is installed on DUT node.
545 :param node: DUT node.
547 :raises: RuntimeError if failed to restart VPP, get VPP version or
551 logger.debug("Verify VPP on node {0}".format(node['host']))
553 DUTSetup.vpp_show_version_verbose(node)
554 DUTSetup.vpp_show_interfaces(node)