--- /dev/null
+HOWTO (will create a wiki page once one is created for CSIT project):
+ - copy Vagrantfile to separate dir on host
+ - vagrant up --parallel
+ sit-back-and-relax
+ - from VM that has access to the same host-only network (192.168.255.0 above)
+ - copy your ssh-key to csit@192.168.255.{101,102,250}
+ - cd ${csit_dir}
+ - virtualenv & pip as in README
+ - PYTHONPATH=`pwd` resources/tools/topology/update_topology.py \
+ topologies/available/vagrant.yaml \
+ -o topologies/available/vagrant_pci.yaml
+ - PYTHONPATH=`pwd` pybot -L TRACE \
+ -v TOPOLOGY_PATH:topologies/available/vagrant_pci.yaml -s \
+ "bridge domain" tests
+ - see tests results
+
# See the License for the specific language governing permissions and
# limitations under the License.
+"""This module exists to provide setup utilities for the framework on topology
+nodes. All tasks required to be run before the actual tests are started is
+supposed to end up here.
+"""
+
import shlex
from subprocess import Popen, PIPE, call
from multiprocessing import Pool
from tempfile import NamedTemporaryFile
from os.path import basename
+
from robot.api import logger
from robot.libraries.BuiltIn import BuiltIn
-from ssh import SSH
-from constants import Constants as con
-from topology import NodeType
-__all__ = ["SetupFramework"]
+from resources.libraries.python.ssh import SSH
+from resources.libraries.python.constants import Constants as con
+from resources.libraries.python.topology import NodeType
+__all__ = ["SetupFramework"]
def pack_framework_dir():
"""Pack the testing WS into temp file, return its name."""
def copy_tarball_to_node(tarball, node):
+ """Copy tarball file from local host to remote node.
+
+ :param tarball: path to tarball to upload
+ :param node: dictionary created from topology
+ :type tarball: string
+ :type node: dict
+ :return: nothing
+ """
logger.console('Copying tarball to {0}'.format(node['host']))
ssh = SSH()
ssh.connect(node)
def extract_tarball_at_node(tarball, node):
+ """Extract tarball at given node.
+
+ Extracts tarball using tar on given node to specific CSIT loocation.
+
+ :param tarball: path to tarball to upload
+ :param node: dictionary created from topology
+ :type tarball: string
+ :type node: dict
+ :return: nothing
+ """
logger.console('Extracting tarball to {0} on {1}'.format(
con.REMOTE_FW_DIR, node['host']))
ssh = SSH()
cmd = 'sudo rm -rf {1}; mkdir {1} ; tar -zxf {0} -C {1}; ' \
'rm -f {0}'.format(tarball, con.REMOTE_FW_DIR)
- (ret_code, stdout, stderr) = ssh.exec_command(cmd, timeout=30)
+ (ret_code, _, stderr) = ssh.exec_command(cmd, timeout=30)
if 0 != ret_code:
logger.error('Unpack error: {0}'.format(stderr))
raise Exception('Failed to unpack {0} at node {1}'.format(
ssh = SSH()
ssh.connect(node)
(ret_code, stdout, stderr) = ssh.exec_command(
- 'cd {0} && rm -rf env && virtualenv env && '
- '. env/bin/activate && '
- 'pip install -r requirements.txt'.format(con.REMOTE_FW_DIR), timeout=100)
+ 'cd {0} && rm -rf env && virtualenv env && . env/bin/activate && '
+ 'pip install -r requirements.txt'.format(con.REMOTE_FW_DIR),
+ timeout=100)
if 0 != ret_code:
logger.error('Virtualenv creation error: {0}'.format(stdout + stderr))
raise Exception('Virtualenv setup failed')
logger.console('Virtualenv created on {0}'.format(node['host']))
def setup_node(args):
+ """Run all set-up methods for a node.
+
+ This method is used as map_async parameter. It receives tuple with all
+ parameters as passed to map_async function.
+
+ :param args: all parameters needed to setup one node
+ :type args: tuple
+ :return: nothing
+ """
tarball, remote_tarball, node = args
copy_tarball_to_node(tarball, node)
extract_tarball_at_node(remote_tarball, node)
if node['type'] == NodeType.TG:
create_env_directory_at_node(node)
-
+ logger.console('Setup of node {0} done'.format(node['host']))
def delete_local_tarball(tarball):
- call(shlex.split('sh -c "rm {0} > /dev/null 2>&1"'.format(tarball)))
+ """Delete local tarball to prevent disk pollution.
+ :param tarball: path to tarball to upload
+ :type tarball: string
+ :return: nothing
+ """
+ call(shlex.split('sh -c "rm {0} > /dev/null 2>&1"'.format(tarball)))
-class SetupFramework(object):
+class SetupFramework(object): # pylint: disable=too-few-public-methods
"""Setup suite run on topology nodes.
Many VAT/CLI based tests need the scripts at remote hosts before executing
def __init__(self):
pass
- def setup_framework(self, nodes):
+ @staticmethod
+ def setup_framework(nodes):
"""Pack the whole directory and extract in temp on each node."""
tarball = pack_framework_dir()
BuiltIn().set_log_level(log_level)
logger.trace('Test framework copied to all topology nodes')
delete_local_tarball(tarball)
+ logger.console('All nodes are ready')
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+import socket
import paramiko
from paramiko import RSAKey
import StringIO
self._ssh.get_transport().getpeername(), end-start))
stdout = ""
- while True:
- buf = chan.recv(self.__MAX_RECV_BUF)
- stdout += buf
- if not buf:
- break
+ try:
+ while True:
+ buf = chan.recv(self.__MAX_RECV_BUF)
+ stdout += buf
+ if not buf:
+ break
+ except socket.timeout:
+ logger.error('Caught timeout exception, current contents '
+ 'of buffer: {0}'.format(stdout))
+ raise
+
stderr = ""
while True:
--- /dev/null
+#!/usr/bin/env python2.7
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This executable python module gathers MAC address data from topology nodes.
+It requires that all interfaces/port elements in topology have driver field.
+This script binds the port in given node to set linux kernel driver and
+extracts MAC address from it."""
+
+import sys
+import os
+import re
+from argparse import ArgumentParser
+
+import yaml
+
+from resources.libraries.python.ssh import SSH
+
+def load_topology(args):
+ """Load topology file referenced to by parameter passed to this script.
+
+ :param args: arguments parsed from commandline
+ :type args: ArgumentParser().parse_args()
+ :return: Python representation of topology yaml
+ :rtype: dict
+ """
+ data = None
+ with open(args.topology, 'r') as stream:
+ try:
+ data = yaml.load(stream)
+ except yaml.YAMLError as exc:
+ print 'Failed to load topology file: {0}'.format(args.topology)
+ print exc
+ raise
+
+ return data
+
+def ssh_no_error(ssh, cmd):
+ """Execute a command over ssh channel, and log and exit if the command
+ fials.
+
+ :param ssh: SSH() object connected to a node
+ :param cmd: Command line to execute on remote node
+ :type ssh: SSH() object
+ :type cmd: str
+ :return: stdout from the SSH command
+ :rtype: str
+ """
+ ret, stdo, stde = ssh.exec_command(cmd)
+ if 0 != ret:
+ print 'Command execution failed: "{}"'.format(cmd)
+ print 'stdout: {0}'.format(stdo)
+ print 'stderr: {0}'.format(stde)
+ raise RuntimeError('Unexpected ssh command failure')
+
+ return stdo
+
+def update_mac_addresses_for_node(node):
+ """For given node loop over all ports with PCI address and look for its MAC
+ address.
+
+ This function firstly unbinds the PCI device from its current driver
+ and binds it to linux kernel driver. After the device is bound to specific
+ linux kernel driver the MAC address is extracted from /sys/bus/pci location
+ and stored within the node dictionary that was passed to this function.
+ :param node: Node from topology
+ :type node: dict
+ :return: None
+ """
+ for port_name, port in node['interfaces'].items():
+ if not port.has_key('driver'):
+ err_msg = '{0} port {1} has no driver element, exiting'.format(
+ node['host'], port_name)
+ raise RuntimeError(err_msg)
+
+ ssh = SSH()
+ ssh.connect(node)
+
+ # TODO: make following SSH commands into one-liner to save on SSH opers
+
+ # First unbind from current driver
+ drvr_dir_path = '/sys/bus/pci/devices/{0}/driver'.format(
+ port['pci_address'])
+ cmd = '''\
+ if [ -d {0} ]; then
+ echo {1} | sudo tee {0}/unbind ;
+ else
+ true Do not have to do anything, port already unbound ;
+ fi'''.format(drvr_dir_path, port['pci_address'])
+ ssh_no_error(ssh, cmd)
+
+ # Then bind to the 'driver' from topology for given port
+ cmd = 'echo {0} | sudo tee /sys/bus/pci/drivers/{1}/bind'.\
+ format(port['pci_address'], port['driver'])
+ ssh_no_error(ssh, cmd)
+
+ # Then extract the mac address and store it in the topology
+ cmd = 'cat /sys/bus/pci/devices/{0}/net/*/address'.format(
+ port['pci_address'])
+ mac = ssh_no_error(ssh, cmd).strip()
+ pattern = re.compile("^([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}$")
+ if not pattern.match(mac):
+ raise RuntimeError('MAC address read from host {0} {1} is in '
+ 'bad format "{2}"'.format(node['host'],
+ port['pci_address'], mac))
+ print '{0}: Found MAC address of PCI device {1}: {2}'.format(
+ node['host'], port['pci_address'], mac)
+ port['mac_address'] = mac
+
+def update_nodes_mac_addresses(topology):
+ """Loop over nodes in topology and get mac addresses for all listed ports
+ based on PCI addresses.
+
+ :param topology: Topology information with nodes
+ :type topology: dict
+ :return: None
+ """
+
+ for node in topology['nodes'].values():
+ update_mac_addresses_for_node(node)
+
+def dump_updated_topology(topology, args):
+ """Writes or prints out updated topology file.
+
+ :param topology: Topology information with nodes
+ :param args: arguments parsed from command line
+ :type topology: dict
+ :type args: ArgumentParser().parse_args()
+ :return: 1 if error occured, 0 if successful
+ :rtype: int
+ """
+
+ if args.output_file:
+ if not args.force:
+ if os.path.isfile(args.output_file):
+ print ('File {0} already exists. If you want to overwrite this '
+ 'file, add -f as a parameter to this script'.format(
+ args.output_file))
+ return 1
+ with open(args.output_file, 'w') as stream:
+ yaml.dump(topology, stream, default_flow_style=False)
+ else:
+ print yaml.dump(topology, default_flow_style=False)
+ return 0
+
+def main():
+ """Main function"""
+ parser = ArgumentParser()
+ parser.add_argument('topology', help="Topology yaml file to read")
+ parser.add_argument('--output-file', '-o', help='Output file')
+ parser.add_argument('-f', '--force', help='Overwrite existing file',
+ action='store_const', const=True)
+ parser.add_argument('--verbose', '-v', action='store_true')
+ args = parser.parse_args()
+
+ topology = load_topology(args)
+ update_nodes_mac_addresses(topology)
+ ret = dump_updated_topology(topology, args)
+
+ return ret
+
+
+if __name__ == "__main__":
+ sys.exit(main())
+
+
--- /dev/null
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# -*- mode: ruby -*-
+# vi: set ts=2 sw=2 sts=2 et ft=ruby :
+
+$user_addition = <<-SHELL
+ sudo deluser csit
+ sudo adduser --disabled-password --gecos "" csit
+ echo csit:csit | sudo chpasswd
+ sudo adduser csit vagrant
+ id csit
+SHELL
+
+$install_prereqs = <<-SHELL
+ sudo apt-get -y update
+ sudo apt-get -y -f install
+ sudo apt-get -y install python-virtualenv python-dev iproute2 debhelper dkms
+ sudo update-alternatives --install /bin/sh sh /bin/bash 100
+SHELL
+
+$install_vpp = <<-SHELL
+ sudo apt-get -y purge vpp\*
+ cd /vagrant
+ if [ -e /vagrant/vpp-*.deb ]; then
+ sudo dpkg -i vpp-*.deb
+ fi
+SHELL
+
+
+def add_dut(config, name, mgmt_ip, net1, net2)
+ config.vm.define name do |node|
+ node.vm.box = "puppetlabs/ubuntu-14.04-64-nocm"
+ node.vm.hostname = name
+ node.vm.provision "shell", inline: $user_addition
+ node.vm.provision "shell", inline: $install_prereqs
+ node.vm.provision "shell", inline: $install_vpp
+
+ node.vm.network "private_network", ip: mgmt_ip
+ node.vm.network "private_network", type: "dhcp", auto_config: false,
+ virtualbox__intnet: net1
+ node.vm.network "private_network", type: "dhcp", auto_config: false,
+ virtualbox__intnet: net2
+ node.vm.provider "virtualbox" do |vb|
+ vb.memory = "2048"
+ vb.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"]
+ vb.customize ["modifyvm", :id, "--nicpromisc4", "allow-all"]
+ end
+ end
+
+end
+
+Vagrant.configure(2) do |config|
+ config.vm.define "tg" do |tg|
+ tg.vm.box = "puppetlabs/ubuntu-14.04-64-nocm"
+ tg.vm.hostname = "tg"
+
+ tg.vm.provision "shell", inline: $user_addition
+ tg.vm.provision "shell", inline: $install_prereqs
+ tg.vm.network "private_network", ip: '192.168.255.100/24'
+ tg.vm.network "private_network", type: "dhcp", auto_config: false,
+ virtualbox__intnet: "tg_dut1"
+ tg.vm.network "private_network", type: "dhcp", auto_config: false,
+ virtualbox__intnet: "tg_dut2"
+ tg.vm.provider "virtualbox" do |vb|
+ vb.memory = "2048"
+ vb.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"]
+ vb.customize ["modifyvm", :id, "--nicpromisc4", "allow-all"]
+ end
+
+ end
+
+ add_dut(config, "dut1", "192.168.255.101/24", "tg_dut1", "dut1_dut2")
+ add_dut(config, "dut2", "192.168.255.102/24", "tg_dut2", "dut1_dut2")
+end
+
--- /dev/null
+#!/bin/bash
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -x
+
+USERNAME=csit
+
+function ssh_do_duts {
+ ssh ${USERNAME}@192.168.255.101 ${@} || exit
+ ssh ${USERNAME}@192.168.255.102 ${@} || exit
+}
+
+rsync -avz ${@} ${USERNAME}@192.168.255.101:/tmp/ || exit
+rsync -avz ${@} ${USERNAME}@192.168.255.102:/tmp/ || exit
+
+ssh_do_duts "sudo apt-get -y purge 'vpp.*' ; exit 0"
+ssh_do_duts "sudo dpkg -i /tmp/vpp*.deb"
+ssh_do_duts "echo 128 | sudo tee /proc/sys/vm/nr_hugepages"
+ssh_do_duts "sudo rm -f /etc/vpp/startup.conf.orig ; sudo cp /etc/vpp/startup.conf /etc/vpp/startup.conf.orig"
+ssh_do_duts "sudo rm /etc/vpp/startup.conf"
+ssh_do_duts "sudo sed -e 's/socket-mem [0-9]*/socket-mem 128/' /etc/vpp/startup.conf.orig | sudo tee /etc/vpp/startup.conf"
+ssh_do_duts "echo heapsize 512M | sudo tee -a /etc/vpp/startup.conf"
+ssh_do_duts "sudo sed -e 's/vm.nr_hugepages=.*/vm.nr_hugepages=128/' -i /etc/sysctl.d/80-vpp.conf"
+ssh_do_duts "sudo sed -e 's/vm.max_map_count=.*/vm.max_map_count=256/' -i /etc/sysctl.d/80-vpp.conf"
+
+
+echo Success!
--- /dev/null
+---
+metadata:
+ version: 0.1
+ schema:
+ - resources/topology_schemas/3_node_topology.sch.yaml
+ - resources/topology_schemas/topology.sch.yaml
+ tags: [vagrant, 3-node]
+
+nodes:
+ TG:
+ type: TG
+ host: "192.168.255.100"
+ port: 22
+ username: csit
+ password: csit
+ interfaces:
+ port3:
+ mac_address: ""
+ pci_address: "0000:00:09.0"
+ link: link1
+ driver: e1000
+ port5:
+ mac_address: ""
+ pci_address: "0000:00:0a.0"
+ link: link2
+ driver: e1000
+ DUT1:
+ type: DUT
+ host: "192.168.255.101"
+ port: 22
+ username: csit
+ password: csit
+ interfaces:
+ port1:
+ mac_address: ""
+ pci_address: "0000:00:09.0"
+ link: link1
+ driver: e1000
+ port3:
+ mac_address: ""
+ pci_address: "0000:00:0a.0"
+ link: link3
+ driver: e1000
+ DUT2:
+ type: DUT
+ host: "192.168.255.102"
+ port: 22
+ username: csit
+ password: csit
+ interfaces:
+ port1:
+ mac_address: ""
+ pci_address: "0000:00:09.0"
+ link: link2
+ driver: e1000
+ port3:
+ mac_address: ""
+ pci_address: "0000:00:0a.0"
+ link: link3
+ driver: e1000
Code Review / csit.git / commitdiff