Make space for upcoming test framework.
Change-Id: I14da6cf95c645d8ee2b71579a658dc7ef3b9f027
Signed-off-by: Damjan Marion <damarion@cisco.com>
+++ /dev/null
-/env
-/outputs
-/output.xml
-/log.html
-/report.html
-*.pyc
+++ /dev/null
-# STEPS TO START DEVELOPING TESTS LOCALLY
- - install virtualenv
- - generate environment using virtualenv:
- # cd $ROOT
- # virtualenv env
- # source env/bin/activate
- - install python requirements for this project by executing:
- # pip install -r requirements.txt
- - make sure user mentioned in topology.py has NOPASSWD sudo access to
- vpp_api_test
-
-
- Done.
-
-# STEPS TO START THE TESTS
-export PYTHONPATH=.
-pybot -L TRACE -V resources/libraries/python/topology.py tests
+++ /dev/null
-#!/usr/bin/env python
-# Copyright (c) 2015 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-import os
-import robot
-from robot.run import RobotFramework
-from robot.conf.settings import RobotSettings
-from robot.running.builder import TestSuiteBuilder
-from robot.running.model import TestSuite
-
-
-def get_suite_list(*datasources, **options):
- class _MyRobotFramework(RobotFramework):
- def main(self, datasources, **options):
- # copied from robot.run.RobotFramework.main
- settings = RobotSettings(options)
- suite = TestSuiteBuilder(settings['SuiteNames'],
- settings['WarnOnSkipped']).build(*datasources)
- suite.configure(**settings.suite_config)
-
- return suite
-
- # Options are in robot.conf.settings
- suite = _MyRobotFramework().execute(*datasources, **options)
- if isinstance(suite, TestSuite):
- suites = []
- suites.append(suite)
- append_new = True
- while append_new:
- append_new = False
- tmp = []
- for s in suites:
- if len(s.suites._items) > 0:
- for i in s.suites._items:
- tmp.append(i)
- append_new = True
- else:
- tmp.append(s)
- suites = tmp
- return suites
- else:
- # TODO: add from robot.errors typ error
- return []
-
-
-def run_suites(test_dir, suites, out_dir="./outputs", **options):
- # TODO: add logic
-
- try:
- for f in os.listdir(out_dir):
- os.remove('/'.join((out_dir, f)))
- except OSError:
- pass
- if not os.path.exists(out_dir):
- os.makedirs(out_dir)
-
- for s in suites:
- longname = s.longname
- varfile=[]
- varfile.append('resources/libraries/python/topology.py')
-
- # TODO: check testcases Tags
-
- with open('{}/{}.out'.format(out_dir, longname), 'w') as out, \
- open('{}/{}.log'.format(out_dir, longname), 'w') as debug:
- robot.run(test_dir,
- suite=[longname],
- output='{}/{}.xml'.format(out_dir, longname),
- debugfile=debug,
- log=None,
- report=None,
- stdout=out,
- variablefile=varfile,
- **options)
-
-
-def parse_outputs(out_dir='./'):
- outs = ['/'.join((out_dir, file)) for file in os.listdir(out_dir) if file.endswith('.xml')]
- robot.rebot(*outs, merge=True)
-
-
-if __name__ == "__main__":
- i = []
- e = []
- # i = ['bd', 'ip']
- # i = ['hw']
- # e = ['hw']
- test_dir = "./tests"
- out_dir = "./outputs"
-
- suite_list = get_suite_list(test_dir, include=i, exclude=e, output=None, dryrun=True)
- run_suites(test_dir, suite_list, include=i, exclude=e, out_dir=out_dir)
- parse_outputs(out_dir=out_dir)
+++ /dev/null
-robotframework
-paramiko
-scp
+++ /dev/null
-#!/bin/bash
-
-echo vpp process
-ps aux | grep vpp
-
-echo Free memory
-free -m
-
-echo List vpp packages
-dpkg -l \*vpp\*
-
-
-echo List /proc/meminfo
-cat /proc/meminfo
-
-
+++ /dev/null
-# Copyright (c) 2015 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-from robot.api import logger
-from topology import NodeType
-from ssh import SSH
-
-class DUTSetup(object):
-
- def __init__(self):
- pass
-
- def setup_all_duts(self, nodes):
- """Prepare all DUTs in given topology for test execution."""
- for node in nodes.values():
- if node['type'] == NodeType.DUT:
- self.setup_dut(node)
-
- def setup_dut(self, node):
- ssh = SSH()
- ssh.connect(node)
-
- ssh.scp('resources/libraries/bash/dut_setup.sh', '/tmp/dut_setup.sh')
- (ret_code, stdout, stderr) = \
- ssh.exec_command('sudo -Sn bash /tmp/dut_setup.sh')
- logger.trace(stdout)
- if 0 != int(ret_code):
- logger.error('DUT {0} setup script failed: "{1}"'.
- format(node['host'], stdout + stderr))
- raise Exception('DUT test setup script failed at node {}'.
- format(node['host']))
+++ /dev/null
-# Copyright (c) 2015 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import shlex
-from ssh import SSH
-from subprocess import Popen, PIPE, call
-from tempfile import NamedTemporaryFile
-from os.path import basename
-from constants import Constants as con
-from robot.api import logger
-
-__all__ = ["SetupFramework"]
-
-class SetupFramework(object):
- """Setup suite run on topology nodes.
-
- Many VAT/CLI based tests need the scripts at remote hosts before executing
- them. This class packs the whole testing directory and copies it over
- to all nodes in topology under /tmp/
-
- """
-
- def __init__(self):
- pass
-
- def __pack_framework_dir(self):
- """Pack the testing WS into temp file, return its name."""
-
- tmpfile = NamedTemporaryFile(suffix=".tgz", prefix="openvpp-testing-")
- file_name = tmpfile.name
- tmpfile.close()
-
- proc = Popen(shlex.split("tar -zcf {0} .".format(file_name)),
- stdout=PIPE, stderr=PIPE)
- (stdout, stderr) = proc.communicate()
-
- logger.debug(stdout)
- logger.debug(stderr)
-
- return_code = proc.wait()
- if 0 != return_code:
- raise Exception("Could not pack testing framework.")
-
- return file_name
-
- def __copy_tarball_to_node(self, tarball, node):
- logger.console('Copying tarball to {0}'.format(node['host']))
- ssh = SSH()
- ssh.connect(node)
-
- ssh.scp(tarball, "/tmp/")
-
- def __extract_tarball_at_node(self, tarball, node):
- logger.console('Extracting tarball to {0} on {1}'.format(
- con.REMOTE_FW_DIR, node['host']))
- ssh = SSH()
- ssh.connect(node)
-
- cmd = 'rm -rf {1}; mkdir {1} ; sudo -Sn tar -zxf {0} -C {1};'.format(
- tarball, con.REMOTE_FW_DIR)
- (ret_code, stdout, stderr) = ssh.exec_command(cmd, timeout=30)
- if 0 != ret_code:
- logger.error('Unpack error: {0}'.format(stderr))
- raise Exception('Failed to unpack {0} at node {1}'.format(
- tarball, node['host']))
-
- def __delete_local_tarball(self, tarball):
- call(shlex.split('sh -c "rm {0} > /dev/null 2>&1"'.format(tarball)))
-
- def setup_framework(self, nodes):
- """Pack the whole directory and extract in temp on each node."""
-
- tarball = self.__pack_framework_dir()
- logger.console('Framework packed to {0}'.format(tarball))
- remote_tarball = "/tmp/{0}".format(basename(tarball))
-
- for node in nodes.values():
- self.__copy_tarball_to_node(tarball, node)
- self.__extract_tarball_at_node(remote_tarball, node)
-
- logger.trace('Test framework copied to all topology nodes')
- self.__delete_local_tarball(tarball)
-
+++ /dev/null
-# Copyright (c) 2015 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import os
-from ssh import SSH
-from robot.api import logger
-
-__all__ = []
-
-class VatExecutor(object):
-
- __TMP_DIR = "/tmp/"
- __VAT_BIN = "vpp_api_test"
-
- def __init__(self):
- self._stdout = None
- self._stderr = None
- self._ret_code = None
-
- def execute_script(self, local_path, node, timeout=10, json_out=True):
- """Copy local_path script to node, execute it and return result.
-
- Returns (rc, stdout, stderr tuple).
- """
-
- ssh = SSH()
- ssh.connect(node)
-
- local_basename = os.path.basename(local_path)
- remote_file_path = self.__TMP_DIR + local_basename
- remote_file_out = remote_file_path + ".out"
-
- ssh.scp(local_path, remote_file_path)
-
- cmd = "sudo -S {vat} {json} < {input}".format(vat=self.__VAT_BIN,
- json="json" if json_out == True else "",
- input=remote_file_path)
- (ret_code, stdout, stderr) = ssh.exec_command(cmd, timeout)
- self._ret_code = ret_code
- self._stdout = stdout
- self._stderr = stderr
-
- logger.trace("Command '{0}' returned {1}'".format(cmd, self._ret_code))
- logger.trace("stdout: '{0}'".format(self._stdout))
- logger.trace("stderr: '{0}'".format(self._stderr))
-
- #TODO: download vpp_api_test output file
- self._delete_files(node, remote_file_path, remote_file_out)
-
- def _delete_files(self, node, *files):
- ssh = SSH()
- ssh.connect(node)
- files = " ".join([str(x) for x in files])
- ssh.exec_command("rm {0}".format(files))
-
- def script_should_have_failed(self):
- if self._ret_code is None:
- raise Exception("First execute the script!")
- if self._ret_code == 0:
- raise AssertionError(
- "Script execution passed, but failure was expected")
-
- def script_should_have_passed(self):
- if self._ret_code is None:
- raise Exception("First execute the script!")
- if self._ret_code != 0:
- raise AssertionError(
- "Script execution failed, but success was expected")
-
- def get_script_stdout(self):
- return self._stdout
-
- def get_script_stderr(self):
- return self._stderr
-
+++ /dev/null
-# Copyright (c) 2015 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-class Constants(object):
- #OpenVPP testing directory location at topology nodes
- REMOTE_FW_DIR = '/tmp/openvpp-testing'
+++ /dev/null
-# Copyright (c) 2015 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import paramiko
-from scp import SCPClient
-from time import time
-from robot.api import logger
-
-__all__ = ["exec_cmd"]
-
-# TODO: Attempt to recycle SSH connections
-# TODO: load priv key
-
-class SSH(object):
-
- __MAX_RECV_BUF = 10*1024*1024
- __existing_connections = {}
-
- def __init__(self):
- self._ssh = paramiko.SSHClient()
- self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
- self._hostname = None
-
- def _node_hash(self, node):
- return hash(frozenset([node['host'], node['port']]))
-
- def connect(self, node):
- """Connect to node prior to running exec_command or scp.
-
- If there already is a connection to the node, this method reuses it.
- """
- self._hostname = node['host']
- node_hash = self._node_hash(node)
- if node_hash in self.__existing_connections:
- self._ssh = self.__existing_connections[node_hash]
- else:
- start = time()
- self._ssh.connect(node['host'], username=node['username'],
- password=node['password'])
- self.__existing_connections[node_hash] = self._ssh
- logger.trace('connect took {} seconds'.format(time() - start))
-
- def exec_command(self, cmd, timeout=10):
- """Execute SSH command on a new channel on the connected Node.
-
- Returns (return_code, stdout, stderr).
- """
- start = time()
- chan = self._ssh.get_transport().open_session()
- if timeout is not None:
- chan.settimeout(int(timeout))
- chan.exec_command(cmd)
- end = time()
- logger.trace('exec_command "{0}" on {1} took {2} seconds'.format(cmd,
- self._hostname, end-start))
-
-
- stdout = ""
- while True:
- buf = chan.recv(self.__MAX_RECV_BUF)
- stdout += buf
- if not buf:
- break
-
- stderr = ""
- while True:
- buf = chan.recv_stderr(self.__MAX_RECV_BUF)
- stderr += buf
- if not buf:
- break
-
- return_code = chan.recv_exit_status()
- logger.trace('chan_recv/_stderr took {} seconds'.format(time()-end))
-
- return (return_code, stdout, stderr)
-
- def scp(self, local_path, remote_path):
- """Copy files from local_path to remote_path.
-
- connect() method has to be called first!
- """
- logger.trace('SCP {0} to {1}:{2}'.format(
- local_path, self._hostname, remote_path))
- # SCPCLient takes a paramiko transport as its only argument
- scp = SCPClient(self._ssh.get_transport())
- start = time()
- scp.put(local_path, remote_path)
- scp.close()
- end = time()
- logger.trace('SCP took {0} seconds'.format(end-start))
-
-def exec_cmd(node, cmd, timeout=None):
- """Convenience function to ssh/exec/return rc & out.
-
- Returns (rc, stdout).
- """
- if node is None:
- raise TypeError('Node parameter is None')
- if cmd is None:
- raise TypeError('Command parameter is None')
- if len(cmd) == 0:
- raise ValueError('Empty command parameter')
-
- ssh = SSH()
- try:
- ssh.connect(node)
- except Exception, e:
- logger.error("Failed to connect to node" + e)
- return None
-
- try:
- (ret_code, stdout, stderr) = ssh.exec_command(cmd, timeout=timeout)
- except Exception, e:
- logger.error(e)
- return None
-
- return (ret_code, stdout, stderr)
-
+++ /dev/null
-# Copyright (c) 2015 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#Defines nodes and topology structure.
-
-__all__ = ["DICT__nodes"]
-
-
-class NodeType(object):
- DUT = 'DUT'
- TG = 'TG'
-
-MOCK_DATA_FOR_NOW = {
- 'nodes' : {
- 'DUT1' : {
- 'type' : NodeType.DUT,
- 'host' : 'wasa-ucs-14',
- 'port' : 22,
- 'username' : '',
- 'password' : '',
- },
- 'DUT2' : {
- 'type' : NodeType.DUT,
- 'host' : 'wasa-ucs-13',
- 'port' : 22,
- 'username' : '',
- 'password' : '',
- },
- 'TG' : {
- 'type' : NodeType.TG,
- 'host' : 'wasa-ucs-12',
- 'port' : 22,
- 'username' : '',
- 'password' : '',
- },
- }
- }
-
-DICT__nodes = MOCK_DATA_FOR_NOW['nodes']
-
+++ /dev/null
-# Copyright (c) 2015 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*** Settings ***
-| Library | resources/libraries/python/DUTSetup.py
-
-*** Keywords ***
-| Setup all DUTs before test
-| | [Documentation] | Setup all DUTs in topology before test execution
-| | Setup All DUTs | ${nodes}
-
+++ /dev/null
-# Copyright (c) 2015 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*** Settings ***
-| Resource | resources/libraries/robot/vat/interfaces.robot
-
-*** Keywords ***
-| VPP reports interfaces on | [Arguments] | ${node}
-| | VPP reports interfaces through VAT on | ${node}
-#| | VPP reports interfaces through ODL on | ${node}
-#| | VPP reports interfaces through DEBUGCLI on | ${node}
+++ /dev/null
-# Copyright (c) 2015 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*** Settings ***
-| Library | resources/libraries/python/VatExecutor.py
-
-*** Variables ***
-| ${VAT_DUMP_INTERFACES} | resources/templates/vat/dump_interfaces.vat
-
-*** Keywords ***
-| VPP reports interfaces through VAT on
-| | [Arguments] | ${node}
-| | Execute Script | ${VAT_DUMP_INTERFACES} | ${node}
-| | Script Should Have Passed
+++ /dev/null
-sw_interface_dump
-dump_interface_table
-quit
+++ /dev/null
-# Copyright (c) 2015 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*** Settings ***
-| Library | resources/libraries/python/SetupFramework.py
-| Suite Setup | Setup Framework | ${nodes}
+++ /dev/null
-# Copyright (c) 2015 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*** Settings ***
-| Resource | resources/libraries/robot/default.robot
-| Resource | resources/libraries/robot/interfaces.robot
-| Test Setup | Setup all DUTs before test
-
-*** Test Cases ***
-| VPP reports interfaces
-| | VPP reports interfaces on | ${nodes['DUT1']}
-
-
-| Loop the test
-| | : FOR | ${INDEX} | in range | 10
-| | | VPP reports interfaces on | ${nodes['DUT1']}
-
+++ /dev/null
-# Copyright (c) 2015 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*** Settings ***
-| Documentation | Dummy test suite to test suite execution
+++ /dev/null
-# Copyright (c) 2015 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*** Settings ***
-| Force Tags | vhost-user | hw
-
-*** Test Cases ***
-| Create vhost user interface on hw
-| | Log | interface created
+++ /dev/null
-# Copyright (c) 2015 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*** Settings ***
-| Force Tags | vhost-user | virl
-
-*** Test Cases ***
-| Create vhost user interface on virl
-| | Log | interface created
Code Review / vpp.git / commitdiff