--- /dev/null
+#!/usr/bin/env python
+
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This is a helper script to make test execution easy."""
+
+from __future__ import print_function
+import sys
+import os
+import time
+from string import ascii_lowercase
+from random import sample
+import argparse
+from pykwalify.core import Core
+from pykwalify.errors import PyKwalifyException
+from yaml import load
+import robot
+from robot.errors import DATA_ERROR, DataError, FRAMEWORK_ERROR, FrameworkError
+from robot.run import RobotFramework
+from robot.conf.settings import RobotSettings
+from robot.running.builder import TestSuiteBuilder
+from robot.running.model import TestSuite
+
+TOPOLOGIES_DIR = './topologies/enabled/'
+TESTS_DIR = './tests'
+OUTPUTS_DIR = './outputs'
+
+
+def get_suite_list(*datasources, **options):
+ """Returns filtered test suites based on include exclude tags
+
+ :param datasources: paths to tests
+ :param options: Robot Framework options (robot.conf.settings.py)
+ :return: list of Robot Framework TestSuites which contain tests
+ """
+ class _MyRobotFramework(RobotFramework):
+ """Custom implementation of RobotFramework main()."""
+ def main(self, datasources, **options):
+ # copied from robot.run.RobotFramework.main
+ settings = RobotSettings(options)
+ test_suite = TestSuiteBuilder(settings['SuiteNames'],
+ settings['WarnOnSkipped'])
+ # pylint: disable=star-args
+ suite = test_suite.build(*datasources)
+ suite.configure(**settings.suite_config)
+
+ return suite
+
+ # get all test cases list without run tests, execute runs overloaded main
+ # function
+ suite = _MyRobotFramework().execute(*datasources, output=None, dryrun=True,
+ **options)
+ if isinstance(suite, TestSuite):
+ suites = []
+ suites.append(suite)
+ append_new = True
+ while append_new:
+ append_new = False
+ tmp = []
+ for suite in suites:
+ # pylint: disable=protected-access
+ if len(suite.suites._items) > 0:
+ for i in suite.suites._items:
+ tmp.append(i)
+ append_new = True
+ else:
+ tmp.append(suite)
+ suites = tmp
+ return suites
+ # TODO: check testcases Tags ? all tests should have same set of tags
+ else:
+ if suite == DATA_ERROR:
+ raise DataError
+ if suite == FRAMEWORK_ERROR:
+ raise FrameworkError
+ return []
+
+
+def run_suites(tests_dir, suites, output_dir, output_prefix='suite',
+ **options):
+ """Execute RF's run with parameters."""
+
+ with open('{}/{}.out'.format(output_dir, output_prefix), 'w') as out:
+ robot.run(tests_dir,
+ suite=[s.longname for s in suites],
+ output='{}/{}.xml'.format(output_dir, output_prefix),
+ debugfile='{}/{}.log'.format(output_dir, output_prefix),
+ log=None,
+ report=None,
+ stdout=out,
+ **options)
+
+
+def parse_outputs(output_dir):
+ """Parse output xmls from all executed tests."""
+
+ outs = [os.path.join(output_dir, file_name)
+ for file_name in os.listdir(output_dir)
+ if file_name.endswith('.xml')]
+ # pylint: disable=star-args
+ robot.rebot(*outs, merge=True)
+
+
+def topology_lookup(topology_paths, topo_dir, validate):
+ """Make topology list and validate topologies against schema
+
+ :param parsed_args: topology list, is empty then scans topologies in
+ topo_dir
+ :param topo_dir: scan directory for topologies
+ :param validate: if True then validate topology
+ :return: list of topologies
+ """
+
+ ret_topologies = []
+ if topology_paths:
+ for topo in topology_paths:
+ if os.path.exists(topo):
+ ret_topologies.append(topo)
+ else:
+ print("Topology file {} doesn't exist".format(topo),
+ file=sys.stderr)
+ else:
+ ret_topologies = [os.path.join(topo_dir, file_name)
+ for file_name in os.listdir(topo_dir)
+ if file_name.lower().endswith('.yaml')]
+
+ if len(ret_topologies) == 0:
+ print('No valid topology found', file=sys.stderr)
+ exit(1)
+
+ # validate topologies against schema
+ exit_on_error = False
+ for topology_name in ret_topologies:
+ try:
+ with open(topology_name) as file_name:
+ yaml_obj = load(file_name)
+ core = Core(source_file=topology_name,
+ schema_files=yaml_obj["metadata"]["schema"])
+ core.validate()
+ except PyKwalifyException as ex:
+ print('Unable to verify topology {}, schema error: {}'.\
+ format(topology_name, ex),
+ file=sys.stderr)
+ exit_on_error = True
+ except KeyError as ex:
+ print('Unable to verify topology {}, key error: {}'.\
+ format(topology_name, ex),
+ file=sys.stderr)
+ exit_on_error = True
+ except Exception as ex:
+ print('Unable to verify topology {}, {}'.format(topology_name, ex),
+ file=sys.stderr)
+ exit_on_error = True
+
+ if exit_on_error and validate:
+ exit(1)
+
+ return ret_topologies
+
+
+def main():
+ """Main function."""
+ parser = argparse.ArgumentParser(description='A test runner')
+ parser.add_argument('-i', '--include', action='append',
+ help='include tests with tag')
+ parser.add_argument('-e', '--exclude', action='append',
+ help='exclude tests with tag')
+ parser.add_argument('-s', '--suite', action='append',
+ help='full name of suite to run')
+ parser.add_argument('-t', '--topology', action='append',
+ help='topology where tests should be run')
+ parser.add_argument('-d', '--test_dir', nargs='?', default=TESTS_DIR,
+ help='where tests are stored')
+ parser.add_argument('-o', '--output_dir', nargs='?', default=OUTPUTS_DIR,
+ help='where results are stored')
+ parser.add_argument('-L', '--loglevel', nargs='?', default='INFO', type=str,
+ choices=['TRACE', 'DEBUG', 'INFO', 'WARN', 'NONE'],
+ help='robot frameworks level for logging')
+ parser.add_argument('-n', '--no_validate', action="store_false",
+ help='Do not exit if topology validation failed')
+
+ args = parser.parse_args()
+
+ i = args.include or []
+ excl = args.exclude or []
+ suite_filter = args.suite or []
+ test_dir = args.test_dir
+
+ # prepare output subdir
+ suite_output_dir = os.path.join(args.output_dir,
+ time.strftime('%y%m%d%H%M%S'))
+ os.makedirs(suite_output_dir)
+
+ topologies = topology_lookup(args.topology, TOPOLOGIES_DIR,
+ args.no_validate)
+ suite_list = get_suite_list(test_dir, include=i, exclude=excl,
+ suite=suite_filter)
+
+ # TODO: do the topology suite mapping magic
+ # for now all tests on single topology
+ if len(topologies) > 1:
+ print('Multiple topologies unsupported yet', file=sys.stderr)
+ exit(1)
+ topology_suite_mapping = {topologies[0]: suite_list}
+
+ # on all topologies, run test
+ # TODO: run parallel
+ for topology_path, topology_suite_list in topology_suite_mapping.items():
+ topology_path_variable = 'TOPOLOGY_PATH:{}'.format(topology_path)
+ variables = [topology_path_variable]
+ print('Runing tests on topology {}'.format(topology_path))
+ run_suites(test_dir, topology_suite_list, variable=variables,
+ output_dir=suite_output_dir,
+ output_prefix=''.join(sample(ascii_lowercase, 5)),
+ include=i, exclude=excl, loglevel=args.loglevel)
+
+ print('Parsing test results')
+ parse_outputs(suite_output_dir)
+
+
+if __name__ == "__main__":
+ main()
Code Review / csit.git / blobdiff