1 # Copyright (c) 2018 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Data pre-processing
16 - extract data from output.xml files generated by Jenkins jobs and store in
18 - provide access to the data.
19 - filter the data using tags,
22 import multiprocessing
28 from robot.api import ExecutionResult, ResultVisitor
29 from robot import errors
30 from collections import OrderedDict
31 from string import replace
33 from jumpavg.AvgStdevMetadataFactory import AvgStdevMetadataFactory
35 from input_data_files import download_and_unzip_data_file
36 from utils import Worker
39 class ExecutionChecker(ResultVisitor):
40 """Class to traverse through the test suite structure.
42 The functionality implemented in this class generates a json structure:
48 "generated": "Timestamp",
49 "version": "SUT version",
50 "job": "Jenkins job name",
51 "build": "Information about the build"
54 "Suite long name 1": {
56 "doc": "Suite 1 documentation",
57 "parent": "Suite 1 parent",
58 "level": "Level of the suite in the suite hierarchy"
60 "Suite long name N": {
62 "doc": "Suite N documentation",
63 "parent": "Suite 2 parent",
64 "level": "Level of the suite in the suite hierarchy"
70 "parent": "Name of the parent of the test",
71 "doc": "Test documentation"
73 "tags": ["tag 1", "tag 2", "tag n"],
74 "type": "PDR" | "NDR" | "TCP" | "MRR" | "BMRR",
75 "throughput": { # Only type: "PDR" | "NDR"
77 "unit": "pps" | "bps" | "percentage"
79 "latency": { # Only type: "PDR" | "NDR"
86 "50": { # Only for NDR
91 "10": { # Only for NDR
103 "50": { # Only for NDR
108 "10": { # Only for NDR
115 "result": { # Only type: "TCP"
117 "unit": "cps" | "rps"
119 "result": { # Only type: "MRR" | "BMRR"
120 "receive-rate": AvgStdevMetadata,
122 "lossTolerance": "lossTolerance", # Only type: "PDR"
123 "vat-history": "DUT1 and DUT2 VAT History"
124 "show-run": "Show Run"
136 "metadata": { # Optional
137 "version": "VPP version",
138 "job": "Jenkins job name",
139 "build": "Information about the build"
143 "doc": "Suite 1 documentation",
144 "parent": "Suite 1 parent",
145 "level": "Level of the suite in the suite hierarchy"
148 "doc": "Suite N documentation",
149 "parent": "Suite 2 parent",
150 "level": "Level of the suite in the suite hierarchy"
156 "parent": "Name of the parent of the test",
157 "doc": "Test documentation"
158 "msg": "Test message"
159 "tags": ["tag 1", "tag 2", "tag n"],
160 "vat-history": "DUT1 and DUT2 VAT History"
161 "show-run": "Show Run"
162 "status": "PASS" | "FAIL"
170 .. note:: ID is the lowercase full path to the test.
173 REGEX_RATE = re.compile(r'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)\s(\w+)')
175 REGEX_LAT_NDR = re.compile(r'^[\D\d]*'
176 r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
177 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
178 r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
179 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
180 r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
181 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]')
183 REGEX_LAT_PDR = re.compile(r'^[\D\d]*'
184 r'LAT_\d+%PDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
185 r'\s\'(-?\d+/-?\d+/-?\d+)\'\][\D\d]*')
187 REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s'
190 REGEX_VERSION_VPP = re.compile(r"(return STDOUT Version:\s*)(.*)")
192 REGEX_VERSION_DPDK = re.compile(r"(return STDOUT testpmd)([\d\D\n]*)"
193 r"(RTE Version: 'DPDK )(.*)(')")
195 REGEX_TCP = re.compile(r'Total\s(rps|cps|throughput):\s([0-9]*).*$')
197 REGEX_MRR = re.compile(r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
198 r'tx\s(\d*),\srx\s(\d*)')
200 REGEX_BMRR = re.compile(r'Maximum Receive Rate trial results'
201 r' in packets per second: \[(.*)\]')
203 REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
205 REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
207 REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
209 def __init__(self, metadata):
212 :param metadata: Key-value pairs to be included in "metadata" part of
217 # Type of message to parse out from the test messages
218 self._msg_type = None
224 self._timestamp = None
226 # Number of VAT History messages found:
228 # 1 - VAT History of DUT1
229 # 2 - VAT History of DUT2
230 self._lookup_kw_nr = 0
231 self._vat_history_lookup_nr = 0
233 # Number of Show Running messages found
235 # 1 - Show run message found
236 self._show_run_lookup_nr = 0
238 # Test ID of currently processed test- the lowercase full path to the
242 # The main data structure
244 "metadata": OrderedDict(),
245 "suites": OrderedDict(),
246 "tests": OrderedDict()
249 # Save the provided metadata
250 for key, val in metadata.items():
251 self._data["metadata"][key] = val
253 # Dictionary defining the methods used to parse different types of
256 "timestamp": self._get_timestamp,
257 "vpp-version": self._get_vpp_version,
258 "dpdk-version": self._get_dpdk_version,
259 "teardown-vat-history": self._get_vat_history,
260 "test-show-runtime": self._get_show_run
265 """Getter - Data parsed from the XML file.
267 :returns: Data parsed from the XML file.
272 def _get_vpp_version(self, msg):
273 """Called when extraction of VPP version is required.
275 :param msg: Message to process.
280 if msg.message.count("return STDOUT Version:"):
281 self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
283 self._data["metadata"]["version"] = self._version
284 self._msg_type = None
286 def _get_dpdk_version(self, msg):
287 """Called when extraction of DPDK version is required.
289 :param msg: Message to process.
294 if msg.message.count("return STDOUT testpmd"):
296 self._version = str(re.search(
297 self.REGEX_VERSION_DPDK, msg.message). group(4))
298 self._data["metadata"]["version"] = self._version
302 self._msg_type = None
304 def _get_timestamp(self, msg):
305 """Called when extraction of timestamp is required.
307 :param msg: Message to process.
312 self._timestamp = msg.timestamp[:14]
313 self._data["metadata"]["generated"] = self._timestamp
314 self._msg_type = None
316 def _get_vat_history(self, msg):
317 """Called when extraction of VAT command history is required.
319 :param msg: Message to process.
323 if msg.message.count("VAT command history:"):
324 self._vat_history_lookup_nr += 1
325 if self._vat_history_lookup_nr == 1:
326 self._data["tests"][self._test_ID]["vat-history"] = str()
328 self._msg_type = None
329 text = re.sub("[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3} "
330 "VAT command history:", "", msg.message, count=1). \
331 replace("\n\n", "\n").replace('\n', ' |br| ').\
332 replace('\r', '').replace('"', "'")
334 self._data["tests"][self._test_ID]["vat-history"] += " |br| "
335 self._data["tests"][self._test_ID]["vat-history"] += \
336 "**DUT" + str(self._vat_history_lookup_nr) + ":** " + text
338 def _get_show_run(self, msg):
339 """Called when extraction of VPP operational data (output of CLI command
340 Show Runtime) is required.
342 :param msg: Message to process.
346 if msg.message.count("return STDOUT Thread "):
347 self._show_run_lookup_nr += 1
348 if self._lookup_kw_nr == 1 and self._show_run_lookup_nr == 1:
349 self._data["tests"][self._test_ID]["show-run"] = str()
350 if self._lookup_kw_nr > 1:
351 self._msg_type = None
352 if self._show_run_lookup_nr == 1:
353 text = msg.message.replace("vat# ", "").\
354 replace("return STDOUT ", "").replace("\n\n", "\n").\
355 replace('\n', ' |br| ').\
356 replace('\r', '').replace('"', "'")
358 self._data["tests"][self._test_ID]["show-run"] += " |br| "
359 self._data["tests"][self._test_ID]["show-run"] += \
360 "**DUT" + str(self._lookup_kw_nr) + ":** |br| " + text
364 def _get_latency(self, msg, test_type):
365 """Get the latency data from the test message.
367 :param msg: Message to be parsed.
368 :param test_type: Type of the test - NDR or PDR.
371 :returns: Latencies parsed from the message.
375 if test_type == "NDR":
376 groups = re.search(self.REGEX_LAT_NDR, msg)
377 groups_range = range(1, 7)
378 elif test_type == "PDR":
379 groups = re.search(self.REGEX_LAT_PDR, msg)
380 groups_range = range(1, 3)
385 for idx in groups_range:
387 lat = [int(item) for item in str(groups.group(idx)).split('/')]
388 except (AttributeError, ValueError):
390 latencies.append(lat)
392 keys = ("min", "avg", "max")
400 latency["direction1"]["100"] = dict(zip(keys, latencies[0]))
401 latency["direction2"]["100"] = dict(zip(keys, latencies[1]))
402 if test_type == "NDR":
403 latency["direction1"]["50"] = dict(zip(keys, latencies[2]))
404 latency["direction2"]["50"] = dict(zip(keys, latencies[3]))
405 latency["direction1"]["10"] = dict(zip(keys, latencies[4]))
406 latency["direction2"]["10"] = dict(zip(keys, latencies[5]))
410 def visit_suite(self, suite):
411 """Implements traversing through the suite and its direct children.
413 :param suite: Suite to process.
417 if self.start_suite(suite) is not False:
418 suite.suites.visit(self)
419 suite.tests.visit(self)
420 self.end_suite(suite)
422 def start_suite(self, suite):
423 """Called when suite starts.
425 :param suite: Suite to process.
431 parent_name = suite.parent.name
432 except AttributeError:
435 doc_str = suite.doc.replace('"', "'").replace('\n', ' ').\
436 replace('\r', '').replace('*[', ' |br| *[').replace("*", "**")
437 doc_str = replace(doc_str, ' |br| *[', '*[', maxreplace=1)
439 self._data["suites"][suite.longname.lower().replace('"', "'").
440 replace(" ", "_")] = {
441 "name": suite.name.lower(),
443 "parent": parent_name,
444 "level": len(suite.longname.split("."))
447 suite.keywords.visit(self)
449 def end_suite(self, suite):
450 """Called when suite ends.
452 :param suite: Suite to process.
458 def visit_test(self, test):
459 """Implements traversing through the test.
461 :param test: Test to process.
465 if self.start_test(test) is not False:
466 test.keywords.visit(self)
469 def start_test(self, test):
470 """Called when test starts.
472 :param test: Test to process.
477 tags = [str(tag) for tag in test.tags]
479 test_result["name"] = test.name.lower()
480 test_result["parent"] = test.parent.name.lower()
481 test_result["tags"] = tags
482 doc_str = test.doc.replace('"', "'").replace('\n', ' '). \
483 replace('\r', '').replace('[', ' |br| [')
484 test_result["doc"] = replace(doc_str, ' |br| [', '[', maxreplace=1)
485 test_result["msg"] = test.message.replace('\n', ' |br| '). \
486 replace('\r', '').replace('"', "'")
487 test_result["status"] = test.status
488 self._test_ID = test.longname.lower()
489 if test.status == "PASS" and ("NDRPDRDISC" in tags or
493 if "NDRDISC" in tags:
495 elif "PDRDISC" in tags:
501 elif "FRMOBL" in tags or "BMRR" in tags:
506 test_result["type"] = test_type
508 # Replace info about cores (e.g. -1c-) with the info about threads
509 # and cores (e.g. -1t1c-) in the long test case names and in the
510 # test case names if necessary.
511 groups = re.search(self.REGEX_TC_NAME_OLD, self._test_ID)
514 for tag in test_result["tags"]:
515 groups = re.search(self.REGEX_TC_TAG, tag)
521 self._test_ID = re.sub(self.REGEX_TC_NAME_NEW,
522 "-{0}-".format(tag_tc.lower()),
525 test_result["name"] = re.sub(self.REGEX_TC_NAME_NEW,
526 "-{0}-".format(tag_tc.lower()),
530 test_result["status"] = "FAIL"
531 logging.error("The test '{0}' has no or more than one "
532 "multi-threading tags.".format(self._test_ID))
535 if test_type in ("NDR", "PDR"):
537 rate_value = str(re.search(
538 self.REGEX_RATE, test.message).group(1))
539 except AttributeError:
542 rate_unit = str(re.search(
543 self.REGEX_RATE, test.message).group(2))
544 except AttributeError:
547 test_result["throughput"] = dict()
548 test_result["throughput"]["value"] = \
549 int(rate_value.split('.')[0])
550 test_result["throughput"]["unit"] = rate_unit
551 test_result["latency"] = \
552 self._get_latency(test.message, test_type)
553 if test_type == "PDR":
554 test_result["lossTolerance"] = str(re.search(
555 self.REGEX_TOLERANCE, test.message).group(1))
557 elif test_type in ("TCP", ):
558 groups = re.search(self.REGEX_TCP, test.message)
559 test_result["result"] = dict()
560 test_result["result"]["value"] = int(groups.group(2))
561 test_result["result"]["unit"] = groups.group(1)
563 elif test_type in ("MRR", "BMRR"):
564 test_result["result"] = dict()
565 groups = re.search(self.REGEX_BMRR, test.message)
566 if groups is not None:
567 items_str = groups.group(1)
568 items_float = [float(item.strip()) for item
569 in items_str.split(",")]
570 test_result["result"]["receive-rate"] = \
571 AvgStdevMetadataFactory.from_data(items_float)
573 groups = re.search(self.REGEX_MRR, test.message)
574 test_result["result"]["receive-rate"] = \
575 AvgStdevMetadataFactory.from_data([
576 float(groups.group(3)) / float(groups.group(1)), ])
578 self._data["tests"][self._test_ID] = test_result
580 def end_test(self, test):
581 """Called when test ends.
583 :param test: Test to process.
589 def visit_keyword(self, keyword):
590 """Implements traversing through the keyword and its child keywords.
592 :param keyword: Keyword to process.
593 :type keyword: Keyword
596 if self.start_keyword(keyword) is not False:
597 self.end_keyword(keyword)
599 def start_keyword(self, keyword):
600 """Called when keyword starts. Default implementation does nothing.
602 :param keyword: Keyword to process.
603 :type keyword: Keyword
607 if keyword.type == "setup":
608 self.visit_setup_kw(keyword)
609 elif keyword.type == "teardown":
610 self._lookup_kw_nr = 0
611 self.visit_teardown_kw(keyword)
613 self._lookup_kw_nr = 0
614 self.visit_test_kw(keyword)
615 except AttributeError:
618 def end_keyword(self, keyword):
619 """Called when keyword ends. Default implementation does nothing.
621 :param keyword: Keyword to process.
622 :type keyword: Keyword
627 def visit_test_kw(self, test_kw):
628 """Implements traversing through the test keyword and its child
631 :param test_kw: Keyword to process.
632 :type test_kw: Keyword
635 for keyword in test_kw.keywords:
636 if self.start_test_kw(keyword) is not False:
637 self.visit_test_kw(keyword)
638 self.end_test_kw(keyword)
640 def start_test_kw(self, test_kw):
641 """Called when test keyword starts. Default implementation does
644 :param test_kw: Keyword to process.
645 :type test_kw: Keyword
648 if test_kw.name.count("Show Runtime Counters On All Duts"):
649 self._lookup_kw_nr += 1
650 self._show_run_lookup_nr = 0
651 self._msg_type = "test-show-runtime"
652 elif test_kw.name.count("Start The L2fwd Test") and not self._version:
653 self._msg_type = "dpdk-version"
656 test_kw.messages.visit(self)
658 def end_test_kw(self, test_kw):
659 """Called when keyword ends. Default implementation does nothing.
661 :param test_kw: Keyword to process.
662 :type test_kw: Keyword
667 def visit_setup_kw(self, setup_kw):
668 """Implements traversing through the teardown keyword and its child
671 :param setup_kw: Keyword to process.
672 :type setup_kw: Keyword
675 for keyword in setup_kw.keywords:
676 if self.start_setup_kw(keyword) is not False:
677 self.visit_setup_kw(keyword)
678 self.end_setup_kw(keyword)
680 def start_setup_kw(self, setup_kw):
681 """Called when teardown keyword starts. Default implementation does
684 :param setup_kw: Keyword to process.
685 :type setup_kw: Keyword
688 if setup_kw.name.count("Show Vpp Version On All Duts") \
689 and not self._version:
690 self._msg_type = "vpp-version"
692 elif setup_kw.name.count("Setup performance global Variables") \
693 and not self._timestamp:
694 self._msg_type = "timestamp"
697 setup_kw.messages.visit(self)
699 def end_setup_kw(self, setup_kw):
700 """Called when keyword ends. Default implementation does nothing.
702 :param setup_kw: Keyword to process.
703 :type setup_kw: Keyword
708 def visit_teardown_kw(self, teardown_kw):
709 """Implements traversing through the teardown keyword and its child
712 :param teardown_kw: Keyword to process.
713 :type teardown_kw: Keyword
716 for keyword in teardown_kw.keywords:
717 if self.start_teardown_kw(keyword) is not False:
718 self.visit_teardown_kw(keyword)
719 self.end_teardown_kw(keyword)
721 def start_teardown_kw(self, teardown_kw):
722 """Called when teardown keyword starts. Default implementation does
725 :param teardown_kw: Keyword to process.
726 :type teardown_kw: Keyword
730 if teardown_kw.name.count("Show Vat History On All Duts"):
731 self._vat_history_lookup_nr = 0
732 self._msg_type = "teardown-vat-history"
733 teardown_kw.messages.visit(self)
735 def end_teardown_kw(self, teardown_kw):
736 """Called when keyword ends. Default implementation does nothing.
738 :param teardown_kw: Keyword to process.
739 :type teardown_kw: Keyword
744 def visit_message(self, msg):
745 """Implements visiting the message.
747 :param msg: Message to process.
751 if self.start_message(msg) is not False:
752 self.end_message(msg)
754 def start_message(self, msg):
755 """Called when message starts. Get required information from messages:
758 :param msg: Message to process.
764 self.parse_msg[self._msg_type](msg)
766 def end_message(self, msg):
767 """Called when message ends. Default implementation does nothing.
769 :param msg: Message to process.
776 class InputData(object):
779 The data is extracted from output.xml files generated by Jenkins jobs and
780 stored in pandas' DataFrames.
786 (as described in ExecutionChecker documentation)
788 (as described in ExecutionChecker documentation)
790 (as described in ExecutionChecker documentation)
793 def __init__(self, spec):
796 :param spec: Specification.
797 :type spec: Specification
804 self._input_data = pd.Series()
808 """Getter - Input data.
811 :rtype: pandas.Series
813 return self._input_data
815 def metadata(self, job, build):
818 :param job: Job which metadata we want.
819 :param build: Build which metadata we want.
823 :rtype: pandas.Series
826 return self.data[job][build]["metadata"]
828 def suites(self, job, build):
831 :param job: Job which suites we want.
832 :param build: Build which suites we want.
836 :rtype: pandas.Series
839 return self.data[job][str(build)]["suites"]
841 def tests(self, job, build):
844 :param job: Job which tests we want.
845 :param build: Build which tests we want.
849 :rtype: pandas.Series
852 return self.data[job][build]["tests"]
855 def _parse_tests(job, build, log):
856 """Process data from robot output.xml file and return JSON structured
859 :param job: The name of job which build output data will be processed.
860 :param build: The build which output data will be processed.
861 :param log: List of log messages.
864 :type log: list of tuples (severity, msg)
865 :returns: JSON data structure.
874 with open(build["file-name"], 'r') as data_file:
876 result = ExecutionResult(data_file)
877 except errors.DataError as err:
878 log.append(("ERROR", "Error occurred while parsing output.xml: "
881 checker = ExecutionChecker(metadata)
882 result.visit(checker)
886 def _download_and_parse_build(self, pid, data_queue, job, build, repeat):
887 """Download and parse the input data file.
889 :param pid: PID of the process executing this method.
890 :param data_queue: Shared memory between processes. Queue which keeps
891 the result data. This data is then read by the main process and used
892 in further processing.
893 :param job: Name of the Jenkins job which generated the processed input
895 :param build: Information about the Jenkins build which generated the
896 processed input file.
897 :param repeat: Repeat the download specified number of times if not
900 :type data_queue: multiprocessing.Manager().Queue()
908 logging.info(" Processing the job/build: {0}: {1}".
909 format(job, build["build"]))
911 logs.append(("INFO", " Processing the job/build: {0}: {1}".
912 format(job, build["build"])))
919 success = download_and_unzip_data_file(self._cfg, job, build, pid,
925 logs.append(("ERROR", "It is not possible to download the input "
926 "data file from the job '{job}', build "
927 "'{build}', or it is damaged. Skipped.".
928 format(job=job, build=build["build"])))
930 logs.append(("INFO", " Processing data from the build '{0}' ...".
931 format(build["build"])))
932 data = InputData._parse_tests(job, build, logs)
934 logs.append(("ERROR", "Input data file from the job '{job}', "
935 "build '{build}' is damaged. Skipped.".
936 format(job=job, build=build["build"])))
941 remove(build["file-name"])
942 except OSError as err:
943 logs.append(("ERROR", "Cannot remove the file '{0}': {1}".
944 format(build["file-name"], err)))
945 logs.append(("INFO", " Done."))
954 data_queue.put(result)
956 def download_and_parse_data(self, repeat=1):
957 """Download the input data files, parse input data from input files and
958 store in pandas' Series.
960 :param repeat: Repeat the download specified number of times if not
965 logging.info("Downloading and parsing input files ...")
967 work_queue = multiprocessing.JoinableQueue()
968 manager = multiprocessing.Manager()
969 data_queue = manager.Queue()
970 cpus = multiprocessing.cpu_count()
973 for cpu in range(cpus):
974 worker = Worker(work_queue,
976 self._download_and_parse_build)
979 workers.append(worker)
980 os.system("taskset -p -c {0} {1} > /dev/null 2>&1".
981 format(cpu, worker.pid))
983 for job, builds in self._cfg.builds.items():
985 work_queue.put((job, build, repeat))
989 logging.info("Done.")
991 while not data_queue.empty():
992 result = data_queue.get()
995 build_nr = result["build"]["build"]
998 data = result["data"]
999 build_data = pd.Series({
1000 "metadata": pd.Series(data["metadata"].values(),
1001 index=data["metadata"].keys()),
1002 "suites": pd.Series(data["suites"].values(),
1003 index=data["suites"].keys()),
1004 "tests": pd.Series(data["tests"].values(),
1005 index=data["tests"].keys())})
1007 if self._input_data.get(job, None) is None:
1008 self._input_data[job] = pd.Series()
1009 self._input_data[job][str(build_nr)] = build_data
1011 self._cfg.set_input_file_name(job, build_nr,
1012 result["build"]["file-name"])
1014 self._cfg.set_input_state(job, build_nr, result["state"])
1016 for item in result["logs"]:
1017 if item[0] == "INFO":
1018 logging.info(item[1])
1019 elif item[0] == "ERROR":
1020 logging.error(item[1])
1021 elif item[0] == "DEBUG":
1022 logging.debug(item[1])
1023 elif item[0] == "CRITICAL":
1024 logging.critical(item[1])
1025 elif item[0] == "WARNING":
1026 logging.warning(item[1])
1030 # Terminate all workers
1031 for worker in workers:
1035 logging.info("Done.")
1038 def _end_of_tag(tag_filter, start=0, closer="'"):
1039 """Return the index of character in the string which is the end of tag.
1041 :param tag_filter: The string where the end of tag is being searched.
1042 :param start: The index where the searching is stated.
1043 :param closer: The character which is the tag closer.
1044 :type tag_filter: str
1047 :returns: The index of the tag closer.
1052 idx_opener = tag_filter.index(closer, start)
1053 return tag_filter.index(closer, idx_opener + 1)
1058 def _condition(tag_filter):
1059 """Create a conditional statement from the given tag filter.
1061 :param tag_filter: Filter based on tags from the element specification.
1062 :type tag_filter: str
1063 :returns: Conditional statement which can be evaluated.
1069 index = InputData._end_of_tag(tag_filter, index)
1073 tag_filter = tag_filter[:index] + " in tags" + tag_filter[index:]
1075 def filter_data(self, element, params=None, data_set="tests",
1076 continue_on_error=False):
1077 """Filter required data from the given jobs and builds.
1079 The output data structure is:
1083 - test (or suite) 1 ID:
1089 - test (or suite) n ID:
1096 :param element: Element which will use the filtered data.
1097 :param params: Parameters which will be included in the output. If None,
1098 all parameters are included.
1099 :param data_set: The set of data to be filtered: tests, suites,
1101 :param continue_on_error: Continue if there is error while reading the
1102 data. The Item will be empty then
1103 :type element: pandas.Series
1106 :type continue_on_error: bool
1107 :returns: Filtered data.
1108 :rtype pandas.Series
1112 if element["filter"] in ("all", "template"):
1115 cond = InputData._condition(element["filter"])
1116 logging.debug(" Filter: {0}".format(cond))
1118 logging.error(" No filter defined.")
1122 params = element.get("parameters", None)
1126 for job, builds in element["data"].items():
1127 data[job] = pd.Series()
1128 for build in builds:
1129 data[job][str(build)] = pd.Series()
1131 data_iter = self.data[job][str(build)][data_set].\
1134 if continue_on_error:
1138 for test_ID, test_data in data_iter:
1139 if eval(cond, {"tags": test_data.get("tags", "")}):
1140 data[job][str(build)][test_ID] = pd.Series()
1142 for param, val in test_data.items():
1143 data[job][str(build)][test_ID][param] = val
1145 for param in params:
1147 data[job][str(build)][test_ID][param] =\
1150 data[job][str(build)][test_ID][param] =\
1154 except (KeyError, IndexError, ValueError) as err:
1155 logging.error(" Missing mandatory parameter in the element "
1156 "specification: {0}".format(err))
1158 except AttributeError:
1161 logging.error(" The filter '{0}' is not correct. Check if all "
1162 "tags are enclosed by apostrophes.".format(cond))
1166 def merge_data(data):
1167 """Merge data from more jobs and builds to a simple data structure.
1169 The output data structure is:
1171 - test (suite) 1 ID:
1177 - test (suite) n ID:
1180 :param data: Data to merge.
1181 :type data: pandas.Series
1182 :returns: Merged data.
1183 :rtype: pandas.Series
1186 logging.info(" Merging data ...")
1188 merged_data = pd.Series()
1189 for _, builds in data.iteritems():
1190 for _, item in builds.iteritems():
1191 for ID, item_data in item.iteritems():
1192 merged_data[ID] = item_data