1 # Copyright (c) 2018 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Data pre-processing
16 - extract data from output.xml files generated by Jenkins jobs and store in
18 - provide access to the data.
19 - filter the data using tags,
22 import multiprocessing
28 from robot.api import ExecutionResult, ResultVisitor
29 from robot import errors
30 from collections import OrderedDict
31 from string import replace
33 from jumpavg.AvgStdevMetadataFactory import AvgStdevMetadataFactory
35 from input_data_files import download_and_unzip_data_file
36 from utils import Worker
39 class ExecutionChecker(ResultVisitor):
40 """Class to traverse through the test suite structure.
42 The functionality implemented in this class generates a json structure:
48 "generated": "Timestamp",
49 "version": "SUT version",
50 "job": "Jenkins job name",
51 "build": "Information about the build"
54 "Suite long name 1": {
56 "doc": "Suite 1 documentation",
57 "parent": "Suite 1 parent",
58 "level": "Level of the suite in the suite hierarchy"
60 "Suite long name N": {
62 "doc": "Suite N documentation",
63 "parent": "Suite 2 parent",
64 "level": "Level of the suite in the suite hierarchy"
70 "parent": "Name of the parent of the test",
71 "doc": "Test documentation"
73 "tags": ["tag 1", "tag 2", "tag n"],
74 "type": "PDR" | "NDR" | "TCP" | "MRR" | "BMRR",
75 "throughput": { # Only type: "PDR" | "NDR"
77 "unit": "pps" | "bps" | "percentage"
79 "latency": { # Only type: "PDR" | "NDR"
86 "50": { # Only for NDR
91 "10": { # Only for NDR
103 "50": { # Only for NDR
108 "10": { # Only for NDR
115 "result": { # Only type: "TCP"
117 "unit": "cps" | "rps"
119 "result": { # Only type: "MRR" | "BMRR"
120 "receive-rate": AvgStdevMetadata,
122 "lossTolerance": "lossTolerance", # Only type: "PDR"
123 "vat-history": "DUT1 and DUT2 VAT History"
124 "show-run": "Show Run"
136 "metadata": { # Optional
137 "version": "VPP version",
138 "job": "Jenkins job name",
139 "build": "Information about the build"
143 "doc": "Suite 1 documentation",
144 "parent": "Suite 1 parent",
145 "level": "Level of the suite in the suite hierarchy"
148 "doc": "Suite N documentation",
149 "parent": "Suite 2 parent",
150 "level": "Level of the suite in the suite hierarchy"
156 "parent": "Name of the parent of the test",
157 "doc": "Test documentation"
158 "msg": "Test message"
159 "tags": ["tag 1", "tag 2", "tag n"],
160 "vat-history": "DUT1 and DUT2 VAT History"
161 "show-run": "Show Run"
162 "status": "PASS" | "FAIL"
170 .. note:: ID is the lowercase full path to the test.
173 REGEX_RATE = re.compile(r'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)\s(\w+)')
175 REGEX_LAT_NDR = re.compile(r'^[\D\d]*'
176 r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
177 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
178 r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
179 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
180 r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
181 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]')
183 REGEX_LAT_PDR = re.compile(r'^[\D\d]*'
184 r'LAT_\d+%PDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
185 r'\s\'(-?\d+/-?\d+/-?\d+)\'\][\D\d]*')
187 REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s'
190 REGEX_VERSION_VPP = re.compile(r"(return STDOUT Version:\s*)(.*)")
192 REGEX_VERSION_DPDK = re.compile(r"(return STDOUT testpmd)([\d\D\n]*)"
193 r"(RTE Version: 'DPDK )(.*)(')")
195 REGEX_TCP = re.compile(r'Total\s(rps|cps|throughput):\s([0-9]*).*$')
197 REGEX_MRR = re.compile(r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
198 r'tx\s(\d*),\srx\s(\d*)')
200 REGEX_BMRR = re.compile(r'Maximum Receive Rate Results \[(.*)\]')
202 REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
204 REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
206 REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
208 def __init__(self, metadata):
211 :param metadata: Key-value pairs to be included in "metadata" part of
216 # Type of message to parse out from the test messages
217 self._msg_type = None
223 self._timestamp = None
225 # Number of VAT History messages found:
227 # 1 - VAT History of DUT1
228 # 2 - VAT History of DUT2
229 self._lookup_kw_nr = 0
230 self._vat_history_lookup_nr = 0
232 # Number of Show Running messages found
234 # 1 - Show run message found
235 self._show_run_lookup_nr = 0
237 # Test ID of currently processed test- the lowercase full path to the
241 # The main data structure
243 "metadata": OrderedDict(),
244 "suites": OrderedDict(),
245 "tests": OrderedDict()
248 # Save the provided metadata
249 for key, val in metadata.items():
250 self._data["metadata"][key] = val
252 # Dictionary defining the methods used to parse different types of
255 "timestamp": self._get_timestamp,
256 "vpp-version": self._get_vpp_version,
257 "dpdk-version": self._get_dpdk_version,
258 "teardown-vat-history": self._get_vat_history,
259 "test-show-runtime": self._get_show_run
264 """Getter - Data parsed from the XML file.
266 :returns: Data parsed from the XML file.
271 def _get_vpp_version(self, msg):
272 """Called when extraction of VPP version is required.
274 :param msg: Message to process.
279 if msg.message.count("return STDOUT Version:"):
280 self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
282 self._data["metadata"]["version"] = self._version
283 self._msg_type = None
285 def _get_dpdk_version(self, msg):
286 """Called when extraction of DPDK version is required.
288 :param msg: Message to process.
293 if msg.message.count("return STDOUT testpmd"):
295 self._version = str(re.search(
296 self.REGEX_VERSION_DPDK, msg.message). group(4))
297 self._data["metadata"]["version"] = self._version
301 self._msg_type = None
303 def _get_timestamp(self, msg):
304 """Called when extraction of timestamp is required.
306 :param msg: Message to process.
311 self._timestamp = msg.timestamp[:14]
312 self._data["metadata"]["generated"] = self._timestamp
313 self._msg_type = None
315 def _get_vat_history(self, msg):
316 """Called when extraction of VAT command history is required.
318 :param msg: Message to process.
322 if msg.message.count("VAT command history:"):
323 self._vat_history_lookup_nr += 1
324 if self._vat_history_lookup_nr == 1:
325 self._data["tests"][self._test_ID]["vat-history"] = str()
327 self._msg_type = None
328 text = re.sub("[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3} "
329 "VAT command history:", "", msg.message, count=1). \
330 replace("\n\n", "\n").replace('\n', ' |br| ').\
331 replace('\r', '').replace('"', "'")
333 self._data["tests"][self._test_ID]["vat-history"] += " |br| "
334 self._data["tests"][self._test_ID]["vat-history"] += \
335 "**DUT" + str(self._vat_history_lookup_nr) + ":** " + text
337 def _get_show_run(self, msg):
338 """Called when extraction of VPP operational data (output of CLI command
339 Show Runtime) is required.
341 :param msg: Message to process.
345 if msg.message.count("return STDOUT Thread "):
346 self._show_run_lookup_nr += 1
347 if self._lookup_kw_nr == 1 and self._show_run_lookup_nr == 1:
348 self._data["tests"][self._test_ID]["show-run"] = str()
349 if self._lookup_kw_nr > 1:
350 self._msg_type = None
351 if self._show_run_lookup_nr == 1:
352 text = msg.message.replace("vat# ", "").\
353 replace("return STDOUT ", "").replace("\n\n", "\n").\
354 replace('\n', ' |br| ').\
355 replace('\r', '').replace('"', "'")
357 self._data["tests"][self._test_ID]["show-run"] += " |br| "
358 self._data["tests"][self._test_ID]["show-run"] += \
359 "**DUT" + str(self._lookup_kw_nr) + ":** |br| " + text
363 def _get_latency(self, msg, test_type):
364 """Get the latency data from the test message.
366 :param msg: Message to be parsed.
367 :param test_type: Type of the test - NDR or PDR.
370 :returns: Latencies parsed from the message.
374 if test_type == "NDR":
375 groups = re.search(self.REGEX_LAT_NDR, msg)
376 groups_range = range(1, 7)
377 elif test_type == "PDR":
378 groups = re.search(self.REGEX_LAT_PDR, msg)
379 groups_range = range(1, 3)
384 for idx in groups_range:
386 lat = [int(item) for item in str(groups.group(idx)).split('/')]
387 except (AttributeError, ValueError):
389 latencies.append(lat)
391 keys = ("min", "avg", "max")
399 latency["direction1"]["100"] = dict(zip(keys, latencies[0]))
400 latency["direction2"]["100"] = dict(zip(keys, latencies[1]))
401 if test_type == "NDR":
402 latency["direction1"]["50"] = dict(zip(keys, latencies[2]))
403 latency["direction2"]["50"] = dict(zip(keys, latencies[3]))
404 latency["direction1"]["10"] = dict(zip(keys, latencies[4]))
405 latency["direction2"]["10"] = dict(zip(keys, latencies[5]))
409 def visit_suite(self, suite):
410 """Implements traversing through the suite and its direct children.
412 :param suite: Suite to process.
416 if self.start_suite(suite) is not False:
417 suite.suites.visit(self)
418 suite.tests.visit(self)
419 self.end_suite(suite)
421 def start_suite(self, suite):
422 """Called when suite starts.
424 :param suite: Suite to process.
430 parent_name = suite.parent.name
431 except AttributeError:
434 doc_str = suite.doc.replace('"', "'").replace('\n', ' ').\
435 replace('\r', '').replace('*[', ' |br| *[').replace("*", "**")
436 doc_str = replace(doc_str, ' |br| *[', '*[', maxreplace=1)
438 self._data["suites"][suite.longname.lower().replace('"', "'").
439 replace(" ", "_")] = {
440 "name": suite.name.lower(),
442 "parent": parent_name,
443 "level": len(suite.longname.split("."))
446 suite.keywords.visit(self)
448 def end_suite(self, suite):
449 """Called when suite ends.
451 :param suite: Suite to process.
457 def visit_test(self, test):
458 """Implements traversing through the test.
460 :param test: Test to process.
464 if self.start_test(test) is not False:
465 test.keywords.visit(self)
468 def start_test(self, test):
469 """Called when test starts.
471 :param test: Test to process.
476 tags = [str(tag) for tag in test.tags]
478 test_result["name"] = test.name.lower()
479 test_result["parent"] = test.parent.name.lower()
480 test_result["tags"] = tags
481 doc_str = test.doc.replace('"', "'").replace('\n', ' '). \
482 replace('\r', '').replace('[', ' |br| [')
483 test_result["doc"] = replace(doc_str, ' |br| [', '[', maxreplace=1)
484 test_result["msg"] = test.message.replace('\n', ' |br| '). \
485 replace('\r', '').replace('"', "'")
486 test_result["status"] = test.status
487 if test.status == "PASS" and ("NDRPDRDISC" in tags or
491 if "NDRDISC" in tags:
493 elif "PDRDISC" in tags:
499 elif "FRMOBL" in tags or "BMRR" in tags:
504 test_result["type"] = test_type
506 if test_type in ("NDR", "PDR"):
508 rate_value = str(re.search(
509 self.REGEX_RATE, test.message).group(1))
510 except AttributeError:
513 rate_unit = str(re.search(
514 self.REGEX_RATE, test.message).group(2))
515 except AttributeError:
518 test_result["throughput"] = dict()
519 test_result["throughput"]["value"] = \
520 int(rate_value.split('.')[0])
521 test_result["throughput"]["unit"] = rate_unit
522 test_result["latency"] = \
523 self._get_latency(test.message, test_type)
524 if test_type == "PDR":
525 test_result["lossTolerance"] = str(re.search(
526 self.REGEX_TOLERANCE, test.message).group(1))
528 elif test_type in ("TCP", ):
529 groups = re.search(self.REGEX_TCP, test.message)
530 test_result["result"] = dict()
531 test_result["result"]["value"] = int(groups.group(2))
532 test_result["result"]["unit"] = groups.group(1)
534 elif test_type in ("MRR", "BMRR"):
535 test_result["result"] = dict()
536 groups = re.search(self.REGEX_BMRR, test.message)
537 if groups is not None:
538 items_str = groups.group(1)
539 items_float = [float(item.strip()) for item
540 in items_str.split(",")]
541 test_result["result"]["receive-rate"] = \
542 AvgStdevMetadataFactory.from_data(items_float)
544 groups = re.search(self.REGEX_MRR, test.message)
545 test_result["result"]["receive-rate"] = \
546 AvgStdevMetadataFactory.from_data([
547 float(groups.group(3)) / float(groups.group(1)), ])
549 self._test_ID = test.longname.lower()
550 self._data["tests"][self._test_ID] = test_result
552 def end_test(self, test):
553 """Called when test ends.
555 :param test: Test to process.
561 def visit_keyword(self, keyword):
562 """Implements traversing through the keyword and its child keywords.
564 :param keyword: Keyword to process.
565 :type keyword: Keyword
568 if self.start_keyword(keyword) is not False:
569 self.end_keyword(keyword)
571 def start_keyword(self, keyword):
572 """Called when keyword starts. Default implementation does nothing.
574 :param keyword: Keyword to process.
575 :type keyword: Keyword
579 if keyword.type == "setup":
580 self.visit_setup_kw(keyword)
581 elif keyword.type == "teardown":
582 self._lookup_kw_nr = 0
583 self.visit_teardown_kw(keyword)
585 self._lookup_kw_nr = 0
586 self.visit_test_kw(keyword)
587 except AttributeError:
590 def end_keyword(self, keyword):
591 """Called when keyword ends. Default implementation does nothing.
593 :param keyword: Keyword to process.
594 :type keyword: Keyword
599 def visit_test_kw(self, test_kw):
600 """Implements traversing through the test keyword and its child
603 :param test_kw: Keyword to process.
604 :type test_kw: Keyword
607 for keyword in test_kw.keywords:
608 if self.start_test_kw(keyword) is not False:
609 self.visit_test_kw(keyword)
610 self.end_test_kw(keyword)
612 def start_test_kw(self, test_kw):
613 """Called when test keyword starts. Default implementation does
616 :param test_kw: Keyword to process.
617 :type test_kw: Keyword
620 if test_kw.name.count("Show Runtime Counters On All Duts"):
621 self._lookup_kw_nr += 1
622 self._show_run_lookup_nr = 0
623 self._msg_type = "test-show-runtime"
624 elif test_kw.name.count("Start The L2fwd Test") and not self._version:
625 self._msg_type = "dpdk-version"
628 test_kw.messages.visit(self)
630 def end_test_kw(self, test_kw):
631 """Called when keyword ends. Default implementation does nothing.
633 :param test_kw: Keyword to process.
634 :type test_kw: Keyword
639 def visit_setup_kw(self, setup_kw):
640 """Implements traversing through the teardown keyword and its child
643 :param setup_kw: Keyword to process.
644 :type setup_kw: Keyword
647 for keyword in setup_kw.keywords:
648 if self.start_setup_kw(keyword) is not False:
649 self.visit_setup_kw(keyword)
650 self.end_setup_kw(keyword)
652 def start_setup_kw(self, setup_kw):
653 """Called when teardown keyword starts. Default implementation does
656 :param setup_kw: Keyword to process.
657 :type setup_kw: Keyword
660 if setup_kw.name.count("Show Vpp Version On All Duts") \
661 and not self._version:
662 self._msg_type = "vpp-version"
664 elif setup_kw.name.count("Setup performance global Variables") \
665 and not self._timestamp:
666 self._msg_type = "timestamp"
669 setup_kw.messages.visit(self)
671 def end_setup_kw(self, setup_kw):
672 """Called when keyword ends. Default implementation does nothing.
674 :param setup_kw: Keyword to process.
675 :type setup_kw: Keyword
680 def visit_teardown_kw(self, teardown_kw):
681 """Implements traversing through the teardown keyword and its child
684 :param teardown_kw: Keyword to process.
685 :type teardown_kw: Keyword
688 for keyword in teardown_kw.keywords:
689 if self.start_teardown_kw(keyword) is not False:
690 self.visit_teardown_kw(keyword)
691 self.end_teardown_kw(keyword)
693 def start_teardown_kw(self, teardown_kw):
694 """Called when teardown keyword starts. Default implementation does
697 :param teardown_kw: Keyword to process.
698 :type teardown_kw: Keyword
702 if teardown_kw.name.count("Show Vat History On All Duts"):
703 self._vat_history_lookup_nr = 0
704 self._msg_type = "teardown-vat-history"
705 teardown_kw.messages.visit(self)
707 def end_teardown_kw(self, teardown_kw):
708 """Called when keyword ends. Default implementation does nothing.
710 :param teardown_kw: Keyword to process.
711 :type teardown_kw: Keyword
716 def visit_message(self, msg):
717 """Implements visiting the message.
719 :param msg: Message to process.
723 if self.start_message(msg) is not False:
724 self.end_message(msg)
726 def start_message(self, msg):
727 """Called when message starts. Get required information from messages:
730 :param msg: Message to process.
736 self.parse_msg[self._msg_type](msg)
738 def end_message(self, msg):
739 """Called when message ends. Default implementation does nothing.
741 :param msg: Message to process.
748 class InputData(object):
751 The data is extracted from output.xml files generated by Jenkins jobs and
752 stored in pandas' DataFrames.
758 (as described in ExecutionChecker documentation)
760 (as described in ExecutionChecker documentation)
762 (as described in ExecutionChecker documentation)
765 def __init__(self, spec):
768 :param spec: Specification.
769 :type spec: Specification
776 self._input_data = pd.Series()
780 """Getter - Input data.
783 :rtype: pandas.Series
785 return self._input_data
787 def metadata(self, job, build):
790 :param job: Job which metadata we want.
791 :param build: Build which metadata we want.
795 :rtype: pandas.Series
798 return self.data[job][build]["metadata"]
800 def suites(self, job, build):
803 :param job: Job which suites we want.
804 :param build: Build which suites we want.
808 :rtype: pandas.Series
811 return self.data[job][str(build)]["suites"]
813 def tests(self, job, build):
816 :param job: Job which tests we want.
817 :param build: Build which tests we want.
821 :rtype: pandas.Series
824 return self.data[job][build]["tests"]
827 def _parse_tests(job, build, log):
828 """Process data from robot output.xml file and return JSON structured
831 :param job: The name of job which build output data will be processed.
832 :param build: The build which output data will be processed.
833 :param log: List of log messages.
836 :type log: list of tuples (severity, msg)
837 :returns: JSON data structure.
846 with open(build["file-name"], 'r') as data_file:
848 result = ExecutionResult(data_file)
849 except errors.DataError as err:
850 log.append(("ERROR", "Error occurred while parsing output.xml: "
853 checker = ExecutionChecker(metadata)
854 result.visit(checker)
858 def _download_and_parse_build(self, pid, data_queue, job, build, repeat):
859 """Download and parse the input data file.
861 :param pid: PID of the process executing this method.
862 :param data_queue: Shared memory between processes. Queue which keeps
863 the result data. This data is then read by the main process and used
864 in further processing.
865 :param job: Name of the Jenkins job which generated the processed input
867 :param build: Information about the Jenkins build which generated the
868 processed input file.
869 :param repeat: Repeat the download specified number of times if not
872 :type data_queue: multiprocessing.Manager().Queue()
880 logging.info(" Processing the job/build: {0}: {1}".
881 format(job, build["build"]))
883 logs.append(("INFO", " Processing the job/build: {0}: {1}".
884 format(job, build["build"])))
891 success = download_and_unzip_data_file(self._cfg, job, build, pid,
897 logs.append(("ERROR", "It is not possible to download the input "
898 "data file from the job '{job}', build "
899 "'{build}', or it is damaged. Skipped.".
900 format(job=job, build=build["build"])))
902 logs.append(("INFO", " Processing data from the build '{0}' ...".
903 format(build["build"])))
904 data = InputData._parse_tests(job, build, logs)
906 logs.append(("ERROR", "Input data file from the job '{job}', "
907 "build '{build}' is damaged. Skipped.".
908 format(job=job, build=build["build"])))
913 remove(build["file-name"])
914 except OSError as err:
915 logs.append(("ERROR", "Cannot remove the file '{0}': {1}".
916 format(build["file-name"], err)))
917 logs.append(("INFO", " Done."))
926 data_queue.put(result)
928 def download_and_parse_data(self, repeat=1):
929 """Download the input data files, parse input data from input files and
930 store in pandas' Series.
932 :param repeat: Repeat the download specified number of times if not
937 logging.info("Downloading and parsing input files ...")
939 work_queue = multiprocessing.JoinableQueue()
940 manager = multiprocessing.Manager()
941 data_queue = manager.Queue()
942 cpus = multiprocessing.cpu_count()
945 for cpu in range(cpus):
946 worker = Worker(work_queue,
948 self._download_and_parse_build)
951 workers.append(worker)
952 os.system("taskset -p -c {0} {1} > /dev/null 2>&1".
953 format(cpu, worker.pid))
955 for job, builds in self._cfg.builds.items():
957 work_queue.put((job, build, repeat))
961 logging.info("Done.")
963 while not data_queue.empty():
964 result = data_queue.get()
967 build_nr = result["build"]["build"]
970 data = result["data"]
971 build_data = pd.Series({
972 "metadata": pd.Series(data["metadata"].values(),
973 index=data["metadata"].keys()),
974 "suites": pd.Series(data["suites"].values(),
975 index=data["suites"].keys()),
976 "tests": pd.Series(data["tests"].values(),
977 index=data["tests"].keys())})
979 if self._input_data.get(job, None) is None:
980 self._input_data[job] = pd.Series()
981 self._input_data[job][str(build_nr)] = build_data
983 self._cfg.set_input_file_name(job, build_nr,
984 result["build"]["file-name"])
986 self._cfg.set_input_state(job, build_nr, result["state"])
988 for item in result["logs"]:
989 if item[0] == "INFO":
990 logging.info(item[1])
991 elif item[0] == "ERROR":
992 logging.error(item[1])
993 elif item[0] == "DEBUG":
994 logging.debug(item[1])
995 elif item[0] == "CRITICAL":
996 logging.critical(item[1])
997 elif item[0] == "WARNING":
998 logging.warning(item[1])
1002 # Terminate all workers
1003 for worker in workers:
1007 logging.info("Done.")
1010 def _end_of_tag(tag_filter, start=0, closer="'"):
1011 """Return the index of character in the string which is the end of tag.
1013 :param tag_filter: The string where the end of tag is being searched.
1014 :param start: The index where the searching is stated.
1015 :param closer: The character which is the tag closer.
1016 :type tag_filter: str
1019 :returns: The index of the tag closer.
1024 idx_opener = tag_filter.index(closer, start)
1025 return tag_filter.index(closer, idx_opener + 1)
1030 def _condition(tag_filter):
1031 """Create a conditional statement from the given tag filter.
1033 :param tag_filter: Filter based on tags from the element specification.
1034 :type tag_filter: str
1035 :returns: Conditional statement which can be evaluated.
1041 index = InputData._end_of_tag(tag_filter, index)
1045 tag_filter = tag_filter[:index] + " in tags" + tag_filter[index:]
1047 def filter_data(self, element, params=None, data_set="tests",
1048 continue_on_error=False):
1049 """Filter required data from the given jobs and builds.
1051 The output data structure is:
1055 - test (or suite) 1 ID:
1061 - test (or suite) n ID:
1068 :param element: Element which will use the filtered data.
1069 :param params: Parameters which will be included in the output. If None,
1070 all parameters are included.
1071 :param data_set: The set of data to be filtered: tests, suites,
1073 :param continue_on_error: Continue if there is error while reading the
1074 data. The Item will be empty then
1075 :type element: pandas.Series
1078 :type continue_on_error: bool
1079 :returns: Filtered data.
1080 :rtype pandas.Series
1084 if element["filter"] in ("all", "template"):
1087 cond = InputData._condition(element["filter"])
1088 logging.debug(" Filter: {0}".format(cond))
1090 logging.error(" No filter defined.")
1094 params = element.get("parameters", None)
1098 for job, builds in element["data"].items():
1099 data[job] = pd.Series()
1100 for build in builds:
1101 data[job][str(build)] = pd.Series()
1103 data_iter = self.data[job][str(build)][data_set].\
1106 if continue_on_error:
1110 for test_ID, test_data in data_iter:
1111 if eval(cond, {"tags": test_data.get("tags", "")}):
1112 data[job][str(build)][test_ID] = pd.Series()
1114 for param, val in test_data.items():
1115 data[job][str(build)][test_ID][param] = val
1117 for param in params:
1119 data[job][str(build)][test_ID][param] =\
1122 data[job][str(build)][test_ID][param] =\
1126 except (KeyError, IndexError, ValueError) as err:
1127 logging.error(" Missing mandatory parameter in the element "
1128 "specification: {0}".format(err))
1130 except AttributeError:
1133 logging.error(" The filter '{0}' is not correct. Check if all "
1134 "tags are enclosed by apostrophes.".format(cond))
1138 def merge_data(data):
1139 """Merge data from more jobs and builds to a simple data structure.
1141 The output data structure is:
1143 - test (suite) 1 ID:
1149 - test (suite) n ID:
1152 :param data: Data to merge.
1153 :type data: pandas.Series
1154 :returns: Merged data.
1155 :rtype: pandas.Series
1158 logging.info(" Merging data ...")
1160 merged_data = pd.Series()
1161 for _, builds in data.iteritems():
1162 for _, item in builds.iteritems():
1163 for ID, item_data in item.iteritems():
1164 merged_data[ID] = item_data