708836aef48f455dd2af8fdc1cef07d5a8fdfcd1
[csit.git] / resources / tools / report_gen / run_robot_json_data.py
1 #!/usr/bin/python
2
3 # Copyright (c) 2017 Cisco and/or its affiliates.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
7 #
8 #     http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15
16 """
17 Script extracts required data from robot framework output file (output.xml) and
18 saves it in JSON format. Its structure is:
19
20 {
21     "metadata": {
22         "vdevice": "VPP version",
23         "data-length": int
24     },
25     "data": {
26         "ID": {
27             "name": "Test name",
28             "parent": "Name of the parent of the test",
29             "tags": ["tag 1", "tag 2", "tag n"],
30             "type": "PDR" | "NDR",
31             "throughput": {
32                 "value": int,
33                 "unit": "pps" | "bps" | "percentage"
34             },
35             "latency": {
36                 "direction1": {
37                     "100": {
38                         "min": int,
39                         "avg": int,
40                         "max": int
41                     },
42                     "50": {  # Only for NDR
43                         "min": int,
44                         "avg": int,
45                         "max": int
46                     },
47                     "10": {  # Only for NDR
48                         "min": int,
49                         "avg": int,
50                         "max": int
51                     }
52                 },
53                 "direction2": {
54                     "100": {
55                         "min": int,
56                         "avg": int,
57                         "max": int
58                     },
59                     "50": {  # Only for NDR
60                         "min": int,
61                         "avg": int,
62                         "max": int
63                     },
64                     "10": {  # Only for NDR
65                         "min": int,
66                         "avg": int,
67                         "max": int
68                     }
69                 }
70             },
71             "lossTolerance": "lossTolerance"  # Only for PDR
72         },
73         "ID" {
74             # next test
75         }
76     }
77 }
78
79 .. note:: ID is the lowercase full path to the test.
80
81 :Example:
82
83 run_robot_json_data.py -i "output.xml" -o "data.json" -v "17.07-rc0~144"
84
85 """
86
87 import argparse
88 import re
89 import sys
90 import json
91
92 from robot.api import ExecutionResult, ResultVisitor
93
94
95 class ExecutionChecker(ResultVisitor):
96     """Class to traverse through the test suite structure.
97
98     The functionality implemented in this class generates a json structure.
99     """
100
101     REGEX_RATE = re.compile(r'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)\s(\w+)')
102
103     REGEX_LAT_NDR = re.compile(r'^[\D\d]*'
104                                r'LAT_\d+%NDR:\s\[\'(-?\d+\/-?\d+\/-?\d+)\','
105                                r'\s\'(-?\d+\/-?\d+\/-?\d+)\'\]\s\n'
106                                r'LAT_\d+%NDR:\s\[\'(-?\d+\/-?\d+\/-?\d+)\','
107                                r'\s\'(-?\d+\/-?\d+\/-?\d+)\'\]\s\n'
108                                r'LAT_\d+%NDR:\s\[\'(-?\d+\/-?\d+\/-?\d+)\','
109                                r'\s\'(-?\d+\/-?\d+\/-?\d+)\'\]')
110
111     REGEX_LAT_PDR = re.compile(r'^[\D\d]*'
112                                r'LAT_\d+%PDR:\s\[\'(-?\d+\/-?\d+\/-?\d+)\','
113                                r'\s\'(-?\d+\/-?\d+\/-?\d+)\'\][\D\d]*')
114
115     REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s'
116                                  r'[\D\d]*')
117
118     def __init__(self, **metadata):
119         """Initialisation.
120
121         :param metadata: Key-value pairs to be included to "metadata" part of
122         JSON structure.
123         :type metadata: dict
124         """
125         self._data = {
126             "metadata": {
127             },
128             "data": {
129             }
130         }
131
132         for key, val in metadata.items():
133             self._data["metadata"][key] = val
134
135     @property
136     def data(self):
137         return self._data
138
139     def _get_latency(self, msg, test_type):
140         """Get the latency data from the test message.
141
142         :param msg: Message to be parsed.
143         :param test_type: Type of the test - NDR or PDR.
144         :type msg: str
145         :type test_type: str
146         :returns: Latencies parsed from the message.
147         :rtype: dict
148         """
149
150         if test_type == "NDR":
151             groups = re.search(self.REGEX_LAT_NDR, msg)
152             groups_range = range(1, 7)
153         elif test_type == "PDR":
154             groups = re.search(self.REGEX_LAT_PDR, msg)
155             groups_range = range(1, 3)
156         else:
157             return {}
158
159         latencies = list()
160         for idx in groups_range:
161             try:
162                 lat = [int(item) for item in str(groups.group(idx)).split('/')]
163             except (AttributeError, ValueError):
164                 lat = [-1, -1, -1]
165             latencies.append(lat)
166
167         keys = ("min", "avg", "max")
168         latency = {
169             "direction1": {
170             },
171             "direction2": {
172             }
173         }
174
175         latency["direction1"]["100"] = dict(zip(keys, latencies[0]))
176         latency["direction2"]["100"] = dict(zip(keys, latencies[1]))
177         if test_type == "NDR":
178             latency["direction1"]["50"] = dict(zip(keys, latencies[2]))
179             latency["direction2"]["50"] = dict(zip(keys, latencies[3]))
180             latency["direction1"]["10"] = dict(zip(keys, latencies[4]))
181             latency["direction2"]["10"] = dict(zip(keys, latencies[5]))
182
183         return latency
184
185     def visit_suite(self, suite):
186         """Implements traversing through the suite and its direct children.
187
188         :param suite: Suite to process.
189         :type suite: Suite
190         :returns: Nothing.
191         """
192         if self.start_suite(suite) is not False:
193             suite.suites.visit(self)
194             suite.tests.visit(self)
195             self.end_suite(suite)
196
197     def start_suite(self, suite):
198         """Called when suite starts.
199
200         :param suite: Suite to process.
201         :type suite: Suite
202         :returns: Nothing.
203         """
204         pass
205
206     def end_suite(self, suite):
207         """Called when suite ends.
208
209         :param suite: Suite to process.
210         :type suite: Suite
211         :returns: Nothing.
212         """
213         pass
214
215     def visit_test(self, test):
216         """Implements traversing through the test.
217
218         :param test: Test to process.
219         :type test: Test
220         :returns: Nothing.
221         """
222         if self.start_test(test) is not False:
223             self.end_test(test)
224
225     def start_test(self, test):
226         """Called when test starts.
227
228         :param test: Test to process.
229         :type test: Test
230         :returns: Nothing.
231         """
232
233         tags = [str(tag) for tag in test.tags]
234         if test.status == "PASS" and "NDRPDRDISC" in tags:
235
236             if "NDRDISC" in tags:
237                 test_type = "NDR"
238             elif "PDRDISC" in tags:
239                 test_type = "PDR"
240             else:
241                 return
242
243             try:
244                 rate_value = str(re.search(
245                     self.REGEX_RATE, test.message).group(1))
246             except AttributeError:
247                 rate_value = "-1"
248             try:
249                 rate_unit = str(re.search(
250                     self.REGEX_RATE, test.message).group(2))
251             except AttributeError:
252                 rate_unit = "-1"
253
254             test_result = dict()
255             test_result["name"] = test.name.lower()
256             test_result["parent"] = test.parent.name.lower()
257             test_result["tags"] = tags
258             test_result["type"] = test_type
259             test_result["throughput"] = dict()
260             test_result["throughput"]["value"] = int(rate_value.split('.')[0])
261             test_result["throughput"]["unit"] = rate_unit
262             test_result["latency"] = self._get_latency(test.message, test_type)
263             if test_type == "PDR":
264                 test_result["lossTolerance"] = str(re.search(
265                     self.REGEX_TOLERANCE, test.message).group(1))
266
267             self._data["data"][test.longname.lower()] = test_result
268
269     def end_test(self, test):
270         """Called when test ends.
271
272         :param test: Test to process.
273         :type test: Test
274         :returns: Nothing.
275         """
276         pass
277
278
279 def parse_tests(args):
280     """Process data from robot output.xml file and return JSON data.
281
282     :param args: Parsed arguments.
283     :type args: ArgumentParser
284     :returns: JSON data structure.
285     :rtype: dict
286     """
287
288     result = ExecutionResult(args.input)
289     checker = ExecutionChecker(vdevice=args.vdevice)
290     result.visit(checker)
291
292     return checker.data
293
294
295 def parse_args():
296     """Parse arguments from cmd line.
297
298     :returns: Parsed arguments.
299     :rtype: ArgumentParser
300     """
301
302     parser = argparse.ArgumentParser(description=__doc__,
303                                      formatter_class=argparse.
304                                      RawDescriptionHelpFormatter)
305     parser.add_argument("-i", "--input",
306                         required=True,
307                         type=argparse.FileType('r'),
308                         help="Robot XML log file.")
309     parser.add_argument("-o", "--output",
310                         required=True,
311                         type=argparse.FileType('w'),
312                         help="JSON output file")
313     parser.add_argument("-v", "--vdevice",
314                         required=False,
315                         default="",
316                         type=str,
317                         help="VPP version")
318
319     return parser.parse_args()
320
321
322 def main():
323     """Main function."""
324
325     args = parse_args()
326     json_data = parse_tests(args)
327     json_data["metadata"]["data-length"] = len(json_data["data"])
328     json.dump(json_data, args.output)
329
330 if __name__ == "__main__":
331     sys.exit(main())

©2016 FD.io a Linux Foundation Collaborative Project. All Rights Reserved.
Linux Foundation is a registered trademark of The Linux Foundation. Linux is a registered trademark of Linus Torvalds.
Please see our privacy policy and terms of use.