resources/tools/report_gen/run_robot_json_data.py

   1 #!/usr/bin/python
   2
   3 # Copyright (c) 2017 Cisco and/or its affiliates.
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at:
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 """
  17 Script extracts required data from robot framework output file (output.xml) and
  18 saves it in JSON format. Its structure is:
  19
  20 {
  21     "metadata": {
  22         "vdevice": "VPP version",
  23         "data-length": int
  24     },
  25     "data": {
  26         "ID": {
  27             "name": "Test name",
  28             "parent": "Name of the parent of the test",
  29             "tags": ["tag 1", "tag 2", "tag n"],
  30             "type": "PDR" | "NDR",
  31             "throughput": {
  32                 "value": int,
  33                 "unit": "pps" | "bps" | "percentage"
  34             },
  35             "latency": {
  36                 "direction1": {
  37                     "100": {
  38                         "min": int,
  39                         "avg": int,
  40                         "max": int
  41                     },
  42                     "50": {  # Only for NDR
  43                         "min": int,
  44                         "avg": int,
  45                         "max": int
  46                     },
  47                     "10": {  # Only for NDR
  48                         "min": int,
  49                         "avg": int,
  50                         "max": int
  51                     }
  52                 },
  53                 "direction2": {
  54                     "100": {
  55                         "min": int,
  56                         "avg": int,
  57                         "max": int
  58                     },
  59                     "50": {  # Only for NDR
  60                         "min": int,
  61                         "avg": int,
  62                         "max": int
  63                     },
  64                     "10": {  # Only for NDR
  65                         "min": int,
  66                         "avg": int,
  67                         "max": int
  68                     }
  69                 }
  70             },
  71             "lossTolerance": "lossTolerance"  # Only for PDR
  72         },
  73         "ID" {
  74             # next test
  75         }
  76     }
  77 }
  78
  79 .. note:: ID is the lowercase full path to the test.
  80
  81 :Example:
  82
  83 run_robot_json_data.py -i "output.xml" -o "data.json" -v "17.07-rc0~144"
  84
  85 """
  86
  87 import argparse
  88 import re
  89 import sys
  90 import json
  91
  92 from robot.api import ExecutionResult, ResultVisitor
  93
  94
  95 class ExecutionChecker(ResultVisitor):
  96     """Class to traverse through the test suite structure.
  97
  98     The functionality implemented in this class generates a json structure.
  99     """
 100
 101     REGEX_RATE = re.compile(r'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)\s(\w+)')
 102
 103     REGEX_LAT_NDR = re.compile(r'^[\D\d]*'
 104                                r'LAT_\d+%NDR:\s\[\'(-?\d+\/-?\d+\/-?\d+)\','
 105                                r'\s\'(-?\d+\/-?\d+\/-?\d+)\'\]\s\n'
 106                                r'LAT_\d+%NDR:\s\[\'(-?\d+\/-?\d+\/-?\d+)\','
 107                                r'\s\'(-?\d+\/-?\d+\/-?\d+)\'\]\s\n'
 108                                r'LAT_\d+%NDR:\s\[\'(-?\d+\/-?\d+\/-?\d+)\','
 109                                r'\s\'(-?\d+\/-?\d+\/-?\d+)\'\]')
 110
 111     REGEX_LAT_PDR = re.compile(r'^[\D\d]*'
 112                                r'LAT_\d+%PDR:\s\[\'(-?\d+\/-?\d+\/-?\d+)\','
 113                                r'\s\'(-?\d+\/-?\d+\/-?\d+)\'\][\D\d]*')
 114
 115     REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s'
 116                                  r'[\D\d]*')
 117
 118     def __init__(self, **metadata):
 119         """Initialisation.
 120
 121         :param metadata: Key-value pairs to be included to "metadata" part of
 122         JSON structure.
 123         :type metadata: dict
 124         """
 125         self._data = {
 126             "metadata": {
 127             },
 128             "data": {
 129             }
 130         }
 131
 132         for key, val in metadata.items():
 133             self._data["metadata"][key] = val
 134
 135     @property
 136     def data(self):
 137         return self._data
 138
 139     def _get_latency(self, msg, test_type):
 140         """Get the latency data from the test message.
 141
 142         :param msg: Message to be parsed.
 143         :param test_type: Type of the test - NDR or PDR.
 144         :type msg: str
 145         :type test_type: str
 146         :returns: Latencies parsed from the message.
 147         :rtype: dict
 148         """
 149
 150         if test_type == "NDR":
 151             groups = re.search(self.REGEX_LAT_NDR, msg)
 152             groups_range = range(1, 7)
 153         elif test_type == "PDR":
 154             groups = re.search(self.REGEX_LAT_PDR, msg)
 155             groups_range = range(1, 3)
 156         else:
 157             return {}
 158
 159         latencies = list()
 160         for idx in groups_range:
 161             try:
 162                 lat = [int(item) for item in str(groups.group(idx)).split('/')]
 163             except (AttributeError, ValueError):
 164                 lat = [-1, -1, -1]
 165             latencies.append(lat)
 166
 167         keys = ("min", "avg", "max")
 168         latency = {
 169             "direction1": {
 170             },
 171             "direction2": {
 172             }
 173         }
 174
 175         latency["direction1"]["100"] = dict(zip(keys, latencies[0]))
 176         latency["direction2"]["100"] = dict(zip(keys, latencies[1]))
 177         if test_type == "NDR":
 178             latency["direction1"]["50"] = dict(zip(keys, latencies[2]))
 179             latency["direction2"]["50"] = dict(zip(keys, latencies[3]))
 180             latency["direction1"]["10"] = dict(zip(keys, latencies[4]))
 181             latency["direction2"]["10"] = dict(zip(keys, latencies[5]))
 182
 183         return latency
 184
 185     def visit_suite(self, suite):
 186         """Implements traversing through the suite and its direct children.
 187
 188         :param suite: Suite to process.
 189         :type suite: Suite
 190         :returns: Nothing.
 191         """
 192         if self.start_suite(suite) is not False:
 193             suite.suites.visit(self)
 194             suite.tests.visit(self)
 195             self.end_suite(suite)
 196
 197     def start_suite(self, suite):
 198         """Called when suite starts.
 199
 200         :param suite: Suite to process.
 201         :type suite: Suite
 202         :returns: Nothing.
 203         """
 204         pass
 205
 206     def end_suite(self, suite):
 207         """Called when suite ends.
 208
 209         :param suite: Suite to process.
 210         :type suite: Suite
 211         :returns: Nothing.
 212         """
 213         pass
 214
 215     def visit_test(self, test):
 216         """Implements traversing through the test.
 217
 218         :param test: Test to process.
 219         :type test: Test
 220         :returns: Nothing.
 221         """
 222         if self.start_test(test) is not False:
 223             self.end_test(test)
 224
 225     def start_test(self, test):
 226         """Called when test starts.
 227
 228         :param test: Test to process.
 229         :type test: Test
 230         :returns: Nothing.
 231         """
 232
 233         tags = [str(tag) for tag in test.tags]
 234         if test.status == "PASS" and "NDRPDRDISC" in tags:
 235
 236             if "NDRDISC" in tags:
 237                 test_type = "NDR"
 238             elif "PDRDISC" in tags:
 239                 test_type = "PDR"
 240             else:
 241                 return
 242
 243             try:
 244                 rate_value = str(re.search(
 245                     self.REGEX_RATE, test.message).group(1))
 246             except AttributeError:
 247                 rate_value = "-1"
 248             try:
 249                 rate_unit = str(re.search(
 250                     self.REGEX_RATE, test.message).group(2))
 251             except AttributeError:
 252                 rate_unit = "-1"
 253
 254             test_result = dict()
 255             test_result["name"] = test.name.lower()
 256             test_result["parent"] = test.parent.name.lower()
 257             test_result["tags"] = tags
 258             test_result["type"] = test_type
 259             test_result["throughput"] = dict()
 260             test_result["throughput"]["value"] = int(rate_value.split('.')[0])
 261             test_result["throughput"]["unit"] = rate_unit
 262             test_result["latency"] = self._get_latency(test.message, test_type)
 263             if test_type == "PDR":
 264                 test_result["lossTolerance"] = str(re.search(
 265                     self.REGEX_TOLERANCE, test.message).group(1))
 266
 267             self._data["data"][test.longname.lower()] = test_result
 268
 269     def end_test(self, test):
 270         """Called when test ends.
 271
 272         :param test: Test to process.
 273         :type test: Test
 274         :returns: Nothing.
 275         """
 276         pass
 277
 278
 279 def parse_tests(args):
 280     """Process data from robot output.xml file and return JSON data.
 281
 282     :param args: Parsed arguments.
 283     :type args: ArgumentParser
 284     :returns: JSON data structure.
 285     :rtype: dict
 286     """
 287
 288     result = ExecutionResult(args.input)
 289     checker = ExecutionChecker(vdevice=args.vdevice)
 290     result.visit(checker)
 291
 292     return checker.data
 293
 294
 295 def parse_args():
 296     """Parse arguments from cmd line.
 297
 298     :returns: Parsed arguments.
 299     :rtype: ArgumentParser
 300     """
 301
 302     parser = argparse.ArgumentParser(description=__doc__,
 303                                      formatter_class=argparse.
 304                                      RawDescriptionHelpFormatter)
 305     parser.add_argument("-i", "--input",
 306                         required=True,
 307                         type=argparse.FileType('r'),
 308                         help="Robot XML log file.")
 309     parser.add_argument("-o", "--output",
 310                         required=True,
 311                         type=argparse.FileType('w'),
 312                         help="JSON output file")
 313     parser.add_argument("-v", "--vdevice",
 314                         required=False,
 315                         default="",
 316                         type=str,
 317                         help="VPP version")
 318
 319     return parser.parse_args()
 320
 321
 322 def main():
 323     """Main function."""
 324
 325     args = parse_args()
 326     json_data = parse_tests(args)
 327     json_data["metadata"]["data-length"] = len(json_data["data"])
 328     json.dump(json_data, args.output)
 329
 330 if __name__ == "__main__":
 331     sys.exit(main())