resources/tools/presentation/generator_CPTA.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Generation of Continuous Performance Trending and Analysis.
  15 """
  16
  17 import multiprocessing
  18 import os
  19 import logging
  20 import csv
  21 import prettytable
  22 import plotly.offline as ploff
  23 import plotly.graph_objs as plgo
  24 import plotly.exceptions as plerr
  25
  26 from collections import OrderedDict
  27 from datetime import datetime
  28
  29 from utils import archive_input_data, execute_command, \
  30     classify_anomalies, Worker
  31
  32
  33 # Command to build the html format of the report
  34 HTML_BUILDER = 'sphinx-build -v -c conf_cpta -a ' \
  35                '-b html -E ' \
  36                '-t html ' \
  37                '-D version="{date}" ' \
  38                '{working_dir} ' \
  39                '{build_dir}/'
  40
  41 # .css file for the html format of the report
  42 THEME_OVERRIDES = """/* override table width restrictions */
  43 .wy-nav-content {
  44     max-width: 1200px !important;
  45 }
  46 .rst-content blockquote {
  47     margin-left: 0px;
  48     line-height: 18px;
  49     margin-bottom: 0px;
  50 }
  51 .wy-menu-vertical a {
  52     display: inline-block;
  53     line-height: 18px;
  54     padding: 0 2em;
  55     display: block;
  56     position: relative;
  57     font-size: 90%;
  58     color: #d9d9d9
  59 }
  60 .wy-menu-vertical li.current a {
  61     color: gray;
  62     border-right: solid 1px #c9c9c9;
  63     padding: 0 3em;
  64 }
  65 .wy-menu-vertical li.toctree-l2.current > a {
  66     background: #c9c9c9;
  67     padding: 0 3em;
  68 }
  69 .wy-menu-vertical li.toctree-l2.current li.toctree-l3 > a {
  70     display: block;
  71     background: #c9c9c9;
  72     padding: 0 4em;
  73 }
  74 .wy-menu-vertical li.toctree-l3.current li.toctree-l4 > a {
  75     display: block;
  76     background: #bdbdbd;
  77     padding: 0 5em;
  78 }
  79 .wy-menu-vertical li.on a, .wy-menu-vertical li.current > a {
  80     color: #404040;
  81     padding: 0 2em;
  82     font-weight: bold;
  83     position: relative;
  84     background: #fcfcfc;
  85     border: none;
  86         border-top-width: medium;
  87         border-bottom-width: medium;
  88         border-top-style: none;
  89         border-bottom-style: none;
  90         border-top-color: currentcolor;
  91         border-bottom-color: currentcolor;
  92     padding-left: 2em -4px;
  93 }
  94 """
  95
  96 COLORS = ["SkyBlue", "Olive", "Purple", "Coral", "Indigo", "Pink",
  97           "Chocolate", "Brown", "Magenta", "Cyan", "Orange", "Black",
  98           "Violet", "Blue", "Yellow", "BurlyWood", "CadetBlue", "Crimson",
  99           "DarkBlue", "DarkCyan", "DarkGreen", "Green", "GoldenRod",
 100           "LightGreen", "LightSeaGreen", "LightSkyBlue", "Maroon",
 101           "MediumSeaGreen", "SeaGreen", "LightSlateGrey",
 102           "SkyBlue", "Olive", "Purple", "Coral", "Indigo", "Pink",
 103           "Chocolate", "Brown", "Magenta", "Cyan", "Orange", "Black",
 104           "Violet", "Blue", "Yellow", "BurlyWood", "CadetBlue", "Crimson",
 105           "DarkBlue", "DarkCyan", "DarkGreen", "Green", "GoldenRod",
 106           "LightGreen", "LightSeaGreen", "LightSkyBlue", "Maroon",
 107           "MediumSeaGreen", "SeaGreen", "LightSlateGrey"
 108           ]
 109
 110
 111 def generate_cpta(spec, data):
 112     """Generate all formats and versions of the Continuous Performance Trending
 113     and Analysis.
 114
 115     :param spec: Specification read from the specification file.
 116     :param data: Full data set.
 117     :type spec: Specification
 118     :type data: InputData
 119     """
 120
 121     logging.info("Generating the Continuous Performance Trending and Analysis "
 122                  "...")
 123
 124     ret_code = _generate_all_charts(spec, data)
 125
 126     cmd = HTML_BUILDER.format(
 127         date=datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC'),
 128         working_dir=spec.environment["paths"]["DIR[WORKING,SRC]"],
 129         build_dir=spec.environment["paths"]["DIR[BUILD,HTML]"])
 130     execute_command(cmd)
 131
 132     with open(spec.environment["paths"]["DIR[CSS_PATCH_FILE]"], "w") as \
 133             css_file:
 134         css_file.write(THEME_OVERRIDES)
 135
 136     with open(spec.environment["paths"]["DIR[CSS_PATCH_FILE2]"], "w") as \
 137             css_file:
 138         css_file.write(THEME_OVERRIDES)
 139
 140     archive_input_data(spec)
 141
 142     logging.info("Done.")
 143
 144     return ret_code
 145
 146
 147 def _generate_trending_traces(in_data, job_name, build_info,
 148                               show_trend_line=True, name="", color=""):
 149     """Generate the trending traces:
 150      - samples,
 151      - outliers, regress, progress
 152      - average of normal samples (trending line)
 153
 154     :param in_data: Full data set.
 155     :param job_name: The name of job which generated the data.
 156     :param build_info: Information about the builds.
 157     :param show_trend_line: Show moving median (trending plot).
 158     :param name: Name of the plot
 159     :param color: Name of the color for the plot.
 160     :type in_data: OrderedDict
 161     :type job_name: str
 162     :type build_info: dict
 163     :type show_trend_line: bool
 164     :type name: str
 165     :type color: str
 166     :returns: Generated traces (list) and the evaluated result.
 167     :rtype: tuple(traces, result)
 168     """
 169
 170     data_x = list(in_data.keys())
 171     data_y = list(in_data.values())
 172
 173     hover_text = list()
 174     xaxis = list()
 175     for idx in data_x:
 176         date = build_info[job_name][str(idx)][0]
 177         hover_str = ("date: {date}<br>"
 178                      "value: {value:,}<br>"
 179                      "{sut}-ref: {build}<br>"
 180                      "csit-ref: mrr-{period}-build-{build_nr}<br>"
 181                      "testbed: {testbed}")
 182         if "dpdk" in job_name:
 183             hover_text.append(hover_str.format(
 184                 date=date,
 185                 value=int(in_data[idx].avg),
 186                 sut="dpdk",
 187                 build=build_info[job_name][str(idx)][1].rsplit('~', 1)[0],
 188                 period="weekly",
 189                 build_nr=idx,
 190                 testbed=build_info[job_name][str(idx)][2]))
 191         elif "vpp" in job_name:
 192             hover_text.append(hover_str.format(
 193                 date=date,
 194                 value=int(in_data[idx].avg),
 195                 sut="vpp",
 196                 build=build_info[job_name][str(idx)][1].rsplit('~', 1)[0],
 197                 period="daily",
 198                 build_nr=idx,
 199                 testbed=build_info[job_name][str(idx)][2]))
 200
 201         xaxis.append(datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]),
 202                               int(date[9:11]), int(date[12:])))
 203
 204     data_pd = OrderedDict()
 205     for key, value in zip(xaxis, data_y):
 206         data_pd[key] = value
 207
 208     anomaly_classification, avgs = classify_anomalies(data_pd)
 209
 210     anomalies = OrderedDict()
 211     anomalies_colors = list()
 212     anomalies_avgs = list()
 213     anomaly_color = {
 214         "regression": 0.0,
 215         "normal": 0.5,
 216         "progression": 1.0
 217     }
 218     if anomaly_classification:
 219         for idx, (key, value) in enumerate(data_pd.iteritems()):
 220             if anomaly_classification[idx] in \
 221                     ("outlier", "regression", "progression"):
 222                 anomalies[key] = value
 223                 anomalies_colors.append(
 224                     anomaly_color[anomaly_classification[idx]])
 225                 anomalies_avgs.append(avgs[idx])
 226         anomalies_colors.extend([0.0, 0.5, 1.0])
 227
 228     # Create traces
 229
 230     trace_samples = plgo.Scatter(
 231         x=xaxis,
 232         y=[y.avg for y in data_y],
 233         mode='markers',
 234         line={
 235             "width": 1
 236         },
 237         showlegend=True,
 238         legendgroup=name,
 239         name="{name}".format(name=name),
 240         marker={
 241             "size": 5,
 242             "color": color,
 243             "symbol": "circle",
 244         },
 245         text=hover_text,
 246         hoverinfo="text"
 247     )
 248     traces = [trace_samples, ]
 249
 250     if show_trend_line:
 251         trace_trend = plgo.Scatter(
 252             x=xaxis,
 253             y=avgs,
 254             mode='lines',
 255             line={
 256                 "shape": "linear",
 257                 "width": 1,
 258                 "color": color,
 259             },
 260             showlegend=False,
 261             legendgroup=name,
 262             name='{name}'.format(name=name),
 263             text=["trend: {0:,}".format(int(avg)) for avg in avgs],
 264             hoverinfo="text+name"
 265         )
 266         traces.append(trace_trend)
 267
 268     trace_anomalies = plgo.Scatter(
 269         x=anomalies.keys(),
 270         y=anomalies_avgs,
 271         mode='markers',
 272         hoverinfo="none",
 273         showlegend=False,
 274         legendgroup=name,
 275         name="{name}-anomalies".format(name=name),
 276         marker={
 277             "size": 15,
 278             "symbol": "circle-open",
 279             "color": anomalies_colors,
 280             "colorscale": [[0.00, "red"],
 281                            [0.33, "red"],
 282                            [0.33, "white"],
 283                            [0.66, "white"],
 284                            [0.66, "green"],
 285                            [1.00, "green"]],
 286             "showscale": True,
 287             "line": {
 288                 "width": 2
 289             },
 290             "colorbar": {
 291                 "y": 0.5,
 292                 "len": 0.8,
 293                 "title": "Circles Marking Data Classification",
 294                 "titleside": 'right',
 295                 "titlefont": {
 296                     "size": 14
 297                 },
 298                 "tickmode": 'array',
 299                 "tickvals": [0.167, 0.500, 0.833],
 300                 "ticktext": ["Regression", "Normal", "Progression"],
 301                 "ticks": "",
 302                 "ticklen": 0,
 303                 "tickangle": -90,
 304                 "thickness": 10
 305             }
 306         }
 307     )
 308     traces.append(trace_anomalies)
 309
 310     if anomaly_classification:
 311         return traces, anomaly_classification[-1]
 312     else:
 313         return traces, None
 314
 315
 316 def _generate_all_charts(spec, input_data):
 317     """Generate all charts specified in the specification file.
 318
 319     :param spec: Specification.
 320     :param input_data: Full data set.
 321     :type spec: Specification
 322     :type input_data: InputData
 323     """
 324
 325     def _generate_chart(_, data_q, graph):
 326         """Generates the chart.
 327         """
 328
 329         logs = list()
 330
 331         logging.info("  Generating the chart '{0}' ...".
 332                      format(graph.get("title", "")))
 333         logs.append(("INFO", "  Generating the chart '{0}' ...".
 334                      format(graph.get("title", ""))))
 335
 336         job_name = graph["data"].keys()[0]
 337
 338         csv_tbl = list()
 339         res = list()
 340
 341         # Transform the data
 342         logs.append(("INFO", "    Creating the data set for the {0} '{1}'.".
 343                      format(graph.get("type", ""), graph.get("title", ""))))
 344         data = input_data.filter_data(graph, continue_on_error=True)
 345         if data is None:
 346             logging.error("No data.")
 347             return
 348
 349         chart_data = dict()
 350         for job, job_data in data.iteritems():
 351             if job != job_name:
 352                 continue
 353             for index, bld in job_data.items():
 354                 for test_name, test in bld.items():
 355                     if chart_data.get(test_name, None) is None:
 356                         chart_data[test_name] = OrderedDict()
 357                     try:
 358                         chart_data[test_name][int(index)] = \
 359                             test["result"]["receive-rate"]
 360                     except (KeyError, TypeError):
 361                         pass
 362
 363         # Add items to the csv table:
 364         for tst_name, tst_data in chart_data.items():
 365             tst_lst = list()
 366             for bld in builds_dict[job_name]:
 367                 itm = tst_data.get(int(bld), '')
 368                 if not isinstance(itm, str):
 369                     itm = itm.avg
 370                 tst_lst.append(str(itm))
 371             csv_tbl.append("{0},".format(tst_name) + ",".join(tst_lst) + '\n')
 372         # Generate traces:
 373         traces = list()
 374         index = 0
 375         for test_name, test_data in chart_data.items():
 376             if not test_data:
 377                 logs.append(("WARNING", "No data for the test '{0}'".
 378                              format(test_name)))
 379                 continue
 380             message = "index: {index}, test: {test}".format(
 381                 index=index, test=test_name)
 382             test_name = test_name.split('.')[-1]
 383             try:
 384                 trace, rslt = _generate_trending_traces(
 385                     test_data,
 386                     job_name=job_name,
 387                     build_info=build_info,
 388                     name='-'.join(test_name.split('-')[2:-1]),
 389                     color=COLORS[index])
 390             except IndexError:
 391                 message = "Out of colors: {}".format(message)
 392                 logs.append(("ERROR", message))
 393                 logging.error(message)
 394                 index += 1
 395                 continue
 396             traces.extend(trace)
 397             res.append(rslt)
 398             index += 1
 399
 400         if traces:
 401             # Generate the chart:
 402             graph["layout"]["xaxis"]["title"] = \
 403                 graph["layout"]["xaxis"]["title"].format(job=job_name)
 404             name_file = "{0}-{1}{2}".format(spec.cpta["output-file"],
 405                                             graph["output-file-name"],
 406                                             spec.cpta["output-file-type"])
 407
 408             logs.append(("INFO", "    Writing the file '{0}' ...".
 409                          format(name_file)))
 410             plpl = plgo.Figure(data=traces, layout=graph["layout"])
 411             try:
 412                 ploff.plot(plpl, show_link=False, auto_open=False,
 413                            filename=name_file)
 414             except plerr.PlotlyEmptyDataError:
 415                 logs.append(("WARNING", "No data for the plot. Skipped."))
 416
 417         data_out = {
 418             "job_name": job_name,
 419             "csv_table": csv_tbl,
 420             "results": res,
 421             "logs": logs
 422         }
 423         data_q.put(data_out)
 424
 425     builds_dict = dict()
 426     for job in spec.input["builds"].keys():
 427         if builds_dict.get(job, None) is None:
 428             builds_dict[job] = list()
 429         for build in spec.input["builds"][job]:
 430             status = build["status"]
 431             if status != "failed" and status != "not found" and \
 432                 status != "removed":
 433                 builds_dict[job].append(str(build["build"]))
 434
 435     # Create "build ID": "date" dict:
 436     build_info = dict()
 437     tb_tbl = spec.environment.get("testbeds", None)
 438     for job_name, job_data in builds_dict.items():
 439         if build_info.get(job_name, None) is None:
 440             build_info[job_name] = OrderedDict()
 441         for build in job_data:
 442             testbed = ""
 443             tb_ip = input_data.metadata(job_name, build).get("testbed", "")
 444             if tb_ip and tb_tbl:
 445                 testbed = tb_tbl.get(tb_ip, "")
 446             build_info[job_name][build] = (
 447                 input_data.metadata(job_name, build).get("generated", ""),
 448                 input_data.metadata(job_name, build).get("version", ""),
 449                 testbed
 450             )
 451
 452     work_queue = multiprocessing.JoinableQueue()
 453     manager = multiprocessing.Manager()
 454     data_queue = manager.Queue()
 455     cpus = multiprocessing.cpu_count()
 456
 457     workers = list()
 458     for cpu in range(cpus):
 459         worker = Worker(work_queue,
 460                         data_queue,
 461                         _generate_chart)
 462         worker.daemon = True
 463         worker.start()
 464         workers.append(worker)
 465         os.system("taskset -p -c {0} {1} > /dev/null 2>&1".
 466                   format(cpu, worker.pid))
 467
 468     for chart in spec.cpta["plots"]:
 469         work_queue.put((chart, ))
 470     work_queue.join()
 471
 472     anomaly_classifications = list()
 473
 474     # Create the header:
 475     csv_tables = dict()
 476     for job_name in builds_dict.keys():
 477         if csv_tables.get(job_name, None) is None:
 478             csv_tables[job_name] = list()
 479         header = "Build Number:," + ",".join(builds_dict[job_name]) + '\n'
 480         csv_tables[job_name].append(header)
 481         build_dates = [x[0] for x in build_info[job_name].values()]
 482         header = "Build Date:," + ",".join(build_dates) + '\n'
 483         csv_tables[job_name].append(header)
 484         versions = [x[1] for x in build_info[job_name].values()]
 485         header = "Version:," + ",".join(versions) + '\n'
 486         csv_tables[job_name].append(header)
 487
 488     while not data_queue.empty():
 489         result = data_queue.get()
 490
 491         anomaly_classifications.extend(result["results"])
 492         csv_tables[result["job_name"]].extend(result["csv_table"])
 493
 494         for item in result["logs"]:
 495             if item[0] == "INFO":
 496                 logging.info(item[1])
 497             elif item[0] == "ERROR":
 498                 logging.error(item[1])
 499             elif item[0] == "DEBUG":
 500                 logging.debug(item[1])
 501             elif item[0] == "CRITICAL":
 502                 logging.critical(item[1])
 503             elif item[0] == "WARNING":
 504                 logging.warning(item[1])
 505
 506     del data_queue
 507
 508     # Terminate all workers
 509     for worker in workers:
 510         worker.terminate()
 511         worker.join()
 512
 513     # Write the tables:
 514     for job_name, csv_table in csv_tables.items():
 515         file_name = spec.cpta["output-file"] + "-" + job_name + "-trending"
 516         with open("{0}.csv".format(file_name), 'w') as file_handler:
 517             file_handler.writelines(csv_table)
 518
 519         txt_table = None
 520         with open("{0}.csv".format(file_name), 'rb') as csv_file:
 521             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 522             line_nr = 0
 523             for row in csv_content:
 524                 if txt_table is None:
 525                     txt_table = prettytable.PrettyTable(row)
 526                 else:
 527                     if line_nr > 1:
 528                         for idx, item in enumerate(row):
 529                             try:
 530                                 row[idx] = str(round(float(item) / 1000000, 2))
 531                             except ValueError:
 532                                 pass
 533                     try:
 534                         txt_table.add_row(row)
 535                     except Exception as err:
 536                         logging.warning("Error occurred while generating TXT "
 537                                         "table:\n{0}".format(err))
 538                 line_nr += 1
 539             txt_table.align["Build Number:"] = "l"
 540         with open("{0}.txt".format(file_name), "w") as txt_file:
 541             txt_file.write(str(txt_table))
 542
 543     # Evaluate result:
 544     if anomaly_classifications:
 545         result = "PASS"
 546         for classification in anomaly_classifications:
 547             if classification == "regression" or classification == "outlier":
 548                 result = "FAIL"
 549                 break
 550     else:
 551         result = "FAIL"
 552
 553     logging.info("Partial results: {0}".format(anomaly_classifications))
 554     logging.info("Result: {0}".format(result))
 555
 556     return result