resources/tools/presentation/generator_CPTA.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Generation of Continuous Performance Trending and Analysis.
  15 """
  16
  17 import multiprocessing
  18 import os
  19 import logging
  20 import csv
  21 import prettytable
  22 import plotly.offline as ploff
  23 import plotly.graph_objs as plgo
  24 import plotly.exceptions as plerr
  25
  26 from collections import OrderedDict
  27 from datetime import datetime
  28 from copy import deepcopy
  29
  30 from utils import archive_input_data, execute_command, \
  31     classify_anomalies, Worker
  32
  33
  34 # Command to build the html format of the report
  35 HTML_BUILDER = 'sphinx-build -v -c conf_cpta -a ' \
  36                '-b html -E ' \
  37                '-t html ' \
  38                '-D version="{date}" ' \
  39                '{working_dir} ' \
  40                '{build_dir}/'
  41
  42 # .css file for the html format of the report
  43 THEME_OVERRIDES = """/* override table width restrictions */
  44 .wy-nav-content {
  45     max-width: 1200px !important;
  46 }
  47 .rst-content blockquote {
  48     margin-left: 0px;
  49     line-height: 18px;
  50     margin-bottom: 0px;
  51 }
  52 .wy-menu-vertical a {
  53     display: inline-block;
  54     line-height: 18px;
  55     padding: 0 2em;
  56     display: block;
  57     position: relative;
  58     font-size: 90%;
  59     color: #d9d9d9
  60 }
  61 .wy-menu-vertical li.current a {
  62     color: gray;
  63     border-right: solid 1px #c9c9c9;
  64     padding: 0 3em;
  65 }
  66 .wy-menu-vertical li.toctree-l2.current > a {
  67     background: #c9c9c9;
  68     padding: 0 3em;
  69 }
  70 .wy-menu-vertical li.toctree-l2.current li.toctree-l3 > a {
  71     display: block;
  72     background: #c9c9c9;
  73     padding: 0 4em;
  74 }
  75 .wy-menu-vertical li.toctree-l3.current li.toctree-l4 > a {
  76     display: block;
  77     background: #bdbdbd;
  78     padding: 0 5em;
  79 }
  80 .wy-menu-vertical li.on a, .wy-menu-vertical li.current > a {
  81     color: #404040;
  82     padding: 0 2em;
  83     font-weight: bold;
  84     position: relative;
  85     background: #fcfcfc;
  86     border: none;
  87         border-top-width: medium;
  88         border-bottom-width: medium;
  89         border-top-style: none;
  90         border-bottom-style: none;
  91         border-top-color: currentcolor;
  92         border-bottom-color: currentcolor;
  93     padding-left: 2em -4px;
  94 }
  95 """
  96
  97 COLORS = ["SkyBlue", "Olive", "Purple", "Coral", "Indigo", "Pink",
  98           "Chocolate", "Brown", "Magenta", "Cyan", "Orange", "Black",
  99           "Violet", "Blue", "Yellow", "BurlyWood", "CadetBlue", "Crimson",
 100           "DarkBlue", "DarkCyan", "DarkGreen", "Green", "GoldenRod",
 101           "LightGreen", "LightSeaGreen", "LightSkyBlue", "Maroon",
 102           "MediumSeaGreen", "SeaGreen", "LightSlateGrey",
 103           "SkyBlue", "Olive", "Purple", "Coral", "Indigo", "Pink",
 104           "Chocolate", "Brown", "Magenta", "Cyan", "Orange", "Black",
 105           "Violet", "Blue", "Yellow", "BurlyWood", "CadetBlue", "Crimson",
 106           "DarkBlue", "DarkCyan", "DarkGreen", "Green", "GoldenRod",
 107           "LightGreen", "LightSeaGreen", "LightSkyBlue", "Maroon",
 108           "MediumSeaGreen", "SeaGreen", "LightSlateGrey"
 109           ]
 110
 111
 112 def generate_cpta(spec, data):
 113     """Generate all formats and versions of the Continuous Performance Trending
 114     and Analysis.
 115
 116     :param spec: Specification read from the specification file.
 117     :param data: Full data set.
 118     :type spec: Specification
 119     :type data: InputData
 120     """
 121
 122     logging.info("Generating the Continuous Performance Trending and Analysis "
 123                  "...")
 124
 125     ret_code = _generate_all_charts(spec, data)
 126
 127     cmd = HTML_BUILDER.format(
 128         date=datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC'),
 129         working_dir=spec.environment["paths"]["DIR[WORKING,SRC]"],
 130         build_dir=spec.environment["paths"]["DIR[BUILD,HTML]"])
 131     execute_command(cmd)
 132
 133     with open(spec.environment["paths"]["DIR[CSS_PATCH_FILE]"], "w") as \
 134             css_file:
 135         css_file.write(THEME_OVERRIDES)
 136
 137     with open(spec.environment["paths"]["DIR[CSS_PATCH_FILE2]"], "w") as \
 138             css_file:
 139         css_file.write(THEME_OVERRIDES)
 140
 141     archive_input_data(spec)
 142
 143     logging.info("Done.")
 144
 145     return ret_code
 146
 147
 148 def _generate_trending_traces(in_data, job_name, build_info,
 149                               show_trend_line=True, name="", color=""):
 150     """Generate the trending traces:
 151      - samples,
 152      - outliers, regress, progress
 153      - average of normal samples (trending line)
 154
 155     :param in_data: Full data set.
 156     :param job_name: The name of job which generated the data.
 157     :param build_info: Information about the builds.
 158     :param show_trend_line: Show moving median (trending plot).
 159     :param name: Name of the plot
 160     :param color: Name of the color for the plot.
 161     :type in_data: OrderedDict
 162     :type job_name: str
 163     :type build_info: dict
 164     :type show_trend_line: bool
 165     :type name: str
 166     :type color: str
 167     :returns: Generated traces (list) and the evaluated result.
 168     :rtype: tuple(traces, result)
 169     """
 170
 171     data_x = list(in_data.keys())
 172     data_y = list(in_data.values())
 173
 174     hover_text = list()
 175     xaxis = list()
 176     for idx in data_x:
 177         date = build_info[job_name][str(idx)][0]
 178         hover_str = ("date: {date}<br>"
 179                      "value: {value:,}<br>"
 180                      "{sut}-ref: {build}<br>"
 181                      "csit-ref: mrr-{period}-build-{build_nr}<br>"
 182                      "testbed: {testbed}")
 183         if "dpdk" in job_name:
 184             hover_text.append(hover_str.format(
 185                 date=date,
 186                 value=int(in_data[idx].avg),
 187                 sut="dpdk",
 188                 build=build_info[job_name][str(idx)][1].rsplit('~', 1)[0],
 189                 period="weekly",
 190                 build_nr=idx,
 191                 testbed=build_info[job_name][str(idx)][2]))
 192         elif "vpp" in job_name:
 193             hover_text.append(hover_str.format(
 194                 date=date,
 195                 value=int(in_data[idx].avg),
 196                 sut="vpp",
 197                 build=build_info[job_name][str(idx)][1].rsplit('~', 1)[0],
 198                 period="daily",
 199                 build_nr=idx,
 200                 testbed=build_info[job_name][str(idx)][2]))
 201
 202         xaxis.append(datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]),
 203                               int(date[9:11]), int(date[12:])))
 204
 205     data_pd = OrderedDict()
 206     for key, value in zip(xaxis, data_y):
 207         data_pd[key] = value
 208
 209     anomaly_classification, avgs = classify_anomalies(data_pd)
 210
 211     anomalies = OrderedDict()
 212     anomalies_colors = list()
 213     anomalies_avgs = list()
 214     anomaly_color = {
 215         "regression": 0.0,
 216         "normal": 0.5,
 217         "progression": 1.0
 218     }
 219     if anomaly_classification:
 220         for idx, (key, value) in enumerate(data_pd.iteritems()):
 221             if anomaly_classification[idx] in \
 222                     ("outlier", "regression", "progression"):
 223                 anomalies[key] = value
 224                 anomalies_colors.append(
 225                     anomaly_color[anomaly_classification[idx]])
 226                 anomalies_avgs.append(avgs[idx])
 227         anomalies_colors.extend([0.0, 0.5, 1.0])
 228
 229     # Create traces
 230
 231     trace_samples = plgo.Scatter(
 232         x=xaxis,
 233         y=[y.avg for y in data_y],
 234         mode='markers',
 235         line={
 236             "width": 1
 237         },
 238         showlegend=True,
 239         legendgroup=name,
 240         name="{name}".format(name=name),
 241         marker={
 242             "size": 5,
 243             "color": color,
 244             "symbol": "circle",
 245         },
 246         text=hover_text,
 247         hoverinfo="text"
 248     )
 249     traces = [trace_samples, ]
 250
 251     if show_trend_line:
 252         trace_trend = plgo.Scatter(
 253             x=xaxis,
 254             y=avgs,
 255             mode='lines',
 256             line={
 257                 "shape": "linear",
 258                 "width": 1,
 259                 "color": color,
 260             },
 261             showlegend=False,
 262             legendgroup=name,
 263             name='{name}'.format(name=name),
 264             text=["trend: {0:,}".format(int(avg)) for avg in avgs],
 265             hoverinfo="text+name"
 266         )
 267         traces.append(trace_trend)
 268
 269     trace_anomalies = plgo.Scatter(
 270         x=anomalies.keys(),
 271         y=anomalies_avgs,
 272         mode='markers',
 273         hoverinfo="none",
 274         showlegend=False,
 275         legendgroup=name,
 276         name="{name}-anomalies".format(name=name),
 277         marker={
 278             "size": 15,
 279             "symbol": "circle-open",
 280             "color": anomalies_colors,
 281             "colorscale": [[0.00, "red"],
 282                            [0.33, "red"],
 283                            [0.33, "white"],
 284                            [0.66, "white"],
 285                            [0.66, "green"],
 286                            [1.00, "green"]],
 287             "showscale": True,
 288             "line": {
 289                 "width": 2
 290             },
 291             "colorbar": {
 292                 "y": 0.5,
 293                 "len": 0.8,
 294                 "title": "Circles Marking Data Classification",
 295                 "titleside": 'right',
 296                 "titlefont": {
 297                     "size": 14
 298                 },
 299                 "tickmode": 'array',
 300                 "tickvals": [0.167, 0.500, 0.833],
 301                 "ticktext": ["Regression", "Normal", "Progression"],
 302                 "ticks": "",
 303                 "ticklen": 0,
 304                 "tickangle": -90,
 305                 "thickness": 10
 306             }
 307         }
 308     )
 309     traces.append(trace_anomalies)
 310
 311     if anomaly_classification:
 312         return traces, anomaly_classification[-1]
 313     else:
 314         return traces, None
 315
 316
 317 def _generate_all_charts(spec, input_data):
 318     """Generate all charts specified in the specification file.
 319
 320     :param spec: Specification.
 321     :param input_data: Full data set.
 322     :type spec: Specification
 323     :type input_data: InputData
 324     """
 325
 326     def _generate_chart(_, data_q, graph):
 327         """Generates the chart.
 328         """
 329
 330         logs = list()
 331
 332         logging.info("  Generating the chart '{0}' ...".
 333                      format(graph.get("title", "")))
 334         logs.append(("INFO", "  Generating the chart '{0}' ...".
 335                      format(graph.get("title", ""))))
 336
 337         job_name = graph["data"].keys()[0]
 338
 339         csv_tbl = list()
 340         res = list()
 341
 342         # Transform the data
 343         logs.append(("INFO", "    Creating the data set for the {0} '{1}'.".
 344                      format(graph.get("type", ""), graph.get("title", ""))))
 345         data = input_data.filter_data(graph, continue_on_error=True)
 346         if data is None:
 347             logging.error("No data.")
 348             return
 349
 350         chart_data = dict()
 351         chart_tags = dict()
 352         for job, job_data in data.iteritems():
 353             if job != job_name:
 354                 continue
 355             for index, bld in job_data.items():
 356                 for test_name, test in bld.items():
 357                     if chart_data.get(test_name, None) is None:
 358                         chart_data[test_name] = OrderedDict()
 359                     try:
 360                         chart_data[test_name][int(index)] = \
 361                             test["result"]["receive-rate"]
 362                         chart_tags[test_name] = test.get("tags", None)
 363                     except (KeyError, TypeError):
 364                         pass
 365
 366         # Add items to the csv table:
 367         for tst_name, tst_data in chart_data.items():
 368             tst_lst = list()
 369             for bld in builds_dict[job_name]:
 370                 itm = tst_data.get(int(bld), '')
 371                 if not isinstance(itm, str):
 372                     itm = itm.avg
 373                 tst_lst.append(str(itm))
 374             csv_tbl.append("{0},".format(tst_name) + ",".join(tst_lst) + '\n')
 375
 376         # Generate traces:
 377         traces = list()
 378         index = 0
 379         groups = graph.get("groups", None)
 380         visibility = list()
 381
 382         if groups:
 383             for group in groups:
 384                 visible = list()
 385                 for tag in group:
 386                     for test_name, test_data in chart_data.items():
 387                         if not test_data:
 388                             logs.append(("WARNING",
 389                                          "No data for the test '{0}'".
 390                                          format(test_name)))
 391                             continue
 392                         if tag in chart_tags[test_name]:
 393                             message = "index: {index}, test: {test}".format(
 394                                 index=index, test=test_name)
 395                             test_name = test_name.split('.')[-1]
 396                             try:
 397                                 trace, rslt = _generate_trending_traces(
 398                                     test_data,
 399                                     job_name=job_name,
 400                                     build_info=build_info,
 401                                     name='-'.join(test_name.split('-')[2:-1]),
 402                                     color=COLORS[index])
 403                             except IndexError:
 404                                 message = "Out of colors: {}".format(message)
 405                                 logs.append(("ERROR", message))
 406                                 logging.error(message)
 407                                 index += 1
 408                                 continue
 409                             traces.extend(trace)
 410                             visible.extend([True for _ in range(len(trace))])
 411                             res.append(rslt)
 412                             index += 1
 413                             break
 414                 visibility.append(visible)
 415         else:
 416             for test_name, test_data in chart_data.items():
 417                 if not test_data:
 418                     logs.append(("WARNING", "No data for the test '{0}'".
 419                                  format(test_name)))
 420                     continue
 421                 message = "index: {index}, test: {test}".format(
 422                     index=index, test=test_name)
 423                 test_name = test_name.split('.')[-1]
 424                 try:
 425                     trace, rslt = _generate_trending_traces(
 426                         test_data,
 427                         job_name=job_name,
 428                         build_info=build_info,
 429                         name='-'.join(test_name.split('-')[2:-1]),
 430                         color=COLORS[index])
 431                 except IndexError:
 432                     message = "Out of colors: {}".format(message)
 433                     logs.append(("ERROR", message))
 434                     logging.error(message)
 435                     index += 1
 436                     continue
 437                 traces.extend(trace)
 438                 res.append(rslt)
 439                 index += 1
 440
 441         if traces:
 442             # Generate the chart:
 443             try:
 444                 layout = deepcopy(graph["layout"])
 445             except KeyError as err:
 446                 logging.error("Finished with error: No layout defined")
 447                 logging.error(repr(err))
 448                 return
 449             if groups:
 450                 show = list()
 451                 for i in range(len(visibility)):
 452                     visible = list()
 453                     for r in range(len(visibility)):
 454                         for _ in range(len(visibility[r])):
 455                             visible.append(i == r)
 456                     show.append(visible)
 457
 458                 buttons = list()
 459                 buttons.append(dict(
 460                     label="All",
 461                     method="update",
 462                     args=[{"visible": [True for _ in range(len(show[0]))]}, ]
 463                 ))
 464                 for i in range(len(groups)):
 465                     try:
 466                         label = graph["group-names"][i]
 467                     except (IndexError, KeyError):
 468                         label = "Group {num}".format(num=i + 1)
 469                     buttons.append(dict(
 470                         label=label,
 471                         method="update",
 472                         args=[{"visible": show[i]}, ]
 473                     ))
 474
 475                 layout['updatemenus'] = list([
 476                     dict(
 477                         active=0,
 478                         type="dropdown",
 479                         direction="down",
 480                         xanchor="left",
 481                         yanchor="bottom",
 482                         x=-0.12,
 483                         y=1.0,
 484                         buttons=buttons
 485                     )
 486                 ])
 487
 488             name_file = "{0}-{1}{2}".format(spec.cpta["output-file"],
 489                                             graph["output-file-name"],
 490                                             spec.cpta["output-file-type"])
 491
 492             logs.append(("INFO", "    Writing the file '{0}' ...".
 493                          format(name_file)))
 494             plpl = plgo.Figure(data=traces, layout=layout)
 495             try:
 496                 ploff.plot(plpl, show_link=False, auto_open=False,
 497                            filename=name_file)
 498             except plerr.PlotlyEmptyDataError:
 499                 logs.append(("WARNING", "No data for the plot. Skipped."))
 500
 501         data_out = {
 502             "job_name": job_name,
 503             "csv_table": csv_tbl,
 504             "results": res,
 505             "logs": logs
 506         }
 507         data_q.put(data_out)
 508
 509     builds_dict = dict()
 510     for job in spec.input["builds"].keys():
 511         if builds_dict.get(job, None) is None:
 512             builds_dict[job] = list()
 513         for build in spec.input["builds"][job]:
 514             status = build["status"]
 515             if status != "failed" and status != "not found" and \
 516                 status != "removed":
 517                 builds_dict[job].append(str(build["build"]))
 518
 519     # Create "build ID": "date" dict:
 520     build_info = dict()
 521     tb_tbl = spec.environment.get("testbeds", None)
 522     for job_name, job_data in builds_dict.items():
 523         if build_info.get(job_name, None) is None:
 524             build_info[job_name] = OrderedDict()
 525         for build in job_data:
 526             testbed = ""
 527             tb_ip = input_data.metadata(job_name, build).get("testbed", "")
 528             if tb_ip and tb_tbl:
 529                 testbed = tb_tbl.get(tb_ip, "")
 530             build_info[job_name][build] = (
 531                 input_data.metadata(job_name, build).get("generated", ""),
 532                 input_data.metadata(job_name, build).get("version", ""),
 533                 testbed
 534             )
 535
 536     work_queue = multiprocessing.JoinableQueue()
 537     manager = multiprocessing.Manager()
 538     data_queue = manager.Queue()
 539     cpus = multiprocessing.cpu_count()
 540
 541     workers = list()
 542     for cpu in range(cpus):
 543         worker = Worker(work_queue,
 544                         data_queue,
 545                         _generate_chart)
 546         worker.daemon = True
 547         worker.start()
 548         workers.append(worker)
 549         os.system("taskset -p -c {0} {1} > /dev/null 2>&1".
 550                   format(cpu, worker.pid))
 551
 552     for chart in spec.cpta["plots"]:
 553         work_queue.put((chart, ))
 554     work_queue.join()
 555
 556     anomaly_classifications = list()
 557
 558     # Create the header:
 559     csv_tables = dict()
 560     for job_name in builds_dict.keys():
 561         if csv_tables.get(job_name, None) is None:
 562             csv_tables[job_name] = list()
 563         header = "Build Number:," + ",".join(builds_dict[job_name]) + '\n'
 564         csv_tables[job_name].append(header)
 565         build_dates = [x[0] for x in build_info[job_name].values()]
 566         header = "Build Date:," + ",".join(build_dates) + '\n'
 567         csv_tables[job_name].append(header)
 568         versions = [x[1] for x in build_info[job_name].values()]
 569         header = "Version:," + ",".join(versions) + '\n'
 570         csv_tables[job_name].append(header)
 571
 572     while not data_queue.empty():
 573         result = data_queue.get()
 574
 575         anomaly_classifications.extend(result["results"])
 576         csv_tables[result["job_name"]].extend(result["csv_table"])
 577
 578         for item in result["logs"]:
 579             if item[0] == "INFO":
 580                 logging.info(item[1])
 581             elif item[0] == "ERROR":
 582                 logging.error(item[1])
 583             elif item[0] == "DEBUG":
 584                 logging.debug(item[1])
 585             elif item[0] == "CRITICAL":
 586                 logging.critical(item[1])
 587             elif item[0] == "WARNING":
 588                 logging.warning(item[1])
 589
 590     del data_queue
 591
 592     # Terminate all workers
 593     for worker in workers:
 594         worker.terminate()
 595         worker.join()
 596
 597     # Write the tables:
 598     for job_name, csv_table in csv_tables.items():
 599         file_name = spec.cpta["output-file"] + "-" + job_name + "-trending"
 600         with open("{0}.csv".format(file_name), 'w') as file_handler:
 601             file_handler.writelines(csv_table)
 602
 603         txt_table = None
 604         with open("{0}.csv".format(file_name), 'rb') as csv_file:
 605             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 606             line_nr = 0
 607             for row in csv_content:
 608                 if txt_table is None:
 609                     txt_table = prettytable.PrettyTable(row)
 610                 else:
 611                     if line_nr > 1:
 612                         for idx, item in enumerate(row):
 613                             try:
 614                                 row[idx] = str(round(float(item) / 1000000, 2))
 615                             except ValueError:
 616                                 pass
 617                     try:
 618                         txt_table.add_row(row)
 619                     except Exception as err:
 620                         logging.warning("Error occurred while generating TXT "
 621                                         "table:\n{0}".format(err))
 622                 line_nr += 1
 623             txt_table.align["Build Number:"] = "l"
 624         with open("{0}.txt".format(file_name), "w") as txt_file:
 625             txt_file.write(str(txt_table))
 626
 627     # Evaluate result:
 628     if anomaly_classifications:
 629         result = "PASS"
 630         for classification in anomaly_classifications:
 631             if classification == "regression" or classification == "outlier":
 632                 result = "FAIL"
 633                 break
 634     else:
 635         result = "FAIL"
 636
 637     logging.info("Partial results: {0}".format(anomaly_classifications))
 638     logging.info("Result: {0}".format(result))
 639
 640     return result