resources/tools/presentation/generator_CPTA.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Generation of Continuous Performance Trending and Analysis.
  15 """
  16
  17 import multiprocessing
  18 import os
  19 import logging
  20 import csv
  21 import prettytable
  22 import plotly.offline as ploff
  23 import plotly.graph_objs as plgo
  24 import plotly.exceptions as plerr
  25
  26 from collections import OrderedDict
  27 from datetime import datetime
  28 from copy import deepcopy
  29
  30 from utils import archive_input_data, execute_command, \
  31     classify_anomalies, Worker
  32
  33
  34 # Command to build the html format of the report
  35 HTML_BUILDER = 'sphinx-build -v -c conf_cpta -a ' \
  36                '-b html -E ' \
  37                '-t html ' \
  38                '-D version="{date}" ' \
  39                '{working_dir} ' \
  40                '{build_dir}/'
  41
  42 # .css file for the html format of the report
  43 THEME_OVERRIDES = """/* override table width restrictions */
  44 .wy-nav-content {
  45     max-width: 1200px !important;
  46 }
  47 .rst-content blockquote {
  48     margin-left: 0px;
  49     line-height: 18px;
  50     margin-bottom: 0px;
  51 }
  52 .wy-menu-vertical a {
  53     display: inline-block;
  54     line-height: 18px;
  55     padding: 0 2em;
  56     display: block;
  57     position: relative;
  58     font-size: 90%;
  59     color: #d9d9d9
  60 }
  61 .wy-menu-vertical li.current a {
  62     color: gray;
  63     border-right: solid 1px #c9c9c9;
  64     padding: 0 3em;
  65 }
  66 .wy-menu-vertical li.toctree-l2.current > a {
  67     background: #c9c9c9;
  68     padding: 0 3em;
  69 }
  70 .wy-menu-vertical li.toctree-l2.current li.toctree-l3 > a {
  71     display: block;
  72     background: #c9c9c9;
  73     padding: 0 4em;
  74 }
  75 .wy-menu-vertical li.toctree-l3.current li.toctree-l4 > a {
  76     display: block;
  77     background: #bdbdbd;
  78     padding: 0 5em;
  79 }
  80 .wy-menu-vertical li.on a, .wy-menu-vertical li.current > a {
  81     color: #404040;
  82     padding: 0 2em;
  83     font-weight: bold;
  84     position: relative;
  85     background: #fcfcfc;
  86     border: none;
  87         border-top-width: medium;
  88         border-bottom-width: medium;
  89         border-top-style: none;
  90         border-bottom-style: none;
  91         border-top-color: currentcolor;
  92         border-bottom-color: currentcolor;
  93     padding-left: 2em -4px;
  94 }
  95 """
  96
  97 COLORS = ["SkyBlue", "Olive", "Purple", "Coral", "Indigo", "Pink",
  98           "Chocolate", "Brown", "Magenta", "Cyan", "Orange", "Black",
  99           "Violet", "Blue", "Yellow", "BurlyWood", "CadetBlue", "Crimson",
 100           "DarkBlue", "DarkCyan", "DarkGreen", "Green", "GoldenRod",
 101           "LightGreen", "LightSeaGreen", "LightSkyBlue", "Maroon",
 102           "MediumSeaGreen", "SeaGreen", "LightSlateGrey",
 103           "SkyBlue", "Olive", "Purple", "Coral", "Indigo", "Pink",
 104           "Chocolate", "Brown", "Magenta", "Cyan", "Orange", "Black",
 105           "Violet", "Blue", "Yellow", "BurlyWood", "CadetBlue", "Crimson",
 106           "DarkBlue", "DarkCyan", "DarkGreen", "Green", "GoldenRod",
 107           "LightGreen", "LightSeaGreen", "LightSkyBlue", "Maroon",
 108           "MediumSeaGreen", "SeaGreen", "LightSlateGrey"
 109           ]
 110
 111
 112 def generate_cpta(spec, data):
 113     """Generate all formats and versions of the Continuous Performance Trending
 114     and Analysis.
 115
 116     :param spec: Specification read from the specification file.
 117     :param data: Full data set.
 118     :type spec: Specification
 119     :type data: InputData
 120     """
 121
 122     logging.info("Generating the Continuous Performance Trending and Analysis "
 123                  "...")
 124
 125     ret_code = _generate_all_charts(spec, data)
 126
 127     cmd = HTML_BUILDER.format(
 128         date=datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC'),
 129         working_dir=spec.environment["paths"]["DIR[WORKING,SRC]"],
 130         build_dir=spec.environment["paths"]["DIR[BUILD,HTML]"])
 131     execute_command(cmd)
 132
 133     with open(spec.environment["paths"]["DIR[CSS_PATCH_FILE]"], "w") as \
 134             css_file:
 135         css_file.write(THEME_OVERRIDES)
 136
 137     with open(spec.environment["paths"]["DIR[CSS_PATCH_FILE2]"], "w") as \
 138             css_file:
 139         css_file.write(THEME_OVERRIDES)
 140
 141     if spec.configuration.get("archive-inputs", True):
 142         archive_input_data(spec)
 143
 144     logging.info("Done.")
 145
 146     return ret_code
 147
 148
 149 def _generate_trending_traces(in_data, job_name, build_info,
 150                               show_trend_line=True, name="", color=""):
 151     """Generate the trending traces:
 152      - samples,
 153      - outliers, regress, progress
 154      - average of normal samples (trending line)
 155
 156     :param in_data: Full data set.
 157     :param job_name: The name of job which generated the data.
 158     :param build_info: Information about the builds.
 159     :param show_trend_line: Show moving median (trending plot).
 160     :param name: Name of the plot
 161     :param color: Name of the color for the plot.
 162     :type in_data: OrderedDict
 163     :type job_name: str
 164     :type build_info: dict
 165     :type show_trend_line: bool
 166     :type name: str
 167     :type color: str
 168     :returns: Generated traces (list) and the evaluated result.
 169     :rtype: tuple(traces, result)
 170     """
 171
 172     data_x = list(in_data.keys())
 173     data_y = list(in_data.values())
 174
 175     hover_text = list()
 176     xaxis = list()
 177     for idx in data_x:
 178         date = build_info[job_name][str(idx)][0]
 179         hover_str = ("date: {date}<br>"
 180                      "value: {value:,}<br>"
 181                      "{sut}-ref: {build}<br>"
 182                      "csit-ref: mrr-{period}-build-{build_nr}<br>"
 183                      "testbed: {testbed}")
 184         if "dpdk" in job_name:
 185             hover_text.append(hover_str.format(
 186                 date=date,
 187                 value=int(in_data[idx].avg),
 188                 sut="dpdk",
 189                 build=build_info[job_name][str(idx)][1].rsplit('~', 1)[0],
 190                 period="weekly",
 191                 build_nr=idx,
 192                 testbed=build_info[job_name][str(idx)][2]))
 193         elif "vpp" in job_name:
 194             hover_text.append(hover_str.format(
 195                 date=date,
 196                 value=int(in_data[idx].avg),
 197                 sut="vpp",
 198                 build=build_info[job_name][str(idx)][1].rsplit('~', 1)[0],
 199                 period="daily",
 200                 build_nr=idx,
 201                 testbed=build_info[job_name][str(idx)][2]))
 202
 203         xaxis.append(datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]),
 204                               int(date[9:11]), int(date[12:])))
 205
 206     data_pd = OrderedDict()
 207     for key, value in zip(xaxis, data_y):
 208         data_pd[key] = value
 209
 210     anomaly_classification, avgs = classify_anomalies(data_pd)
 211
 212     anomalies = OrderedDict()
 213     anomalies_colors = list()
 214     anomalies_avgs = list()
 215     anomaly_color = {
 216         "regression": 0.0,
 217         "normal": 0.5,
 218         "progression": 1.0
 219     }
 220     if anomaly_classification:
 221         for idx, (key, value) in enumerate(data_pd.iteritems()):
 222             if anomaly_classification[idx] in \
 223                     ("outlier", "regression", "progression"):
 224                 anomalies[key] = value
 225                 anomalies_colors.append(
 226                     anomaly_color[anomaly_classification[idx]])
 227                 anomalies_avgs.append(avgs[idx])
 228         anomalies_colors.extend([0.0, 0.5, 1.0])
 229
 230     # Create traces
 231
 232     trace_samples = plgo.Scatter(
 233         x=xaxis,
 234         y=[y.avg for y in data_y],
 235         mode='markers',
 236         line={
 237             "width": 1
 238         },
 239         showlegend=True,
 240         legendgroup=name,
 241         name="{name}".format(name=name),
 242         marker={
 243             "size": 5,
 244             "color": color,
 245             "symbol": "circle",
 246         },
 247         text=hover_text,
 248         hoverinfo="text"
 249     )
 250     traces = [trace_samples, ]
 251
 252     if show_trend_line:
 253         trace_trend = plgo.Scatter(
 254             x=xaxis,
 255             y=avgs,
 256             mode='lines',
 257             line={
 258                 "shape": "linear",
 259                 "width": 1,
 260                 "color": color,
 261             },
 262             showlegend=False,
 263             legendgroup=name,
 264             name='{name}'.format(name=name),
 265             text=["trend: {0:,}".format(int(avg)) for avg in avgs],
 266             hoverinfo="text+name"
 267         )
 268         traces.append(trace_trend)
 269
 270     trace_anomalies = plgo.Scatter(
 271         x=anomalies.keys(),
 272         y=anomalies_avgs,
 273         mode='markers',
 274         hoverinfo="none",
 275         showlegend=False,
 276         legendgroup=name,
 277         name="{name}-anomalies".format(name=name),
 278         marker={
 279             "size": 15,
 280             "symbol": "circle-open",
 281             "color": anomalies_colors,
 282             "colorscale": [[0.00, "red"],
 283                            [0.33, "red"],
 284                            [0.33, "white"],
 285                            [0.66, "white"],
 286                            [0.66, "green"],
 287                            [1.00, "green"]],
 288             "showscale": True,
 289             "line": {
 290                 "width": 2
 291             },
 292             "colorbar": {
 293                 "y": 0.5,
 294                 "len": 0.8,
 295                 "title": "Circles Marking Data Classification",
 296                 "titleside": 'right',
 297                 "titlefont": {
 298                     "size": 14
 299                 },
 300                 "tickmode": 'array',
 301                 "tickvals": [0.167, 0.500, 0.833],
 302                 "ticktext": ["Regression", "Normal", "Progression"],
 303                 "ticks": "",
 304                 "ticklen": 0,
 305                 "tickangle": -90,
 306                 "thickness": 10
 307             }
 308         }
 309     )
 310     traces.append(trace_anomalies)
 311
 312     if anomaly_classification:
 313         return traces, anomaly_classification[-1]
 314     else:
 315         return traces, None
 316
 317
 318 def _generate_all_charts(spec, input_data):
 319     """Generate all charts specified in the specification file.
 320
 321     :param spec: Specification.
 322     :param input_data: Full data set.
 323     :type spec: Specification
 324     :type input_data: InputData
 325     """
 326
 327     def _generate_chart(_, data_q, graph):
 328         """Generates the chart.
 329         """
 330
 331         logs = list()
 332
 333         logging.info("  Generating the chart '{0}' ...".
 334                      format(graph.get("title", "")))
 335         logs.append(("INFO", "  Generating the chart '{0}' ...".
 336                      format(graph.get("title", ""))))
 337
 338         job_name = graph["data"].keys()[0]
 339
 340         csv_tbl = list()
 341         res = list()
 342
 343         # Transform the data
 344         logs.append(("INFO", "    Creating the data set for the {0} '{1}'.".
 345                      format(graph.get("type", ""), graph.get("title", ""))))
 346         data = input_data.filter_data(graph, continue_on_error=True)
 347         if data is None:
 348             logging.error("No data.")
 349             return
 350
 351         chart_data = dict()
 352         chart_tags = dict()
 353         for job, job_data in data.iteritems():
 354             if job != job_name:
 355                 continue
 356             for index, bld in job_data.items():
 357                 for test_name, test in bld.items():
 358                     if chart_data.get(test_name, None) is None:
 359                         chart_data[test_name] = OrderedDict()
 360                     try:
 361                         chart_data[test_name][int(index)] = \
 362                             test["result"]["receive-rate"]
 363                         chart_tags[test_name] = test.get("tags", None)
 364                     except (KeyError, TypeError):
 365                         pass
 366
 367         # Add items to the csv table:
 368         for tst_name, tst_data in chart_data.items():
 369             tst_lst = list()
 370             for bld in builds_dict[job_name]:
 371                 itm = tst_data.get(int(bld), '')
 372                 if not isinstance(itm, str):
 373                     itm = itm.avg
 374                 tst_lst.append(str(itm))
 375             csv_tbl.append("{0},".format(tst_name) + ",".join(tst_lst) + '\n')
 376
 377         # Generate traces:
 378         traces = list()
 379         index = 0
 380         groups = graph.get("groups", None)
 381         visibility = list()
 382
 383         if groups:
 384             for group in groups:
 385                 visible = list()
 386                 for tag in group:
 387                     for test_name, test_data in chart_data.items():
 388                         if not test_data:
 389                             logs.append(("WARNING",
 390                                          "No data for the test '{0}'".
 391                                          format(test_name)))
 392                             continue
 393                         if tag in chart_tags[test_name]:
 394                             message = "index: {index}, test: {test}".format(
 395                                 index=index, test=test_name)
 396                             test_name = test_name.split('.')[-1]
 397                             try:
 398                                 trace, rslt = _generate_trending_traces(
 399                                     test_data,
 400                                     job_name=job_name,
 401                                     build_info=build_info,
 402                                     name='-'.join(test_name.split('-')[2:-1]),
 403                                     color=COLORS[index])
 404                             except IndexError:
 405                                 message = "Out of colors: {}".format(message)
 406                                 logs.append(("ERROR", message))
 407                                 logging.error(message)
 408                                 index += 1
 409                                 continue
 410                             traces.extend(trace)
 411                             visible.extend([True for _ in range(len(trace))])
 412                             res.append(rslt)
 413                             index += 1
 414                             break
 415                 visibility.append(visible)
 416         else:
 417             for test_name, test_data in chart_data.items():
 418                 if not test_data:
 419                     logs.append(("WARNING", "No data for the test '{0}'".
 420                                  format(test_name)))
 421                     continue
 422                 message = "index: {index}, test: {test}".format(
 423                     index=index, test=test_name)
 424                 test_name = test_name.split('.')[-1]
 425                 try:
 426                     trace, rslt = _generate_trending_traces(
 427                         test_data,
 428                         job_name=job_name,
 429                         build_info=build_info,
 430                         name='-'.join(test_name.split('-')[2:-1]),
 431                         color=COLORS[index])
 432                 except IndexError:
 433                     message = "Out of colors: {}".format(message)
 434                     logs.append(("ERROR", message))
 435                     logging.error(message)
 436                     index += 1
 437                     continue
 438                 traces.extend(trace)
 439                 res.append(rslt)
 440                 index += 1
 441
 442         if traces:
 443             # Generate the chart:
 444             try:
 445                 layout = deepcopy(graph["layout"])
 446             except KeyError as err:
 447                 logging.error("Finished with error: No layout defined")
 448                 logging.error(repr(err))
 449                 return
 450             if groups:
 451                 show = list()
 452                 for i in range(len(visibility)):
 453                     visible = list()
 454                     for r in range(len(visibility)):
 455                         for _ in range(len(visibility[r])):
 456                             visible.append(i == r)
 457                     show.append(visible)
 458
 459                 buttons = list()
 460                 buttons.append(dict(
 461                     label="All",
 462                     method="update",
 463                     args=[{"visible": [True for _ in range(len(show[0]))]}, ]
 464                 ))
 465                 for i in range(len(groups)):
 466                     try:
 467                         label = graph["group-names"][i]
 468                     except (IndexError, KeyError):
 469                         label = "Group {num}".format(num=i + 1)
 470                     buttons.append(dict(
 471                         label=label,
 472                         method="update",
 473                         args=[{"visible": show[i]}, ]
 474                     ))
 475
 476                 layout['updatemenus'] = list([
 477                     dict(
 478                         active=0,
 479                         type="dropdown",
 480                         direction="down",
 481                         xanchor="left",
 482                         yanchor="bottom",
 483                         x=-0.12,
 484                         y=1.0,
 485                         buttons=buttons
 486                     )
 487                 ])
 488
 489             name_file = "{0}-{1}{2}".format(spec.cpta["output-file"],
 490                                             graph["output-file-name"],
 491                                             spec.cpta["output-file-type"])
 492
 493             logs.append(("INFO", "    Writing the file '{0}' ...".
 494                          format(name_file)))
 495             plpl = plgo.Figure(data=traces, layout=layout)
 496             try:
 497                 ploff.plot(plpl, show_link=False, auto_open=False,
 498                            filename=name_file)
 499             except plerr.PlotlyEmptyDataError:
 500                 logs.append(("WARNING", "No data for the plot. Skipped."))
 501
 502         data_out = {
 503             "job_name": job_name,
 504             "csv_table": csv_tbl,
 505             "results": res,
 506             "logs": logs
 507         }
 508         data_q.put(data_out)
 509
 510     builds_dict = dict()
 511     for job in spec.input["builds"].keys():
 512         if builds_dict.get(job, None) is None:
 513             builds_dict[job] = list()
 514         for build in spec.input["builds"][job]:
 515             status = build["status"]
 516             if status != "failed" and status != "not found" and \
 517                 status != "removed":
 518                 builds_dict[job].append(str(build["build"]))
 519
 520     # Create "build ID": "date" dict:
 521     build_info = dict()
 522     tb_tbl = spec.environment.get("testbeds", None)
 523     for job_name, job_data in builds_dict.items():
 524         if build_info.get(job_name, None) is None:
 525             build_info[job_name] = OrderedDict()
 526         for build in job_data:
 527             testbed = ""
 528             tb_ip = input_data.metadata(job_name, build).get("testbed", "")
 529             if tb_ip and tb_tbl:
 530                 testbed = tb_tbl.get(tb_ip, "")
 531             build_info[job_name][build] = (
 532                 input_data.metadata(job_name, build).get("generated", ""),
 533                 input_data.metadata(job_name, build).get("version", ""),
 534                 testbed
 535             )
 536
 537     work_queue = multiprocessing.JoinableQueue()
 538     manager = multiprocessing.Manager()
 539     data_queue = manager.Queue()
 540     cpus = multiprocessing.cpu_count()
 541
 542     workers = list()
 543     for cpu in range(cpus):
 544         worker = Worker(work_queue,
 545                         data_queue,
 546                         _generate_chart)
 547         worker.daemon = True
 548         worker.start()
 549         workers.append(worker)
 550         os.system("taskset -p -c {0} {1} > /dev/null 2>&1".
 551                   format(cpu, worker.pid))
 552
 553     for chart in spec.cpta["plots"]:
 554         work_queue.put((chart, ))
 555     work_queue.join()
 556
 557     anomaly_classifications = list()
 558
 559     # Create the header:
 560     csv_tables = dict()
 561     for job_name in builds_dict.keys():
 562         if csv_tables.get(job_name, None) is None:
 563             csv_tables[job_name] = list()
 564         header = "Build Number:," + ",".join(builds_dict[job_name]) + '\n'
 565         csv_tables[job_name].append(header)
 566         build_dates = [x[0] for x in build_info[job_name].values()]
 567         header = "Build Date:," + ",".join(build_dates) + '\n'
 568         csv_tables[job_name].append(header)
 569         versions = [x[1] for x in build_info[job_name].values()]
 570         header = "Version:," + ",".join(versions) + '\n'
 571         csv_tables[job_name].append(header)
 572
 573     while not data_queue.empty():
 574         result = data_queue.get()
 575
 576         anomaly_classifications.extend(result["results"])
 577         csv_tables[result["job_name"]].extend(result["csv_table"])
 578
 579         for item in result["logs"]:
 580             if item[0] == "INFO":
 581                 logging.info(item[1])
 582             elif item[0] == "ERROR":
 583                 logging.error(item[1])
 584             elif item[0] == "DEBUG":
 585                 logging.debug(item[1])
 586             elif item[0] == "CRITICAL":
 587                 logging.critical(item[1])
 588             elif item[0] == "WARNING":
 589                 logging.warning(item[1])
 590
 591     del data_queue
 592
 593     # Terminate all workers
 594     for worker in workers:
 595         worker.terminate()
 596         worker.join()
 597
 598     # Write the tables:
 599     for job_name, csv_table in csv_tables.items():
 600         file_name = spec.cpta["output-file"] + "-" + job_name + "-trending"
 601         with open("{0}.csv".format(file_name), 'w') as file_handler:
 602             file_handler.writelines(csv_table)
 603
 604         txt_table = None
 605         with open("{0}.csv".format(file_name), 'rb') as csv_file:
 606             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 607             line_nr = 0
 608             for row in csv_content:
 609                 if txt_table is None:
 610                     txt_table = prettytable.PrettyTable(row)
 611                 else:
 612                     if line_nr > 1:
 613                         for idx, item in enumerate(row):
 614                             try:
 615                                 row[idx] = str(round(float(item) / 1000000, 2))
 616                             except ValueError:
 617                                 pass
 618                     try:
 619                         txt_table.add_row(row)
 620                     except Exception as err:
 621                         logging.warning("Error occurred while generating TXT "
 622                                         "table:\n{0}".format(err))
 623                 line_nr += 1
 624             txt_table.align["Build Number:"] = "l"
 625         with open("{0}.txt".format(file_name), "w") as txt_file:
 626             txt_file.write(str(txt_table))
 627
 628     # Evaluate result:
 629     if anomaly_classifications:
 630         result = "PASS"
 631         for classification in anomaly_classifications:
 632             if classification == "regression" or classification == "outlier":
 633                 result = "FAIL"
 634                 break
 635     else:
 636         result = "FAIL"
 637
 638     logging.info("Partial results: {0}".format(anomaly_classifications))
 639     logging.info("Result: {0}".format(result))
 640
 641     return result