resources/tools/presentation/generator_cpta.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Generation of Continuous Performance Trending and Analysis.
  15 """
  16
  17 import logging
  18 import csv
  19
  20 from collections import OrderedDict
  21 from datetime import datetime
  22 from copy import deepcopy
  23
  24 import prettytable
  25 import plotly.offline as ploff
  26 import plotly.graph_objs as plgo
  27 import plotly.exceptions as plerr
  28
  29 from pal_utils import archive_input_data, execute_command, classify_anomalies
  30
  31
  32 # Command to build the html format of the report
  33 HTML_BUILDER = u'sphinx-build -v -c conf_cpta -a ' \
  34                u'-b html -E ' \
  35                u'-t html ' \
  36                u'-D version="{date}" ' \
  37                u'{working_dir} ' \
  38                u'{build_dir}/'
  39
  40 # .css file for the html format of the report
  41 THEME_OVERRIDES = u"""/* override table width restrictions */
  42 .wy-nav-content {
  43     max-width: 1200px !important;
  44 }
  45 .rst-content blockquote {
  46     margin-left: 0px;
  47     line-height: 18px;
  48     margin-bottom: 0px;
  49 }
  50 .wy-menu-vertical a {
  51     display: inline-block;
  52     line-height: 18px;
  53     padding: 0 2em;
  54     display: block;
  55     position: relative;
  56     font-size: 90%;
  57     color: #d9d9d9
  58 }
  59 .wy-menu-vertical li.current a {
  60     color: gray;
  61     border-right: solid 1px #c9c9c9;
  62     padding: 0 3em;
  63 }
  64 .wy-menu-vertical li.toctree-l2.current > a {
  65     background: #c9c9c9;
  66     padding: 0 3em;
  67 }
  68 .wy-menu-vertical li.toctree-l2.current li.toctree-l3 > a {
  69     display: block;
  70     background: #c9c9c9;
  71     padding: 0 4em;
  72 }
  73 .wy-menu-vertical li.toctree-l3.current li.toctree-l4 > a {
  74     display: block;
  75     background: #bdbdbd;
  76     padding: 0 5em;
  77 }
  78 .wy-menu-vertical li.on a, .wy-menu-vertical li.current > a {
  79     color: #404040;
  80     padding: 0 2em;
  81     font-weight: bold;
  82     position: relative;
  83     background: #fcfcfc;
  84     border: none;
  85         border-top-width: medium;
  86         border-bottom-width: medium;
  87         border-top-style: none;
  88         border-bottom-style: none;
  89         border-top-color: currentcolor;
  90         border-bottom-color: currentcolor;
  91     padding-left: 2em -4px;
  92 }
  93 """
  94
  95 COLORS = (
  96     u"#1A1110",
  97     u"#DA2647",
  98     u"#214FC6",
  99     u"#01786F",
 100     u"#BD8260",
 101     u"#FFD12A",
 102     u"#A6E7FF",
 103     u"#738276",
 104     u"#C95A49",
 105     u"#FC5A8D",
 106     u"#CEC8EF",
 107     u"#391285",
 108     u"#6F2DA8",
 109     u"#FF878D",
 110     u"#45A27D",
 111     u"#FFD0B9",
 112     u"#FD5240",
 113     u"#DB91EF",
 114     u"#44D7A8",
 115     u"#4F86F7",
 116     u"#84DE02",
 117     u"#FFCFF1",
 118     u"#614051"
 119 )
 120
 121
 122 def generate_cpta(spec, data):
 123     """Generate all formats and versions of the Continuous Performance Trending
 124     and Analysis.
 125
 126     :param spec: Specification read from the specification file.
 127     :param data: Full data set.
 128     :type spec: Specification
 129     :type data: InputData
 130     """
 131
 132     logging.info(u"Generating the Continuous Performance Trending and Analysis "
 133                  u"...")
 134
 135     ret_code = _generate_all_charts(spec, data)
 136
 137     cmd = HTML_BUILDER.format(
 138         date=datetime.utcnow().strftime(u'%Y-%m-%d %H:%M UTC'),
 139         working_dir=spec.environment[u'paths'][u'DIR[WORKING,SRC]'],
 140         build_dir=spec.environment[u'paths'][u'DIR[BUILD,HTML]'])
 141     execute_command(cmd)
 142
 143     with open(spec.environment[u'paths'][u'DIR[CSS_PATCH_FILE]'], u'w') as \
 144             css_file:
 145         css_file.write(THEME_OVERRIDES)
 146
 147     with open(spec.environment[u'paths'][u'DIR[CSS_PATCH_FILE2]'], u'w') as \
 148             css_file:
 149         css_file.write(THEME_OVERRIDES)
 150
 151     if spec.configuration.get(u"archive-inputs", True):
 152         archive_input_data(spec)
 153
 154     logging.info(u"Done.")
 155
 156     return ret_code
 157
 158
 159 def _generate_trending_traces(in_data, job_name, build_info,
 160                               name=u"", color=u"", incl_tests=u"MRR"):
 161     """Generate the trending traces:
 162      - samples,
 163      - outliers, regress, progress
 164      - average of normal samples (trending line)
 165
 166     :param in_data: Full data set.
 167     :param job_name: The name of job which generated the data.
 168     :param build_info: Information about the builds.
 169     :param name: Name of the plot
 170     :param color: Name of the color for the plot.
 171     :param incl_tests: Included tests, accepted values: MRR, NDR, PDR
 172     :type in_data: OrderedDict
 173     :type job_name: str
 174     :type build_info: dict
 175     :type name: str
 176     :type color: str
 177     :type incl_tests: str
 178     :returns: Generated traces (list) and the evaluated result.
 179     :rtype: tuple(traces, result)
 180     """
 181
 182     if incl_tests not in (u"MRR", u"NDR", u"PDR"):
 183         return list(), None
 184
 185     data_x = list(in_data.keys())
 186     data_y_pps = list()
 187     data_y_mpps = list()
 188     data_y_stdev = list()
 189     for item in in_data.values():
 190         data_y_pps.append(float(item[u"receive-rate"]))
 191         data_y_stdev.append(float(item[u"receive-stdev"]) / 1e6)
 192         data_y_mpps.append(float(item[u"receive-rate"]) / 1e6)
 193
 194     hover_text = list()
 195     xaxis = list()
 196     for index, key in enumerate(data_x):
 197         str_key = str(key)
 198         date = build_info[job_name][str_key][0]
 199         hover_str = (u"date: {date}<br>"
 200                      u"{property} [Mpps]: {value:.3f}<br>"
 201                      u"<stdev>"
 202                      u"{sut}-ref: {build}<br>"
 203                      u"csit-ref: {test}-{period}-build-{build_nr}<br>"
 204                      u"testbed: {testbed}")
 205         if incl_tests == u"MRR":
 206             hover_str = hover_str.replace(
 207                 u"<stdev>", f"stdev [Mpps]: {data_y_stdev[index]:.3f}<br>"
 208             )
 209         else:
 210             hover_str = hover_str.replace(u"<stdev>", u"")
 211         if u"dpdk" in job_name:
 212             hover_text.append(hover_str.format(
 213                 date=date,
 214                 property=u"average" if incl_tests == u"MRR" else u"throughput",
 215                 value=data_y_mpps[index],
 216                 sut=u"dpdk",
 217                 build=build_info[job_name][str_key][1].rsplit(u'~', 1)[0],
 218                 test=incl_tests.lower(),
 219                 period=u"weekly",
 220                 build_nr=str_key,
 221                 testbed=build_info[job_name][str_key][2]))
 222         elif u"vpp" in job_name:
 223             hover_text.append(hover_str.format(
 224                 date=date,
 225                 property=u"average" if incl_tests == u"MRR" else u"throughput",
 226                 value=data_y_mpps[index],
 227                 sut=u"vpp",
 228                 build=build_info[job_name][str_key][1].rsplit(u'~', 1)[0],
 229                 test=incl_tests.lower(),
 230                 period=u"daily" if incl_tests == u"MRR" else u"weekly",
 231                 build_nr=str_key,
 232                 testbed=build_info[job_name][str_key][2]))
 233
 234         xaxis.append(datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]),
 235                               int(date[9:11]), int(date[12:])))
 236
 237     data_pd = OrderedDict()
 238     for key, value in zip(xaxis, data_y_pps):
 239         data_pd[key] = value
 240
 241     anomaly_classification, avgs_pps, stdevs_pps = classify_anomalies(data_pd)
 242     avgs_mpps = [avg_pps / 1e6 for avg_pps in avgs_pps]
 243     stdevs_mpps = [stdev_pps / 1e6 for stdev_pps in stdevs_pps]
 244
 245     anomalies = OrderedDict()
 246     anomalies_colors = list()
 247     anomalies_avgs = list()
 248     anomaly_color = {
 249         u"regression": 0.0,
 250         u"normal": 0.5,
 251         u"progression": 1.0
 252     }
 253     if anomaly_classification:
 254         for index, (key, value) in enumerate(data_pd.items()):
 255             if anomaly_classification[index] in (u"regression", u"progression"):
 256                 anomalies[key] = value / 1e6
 257                 anomalies_colors.append(
 258                     anomaly_color[anomaly_classification[index]])
 259                 anomalies_avgs.append(avgs_mpps[index])
 260         anomalies_colors.extend([0.0, 0.5, 1.0])
 261
 262     # Create traces
 263
 264     trace_samples = plgo.Scatter(
 265         x=xaxis,
 266         y=data_y_mpps,
 267         mode=u"markers",
 268         line={
 269             u"width": 1
 270         },
 271         showlegend=True,
 272         legendgroup=name,
 273         name=f"{name}",
 274         marker={
 275             u"size": 5,
 276             u"color": color,
 277             u"symbol": u"circle",
 278         },
 279         text=hover_text,
 280         hoverinfo=u"text+name"
 281     )
 282     traces = [trace_samples, ]
 283
 284     trend_hover_text = list()
 285     for idx in range(len(data_x)):
 286         trend_hover_str = (
 287             f"trend [Mpps]: {avgs_mpps[idx]:.3f}<br>"
 288             f"stdev [Mpps]: {stdevs_mpps[idx]:.3f}"
 289         )
 290         trend_hover_text.append(trend_hover_str)
 291
 292     trace_trend = plgo.Scatter(
 293         x=xaxis,
 294         y=avgs_mpps,
 295         mode=u"lines",
 296         line={
 297             u"shape": u"linear",
 298             u"width": 1,
 299             u"color": color,
 300         },
 301         showlegend=False,
 302         legendgroup=name,
 303         name=f"{name}",
 304         text=trend_hover_text,
 305         hoverinfo=u"text+name"
 306     )
 307     traces.append(trace_trend)
 308
 309     trace_anomalies = plgo.Scatter(
 310         x=list(anomalies.keys()),
 311         y=anomalies_avgs,
 312         mode=u"markers",
 313         hoverinfo=u"none",
 314         showlegend=False,
 315         legendgroup=name,
 316         name=f"{name}-anomalies",
 317         marker={
 318             u"size": 15,
 319             u"symbol": u"circle-open",
 320             u"color": anomalies_colors,
 321             u"colorscale": [
 322                 [0.00, u"red"],
 323                 [0.33, u"red"],
 324                 [0.33, u"white"],
 325                 [0.66, u"white"],
 326                 [0.66, u"green"],
 327                 [1.00, u"green"]
 328             ],
 329             u"showscale": True,
 330             u"line": {
 331                 u"width": 2
 332             },
 333             u"colorbar": {
 334                 u"y": 0.5,
 335                 u"len": 0.8,
 336                 u"title": u"Circles Marking Data Classification",
 337                 u"titleside": u"right",
 338                 u"titlefont": {
 339                     u"size": 14
 340                 },
 341                 u"tickmode": u"array",
 342                 u"tickvals": [0.167, 0.500, 0.833],
 343                 u"ticktext": [u"Regression", u"Normal", u"Progression"],
 344                 u"ticks": u"",
 345                 u"ticklen": 0,
 346                 u"tickangle": -90,
 347                 u"thickness": 10
 348             }
 349         }
 350     )
 351     traces.append(trace_anomalies)
 352
 353     if anomaly_classification:
 354         return traces, anomaly_classification[-1]
 355
 356     return traces, None
 357
 358
 359 def _generate_all_charts(spec, input_data):
 360     """Generate all charts specified in the specification file.
 361
 362     :param spec: Specification.
 363     :param input_data: Full data set.
 364     :type spec: Specification
 365     :type input_data: InputData
 366     """
 367
 368     def _generate_chart(graph):
 369         """Generates the chart.
 370
 371         :param graph: The graph to be generated
 372         :type graph: dict
 373         :returns: Dictionary with the job name, csv table with results and
 374             list of tests classification results.
 375         :rtype: dict
 376         """
 377
 378         logging.info(f"  Generating the chart {graph.get(u'title', u'')} ...")
 379
 380         incl_tests = graph.get(u"include-tests", u"MRR")
 381
 382         job_name = list(graph[u"data"].keys())[0]
 383
 384         csv_tbl = list()
 385         res = dict()
 386
 387         # Transform the data
 388         logging.info(
 389             f"    Creating the data set for the {graph.get(u'type', u'')} "
 390             f"{graph.get(u'title', u'')}."
 391         )
 392
 393         if graph.get(u"include", None):
 394             data = input_data.filter_tests_by_name(
 395                 graph,
 396                 params=[u"type", u"result", u"throughput", u"tags"],
 397                 continue_on_error=True
 398             )
 399         else:
 400             data = input_data.filter_data(
 401                 graph,
 402                 params=[u"type", u"result", u"throughput", u"tags"],
 403                 continue_on_error=True)
 404
 405         if data is None or data.empty:
 406             logging.error(u"No data.")
 407             return dict()
 408
 409         chart_data = dict()
 410         chart_tags = dict()
 411         for job, job_data in data.items():
 412             if job != job_name:
 413                 continue
 414             for index, bld in job_data.items():
 415                 for test_name, test in bld.items():
 416                     if chart_data.get(test_name, None) is None:
 417                         chart_data[test_name] = OrderedDict()
 418                     try:
 419                         if incl_tests == u"MRR":
 420                             rate = test[u"result"][u"receive-rate"]
 421                             stdev = test[u"result"][u"receive-stdev"]
 422                         elif incl_tests == u"NDR":
 423                             rate = test[u"throughput"][u"NDR"][u"LOWER"]
 424                             stdev = float(u"nan")
 425                         elif incl_tests == u"PDR":
 426                             rate = test[u"throughput"][u"PDR"][u"LOWER"]
 427                             stdev = float(u"nan")
 428                         else:
 429                             continue
 430                         chart_data[test_name][int(index)] = {
 431                             u"receive-rate": rate,
 432                             u"receive-stdev": stdev
 433                         }
 434                         chart_tags[test_name] = test.get(u"tags", None)
 435                     except (KeyError, TypeError):
 436                         pass
 437
 438         # Add items to the csv table:
 439         for tst_name, tst_data in chart_data.items():
 440             tst_lst = list()
 441             for bld in builds_dict[job_name]:
 442                 itm = tst_data.get(int(bld), dict())
 443                 # CSIT-1180: Itm will be list, compute stats.
 444                 try:
 445                     tst_lst.append(str(itm.get(u"receive-rate", u"")))
 446                 except AttributeError:
 447                     tst_lst.append(u"")
 448             csv_tbl.append(f"{tst_name}," + u",".join(tst_lst) + u'\n')
 449
 450         # Generate traces:
 451         traces = list()
 452         index = 0
 453         groups = graph.get(u"groups", None)
 454         visibility = list()
 455
 456         if groups:
 457             for group in groups:
 458                 visible = list()
 459                 for tag in group:
 460                     for tst_name, test_data in chart_data.items():
 461                         logging.info(tst_name)
 462                         logging.info(tst_data)
 463                         logging.info(u"---------------------------------------")
 464                         if not test_data:
 465                             logging.warning(f"No data for the test {tst_name}")
 466                             continue
 467                         if tag not in chart_tags[tst_name]:
 468                             continue
 469                         try:
 470                             trace, rslt = _generate_trending_traces(
 471                                 test_data,
 472                                 job_name=job_name,
 473                                 build_info=build_info,
 474                                 name=u'-'.join(tst_name.split(u'.')[-1].
 475                                                split(u'-')[2:-1]),
 476                                 color=COLORS[index],
 477                                 incl_tests=incl_tests
 478                             )
 479                         except IndexError:
 480                             logging.error(f"Out of colors: index: "
 481                                           f"{index}, test: {tst_name}")
 482                             index += 1
 483                             continue
 484                         traces.extend(trace)
 485                         visible.extend([True for _ in range(len(trace))])
 486                         res[tst_name] = rslt
 487                         index += 1
 488                         break
 489                 visibility.append(visible)
 490         else:
 491             for tst_name, test_data in chart_data.items():
 492                 if not test_data:
 493                     logging.warning(f"No data for the test {tst_name}")
 494                     continue
 495                 try:
 496                     trace, rslt = _generate_trending_traces(
 497                         test_data,
 498                         job_name=job_name,
 499                         build_info=build_info,
 500                         name=u'-'.join(
 501                             tst_name.split(u'.')[-1].split(u'-')[2:-1]),
 502                         color=COLORS[index],
 503                         incl_tests=incl_tests
 504                     )
 505                 except IndexError:
 506                     logging.error(
 507                         f"Out of colors: index: {index}, test: {tst_name}"
 508                     )
 509                     index += 1
 510                     continue
 511                 traces.extend(trace)
 512                 res[tst_name] = rslt
 513                 index += 1
 514
 515         if traces:
 516             # Generate the chart:
 517             try:
 518                 layout = deepcopy(graph[u"layout"])
 519             except KeyError as err:
 520                 logging.error(u"Finished with error: No layout defined")
 521                 logging.error(repr(err))
 522                 return dict()
 523             if groups:
 524                 show = list()
 525                 for i in range(len(visibility)):
 526                     visible = list()
 527                     for vis_idx, _ in enumerate(visibility):
 528                         for _ in range(len(visibility[vis_idx])):
 529                             visible.append(i == vis_idx)
 530                     show.append(visible)
 531
 532                 buttons = list()
 533                 buttons.append(dict(
 534                     label=u"All",
 535                     method=u"update",
 536                     args=[{u"visible": [True for _ in range(len(show[0]))]}, ]
 537                 ))
 538                 for i in range(len(groups)):
 539                     try:
 540                         label = graph[u"group-names"][i]
 541                     except (IndexError, KeyError):
 542                         label = f"Group {i + 1}"
 543                     buttons.append(dict(
 544                         label=label,
 545                         method=u"update",
 546                         args=[{u"visible": show[i]}, ]
 547                     ))
 548
 549                 layout[u"updatemenus"] = list([
 550                     dict(
 551                         active=0,
 552                         type=u"dropdown",
 553                         direction=u"down",
 554                         xanchor=u"left",
 555                         yanchor=u"bottom",
 556                         x=-0.12,
 557                         y=1.0,
 558                         buttons=buttons
 559                     )
 560                 ])
 561
 562             name_file = (
 563                 f"{spec.cpta[u'output-file']}/{graph[u'output-file-name']}"
 564                 f"{spec.cpta[u'output-file-type']}")
 565
 566             logging.info(f"    Writing the file {name_file} ...")
 567             plpl = plgo.Figure(data=traces, layout=layout)
 568             try:
 569                 ploff.plot(plpl, show_link=False, auto_open=False,
 570                            filename=name_file)
 571             except plerr.PlotlyEmptyDataError:
 572                 logging.warning(u"No data for the plot. Skipped.")
 573
 574         return {u"job_name": job_name, u"csv_table": csv_tbl, u"results": res}
 575
 576     builds_dict = dict()
 577     for job in spec.input[u"builds"].keys():
 578         if builds_dict.get(job, None) is None:
 579             builds_dict[job] = list()
 580         for build in spec.input[u"builds"][job]:
 581             status = build[u"status"]
 582             if status not in (u"failed", u"not found", u"removed", None):
 583                 builds_dict[job].append(str(build[u"build"]))
 584
 585     # Create "build ID": "date" dict:
 586     build_info = dict()
 587     tb_tbl = spec.environment.get(u"testbeds", None)
 588     for job_name, job_data in builds_dict.items():
 589         if build_info.get(job_name, None) is None:
 590             build_info[job_name] = OrderedDict()
 591         for build in job_data:
 592             testbed = u""
 593             tb_ip = input_data.metadata(job_name, build).get(u"testbed", u"")
 594             if tb_ip and tb_tbl:
 595                 testbed = tb_tbl.get(tb_ip, u"")
 596             build_info[job_name][build] = (
 597                 input_data.metadata(job_name, build).get(u"generated", u""),
 598                 input_data.metadata(job_name, build).get(u"version", u""),
 599                 testbed
 600             )
 601
 602     anomaly_classifications = dict()
 603
 604     # Create the table header:
 605     csv_tables = dict()
 606     for job_name in builds_dict:
 607         if csv_tables.get(job_name, None) is None:
 608             csv_tables[job_name] = list()
 609         header = f"Build Number:,{u','.join(builds_dict[job_name])}\n"
 610         csv_tables[job_name].append(header)
 611         build_dates = [x[0] for x in build_info[job_name].values()]
 612         header = f"Build Date:,{u','.join(build_dates)}\n"
 613         csv_tables[job_name].append(header)
 614         versions = [x[1] for x in build_info[job_name].values()]
 615         header = f"Version:,{u','.join(versions)}\n"
 616         csv_tables[job_name].append(header)
 617
 618     for chart in spec.cpta[u"plots"]:
 619         result = _generate_chart(chart)
 620         if not result:
 621             continue
 622
 623         csv_tables[result[u"job_name"]].extend(result[u"csv_table"])
 624
 625         if anomaly_classifications.get(result[u"job_name"], None) is None:
 626             anomaly_classifications[result[u"job_name"]] = dict()
 627         anomaly_classifications[result[u"job_name"]].update(result[u"results"])
 628
 629     # Write the tables:
 630     for job_name, csv_table in csv_tables.items():
 631         file_name = f"{spec.cpta[u'output-file']}/{job_name}-trending"
 632         with open(f"{file_name}.csv", u"wt") as file_handler:
 633             file_handler.writelines(csv_table)
 634
 635         txt_table = None
 636         with open(f"{file_name}.csv", u"rt") as csv_file:
 637             csv_content = csv.reader(csv_file, delimiter=u',', quotechar=u'"')
 638             line_nr = 0
 639             for row in csv_content:
 640                 if txt_table is None:
 641                     txt_table = prettytable.PrettyTable(row)
 642                 else:
 643                     if line_nr > 1:
 644                         for idx, item in enumerate(row):
 645                             try:
 646                                 row[idx] = str(round(float(item) / 1000000, 2))
 647                             except ValueError:
 648                                 pass
 649                     try:
 650                         txt_table.add_row(row)
 651                     # PrettyTable raises Exception
 652                     except Exception as err:
 653                         logging.warning(
 654                             f"Error occurred while generating TXT table:\n{err}"
 655                         )
 656                 line_nr += 1
 657             txt_table.align[u"Build Number:"] = u"l"
 658         with open(f"{file_name}.txt", u"wt") as txt_file:
 659             txt_file.write(str(txt_table))
 660
 661     # Evaluate result:
 662     if anomaly_classifications:
 663         result = u"PASS"
 664         for job_name, job_data in anomaly_classifications.items():
 665             file_name = \
 666                 f"{spec.cpta[u'output-file']}/regressions-{job_name}.txt"
 667             with open(file_name, u'w') as txt_file:
 668                 for test_name, classification in job_data.items():
 669                     if classification == u"regression":
 670                         txt_file.write(test_name + u'\n')
 671                     if classification in (u"regression", u"outlier"):
 672                         result = u"FAIL"
 673             file_name = \
 674                 f"{spec.cpta[u'output-file']}/progressions-{job_name}.txt"
 675             with open(file_name, u'w') as txt_file:
 676                 for test_name, classification in job_data.items():
 677                     if classification == u"progression":
 678                         txt_file.write(test_name + u'\n')
 679     else:
 680         result = u"FAIL"
 681
 682     logging.info(f"Partial results: {anomaly_classifications}")
 683     logging.info(f"Result: {result}")
 684
 685     return result