resources/tools/presentation/generator_cpta.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Generation of Continuous Performance Trending and Analysis.
  15 """
  16
  17 import logging
  18 import csv
  19
  20 from collections import OrderedDict
  21 from datetime import datetime
  22 from copy import deepcopy
  23
  24 import prettytable
  25 import plotly.offline as ploff
  26 import plotly.graph_objs as plgo
  27 import plotly.exceptions as plerr
  28
  29 from pal_utils import archive_input_data, execute_command, classify_anomalies
  30
  31
  32 # Command to build the html format of the report
  33 HTML_BUILDER = u'sphinx-build -v -c conf_cpta -a ' \
  34                u'-b html -E ' \
  35                u'-t html ' \
  36                u'-D version="{date}" ' \
  37                u'{working_dir} ' \
  38                u'{build_dir}/'
  39
  40 # .css file for the html format of the report
  41 THEME_OVERRIDES = u"""/* override table width restrictions */
  42 .wy-nav-content {
  43     max-width: 1200px !important;
  44 }
  45 .rst-content blockquote {
  46     margin-left: 0px;
  47     line-height: 18px;
  48     margin-bottom: 0px;
  49 }
  50 .wy-menu-vertical a {
  51     display: inline-block;
  52     line-height: 18px;
  53     padding: 0 2em;
  54     display: block;
  55     position: relative;
  56     font-size: 90%;
  57     color: #d9d9d9
  58 }
  59 .wy-menu-vertical li.current a {
  60     color: gray;
  61     border-right: solid 1px #c9c9c9;
  62     padding: 0 3em;
  63 }
  64 .wy-menu-vertical li.toctree-l2.current > a {
  65     background: #c9c9c9;
  66     padding: 0 3em;
  67 }
  68 .wy-menu-vertical li.toctree-l2.current li.toctree-l3 > a {
  69     display: block;
  70     background: #c9c9c9;
  71     padding: 0 4em;
  72 }
  73 .wy-menu-vertical li.toctree-l3.current li.toctree-l4 > a {
  74     display: block;
  75     background: #bdbdbd;
  76     padding: 0 5em;
  77 }
  78 .wy-menu-vertical li.on a, .wy-menu-vertical li.current > a {
  79     color: #404040;
  80     padding: 0 2em;
  81     font-weight: bold;
  82     position: relative;
  83     background: #fcfcfc;
  84     border: none;
  85         border-top-width: medium;
  86         border-bottom-width: medium;
  87         border-top-style: none;
  88         border-bottom-style: none;
  89         border-top-color: currentcolor;
  90         border-bottom-color: currentcolor;
  91     padding-left: 2em -4px;
  92 }
  93 """
  94
  95 COLORS = (
  96     u"#1A1110",
  97     u"#DA2647",
  98     u"#214FC6",
  99     u"#01786F",
 100     u"#BD8260",
 101     u"#FFD12A",
 102     u"#A6E7FF",
 103     u"#738276",
 104     u"#C95A49",
 105     u"#FC5A8D",
 106     u"#CEC8EF",
 107     u"#391285",
 108     u"#6F2DA8",
 109     u"#FF878D",
 110     u"#45A27D",
 111     u"#FFD0B9",
 112     u"#FD5240",
 113     u"#DB91EF",
 114     u"#44D7A8",
 115     u"#4F86F7",
 116     u"#84DE02",
 117     u"#FFCFF1",
 118     u"#614051"
 119 )
 120
 121
 122 def generate_cpta(spec, data):
 123     """Generate all formats and versions of the Continuous Performance Trending
 124     and Analysis.
 125
 126     :param spec: Specification read from the specification file.
 127     :param data: Full data set.
 128     :type spec: Specification
 129     :type data: InputData
 130     """
 131
 132     logging.info(u"Generating the Continuous Performance Trending and Analysis "
 133                  u"...")
 134
 135     ret_code = _generate_all_charts(spec, data)
 136
 137     cmd = HTML_BUILDER.format(
 138         date=datetime.utcnow().strftime(u'%Y-%m-%d %H:%M UTC'),
 139         working_dir=spec.environment[u'paths'][u'DIR[WORKING,SRC]'],
 140         build_dir=spec.environment[u'paths'][u'DIR[BUILD,HTML]'])
 141     execute_command(cmd)
 142
 143     with open(spec.environment[u'paths'][u'DIR[CSS_PATCH_FILE]'], u'w') as \
 144             css_file:
 145         css_file.write(THEME_OVERRIDES)
 146
 147     with open(spec.environment[u'paths'][u'DIR[CSS_PATCH_FILE2]'], u'w') as \
 148             css_file:
 149         css_file.write(THEME_OVERRIDES)
 150
 151     if spec.configuration.get(u"archive-inputs", True):
 152         archive_input_data(spec)
 153
 154     logging.info(u"Done.")
 155
 156     return ret_code
 157
 158
 159 def _generate_trending_traces(in_data, job_name, build_info,
 160                               name=u"", color=u"", incl_tests=u"MRR"):
 161     """Generate the trending traces:
 162      - samples,
 163      - outliers, regress, progress
 164      - average of normal samples (trending line)
 165
 166     :param in_data: Full data set.
 167     :param job_name: The name of job which generated the data.
 168     :param build_info: Information about the builds.
 169     :param name: Name of the plot
 170     :param color: Name of the color for the plot.
 171     :param incl_tests: Included tests, accepted values: MRR, NDR, PDR
 172     :type in_data: OrderedDict
 173     :type job_name: str
 174     :type build_info: dict
 175     :type name: str
 176     :type color: str
 177     :type incl_tests: str
 178     :returns: Generated traces (list) and the evaluated result.
 179     :rtype: tuple(traces, result)
 180     """
 181
 182     if incl_tests not in (u"MRR", u"NDR", u"PDR"):
 183         return list(), None
 184
 185     data_x = list(in_data.keys())
 186     data_y_pps = list()
 187     data_y_mpps = list()
 188     data_y_stdev = list()
 189     for item in in_data.values():
 190         data_y_pps.append(float(item[u"receive-rate"]))
 191         data_y_stdev.append(float(item[u"receive-stdev"]) / 1e6)
 192         data_y_mpps.append(float(item[u"receive-rate"]) / 1e6)
 193
 194     hover_text = list()
 195     xaxis = list()
 196     for index, key in enumerate(data_x):
 197         str_key = str(key)
 198         date = build_info[job_name][str_key][0]
 199         hover_str = (u"date: {date}<br>"
 200                      u"{property} [Mpps]: {value:.3f}<br>"
 201                      u"<stdev>"
 202                      u"{sut}-ref: {build}<br>"
 203                      u"csit-ref: {test}-{period}-build-{build_nr}<br>"
 204                      u"testbed: {testbed}")
 205         if incl_tests == u"MRR":
 206             hover_str = hover_str.replace(
 207                 u"<stdev>", f"stdev [Mpps]: {data_y_stdev[index]:.3f}<br>"
 208             )
 209         else:
 210             hover_str = hover_str.replace(u"<stdev>", u"")
 211         if u"dpdk" in job_name:
 212             hover_text.append(hover_str.format(
 213                 date=date,
 214                 property=u"average" if incl_tests == u"MRR" else u"throughput",
 215                 value=data_y_mpps[index],
 216                 sut=u"dpdk",
 217                 build=build_info[job_name][str_key][1].rsplit(u'~', 1)[0],
 218                 test=incl_tests.lower(),
 219                 period=u"weekly",
 220                 build_nr=str_key,
 221                 testbed=build_info[job_name][str_key][2]))
 222         elif u"vpp" in job_name:
 223             hover_text.append(hover_str.format(
 224                 date=date,
 225                 property=u"average" if incl_tests == u"MRR" else u"throughput",
 226                 value=data_y_mpps[index],
 227                 sut=u"vpp",
 228                 build=build_info[job_name][str_key][1].rsplit(u'~', 1)[0],
 229                 test=incl_tests.lower(),
 230                 period=u"daily" if incl_tests == u"MRR" else u"weekly",
 231                 build_nr=str_key,
 232                 testbed=build_info[job_name][str_key][2]))
 233
 234         xaxis.append(datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]),
 235                               int(date[9:11]), int(date[12:])))
 236
 237     data_pd = OrderedDict()
 238     for key, value in zip(xaxis, data_y_pps):
 239         data_pd[key] = value
 240
 241     anomaly_classification, avgs_pps, stdevs_pps = classify_anomalies(data_pd)
 242     avgs_mpps = [avg_pps / 1e6 for avg_pps in avgs_pps]
 243     stdevs_mpps = [stdev_pps / 1e6 for stdev_pps in stdevs_pps]
 244
 245     anomalies = OrderedDict()
 246     anomalies_colors = list()
 247     anomalies_avgs = list()
 248     anomaly_color = {
 249         u"regression": 0.0,
 250         u"normal": 0.5,
 251         u"progression": 1.0
 252     }
 253     if anomaly_classification:
 254         for index, (key, value) in enumerate(data_pd.items()):
 255             if anomaly_classification[index] in (u"regression", u"progression"):
 256                 anomalies[key] = value / 1e6
 257                 anomalies_colors.append(
 258                     anomaly_color[anomaly_classification[index]])
 259                 anomalies_avgs.append(avgs_mpps[index])
 260         anomalies_colors.extend([0.0, 0.5, 1.0])
 261
 262     # Create traces
 263
 264     trace_samples = plgo.Scatter(
 265         x=xaxis,
 266         y=data_y_mpps,
 267         mode=u"markers",
 268         line={
 269             u"width": 1
 270         },
 271         showlegend=True,
 272         legendgroup=name,
 273         name=f"{name}",
 274         marker={
 275             u"size": 5,
 276             u"color": color,
 277             u"symbol": u"circle",
 278         },
 279         text=hover_text,
 280         hoverinfo=u"text+name"
 281     )
 282     traces = [trace_samples, ]
 283
 284     trend_hover_text = list()
 285     for idx in range(len(data_x)):
 286         trend_hover_str = (
 287             f"trend [Mpps]: {avgs_mpps[idx]:.3f}<br>"
 288             f"stdev [Mpps]: {stdevs_mpps[idx]:.3f}"
 289         )
 290         trend_hover_text.append(trend_hover_str)
 291
 292     trace_trend = plgo.Scatter(
 293         x=xaxis,
 294         y=avgs_mpps,
 295         mode=u"lines",
 296         line={
 297             u"shape": u"linear",
 298             u"width": 1,
 299             u"color": color,
 300         },
 301         showlegend=False,
 302         legendgroup=name,
 303         name=f"{name}",
 304         text=trend_hover_text,
 305         hoverinfo=u"text+name"
 306     )
 307     traces.append(trace_trend)
 308
 309     trace_anomalies = plgo.Scatter(
 310         x=list(anomalies.keys()),
 311         y=anomalies_avgs,
 312         mode=u"markers",
 313         hoverinfo=u"none",
 314         showlegend=False,
 315         legendgroup=name,
 316         name=f"{name}-anomalies",
 317         marker={
 318             u"size": 15,
 319             u"symbol": u"circle-open",
 320             u"color": anomalies_colors,
 321             u"colorscale": [
 322                 [0.00, u"red"],
 323                 [0.33, u"red"],
 324                 [0.33, u"white"],
 325                 [0.66, u"white"],
 326                 [0.66, u"green"],
 327                 [1.00, u"green"]
 328             ],
 329             u"showscale": True,
 330             u"line": {
 331                 u"width": 2
 332             },
 333             u"colorbar": {
 334                 u"y": 0.5,
 335                 u"len": 0.8,
 336                 u"title": u"Circles Marking Data Classification",
 337                 u"titleside": u"right",
 338                 u"titlefont": {
 339                     u"size": 14
 340                 },
 341                 u"tickmode": u"array",
 342                 u"tickvals": [0.167, 0.500, 0.833],
 343                 u"ticktext": [u"Regression", u"Normal", u"Progression"],
 344                 u"ticks": u"",
 345                 u"ticklen": 0,
 346                 u"tickangle": -90,
 347                 u"thickness": 10
 348             }
 349         }
 350     )
 351     traces.append(trace_anomalies)
 352
 353     if anomaly_classification:
 354         return traces, anomaly_classification[-1]
 355
 356     return traces, None
 357
 358
 359 def _generate_all_charts(spec, input_data):
 360     """Generate all charts specified in the specification file.
 361
 362     :param spec: Specification.
 363     :param input_data: Full data set.
 364     :type spec: Specification
 365     :type input_data: InputData
 366     """
 367
 368     def _generate_chart(graph):
 369         """Generates the chart.
 370
 371         :param graph: The graph to be generated
 372         :type graph: dict
 373         :returns: Dictionary with the job name, csv table with results and
 374             list of tests classification results.
 375         :rtype: dict
 376         """
 377
 378         logging.info(f"  Generating the chart {graph.get(u'title', u'')} ...")
 379
 380         incl_tests = graph.get(u"include-tests", u"MRR")
 381
 382         job_name = list(graph[u"data"].keys())[0]
 383
 384         csv_tbl = list()
 385         res = dict()
 386
 387         # Transform the data
 388         logging.info(
 389             f"    Creating the data set for the {graph.get(u'type', u'')} "
 390             f"{graph.get(u'title', u'')}."
 391         )
 392
 393         if graph.get(u"include", None):
 394             data = input_data.filter_tests_by_name(
 395                 graph,
 396                 params=[u"type", u"result", u"throughput", u"tags"],
 397                 continue_on_error=True
 398             )
 399         else:
 400             data = input_data.filter_data(
 401                 graph,
 402                 params=[u"type", u"result", u"throughput", u"tags"],
 403                 continue_on_error=True)
 404
 405         if data is None or data.empty:
 406             logging.error(u"No data.")
 407             return dict()
 408
 409         chart_data = dict()
 410         chart_tags = dict()
 411         for job, job_data in data.items():
 412             if job != job_name:
 413                 continue
 414             for index, bld in job_data.items():
 415                 for test_name, test in bld.items():
 416                     if chart_data.get(test_name, None) is None:
 417                         chart_data[test_name] = OrderedDict()
 418                     try:
 419                         if incl_tests == u"MRR":
 420                             rate = test[u"result"][u"receive-rate"]
 421                             stdev = test[u"result"][u"receive-stdev"]
 422                         elif incl_tests == u"NDR":
 423                             rate = test[u"throughput"][u"NDR"][u"LOWER"]
 424                             stdev = float(u"nan")
 425                         elif incl_tests == u"PDR":
 426                             rate = test[u"throughput"][u"PDR"][u"LOWER"]
 427                             stdev = float(u"nan")
 428                         else:
 429                             continue
 430                         chart_data[test_name][int(index)] = {
 431                             u"receive-rate": rate,
 432                             u"receive-stdev": stdev
 433                         }
 434                         chart_tags[test_name] = test.get(u"tags", None)
 435                     except (KeyError, TypeError):
 436                         pass
 437
 438         # Add items to the csv table:
 439         for tst_name, tst_data in chart_data.items():
 440             tst_lst = list()
 441             for bld in builds_dict[job_name]:
 442                 itm = tst_data.get(int(bld), dict())
 443                 # CSIT-1180: Itm will be list, compute stats.
 444                 try:
 445                     tst_lst.append(str(itm.get(u"receive-rate", u"")))
 446                 except AttributeError:
 447                     tst_lst.append(u"")
 448             csv_tbl.append(f"{tst_name}," + u",".join(tst_lst) + u'\n')
 449
 450         # Generate traces:
 451         traces = list()
 452         index = 0
 453         groups = graph.get(u"groups", None)
 454         visibility = list()
 455
 456         if groups:
 457             for group in groups:
 458                 visible = list()
 459                 for tag in group:
 460                     for tst_name, test_data in chart_data.items():
 461                         if not test_data:
 462                             logging.warning(f"No data for the test {tst_name}")
 463                             continue
 464                         if tag not in chart_tags[tst_name]:
 465                             continue
 466                         try:
 467                             trace, rslt = _generate_trending_traces(
 468                                 test_data,
 469                                 job_name=job_name,
 470                                 build_info=build_info,
 471                                 name=u'-'.join(tst_name.split(u'.')[-1].
 472                                                split(u'-')[2:-1]),
 473                                 color=COLORS[index],
 474                                 incl_tests=incl_tests
 475                             )
 476                         except IndexError:
 477                             logging.error(f"Out of colors: index: "
 478                                           f"{index}, test: {tst_name}")
 479                             index += 1
 480                             continue
 481                         traces.extend(trace)
 482                         visible.extend([True for _ in range(len(trace))])
 483                         res[tst_name] = rslt
 484                         index += 1
 485                         break
 486                 visibility.append(visible)
 487         else:
 488             for tst_name, test_data in chart_data.items():
 489                 if not test_data:
 490                     logging.warning(f"No data for the test {tst_name}")
 491                     continue
 492                 try:
 493                     trace, rslt = _generate_trending_traces(
 494                         test_data,
 495                         job_name=job_name,
 496                         build_info=build_info,
 497                         name=u'-'.join(
 498                             tst_name.split(u'.')[-1].split(u'-')[2:-1]),
 499                         color=COLORS[index],
 500                         incl_tests=incl_tests
 501                     )
 502                 except IndexError:
 503                     logging.error(
 504                         f"Out of colors: index: {index}, test: {tst_name}"
 505                     )
 506                     index += 1
 507                     continue
 508                 traces.extend(trace)
 509                 res[tst_name] = rslt
 510                 index += 1
 511
 512         if traces:
 513             # Generate the chart:
 514             try:
 515                 layout = deepcopy(graph[u"layout"])
 516             except KeyError as err:
 517                 logging.error(u"Finished with error: No layout defined")
 518                 logging.error(repr(err))
 519                 return dict()
 520             if groups:
 521                 show = list()
 522                 for i in range(len(visibility)):
 523                     visible = list()
 524                     for vis_idx, _ in enumerate(visibility):
 525                         for _ in range(len(visibility[vis_idx])):
 526                             visible.append(i == vis_idx)
 527                     show.append(visible)
 528
 529                 buttons = list()
 530                 buttons.append(dict(
 531                     label=u"All",
 532                     method=u"update",
 533                     args=[{u"visible": [True for _ in range(len(show[0]))]}, ]
 534                 ))
 535                 for i in range(len(groups)):
 536                     try:
 537                         label = graph[u"group-names"][i]
 538                     except (IndexError, KeyError):
 539                         label = f"Group {i + 1}"
 540                     buttons.append(dict(
 541                         label=label,
 542                         method=u"update",
 543                         args=[{u"visible": show[i]}, ]
 544                     ))
 545
 546                 layout[u"updatemenus"] = list([
 547                     dict(
 548                         active=0,
 549                         type=u"dropdown",
 550                         direction=u"down",
 551                         xanchor=u"left",
 552                         yanchor=u"bottom",
 553                         x=-0.12,
 554                         y=1.0,
 555                         buttons=buttons
 556                     )
 557                 ])
 558
 559             name_file = (
 560                 f"{spec.cpta[u'output-file']}/{graph[u'output-file-name']}"
 561                 f"{spec.cpta[u'output-file-type']}")
 562
 563             logging.info(f"    Writing the file {name_file} ...")
 564             plpl = plgo.Figure(data=traces, layout=layout)
 565             try:
 566                 ploff.plot(plpl, show_link=False, auto_open=False,
 567                            filename=name_file)
 568             except plerr.PlotlyEmptyDataError:
 569                 logging.warning(u"No data for the plot. Skipped.")
 570
 571         return {u"job_name": job_name, u"csv_table": csv_tbl, u"results": res}
 572
 573     builds_dict = dict()
 574     for job in spec.input[u"builds"].keys():
 575         if builds_dict.get(job, None) is None:
 576             builds_dict[job] = list()
 577         for build in spec.input[u"builds"][job]:
 578             status = build[u"status"]
 579             if status not in (u"failed", u"not found", u"removed", None):
 580                 builds_dict[job].append(str(build[u"build"]))
 581
 582     # Create "build ID": "date" dict:
 583     build_info = dict()
 584     tb_tbl = spec.environment.get(u"testbeds", None)
 585     for job_name, job_data in builds_dict.items():
 586         if build_info.get(job_name, None) is None:
 587             build_info[job_name] = OrderedDict()
 588         for build in job_data:
 589             testbed = u""
 590             tb_ip = input_data.metadata(job_name, build).get(u"testbed", u"")
 591             if tb_ip and tb_tbl:
 592                 testbed = tb_tbl.get(tb_ip, u"")
 593             build_info[job_name][build] = (
 594                 input_data.metadata(job_name, build).get(u"generated", u""),
 595                 input_data.metadata(job_name, build).get(u"version", u""),
 596                 testbed
 597             )
 598
 599     anomaly_classifications = dict()
 600
 601     # Create the table header:
 602     csv_tables = dict()
 603     for job_name in builds_dict:
 604         if csv_tables.get(job_name, None) is None:
 605             csv_tables[job_name] = list()
 606         header = f"Build Number:,{u','.join(builds_dict[job_name])}\n"
 607         csv_tables[job_name].append(header)
 608         build_dates = [x[0] for x in build_info[job_name].values()]
 609         header = f"Build Date:,{u','.join(build_dates)}\n"
 610         csv_tables[job_name].append(header)
 611         versions = [x[1] for x in build_info[job_name].values()]
 612         header = f"Version:,{u','.join(versions)}\n"
 613         csv_tables[job_name].append(header)
 614
 615     for chart in spec.cpta[u"plots"]:
 616         result = _generate_chart(chart)
 617         if not result:
 618             continue
 619
 620         csv_tables[result[u"job_name"]].extend(result[u"csv_table"])
 621
 622         if anomaly_classifications.get(result[u"job_name"], None) is None:
 623             anomaly_classifications[result[u"job_name"]] = dict()
 624         anomaly_classifications[result[u"job_name"]].update(result[u"results"])
 625
 626     # Write the tables:
 627     for job_name, csv_table in csv_tables.items():
 628         file_name = f"{spec.cpta[u'output-file']}/{job_name}-trending"
 629         with open(f"{file_name}.csv", u"wt") as file_handler:
 630             file_handler.writelines(csv_table)
 631
 632         txt_table = None
 633         with open(f"{file_name}.csv", u"rt") as csv_file:
 634             csv_content = csv.reader(csv_file, delimiter=u',', quotechar=u'"')
 635             line_nr = 0
 636             for row in csv_content:
 637                 if txt_table is None:
 638                     txt_table = prettytable.PrettyTable(row)
 639                 else:
 640                     if line_nr > 1:
 641                         for idx, item in enumerate(row):
 642                             try:
 643                                 row[idx] = str(round(float(item) / 1000000, 2))
 644                             except ValueError:
 645                                 pass
 646                     try:
 647                         txt_table.add_row(row)
 648                     # PrettyTable raises Exception
 649                     except Exception as err:
 650                         logging.warning(
 651                             f"Error occurred while generating TXT table:\n{err}"
 652                         )
 653                 line_nr += 1
 654             txt_table.align[u"Build Number:"] = u"l"
 655         with open(f"{file_name}.txt", u"wt") as txt_file:
 656             txt_file.write(str(txt_table))
 657
 658     # Evaluate result:
 659     if anomaly_classifications:
 660         result = u"PASS"
 661         for job_name, job_data in anomaly_classifications.items():
 662             file_name = \
 663                 f"{spec.cpta[u'output-file']}/regressions-{job_name}.txt"
 664             with open(file_name, u'w') as txt_file:
 665                 for test_name, classification in job_data.items():
 666                     if classification == u"regression":
 667                         txt_file.write(test_name + u'\n')
 668                     if classification in (u"regression", u"outlier"):
 669                         result = u"FAIL"
 670             file_name = \
 671                 f"{spec.cpta[u'output-file']}/progressions-{job_name}.txt"
 672             with open(file_name, u'w') as txt_file:
 673                 for test_name, classification in job_data.items():
 674                     if classification == u"progression":
 675                         txt_file.write(test_name + u'\n')
 676     else:
 677         result = u"FAIL"
 678
 679     logging.info(f"Partial results: {anomaly_classifications}")
 680     logging.info(f"Result: {result}")
 681
 682     return result