resources/tools/presentation/generator_CPTA.py

   1 # Copyright (c) 2018 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Generation of Continuous Performance Trending and Analysis.
  15 """
  16
  17 import multiprocessing
  18 import os
  19 import logging
  20 import csv
  21 import prettytable
  22 import plotly.offline as ploff
  23 import plotly.graph_objs as plgo
  24 import plotly.exceptions as plerr
  25
  26 from collections import OrderedDict
  27 from datetime import datetime
  28
  29 from utils import archive_input_data, execute_command, \
  30     classify_anomalies, Worker
  31
  32
  33 # Command to build the html format of the report
  34 HTML_BUILDER = 'sphinx-build -v -c conf_cpta -a ' \
  35                '-b html -E ' \
  36                '-t html ' \
  37                '-D version="{date}" ' \
  38                '{working_dir} ' \
  39                '{build_dir}/'
  40
  41 # .css file for the html format of the report
  42 THEME_OVERRIDES = """/* override table width restrictions */
  43 .wy-nav-content {
  44     max-width: 1200px !important;
  45 }
  46 .rst-content blockquote {
  47     margin-left: 0px;
  48     line-height: 18px;
  49     margin-bottom: 0px;
  50 }
  51 .wy-menu-vertical a {
  52     display: inline-block;
  53     line-height: 18px;
  54     padding: 0 2em;
  55     display: block;
  56     position: relative;
  57     font-size: 90%;
  58     color: #d9d9d9
  59 }
  60 .wy-menu-vertical li.current a {
  61     color: gray;
  62     border-right: solid 1px #c9c9c9;
  63     padding: 0 3em;
  64 }
  65 .wy-menu-vertical li.toctree-l2.current > a {
  66     background: #c9c9c9;
  67     padding: 0 3em;
  68 }
  69 .wy-menu-vertical li.toctree-l2.current li.toctree-l3 > a {
  70     display: block;
  71     background: #c9c9c9;
  72     padding: 0 4em;
  73 }
  74 .wy-menu-vertical li.toctree-l3.current li.toctree-l4 > a {
  75     display: block;
  76     background: #bdbdbd;
  77     padding: 0 5em;
  78 }
  79 .wy-menu-vertical li.on a, .wy-menu-vertical li.current > a {
  80     color: #404040;
  81     padding: 0 2em;
  82     font-weight: bold;
  83     position: relative;
  84     background: #fcfcfc;
  85     border: none;
  86         border-top-width: medium;
  87         border-bottom-width: medium;
  88         border-top-style: none;
  89         border-bottom-style: none;
  90         border-top-color: currentcolor;
  91         border-bottom-color: currentcolor;
  92     padding-left: 2em -4px;
  93 }
  94 """
  95
  96 COLORS = ["SkyBlue", "Olive", "Purple", "Coral", "Indigo", "Pink",
  97           "Chocolate", "Brown", "Magenta", "Cyan", "Orange", "Black",
  98           "Violet", "Blue", "Yellow", "BurlyWood", "CadetBlue", "Crimson",
  99           "DarkBlue", "DarkCyan", "DarkGreen", "Green", "GoldenRod",
 100           "LightGreen", "LightSeaGreen", "LightSkyBlue", "Maroon",
 101           "MediumSeaGreen", "SeaGreen", "LightSlateGrey",
 102           "SkyBlue", "Olive", "Purple", "Coral", "Indigo", "Pink",
 103           "Chocolate", "Brown", "Magenta", "Cyan", "Orange", "Black",
 104           "Violet", "Blue", "Yellow", "BurlyWood", "CadetBlue", "Crimson",
 105           "DarkBlue", "DarkCyan", "DarkGreen", "Green", "GoldenRod",
 106           "LightGreen", "LightSeaGreen", "LightSkyBlue", "Maroon",
 107           "MediumSeaGreen", "SeaGreen", "LightSlateGrey"
 108           ]
 109
 110
 111 def generate_cpta(spec, data):
 112     """Generate all formats and versions of the Continuous Performance Trending
 113     and Analysis.
 114
 115     :param spec: Specification read from the specification file.
 116     :param data: Full data set.
 117     :type spec: Specification
 118     :type data: InputData
 119     """
 120
 121     logging.info("Generating the Continuous Performance Trending and Analysis "
 122                  "...")
 123
 124     ret_code = _generate_all_charts(spec, data)
 125
 126     cmd = HTML_BUILDER.format(
 127         date=datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC'),
 128         working_dir=spec.environment["paths"]["DIR[WORKING,SRC]"],
 129         build_dir=spec.environment["paths"]["DIR[BUILD,HTML]"])
 130     execute_command(cmd)
 131
 132     with open(spec.environment["paths"]["DIR[CSS_PATCH_FILE]"], "w") as \
 133             css_file:
 134         css_file.write(THEME_OVERRIDES)
 135
 136     with open(spec.environment["paths"]["DIR[CSS_PATCH_FILE2]"], "w") as \
 137             css_file:
 138         css_file.write(THEME_OVERRIDES)
 139
 140     archive_input_data(spec)
 141
 142     logging.info("Done.")
 143
 144     return ret_code
 145
 146
 147 def _generate_trending_traces(in_data, job_name, build_info,
 148                               show_trend_line=True, name="", color=""):
 149     """Generate the trending traces:
 150      - samples,
 151      - outliers, regress, progress
 152      - average of normal samples (trending line)
 153
 154     :param in_data: Full data set.
 155     :param job_name: The name of job which generated the data.
 156     :param build_info: Information about the builds.
 157     :param show_trend_line: Show moving median (trending plot).
 158     :param name: Name of the plot
 159     :param color: Name of the color for the plot.
 160     :type in_data: OrderedDict
 161     :type job_name: str
 162     :type build_info: dict
 163     :type show_trend_line: bool
 164     :type name: str
 165     :type color: str
 166     :returns: Generated traces (list) and the evaluated result.
 167     :rtype: tuple(traces, result)
 168     """
 169
 170     data_x = list(in_data.keys())
 171     data_y = list(in_data.values())
 172
 173     hover_text = list()
 174     xaxis = list()
 175     for idx in data_x:
 176         date = build_info[job_name][str(idx)][0]
 177         hover_str = ("date: {0}<br>"
 178                      "value: {1:,}<br>"
 179                      "{2}-ref: {3}<br>"
 180                      "csit-ref: mrr-{4}-build-{5}")
 181         if "dpdk" in job_name:
 182             hover_text.append(hover_str.format(
 183                 date,
 184                 int(in_data[idx].avg),
 185                 "dpdk",
 186                 build_info[job_name][str(idx)][1].
 187                 rsplit('~', 1)[0],
 188                 "weekly",
 189                 idx))
 190         elif "vpp" in job_name:
 191             hover_text.append(hover_str.format(
 192                 date,
 193                 int(in_data[idx].avg),
 194                 "vpp",
 195                 build_info[job_name][str(idx)][1].
 196                 rsplit('~', 1)[0],
 197                 "daily",
 198                 idx))
 199
 200         xaxis.append(datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]),
 201                               int(date[9:11]), int(date[12:])))
 202
 203     data_pd = OrderedDict()
 204     for key, value in zip(xaxis, data_y):
 205         data_pd[key] = value
 206
 207     anomaly_classification, avgs = classify_anomalies(data_pd)
 208
 209     anomalies = OrderedDict()
 210     anomalies_colors = list()
 211     anomalies_avgs = list()
 212     anomaly_color = {
 213         "regression": 0.0,
 214         "normal": 0.5,
 215         "progression": 1.0
 216     }
 217     if anomaly_classification:
 218         for idx, (key, value) in enumerate(data_pd.iteritems()):
 219             if anomaly_classification[idx] in \
 220                     ("outlier", "regression", "progression"):
 221                 anomalies[key] = value
 222                 anomalies_colors.append(
 223                     anomaly_color[anomaly_classification[idx]])
 224                 anomalies_avgs.append(avgs[idx])
 225         anomalies_colors.extend([0.0, 0.5, 1.0])
 226
 227     # Create traces
 228
 229     trace_samples = plgo.Scatter(
 230         x=xaxis,
 231         y=[y.avg for y in data_y],
 232         mode='markers',
 233         line={
 234             "width": 1
 235         },
 236         showlegend=True,
 237         legendgroup=name,
 238         name="{name}".format(name=name),
 239         marker={
 240             "size": 5,
 241             "color": color,
 242             "symbol": "circle",
 243         },
 244         text=hover_text,
 245         hoverinfo="text"
 246     )
 247     traces = [trace_samples, ]
 248
 249     if show_trend_line:
 250         trace_trend = plgo.Scatter(
 251             x=xaxis,
 252             y=avgs,
 253             mode='lines',
 254             line={
 255                 "shape": "linear",
 256                 "width": 1,
 257                 "color": color,
 258             },
 259             showlegend=False,
 260             legendgroup=name,
 261             name='{name}'.format(name=name),
 262             text=["trend: {0:,}".format(int(avg)) for avg in avgs],
 263             hoverinfo="text+name"
 264         )
 265         traces.append(trace_trend)
 266
 267     trace_anomalies = plgo.Scatter(
 268         x=anomalies.keys(),
 269         y=anomalies_avgs,
 270         mode='markers',
 271         hoverinfo="none",
 272         showlegend=False,
 273         legendgroup=name,
 274         name="{name}-anomalies".format(name=name),
 275         marker={
 276             "size": 15,
 277             "symbol": "circle-open",
 278             "color": anomalies_colors,
 279             "colorscale": [[0.00, "red"],
 280                            [0.33, "red"],
 281                            [0.33, "white"],
 282                            [0.66, "white"],
 283                            [0.66, "green"],
 284                            [1.00, "green"]],
 285             "showscale": True,
 286             "line": {
 287                 "width": 2
 288             },
 289             "colorbar": {
 290                 "y": 0.5,
 291                 "len": 0.8,
 292                 "title": "Circles Marking Data Classification",
 293                 "titleside": 'right',
 294                 "titlefont": {
 295                     "size": 14
 296                 },
 297                 "tickmode": 'array',
 298                 "tickvals": [0.167, 0.500, 0.833],
 299                 "ticktext": ["Regression", "Normal", "Progression"],
 300                 "ticks": "",
 301                 "ticklen": 0,
 302                 "tickangle": -90,
 303                 "thickness": 10
 304             }
 305         }
 306     )
 307     traces.append(trace_anomalies)
 308
 309     if anomaly_classification:
 310         return traces, anomaly_classification[-1]
 311     else:
 312         return traces, None
 313
 314
 315 def _generate_all_charts(spec, input_data):
 316     """Generate all charts specified in the specification file.
 317
 318     :param spec: Specification.
 319     :param input_data: Full data set.
 320     :type spec: Specification
 321     :type input_data: InputData
 322     """
 323
 324     def _generate_chart(_, data_q, graph):
 325         """Generates the chart.
 326         """
 327
 328         logs = list()
 329
 330         logging.info("  Generating the chart '{0}' ...".
 331                      format(graph.get("title", "")))
 332         logs.append(("INFO", "  Generating the chart '{0}' ...".
 333                      format(graph.get("title", ""))))
 334
 335         job_name = graph["data"].keys()[0]
 336
 337         csv_tbl = list()
 338         res = list()
 339
 340         # Transform the data
 341         logs.append(("INFO", "    Creating the data set for the {0} '{1}'.".
 342                      format(graph.get("type", ""), graph.get("title", ""))))
 343         data = input_data.filter_data(graph, continue_on_error=True)
 344         if data is None:
 345             logging.error("No data.")
 346             return
 347
 348         chart_data = dict()
 349         for job, job_data in data.iteritems():
 350             if job != job_name:
 351                 continue
 352             for index, bld in job_data.items():
 353                 for test_name, test in bld.items():
 354                     if chart_data.get(test_name, None) is None:
 355                         chart_data[test_name] = OrderedDict()
 356                     try:
 357                         chart_data[test_name][int(index)] = \
 358                             test["result"]["receive-rate"]
 359                     except (KeyError, TypeError):
 360                         pass
 361
 362         # Add items to the csv table:
 363         for tst_name, tst_data in chart_data.items():
 364             tst_lst = list()
 365             for bld in builds_dict[job_name]:
 366                 itm = tst_data.get(int(bld), '')
 367                 if not isinstance(itm, str):
 368                     itm = itm.avg
 369                 tst_lst.append(str(itm))
 370             csv_tbl.append("{0},".format(tst_name) + ",".join(tst_lst) + '\n')
 371         # Generate traces:
 372         traces = list()
 373         index = 0
 374         for test_name, test_data in chart_data.items():
 375             if not test_data:
 376                 logs.append(("WARNING", "No data for the test '{0}'".
 377                              format(test_name)))
 378                 continue
 379             message = "index: {index}, test: {test}".format(
 380                 index=index, test=test_name)
 381             test_name = test_name.split('.')[-1]
 382             try:
 383                 trace, rslt = _generate_trending_traces(
 384                     test_data,
 385                     job_name=job_name,
 386                     build_info=build_info,
 387                     name='-'.join(test_name.split('-')[2:-1]),
 388                     color=COLORS[index])
 389             except IndexError:
 390                 message = "Out of colors: {}".format(message)
 391                 logs.append(("ERROR", message))
 392                 logging.error(message)
 393                 index += 1
 394                 continue
 395             traces.extend(trace)
 396             res.append(rslt)
 397             index += 1
 398
 399         if traces:
 400             # Generate the chart:
 401             graph["layout"]["xaxis"]["title"] = \
 402                 graph["layout"]["xaxis"]["title"].format(job=job_name)
 403             name_file = "{0}-{1}{2}".format(spec.cpta["output-file"],
 404                                             graph["output-file-name"],
 405                                             spec.cpta["output-file-type"])
 406
 407             logs.append(("INFO", "    Writing the file '{0}' ...".
 408                          format(name_file)))
 409             plpl = plgo.Figure(data=traces, layout=graph["layout"])
 410             try:
 411                 ploff.plot(plpl, show_link=False, auto_open=False,
 412                            filename=name_file)
 413             except plerr.PlotlyEmptyDataError:
 414                 logs.append(("WARNING", "No data for the plot. Skipped."))
 415
 416         data_out = {
 417             "job_name": job_name,
 418             "csv_table": csv_tbl,
 419             "results": res,
 420             "logs": logs
 421         }
 422         data_q.put(data_out)
 423
 424     builds_dict = dict()
 425     for job in spec.input["builds"].keys():
 426         if builds_dict.get(job, None) is None:
 427             builds_dict[job] = list()
 428         for build in spec.input["builds"][job]:
 429             status = build["status"]
 430             if status != "failed" and status != "not found":
 431                 builds_dict[job].append(str(build["build"]))
 432
 433     # Create "build ID": "date" dict:
 434     build_info = dict()
 435     for job_name, job_data in builds_dict.items():
 436         if build_info.get(job_name, None) is None:
 437             build_info[job_name] = OrderedDict()
 438         for build in job_data:
 439             build_info[job_name][build] = (
 440                 input_data.metadata(job_name, build).get("generated", ""),
 441                 input_data.metadata(job_name, build).get("version", "")
 442             )
 443
 444     work_queue = multiprocessing.JoinableQueue()
 445     manager = multiprocessing.Manager()
 446     data_queue = manager.Queue()
 447     cpus = multiprocessing.cpu_count()
 448
 449     workers = list()
 450     for cpu in range(cpus):
 451         worker = Worker(work_queue,
 452                         data_queue,
 453                         _generate_chart)
 454         worker.daemon = True
 455         worker.start()
 456         workers.append(worker)
 457         os.system("taskset -p -c {0} {1} > /dev/null 2>&1".
 458                   format(cpu, worker.pid))
 459
 460     for chart in spec.cpta["plots"]:
 461         work_queue.put((chart, ))
 462     work_queue.join()
 463
 464     anomaly_classifications = list()
 465
 466     # Create the header:
 467     csv_tables = dict()
 468     for job_name in builds_dict.keys():
 469         if csv_tables.get(job_name, None) is None:
 470             csv_tables[job_name] = list()
 471         header = "Build Number:," + ",".join(builds_dict[job_name]) + '\n'
 472         csv_tables[job_name].append(header)
 473         build_dates = [x[0] for x in build_info[job_name].values()]
 474         header = "Build Date:," + ",".join(build_dates) + '\n'
 475         csv_tables[job_name].append(header)
 476         versions = [x[1] for x in build_info[job_name].values()]
 477         header = "Version:," + ",".join(versions) + '\n'
 478         csv_tables[job_name].append(header)
 479
 480     while not data_queue.empty():
 481         result = data_queue.get()
 482
 483         anomaly_classifications.extend(result["results"])
 484         csv_tables[result["job_name"]].extend(result["csv_table"])
 485
 486         for item in result["logs"]:
 487             if item[0] == "INFO":
 488                 logging.info(item[1])
 489             elif item[0] == "ERROR":
 490                 logging.error(item[1])
 491             elif item[0] == "DEBUG":
 492                 logging.debug(item[1])
 493             elif item[0] == "CRITICAL":
 494                 logging.critical(item[1])
 495             elif item[0] == "WARNING":
 496                 logging.warning(item[1])
 497
 498     del data_queue
 499
 500     # Terminate all workers
 501     for worker in workers:
 502         worker.terminate()
 503         worker.join()
 504
 505     # Write the tables:
 506     for job_name, csv_table in csv_tables.items():
 507         file_name = spec.cpta["output-file"] + "-" + job_name + "-trending"
 508         with open("{0}.csv".format(file_name), 'w') as file_handler:
 509             file_handler.writelines(csv_table)
 510
 511         txt_table = None
 512         with open("{0}.csv".format(file_name), 'rb') as csv_file:
 513             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 514             line_nr = 0
 515             for row in csv_content:
 516                 if txt_table is None:
 517                     txt_table = prettytable.PrettyTable(row)
 518                 else:
 519                     if line_nr > 1:
 520                         for idx, item in enumerate(row):
 521                             try:
 522                                 row[idx] = str(round(float(item) / 1000000, 2))
 523                             except ValueError:
 524                                 pass
 525                     try:
 526                         txt_table.add_row(row)
 527                     except Exception as err:
 528                         logging.warning("Error occurred while generating TXT "
 529                                         "table:\n{0}".format(err))
 530                 line_nr += 1
 531             txt_table.align["Build Number:"] = "l"
 532         with open("{0}.txt".format(file_name), "w") as txt_file:
 533             txt_file.write(str(txt_table))
 534
 535     # Evaluate result:
 536     if anomaly_classifications:
 537         result = "PASS"
 538         for classification in anomaly_classifications:
 539             if classification == "regression" or classification == "outlier":
 540                 result = "FAIL"
 541                 break
 542     else:
 543         result = "FAIL"
 544
 545     logging.info("Partial results: {0}".format(anomaly_classifications))
 546     logging.info("Result: {0}".format(result))
 547
 548     return result