resources/tools/presentation/generator_CPTA.py

   1 # Copyright (c) 2018 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Generation of Continuous Performance Trending and Analysis.
  15 """
  16
  17 import multiprocessing
  18 import os
  19 import logging
  20 import csv
  21 import prettytable
  22 import plotly.offline as ploff
  23 import plotly.graph_objs as plgo
  24 import plotly.exceptions as plerr
  25
  26 from collections import OrderedDict
  27 from datetime import datetime
  28
  29 from utils import archive_input_data, execute_command, \
  30     classify_anomalies, Worker
  31
  32
  33 # Command to build the html format of the report
  34 HTML_BUILDER = 'sphinx-build -v -c conf_cpta -a ' \
  35                '-b html -E ' \
  36                '-t html ' \
  37                '-D version="{date}" ' \
  38                '{working_dir} ' \
  39                '{build_dir}/'
  40
  41 # .css file for the html format of the report
  42 THEME_OVERRIDES = """/* override table width restrictions */
  43 .wy-nav-content {
  44     max-width: 1200px !important;
  45 }
  46 """
  47
  48 COLORS = ["SkyBlue", "Olive", "Purple", "Coral", "Indigo", "Pink",
  49           "Chocolate", "Brown", "Magenta", "Cyan", "Orange", "Black",
  50           "Violet", "Blue", "Yellow", "BurlyWood", "CadetBlue", "Crimson",
  51           "DarkBlue", "DarkCyan", "DarkGreen", "Green", "GoldenRod",
  52           "LightGreen", "LightSeaGreen", "LightSkyBlue", "Maroon",
  53           "MediumSeaGreen", "SeaGreen", "LightSlateGrey",
  54           "SkyBlue", "Olive", "Purple", "Coral", "Indigo", "Pink",
  55           "Chocolate", "Brown", "Magenta", "Cyan", "Orange", "Black",
  56           "Violet", "Blue", "Yellow", "BurlyWood", "CadetBlue", "Crimson",
  57           "DarkBlue", "DarkCyan", "DarkGreen", "Green", "GoldenRod",
  58           "LightGreen", "LightSeaGreen", "LightSkyBlue", "Maroon",
  59           "MediumSeaGreen", "SeaGreen", "LightSlateGrey"
  60           ]
  61
  62
  63 def generate_cpta(spec, data):
  64     """Generate all formats and versions of the Continuous Performance Trending
  65     and Analysis.
  66
  67     :param spec: Specification read from the specification file.
  68     :param data: Full data set.
  69     :type spec: Specification
  70     :type data: InputData
  71     """
  72
  73     logging.info("Generating the Continuous Performance Trending and Analysis "
  74                  "...")
  75
  76     ret_code = _generate_all_charts(spec, data)
  77
  78     cmd = HTML_BUILDER.format(
  79         date=datetime.utcnow().strftime('%m/%d/%Y %H:%M UTC'),
  80         working_dir=spec.environment["paths"]["DIR[WORKING,SRC]"],
  81         build_dir=spec.environment["paths"]["DIR[BUILD,HTML]"])
  82     execute_command(cmd)
  83
  84     with open(spec.environment["paths"]["DIR[CSS_PATCH_FILE]"], "w") as \
  85             css_file:
  86         css_file.write(THEME_OVERRIDES)
  87
  88     with open(spec.environment["paths"]["DIR[CSS_PATCH_FILE2]"], "w") as \
  89             css_file:
  90         css_file.write(THEME_OVERRIDES)
  91
  92     archive_input_data(spec)
  93
  94     logging.info("Done.")
  95
  96     return ret_code
  97
  98
  99 def _generate_trending_traces(in_data, job_name, build_info,
 100                               show_trend_line=True, name="", color=""):
 101     """Generate the trending traces:
 102      - samples,
 103      - outliers, regress, progress
 104      - average of normal samples (trending line)
 105
 106     :param in_data: Full data set.
 107     :param job_name: The name of job which generated the data.
 108     :param build_info: Information about the builds.
 109     :param show_trend_line: Show moving median (trending plot).
 110     :param name: Name of the plot
 111     :param color: Name of the color for the plot.
 112     :type in_data: OrderedDict
 113     :type job_name: str
 114     :type build_info: dict
 115     :type show_trend_line: bool
 116     :type name: str
 117     :type color: str
 118     :returns: Generated traces (list) and the evaluated result.
 119     :rtype: tuple(traces, result)
 120     """
 121
 122     data_x = list(in_data.keys())
 123     data_y = list(in_data.values())
 124
 125     hover_text = list()
 126     xaxis = list()
 127     for idx in data_x:
 128         date = build_info[job_name][str(idx)][0]
 129         hover_str = ("date: {0}<br>"
 130                      "value: {1:,}<br>"
 131                      "{2}-ref: {3}<br>"
 132                      "csit-ref: mrr-{4}-build-{5}")
 133         if "dpdk" in job_name:
 134             hover_text.append(hover_str.format(
 135                 date,
 136                 int(in_data[idx].avg),
 137                 "dpdk",
 138                 build_info[job_name][str(idx)][1].
 139                 rsplit('~', 1)[0],
 140                 "weekly",
 141                 idx))
 142         elif "vpp" in job_name:
 143             hover_text.append(hover_str.format(
 144                 date,
 145                 int(in_data[idx].avg),
 146                 "vpp",
 147                 build_info[job_name][str(idx)][1].
 148                 rsplit('~', 1)[0],
 149                 "daily",
 150                 idx))
 151
 152         xaxis.append(datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]),
 153                               int(date[9:11]), int(date[12:])))
 154
 155     data_pd = OrderedDict()
 156     for key, value in zip(xaxis, data_y):
 157         data_pd[key] = value
 158
 159     anomaly_classification, avgs = classify_anomalies(data_pd)
 160
 161     anomalies = OrderedDict()
 162     anomalies_colors = list()
 163     anomalies_avgs = list()
 164     anomaly_color = {
 165         "regression": 0.0,
 166         "normal": 0.5,
 167         "progression": 1.0
 168     }
 169     if anomaly_classification:
 170         for idx, (key, value) in enumerate(data_pd.iteritems()):
 171             if anomaly_classification[idx] in \
 172                     ("outlier", "regression", "progression"):
 173                 anomalies[key] = value
 174                 anomalies_colors.append(
 175                     anomaly_color[anomaly_classification[idx]])
 176                 anomalies_avgs.append(avgs[idx])
 177         anomalies_colors.extend([0.0, 0.5, 1.0])
 178
 179     # Create traces
 180
 181     trace_samples = plgo.Scatter(
 182         x=xaxis,
 183         y=[y.avg for y in data_y],
 184         mode='markers',
 185         line={
 186             "width": 1
 187         },
 188         showlegend=True,
 189         legendgroup=name,
 190         name="{name}".format(name=name),
 191         marker={
 192             "size": 5,
 193             "color": color,
 194             "symbol": "circle",
 195         },
 196         text=hover_text,
 197         hoverinfo="text"
 198     )
 199     traces = [trace_samples, ]
 200
 201     if show_trend_line:
 202         trace_trend = plgo.Scatter(
 203             x=xaxis,
 204             y=avgs,
 205             mode='lines',
 206             line={
 207                 "shape": "linear",
 208                 "width": 1,
 209                 "color": color,
 210             },
 211             showlegend=False,
 212             legendgroup=name,
 213             name='{name}'.format(name=name),
 214             text=["trend: {0:,}".format(int(avg)) for avg in avgs],
 215             hoverinfo="text+name"
 216         )
 217         traces.append(trace_trend)
 218
 219     trace_anomalies = plgo.Scatter(
 220         x=anomalies.keys(),
 221         y=anomalies_avgs,
 222         mode='markers',
 223         hoverinfo="none",
 224         showlegend=False,
 225         legendgroup=name,
 226         name="{name}-anomalies".format(name=name),
 227         marker={
 228             "size": 15,
 229             "symbol": "circle-open",
 230             "color": anomalies_colors,
 231             "colorscale": [[0.00, "red"],
 232                            [0.33, "red"],
 233                            [0.33, "white"],
 234                            [0.66, "white"],
 235                            [0.66, "green"],
 236                            [1.00, "green"]],
 237             "showscale": True,
 238             "line": {
 239                 "width": 2
 240             },
 241             "colorbar": {
 242                 "y": 0.5,
 243                 "len": 0.8,
 244                 "title": "Circles Marking Data Classification",
 245                 "titleside": 'right',
 246                 "titlefont": {
 247                     "size": 14
 248                 },
 249                 "tickmode": 'array',
 250                 "tickvals": [0.167, 0.500, 0.833],
 251                 "ticktext": ["Regression", "Normal", "Progression"],
 252                 "ticks": "",
 253                 "ticklen": 0,
 254                 "tickangle": -90,
 255                 "thickness": 10
 256             }
 257         }
 258     )
 259     traces.append(trace_anomalies)
 260
 261     if anomaly_classification:
 262         return traces, anomaly_classification[-1]
 263     else:
 264         return traces, None
 265
 266
 267 def _generate_all_charts(spec, input_data):
 268     """Generate all charts specified in the specification file.
 269
 270     :param spec: Specification.
 271     :param input_data: Full data set.
 272     :type spec: Specification
 273     :type input_data: InputData
 274     """
 275
 276     def _generate_chart(_, data_q, graph):
 277         """Generates the chart.
 278         """
 279
 280         logs = list()
 281
 282         logging.info("  Generating the chart '{0}' ...".
 283                      format(graph.get("title", "")))
 284         logs.append(("INFO", "  Generating the chart '{0}' ...".
 285                      format(graph.get("title", ""))))
 286
 287         job_name = graph["data"].keys()[0]
 288
 289         csv_tbl = list()
 290         res = list()
 291
 292         # Transform the data
 293         logs.append(("INFO", "    Creating the data set for the {0} '{1}'.".
 294                      format(graph.get("type", ""), graph.get("title", ""))))
 295         data = input_data.filter_data(graph, continue_on_error=True)
 296         if data is None:
 297             logging.error("No data.")
 298             return
 299
 300         chart_data = dict()
 301         for job, job_data in data.iteritems():
 302             if job != job_name:
 303                 continue
 304             for index, bld in job_data.items():
 305                 for test_name, test in bld.items():
 306                     if chart_data.get(test_name, None) is None:
 307                         chart_data[test_name] = OrderedDict()
 308                     try:
 309                         chart_data[test_name][int(index)] = \
 310                             test["result"]["receive-rate"]
 311                     except (KeyError, TypeError):
 312                         pass
 313
 314         # Add items to the csv table:
 315         for tst_name, tst_data in chart_data.items():
 316             tst_lst = list()
 317             for bld in builds_dict[job_name]:
 318                 itm = tst_data.get(int(bld), '')
 319                 if not isinstance(itm, str):
 320                     itm = itm.avg
 321                 tst_lst.append(str(itm))
 322             csv_tbl.append("{0},".format(tst_name) + ",".join(tst_lst) + '\n')
 323         # Generate traces:
 324         traces = list()
 325         index = 0
 326         for test_name, test_data in chart_data.items():
 327             if not test_data:
 328                 logs.append(("WARNING", "No data for the test '{0}'".
 329                              format(test_name)))
 330                 continue
 331             message = "index: {index}, test: {test}".format(
 332                 index=index, test=test_name)
 333             test_name = test_name.split('.')[-1]
 334             try:
 335                 trace, rslt = _generate_trending_traces(
 336                     test_data,
 337                     job_name=job_name,
 338                     build_info=build_info,
 339                     name='-'.join(test_name.split('-')[2:-1]),
 340                     color=COLORS[index])
 341             except IndexError:
 342                 message = "Out of colors: {}".format(message)
 343                 logs.append(("ERROR", message))
 344                 logging.error(message)
 345                 index += 1
 346                 continue
 347             traces.extend(trace)
 348             res.append(rslt)
 349             index += 1
 350
 351         if traces:
 352             # Generate the chart:
 353             graph["layout"]["xaxis"]["title"] = \
 354                 graph["layout"]["xaxis"]["title"].format(job=job_name)
 355             name_file = "{0}-{1}{2}".format(spec.cpta["output-file"],
 356                                             graph["output-file-name"],
 357                                             spec.cpta["output-file-type"])
 358
 359             logs.append(("INFO", "    Writing the file '{0}' ...".
 360                          format(name_file)))
 361             plpl = plgo.Figure(data=traces, layout=graph["layout"])
 362             try:
 363                 ploff.plot(plpl, show_link=False, auto_open=False,
 364                            filename=name_file)
 365             except plerr.PlotlyEmptyDataError:
 366                 logs.append(("WARNING", "No data for the plot. Skipped."))
 367
 368         data_out = {
 369             "job_name": job_name,
 370             "csv_table": csv_tbl,
 371             "results": res,
 372             "logs": logs
 373         }
 374         data_q.put(data_out)
 375
 376     builds_dict = dict()
 377     for job in spec.input["builds"].keys():
 378         if builds_dict.get(job, None) is None:
 379             builds_dict[job] = list()
 380         for build in spec.input["builds"][job]:
 381             status = build["status"]
 382             if status != "failed" and status != "not found":
 383                 builds_dict[job].append(str(build["build"]))
 384
 385     # Create "build ID": "date" dict:
 386     build_info = dict()
 387     for job_name, job_data in builds_dict.items():
 388         if build_info.get(job_name, None) is None:
 389             build_info[job_name] = OrderedDict()
 390         for build in job_data:
 391             build_info[job_name][build] = (
 392                 input_data.metadata(job_name, build).get("generated", ""),
 393                 input_data.metadata(job_name, build).get("version", "")
 394             )
 395
 396     work_queue = multiprocessing.JoinableQueue()
 397     manager = multiprocessing.Manager()
 398     data_queue = manager.Queue()
 399     cpus = multiprocessing.cpu_count()
 400
 401     workers = list()
 402     for cpu in range(cpus):
 403         worker = Worker(work_queue,
 404                         data_queue,
 405                         _generate_chart)
 406         worker.daemon = True
 407         worker.start()
 408         workers.append(worker)
 409         os.system("taskset -p -c {0} {1} > /dev/null 2>&1".
 410                   format(cpu, worker.pid))
 411
 412     for chart in spec.cpta["plots"]:
 413         work_queue.put((chart, ))
 414     work_queue.join()
 415
 416     anomaly_classifications = list()
 417
 418     # Create the header:
 419     csv_tables = dict()
 420     for job_name in builds_dict.keys():
 421         if csv_tables.get(job_name, None) is None:
 422             csv_tables[job_name] = list()
 423         header = "Build Number:," + ",".join(builds_dict[job_name]) + '\n'
 424         csv_tables[job_name].append(header)
 425         build_dates = [x[0] for x in build_info[job_name].values()]
 426         header = "Build Date:," + ",".join(build_dates) + '\n'
 427         csv_tables[job_name].append(header)
 428         versions = [x[1] for x in build_info[job_name].values()]
 429         header = "Version:," + ",".join(versions) + '\n'
 430         csv_tables[job_name].append(header)
 431
 432     while not data_queue.empty():
 433         result = data_queue.get()
 434
 435         anomaly_classifications.extend(result["results"])
 436         csv_tables[result["job_name"]].extend(result["csv_table"])
 437
 438         for item in result["logs"]:
 439             if item[0] == "INFO":
 440                 logging.info(item[1])
 441             elif item[0] == "ERROR":
 442                 logging.error(item[1])
 443             elif item[0] == "DEBUG":
 444                 logging.debug(item[1])
 445             elif item[0] == "CRITICAL":
 446                 logging.critical(item[1])
 447             elif item[0] == "WARNING":
 448                 logging.warning(item[1])
 449
 450     del data_queue
 451
 452     # Terminate all workers
 453     for worker in workers:
 454         worker.terminate()
 455         worker.join()
 456
 457     # Write the tables:
 458     for job_name, csv_table in csv_tables.items():
 459         file_name = spec.cpta["output-file"] + "-" + job_name + "-trending"
 460         with open("{0}.csv".format(file_name), 'w') as file_handler:
 461             file_handler.writelines(csv_table)
 462
 463         txt_table = None
 464         with open("{0}.csv".format(file_name), 'rb') as csv_file:
 465             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 466             line_nr = 0
 467             for row in csv_content:
 468                 if txt_table is None:
 469                     txt_table = prettytable.PrettyTable(row)
 470                 else:
 471                     if line_nr > 1:
 472                         for idx, item in enumerate(row):
 473                             try:
 474                                 row[idx] = str(round(float(item) / 1000000, 2))
 475                             except ValueError:
 476                                 pass
 477                     try:
 478                         txt_table.add_row(row)
 479                     except Exception as err:
 480                         logging.warning("Error occurred while generating TXT "
 481                                         "table:\n{0}".format(err))
 482                 line_nr += 1
 483             txt_table.align["Build Number:"] = "l"
 484         with open("{0}.txt".format(file_name), "w") as txt_file:
 485             txt_file.write(str(txt_table))
 486
 487     # Evaluate result:
 488     if anomaly_classifications:
 489         result = "PASS"
 490         for classification in anomaly_classifications:
 491             if classification == "regression" or classification == "outlier":
 492                 result = "FAIL"
 493                 break
 494     else:
 495         result = "FAIL"
 496
 497     logging.info("Partial results: {0}".format(anomaly_classifications))
 498     logging.info("Result: {0}".format(result))
 499
 500     return result