resources/tools/presentation/generator_CPTA.py

   1 # Copyright (c) 2018 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Generation of Continuous Performance Trending and Analysis.
  15 """
  16
  17 import multiprocessing
  18 import os
  19 import logging
  20 import csv
  21 import prettytable
  22 import plotly.offline as ploff
  23 import plotly.graph_objs as plgo
  24 import plotly.exceptions as plerr
  25 import pandas as pd
  26
  27 from collections import OrderedDict
  28 from datetime import datetime
  29
  30 from utils import split_outliers, archive_input_data, execute_command,\
  31     classify_anomalies, Worker
  32
  33
  34 # Command to build the html format of the report
  35 HTML_BUILDER = 'sphinx-build -v -c conf_cpta -a ' \
  36                '-b html -E ' \
  37                '-t html ' \
  38                '-D version="{date}" ' \
  39                '{working_dir} ' \
  40                '{build_dir}/'
  41
  42 # .css file for the html format of the report
  43 THEME_OVERRIDES = """/* override table width restrictions */
  44 .wy-nav-content {
  45     max-width: 1200px !important;
  46 }
  47 """
  48
  49 COLORS = ["SkyBlue", "Olive", "Purple", "Coral", "Indigo", "Pink",
  50           "Chocolate", "Brown", "Magenta", "Cyan", "Orange", "Black",
  51           "Violet", "Blue", "Yellow"]
  52
  53
  54 def generate_cpta(spec, data):
  55     """Generate all formats and versions of the Continuous Performance Trending
  56     and Analysis.
  57
  58     :param spec: Specification read from the specification file.
  59     :param data: Full data set.
  60     :type spec: Specification
  61     :type data: InputData
  62     """
  63
  64     logging.info("Generating the Continuous Performance Trending and Analysis "
  65                  "...")
  66
  67     ret_code = _generate_all_charts(spec, data)
  68
  69     cmd = HTML_BUILDER.format(
  70         date=datetime.utcnow().strftime('%m/%d/%Y %H:%M UTC'),
  71         working_dir=spec.environment["paths"]["DIR[WORKING,SRC]"],
  72         build_dir=spec.environment["paths"]["DIR[BUILD,HTML]"])
  73     execute_command(cmd)
  74
  75     with open(spec.environment["paths"]["DIR[CSS_PATCH_FILE]"], "w") as \
  76             css_file:
  77         css_file.write(THEME_OVERRIDES)
  78
  79     with open(spec.environment["paths"]["DIR[CSS_PATCH_FILE2]"], "w") as \
  80             css_file:
  81         css_file.write(THEME_OVERRIDES)
  82
  83     archive_input_data(spec)
  84
  85     logging.info("Done.")
  86
  87     return ret_code
  88
  89
  90 def _generate_trending_traces(in_data, build_info, moving_win_size=10,
  91                               show_trend_line=True, name="", color=""):
  92     """Generate the trending traces:
  93      - samples,
  94      - trimmed moving median (trending line)
  95      - outliers, regress, progress
  96
  97     :param in_data: Full data set.
  98     :param build_info: Information about the builds.
  99     :param moving_win_size: Window size.
 100     :param show_trend_line: Show moving median (trending plot).
 101     :param name: Name of the plot
 102     :param color: Name of the color for the plot.
 103     :type in_data: OrderedDict
 104     :type build_info: dict
 105     :type moving_win_size: int
 106     :type show_trend_line: bool
 107     :type name: str
 108     :type color: str
 109     :returns: Generated traces (list) and the evaluated result.
 110     :rtype: tuple(traces, result)
 111     """
 112
 113     data_x = list(in_data.keys())
 114     data_y = list(in_data.values())
 115
 116     hover_text = list()
 117     xaxis = list()
 118     for idx in data_x:
 119         hover_text.append("vpp-ref: {0}<br>csit-ref: mrr-daily-build-{1}".
 120                           format(build_info[str(idx)][1].rsplit('~', 1)[0],
 121                                  idx))
 122         date = build_info[str(idx)][0]
 123         xaxis.append(datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]),
 124                               int(date[9:11]), int(date[12:])))
 125
 126     data_pd = pd.Series(data_y, index=xaxis)
 127
 128     t_data, outliers = split_outliers(data_pd, outlier_const=1.5,
 129                                       window=moving_win_size)
 130     anomaly_classification = classify_anomalies(t_data, window=moving_win_size)
 131
 132     anomalies = pd.Series()
 133     anomalies_colors = list()
 134     anomaly_color = {
 135         "outlier": 0.0,
 136         "regression": 0.33,
 137         "normal": 0.66,
 138         "progression": 1.0
 139     }
 140     if anomaly_classification:
 141         for idx, item in enumerate(data_pd.items()):
 142             if anomaly_classification[idx] in \
 143                     ("outlier", "regression", "progression"):
 144                 anomalies = anomalies.append(pd.Series([item[1], ],
 145                                                        index=[item[0], ]))
 146                 anomalies_colors.append(
 147                     anomaly_color[anomaly_classification[idx]])
 148         anomalies_colors.extend([0.0, 0.33, 0.66, 1.0])
 149
 150     # Create traces
 151
 152     trace_samples = plgo.Scatter(
 153         x=xaxis,
 154         y=data_y,
 155         mode='markers',
 156         line={
 157             "width": 1
 158         },
 159         legendgroup=name,
 160         name="{name}-thput".format(name=name),
 161         marker={
 162             "size": 5,
 163             "color": color,
 164             "symbol": "circle",
 165         },
 166         text=hover_text,
 167         hoverinfo="x+y+text+name"
 168     )
 169     traces = [trace_samples, ]
 170
 171     trace_anomalies = plgo.Scatter(
 172         x=anomalies.keys(),
 173         y=anomalies.values,
 174         mode='markers',
 175         hoverinfo="none",
 176         showlegend=True,
 177         legendgroup=name,
 178         name="{name}-anomalies".format(name=name),
 179         marker={
 180             "size": 15,
 181             "symbol": "circle-open",
 182             "color": anomalies_colors,
 183             "colorscale": [[0.00, "grey"],
 184                            [0.25, "grey"],
 185                            [0.25, "red"],
 186                            [0.50, "red"],
 187                            [0.50, "white"],
 188                            [0.75, "white"],
 189                            [0.75, "green"],
 190                            [1.00, "green"]],
 191             "showscale": True,
 192             "line": {
 193                 "width": 2
 194             },
 195             "colorbar": {
 196                 "y": 0.5,
 197                 "len": 0.8,
 198                 "title": "Circles Marking Data Classification",
 199                 "titleside": 'right',
 200                 "titlefont": {
 201                     "size": 14
 202                 },
 203                 "tickmode": 'array',
 204                 "tickvals": [0.125, 0.375, 0.625, 0.875],
 205                 "ticktext": ["Outlier", "Regression", "Normal", "Progression"],
 206                 "ticks": "",
 207                 "ticklen": 0,
 208                 "tickangle": -90,
 209                 "thickness": 10
 210             }
 211         }
 212     )
 213     traces.append(trace_anomalies)
 214
 215     if show_trend_line:
 216         data_trend = t_data.rolling(window=moving_win_size,
 217                                     min_periods=2).median()
 218         trace_trend = plgo.Scatter(
 219             x=data_trend.keys(),
 220             y=data_trend.tolist(),
 221             mode='lines',
 222             line={
 223                 "shape": "spline",
 224                 "width": 1,
 225                 "color": color,
 226             },
 227             legendgroup=name,
 228             name='{name}-trend'.format(name=name)
 229         )
 230         traces.append(trace_trend)
 231
 232     return traces, anomaly_classification[-1]
 233
 234
 235 def _generate_all_charts(spec, input_data):
 236     """Generate all charts specified in the specification file.
 237
 238     :param spec: Specification.
 239     :param input_data: Full data set.
 240     :type spec: Specification
 241     :type input_data: InputData
 242     """
 243
 244     def _generate_chart(_, data_q, graph):
 245         """Generates the chart.
 246         """
 247
 248         logs = list()
 249
 250         logging.info("  Generating the chart '{0}' ...".
 251                      format(graph.get("title", "")))
 252         logs.append(("INFO", "  Generating the chart '{0}' ...".
 253                      format(graph.get("title", ""))))
 254
 255         job_name = spec.cpta["data"].keys()[0]
 256
 257         csv_tbl = list()
 258         res = list()
 259
 260         # Transform the data
 261         logs.append(("INFO", "    Creating the data set for the {0} '{1}'.".
 262                      format(graph.get("type", ""), graph.get("title", ""))))
 263         data = input_data.filter_data(graph, continue_on_error=True)
 264         if data is None:
 265             logging.error("No data.")
 266             return
 267
 268         chart_data = dict()
 269         for job in data:
 270             for index, bld in job.items():
 271                 for test_name, test in bld.items():
 272                     if chart_data.get(test_name, None) is None:
 273                         chart_data[test_name] = OrderedDict()
 274                     try:
 275                         chart_data[test_name][int(index)] = \
 276                             test["result"]["throughput"]
 277                     except (KeyError, TypeError):
 278                         pass
 279
 280         # Add items to the csv table:
 281         for tst_name, tst_data in chart_data.items():
 282             tst_lst = list()
 283             for bld in builds_lst:
 284                 itm = tst_data.get(int(bld), '')
 285                 tst_lst.append(str(itm))
 286             csv_tbl.append("{0},".format(tst_name) + ",".join(tst_lst) + '\n')
 287         # Generate traces:
 288         traces = list()
 289         win_size = 14
 290         index = 0
 291         for test_name, test_data in chart_data.items():
 292             if not test_data:
 293                 logs.append(("WARNING", "No data for the test '{0}'".
 294                              format(test_name)))
 295                 continue
 296             test_name = test_name.split('.')[-1]
 297             trace, rslt = _generate_trending_traces(
 298                 test_data,
 299                 build_info=build_info,
 300                 moving_win_size=win_size,
 301                 name='-'.join(test_name.split('-')[3:-1]),
 302                 color=COLORS[index])
 303             traces.extend(trace)
 304             res.append(rslt)
 305             index += 1
 306
 307         if traces:
 308             # Generate the chart:
 309             graph["layout"]["xaxis"]["title"] = \
 310                 graph["layout"]["xaxis"]["title"].format(job=job_name)
 311             name_file = "{0}-{1}{2}".format(spec.cpta["output-file"],
 312                                             graph["output-file-name"],
 313                                             spec.cpta["output-file-type"])
 314
 315             logs.append(("INFO", "    Writing the file '{0}' ...".
 316                          format(name_file)))
 317             plpl = plgo.Figure(data=traces, layout=graph["layout"])
 318             try:
 319                 ploff.plot(plpl, show_link=False, auto_open=False,
 320                            filename=name_file)
 321             except plerr.PlotlyEmptyDataError:
 322                 logs.append(("WARNING", "No data for the plot. Skipped."))
 323
 324         data_out = {
 325             "csv_table": csv_tbl,
 326             "results": res,
 327             "logs": logs
 328         }
 329         data_q.put(data_out)
 330
 331     job_name = spec.cpta["data"].keys()[0]
 332
 333     builds_lst = list()
 334     for build in spec.input["builds"][job_name]:
 335         status = build["status"]
 336         if status != "failed" and status != "not found":
 337             builds_lst.append(str(build["build"]))
 338
 339     # Get "build ID": "date" dict:
 340     build_info = OrderedDict()
 341     for build in builds_lst:
 342         try:
 343             build_info[build] = (
 344                 input_data.metadata(job_name, build)["generated"][:14],
 345                 input_data.metadata(job_name, build)["version"]
 346             )
 347         except KeyError:
 348             build_info[build] = ("", "")
 349
 350     work_queue = multiprocessing.JoinableQueue()
 351     manager = multiprocessing.Manager()
 352     data_queue = manager.Queue()
 353     cpus = multiprocessing.cpu_count()
 354
 355     workers = list()
 356     for cpu in range(cpus):
 357         worker = Worker(work_queue,
 358                         data_queue,
 359                         _generate_chart)
 360         worker.daemon = True
 361         worker.start()
 362         workers.append(worker)
 363         os.system("taskset -p -c {0} {1} > /dev/null 2>&1".
 364                   format(cpu, worker.pid))
 365
 366     for chart in spec.cpta["plots"]:
 367         work_queue.put((chart, ))
 368     work_queue.join()
 369
 370     anomaly_classifications = list()
 371
 372     # Create the header:
 373     csv_table = list()
 374     header = "Build Number:," + ",".join(builds_lst) + '\n'
 375     csv_table.append(header)
 376     build_dates = [x[0] for x in build_info.values()]
 377     header = "Build Date:," + ",".join(build_dates) + '\n'
 378     csv_table.append(header)
 379     vpp_versions = [x[1] for x in build_info.values()]
 380     header = "VPP Version:," + ",".join(vpp_versions) + '\n'
 381     csv_table.append(header)
 382
 383     while not data_queue.empty():
 384         result = data_queue.get()
 385
 386         anomaly_classifications.extend(result["results"])
 387         csv_table.extend(result["csv_table"])
 388
 389         for item in result["logs"]:
 390             if item[0] == "INFO":
 391                 logging.info(item[1])
 392             elif item[0] == "ERROR":
 393                 logging.error(item[1])
 394             elif item[0] == "DEBUG":
 395                 logging.debug(item[1])
 396             elif item[0] == "CRITICAL":
 397                 logging.critical(item[1])
 398             elif item[0] == "WARNING":
 399                 logging.warning(item[1])
 400
 401     del data_queue
 402
 403     # Terminate all workers
 404     for worker in workers:
 405         worker.terminate()
 406         worker.join()
 407
 408     # Write the tables:
 409     file_name = spec.cpta["output-file"] + "-trending"
 410     with open("{0}.csv".format(file_name), 'w') as file_handler:
 411         file_handler.writelines(csv_table)
 412
 413     txt_table = None
 414     with open("{0}.csv".format(file_name), 'rb') as csv_file:
 415         csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 416         line_nr = 0
 417         for row in csv_content:
 418             if txt_table is None:
 419                 txt_table = prettytable.PrettyTable(row)
 420             else:
 421                 if line_nr > 1:
 422                     for idx, item in enumerate(row):
 423                         try:
 424                             row[idx] = str(round(float(item) / 1000000, 2))
 425                         except ValueError:
 426                             pass
 427                 try:
 428                     txt_table.add_row(row)
 429                 except Exception as err:
 430                     logging.warning("Error occurred while generating TXT table:"
 431                                     "\n{0}".format(err))
 432             line_nr += 1
 433         txt_table.align["Build Number:"] = "l"
 434     with open("{0}.txt".format(file_name), "w") as txt_file:
 435         txt_file.write(str(txt_table))
 436
 437     # Evaluate result:
 438     if anomaly_classifications:
 439         result = "PASS"
 440         for classification in anomaly_classifications:
 441             if classification == "regression" or classification == "outlier":
 442                 result = "FAIL"
 443                 break
 444     else:
 445         result = "FAIL"
 446
 447     logging.info("Partial results: {0}".format(anomaly_classifications))
 448     logging.info("Result: {0}".format(result))
 449
 450     return result