resources/tools/presentation/new/generator_CPTA.py

   1 # Copyright (c) 2018 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Generation of Continuous Performance Trending and Analysis.
  15 """
  16
  17 import multiprocessing
  18 import os
  19 import logging
  20 import csv
  21 import prettytable
  22 import plotly.offline as ploff
  23 import plotly.graph_objs as plgo
  24 import plotly.exceptions as plerr
  25 import pandas as pd
  26
  27 from collections import OrderedDict
  28 from datetime import datetime
  29
  30 from utils import archive_input_data, execute_command,\
  31     classify_anomalies, Worker
  32
  33
  34 # Command to build the html format of the report
  35 HTML_BUILDER = 'sphinx-build -v -c conf_cpta -a ' \
  36                '-b html -E ' \
  37                '-t html ' \
  38                '-D version="{date}" ' \
  39                '{working_dir} ' \
  40                '{build_dir}/'
  41
  42 # .css file for the html format of the report
  43 THEME_OVERRIDES = """/* override table width restrictions */
  44 .wy-nav-content {
  45     max-width: 1200px !important;
  46 }
  47 """
  48
  49 COLORS = ["SkyBlue", "Olive", "Purple", "Coral", "Indigo", "Pink",
  50           "Chocolate", "Brown", "Magenta", "Cyan", "Orange", "Black",
  51           "Violet", "Blue", "Yellow"]
  52
  53
  54 def generate_cpta(spec, data):
  55     """Generate all formats and versions of the Continuous Performance Trending
  56     and Analysis.
  57
  58     :param spec: Specification read from the specification file.
  59     :param data: Full data set.
  60     :type spec: Specification
  61     :type data: InputData
  62     """
  63
  64     logging.info("Generating the Continuous Performance Trending and Analysis "
  65                  "...")
  66
  67     ret_code = _generate_all_charts(spec, data)
  68
  69     cmd = HTML_BUILDER.format(
  70         date=datetime.utcnow().strftime('%m/%d/%Y %H:%M UTC'),
  71         working_dir=spec.environment["paths"]["DIR[WORKING,SRC]"],
  72         build_dir=spec.environment["paths"]["DIR[BUILD,HTML]"])
  73     execute_command(cmd)
  74
  75     with open(spec.environment["paths"]["DIR[CSS_PATCH_FILE]"], "w") as \
  76             css_file:
  77         css_file.write(THEME_OVERRIDES)
  78
  79     with open(spec.environment["paths"]["DIR[CSS_PATCH_FILE2]"], "w") as \
  80             css_file:
  81         css_file.write(THEME_OVERRIDES)
  82
  83     archive_input_data(spec)
  84
  85     logging.info("Done.")
  86
  87     return ret_code
  88
  89
  90 def _generate_trending_traces(in_data, build_info, moving_win_size=10,
  91                               show_trend_line=True, name="", color=""):
  92     """Generate the trending traces:
  93      - samples,
  94      - trimmed moving median (trending line)
  95      - outliers, regress, progress
  96
  97     :param in_data: Full data set.
  98     :param build_info: Information about the builds.
  99     :param moving_win_size: Window size.
 100     :param show_trend_line: Show moving median (trending plot).
 101     :param name: Name of the plot
 102     :param color: Name of the color for the plot.
 103     :type in_data: OrderedDict
 104     :type build_info: dict
 105     :type moving_win_size: int
 106     :type show_trend_line: bool
 107     :type name: str
 108     :type color: str
 109     :returns: Generated traces (list) and the evaluated result.
 110     :rtype: tuple(traces, result)
 111     """
 112
 113     data_x = list(in_data.keys())
 114     data_y = list(in_data.values())
 115
 116     hover_text = list()
 117     xaxis = list()
 118     for idx in data_x:
 119         hover_text.append("vpp-ref: {0}<br>csit-ref: mrr-daily-build-{1}".
 120                           format(build_info[str(idx)][1].rsplit('~', 1)[0],
 121                                  idx))
 122         date = build_info[str(idx)][0]
 123         xaxis.append(datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]),
 124                               int(date[9:11]), int(date[12:])))
 125
 126     data_pd = pd.Series(data_y, index=xaxis)
 127
 128     anomaly_classification, avgs = classify_anomalies(data_pd)
 129
 130     anomalies = pd.Series()
 131     anomalies_colors = list()
 132     anomalies_avgs = list()
 133     anomaly_color = {
 134         "outlier": 0.0,
 135         "regression": 0.33,
 136         "normal": 0.66,
 137         "progression": 1.0
 138     }
 139     if anomaly_classification:
 140         for idx, item in enumerate(data_pd.items()):
 141             if anomaly_classification[idx] in \
 142                     ("outlier", "regression", "progression"):
 143                 anomalies = anomalies.append(pd.Series([item[1], ],
 144                                                        index=[item[0], ]))
 145                 anomalies_colors.append(
 146                     anomaly_color[anomaly_classification[idx]])
 147                 anomalies_avgs.append(avgs[idx])
 148         anomalies_colors.extend([0.0, 0.33, 0.66, 1.0])
 149
 150     # Create traces
 151
 152     trace_samples = plgo.Scatter(
 153         x=xaxis,
 154         y=data_y,
 155         mode='markers',
 156         line={
 157             "width": 1
 158         },
 159         legendgroup=name,
 160         name="{name}-thput".format(name=name),
 161         marker={
 162             "size": 5,
 163             "color": color,
 164             "symbol": "circle",
 165         },
 166         text=hover_text,
 167         hoverinfo="x+y+text+name"
 168     )
 169     traces = [trace_samples, ]
 170
 171     if show_trend_line:
 172         trace_trend = plgo.Scatter(
 173             x=xaxis,
 174             y=avgs,
 175             mode='lines',
 176             line={
 177                 "shape": "linear",
 178                 "width": 1,
 179                 "color": color,
 180             },
 181             legendgroup=name,
 182             name='{name}-trend'.format(name=name)
 183         )
 184         traces.append(trace_trend)
 185
 186     trace_anomalies = plgo.Scatter(
 187         x=anomalies.keys(),
 188         y=anomalies_avgs,
 189         mode='markers',
 190         hoverinfo="none",
 191         showlegend=True,
 192         legendgroup=name,
 193         name="{name}-anomalies".format(name=name),
 194         marker={
 195             "size": 15,
 196             "symbol": "circle-open",
 197             "color": anomalies_colors,
 198             "colorscale": [[0.00, "grey"],
 199                            [0.25, "grey"],
 200                            [0.25, "red"],
 201                            [0.50, "red"],
 202                            [0.50, "white"],
 203                            [0.75, "white"],
 204                            [0.75, "green"],
 205                            [1.00, "green"]],
 206             "showscale": True,
 207             "line": {
 208                 "width": 2
 209             },
 210             "colorbar": {
 211                 "y": 0.5,
 212                 "len": 0.8,
 213                 "title": "Circles Marking Data Classification",
 214                 "titleside": 'right',
 215                 "titlefont": {
 216                     "size": 14
 217                 },
 218                 "tickmode": 'array',
 219                 "tickvals": [0.125, 0.375, 0.625, 0.875],
 220                 "ticktext": ["Outlier", "Regression", "Normal", "Progression"],
 221                 "ticks": "",
 222                 "ticklen": 0,
 223                 "tickangle": -90,
 224                 "thickness": 10
 225             }
 226         }
 227     )
 228     traces.append(trace_anomalies)
 229
 230     return traces, anomaly_classification[-1]
 231
 232
 233 def _generate_all_charts(spec, input_data):
 234     """Generate all charts specified in the specification file.
 235
 236     :param spec: Specification.
 237     :param input_data: Full data set.
 238     :type spec: Specification
 239     :type input_data: InputData
 240     """
 241
 242     def _generate_chart(_, data_q, graph):
 243         """Generates the chart.
 244         """
 245
 246         logs = list()
 247
 248         logging.info("  Generating the chart '{0}' ...".
 249                      format(graph.get("title", "")))
 250         logs.append(("INFO", "  Generating the chart '{0}' ...".
 251                      format(graph.get("title", ""))))
 252
 253         job_name = spec.cpta["data"].keys()[0]
 254
 255         csv_tbl = list()
 256         res = list()
 257
 258         # Transform the data
 259         logs.append(("INFO", "    Creating the data set for the {0} '{1}'.".
 260                      format(graph.get("type", ""), graph.get("title", ""))))
 261         data = input_data.filter_data(graph, continue_on_error=True)
 262         if data is None:
 263             logging.error("No data.")
 264             return
 265
 266         chart_data = dict()
 267         for job in data:
 268             for index, bld in job.items():
 269                 for test_name, test in bld.items():
 270                     if chart_data.get(test_name, None) is None:
 271                         chart_data[test_name] = OrderedDict()
 272                     try:
 273                         chart_data[test_name][int(index)] = \
 274                             test["result"]["throughput"]
 275                     except (KeyError, TypeError):
 276                         pass
 277
 278         # Add items to the csv table:
 279         for tst_name, tst_data in chart_data.items():
 280             tst_lst = list()
 281             for bld in builds_lst:
 282                 itm = tst_data.get(int(bld), '')
 283                 tst_lst.append(str(itm))
 284             csv_tbl.append("{0},".format(tst_name) + ",".join(tst_lst) + '\n')
 285         # Generate traces:
 286         traces = list()
 287         win_size = 14
 288         index = 0
 289         for test_name, test_data in chart_data.items():
 290             if not test_data:
 291                 logs.append(("WARNING", "No data for the test '{0}'".
 292                              format(test_name)))
 293                 continue
 294             test_name = test_name.split('.')[-1]
 295             trace, rslt = _generate_trending_traces(
 296                 test_data,
 297                 build_info=build_info,
 298                 moving_win_size=win_size,
 299                 name='-'.join(test_name.split('-')[3:-1]),
 300                 color=COLORS[index])
 301             traces.extend(trace)
 302             res.append(rslt)
 303             index += 1
 304
 305         if traces:
 306             # Generate the chart:
 307             graph["layout"]["xaxis"]["title"] = \
 308                 graph["layout"]["xaxis"]["title"].format(job=job_name)
 309             name_file = "{0}-{1}{2}".format(spec.cpta["output-file"],
 310                                             graph["output-file-name"],
 311                                             spec.cpta["output-file-type"])
 312
 313             logs.append(("INFO", "    Writing the file '{0}' ...".
 314                          format(name_file)))
 315             plpl = plgo.Figure(data=traces, layout=graph["layout"])
 316             try:
 317                 ploff.plot(plpl, show_link=False, auto_open=False,
 318                            filename=name_file)
 319             except plerr.PlotlyEmptyDataError:
 320                 logs.append(("WARNING", "No data for the plot. Skipped."))
 321
 322         data_out = {
 323             "csv_table": csv_tbl,
 324             "results": res,
 325             "logs": logs
 326         }
 327         data_q.put(data_out)
 328
 329     job_name = spec.cpta["data"].keys()[0]
 330
 331     builds_lst = list()
 332     for build in spec.input["builds"][job_name]:
 333         status = build["status"]
 334         if status != "failed" and status != "not found":
 335             builds_lst.append(str(build["build"]))
 336
 337     # Get "build ID": "date" dict:
 338     build_info = OrderedDict()
 339     for build in builds_lst:
 340         try:
 341             build_info[build] = (
 342                 input_data.metadata(job_name, build)["generated"][:14],
 343                 input_data.metadata(job_name, build)["version"]
 344             )
 345         except KeyError:
 346             build_info[build] = ("", "")
 347
 348     work_queue = multiprocessing.JoinableQueue()
 349     manager = multiprocessing.Manager()
 350     data_queue = manager.Queue()
 351     cpus = multiprocessing.cpu_count()
 352
 353     workers = list()
 354     for cpu in range(cpus):
 355         worker = Worker(work_queue,
 356                         data_queue,
 357                         _generate_chart)
 358         worker.daemon = True
 359         worker.start()
 360         workers.append(worker)
 361         os.system("taskset -p -c {0} {1} > /dev/null 2>&1".
 362                   format(cpu, worker.pid))
 363
 364     for chart in spec.cpta["plots"]:
 365         work_queue.put((chart, ))
 366     work_queue.join()
 367
 368     anomaly_classifications = list()
 369
 370     # Create the header:
 371     csv_table = list()
 372     header = "Build Number:," + ",".join(builds_lst) + '\n'
 373     csv_table.append(header)
 374     build_dates = [x[0] for x in build_info.values()]
 375     header = "Build Date:," + ",".join(build_dates) + '\n'
 376     csv_table.append(header)
 377     vpp_versions = [x[1] for x in build_info.values()]
 378     header = "VPP Version:," + ",".join(vpp_versions) + '\n'
 379     csv_table.append(header)
 380
 381     while not data_queue.empty():
 382         result = data_queue.get()
 383
 384         anomaly_classifications.extend(result["results"])
 385         csv_table.extend(result["csv_table"])
 386
 387         for item in result["logs"]:
 388             if item[0] == "INFO":
 389                 logging.info(item[1])
 390             elif item[0] == "ERROR":
 391                 logging.error(item[1])
 392             elif item[0] == "DEBUG":
 393                 logging.debug(item[1])
 394             elif item[0] == "CRITICAL":
 395                 logging.critical(item[1])
 396             elif item[0] == "WARNING":
 397                 logging.warning(item[1])
 398
 399     del data_queue
 400
 401     # Terminate all workers
 402     for worker in workers:
 403         worker.terminate()
 404         worker.join()
 405
 406     # Write the tables:
 407     file_name = spec.cpta["output-file"] + "-trending"
 408     with open("{0}.csv".format(file_name), 'w') as file_handler:
 409         file_handler.writelines(csv_table)
 410
 411     txt_table = None
 412     with open("{0}.csv".format(file_name), 'rb') as csv_file:
 413         csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 414         line_nr = 0
 415         for row in csv_content:
 416             if txt_table is None:
 417                 txt_table = prettytable.PrettyTable(row)
 418             else:
 419                 if line_nr > 1:
 420                     for idx, item in enumerate(row):
 421                         try:
 422                             row[idx] = str(round(float(item) / 1000000, 2))
 423                         except ValueError:
 424                             pass
 425                 try:
 426                     txt_table.add_row(row)
 427                 except Exception as err:
 428                     logging.warning("Error occurred while generating TXT table:"
 429                                     "\n{0}".format(err))
 430             line_nr += 1
 431         txt_table.align["Build Number:"] = "l"
 432     with open("{0}.txt".format(file_name), "w") as txt_file:
 433         txt_file.write(str(txt_table))
 434
 435     # Evaluate result:
 436     if anomaly_classifications:
 437         result = "PASS"
 438         for classification in anomaly_classifications:
 439             if classification == "regression" or classification == "outlier":
 440                 result = "FAIL"
 441                 break
 442     else:
 443         result = "FAIL"
 444
 445     logging.info("Partial results: {0}".format(anomaly_classifications))
 446     logging.info("Result: {0}".format(result))
 447
 448     return result