resources/tools/presentation/generator_cpta.py

   1 # Copyright (c) 2021 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Generation of Continuous Performance Trending and Analysis.
  15 """
  16
  17 import re
  18 import logging
  19 import csv
  20
  21 from collections import OrderedDict
  22 from datetime import datetime
  23 from copy import deepcopy
  24
  25 import prettytable
  26 import plotly.offline as ploff
  27 import plotly.graph_objs as plgo
  28 import plotly.exceptions as plerr
  29
  30 from pal_utils import archive_input_data, execute_command, classify_anomalies
  31
  32
  33 # Command to build the html format of the report
  34 HTML_BUILDER = u'sphinx-build -v -c sphinx_conf/trending -a ' \
  35                u'-b html -E ' \
  36                u'-t html ' \
  37                u'-D version="{date}" ' \
  38                u'{working_dir} ' \
  39                u'{build_dir}/'
  40
  41 # .css file for the html format of the report
  42 THEME_OVERRIDES = u"""/* override table width restrictions */
  43 .wy-nav-content {
  44     max-width: 1200px !important;
  45 }
  46 .rst-content blockquote {
  47     margin-left: 0px;
  48     line-height: 18px;
  49     margin-bottom: 0px;
  50 }
  51 .wy-menu-vertical a {
  52     display: inline-block;
  53     line-height: 18px;
  54     padding: 0 2em;
  55     display: block;
  56     position: relative;
  57     font-size: 90%;
  58     color: #d9d9d9
  59 }
  60 .wy-menu-vertical li.current a {
  61     color: gray;
  62     border-right: solid 1px #c9c9c9;
  63     padding: 0 3em;
  64 }
  65 .wy-menu-vertical li.toctree-l2.current > a {
  66     background: #c9c9c9;
  67     padding: 0 3em;
  68 }
  69 .wy-menu-vertical li.toctree-l2.current li.toctree-l3 > a {
  70     display: block;
  71     background: #c9c9c9;
  72     padding: 0 4em;
  73 }
  74 .wy-menu-vertical li.toctree-l3.current li.toctree-l4 > a {
  75     display: block;
  76     background: #bdbdbd;
  77     padding: 0 5em;
  78 }
  79 .wy-menu-vertical li.on a, .wy-menu-vertical li.current > a {
  80     color: #404040;
  81     padding: 0 2em;
  82     font-weight: bold;
  83     position: relative;
  84     background: #fcfcfc;
  85     border: none;
  86         border-top-width: medium;
  87         border-bottom-width: medium;
  88         border-top-style: none;
  89         border-bottom-style: none;
  90         border-top-color: currentcolor;
  91         border-bottom-color: currentcolor;
  92     padding-left: 2em -4px;
  93 }
  94 """
  95
  96 COLORS = (
  97     u"#1A1110",
  98     u"#DA2647",
  99     u"#214FC6",
 100     u"#01786F",
 101     u"#BD8260",
 102     u"#FFD12A",
 103     u"#A6E7FF",
 104     u"#738276",
 105     u"#C95A49",
 106     u"#FC5A8D",
 107     u"#CEC8EF",
 108     u"#391285",
 109     u"#6F2DA8",
 110     u"#FF878D",
 111     u"#45A27D",
 112     u"#FFD0B9",
 113     u"#FD5240",
 114     u"#DB91EF",
 115     u"#44D7A8",
 116     u"#4F86F7",
 117     u"#84DE02",
 118     u"#FFCFF1",
 119     u"#614051"
 120 )
 121
 122
 123 def generate_cpta(spec, data):
 124     """Generate all formats and versions of the Continuous Performance Trending
 125     and Analysis.
 126
 127     :param spec: Specification read from the specification file.
 128     :param data: Full data set.
 129     :type spec: Specification
 130     :type data: InputData
 131     """
 132
 133     logging.info(u"Generating the Continuous Performance Trending and Analysis "
 134                  u"...")
 135
 136     ret_code = _generate_all_charts(spec, data)
 137
 138     cmd = HTML_BUILDER.format(
 139         date=datetime.utcnow().strftime(u'%Y-%m-%d %H:%M UTC'),
 140         working_dir=spec.environment[u'paths'][u'DIR[WORKING,SRC]'],
 141         build_dir=spec.environment[u'paths'][u'DIR[BUILD,HTML]'])
 142     execute_command(cmd)
 143
 144     with open(spec.environment[u'paths'][u'DIR[CSS_PATCH_FILE]'], u'w') as \
 145             css_file:
 146         css_file.write(THEME_OVERRIDES)
 147
 148     with open(spec.environment[u'paths'][u'DIR[CSS_PATCH_FILE2]'], u'w') as \
 149             css_file:
 150         css_file.write(THEME_OVERRIDES)
 151
 152     if spec.environment.get(u"archive-inputs", False):
 153         archive_input_data(spec)
 154
 155     logging.info(u"Done.")
 156
 157     return ret_code
 158
 159
 160 def _generate_trending_traces(in_data, job_name, build_info,
 161                               name=u"", color=u"", incl_tests=u"mrr"):
 162     """Generate the trending traces:
 163      - samples,
 164      - outliers, regress, progress
 165      - average of normal samples (trending line)
 166
 167     :param in_data: Full data set.
 168     :param job_name: The name of job which generated the data.
 169     :param build_info: Information about the builds.
 170     :param name: Name of the plot
 171     :param color: Name of the color for the plot.
 172     :param incl_tests: Included tests, accepted values: mrr, ndr, pdr
 173     :type in_data: OrderedDict
 174     :type job_name: str
 175     :type build_info: dict
 176     :type name: str
 177     :type color: str
 178     :type incl_tests: str
 179     :returns: Generated traces (list) and the evaluated result.
 180     :rtype: tuple(traces, result)
 181     """
 182
 183     if incl_tests not in (u"mrr", u"ndr", u"pdr"):
 184         return list(), None
 185
 186     data_x = list(in_data.keys())
 187     data_y_pps = list()
 188     data_y_mpps = list()
 189     data_y_stdev = list()
 190     for item in in_data.values():
 191         data_y_pps.append(float(item[u"receive-rate"]))
 192         data_y_stdev.append(float(item[u"receive-stdev"]) / 1e6)
 193         data_y_mpps.append(float(item[u"receive-rate"]) / 1e6)
 194
 195     hover_text = list()
 196     xaxis = list()
 197     for index, key in enumerate(data_x):
 198         str_key = str(key)
 199         date = build_info[job_name][str_key][0]
 200         hover_str = (u"date: {date}<br>"
 201                      u"{property} [Mpps]: {value:.3f}<br>"
 202                      u"<stdev>"
 203                      u"{sut}-ref: {build}<br>"
 204                      u"csit-ref: {test}-{period}-build-{build_nr}<br>"
 205                      u"testbed: {testbed}")
 206         if incl_tests == u"mrr":
 207             hover_str = hover_str.replace(
 208                 u"<stdev>", f"stdev [Mpps]: {data_y_stdev[index]:.3f}<br>"
 209             )
 210         else:
 211             hover_str = hover_str.replace(u"<stdev>", u"")
 212         if u"-cps" in name:
 213             hover_str = hover_str.replace(u"[Mpps]", u"[Mcps]")
 214         if u"dpdk" in job_name:
 215             hover_text.append(hover_str.format(
 216                 date=date,
 217                 property=u"average" if incl_tests == u"mrr" else u"throughput",
 218                 value=data_y_mpps[index],
 219                 sut=u"dpdk",
 220                 build=build_info[job_name][str_key][1].rsplit(u'~', 1)[0],
 221                 test=incl_tests,
 222                 period=u"weekly",
 223                 build_nr=str_key,
 224                 testbed=build_info[job_name][str_key][2]))
 225         elif u"vpp" in job_name:
 226             hover_str = hover_str.format(
 227                 date=date,
 228                 property=u"average" if incl_tests == u"mrr" else u"throughput",
 229                 value=data_y_mpps[index],
 230                 sut=u"vpp",
 231                 build=build_info[job_name][str_key][1].rsplit(u'~', 1)[0],
 232                 test=incl_tests,
 233                 period=u"daily" if incl_tests == u"mrr" else u"weekly",
 234                 build_nr=str_key,
 235                 testbed=build_info[job_name][str_key][2])
 236             if u"-cps" in name:
 237                 hover_str = hover_str.replace(u"throughput", u"connection rate")
 238             hover_text.append(hover_str)
 239
 240         xaxis.append(datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]),
 241                               int(date[9:11]), int(date[12:])))
 242
 243     data_pd = OrderedDict()
 244     for key, value in zip(xaxis, data_y_pps):
 245         data_pd[key] = value
 246
 247     anomaly_classification, avgs_pps, stdevs_pps = classify_anomalies(data_pd)
 248     avgs_mpps = [avg_pps / 1e6 for avg_pps in avgs_pps]
 249     stdevs_mpps = [stdev_pps / 1e6 for stdev_pps in stdevs_pps]
 250
 251     anomalies = OrderedDict()
 252     anomalies_colors = list()
 253     anomalies_avgs = list()
 254     anomaly_color = {
 255         u"regression": 0.0,
 256         u"normal": 0.5,
 257         u"progression": 1.0
 258     }
 259     if anomaly_classification:
 260         for index, (key, value) in enumerate(data_pd.items()):
 261             if anomaly_classification[index] in (u"regression", u"progression"):
 262                 anomalies[key] = value / 1e6
 263                 anomalies_colors.append(
 264                     anomaly_color[anomaly_classification[index]])
 265                 anomalies_avgs.append(avgs_mpps[index])
 266         anomalies_colors.extend([0.0, 0.5, 1.0])
 267
 268     # Create traces
 269
 270     trace_samples = plgo.Scatter(
 271         x=xaxis,
 272         y=data_y_mpps,
 273         mode=u"markers",
 274         line={
 275             u"width": 1
 276         },
 277         showlegend=True,
 278         legendgroup=name,
 279         name=f"{name}",
 280         marker={
 281             u"size": 5,
 282             u"color": color,
 283             u"symbol": u"circle",
 284         },
 285         text=hover_text,
 286         hoverinfo=u"text+name"
 287     )
 288     traces = [trace_samples, ]
 289
 290     trend_hover_text = list()
 291     for idx in range(len(data_x)):
 292         trend_hover_str = (
 293             f"trend [Mpps]: {avgs_mpps[idx]:.3f}<br>"
 294             f"stdev [Mpps]: {stdevs_mpps[idx]:.3f}"
 295         )
 296         trend_hover_text.append(trend_hover_str)
 297
 298     trace_trend = plgo.Scatter(
 299         x=xaxis,
 300         y=avgs_mpps,
 301         mode=u"lines",
 302         line={
 303             u"shape": u"linear",
 304             u"width": 1,
 305             u"color": color,
 306         },
 307         showlegend=False,
 308         legendgroup=name,
 309         name=f"{name}",
 310         text=trend_hover_text,
 311         hoverinfo=u"text+name"
 312     )
 313     traces.append(trace_trend)
 314
 315     trace_anomalies = plgo.Scatter(
 316         x=list(anomalies.keys()),
 317         y=anomalies_avgs,
 318         mode=u"markers",
 319         hoverinfo=u"none",
 320         showlegend=False,
 321         legendgroup=name,
 322         name=f"{name}-anomalies",
 323         marker={
 324             u"size": 15,
 325             u"symbol": u"circle-open",
 326             u"color": anomalies_colors,
 327             u"colorscale": [
 328                 [0.00, u"red"],
 329                 [0.33, u"red"],
 330                 [0.33, u"white"],
 331                 [0.66, u"white"],
 332                 [0.66, u"green"],
 333                 [1.00, u"green"]
 334             ],
 335             u"showscale": True,
 336             u"line": {
 337                 u"width": 2
 338             },
 339             u"colorbar": {
 340                 u"y": 0.5,
 341                 u"len": 0.8,
 342                 u"title": u"Circles Marking Data Classification",
 343                 u"titleside": u"right",
 344                 u"titlefont": {
 345                     u"size": 14
 346                 },
 347                 u"tickmode": u"array",
 348                 u"tickvals": [0.167, 0.500, 0.833],
 349                 u"ticktext": [u"Regression", u"Normal", u"Progression"],
 350                 u"ticks": u"",
 351                 u"ticklen": 0,
 352                 u"tickangle": -90,
 353                 u"thickness": 10
 354             }
 355         }
 356     )
 357     traces.append(trace_anomalies)
 358
 359     if anomaly_classification:
 360         return traces, anomaly_classification[-1]
 361
 362     return traces, None
 363
 364
 365 def _generate_all_charts(spec, input_data):
 366     """Generate all charts specified in the specification file.
 367
 368     :param spec: Specification.
 369     :param input_data: Full data set.
 370     :type spec: Specification
 371     :type input_data: InputData
 372     """
 373
 374     def _generate_chart(graph):
 375         """Generates the chart.
 376
 377         :param graph: The graph to be generated
 378         :type graph: dict
 379         :returns: Dictionary with the job name, csv table with results and
 380             list of tests classification results.
 381         :rtype: dict
 382         """
 383
 384         logging.info(f"  Generating the chart {graph.get(u'title', u'')} ...")
 385
 386         job_name = list(graph[u"data"].keys())[0]
 387
 388         # Transform the data
 389         logging.info(
 390             f"    Creating the data set for the {graph.get(u'type', u'')} "
 391             f"{graph.get(u'title', u'')}."
 392         )
 393
 394         data = input_data.filter_tests_by_name(
 395             graph,
 396             params=[u"type", u"result", u"throughput", u"tags"],
 397             continue_on_error=True
 398         )
 399
 400         if data is None or data.empty:
 401             logging.error(u"No data.")
 402             return dict()
 403
 404         return_lst = list()
 405
 406         for ttype in graph.get(u"test-type", (u"mrr", )):
 407             for core in graph.get(u"core", tuple()):
 408                 csv_tbl = list()
 409                 res = dict()
 410                 chart_data = dict()
 411                 chart_tags = dict()
 412                 for item in graph.get(u"include", tuple()):
 413                     reg_ex = re.compile(str(item.format(core=core)).lower())
 414                     for job, job_data in data.items():
 415                         if job != job_name:
 416                             continue
 417                         for index, bld in job_data.items():
 418                             for test_id, test in bld.items():
 419                                 if not re.match(reg_ex, str(test_id).lower()):
 420                                     continue
 421                                 if chart_data.get(test_id, None) is None:
 422                                     chart_data[test_id] = OrderedDict()
 423                                 try:
 424                                     if ttype == u"mrr":
 425                                         rate = test[u"result"][u"receive-rate"]
 426                                         stdev = \
 427                                             test[u"result"][u"receive-stdev"]
 428                                     elif ttype == u"ndr":
 429                                         rate = \
 430                                             test["throughput"][u"NDR"][u"LOWER"]
 431                                         stdev = float(u"nan")
 432                                     elif ttype == u"pdr":
 433                                         rate = \
 434                                             test["throughput"][u"PDR"][u"LOWER"]
 435                                         stdev = float(u"nan")
 436                                     else:
 437                                         continue
 438                                     chart_data[test_id][int(index)] = {
 439                                         u"receive-rate": rate,
 440                                         u"receive-stdev": stdev
 441                                     }
 442                                     chart_tags[test_id] = \
 443                                         test.get(u"tags", None)
 444                                 except (KeyError, TypeError):
 445                                     pass
 446
 447                 # Add items to the csv table:
 448                 for tst_name, tst_data in chart_data.items():
 449                     tst_lst = list()
 450                     for bld in builds_dict[job_name]:
 451                         itm = tst_data.get(int(bld), dict())
 452                         # CSIT-1180: Itm will be list, compute stats.
 453                         try:
 454                             tst_lst.append(str(itm.get(u"receive-rate", u"")))
 455                         except AttributeError:
 456                             tst_lst.append(u"")
 457                     csv_tbl.append(f"{tst_name}," + u",".join(tst_lst) + u'\n')
 458
 459                 # Generate traces:
 460                 traces = list()
 461                 index = 0
 462                 groups = graph.get(u"groups", None)
 463                 visibility = list()
 464
 465                 if groups:
 466                     for group in groups:
 467                         visible = list()
 468                         for tag in group:
 469                             for tst_name, test_data in chart_data.items():
 470                                 if not test_data:
 471                                     logging.warning(
 472                                         f"No data for the test {tst_name}"
 473                                     )
 474                                     continue
 475                                 if tag not in chart_tags[tst_name]:
 476                                     continue
 477                                 try:
 478                                     trace, rslt = _generate_trending_traces(
 479                                         test_data,
 480                                         job_name=job_name,
 481                                         build_info=build_info,
 482                                         name=u'-'.join(tst_name.split(u'.')[-1].
 483                                                        split(u'-')[2:-1]),
 484                                         color=COLORS[index],
 485                                         incl_tests=ttype
 486                                     )
 487                                 except IndexError:
 488                                     logging.error(f"Out of colors: index: "
 489                                                   f"{index}, test: {tst_name}")
 490                                     index += 1
 491                                     continue
 492                                 traces.extend(trace)
 493                                 visible.extend(
 494                                     [True for _ in range(len(trace))]
 495                                 )
 496                                 res[tst_name] = rslt
 497                                 index += 1
 498                                 break
 499                         visibility.append(visible)
 500                 else:
 501                     for tst_name, test_data in chart_data.items():
 502                         if not test_data:
 503                             logging.warning(f"No data for the test {tst_name}")
 504                             continue
 505                         try:
 506                             trace, rslt = _generate_trending_traces(
 507                                 test_data,
 508                                 job_name=job_name,
 509                                 build_info=build_info,
 510                                 name=u'-'.join(
 511                                     tst_name.split(u'.')[-1].split(u'-')[2:-1]),
 512                                 color=COLORS[index],
 513                                 incl_tests=ttype
 514                             )
 515                         except IndexError:
 516                             logging.error(
 517                                 f"Out of colors: index: "
 518                                 f"{index}, test: {tst_name}"
 519                             )
 520                             index += 1
 521                             continue
 522                         traces.extend(trace)
 523                         res[tst_name] = rslt
 524                         index += 1
 525
 526                 if traces:
 527                     # Generate the chart:
 528                     try:
 529                         layout = deepcopy(graph[u"layout"])
 530                     except KeyError as err:
 531                         logging.error(u"Finished with error: No layout defined")
 532                         logging.error(repr(err))
 533                         return dict()
 534                     if groups:
 535                         show = list()
 536                         for i in range(len(visibility)):
 537                             visible = list()
 538                             for vis_idx, _ in enumerate(visibility):
 539                                 for _ in range(len(visibility[vis_idx])):
 540                                     visible.append(i == vis_idx)
 541                             show.append(visible)
 542
 543                         buttons = list()
 544                         buttons.append(dict(
 545                             label=u"All",
 546                             method=u"update",
 547                             args=[{u"visible":
 548                                        [True for _ in range(len(show[0]))]}, ]
 549                         ))
 550                         for i in range(len(groups)):
 551                             try:
 552                                 label = graph[u"group-names"][i]
 553                             except (IndexError, KeyError):
 554                                 label = f"Group {i + 1}"
 555                             buttons.append(dict(
 556                                 label=label,
 557                                 method=u"update",
 558                                 args=[{u"visible": show[i]}, ]
 559                             ))
 560
 561                         layout[u"updatemenus"] = list([
 562                             dict(
 563                                 active=0,
 564                                 type=u"dropdown",
 565                                 direction=u"down",
 566                                 xanchor=u"left",
 567                                 yanchor=u"bottom",
 568                                 x=-0.12,
 569                                 y=1.0,
 570                                 buttons=buttons
 571                             )
 572                         ])
 573
 574                     name_file = (
 575                         f"{spec.cpta[u'output-file']}/"
 576                         f"{graph[u'output-file-name']}.html"
 577                     )
 578                     name_file = name_file.format(core=core, test_type=ttype)
 579
 580                     logging.info(f"    Writing the file {name_file}")
 581                     plpl = plgo.Figure(data=traces, layout=layout)
 582                     try:
 583                         ploff.plot(
 584                             plpl,
 585                             show_link=False,
 586                             auto_open=False,
 587                             filename=name_file
 588                         )
 589                     except plerr.PlotlyEmptyDataError:
 590                         logging.warning(u"No data for the plot. Skipped.")
 591
 592                 return_lst.append(
 593                     {
 594                         u"job_name": job_name,
 595                         u"csv_table": csv_tbl,
 596                         u"results": res
 597                     }
 598                 )
 599
 600         return return_lst
 601
 602     builds_dict = dict()
 603     for job, builds in spec.input.items():
 604         if builds_dict.get(job, None) is None:
 605             builds_dict[job] = list()
 606         for build in builds:
 607             if build[u"status"] not in (u"failed", u"not found", u"removed",
 608                                         None):
 609                 builds_dict[job].append(str(build[u"build"]))
 610
 611     # Create "build ID": "date" dict:
 612     build_info = dict()
 613     tb_tbl = spec.environment.get(u"testbeds", None)
 614     for job_name, job_data in builds_dict.items():
 615         if build_info.get(job_name, None) is None:
 616             build_info[job_name] = OrderedDict()
 617         for build in job_data:
 618             testbed = u""
 619             tb_ip = input_data.metadata(job_name, build).get(u"testbed", u"")
 620             if tb_ip and tb_tbl:
 621                 testbed = tb_tbl.get(tb_ip, u"")
 622             build_info[job_name][build] = (
 623                 input_data.metadata(job_name, build).get(u"generated", u""),
 624                 input_data.metadata(job_name, build).get(u"version", u""),
 625                 testbed
 626             )
 627
 628     anomaly_classifications = dict()
 629
 630     # Create the table header:
 631     csv_tables = dict()
 632     for job_name in builds_dict:
 633         if csv_tables.get(job_name, None) is None:
 634             csv_tables[job_name] = list()
 635         header = f"Build Number:,{u','.join(builds_dict[job_name])}\n"
 636         csv_tables[job_name].append(header)
 637         build_dates = [x[0] for x in build_info[job_name].values()]
 638         header = f"Build Date:,{u','.join(build_dates)}\n"
 639         csv_tables[job_name].append(header)
 640         versions = [x[1] for x in build_info[job_name].values()]
 641         header = f"Version:,{u','.join(versions)}\n"
 642         csv_tables[job_name].append(header)
 643
 644     for chart in spec.cpta[u"plots"]:
 645         results = _generate_chart(chart)
 646         if not results:
 647             continue
 648
 649         for result in results:
 650             csv_tables[result[u"job_name"]].extend(result[u"csv_table"])
 651
 652             if anomaly_classifications.get(result[u"job_name"], None) is None:
 653                 anomaly_classifications[result[u"job_name"]] = dict()
 654             anomaly_classifications[result[u"job_name"]].\
 655                 update(result[u"results"])
 656
 657     # Write the tables:
 658     for job_name, csv_table in csv_tables.items():
 659         file_name = f"{spec.cpta[u'output-file']}/{job_name}-trending"
 660         with open(f"{file_name}.csv", u"wt") as file_handler:
 661             file_handler.writelines(csv_table)
 662
 663         txt_table = None
 664         with open(f"{file_name}.csv", u"rt") as csv_file:
 665             csv_content = csv.reader(csv_file, delimiter=u',', quotechar=u'"')
 666             line_nr = 0
 667             for row in csv_content:
 668                 if txt_table is None:
 669                     txt_table = prettytable.PrettyTable(row)
 670                 else:
 671                     if line_nr > 1:
 672                         for idx, item in enumerate(row):
 673                             try:
 674                                 row[idx] = str(round(float(item) / 1000000, 2))
 675                             except ValueError:
 676                                 pass
 677                     try:
 678                         txt_table.add_row(row)
 679                     # PrettyTable raises Exception
 680                     except Exception as err:
 681                         logging.warning(
 682                             f"Error occurred while generating TXT table:\n{err}"
 683                         )
 684                 line_nr += 1
 685             txt_table.align[u"Build Number:"] = u"l"
 686         with open(f"{file_name}.txt", u"wt") as txt_file:
 687             txt_file.write(str(txt_table))
 688
 689     # Evaluate result:
 690     if anomaly_classifications:
 691         result = u"PASS"
 692         for job_name, job_data in anomaly_classifications.items():
 693             file_name = \
 694                 f"{spec.cpta[u'output-file']}/regressions-{job_name}.txt"
 695             with open(file_name, u'w') as txt_file:
 696                 for test_name, classification in job_data.items():
 697                     if classification == u"regression":
 698                         txt_file.write(test_name + u'\n')
 699                     if classification in (u"regression", u"outlier"):
 700                         result = u"FAIL"
 701             file_name = \
 702                 f"{spec.cpta[u'output-file']}/progressions-{job_name}.txt"
 703             with open(file_name, u'w') as txt_file:
 704                 for test_name, classification in job_data.items():
 705                     if classification == u"progression":
 706                         txt_file.write(test_name + u'\n')
 707     else:
 708         result = u"FAIL"
 709
 710     logging.info(f"Partial results: {anomaly_classifications}")
 711     logging.info(f"Result: {result}")
 712
 713     return result