docs/model/current/schema/test_case.info.schema.yaml

   1 # Copyright (c) 2023 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 ---
  15
  16 $id: https://fd.io/FIXME/CSIT/UTI/test_case/info/1.3.0
  17 $schema: https://json-schema.org/draft/2020-12/schema
  18 description: >-
  19     Schema for output of test case.
  20 allOf:
  21 -   description: >-
  22         The main structure, without conditional relations between fields yet.
  23     type: object
  24     additionalProperties: false
  25     properties:
  26         duration:
  27             description: >-
  28                 A derived quantity. Difference between start_time and end_time,
  29                 in seconds.
  30             $ref: "#/$defs/types/nonnegative_number"
  31         dut_type:
  32             description: >-
  33                 DUT type used, e.g. VPP or DPDK.
  34             type: string
  35             minLength: 1
  36         dut_version:
  37             description: >-
  38                 Version string appropriate to DUT type used.
  39             type: string
  40             minLength: 1
  41         end_time:
  42             description: >-
  43                 UTC date and time in RFC 3339 format, specifying calendar time
  44                 just before test case ended (at the end of test case teardown).
  45             $ref: "#/$defs/types/date_time"
  46         hosts:
  47             description: >-
  48                 Array of hosts this test interacted with.
  49                 This can be used for identifying testbed number.
  50                 Valid tests shoud interact with at least one DUT or TG.
  51                 The array is usually sorted, but that is not a requirement.
  52             type: array
  53             minItems: 1
  54             items:
  55                 description: >-
  56                     Host identifier, usually numeric IPv4 address.
  57                 type: string
  58         telemetry:
  59             description: >-
  60                 Array of telemetry entries. Each entry represent one captured
  61                 metric.
  62             type: array
  63             minItems: 0
  64             items:
  65                 description: >-
  66                     Telemetry entry.
  67                 type: string
  68         message:
  69             description: >-
  70                 If passed is true, this value is empty.
  71                 Otherwise, value taken directly from TEST_MESSAGE
  72                 Robot variable, read at the end of test case
  73                 (in test teardown, before export and validation).
  74                 It contains information from the exception
  75                 that caused the failure, probably with additional
  76                 exceptions from teardown keywords.
  77             type: string
  78         passed:
  79             description: >-
  80                 Value set accordingly to TEST_STATUS Robot variable,
  81                 true if and only if the status is "PASS".
  82                 The status is read at the end of test case
  83                 (in test teardown, before export and validation).
  84             type: boolean
  85         result:
  86             type: object
  87             allOf:
  88             -   description: >-
  89                     Sub-schema common for all cases,
  90                     only result type identifier defined here.
  91                 properties:
  92                     type:
  93                         description: >-
  94                             Identifier of which result type case is applied.
  95                         type: string
  96                 required:
  97                 -   type
  98             -   oneOf:
  99                 -   description: >-
 100                         Result type for unknown case.
 101                         This case represents a test with no specific result
 102                         (outside message), e.g. device test;
 103                         or a test with result not parsed into
 104                         this version of model yet, e.g. GSO test.
 105                     additionalProperties: false
 106                     properties:
 107                         type:
 108                             const: unknown
 109                 -   description: >-
 110                         Result type MRR case.
 111                     additionalProperties: false
 112                     properties:
 113                         type:
 114                             const: mrr
 115                         receive_rate:
 116                             description: >-
 117                                 The results refer to receive rates for multiple
 118                                 MRR trials. For PPS, these are aggregate
 119                                 (bidirectional) rates.
 120                                 Currently, the tests are exporting
 121                                 approximated receive rates.
 122                                 That means the actual trial duration
 123                                 is measured (as opposed to trusting traffic
 124                                 generator to honor its target duration),
 125                                 so the resulting values contain noise
 126                                 from time measurement, and can be lower
 127                                 than the real performance
 128                                 (due to various time overheads).
 129                                 Bandwidth values are supported, but currently
 130                                 Robot does not export them.
 131                             $ref: "#/$defs/types/rate_list_with_bandwidth"
 132                     required:
 133                     -   type
 134                     -   receive_rate
 135                 -   description: >-
 136                         Result type NDRPDR case.
 137                     additionalProperties: false
 138                     properties:
 139                         type:
 140                             const: ndrpdr
 141                         ndr:
 142                             description: >-
 143                                 The results refer to search for NDR
 144                                 (Non Drop Rate). For PPS, this is aggregate
 145                                 (bidirectional) rate.
 146                                 Each bound was used as the target load value
 147                                 in a full-duration trial measurement.
 148                                 The accepted loss ratio for NDR is exact zero.
 149                                 Note that packets the Traffic Generator
 150                                 did not send are also counted as lost packets.
 151                             $ref: "#/$defs/macros/lower_and_upper_rate"
 152                         pdr:
 153                             description: >-
 154                                 The results refer to search for PDR
 155                                 (Partial Drop Rate). For PPS, this is aggregate
 156                                 (bidirectional) rate.
 157                                 Each bound was used as the target load value
 158                                 in a full-duration trial measurement.
 159                                 The accepted loss ratio for PDR is 0.5%.
 160                                 Note that packets the Traffic Generator
 161                                 did not send are also counted as lost packets.
 162                             $ref: "#/$defs/macros/lower_and_upper_rate"
 163                         latency_forward:
 164                             description: >-
 165                                 Object with results related to latency part
 166                                 of NDRPDR test, for forward traffic direction.
 167                                 It is the direction used in unidirectional
 168                                 traffic profiles.
 169                                 ASTF profiles and IMIX STL profiles
 170                                 do not support latency information,
 171                                 so for those tests this object is missing.
 172                                 It is also missing if Traffic Generator
 173                                 fails to return valid latency results
 174                                 for any other reasons,
 175                                 e.g. latency rate is too high for CPU/NIC used.
 176                             $ref: "#/$defs/macros/latency_for_loads"
 177                         latency_reverse:
 178                             description: >-
 179                                 Object with results related to latency part
 180                                 of NDRPDR test, for reverse traffic diration.
 181                                 This object is not present
 182                                 when unidirectional traffic profiles are used.
 183                                 ASTF profiles and IMIX STL profiles
 184                                 do not support latency information,
 185                                 so for those tests this object is missing.
 186                                 It is also missing if Traffic Generator
 187                                 fails to return valid latency results
 188                                 for any other reasons,
 189                                 e.g. latency rate is too high for CPU/NIC used.
 190                             $ref: "#/$defs/macros/latency_for_loads"
 191                     required:
 192                     -   type
 193                     -   ndr
 194                     -   pdr
 195                 -   description: >-
 196                         Result type SOAK case.
 197                     additionalProperties: false
 198                     properties:
 199                         type:
 200                             const: soak
 201                         critical_rate:
 202                             description: >-
 203                                 The results refer to bayesian estimate
 204                                 of critical rate corresponding to
 205                                 average loss ratio of 10^-7.
 206                                 For PPS, this is aggregate (bidirectional) rate.
 207                                 The bounds are computed from
 208                                 trial measurement results,
 209                                 but are not equal to any target load used.
 210                                 Note that packets the Traffic Generator
 211                                 did not send are also counted as lost packets.
 212                             $ref: "#/$defs/macros/lower_and_upper_rate"
 213                     required:
 214                     -   type
 215                     -   critical_rate
 216                 -   description: >-
 217                         Result type RECONF case.
 218                     additionalProperties: false
 219                     properties:
 220                         type:
 221                             const: reconf
 222                         aggregate_rate:
 223                             description: >-
 224                                 Load used when reconfiguring, found as NDR lower
 225                                 bound. This is an aggregate (bidirectional)
 226                                 rate. Note that packets which the Traffic
 227                                 Generator did not send, are also counted as lost
 228                                 packets.
 229                             $ref: "#/$defs/types/rate_with_bandwidth"
 230                         loss:
 231                             description: >-
 232                                 Number of packets lost during reconfiguration,
 233                                 with the time that equals packet loss divided by
 234                                 packet rate.
 235                             $ref: "#/$defs/types/packet_with_time"
 236                     required:
 237                     -   type
 238                     -   loss
 239                     -   aggregate_rate
 240                 -   description: >-
 241                         Result type HOSTSTACK case.
 242                     additionalProperties: false
 243                     properties:
 244                         type:
 245                             const: hoststack
 246                         bandwidth:
 247                             description: >-
 248                                 Goodput measured in bits per second.
 249                             $ref: "#/$defs/types/bandwidth"
 250                         completed_requests:
 251                             description: >-
 252                                 Number of completed requests.
 253                             $ref: "#/$defs/types/count_requests"
 254                         failed_requests:
 255                             description: >-
 256                                 Number of failed requests.
 257                             $ref: "#/$defs/types/count_requests"
 258                         retransmits:
 259                             description: >-
 260                                 Number of retransmits.
 261                             $ref: "#/$defs/types/count_packets"
 262                         latency:
 263                             description: >-
 264                                 Value and unit of latency.
 265                             $ref: "#/$defs/types/value_with_unit"
 266                         duration:
 267                             description: >-
 268                                 The relative time difference (in seconds)
 269                                 between program start and end.
 270                             $ref: "#/$defs/types/time_quantity"
 271                         rate:
 272                             description: >-
 273                                 RPS or CPS rate, with corresponding unit, as
 274                                 reported by TG.
 275                             $ref: "#/$defs/types/rate_without_bandwidth"
 276                     required:
 277                     -   type
 278                     -   bandwidth
 279         start_time:
 280             description: >-
 281                 UTC date and time in RFC 3339 format, specifying calendar time
 282                 just after test case started (at the start of test setup).
 283             $ref: "#/$defs/types/date_time"
 284         tags:
 285             description: >-
 286                 The list of strings comes directly
 287                 from Robot variable TEST_TAGS.
 288                 The content should include both static and dynamic tags
 289                 at the end of test case (teardown).
 290             type: array
 291             items:
 292                 type: string
 293         test_documentation:
 294             description: >-
 295                 Value taken directly from TEST_DOCUMENTATION Robot variable.
 296                 The content is what you see in suite file
 297                 at test case definition, which is usually empty
 298                 as CSIT uses data driven test cases.
 299             type: string
 300         test_id:
 301             description: >-
 302                 A derived quantity.
 303                 It is the most complete and unique identifier for a test case.
 304                 This property has a value, of the following form:
 305                 {suite_name}.{test_name}
 306                 Here, suite name comes from SUITE_NAME robot variable,
 307                 test name comes from TEST_NAME robot variable,
 308                 but both are converted to lower case,
 309                 and spaces are replaced by underscores.
 310             type: string
 311             minLength: 3
 312         test_name_long:
 313             description: >-
 314                 A derived quantity.
 315                 This property has a value, of the following form:
 316                 {nic_short_name}-{frame_size}-{threads_and_cores}-{suite_part}
 317                 Here, suite part is very similar to suite tag,
 318                 but additionally may contain a prefix describing NIC driver used
 319                 (if it is not the default one, drv_vfio_pci for VPP tests).
 320                 Any space is replaced by underscore and letters are lower case.
 321             type: string
 322             minLength: 3
 323         test_name_short:
 324             description: >-
 325                 A derived quantity.
 326                 This property has a value very similar to suite tag,
 327                 but additionally may contain a prefix describing NIC driver used
 328                 (if it is not the default one, drv_vfio_pci for VPP tests).
 329                 Any space is replaced by underscore and letters are lower case.
 330             type: string
 331             minLength: 3
 332         test_type:
 333             description: >-
 334                 A derived quantity.
 335                 Test type identifier, PAL uses it to group similar tests,
 336                 e.g. for comparison tables.
 337                 Ideally, this information should be parseable from test name,
 338                 but the current naming scheme is not simple/consistent enough.
 339                 The current implementation queries the robot test tags.
 340                 The resulting value is frequently identical to result type,
 341                 but this schema version does not require any relation there,
 342                 as PAL may want to group tests differently.
 343             type: string
 344             enum:
 345             -   device
 346             -   gso
 347             -   hoststack
 348             -   mrr
 349             -   ndrpdr
 350             -   reconf
 351             -   soak
 352         tg_type:
 353             description: >-
 354                 TG type used, e.g. TREX.
 355             type: string
 356             minLength: 1
 357         tg_version:
 358             description: >-
 359                 Version string appropriate to TG type used.
 360             type: string
 361             minLength: 1
 362         version:
 363             description: >-
 364                 CSIT model version (semver format)
 365                 the exporting code adhered to.
 366             type: string
 367             const: 1.3.0
 368     required:
 369     -   duration
 370     -   dut_type
 371     -   dut_version
 372     -   end_time
 373     -   hosts
 374     -   telemetry
 375     -   message
 376     -   passed
 377     -   result
 378     -   start_time
 379     -   tags
 380     -   test_documentation
 381     -   test_id
 382     -   test_name_long
 383     -   test_name_short
 384     -   test_type
 385     -   tg_type
 386     -   tg_version
 387     -   version
 388 -   description: >-
 389         Subschema validating relation between status and message.
 390     oneOf:
 391     -   description: >-
 392             Subschema for passing tests, message has to be empty.
 393         type: object
 394         properties:
 395             passed:
 396                 const: true
 397             message:
 398                 const: ""
 399     -   description: >-
 400             Subschema for failing tests, mesage cannot be empty.
 401         type: object
 402         properties:
 403             passed:
 404                 const: false
 405             message:
 406                 minLength: 1
 407
 408 $defs:
 409     types:
 410         nonnegative_number:
 411             type: number
 412             minimum: 0
 413         positive_number:
 414             type: number
 415             minimum: 1
 416         nonnegative_integer:
 417             type: integer
 418             minimum: 0
 419         positive_integer:
 420             type: integer
 421             minimum: 1
 422         date_time:
 423             type: string
 424             format: date-time
 425         empty_array:
 426             type: array
 427             maxItems: 0
 428         rate_unit:
 429             description: >-
 430                 Packets per second (pps),
 431                 connections per second (cps),
 432                 requests per second (rps).
 433             type: string
 434             enum:
 435             -   pps
 436             -   cps
 437             -   rps
 438         bandwidth_unit:
 439             description: >-
 440                 Unit of measurement for bandwidth values.
 441                 Currently a constant, but later versions of model
 442                 may allow more units.
 443             enum:
 444             -   bps
 445         count_packets:
 446             description: >-
 447                 Type, for counting packets.
 448             allOf:
 449             -   $ref: "#/$defs/types/value_with_unit"
 450             -   properties:
 451                     value:
 452                         description: >-
 453                             A number of packets of interest.
 454                     unit:
 455                         description: >-
 456                             Unit suitable for displaying packet counts.
 457                         enum:
 458                         -   packets
 459         count_requests:
 460             description: >-
 461                 Type, for counting requests.
 462             allOf:
 463             -   $ref: "#/$defs/types/value_with_unit"
 464             -   properties:
 465                     value:
 466                         description: >-
 467                             A number of requests of interest.
 468                     unit:
 469                         description: >-
 470                             Unit suitable for displaying request counts.
 471                         enum:
 472                         -   requests
 473         time_quantity:
 474             description: >-
 475                 Reusable type, for various time quantites.
 476             allOf:
 477             -   $ref: "#/$defs/types/value_with_unit"
 478             -   properties:
 479                     value:
 480                         description: >-
 481                             Unless specified otherwise, this is a duration
 482                             between two events.
 483                     unit:
 484                         description: >-
 485                             Only seconds are the unit supported for time
 486                             quantities.
 487                         enum:
 488                         -   s
 489         value_with_unit:
 490             description: >-
 491                 Reusable composite type, value together with its
 492                 unit of measurement.
 493             type: object
 494             additionalProperties: false
 495             properties:
 496                 value:
 497                     description: >-
 498                         Numeric value, context specified elsewhere.
 499                         The only assumption is that value is not negative.
 500                     $ref: "#/$defs/types/nonnegative_number"
 501                 unit:
 502                     description: >-
 503                         Unit of measurement for the value.
 504                         Context and allowed values are specified elsewhere.
 505                     type: string
 506             required:
 507             -   value
 508             -   unit
 509         rate_without_bandwidth:
 510             description: >-
 511                 Reusable type, for various rate quantites.
 512             allOf:
 513             -   $ref: "#/$defs/types/value_with_unit"
 514             -   properties:
 515                     value:
 516                         description: >-
 517                             Unless specified otherwise,
 518                             this is the aggregated rate
 519                             (sum of both traffic directions).
 520                             Depending on the usage, the value can express
 521                             intended load, offered load, receive rate,
 522                             and various approximations
 523                             or estimated bounds thereof.
 524                     unit:
 525                         description: >-
 526                             A transaction rate unit the value is expressed in.
 527                         $ref: "#/$defs/types/rate_unit"
 528         bandwidth:
 529             description: >-
 530                 Reusable type, for various bandwidth quantites.
 531             allOf:
 532             -   $ref: "#/$defs/types/value_with_unit"
 533             -   properties:
 534                     value:
 535                         description: >-
 536                             Bandwidth value computed from the corresponding
 537                             rate.
 538                     unit:
 539                         $ref: "#/$defs/types/bandwidth_unit"
 540         rate_with_bandwidth:
 541             description: >-
 542                 Reusable composite type, joining primary rate
 543                 with optional derived bandwidth.
 544                 Not all test types currently compute bandwidth,
 545                 even if rate unit is pps.
 546             type: object
 547             additionalProperties: false
 548             properties:
 549                 rate:
 550                     $ref: "#/$defs/types/rate_without_bandwidth"
 551                 bandwidth:
 552                     $ref: "#/$defs/types/bandwidth"
 553             required:
 554             - rate
 555         packet_with_time:
 556             description: >-
 557                 Reusable composite type, joining packet count with the
 558                 time quantity.
 559             type: object
 560             additionalProperties: false
 561             properties:
 562                 packet:
 563                     $ref: "#/$defs/types/count_packets"
 564                 time:
 565                     $ref: "#/$defs/types/time_quantity"
 566             required:
 567             - packet
 568             - time
 569         value_list_with_unit_and_stats:
 570             description: >-
 571                 Reusable composite type, multiple values together with their
 572                 unit of measurement and derived statistics.
 573             type: object
 574             additionalProperties: false
 575             properties:
 576                 values:
 577                     description: >-
 578                         List of values of the same unit, useful for MRR.
 579                     type: array
 580                     minItmes: 1
 581                     items:
 582                         description: >-
 583                             Numeric value, context specified elsewhere. The only
 584                             assumption is that the value is nonnegative.
 585                         $ref: "#/$defs/types/nonnegative_number"
 586                 avg:
 587                     description: >-
 588                         A derived quantity. It is the arithmetic average of the
 589                         values list.
 590                     $ref: "#/$defs/types/nonnegative_number"
 591                 stdev:
 592                     description: >-
 593                         A derived quantity. It is the standard deviation for the
 594                         values list, as computed by jumpavg library.
 595                     $ref: "#/$defs/types/nonnegative_number"
 596                 unit:
 597                     description: >-
 598                         Unit of measurement for the values.
 599                         Context and allowed values are specified elsewhere.
 600                     type: string
 601             required:
 602             -   values
 603             -   avg
 604             -   stdev
 605             -   unit
 606         rate_list_without_bandwidth:
 607             description: >-
 608                 Reusable composite type, multiple rate values.
 609             allOf:
 610             -   $ref: "#/$defs/types/value_list_with_unit_and_stats"
 611             -   properties:
 612                     values:
 613                         items:
 614                             description: >-
 615                                 Unless specified otherwise,
 616                                 this is the aggregated rate
 617                                 (sum of both traffic directions).
 618                                 Depending on the usage, the value can express
 619                                 intended load, offered load, receive rate,
 620                                 and various approximations or estimated bounds
 621                                 thereof.
 622                     unit:
 623                         $ref: "#/$defs/types/rate_unit"
 624         bandwidth_list:
 625             description: >-
 626                 Reusable composite type, multiple bandwidth values. This is a
 627                 derived quantity.
 628             allOf:
 629             -   $ref: "#/$defs/types/value_list_with_unit_and_stats"
 630             -   properties:
 631                     values:
 632                         items:
 633                             description: >-
 634                                 Unless specified otherwise,
 635                                 this is the aggregated bandwidth
 636                                 (sum of both traffic directions).
 637                                 Depending on the usage, the value can express
 638                                 intended load, offered load, receive rate,
 639                                 and various approximations or estimated bounds
 640                                 thereof.
 641                     unit:
 642                         $ref: "#/$defs/types/bandwidth_unit"
 643         rate_list_with_bandwidth:
 644             description: >-
 645                 Reusable composite type, joining primary rates
 646                 with optional derived bandwidths (and stats).
 647                 No test types currently computes the bandwidth part.
 648             type: object
 649             additionalProperties: false
 650             properties:
 651                 rate:
 652                     $ref: "#/$defs/types/rate_list_without_bandwidth"
 653                 bandwidth:
 654                     $ref: "#/$defs/types/bandwidth_list"
 655             required:
 656             - rate
 657     macros:
 658         lower_and_upper_rate:
 659             type: object
 660             additionalProperties: false
 661             properties:
 662                 lower:
 663                     description: >-
 664                         The lower bound (or min_rate) for the estimate
 665                         of a particular searched value.
 666                     $ref: "#/$defs/types/rate_with_bandwidth"
 667                 upper:
 668                     description: >-
 669                         The upper bound (or max_rate) for the estimate
 670                         of a particular searched value.
 671                     $ref: "#/$defs/types/rate_with_bandwidth"
 672             required:
 673             -   lower
 674             -   upper
 675         latency_numbers:
 676             type: object
 677             additionalProperties: false
 678             properties:
 679                 min:
 680                     description: >-
 681                         Rounded minimal latency time measured in this trial.
 682                         See unit property for the unit of measurement.
 683                     $ref: "#/$defs/types/nonnegative_integer"
 684                 max:
 685                     description: >-
 686                         Rounded maximal latency time measured in this trial.
 687                         See unit property for the unit of measurement.
 688                         Zero value is not allowed, as that is one of symptoms
 689                         of Traffic Generator failing to get proper latency.
 690                     $ref: "#/$defs/types/positive_integer"
 691                 avg:
 692                     description: >-
 693                         Rounded average latency time measured in this trial.
 694                         See unit property for the unit of measurement.
 695                     $ref: "#/$defs/types/nonnegative_integer"
 696                 hdrh:
 697                     description: >-
 698                         Base64-encoded compressed representation of HDRHistogram
 699                         of all latency sample times encountered
 700                         in this latency trial.
 701                         See unit property for the unit of measurement.
 702                         Note that some bins can be several units wide.
 703                     type: string
 704                 unit:
 705                     description: >-
 706                         Unit of measurement for latency times.
 707                         Currently a constant, but later versions
 708                         of the model may allow more values.
 709                     type: string
 710                     enum:
 711                     -   us
 712             required:
 713             -   avg
 714             -   hdrh
 715             -   max
 716             -   min
 717             -   unit
 718         latency_for_loads:
 719             type: object
 720             additionalProperties: false
 721             properties:
 722                 pdr_0:
 723                     description: >-
 724                         Object related to latency measurement performed
 725                         at minimal rate (currently 9000 pps per direction).
 726                     $ref: "#/$defs/macros/latency_numbers"
 727                 pdr_10:
 728                     description: >-
 729                         Object related to latency measurement performed
 730                         at 10% of PDR lower bound, if needed rounded up
 731                         to minimal rate (currently 9000 pps per direction).
 732                     $ref: "#/$defs/macros/latency_numbers"
 733                 pdr_50:
 734                     description: >-
 735                         Object related to latency measurement performed
 736                         at 50% of PDR lower bound, if needed rounded up
 737                         to minimal rate (currently 9000 pps per direction).
 738                     $ref: "#/$defs/macros/latency_numbers"
 739                 pdr_90:
 740                     description: >-
 741                         Object related to latency measurement performed
 742                         at 90% of PDR lower bound, if needed rounded up
 743                         to minimal rate (currently 9000 pps per direction).
 744                     $ref: "#/$defs/macros/latency_numbers"
 745             required:
 746             -   pdr_0
 747             -   pdr_10
 748             -   pdr_50
 749             -   pdr_90