docs/model/current/schema/test_case.info.schema.yaml

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 ---
  15
  16 $id: https://fd.io/FIXME/CSIT/UTI/test_case/info/1.2.0
  17 $schema: https://json-schema.org/draft/2020-12/schema
  18 description: >-
  19     Schema for output of test case.
  20 allOf:
  21 -   description: >-
  22         The main structure, without conditional relations between fields yet.
  23     type: object
  24     additionalProperties: false
  25     properties:
  26         duration:
  27             description: >-
  28                 A derived quantity. Difference between start_time and end_time,
  29                 in seconds.
  30             $ref: "#/$defs/types/nonnegative_number"
  31         dut_type:
  32             description: >-
  33                 DUT type used, e.g. VPP or DPDK.
  34             type: string
  35             minLength: 1
  36         dut_version:
  37             description: >-
  38                 Version string appropriate to DUT type used.
  39             type: string
  40             minLength: 1
  41         end_time:
  42             description: >-
  43                 UTC date and time in RFC 3339 format, specifying calendar time
  44                 just before test case ended (at the end of test case teardown).
  45             $ref: "#/$defs/types/date_time"
  46         hosts:
  47             description: >-
  48                 Array of hosts this test interacted with.
  49                 This can be used for identifying testbed number.
  50                 Valid tests shoud interact with at least one DUT or TG.
  51                 The array is usually sorted, but that is not a requirement.
  52             type: array
  53             minItems: 1
  54             items:
  55                 description: >-
  56                     Host identifier, usually numeric IPv4 address.
  57                 type: string
  58         telemetry:
  59             description: >-
  60                 Array of telemetry entries. Each entry represent one captured
  61                 metric.
  62             type: array
  63             minItems: 0
  64             items:
  65                 description: >-
  66                     Telemetry entry.
  67                 type: string
  68         message:
  69             description: >-
  70                 If passed is true, this value is empty.
  71                 Otherwise, value taken directly from TEST_MESSAGE
  72                 Robot variable, read at the end of test case
  73                 (in test teardown, before export and validation).
  74                 It contains information from the exception
  75                 that caused the failure, probably with additional
  76                 exceptions from teardown keywords.
  77             type: string
  78         passed:
  79             description: >-
  80                 Value set accordingly to TEST_STATUS Robot variable,
  81                 true if and only if the status is "PASS".
  82                 The status is read at the end of test case
  83                 (in test teardown, before export and validation).
  84             type: boolean
  85         result:
  86             type: object
  87             allOf:
  88             -   description: >-
  89                     Sub-schema common for all cases,
  90                     only result type identifier defined here.
  91                 properties:
  92                     type:
  93                         description: >-
  94                             Identifier of which result type case is applied.
  95                         type: string
  96                 required:
  97                 -   type
  98             -   oneOf:
  99                 -   description: >-
 100                         Result type for unknown case.
 101                         This case represents a test with no specific result
 102                         (outside message), e.g. device test;
 103                         or a test with result not parsed into
 104                         this version of model yet, e.g. GSO test.
 105                     additionalProperties: false
 106                     properties:
 107                         type:
 108                             const: unknown
 109                 -   description: >-
 110                         Result type MRR case.
 111                     additionalProperties: false
 112                     properties:
 113                         type:
 114                             const: mrr
 115                         receive_rate:
 116                             description: >-
 117                                 The results refer to receive rates for multiple
 118                                 MRR trials. For PPS, these are aggregate
 119                                 (bidirectional) rates.
 120                                 Currently, the tests are exporting
 121                                 approximated receive rates.
 122                                 That means the actual trial duration
 123                                 is measured (as opposed to trusting traffic
 124                                 generator to honor its target duration),
 125                                 so the resulting values contain noise
 126                                 from time measurement, and can be lower
 127                                 than the real performance
 128                                 (due to various time overheads).
 129                                 Bandwidth values are supported, but currently
 130                                 Robot does not export them.
 131                             $ref: "#/$defs/types/rate_list_with_bandwidth"
 132                     required:
 133                     -   type
 134                     -   receive_rate
 135                 -   description: >-
 136                         Result type NDRPDR case.
 137                     additionalProperties: false
 138                     properties:
 139                         type:
 140                             const: ndrpdr
 141                         ndr:
 142                             description: >-
 143                                 The results refer to search for NDR
 144                                 (Non Drop Rate). For PPS, this is aggregate
 145                                 (bidirectional) rate.
 146                                 Each bound was used as the target load value
 147                                 in a full-duration trial measurement.
 148                                 The accepted loss ratio for NDR is exact zero.
 149                                 Note that packets the Traffic Generator
 150                                 did not send are also counted as lost packets.
 151                             $ref: "#/$defs/macros/lower_and_upper_rate"
 152                         pdr:
 153                             description: >-
 154                                 The results refer to search for PDR
 155                                 (Partial Drop Rate). For PPS, this is aggregate
 156                                 (bidirectional) rate.
 157                                 Each bound was used as the target load value
 158                                 in a full-duration trial measurement.
 159                                 The accepted loss ratio for PDR is 0.5%.
 160                                 Note that packets the Traffic Generator
 161                                 did not send are also counted as lost packets.
 162                             $ref: "#/$defs/macros/lower_and_upper_rate"
 163                         latency_forward:
 164                             description: >-
 165                                 Object with results related to latency part
 166                                 of NDRPDR test, for forward traffic direction.
 167                                 It is the direction used in unidirectional
 168                                 traffic profiles.
 169                                 ASTF profiles and IMIX STL profiles
 170                                 do not support latency information,
 171                                 so for those tests this object is missing.
 172                                 It is also missing if Traffic Generator
 173                                 fails to return valid latency results
 174                                 for any other reasons,
 175                                 e.g. latency rate is too high for CPU/NIC used.
 176                             $ref: "#/$defs/macros/latency_for_loads"
 177                         latency_reverse:
 178                             description: >-
 179                                 Object with results related to latency part
 180                                 of NDRPDR test, for reverse traffic diration.
 181                                 This object is not present
 182                                 when unidirectional traffic profiles are used.
 183                                 ASTF profiles and IMIX STL profiles
 184                                 do not support latency information,
 185                                 so for those tests this object is missing.
 186                                 It is also missing if Traffic Generator
 187                                 fails to return valid latency results
 188                                 for any other reasons,
 189                                 e.g. latency rate is too high for CPU/NIC used.
 190                             $ref: "#/$defs/macros/latency_for_loads"
 191                     required:
 192                     -   type
 193                     -   ndr
 194                     -   pdr
 195                 -   description: >-
 196                         Result type SOAK case.
 197                     additionalProperties: false
 198                     properties:
 199                         type:
 200                             const: soak
 201                         critical_rate:
 202                             description: >-
 203                                 The results refer to bayesian estimate
 204                                 of critical rate corresponding to
 205                                 average loss ratio of 10^-7.
 206                                 For PPS, this is aggregate (bidirectional) rate.
 207                                 The bounds are computed from
 208                                 trial measurement results,
 209                                 but are not equal to any target load used.
 210                                 Note that packets the Traffic Generator
 211                                 did not send are also counted as lost packets.
 212                             $ref: "#/$defs/macros/lower_and_upper_rate"
 213                     required:
 214                     -   type
 215                     -   critical_rate
 216                 -   description: >-
 217                         Result type RECONF case.
 218                     additionalProperties: false
 219                     properties:
 220                         type:
 221                             const: reconf
 222                         aggregate_rate:
 223                             description: >-
 224                                 Load used when reconfiguring, found as NDR lower
 225                                 bound. This is an aggregate (bidirectional)
 226                                 rate. Note that packets which the Traffic
 227                                 Generator did not send, are also counted as lost
 228                                 packets.
 229                             $ref: "#/$defs/types/rate_with_bandwidth"
 230                         loss:
 231                             description: >-
 232                                 Number of packets lost during reconfiguration,
 233                                 with the time that equals packet loss divided by
 234                                 packet rate.
 235                             $ref: "#/$defs/types/packet_with_time"
 236                     required:
 237                     -   loss
 238                     -   aggregate_rate
 239         start_time:
 240             description: >-
 241                 UTC date and time in RFC 3339 format, specifying calendar time
 242                 just after test case started (at the start of test setup).
 243             $ref: "#/$defs/types/date_time"
 244         tags:
 245             description: >-
 246                 The list of strings comes directly
 247                 from Robot variable TEST_TAGS.
 248                 The content should include both static and dynamic tags
 249                 at the end of test case (teardown).
 250             type: array
 251             items:
 252                 type: string
 253         test_documentation:
 254             description: >-
 255                 Value taken directly from TEST_DOCUMENTATION Robot variable.
 256                 The content is what you see in suite file
 257                 at test case definition, which is usually empty
 258                 as CSIT uses data driven test cases.
 259             type: string
 260         test_id:
 261             description: >-
 262                 A derived quantity.
 263                 It is the most complete and unique identifier for a test case.
 264                 This property has a value, of the following form:
 265                 {suite_name}.{test_name}
 266                 Here, suite name comes from SUITE_NAME robot variable,
 267                 test name comes from TEST_NAME robot variable,
 268                 but both are converted to lower case,
 269                 and spaces are replaced by underscores.
 270             type: string
 271             minLength: 3
 272         test_name_long:
 273             description: >-
 274                 A derived quantity.
 275                 This property has a value, of the following form:
 276                 {nic_short_name}-{frame_size}-{threads_and_cores}-{suite_part}
 277                 Here, suite part is very similar to suite tag,
 278                 but additionally may contain a prefix describing NIC driver used
 279                 (if it is not the default one, drv_vfio_pci for VPP tests).
 280                 Any space is replaced by underscore and letters are lower case.
 281             type: string
 282             minLength: 3
 283         test_name_short:
 284             description: >-
 285                 A derived quantity.
 286                 This property has a value very similar to suite tag,
 287                 but additionally may contain a prefix describing NIC driver used
 288                 (if it is not the default one, drv_vfio_pci for VPP tests).
 289                 Any space is replaced by underscore and letters are lower case.
 290             type: string
 291             minLength: 3
 292         test_type:
 293             description: >-
 294                 A derived quantity.
 295                 Test type identifier, PAL uses it to group similar tests,
 296                 e.g. for comparison tables.
 297                 Ideally, this information should be parseable from test name,
 298                 but the current naming scheme is not simple/consistent enough.
 299                 The current implementation queries the robot test tags.
 300                 The resulting value is frequently identical to result type,
 301                 but this schema version does not require any relation there,
 302                 as PAL may want to group tests differently.
 303             type: string
 304             enum:
 305             -   device
 306             -   gso
 307             -   hoststack
 308             -   mrr
 309             -   ndrpdr
 310             -   reconf
 311             -   soak
 312             -   vsap
 313         tg_type:
 314             description: >-
 315                 TG type used, e.g. TREX.
 316             type: string
 317             minLength: 1
 318         tg_version:
 319             description: >-
 320                 Version string appropriate to TG type used.
 321             type: string
 322             minLength: 1
 323         version:
 324             description: >-
 325                 CSIT model version (semver format)
 326                 the exporting code adhered to.
 327             type: string
 328             const: 1.2.0
 329     required:
 330     -   duration
 331     -   dut_type
 332     -   dut_version
 333     -   end_time
 334     -   hosts
 335     -   telemetry
 336     -   message
 337     -   passed
 338     -   result
 339     -   start_time
 340     -   tags
 341     -   test_documentation
 342     -   test_id
 343     -   test_name_long
 344     -   test_name_short
 345     -   test_type
 346     -   tg_type
 347     -   tg_version
 348     -   version
 349 -   description: >-
 350         Subschema validating relation between status and message.
 351     oneOf:
 352     -   description: >-
 353             Subschema for passing tests, message has to be empty.
 354         type: object
 355         properties:
 356             passed:
 357                 const: true
 358             message:
 359                 const: ""
 360     -   description: >-
 361             Subschema for failing tests, mesage cannot be empty.
 362         type: object
 363         properties:
 364             passed:
 365                 const: false
 366             message:
 367                 minLength: 1
 368
 369 $defs:
 370     types:
 371         nonnegative_number:
 372             type: number
 373             minimum: 0
 374         positive_number:
 375             type: number
 376             minimum: 1
 377         nonnegative_integer:
 378             type: integer
 379             minimum: 0
 380         positive_integer:
 381             type: integer
 382             minimum: 1
 383         date_time:
 384             type: string
 385             format: date-time
 386         empty_array:
 387             type: array
 388             maxItems: 0
 389         rate_unit:
 390             description: >-
 391                 Packets per second (pps) or connections per second (cps).
 392             type: string
 393             enum:
 394             -   pps
 395             -   cps
 396         bandwidth_unit:
 397             description: >-
 398                 Unit of measurement for bandwidth values.
 399                 Currently a constant, but later versions of model
 400                 may allow more units.
 401             enum:
 402             -   bps
 403         count_packets:
 404             description: >-
 405                 Type, for counting packets.
 406             allOf:
 407             -   $ref: "#/$defs/types/value_with_unit"
 408             -   properties:
 409                     value:
 410                         description: >-
 411                             A number of packets of interest.
 412                     unit:
 413                         description: >-
 414                             Unit suitable for displaying packet counts.
 415                         enum:
 416                         -   packets
 417         time_quantity:
 418             description: >-
 419                 Reusable type, for various time quantites.
 420             allOf:
 421             -   $ref: "#/$defs/types/value_with_unit"
 422             -   properties:
 423                     value:
 424                         description: >-
 425                             Unless specified otherwise, this is a duration
 426                             between two events.
 427                     unit:
 428                         description: >-
 429                             Only seconds are the unit supported for time
 430                             quantities.
 431                         enum:
 432                         -   s
 433         value_with_unit:
 434             description: >-
 435                 Reusable composite type, value together with its
 436                 unit of measurement.
 437             type: object
 438             additionalProperties: false
 439             properties:
 440                 value:
 441                     description: >-
 442                         Numeric value, context specified elsewhere.
 443                         The only assumption is that value is not negative.
 444                     $ref: "#/$defs/types/nonnegative_number"
 445                 unit:
 446                     description: >-
 447                         Unit of measurement for the value.
 448                         Context and allowed values are specified elsewhere.
 449                     type: string
 450             required:
 451             -   value
 452             -   unit
 453         rate_without_bandwidth:
 454             description: >-
 455                 Reusable type, for various rate quantites.
 456             allOf:
 457             -   $ref: "#/$defs/types/value_with_unit"
 458             -   properties:
 459                     value:
 460                         description: >-
 461                             Unless specified otherwise,
 462                             this is the aggregated rate
 463                             (sum of both traffic directions).
 464                             Depending on the usage, the value can express
 465                             intended load, offered load, receive rate,
 466                             and various approximations
 467                             or estimated bounds thereof.
 468                     unit:
 469                         description: >-
 470                             A transaction rate unit the value is expressed in.
 471                         $ref: "#/$defs/types/rate_unit"
 472         bandwidth:
 473             description: >-
 474                 Reusable type, for various bandwidth quantites.
 475             allOf:
 476             -   $ref: "#/$defs/types/value_with_unit"
 477             -   properties:
 478                     value:
 479                         description: >-
 480                             Bandwidth value computed
 481                             from the corresponding rate.
 482                     unit:
 483                         $ref: "#/$defs/types/bandwidth_unit"
 484         rate_with_bandwidth:
 485             description: >-
 486                 Reusable composite type, joining primary rate
 487                 with optional derived bandwidth.
 488                 Not all test types currently compute bandwidth,
 489                 even if rate unit is pps.
 490             type: object
 491             additionalProperties: false
 492             properties:
 493                 rate:
 494                     $ref: "#/$defs/types/rate_without_bandwidth"
 495                 bandwidth:
 496                     $ref: "#/$defs/types/bandwidth"
 497             required:
 498             - rate
 499         packet_with_time:
 500             description: >-
 501                 Reusable composite type, joining packet count with the
 502                 time quantity.
 503             type: object
 504             additionalProperties: false
 505             properties:
 506                 packet:
 507                     $ref: "#/$defs/types/count_packets"
 508                 time:
 509                     $ref: "#/$defs/types/time_quantity"
 510             required:
 511             - packet
 512             - time
 513         value_list_with_unit_and_stats:
 514             description: >-
 515                 Reusable composite type, multiple values together with their
 516                 unit of measurement and derived statistics.
 517             type: object
 518             additionalProperties: false
 519             properties:
 520                 values:
 521                     description: >-
 522                         List of values of the same unit, useful for MRR.
 523                     type: array
 524                     minItmes: 1
 525                     items:
 526                         description: >-
 527                             Numeric value, context specified elsewhere. The only
 528                             assumption is that the value is nonnegative.
 529                         $ref: "#/$defs/types/nonnegative_number"
 530                 avg:
 531                     description: >-
 532                         A derived quantity. It is the arithmetic average of the
 533                         values list.
 534                     $ref: "#/$defs/types/nonnegative_number"
 535                 stdev:
 536                     description: >-
 537                         A derived quantity. It is the standard deviation for the
 538                         values list, as computed by jumpavg library.
 539                     $ref: "#/$defs/types/nonnegative_number"
 540                 unit:
 541                     description: >-
 542                         Unit of measurement for the values.
 543                         Context and allowed values are specified elsewhere.
 544                     type: string
 545             required:
 546             -   values
 547             -   avg
 548             -   stdev
 549             -   unit
 550         rate_list_without_bandwidth:
 551             description: >-
 552                 Reusable composite type, multiple rate values.
 553             allOf:
 554             -   $ref: "#/$defs/types/value_list_with_unit_and_stats"
 555             -   properties:
 556                     values:
 557                         items:
 558                             description: >-
 559                                 Unless specified otherwise,
 560                                 this is the aggregated rate
 561                                 (sum of both traffic directions).
 562                                 Depending on the usage, the value can express
 563                                 intended load, offered load, receive rate,
 564                                 and various approximations or estimated bounds
 565                                 thereof.
 566                     unit:
 567                         $ref: "#/$defs/types/rate_unit"
 568         bandwidth_list:
 569             description: >-
 570                 Reusable composite type, multiple bandwidth values. This is a
 571                 derived quantity.
 572             allOf:
 573             -   $ref: "#/$defs/types/value_list_with_unit_and_stats"
 574             -   properties:
 575                     values:
 576                         items:
 577                             description: >-
 578                                 Unless specified otherwise,
 579                                 this is the aggregated bandwidth
 580                                 (sum of both traffic directions).
 581                                 Depending on the usage, the value can express
 582                                 intended load, offered load, receive rate,
 583                                 and various approximations or estimated bounds
 584                                 thereof.
 585                     unit:
 586                         $ref: "#/$defs/types/bandwidth_unit"
 587         rate_list_with_bandwidth:
 588             description: >-
 589                 Reusable composite type, joining primary rates
 590                 with optional derived bandwidths (and stats).
 591                 No test types currently computes the bandwidth part.
 592             type: object
 593             additionalProperties: false
 594             properties:
 595                 rate:
 596                     $ref: "#/$defs/types/rate_list_without_bandwidth"
 597                 bandwidth:
 598                     $ref: "#/$defs/types/bandwidth_list"
 599             required:
 600             - rate
 601     macros:
 602         lower_and_upper_rate:
 603             type: object
 604             additionalProperties: false
 605             properties:
 606                 lower:
 607                     description: >-
 608                         The lower bound (or min_rate) for the estimate
 609                         of a particular searched value.
 610                     $ref: "#/$defs/types/rate_with_bandwidth"
 611                 upper:
 612                     description: >-
 613                         The upper bound (or max_rate) for the estimate
 614                         of a particular searched value.
 615                     $ref: "#/$defs/types/rate_with_bandwidth"
 616             required:
 617             -   lower
 618             -   upper
 619         latency_numbers:
 620             type: object
 621             additionalProperties: false
 622             properties:
 623                 min:
 624                     description: >-
 625                         Rounded minimal latency time measured in this trial.
 626                         See unit property for the unit of measurement.
 627                     $ref: "#/$defs/types/nonnegative_integer"
 628                 max:
 629                     description: >-
 630                         Rounded maximal latency time measured in this trial.
 631                         See unit property for the unit of measurement.
 632                         Zero value is not allowed, as that is one of symptoms
 633                         of Traffic Generator failing to get proper latency.
 634                     $ref: "#/$defs/types/positive_integer"
 635                 avg:
 636                     description: >-
 637                         Rounded average latency time measured in this trial.
 638                         See unit property for the unit of measurement.
 639                     $ref: "#/$defs/types/nonnegative_integer"
 640                 hdrh:
 641                     description: >-
 642                         Base64-encoded compressed representation of HDRHistogram
 643                         of all latency sample times encountered
 644                         in this latency trial.
 645                         See unit property for the unit of measurement.
 646                         Note that some bins can be several units wide.
 647                     type: string
 648                 unit:
 649                     description: >-
 650                         Unit of measurement for latency times.
 651                         Currently a constant, but later versions
 652                         of the model may allow more values.
 653                     type: string
 654                     enum:
 655                     -   us
 656             required:
 657             -   avg
 658             -   hdrh
 659             -   max
 660             -   min
 661             -   unit
 662         latency_for_loads:
 663             type: object
 664             additionalProperties: false
 665             properties:
 666                 pdr_0:
 667                     description: >-
 668                         Object related to latency measurement performed
 669                         at minimal rate (currently 9000 pps per direction).
 670                     $ref: "#/$defs/macros/latency_numbers"
 671                 pdr_10:
 672                     description: >-
 673                         Object related to latency measurement performed
 674                         at 10% of PDR lower bound, if needed rounded up
 675                         to minimal rate (currently 9000 pps per direction).
 676                     $ref: "#/$defs/macros/latency_numbers"
 677                 pdr_50:
 678                     description: >-
 679                         Object related to latency measurement performed
 680                         at 50% of PDR lower bound, if needed rounded up
 681                         to minimal rate (currently 9000 pps per direction).
 682                     $ref: "#/$defs/macros/latency_numbers"
 683                 pdr_90:
 684                     description: >-
 685                         Object related to latency measurement performed
 686                         at 90% of PDR lower bound, if needed rounded up
 687                         to minimal rate (currently 9000 pps per direction).
 688                     $ref: "#/$defs/macros/latency_numbers"
 689             required:
 690             -   pdr_0
 691             -   pdr_10
 692             -   pdr_50
 693             -   pdr_90