docs/model/current/schema/test_case.info.schema.yaml

   1 # Copyright (c) 2021 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 ---
  15
  16 $id: https://fd.io/FIXME/CSIT/UTI/test_case/info/1.0.0
  17 $schema: https://json-schema.org/draft/2020-12/schema
  18 description: >-
  19     Schema for info output of test case.
  20 allOf:
  21 -   description: >-
  22         The main structure, without conditional relations between fields yet.
  23     type: object
  24     additionalProperties: false
  25     properties:
  26         duration:
  27             description: >-
  28                 A derived quantity, present only in info output files.
  29                 Difference between start_time and end_time, in seconds.
  30             $ref: "#/$defs/types/nonnegative_number"
  31         dut_type:
  32             description: >-
  33                 DUT type used, e.g. VPP or DPDK.
  34             type: string
  35         dut_version:
  36             description: >-
  37                 Version string appropriate to DUT type used.
  38             type: string
  39         end_time:
  40             description: >-
  41                 UTC date and time in RFC 3339 format, specifying calendar time
  42                 just before test case ended (at the end of test case teardown).
  43             $ref: "#/$defs/types/date_time"
  44         hosts:
  45             description: >-
  46                 Array of hosts this test interacted with.
  47                 This can be used for identifying testbed number.
  48                 Valid tests shoud interact with at least one DUT or TG.
  49                 The array is usually sorted, but that is not a requirement.
  50             type: array
  51             minItems: 1
  52             items:
  53                 description: >-
  54                     Host identifier, usually numeric IPv4 address.
  55                 type: string
  56         log:
  57             description: >-
  58                 No log items are implemented in the current version,
  59                 but the (empty) list is present to simplify logic
  60                 in multi-version importers.
  61             $ref: "#/$defs/types/empty_array"
  62         message:
  63             description: >-
  64                 If passed is true, this value is empty.
  65                 Otherwise, value taken directly from TEST_MESSAGE
  66                 Robot variable, read at the end of test case
  67                 (in test teardown, before export and validation).
  68                 It contains information from the exception
  69                 that caused the failure, probably with additional
  70                 exceptions from teardown keywords.
  71             type: string
  72         passed:
  73             description: >-
  74                 Value set accordingly to TEST_STATUS Robot variable,
  75                 true if and only if the status is "PASS".
  76                 The status is read at the end of test case
  77                 (in test teardown, before export and validation).
  78             type: boolean
  79         result:
  80             type: object
  81             allOf:
  82             -   description: >-
  83                     Sub-schema common for all cases,
  84                     only result type identifier defined here.
  85                 properties:
  86                     type:
  87                         description: >-
  88                             Identifier of which result type case is applied.
  89                         type: string
  90                 required:
  91                 -   type
  92             -   oneOf:
  93                 -   description: >-
  94                         Result type for unknown case.
  95                         This case represents a test with no specific result
  96                         (outside message), e.g. device test;
  97                         or a test with result not parsed into
  98                         this version of model yet, e.g. GSO test.
  99                     additionalProperties: false
 100                     properties:
 101                         type:
 102                             const: unknown
 103                 -   description: >-
 104                         Result type MRR case.
 105                     additionalProperties: false
 106                     properties:
 107                         type:
 108                             const: mrr
 109                         receive_rate:
 110                             description: >-
 111                                 The results refer to receive rates for multiple
 112                                 MRR trials. For PPS, these are aggregate
 113                                 (bidirectional) rates.
 114                                 Currently, the tests are exporting
 115                                 approximated receive rates.
 116                                 That means the actual trial duration
 117                                 is measured (as opposed to trusting traffic
 118                                 generator to honor its target duration),
 119                                 so the resulting values contain noise
 120                                 from time measurement, and can be lower
 121                                 than the real performance
 122                                 (due to various time overheads).
 123                                 Bandwidth values are supported, but currently
 124                                 Robot does not export them.
 125                             $ref: "#/$defs/types/rate_list_with_bandwidth"
 126                     required:
 127                     -   type
 128                     -   receive_rate
 129                 -   description: >-
 130                         Result type NDRPDR case.
 131                     additionalProperties: false
 132                     properties:
 133                         type:
 134                             const: ndrpdr
 135                         ndr:
 136                             description: >-
 137                                 The results refer to search for NDR
 138                                 (Non Drop Rate). For PPS, this is aggregate
 139                                 (bidirectional) rate.
 140                                 Each bound was used as the target load value
 141                                 in a full-duration trial measurement.
 142                                 The accepted loss ratio for NDR is exact zero.
 143                                 Note that packets the Traffic Generator
 144                                 did not send are also counted as lost packets.
 145                             $ref: "#/$defs/macros/lower_and_upper_rate"
 146                         pdr:
 147                             description: >-
 148                                 The results refer to search for PDR
 149                                 (Partial Drop Rate). For PPS, this is aggregate
 150                                 (bidirectional) rate.
 151                                 Each bound was used as the target load value
 152                                 in a full-duration trial measurement.
 153                                 The accepted loss ratio for PDR is 0.5%.
 154                                 Note that packets the Traffic Generator
 155                                 did not send are also counted as lost packets.
 156                             $ref: "#/$defs/macros/lower_and_upper_rate"
 157                         latency_forward:
 158                             description: >-
 159                                 Object with results related to latency part
 160                                 of NDRPDR test, for forward traffic direction.
 161                                 It is the direction used in unidirectional
 162                                 traffic profiles.
 163                                 ASTF profiles and IMIX STL profiles
 164                                 do not support latency information,
 165                                 so for those tests this object is missing.
 166                                 It is also missing if Traffic Generator
 167                                 fails to return valid latency results
 168                                 for any other reasons,
 169                                 e.g. latency rate is too high for CPU/NIC used.
 170                             $ref: "#/$defs/macros/latency_for_loads"
 171                         latency_reverse:
 172                             description: >-
 173                                 Object with results related to latency part
 174                                 of NDRPDR test, for reverse traffic diration.
 175                                 This object is not present
 176                                 when unidirectional traffic profiles are used.
 177                                 ASTF profiles and IMIX STL profiles
 178                                 do not support latency information,
 179                                 so for those tests this object is missing.
 180                                 It is also missing if Traffic Generator
 181                                 fails to return valid latency results
 182                                 for any other reasons,
 183                                 e.g. latency rate is too high for CPU/NIC used.
 184                             $ref: "#/$defs/macros/latency_for_loads"
 185                     required:
 186                     -   type
 187                     -   ndr
 188                     -   pdr
 189                 -   description: >-
 190                         Result type SOAK case.
 191                     additionalProperties: false
 192                     properties:
 193                         type:
 194                             const: soak
 195                         critical_rate:
 196                             description: >-
 197                                 The results refer to bayesian estimate
 198                                 of critical rate corresponding to
 199                                 average loss ratio of 10^-7.
 200                                 For PPS, this is aggregate (bidirectional) rate.
 201                                 The bounds are computed from
 202                                 trial measurement results,
 203                                 but are not equal to any target load used.
 204                                 Note that packets the Traffic Generator
 205                                 did not send are also counted as lost packets.
 206                             $ref: "#/$defs/macros/lower_and_upper_rate"
 207                     required:
 208                     -   type
 209                     -   critical_rate
 210         start_time:
 211             description: >-
 212                 UTC date and time in RFC 3339 format, specifying calendar time
 213                 just after test case started (at the start of test setup).
 214             $ref: "#/$defs/types/date_time"
 215         tags:
 216             description: >-
 217                 The list of strings comes directly
 218                 from Robot variable TEST_TAGS.
 219                 The content should include both static and dynamic tags
 220                 at the end of test case (teardown).
 221             type: array
 222             items:
 223                 type: string
 224         test_documentation:
 225             description: >-
 226                 Value taken directly from TEST_DOCUMENTATION Robot variable.
 227                 The content is what you see in suite file
 228                 at test case definition, which is usually empty
 229                 as CSIT uses data driven test cases.
 230             type: string
 231         test_id:
 232             description: >-
 233                 A derived quantity, present only in info output files.
 234                 It is the most complete and unique identifier for a test case.
 235                 This property has a value, of the following form:
 236                 {suite_name}.{test_name}
 237                 Here, suite name comes from SUITE_NAME robot variable,
 238                 test name comes from TEST_NAME robot variable,
 239                 but both are converted to lower case,
 240                 and spaces are replaced by underscores.
 241             type: string
 242             minLength: 3
 243         test_name_long:
 244             description: >-
 245                 A derived quantity, present only in info output files.
 246                 This property has a value, of the following form:
 247                 {nic_short_name}-{frame_size}-{threads_and_cores}-{suite_part}
 248                 Here, suite part is very similar to suite tag,
 249                 but additionally may contain a prefix describing NIC driver used
 250                 (if it is not the default one, drv_vfio_pci for VPP tests).
 251                 Any space is replaced by underscore and letters are lower case.
 252             type: string
 253             minLength: 3
 254         test_name_short:
 255             description: >-
 256                 A derived quantity, present only in info output files.
 257                 This property has a value very similar to suite tag,
 258                 but additionally may contain a prefix describing NIC driver used
 259                 (if it is not the default one, drv_vfio_pci for VPP tests).
 260                 Any space is replaced by underscore and letters are lower case.
 261             type: string
 262             minLength: 3
 263         test_type:
 264             description: >-
 265                 A derived quantity, present only in info output files.
 266                 Test type identifier, PAL uses it to group similar tests,
 267                 e.g. for comparison tables.
 268                 Ideally, this information should be parseable from test name,
 269                 but the current naming scheme is not simple/consistent enough.
 270                 The current implementation queries the robot test tags.
 271                 The resulting value is frequently identical to result type,
 272                 but this schema version does not require any relation there,
 273                 as PAL may want to group tests differently.
 274             type: string
 275             enum:
 276             -   device
 277             -   gso
 278             -   hoststack
 279             -   mrr
 280             -   ndrpdr
 281             -   reconf
 282             -   soak
 283             -   vsap
 284         version:
 285             description: >-
 286                 CSIT model version (semver format)
 287                 the exporting code adhered to.
 288             type: string
 289             const: 1.0.0
 290     required:
 291     -   duration
 292     -   dut_type
 293     -   dut_version
 294     -   end_time
 295     -   hosts
 296     -   log
 297     -   message
 298     -   passed
 299     -   result
 300     -   start_time
 301     -   tags
 302     -   test_documentation
 303     -   test_id
 304     -   test_name_long
 305     -   test_name_short
 306     -   test_type
 307     -   version
 308 -   description: >-
 309         Subschema validating relation between status and message.
 310     oneOf:
 311     -   description: >-
 312             Subschema for passing tests, message has to be empty.
 313         type: object
 314         properties:
 315             passed:
 316                 const: true
 317             message:
 318                 const: ""
 319     -   description: >-
 320             Subschema for failing tests, mesage cannot be empty.
 321         type: object
 322         properties:
 323             passed:
 324                 const: false
 325             message:
 326                 minLength: 1
 327 -   description: >-
 328         Subschema validating relation between dut_type and dut_version.
 329     oneOf:
 330     -   description: >-
 331             Subschema for tests with no DUT, e.g. TRex self-test.
 332         type: object
 333         properties:
 334             dut_type:
 335                 const: none
 336             dut_version:
 337                 const: ""
 338     -   description: >-
 339             Subschema for DUT type VPP.
 340         type: object
 341         properties:
 342             dut_type:
 343                 const: VPP
 344             dut_version:
 345                 minLength: 1
 346     -   description: >-
 347             Subschema for DUT type DPDK.
 348         type: object
 349         properties:
 350             dut_type:
 351                 const: DPDK
 352             dut_version:
 353                 minLength: 1
 354
 355 $defs:
 356     types:
 357         nonnegative_number:
 358             type: number
 359             minimum: 0
 360         positive_number:
 361             type: number
 362             minimum: 1
 363         nonnegative_integer:
 364             type: integer
 365             minimum: 0
 366         positive_integer:
 367             type: integer
 368             minimum: 1
 369         date_time:
 370             type: string
 371             format: date-time
 372         empty_array:
 373             type: array
 374             maxItems: 0
 375         rate_unit:
 376             description: >-
 377                 Packets per second (pps) or connections per second (cps).
 378             type: string
 379             enum:
 380             -   pps
 381             -   cps
 382         bandwidth_unit:
 383             description: >-
 384                 Unit of measurement for bandwidth values.
 385                 Currently a constant, but later versions of model
 386                 may allow more units.
 387             enum:
 388             -   bps
 389         value_with_unit:
 390             description: >-
 391                 Reusable composite type, value together with its
 392                 unit of measurement.
 393             type: object
 394             additionalProperties: false
 395             properties:
 396                 value:
 397                     description: >-
 398                         Numeric value, context specified elsewhere.
 399                         The only assumption is that value is not negative.
 400                     $ref: "#/$defs/types/nonnegative_number"
 401                 unit:
 402                     description: >-
 403                         Unit of measurement for the value.
 404                         Context and allowed values are specified elsewhere.
 405                     type: string
 406             required:
 407             -   value
 408             -   unit
 409         rate_without_bandwidth:
 410             description: >-
 411                 Reusable type, for various rate quantites.
 412             allOf:
 413             -   $ref: "#/$defs/types/value_with_unit"
 414             -   properties:
 415                     value:
 416                         description: >-
 417                             Unless specified otherwise,
 418                             this is the aggregated rate
 419                             (sum of both traffic directions).
 420                             Depending on the usage, the value can express
 421                             intended load, offered load, receive rate,
 422                             and various approximations
 423                             or estimated bounds thereof.
 424                     unit:
 425                         description: >-
 426                             A transaction rate unit the value is expressed in.
 427                         $ref: "#/$defs/types/rate_unit"
 428         bandwidth:
 429             description: >-
 430                 Reusable type, for various bandwidth quantites.
 431             allOf:
 432             -   $ref: "#/$defs/types/value_with_unit"
 433             -   properties:
 434                     value:
 435                         description: >-
 436                             Bandwidth value computed
 437                             from the corresponding rate.
 438                     unit:
 439                         $ref: "#/$defs/types/bandwidth_unit"
 440         rate_with_bandwidth:
 441             description: >-
 442                 Reusable composite type, joining primary rate
 443                 with optional derived bandwidth.
 444                 Not all test types currently compute bandwidth,
 445                 even if rate unit is pps.
 446             type: object
 447             additionalProperties: false
 448             properties:
 449                 rate:
 450                     $ref: "#/$defs/types/rate_without_bandwidth"
 451                 bandwidth:
 452                     $ref: "#/$defs/types/bandwidth"
 453             required:
 454             - rate
 455         value_list_with_unit_and_stats:
 456             description: >-
 457                 Reusable composite type, multiple values together with their
 458                 unit of measurement and derived statistics.
 459             type: object
 460             additionalProperties: false
 461             properties:
 462                 values:
 463                     description: >-
 464                         List of values of the same unit, useful for MRR.
 465                     type: array
 466                     minItmes: 1
 467                     items:
 468                         description: >-
 469                             Numeric value, context specified elsewhere.
 470                             The only assumption is that the value is nonnegative.
 471                         $ref: "#/$defs/types/nonnegative_number"
 472                 avg:
 473                     description: >-
 474                         A derived quantity, present only in info output files.
 475                         It is the arithmetic average of the values list.
 476                     $ref: "#/$defs/types/nonnegative_number"
 477                 stdev:
 478                     description: >-
 479                         A derived quantity, present only in info output files.
 480                         It is the standard deviation for the values list,
 481                         as computed by jumpavg library.
 482                     $ref: "#/$defs/types/nonnegative_number"
 483                 unit:
 484                     description: >-
 485                         Unit of measurement for the values.
 486                         Context and allowed values are specified elsewhere.
 487                     type: string
 488             required:
 489             -   values
 490             -   avg
 491             -   stdev
 492             -   unit
 493         rate_list_without_bandwidth:
 494             description: >-
 495                 Reusable composite type, multiple rate values.
 496             allOf:
 497             -   $ref: "#/$defs/types/value_list_with_unit_and_stats"
 498             -   properties:
 499                     values:
 500                         items:
 501                             description: >-
 502                                 Unless specified otherwise,
 503                                 this is the aggregated rate
 504                                 (sum of both traffic directions).
 505                                 Depending on the usage, the value can express
 506                                 intended load, offered load, receive rate,
 507                                 and various approximations or estimated bounds
 508                                 thereof.
 509                     unit:
 510                         $ref: "#/$defs/types/rate_unit"
 511         bandwidth_list:
 512             description: >-
 513                 Reusable composite type, multiple bandwidth values.
 514                 This is a derived entity, thus it only appears in info output,
 515                 and only if rate unit is pps.
 516             allOf:
 517             -   $ref: "#/$defs/types/value_list_with_unit_and_stats"
 518             -   properties:
 519                     values:
 520                         items:
 521                             description: >-
 522                                 Unless specified otherwise,
 523                                 this is the aggregated bandwidth
 524                                 (sum of both traffic directions).
 525                                 Depending on the usage, the value can express
 526                                 intended load, offered load, receive rate,
 527                                 and various approximations or estimated bounds
 528                                 thereof.
 529                     unit:
 530                         $ref: "#/$defs/types/bandwidth_unit"
 531         rate_list_with_bandwidth:
 532             description: >-
 533                 Reusable composite type, joining primary rates
 534                 with optional derived bandwidths (and stats).
 535                 No test types currently computes the bandwidth part.
 536             type: object
 537             additionalProperties: false
 538             properties:
 539                 rate:
 540                     $ref: "#/$defs/types/rate_list_without_bandwidth"
 541                 bandwidth:
 542                     $ref: "#/$defs/types/bandwidth_list"
 543             required:
 544             - rate
 545     macros:
 546         lower_and_upper_rate:
 547             type: object
 548             additionalProperties: false
 549             properties:
 550                 lower:
 551                     description: >-
 552                         The lower bound (or min_rate) for the estimate
 553                         of a particular searched value.
 554                     $ref: "#/$defs/types/rate_with_bandwidth"
 555                 upper:
 556                     description: >-
 557                         The upper bound (or max_rate) for the estimate
 558                         of a particular searched value.
 559                     $ref: "#/$defs/types/rate_with_bandwidth"
 560             required:
 561             -   lower
 562             -   upper
 563         latency_numbers:
 564             type: object
 565             additionalProperties: false
 566             properties:
 567                 min:
 568                     description: >-
 569                         Rounded minimal latency time measured in this trial.
 570                         See unit property for the unit of measurement.
 571                     $ref: "#/$defs/types/nonnegative_integer"
 572                 max:
 573                     description: >-
 574                         Rounded maximal latency time measured in this trial.
 575                         See unit property for the unit of measurement.
 576                         Zero value is not allowed, as that is one of symptoms
 577                         of Traffic Generator failing to get proper latency.
 578                     $ref: "#/$defs/types/positive_integer"
 579                 avg:
 580                     description: >-
 581                         Rounded average latency time measured in this trial.
 582                         See unit property for the unit of measurement.
 583                     $ref: "#/$defs/types/nonnegative_integer"
 584                 hdrh:
 585                     description: >-
 586                         Base64-encoded compressed representation of HDRHistogram
 587                         of all latency sample times encountered
 588                         in this latency trial.
 589                         See unit property for the unit of measurement.
 590                         Note that some bins can be several units wide.
 591                     type: string
 592                 unit:
 593                     description: >-
 594                         Unit of measurement for latency times.
 595                         Currently a constant, but later versions
 596                         of the model may allow more values.
 597                     type: string
 598                     enum:
 599                     -   us
 600             required:
 601             -   avg
 602             -   hdrh
 603             -   max
 604             -   min
 605             -   unit
 606         latency_for_loads:
 607             type: object
 608             additionalProperties: false
 609             properties:
 610                 pdr_0:
 611                     description: >-
 612                         Object related to latency measurement performed
 613                         at minimal rate (currently 9000 pps per direction).
 614                     $ref: "#/$defs/macros/latency_numbers"
 615                 pdr_10:
 616                     description: >-
 617                         Object related to latency measurement performed
 618                         at 10% of PDR lower bound, if needed rounded up
 619                         to minimal rate (currently 9000 pps per direction).
 620                     $ref: "#/$defs/macros/latency_numbers"
 621                 pdr_50:
 622                     description: >-
 623                         Object related to latency measurement performed
 624                         at 50% of PDR lower bound, if needed rounded up
 625                         to minimal rate (currently 9000 pps per direction).
 626                     $ref: "#/$defs/macros/latency_numbers"
 627                 pdr_90:
 628                     description: >-
 629                         Object related to latency measurement performed
 630                         at 90% of PDR lower bound, if needed rounded up
 631                         to minimal rate (currently 9000 pps per direction).
 632                     $ref: "#/$defs/macros/latency_numbers"
 633             required:
 634             -   pdr_0
 635             -   pdr_10
 636             -   pdr_50
 637             -   pdr_90