docs/model/current/schema/test_case.info.schema.yaml

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 ---
  15
  16 $id: https://fd.io/FIXME/CSIT/UTI/test_case/info/1.0.1
  17 $schema: https://json-schema.org/draft/2020-12/schema
  18 description: >-
  19     Schema for info output of test case.
  20 allOf:
  21 -   description: >-
  22         The main structure, without conditional relations between fields yet.
  23     type: object
  24     additionalProperties: false
  25     properties:
  26         duration:
  27             description: >-
  28                 A derived quantity, present only in info output files.
  29                 Difference between start_time and end_time, in seconds.
  30             $ref: "#/$defs/types/nonnegative_number"
  31         dut_type:
  32             description: >-
  33                 DUT type used, e.g. VPP or DPDK.
  34             type: string
  35             minLength: 1
  36         dut_version:
  37             description: >-
  38                 Version string appropriate to DUT type used.
  39             type: string
  40             minLength: 1
  41         end_time:
  42             description: >-
  43                 UTC date and time in RFC 3339 format, specifying calendar time
  44                 just before test case ended (at the end of test case teardown).
  45             $ref: "#/$defs/types/date_time"
  46         hosts:
  47             description: >-
  48                 Array of hosts this test interacted with.
  49                 This can be used for identifying testbed number.
  50                 Valid tests shoud interact with at least one DUT or TG.
  51                 The array is usually sorted, but that is not a requirement.
  52             type: array
  53             minItems: 1
  54             items:
  55                 description: >-
  56                     Host identifier, usually numeric IPv4 address.
  57                 type: string
  58         log:
  59             description: >-
  60                 No log items are implemented in the current version,
  61                 but the (empty) list is present to simplify logic
  62                 in multi-version importers.
  63             $ref: "#/$defs/types/empty_array"
  64         message:
  65             description: >-
  66                 If passed is true, this value is empty.
  67                 Otherwise, value taken directly from TEST_MESSAGE
  68                 Robot variable, read at the end of test case
  69                 (in test teardown, before export and validation).
  70                 It contains information from the exception
  71                 that caused the failure, probably with additional
  72                 exceptions from teardown keywords.
  73             type: string
  74         passed:
  75             description: >-
  76                 Value set accordingly to TEST_STATUS Robot variable,
  77                 true if and only if the status is "PASS".
  78                 The status is read at the end of test case
  79                 (in test teardown, before export and validation).
  80             type: boolean
  81         result:
  82             type: object
  83             allOf:
  84             -   description: >-
  85                     Sub-schema common for all cases,
  86                     only result type identifier defined here.
  87                 properties:
  88                     type:
  89                         description: >-
  90                             Identifier of which result type case is applied.
  91                         type: string
  92                 required:
  93                 -   type
  94             -   oneOf:
  95                 -   description: >-
  96                         Result type for unknown case.
  97                         This case represents a test with no specific result
  98                         (outside message), e.g. device test;
  99                         or a test with result not parsed into
 100                         this version of model yet, e.g. GSO test.
 101                     additionalProperties: false
 102                     properties:
 103                         type:
 104                             const: unknown
 105                 -   description: >-
 106                         Result type MRR case.
 107                     additionalProperties: false
 108                     properties:
 109                         type:
 110                             const: mrr
 111                         receive_rate:
 112                             description: >-
 113                                 The results refer to receive rates for multiple
 114                                 MRR trials. For PPS, these are aggregate
 115                                 (bidirectional) rates.
 116                                 Currently, the tests are exporting
 117                                 approximated receive rates.
 118                                 That means the actual trial duration
 119                                 is measured (as opposed to trusting traffic
 120                                 generator to honor its target duration),
 121                                 so the resulting values contain noise
 122                                 from time measurement, and can be lower
 123                                 than the real performance
 124                                 (due to various time overheads).
 125                                 Bandwidth values are supported, but currently
 126                                 Robot does not export them.
 127                             $ref: "#/$defs/types/rate_list_with_bandwidth"
 128                     required:
 129                     -   type
 130                     -   receive_rate
 131                 -   description: >-
 132                         Result type NDRPDR case.
 133                     additionalProperties: false
 134                     properties:
 135                         type:
 136                             const: ndrpdr
 137                         ndr:
 138                             description: >-
 139                                 The results refer to search for NDR
 140                                 (Non Drop Rate). For PPS, this is aggregate
 141                                 (bidirectional) rate.
 142                                 Each bound was used as the target load value
 143                                 in a full-duration trial measurement.
 144                                 The accepted loss ratio for NDR is exact zero.
 145                                 Note that packets the Traffic Generator
 146                                 did not send are also counted as lost packets.
 147                             $ref: "#/$defs/macros/lower_and_upper_rate"
 148                         pdr:
 149                             description: >-
 150                                 The results refer to search for PDR
 151                                 (Partial Drop Rate). For PPS, this is aggregate
 152                                 (bidirectional) rate.
 153                                 Each bound was used as the target load value
 154                                 in a full-duration trial measurement.
 155                                 The accepted loss ratio for PDR is 0.5%.
 156                                 Note that packets the Traffic Generator
 157                                 did not send are also counted as lost packets.
 158                             $ref: "#/$defs/macros/lower_and_upper_rate"
 159                         latency_forward:
 160                             description: >-
 161                                 Object with results related to latency part
 162                                 of NDRPDR test, for forward traffic direction.
 163                                 It is the direction used in unidirectional
 164                                 traffic profiles.
 165                                 ASTF profiles and IMIX STL profiles
 166                                 do not support latency information,
 167                                 so for those tests this object is missing.
 168                                 It is also missing if Traffic Generator
 169                                 fails to return valid latency results
 170                                 for any other reasons,
 171                                 e.g. latency rate is too high for CPU/NIC used.
 172                             $ref: "#/$defs/macros/latency_for_loads"
 173                         latency_reverse:
 174                             description: >-
 175                                 Object with results related to latency part
 176                                 of NDRPDR test, for reverse traffic diration.
 177                                 This object is not present
 178                                 when unidirectional traffic profiles are used.
 179                                 ASTF profiles and IMIX STL profiles
 180                                 do not support latency information,
 181                                 so for those tests this object is missing.
 182                                 It is also missing if Traffic Generator
 183                                 fails to return valid latency results
 184                                 for any other reasons,
 185                                 e.g. latency rate is too high for CPU/NIC used.
 186                             $ref: "#/$defs/macros/latency_for_loads"
 187                     required:
 188                     -   type
 189                     -   ndr
 190                     -   pdr
 191                 -   description: >-
 192                         Result type SOAK case.
 193                     additionalProperties: false
 194                     properties:
 195                         type:
 196                             const: soak
 197                         critical_rate:
 198                             description: >-
 199                                 The results refer to bayesian estimate
 200                                 of critical rate corresponding to
 201                                 average loss ratio of 10^-7.
 202                                 For PPS, this is aggregate (bidirectional) rate.
 203                                 The bounds are computed from
 204                                 trial measurement results,
 205                                 but are not equal to any target load used.
 206                                 Note that packets the Traffic Generator
 207                                 did not send are also counted as lost packets.
 208                             $ref: "#/$defs/macros/lower_and_upper_rate"
 209                     required:
 210                     -   type
 211                     -   critical_rate
 212         start_time:
 213             description: >-
 214                 UTC date and time in RFC 3339 format, specifying calendar time
 215                 just after test case started (at the start of test setup).
 216             $ref: "#/$defs/types/date_time"
 217         tags:
 218             description: >-
 219                 The list of strings comes directly
 220                 from Robot variable TEST_TAGS.
 221                 The content should include both static and dynamic tags
 222                 at the end of test case (teardown).
 223             type: array
 224             items:
 225                 type: string
 226         test_documentation:
 227             description: >-
 228                 Value taken directly from TEST_DOCUMENTATION Robot variable.
 229                 The content is what you see in suite file
 230                 at test case definition, which is usually empty
 231                 as CSIT uses data driven test cases.
 232             type: string
 233         test_id:
 234             description: >-
 235                 A derived quantity, present only in info output files.
 236                 It is the most complete and unique identifier for a test case.
 237                 This property has a value, of the following form:
 238                 {suite_name}.{test_name}
 239                 Here, suite name comes from SUITE_NAME robot variable,
 240                 test name comes from TEST_NAME robot variable,
 241                 but both are converted to lower case,
 242                 and spaces are replaced by underscores.
 243             type: string
 244             minLength: 3
 245         test_name_long:
 246             description: >-
 247                 A derived quantity, present only in info output files.
 248                 This property has a value, of the following form:
 249                 {nic_short_name}-{frame_size}-{threads_and_cores}-{suite_part}
 250                 Here, suite part is very similar to suite tag,
 251                 but additionally may contain a prefix describing NIC driver used
 252                 (if it is not the default one, drv_vfio_pci for VPP tests).
 253                 Any space is replaced by underscore and letters are lower case.
 254             type: string
 255             minLength: 3
 256         test_name_short:
 257             description: >-
 258                 A derived quantity, present only in info output files.
 259                 This property has a value very similar to suite tag,
 260                 but additionally may contain a prefix describing NIC driver used
 261                 (if it is not the default one, drv_vfio_pci for VPP tests).
 262                 Any space is replaced by underscore and letters are lower case.
 263             type: string
 264             minLength: 3
 265         test_type:
 266             description: >-
 267                 A derived quantity, present only in info output files.
 268                 Test type identifier, PAL uses it to group similar tests,
 269                 e.g. for comparison tables.
 270                 Ideally, this information should be parseable from test name,
 271                 but the current naming scheme is not simple/consistent enough.
 272                 The current implementation queries the robot test tags.
 273                 The resulting value is frequently identical to result type,
 274                 but this schema version does not require any relation there,
 275                 as PAL may want to group tests differently.
 276             type: string
 277             enum:
 278             -   device
 279             -   gso
 280             -   hoststack
 281             -   mrr
 282             -   ndrpdr
 283             -   reconf
 284             -   soak
 285             -   vsap
 286         tg_type:
 287             description: >-
 288                 TG type used, e.g. TREX.
 289             type: string
 290             minLength: 1
 291         tg_version:
 292             description: >-
 293                 Version string appropriate to TG type used.
 294             type: string
 295             minLength: 1
 296         version:
 297             description: >-
 298                 CSIT model version (semver format)
 299                 the exporting code adhered to.
 300             type: string
 301             const: 1.0.1
 302     required:
 303     -   duration
 304     -   dut_type
 305     -   dut_version
 306     -   end_time
 307     -   hosts
 308     -   log
 309     -   message
 310     -   passed
 311     -   result
 312     -   start_time
 313     -   tags
 314     -   test_documentation
 315     -   test_id
 316     -   test_name_long
 317     -   test_name_short
 318     -   test_type
 319     -   tg_type
 320     -   tg_version
 321     -   version
 322 -   description: >-
 323         Subschema validating relation between status and message.
 324     oneOf:
 325     -   description: >-
 326             Subschema for passing tests, message has to be empty.
 327         type: object
 328         properties:
 329             passed:
 330                 const: true
 331             message:
 332                 const: ""
 333     -   description: >-
 334             Subschema for failing tests, mesage cannot be empty.
 335         type: object
 336         properties:
 337             passed:
 338                 const: false
 339             message:
 340                 minLength: 1
 341
 342 $defs:
 343     types:
 344         nonnegative_number:
 345             type: number
 346             minimum: 0
 347         positive_number:
 348             type: number
 349             minimum: 1
 350         nonnegative_integer:
 351             type: integer
 352             minimum: 0
 353         positive_integer:
 354             type: integer
 355             minimum: 1
 356         date_time:
 357             type: string
 358             format: date-time
 359         empty_array:
 360             type: array
 361             maxItems: 0
 362         rate_unit:
 363             description: >-
 364                 Packets per second (pps) or connections per second (cps).
 365             type: string
 366             enum:
 367             -   pps
 368             -   cps
 369         bandwidth_unit:
 370             description: >-
 371                 Unit of measurement for bandwidth values.
 372                 Currently a constant, but later versions of model
 373                 may allow more units.
 374             enum:
 375             -   bps
 376         value_with_unit:
 377             description: >-
 378                 Reusable composite type, value together with its
 379                 unit of measurement.
 380             type: object
 381             additionalProperties: false
 382             properties:
 383                 value:
 384                     description: >-
 385                         Numeric value, context specified elsewhere.
 386                         The only assumption is that value is not negative.
 387                     $ref: "#/$defs/types/nonnegative_number"
 388                 unit:
 389                     description: >-
 390                         Unit of measurement for the value.
 391                         Context and allowed values are specified elsewhere.
 392                     type: string
 393             required:
 394             -   value
 395             -   unit
 396         rate_without_bandwidth:
 397             description: >-
 398                 Reusable type, for various rate quantites.
 399             allOf:
 400             -   $ref: "#/$defs/types/value_with_unit"
 401             -   properties:
 402                     value:
 403                         description: >-
 404                             Unless specified otherwise,
 405                             this is the aggregated rate
 406                             (sum of both traffic directions).
 407                             Depending on the usage, the value can express
 408                             intended load, offered load, receive rate,
 409                             and various approximations
 410                             or estimated bounds thereof.
 411                     unit:
 412                         description: >-
 413                             A transaction rate unit the value is expressed in.
 414                         $ref: "#/$defs/types/rate_unit"
 415         bandwidth:
 416             description: >-
 417                 Reusable type, for various bandwidth quantites.
 418             allOf:
 419             -   $ref: "#/$defs/types/value_with_unit"
 420             -   properties:
 421                     value:
 422                         description: >-
 423                             Bandwidth value computed
 424                             from the corresponding rate.
 425                     unit:
 426                         $ref: "#/$defs/types/bandwidth_unit"
 427         rate_with_bandwidth:
 428             description: >-
 429                 Reusable composite type, joining primary rate
 430                 with optional derived bandwidth.
 431                 Not all test types currently compute bandwidth,
 432                 even if rate unit is pps.
 433             type: object
 434             additionalProperties: false
 435             properties:
 436                 rate:
 437                     $ref: "#/$defs/types/rate_without_bandwidth"
 438                 bandwidth:
 439                     $ref: "#/$defs/types/bandwidth"
 440             required:
 441             - rate
 442         value_list_with_unit_and_stats:
 443             description: >-
 444                 Reusable composite type, multiple values together with their
 445                 unit of measurement and derived statistics.
 446             type: object
 447             additionalProperties: false
 448             properties:
 449                 values:
 450                     description: >-
 451                         List of values of the same unit, useful for MRR.
 452                     type: array
 453                     minItmes: 1
 454                     items:
 455                         description: >-
 456                             Numeric value, context specified elsewhere.
 457                             The only assumption is that the value is nonnegative.
 458                         $ref: "#/$defs/types/nonnegative_number"
 459                 avg:
 460                     description: >-
 461                         A derived quantity, present only in info output files.
 462                         It is the arithmetic average of the values list.
 463                     $ref: "#/$defs/types/nonnegative_number"
 464                 stdev:
 465                     description: >-
 466                         A derived quantity, present only in info output files.
 467                         It is the standard deviation for the values list,
 468                         as computed by jumpavg library.
 469                     $ref: "#/$defs/types/nonnegative_number"
 470                 unit:
 471                     description: >-
 472                         Unit of measurement for the values.
 473                         Context and allowed values are specified elsewhere.
 474                     type: string
 475             required:
 476             -   values
 477             -   avg
 478             -   stdev
 479             -   unit
 480         rate_list_without_bandwidth:
 481             description: >-
 482                 Reusable composite type, multiple rate values.
 483             allOf:
 484             -   $ref: "#/$defs/types/value_list_with_unit_and_stats"
 485             -   properties:
 486                     values:
 487                         items:
 488                             description: >-
 489                                 Unless specified otherwise,
 490                                 this is the aggregated rate
 491                                 (sum of both traffic directions).
 492                                 Depending on the usage, the value can express
 493                                 intended load, offered load, receive rate,
 494                                 and various approximations or estimated bounds
 495                                 thereof.
 496                     unit:
 497                         $ref: "#/$defs/types/rate_unit"
 498         bandwidth_list:
 499             description: >-
 500                 Reusable composite type, multiple bandwidth values.
 501                 This is a derived entity, thus it only appears in info output,
 502                 and only if rate unit is pps.
 503             allOf:
 504             -   $ref: "#/$defs/types/value_list_with_unit_and_stats"
 505             -   properties:
 506                     values:
 507                         items:
 508                             description: >-
 509                                 Unless specified otherwise,
 510                                 this is the aggregated bandwidth
 511                                 (sum of both traffic directions).
 512                                 Depending on the usage, the value can express
 513                                 intended load, offered load, receive rate,
 514                                 and various approximations or estimated bounds
 515                                 thereof.
 516                     unit:
 517                         $ref: "#/$defs/types/bandwidth_unit"
 518         rate_list_with_bandwidth:
 519             description: >-
 520                 Reusable composite type, joining primary rates
 521                 with optional derived bandwidths (and stats).
 522                 No test types currently computes the bandwidth part.
 523             type: object
 524             additionalProperties: false
 525             properties:
 526                 rate:
 527                     $ref: "#/$defs/types/rate_list_without_bandwidth"
 528                 bandwidth:
 529                     $ref: "#/$defs/types/bandwidth_list"
 530             required:
 531             - rate
 532     macros:
 533         lower_and_upper_rate:
 534             type: object
 535             additionalProperties: false
 536             properties:
 537                 lower:
 538                     description: >-
 539                         The lower bound (or min_rate) for the estimate
 540                         of a particular searched value.
 541                     $ref: "#/$defs/types/rate_with_bandwidth"
 542                 upper:
 543                     description: >-
 544                         The upper bound (or max_rate) for the estimate
 545                         of a particular searched value.
 546                     $ref: "#/$defs/types/rate_with_bandwidth"
 547             required:
 548             -   lower
 549             -   upper
 550         latency_numbers:
 551             type: object
 552             additionalProperties: false
 553             properties:
 554                 min:
 555                     description: >-
 556                         Rounded minimal latency time measured in this trial.
 557                         See unit property for the unit of measurement.
 558                     $ref: "#/$defs/types/nonnegative_integer"
 559                 max:
 560                     description: >-
 561                         Rounded maximal latency time measured in this trial.
 562                         See unit property for the unit of measurement.
 563                         Zero value is not allowed, as that is one of symptoms
 564                         of Traffic Generator failing to get proper latency.
 565                     $ref: "#/$defs/types/positive_integer"
 566                 avg:
 567                     description: >-
 568                         Rounded average latency time measured in this trial.
 569                         See unit property for the unit of measurement.
 570                     $ref: "#/$defs/types/nonnegative_integer"
 571                 hdrh:
 572                     description: >-
 573                         Base64-encoded compressed representation of HDRHistogram
 574                         of all latency sample times encountered
 575                         in this latency trial.
 576                         See unit property for the unit of measurement.
 577                         Note that some bins can be several units wide.
 578                     type: string
 579                 unit:
 580                     description: >-
 581                         Unit of measurement for latency times.
 582                         Currently a constant, but later versions
 583                         of the model may allow more values.
 584                     type: string
 585                     enum:
 586                     -   us
 587             required:
 588             -   avg
 589             -   hdrh
 590             -   max
 591             -   min
 592             -   unit
 593         latency_for_loads:
 594             type: object
 595             additionalProperties: false
 596             properties:
 597                 pdr_0:
 598                     description: >-
 599                         Object related to latency measurement performed
 600                         at minimal rate (currently 9000 pps per direction).
 601                     $ref: "#/$defs/macros/latency_numbers"
 602                 pdr_10:
 603                     description: >-
 604                         Object related to latency measurement performed
 605                         at 10% of PDR lower bound, if needed rounded up
 606                         to minimal rate (currently 9000 pps per direction).
 607                     $ref: "#/$defs/macros/latency_numbers"
 608                 pdr_50:
 609                     description: >-
 610                         Object related to latency measurement performed
 611                         at 50% of PDR lower bound, if needed rounded up
 612                         to minimal rate (currently 9000 pps per direction).
 613                     $ref: "#/$defs/macros/latency_numbers"
 614                 pdr_90:
 615                     description: >-
 616                         Object related to latency measurement performed
 617                         at 90% of PDR lower bound, if needed rounded up
 618                         to minimal rate (currently 9000 pps per direction).
 619                     $ref: "#/$defs/macros/latency_numbers"
 620             required:
 621             -   pdr_0
 622             -   pdr_10
 623             -   pdr_50
 624             -   pdr_90