docs/model/current/schema/test_case.info.schema.yaml

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 ---
  15
  16 $id: https://fd.io/FIXME/CSIT/UTI/test_case/info/1.1.0
  17 $schema: https://json-schema.org/draft/2020-12/schema
  18 description: >-
  19     Schema for info output of test case.
  20 allOf:
  21 -   description: >-
  22         The main structure, without conditional relations between fields yet.
  23     type: object
  24     additionalProperties: false
  25     properties:
  26         duration:
  27             description: >-
  28                 A derived quantity, present only in info output files.
  29                 Difference between start_time and end_time, in seconds.
  30             $ref: "#/$defs/types/nonnegative_number"
  31         dut_type:
  32             description: >-
  33                 DUT type used, e.g. VPP or DPDK.
  34             type: string
  35             minLength: 1
  36         dut_version:
  37             description: >-
  38                 Version string appropriate to DUT type used.
  39             type: string
  40             minLength: 1
  41         end_time:
  42             description: >-
  43                 UTC date and time in RFC 3339 format, specifying calendar time
  44                 just before test case ended (at the end of test case teardown).
  45             $ref: "#/$defs/types/date_time"
  46         hosts:
  47             description: >-
  48                 Array of hosts this test interacted with.
  49                 This can be used for identifying testbed number.
  50                 Valid tests shoud interact with at least one DUT or TG.
  51                 The array is usually sorted, but that is not a requirement.
  52             type: array
  53             minItems: 1
  54             items:
  55                 description: >-
  56                     Host identifier, usually numeric IPv4 address.
  57                 type: string
  58         telemetry:
  59             description: >-
  60                 Array of telemetry entries. Each entry represent one captured
  61                 metric.
  62             type: array
  63             minItems: 0
  64             items:
  65                 description: >-
  66                     Telemetry entry.
  67                 type: string
  68         message:
  69             description: >-
  70                 If passed is true, this value is empty.
  71                 Otherwise, value taken directly from TEST_MESSAGE
  72                 Robot variable, read at the end of test case
  73                 (in test teardown, before export and validation).
  74                 It contains information from the exception
  75                 that caused the failure, probably with additional
  76                 exceptions from teardown keywords.
  77             type: string
  78         passed:
  79             description: >-
  80                 Value set accordingly to TEST_STATUS Robot variable,
  81                 true if and only if the status is "PASS".
  82                 The status is read at the end of test case
  83                 (in test teardown, before export and validation).
  84             type: boolean
  85         result:
  86             type: object
  87             allOf:
  88             -   description: >-
  89                     Sub-schema common for all cases,
  90                     only result type identifier defined here.
  91                 properties:
  92                     type:
  93                         description: >-
  94                             Identifier of which result type case is applied.
  95                         type: string
  96                 required:
  97                 -   type
  98             -   oneOf:
  99                 -   description: >-
 100                         Result type for unknown case.
 101                         This case represents a test with no specific result
 102                         (outside message), e.g. device test;
 103                         or a test with result not parsed into
 104                         this version of model yet, e.g. GSO test.
 105                     additionalProperties: false
 106                     properties:
 107                         type:
 108                             const: unknown
 109                 -   description: >-
 110                         Result type MRR case.
 111                     additionalProperties: false
 112                     properties:
 113                         type:
 114                             const: mrr
 115                         receive_rate:
 116                             description: >-
 117                                 The results refer to receive rates for multiple
 118                                 MRR trials. For PPS, these are aggregate
 119                                 (bidirectional) rates.
 120                                 Currently, the tests are exporting
 121                                 approximated receive rates.
 122                                 That means the actual trial duration
 123                                 is measured (as opposed to trusting traffic
 124                                 generator to honor its target duration),
 125                                 so the resulting values contain noise
 126                                 from time measurement, and can be lower
 127                                 than the real performance
 128                                 (due to various time overheads).
 129                                 Bandwidth values are supported, but currently
 130                                 Robot does not export them.
 131                             $ref: "#/$defs/types/rate_list_with_bandwidth"
 132                     required:
 133                     -   type
 134                     -   receive_rate
 135                 -   description: >-
 136                         Result type NDRPDR case.
 137                     additionalProperties: false
 138                     properties:
 139                         type:
 140                             const: ndrpdr
 141                         ndr:
 142                             description: >-
 143                                 The results refer to search for NDR
 144                                 (Non Drop Rate). For PPS, this is aggregate
 145                                 (bidirectional) rate.
 146                                 Each bound was used as the target load value
 147                                 in a full-duration trial measurement.
 148                                 The accepted loss ratio for NDR is exact zero.
 149                                 Note that packets the Traffic Generator
 150                                 did not send are also counted as lost packets.
 151                             $ref: "#/$defs/macros/lower_and_upper_rate"
 152                         pdr:
 153                             description: >-
 154                                 The results refer to search for PDR
 155                                 (Partial Drop Rate). For PPS, this is aggregate
 156                                 (bidirectional) rate.
 157                                 Each bound was used as the target load value
 158                                 in a full-duration trial measurement.
 159                                 The accepted loss ratio for PDR is 0.5%.
 160                                 Note that packets the Traffic Generator
 161                                 did not send are also counted as lost packets.
 162                             $ref: "#/$defs/macros/lower_and_upper_rate"
 163                         latency_forward:
 164                             description: >-
 165                                 Object with results related to latency part
 166                                 of NDRPDR test, for forward traffic direction.
 167                                 It is the direction used in unidirectional
 168                                 traffic profiles.
 169                                 ASTF profiles and IMIX STL profiles
 170                                 do not support latency information,
 171                                 so for those tests this object is missing.
 172                                 It is also missing if Traffic Generator
 173                                 fails to return valid latency results
 174                                 for any other reasons,
 175                                 e.g. latency rate is too high for CPU/NIC used.
 176                             $ref: "#/$defs/macros/latency_for_loads"
 177                         latency_reverse:
 178                             description: >-
 179                                 Object with results related to latency part
 180                                 of NDRPDR test, for reverse traffic diration.
 181                                 This object is not present
 182                                 when unidirectional traffic profiles are used.
 183                                 ASTF profiles and IMIX STL profiles
 184                                 do not support latency information,
 185                                 so for those tests this object is missing.
 186                                 It is also missing if Traffic Generator
 187                                 fails to return valid latency results
 188                                 for any other reasons,
 189                                 e.g. latency rate is too high for CPU/NIC used.
 190                             $ref: "#/$defs/macros/latency_for_loads"
 191                     required:
 192                     -   type
 193                     -   ndr
 194                     -   pdr
 195                 -   description: >-
 196                         Result type SOAK case.
 197                     additionalProperties: false
 198                     properties:
 199                         type:
 200                             const: soak
 201                         critical_rate:
 202                             description: >-
 203                                 The results refer to bayesian estimate
 204                                 of critical rate corresponding to
 205                                 average loss ratio of 10^-7.
 206                                 For PPS, this is aggregate (bidirectional) rate.
 207                                 The bounds are computed from
 208                                 trial measurement results,
 209                                 but are not equal to any target load used.
 210                                 Note that packets the Traffic Generator
 211                                 did not send are also counted as lost packets.
 212                             $ref: "#/$defs/macros/lower_and_upper_rate"
 213                     required:
 214                     -   type
 215                     -   critical_rate
 216         start_time:
 217             description: >-
 218                 UTC date and time in RFC 3339 format, specifying calendar time
 219                 just after test case started (at the start of test setup).
 220             $ref: "#/$defs/types/date_time"
 221         tags:
 222             description: >-
 223                 The list of strings comes directly
 224                 from Robot variable TEST_TAGS.
 225                 The content should include both static and dynamic tags
 226                 at the end of test case (teardown).
 227             type: array
 228             items:
 229                 type: string
 230         test_documentation:
 231             description: >-
 232                 Value taken directly from TEST_DOCUMENTATION Robot variable.
 233                 The content is what you see in suite file
 234                 at test case definition, which is usually empty
 235                 as CSIT uses data driven test cases.
 236             type: string
 237         test_id:
 238             description: >-
 239                 A derived quantity, present only in info output files.
 240                 It is the most complete and unique identifier for a test case.
 241                 This property has a value, of the following form:
 242                 {suite_name}.{test_name}
 243                 Here, suite name comes from SUITE_NAME robot variable,
 244                 test name comes from TEST_NAME robot variable,
 245                 but both are converted to lower case,
 246                 and spaces are replaced by underscores.
 247             type: string
 248             minLength: 3
 249         test_name_long:
 250             description: >-
 251                 A derived quantity, present only in info output files.
 252                 This property has a value, of the following form:
 253                 {nic_short_name}-{frame_size}-{threads_and_cores}-{suite_part}
 254                 Here, suite part is very similar to suite tag,
 255                 but additionally may contain a prefix describing NIC driver used
 256                 (if it is not the default one, drv_vfio_pci for VPP tests).
 257                 Any space is replaced by underscore and letters are lower case.
 258             type: string
 259             minLength: 3
 260         test_name_short:
 261             description: >-
 262                 A derived quantity, present only in info output files.
 263                 This property has a value very similar to suite tag,
 264                 but additionally may contain a prefix describing NIC driver used
 265                 (if it is not the default one, drv_vfio_pci for VPP tests).
 266                 Any space is replaced by underscore and letters are lower case.
 267             type: string
 268             minLength: 3
 269         test_type:
 270             description: >-
 271                 A derived quantity, present only in info output files.
 272                 Test type identifier, PAL uses it to group similar tests,
 273                 e.g. for comparison tables.
 274                 Ideally, this information should be parseable from test name,
 275                 but the current naming scheme is not simple/consistent enough.
 276                 The current implementation queries the robot test tags.
 277                 The resulting value is frequently identical to result type,
 278                 but this schema version does not require any relation there,
 279                 as PAL may want to group tests differently.
 280             type: string
 281             enum:
 282             -   device
 283             -   gso
 284             -   hoststack
 285             -   mrr
 286             -   ndrpdr
 287             -   reconf
 288             -   soak
 289             -   vsap
 290         tg_type:
 291             description: >-
 292                 TG type used, e.g. TREX.
 293             type: string
 294             minLength: 1
 295         tg_version:
 296             description: >-
 297                 Version string appropriate to TG type used.
 298             type: string
 299             minLength: 1
 300         version:
 301             description: >-
 302                 CSIT model version (semver format)
 303                 the exporting code adhered to.
 304             type: string
 305             const: 1.1.0
 306     required:
 307     -   duration
 308     -   dut_type
 309     -   dut_version
 310     -   end_time
 311     -   hosts
 312     -   telemetry
 313     -   message
 314     -   passed
 315     -   result
 316     -   start_time
 317     -   tags
 318     -   test_documentation
 319     -   test_id
 320     -   test_name_long
 321     -   test_name_short
 322     -   test_type
 323     -   tg_type
 324     -   tg_version
 325     -   version
 326 -   description: >-
 327         Subschema validating relation between status and message.
 328     oneOf:
 329     -   description: >-
 330             Subschema for passing tests, message has to be empty.
 331         type: object
 332         properties:
 333             passed:
 334                 const: true
 335             message:
 336                 const: ""
 337     -   description: >-
 338             Subschema for failing tests, mesage cannot be empty.
 339         type: object
 340         properties:
 341             passed:
 342                 const: false
 343             message:
 344                 minLength: 1
 345
 346 $defs:
 347     types:
 348         nonnegative_number:
 349             type: number
 350             minimum: 0
 351         positive_number:
 352             type: number
 353             minimum: 1
 354         nonnegative_integer:
 355             type: integer
 356             minimum: 0
 357         positive_integer:
 358             type: integer
 359             minimum: 1
 360         date_time:
 361             type: string
 362             format: date-time
 363         empty_array:
 364             type: array
 365             maxItems: 0
 366         rate_unit:
 367             description: >-
 368                 Packets per second (pps) or connections per second (cps).
 369             type: string
 370             enum:
 371             -   pps
 372             -   cps
 373         bandwidth_unit:
 374             description: >-
 375                 Unit of measurement for bandwidth values.
 376                 Currently a constant, but later versions of model
 377                 may allow more units.
 378             enum:
 379             -   bps
 380         value_with_unit:
 381             description: >-
 382                 Reusable composite type, value together with its
 383                 unit of measurement.
 384             type: object
 385             additionalProperties: false
 386             properties:
 387                 value:
 388                     description: >-
 389                         Numeric value, context specified elsewhere.
 390                         The only assumption is that value is not negative.
 391                     $ref: "#/$defs/types/nonnegative_number"
 392                 unit:
 393                     description: >-
 394                         Unit of measurement for the value.
 395                         Context and allowed values are specified elsewhere.
 396                     type: string
 397             required:
 398             -   value
 399             -   unit
 400         rate_without_bandwidth:
 401             description: >-
 402                 Reusable type, for various rate quantites.
 403             allOf:
 404             -   $ref: "#/$defs/types/value_with_unit"
 405             -   properties:
 406                     value:
 407                         description: >-
 408                             Unless specified otherwise,
 409                             this is the aggregated rate
 410                             (sum of both traffic directions).
 411                             Depending on the usage, the value can express
 412                             intended load, offered load, receive rate,
 413                             and various approximations
 414                             or estimated bounds thereof.
 415                     unit:
 416                         description: >-
 417                             A transaction rate unit the value is expressed in.
 418                         $ref: "#/$defs/types/rate_unit"
 419         bandwidth:
 420             description: >-
 421                 Reusable type, for various bandwidth quantites.
 422             allOf:
 423             -   $ref: "#/$defs/types/value_with_unit"
 424             -   properties:
 425                     value:
 426                         description: >-
 427                             Bandwidth value computed
 428                             from the corresponding rate.
 429                     unit:
 430                         $ref: "#/$defs/types/bandwidth_unit"
 431         rate_with_bandwidth:
 432             description: >-
 433                 Reusable composite type, joining primary rate
 434                 with optional derived bandwidth.
 435                 Not all test types currently compute bandwidth,
 436                 even if rate unit is pps.
 437             type: object
 438             additionalProperties: false
 439             properties:
 440                 rate:
 441                     $ref: "#/$defs/types/rate_without_bandwidth"
 442                 bandwidth:
 443                     $ref: "#/$defs/types/bandwidth"
 444             required:
 445             - rate
 446         value_list_with_unit_and_stats:
 447             description: >-
 448                 Reusable composite type, multiple values together with their
 449                 unit of measurement and derived statistics.
 450             type: object
 451             additionalProperties: false
 452             properties:
 453                 values:
 454                     description: >-
 455                         List of values of the same unit, useful for MRR.
 456                     type: array
 457                     minItmes: 1
 458                     items:
 459                         description: >-
 460                             Numeric value, context specified elsewhere.
 461                             The only assumption is that the value is nonnegative.
 462                         $ref: "#/$defs/types/nonnegative_number"
 463                 avg:
 464                     description: >-
 465                         A derived quantity, present only in info output files.
 466                         It is the arithmetic average of the values list.
 467                     $ref: "#/$defs/types/nonnegative_number"
 468                 stdev:
 469                     description: >-
 470                         A derived quantity, present only in info output files.
 471                         It is the standard deviation for the values list,
 472                         as computed by jumpavg library.
 473                     $ref: "#/$defs/types/nonnegative_number"
 474                 unit:
 475                     description: >-
 476                         Unit of measurement for the values.
 477                         Context and allowed values are specified elsewhere.
 478                     type: string
 479             required:
 480             -   values
 481             -   avg
 482             -   stdev
 483             -   unit
 484         rate_list_without_bandwidth:
 485             description: >-
 486                 Reusable composite type, multiple rate values.
 487             allOf:
 488             -   $ref: "#/$defs/types/value_list_with_unit_and_stats"
 489             -   properties:
 490                     values:
 491                         items:
 492                             description: >-
 493                                 Unless specified otherwise,
 494                                 this is the aggregated rate
 495                                 (sum of both traffic directions).
 496                                 Depending on the usage, the value can express
 497                                 intended load, offered load, receive rate,
 498                                 and various approximations or estimated bounds
 499                                 thereof.
 500                     unit:
 501                         $ref: "#/$defs/types/rate_unit"
 502         bandwidth_list:
 503             description: >-
 504                 Reusable composite type, multiple bandwidth values.
 505                 This is a derived entity, thus it only appears in info output,
 506                 and only if rate unit is pps.
 507             allOf:
 508             -   $ref: "#/$defs/types/value_list_with_unit_and_stats"
 509             -   properties:
 510                     values:
 511                         items:
 512                             description: >-
 513                                 Unless specified otherwise,
 514                                 this is the aggregated bandwidth
 515                                 (sum of both traffic directions).
 516                                 Depending on the usage, the value can express
 517                                 intended load, offered load, receive rate,
 518                                 and various approximations or estimated bounds
 519                                 thereof.
 520                     unit:
 521                         $ref: "#/$defs/types/bandwidth_unit"
 522         rate_list_with_bandwidth:
 523             description: >-
 524                 Reusable composite type, joining primary rates
 525                 with optional derived bandwidths (and stats).
 526                 No test types currently computes the bandwidth part.
 527             type: object
 528             additionalProperties: false
 529             properties:
 530                 rate:
 531                     $ref: "#/$defs/types/rate_list_without_bandwidth"
 532                 bandwidth:
 533                     $ref: "#/$defs/types/bandwidth_list"
 534             required:
 535             - rate
 536     macros:
 537         lower_and_upper_rate:
 538             type: object
 539             additionalProperties: false
 540             properties:
 541                 lower:
 542                     description: >-
 543                         The lower bound (or min_rate) for the estimate
 544                         of a particular searched value.
 545                     $ref: "#/$defs/types/rate_with_bandwidth"
 546                 upper:
 547                     description: >-
 548                         The upper bound (or max_rate) for the estimate
 549                         of a particular searched value.
 550                     $ref: "#/$defs/types/rate_with_bandwidth"
 551             required:
 552             -   lower
 553             -   upper
 554         latency_numbers:
 555             type: object
 556             additionalProperties: false
 557             properties:
 558                 min:
 559                     description: >-
 560                         Rounded minimal latency time measured in this trial.
 561                         See unit property for the unit of measurement.
 562                     $ref: "#/$defs/types/nonnegative_integer"
 563                 max:
 564                     description: >-
 565                         Rounded maximal latency time measured in this trial.
 566                         See unit property for the unit of measurement.
 567                         Zero value is not allowed, as that is one of symptoms
 568                         of Traffic Generator failing to get proper latency.
 569                     $ref: "#/$defs/types/positive_integer"
 570                 avg:
 571                     description: >-
 572                         Rounded average latency time measured in this trial.
 573                         See unit property for the unit of measurement.
 574                     $ref: "#/$defs/types/nonnegative_integer"
 575                 hdrh:
 576                     description: >-
 577                         Base64-encoded compressed representation of HDRHistogram
 578                         of all latency sample times encountered
 579                         in this latency trial.
 580                         See unit property for the unit of measurement.
 581                         Note that some bins can be several units wide.
 582                     type: string
 583                 unit:
 584                     description: >-
 585                         Unit of measurement for latency times.
 586                         Currently a constant, but later versions
 587                         of the model may allow more values.
 588                     type: string
 589                     enum:
 590                     -   us
 591             required:
 592             -   avg
 593             -   hdrh
 594             -   max
 595             -   min
 596             -   unit
 597         latency_for_loads:
 598             type: object
 599             additionalProperties: false
 600             properties:
 601                 pdr_0:
 602                     description: >-
 603                         Object related to latency measurement performed
 604                         at minimal rate (currently 9000 pps per direction).
 605                     $ref: "#/$defs/macros/latency_numbers"
 606                 pdr_10:
 607                     description: >-
 608                         Object related to latency measurement performed
 609                         at 10% of PDR lower bound, if needed rounded up
 610                         to minimal rate (currently 9000 pps per direction).
 611                     $ref: "#/$defs/macros/latency_numbers"
 612                 pdr_50:
 613                     description: >-
 614                         Object related to latency measurement performed
 615                         at 50% of PDR lower bound, if needed rounded up
 616                         to minimal rate (currently 9000 pps per direction).
 617                     $ref: "#/$defs/macros/latency_numbers"
 618                 pdr_90:
 619                     description: >-
 620                         Object related to latency measurement performed
 621                         at 90% of PDR lower bound, if needed rounded up
 622                         to minimal rate (currently 9000 pps per direction).
 623                     $ref: "#/$defs/macros/latency_numbers"
 624             required:
 625             -   pdr_0
 626             -   pdr_10
 627             -   pdr_50
 628             -   pdr_90