jjb/csit/csit-perf.yaml

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Copyright (c) 2019 PANTHEON.tech s.r.o. and/or its affiliates.
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at:
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 ---
  16 - project:
  17     name: csit-perf
  18     description: 'CSIT Jenkins jobs.'
  19
  20     jobs:
  21       - 'csit-dpdk-perf-verify-{stream}-{node-arch}':
  22           node-arch:
  23             - 2n-clx
  24             - 2n-skx
  25             - 2n-dnv
  26             - 3n-skx
  27             - 3n-hsw
  28             - 3n-dnv
  29             - 3n-tsh:
  30                executor: 'ubuntu1804arm-us'
  31       - 'csit-dpdk-perf-mrr-weekly-master{node-arch}':
  32           node-arch:
  33             - ''
  34             - -2n-clx
  35             - -2n-skx
  36             - -3n-skx
  37           periodicity: '0 0 * * 6'
  38       - 'csit-vpp-perf-verify-{stream}-{node-arch}':
  39           node-arch:
  40             - 2n-clx
  41             - 2n-skx
  42             - 2n-dnv
  43             - 3n-skx
  44             - 3n-hsw
  45             - 3n-dnv
  46             - 3n-tsh:
  47                executor: 'ubuntu1804arm-us'
  48       - 'csit-vpp-perf-mrr-daily-master{node-arch}':
  49           node-arch:
  50             - '':
  51                 periodicity: '0 14 * * *'
  52             - -2n-clx:
  53                 periodicity: '0 14 * * *'
  54             - -2n-skx:
  55                 periodicity: '0 14 * * 1,4'
  56             - -2n-dnv:
  57                 periodicity: '0 14 * * *'
  58             - -3n-skx:
  59                 periodicity: '0 14 * * 1,4'
  60             - -3n-dnv:
  61                 periodicity: '0 14 * * *'
  62             - -3n-tsh:
  63                 executor: 'ubuntu1804arm-us'
  64                 periodicity: '0 14 * * 6'
  65       - 'csit-vpp-perf-mrr-weekly-{stream}-{node-arch}':
  66           node-arch:
  67             - 2n-clx
  68             - 2n-skx
  69             - 3n-skx
  70           stream:
  71             - master:
  72                 branch: 'master'
  73                 branch-id: 'oper'
  74             - '1908':
  75                 branch: 'rls1908'
  76                 branch-id: 'oper-rls1908'
  77           periodicity: '0 0 * * 6'
  78       - 'csit-vpp-perf-ndrpdr-weekly-master-{node-arch}':
  79           node-arch:
  80             - 2n-clx
  81             - 2n-skx
  82             - 3n-skx
  83             - 3n-hsw
  84           periodicity: '0 0 * * 6'
  85
  86     project: 'csit'
  87     executor: 'ubuntu1804-us'
  88     branch: 'master'
  89     branch-id: 'oper'
  90     csit-perf-trial-duration: "1.0"
  91     csit-perf-trial-multiplicity: "10"
  92     stream:
  93       - master:
  94           branch: 'master'
  95           branch-id: 'oper'
  96       - '1908_1':
  97           branch: 'rls1908_1'
  98           branch-id: 'oper-rls1908_1'
  99       - '2001':
 100           branch: 'rls2001'
 101           branch-id: 'oper-rls2001'
 102     build-artifact-num-to-keep: 10
 103
 104
 105 - job-template:
 106     name: 'csit-vpp-perf-verify-{stream}-{node-arch}'
 107
 108     description: |
 109       <ul>
 110           <li>objective
 111               <ul>
 112                   <li>perf verify - per csit gerrit patch
 113               </ul>
 114           <li>job
 115               <ul>
 116                   <li>name: <b>csit-vpp-perf-verify-{stream}-{node-arch}</b>
 117                   <li>trigger: keyword in comment to csit patch:
 118                       <b>csit-{node-arch}-perftest tag_list</b>
 119               </ul>
 120           <li>executor
 121               <ul>
 122                   <li>{executor}
 123               </ul>
 124           <li>perf test-cases selection
 125               <ul>
 126                   <li>test-cases selected using trigger keyword followed by tags
 127                   <li>all test-cases are labelled with CSIT RF topic tags
 128                       defined in <i>csit/docs/tag_documentation.rst</i>
 129               </ul>
 130           <li>proposed keywords
 131               <ul>
 132                   <li>focus on testing specified vpp packet processing path
 133                   <li>format: csit-{node-arch}-perftest tag_list
 134               </ul>
 135           <li>default tags for vpp startup config and perf test type
 136               <ul>
 137                   <li>IP4BASE, IP6BASE, L2BDBASE
 138                   <li>64B/78B: 64B/78B packet size
 139                   <li>1C: one core
 140                   <li>X710: x710 Fortville NIC
 141                   <li>MRR: MRR throughput
 142               </ul>
 143           <li>results
 144               <ul>
 145                   <li>jjb verify: PASS/FAIL
 146                   <li>present: no
 147                   <li>analyse: no
 148               </ul>
 149       </ul>
 150
 151     project-type: freestyle
 152     node: '{executor}'
 153     concurrent: true
 154     archive-artifacts: 'archive/*.*'
 155     latest-only: false
 156
 157     build-discarder:
 158       daysToKeep: '{build-days-to-keep}'
 159       numToKeep: '{build-num-to-keep}'
 160       artifactDaysToKeep: '{build-artifact-days-to-keep}'
 161       artifactNumToKeep: '{build-artifact-num-to-keep}'
 162
 163     parameters:
 164       - project-parameter:
 165           project: '{project}'
 166       - gerrit-parameter:
 167           branch: '{branch}'
 168       - gerrit-refspec-parameter
 169       - gerrit-event-type-parameter
 170       - gerrit-event-comment-text-parameter
 171       - csit-perf-trial-duration-parameter:
 172           csit-perf-trial-duration: "{csit-perf-trial-duration}"
 173       - csit-perf-trial-multiplicity-parameter:
 174           csit-perf-trial-multiplicity: "{csit-perf-trial-multiplicity}"
 175       - string:
 176           name: ARCHIVE_ARTIFACTS
 177           default: '{archive-artifacts}'
 178           description: Artifacts to archive to the logs server.
 179       # This is a manually-triggered verify job, part of API coverage.
 180       # CRC checking is needed to ensure vpp crc job breaks only when intended.
 181       # Can be removed when this job is no longer needed for full API coverage.
 182       - csit-fail-on-crc-mismatch-parameter:
 183           fail-on-crc-mismatch: 'True'
 184
 185     scm:
 186       - gerrit-trigger-scm:
 187           credentials-id: 'jenkins-gerrit-credentials'
 188           refspec: '$GERRIT_REFSPEC'
 189           choosing-strategy: 'gerrit'
 190
 191     wrappers:
 192       - fdio-infra-wrappers-non-activity-timeout:
 193           build-timeout: '{build-timeout}'
 194
 195     triggers:
 196       - gerrit:
 197           server-name: 'Primary'
 198           trigger-on:
 199             - comment-added-contains-event:
 200                 comment-contains-value: 'csit-{node-arch}-perftest'
 201           projects:
 202             - project-compare-type: 'ANT'
 203               project-pattern: '{project}'
 204               branches:
 205                 - branch-compare-type: 'ANT'
 206                   branch-pattern: '**/{branch}'
 207           skip-vote:
 208             successful: false
 209             failed: false
 210             unstable: false
 211             notbuilt: false
 212
 213     builders:
 214       - shell:
 215           !include-raw-escape:
 216           - ../scripts/csit/perf-verify.sh
 217
 218     publishers:
 219       - robot:
 220           output-path: archive
 221           other-files:
 222             - '*.*'
 223
 224       - fdio-infra-shiplogs:
 225           maven-version: 'mvn33-new'
 226
 227 - job-template:
 228     name: 'csit-vpp-perf-mrr-daily-master{node-arch}'
 229
 230     description: |
 231       <ul>
 232           <li>objective
 233               <ul>
 234                   <li>check vpp perf mrr daily
 235               </ul>
 236           <li>executor
 237               <ul>
 238                   <li>{executor}
 239               </ul>
 240           <li>perf test-cases selection
 241               <ul>
 242                   <li>all CSIT VPP MRR perf suites
 243               </ul>
 244           <li>results
 245               <ul>
 246                   <li>jjb verify: PASS/FAIL
 247                   <li>present: no
 248                   <li>analyse: csit project (catch malfunctions/degradations)
 249               </ul>
 250       </ul>
 251
 252     project-type: freestyle
 253     node: '{executor}'
 254     concurrent: false
 255     archive-artifacts: 'csit/archive/*.*'
 256     latest-only: false
 257
 258     build-discarder:
 259       daysToKeep: '{build-days-to-keep}'
 260       numToKeep: '{build-num-to-keep}'
 261       artifactDaysToKeep: '{build-artifact-days-to-keep}'
 262       artifactNumToKeep: '{build-artifact-num-to-keep}'
 263
 264     parameters:
 265       - project-parameter:
 266           project: '{project}'
 267       - gerrit-parameter:
 268           branch: '{branch}'
 269       - csit-perf-trial-duration-parameter:
 270           csit-perf-trial-duration: "{csit-perf-trial-duration}"
 271       - csit-perf-trial-multiplicity-parameter:
 272           csit-perf-trial-multiplicity: "{csit-perf-trial-multiplicity}"
 273       - string:
 274           name: ARCHIVE_ARTIFACTS
 275           default: '{archive-artifacts}'
 276           description: Artifacts to archive to the logs server.
 277
 278     wrappers:
 279       - fdio-infra-wrappers-non-activity-timeout:
 280           build-timeout: '{build-timeout}'
 281
 282     triggers:
 283       - timed: '{periodicity}'
 284
 285     builders:
 286       - shell: 'echo "BRANCH_ID={branch-id}" > $WORKSPACE/variables.prop'
 287       - inject:
 288           properties-file: variables.prop
 289       - shell:
 290           !include-raw-escape:
 291           - ../scripts/csit/perf-timed.sh
 292
 293     publishers:
 294       - robot:
 295           output-path: 'csit/archive'
 296           other-files:
 297             - '*.*'
 298
 299       - fdio-infra-shiplogs:
 300           maven-version: 'mvn33-new'
 301
 302 - job-template:
 303     name: 'csit-vpp-perf-mrr-weekly-{stream}-{node-arch}'
 304
 305     description: |
 306       <ul>
 307           <li>objective
 308               <ul>
 309                   <li>check vpp perf mrr weekly on {node-arch}
 310               </ul>
 311           <li>executor
 312               <ul>
 313                   <li>{executor}
 314               </ul>
 315           <li>perf test-cases selection
 316               <ul>
 317                   <li>NF Density CSIT VPP MRR perf suites
 318               </ul>
 319           <li>results
 320               <ul>
 321                   <li>jjb verify: PASS/FAIL
 322                   <li>present: no
 323                   <li>analyse: csit project (catch malfunctions/degradations)
 324               </ul>
 325       </ul>
 326
 327     project-type: freestyle
 328     node: '{executor}'
 329     concurrent: false
 330     archive-artifacts: 'csit/archive/*.*'
 331     latest-only: false
 332
 333     build-discarder:
 334       daysToKeep: '{build-days-to-keep}'
 335       numToKeep: '{build-num-to-keep}'
 336       artifactDaysToKeep: '{build-artifact-days-to-keep}'
 337       artifactNumToKeep: '{build-artifact-num-to-keep}'
 338
 339     parameters:
 340       - project-parameter:
 341           project: '{project}'
 342       - gerrit-parameter:
 343           branch: '{branch}'
 344       - csit-perf-trial-duration-parameter:
 345           csit-perf-trial-duration: "{csit-perf-trial-duration}"
 346       - csit-perf-trial-multiplicity-parameter:
 347           csit-perf-trial-multiplicity: "{csit-perf-trial-multiplicity}"
 348       - string:
 349           name: ARCHIVE_ARTIFACTS
 350           default: '{archive-artifacts}'
 351           description: Artifacts to archive to the logs server.
 352
 353     wrappers:
 354       - fdio-infra-wrappers-non-activity-timeout:
 355           build-timeout: '{build-timeout}'
 356
 357     triggers:
 358       - timed: '{periodicity}'
 359
 360     builders:
 361       - shell: 'echo "BRANCH_ID={branch-id}" > $WORKSPACE/variables.prop'
 362       - inject:
 363           properties-file: variables.prop
 364       - shell:
 365           !include-raw-escape:
 366           - ../scripts/csit/perf-timed.sh
 367
 368     publishers:
 369       - robot:
 370           output-path: 'csit/archive'
 371           other-files:
 372             - '*.*'
 373
 374       - fdio-infra-shiplogs:
 375           maven-version: 'mvn33-new'
 376
 377 - job-template:
 378     name: 'csit-vpp-perf-ndrpdr-weekly-master-{node-arch}'
 379
 380     description: |
 381       <ul>
 382           <li>objective
 383               <ul>
 384                   <li>check vpp perf ndrpdr weekly on {node-arch}
 385               </ul>
 386           <li>executor
 387               <ul>
 388                   <li>{executor}
 389               </ul>
 390           <li>perf test-cases selection
 391               <ul>
 392                   <li>pre-selected CSIT VPP NDRPDR perf suites
 393               </ul>
 394           <li>results
 395               <ul>
 396                   <li>jjb verify: PASS/FAIL
 397                   <li>present: no
 398                   <li>analyse: csit project (catch malfunctions/degradations)
 399               </ul>
 400       </ul>
 401     project-type: freestyle
 402     node: '{executor}'
 403     concurrent: false
 404     archive-artifacts: 'csit/archive/*.*'
 405     latest-only: false
 406
 407     build-discarder:
 408       daysToKeep: '{build-days-to-keep}'
 409       numToKeep: '{build-num-to-keep}'
 410       artifactDaysToKeep: '{build-artifact-days-to-keep}'
 411       artifactNumToKeep: '{build-artifact-num-to-keep}'
 412
 413     parameters:
 414       - project-parameter:
 415           project: '{project}'
 416       - gerrit-parameter:
 417           branch: '{branch}'
 418       - string:
 419           name: ARCHIVE_ARTIFACTS
 420           default: '{archive-artifacts}'
 421           description: Artifacts to archive to the logs server.
 422
 423     wrappers:
 424       - fdio-infra-wrappers-non-activity-timeout:
 425           build-timeout: '{build-timeout}'
 426
 427     triggers:
 428       - timed: '{periodicity}'
 429
 430     builders:
 431       - shell: 'echo "BRANCH_ID={branch-id}" > $WORKSPACE/variables.prop'
 432       - inject:
 433           properties-file: variables.prop
 434       - shell:
 435           !include-raw-escape:
 436           - ../scripts/csit/perf-timed.sh
 437
 438     publishers:
 439       - robot:
 440           output-path: 'csit/archive'
 441           other-files:
 442             - '*.*'
 443
 444       - fdio-infra-shiplogs:
 445           maven-version: 'mvn33-new'
 446
 447 - job-template:
 448     name: 'csit-dpdk-perf-verify-{stream}-{node-arch}'
 449
 450     description: |
 451       <ul>
 452           <li>objective
 453               <ul>
 454                   <li>perf verify - per csit gerrit patch
 455               </ul>
 456           <li>job
 457               <ul>
 458                   <li>name: <b>csit-dpdk-perf-verify-{stream}-{node-arch}</b>
 459                   <li>trigger: keyword in comment to csit patch:
 460                       <b>csit-dpdk-{node-arch}-perftest tag_list</b>
 461               </ul>
 462           <li>executor
 463               <ul>
 464                   <li>{executor}
 465               </ul>
 466           <li>perf test-cases selection
 467               <ul>
 468                   <li>test-cases selected using trigger keyword followed by tags
 469                   <li>all test-cases are labelled with CSIT RF topic tags
 470                       defined in <i>csit/docs/tag_documentation.rst</i>
 471               </ul>
 472           <li>default tags for dpdk perf test type
 473               <ul>
 474                   <li>IP4BASE, L2XCBASE
 475                   <li>64B: 64B packet size
 476                   <li>1C: one core
 477                   <li>X710: x710 Fortville NIC
 478                   <li>MRR: MRR throughput
 479               </ul>
 480           <li>results
 481               <ul>
 482                   <li>jjb verify: PASS/FAIL
 483                   <li>present: no
 484                   <li>analyse: no
 485               </ul>
 486       </ul>
 487
 488     project-type: freestyle
 489     node: '{executor}'
 490     concurrent: true
 491     archive-artifacts: 'archive/*.*'
 492     latest-only: false
 493
 494     build-discarder:
 495       daysToKeep: '{build-days-to-keep}'
 496       numToKeep: '{build-num-to-keep}'
 497       artifactDaysToKeep: '{build-artifact-days-to-keep}'
 498       artifactNumToKeep: '{build-artifact-num-to-keep}'
 499
 500     parameters:
 501       - project-parameter:
 502           project: '{project}'
 503       - gerrit-parameter:
 504           branch: '{branch}'
 505       - csit-perf-trial-duration-parameter:
 506           csit-perf-trial-duration: "{csit-perf-trial-duration}"
 507       - csit-perf-trial-multiplicity-parameter:
 508           csit-perf-trial-multiplicity: "{csit-perf-trial-multiplicity}"
 509       - string:
 510           name: ARCHIVE_ARTIFACTS
 511           default: '{archive-artifacts}'
 512           description: Artifacts to archive to the logs server.
 513
 514     scm:
 515       - gerrit-trigger-scm:
 516           credentials-id: 'jenkins-gerrit-credentials'
 517           refspec: '$GERRIT_REFSPEC'
 518           choosing-strategy: 'gerrit'
 519
 520     wrappers:
 521       - fdio-infra-wrappers-non-activity-timeout:
 522           build-timeout: '{build-timeout}'
 523
 524     triggers:
 525       - gerrit:
 526           server-name: 'Primary'
 527           trigger-on:
 528             - comment-added-contains-event:
 529                 comment-contains-value: 'csit-dpdk-{node-arch}-perftest'
 530           projects:
 531             - project-compare-type: 'ANT'
 532               project-pattern: '{project}'
 533               branches:
 534                 - branch-compare-type: 'ANT'
 535                   branch-pattern: '**/{branch}'
 536           skip-vote:
 537             successful: false
 538             failed: false
 539             unstable: false
 540             notbuilt: false
 541
 542     builders:
 543       - shell:
 544           !include-raw-escape:
 545           - ../scripts/csit/perf-verify.sh
 546
 547     publishers:
 548       - robot:
 549           output-path: archive
 550           other-files:
 551             - '*.*'
 552
 553       - fdio-infra-shiplogs:
 554           maven-version: 'mvn33-new'
 555
 556 - job-template:
 557     name: 'csit-dpdk-perf-mrr-weekly-master{node-arch}'
 558
 559     description: |
 560       <ul>
 561           <li>objective
 562               <ul>
 563                   <li>check dpdk perf mrr weekly
 564               </ul>
 565           <li>executor
 566               <ul>
 567                   <li>{executor}
 568               </ul>
 569           <li>perf test-cases selection
 570               <ul>
 571                   <li>all CSIT DPDK MRR perf suites
 572               </ul>
 573           <li>results
 574               <ul>
 575                   <li>jjb verify: PASS/FAIL
 576                   <li>present: no
 577                   <li>analyse: csit project (catch malfunctions/degradations)
 578               </ul>
 579       </ul>
 580
 581     project-type: freestyle
 582     node: '{executor}'
 583     concurrent: false
 584     archive-artifacts: 'csit/archive/*.*'
 585     latest-only: false
 586
 587     build-discarder:
 588       daysToKeep: '{build-days-to-keep}'
 589       numToKeep: '{build-num-to-keep}'
 590       artifactDaysToKeep: '{build-artifact-days-to-keep}'
 591       artifactNumToKeep: '{build-artifact-num-to-keep}'
 592
 593     parameters:
 594       - project-parameter:
 595           project: '{project}'
 596       - gerrit-parameter:
 597           branch: '{branch}'
 598       - csit-perf-trial-duration-parameter:
 599           csit-perf-trial-duration: "{csit-perf-trial-duration}"
 600       - csit-perf-trial-multiplicity-parameter:
 601           csit-perf-trial-multiplicity: "{csit-perf-trial-multiplicity}"
 602       - string:
 603           name: ARCHIVE_ARTIFACTS
 604           default: '{archive-artifacts}'
 605           description: Artifacts to archive to the logs server.
 606
 607     wrappers:
 608       - fdio-infra-wrappers-non-activity-timeout:
 609           build-timeout: '{build-timeout}'
 610
 611     triggers:
 612       - timed: '{periodicity}'
 613
 614     builders:
 615       - shell: 'echo "BRANCH_ID={branch-id}" > $WORKSPACE/variables.prop'
 616       - inject:
 617           properties-file: variables.prop
 618       - shell:
 619           !include-raw-escape:
 620           - ../scripts/csit/perf-timed.sh
 621
 622     publishers:
 623       - robot:
 624           output-path: 'csit/archive'
 625           other-files:
 626             - '*.*'
 627
 628       - fdio-infra-shiplogs:
 629           maven-version: 'mvn33-new'