jjb/csit/csit-perf.yaml

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Copyright (c) 2019 PANTHEON.tech s.r.o. and/or its affiliates.
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at:
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 ---
  16 - project:
  17     name: csit-perf
  18     description: 'CSIT Jenkins jobs.'
  19
  20     jobs:
  21       - 'csit-vpp-perf-verify-{stream}-{node-arch}'
  22       - 'csit-vpp-perf-mrr-daily-master{node-arch}':
  23           node-arch:
  24             - ''
  25             - -2n-clx
  26             - -2n-skx
  27             - -2n-dnv
  28             - -3n-skx
  29             - -3n-dnv
  30             - -3n-tsh:
  31                 executor: 'ubuntu1804arm-us'
  32                 vpp-mrr-daily-periodicity: '0 14 * * 0'
  33       - 'csit-vpp-perf-mrr-weekly-master-{node-arch}':
  34           node-arch:
  35             - 2n-clx
  36             - 2n-skx
  37       - 'csit-vpp-perf-ndrpdr-weekly-master-{node-arch}':
  38           node-arch:
  39             - 2n-clx
  40             - 2n-skx
  41             - 3n-skx
  42             - 3n-hsw
  43       - 'csit-dpdk-perf-verify-{stream}-{node-arch}'
  44       - 'csit-dpdk-perf-mrr-weekly-master{node-arch}':
  45           node-arch:
  46             - ''
  47             - -2n-clx
  48             - -2n-skx
  49             - -3n-skx
  50       - 'csit-ligato-perf-verify-{stream}-{node-arch}'
  51
  52     project: 'csit'
  53     executor: 'ubuntu1804-us'
  54     ligato-executor: 'ubuntu1804-basebuild-4c-4g'
  55     branch: 'master'
  56     branch-id: 'oper'
  57     vpp-mrr-daily-periodicity: '0 2,14 * * *'
  58     stream:
  59       - master:
  60           branch: 'master'
  61           branch-id: 'oper'
  62           executor: 'ubuntu1804-us'
  63       - '1901':
  64           branch: 'rls1901'
  65           branch-id: 'oper-rls1901'
  66           executor: 'ubuntu1604-basebuild-8c-32g'
  67       - '1904':
  68           branch: 'rls1904'
  69           branch-id: 'oper-rls1904'
  70           executor: 'ubuntu1804-us'
  71       - '1908_1':
  72           branch: 'rls1908_1'
  73           branch-id: 'oper-rls1908_1'
  74           executor: 'ubuntu1804-us'
  75
  76     build-artifact-num-to-keep: 10
  77     node-arch:
  78       - 2n-clx
  79       - 2n-skx
  80       - 2n-dnv
  81       - 3n-skx
  82       - 3n-hsw
  83       - 3n-dnv
  84       - 3n-tsh:
  85           executor: 'ubuntu1804arm-us'
  86           ligato-executor: 'ubuntu1804arm-us'
  87
  88 - job-template:
  89     name: 'csit-vpp-perf-verify-{stream}-{node-arch}'
  90
  91     description: |
  92       <ul>
  93           <li>objective
  94               <ul>
  95                   <li>perf verify - per csit gerrit patch
  96               </ul>
  97           <li>job
  98               <ul>
  99                   <li>name: <b>csit-vpp-perf-verify-{stream}-{node-arch}</b>
 100                   <li>trigger: keyword in comment to csit patch:
 101                       <b>csit-{node-arch}-perftest tag_list</b>
 102               </ul>
 103           <li>perf test-cases selection
 104               <ul>
 105                   <li>test-cases selected using trigger keyword followed by tags
 106                   <li>all test-cases are labelled with CSIT RF topic tags
 107                       defined in <i>csit/docs/tag_documentation.rst</i>
 108               </ul>
 109           <li>proposed keywords
 110               <ul>
 111                   <li>focus on testing specified vpp packet processing path
 112                   <li>format: csit-{node-arch}-perftest tag_list
 113               </ul>
 114           <li>default tags for vpp startup config and perf test type
 115               <ul>
 116                   <li>IP4BASE, IP6BASE, L2BDBASE
 117                   <li>64B/78B: 64B/78B packet size
 118                   <li>1C: one core
 119                   <li>X710: x710 Fortville NIC
 120                   <li>MRR: MRR throughput
 121               </ul>
 122           <li>results
 123               <ul>
 124                   <li>jjb verify: PASS/FAIL
 125                   <li>present: no
 126                   <li>analyse: no
 127               </ul>
 128       </ul>
 129
 130     project-type: freestyle
 131     node: '{executor}'
 132     concurrent: true
 133     archive-artifacts: 'archive/*.*'
 134     latest-only: false
 135
 136     build-discarder:
 137       daysToKeep: '{build-days-to-keep}'
 138       numToKeep: '{build-num-to-keep}'
 139       artifactDaysToKeep: '{build-artifact-days-to-keep}'
 140       artifactNumToKeep: '{build-artifact-num-to-keep}'
 141
 142     parameters:
 143       - project-parameter:
 144           project: '{project}'
 145       - gerrit-parameter:
 146           branch: '{branch}'
 147       - gerrit-refspec-parameter
 148       - gerrit-event-type-parameter
 149       - gerrit-event-comment-text-parameter
 150       - string:
 151           name: ARCHIVE_ARTIFACTS
 152           default: '{archive-artifacts}'
 153           description: Artifacts to archive to the logs server.
 154       # This is a manually-triggered verify job, part of API coverage.
 155       # CRC checking is needed to ensure vpp crc job breaks only when intended.
 156       # Can be removed when this job is no longer needed for full API coverage.
 157       - csit-fail-on-crc-mismatch-parameter:
 158           fail-on-crc-mismatch: 'True'
 159
 160     scm:
 161       - gerrit-trigger-scm:
 162           credentials-id: 'jenkins-gerrit-credentials'
 163           refspec: '$GERRIT_REFSPEC'
 164           choosing-strategy: 'gerrit'
 165
 166     wrappers:
 167       - fdio-infra-wrappers-non-activity-timeout:
 168           build-timeout: '{build-timeout}'
 169
 170     triggers:
 171       - gerrit:
 172           server-name: 'Primary'
 173           trigger-on:
 174             - comment-added-contains-event:
 175                 comment-contains-value: 'csit-{node-arch}-perftest'
 176           projects:
 177             - project-compare-type: 'ANT'
 178               project-pattern: '{project}'
 179               branches:
 180                 - branch-compare-type: 'ANT'
 181                   branch-pattern: '**/{branch}'
 182           skip-vote:
 183             successful: false
 184             failed: false
 185             unstable: false
 186             notbuilt: false
 187
 188     builders:
 189       - shell:
 190           !include-raw-escape: jjb/csit/include-raw-csit-perf-verify.sh
 191
 192     publishers:
 193       - robot:
 194           output-path: archive
 195           other-files:
 196             - '*.*'
 197
 198       - fdio-infra-shiplogs:
 199           maven-version: 'mvn33-new'
 200
 201 - job-template:
 202     name: 'csit-vpp-perf-mrr-daily-master{node-arch}'
 203
 204     description: |
 205       <ul>
 206           <li>objective
 207               <ul>
 208                   <li>check vpp perf mrr daily
 209               </ul>
 210           <li>perf test-cases selection
 211               <ul>
 212                   <li>all CSIT VPP MRR perf suites
 213               </ul>
 214           <li>results
 215               <ul>
 216                   <li>jjb verify: PASS/FAIL
 217                   <li>present: no
 218                   <li>analyse: csit project (catch malfunctions/degradations)
 219               </ul>
 220       </ul>
 221
 222     project-type: freestyle
 223     node: '{executor}'
 224     concurrent: false
 225     archive-artifacts: 'csit/archive/*.*'
 226     latest-only: false
 227
 228     build-discarder:
 229       daysToKeep: '{build-days-to-keep}'
 230       numToKeep: '{build-num-to-keep}'
 231       artifactDaysToKeep: '{build-artifact-days-to-keep}'
 232       artifactNumToKeep: '{build-artifact-num-to-keep}'
 233
 234     parameters:
 235       - project-parameter:
 236           project: '{project}'
 237       - gerrit-parameter:
 238           branch: '{branch}'
 239       - string:
 240           name: ARCHIVE_ARTIFACTS
 241           default: '{archive-artifacts}'
 242           description: Artifacts to archive to the logs server.
 243
 244     wrappers:
 245       - fdio-infra-wrappers-non-activity-timeout:
 246           build-timeout: '{build-timeout}'
 247
 248     triggers:
 249       - timed: '{vpp-mrr-daily-periodicity}'
 250
 251     builders:
 252       - shell: 'echo "BRANCH_ID={branch-id}" > $WORKSPACE/variables.prop'
 253       - inject:
 254           properties-file: variables.prop
 255       - shell:
 256           !include-raw-escape: jjb/csit/include-raw-csit-perf-timed.sh
 257
 258     publishers:
 259       - robot:
 260           output-path: 'csit/archive'
 261           other-files:
 262             - '*.*'
 263
 264       - fdio-infra-shiplogs:
 265           maven-version: 'mvn33-new'
 266
 267 - job-template:
 268     name: 'csit-vpp-perf-mrr-weekly-master-{node-arch}'
 269
 270     description: |
 271       <ul>
 272           <li>objective
 273               <ul>
 274                   <li>check vpp perf mrr weekly on {node-arch}
 275               </ul>
 276           <li>perf test-cases selection
 277               <ul>
 278                   <li>NF Density CSIT VPP MRR perf suites
 279               </ul>
 280           <li>results
 281               <ul>
 282                   <li>jjb verify: PASS/FAIL
 283                   <li>present: no
 284                   <li>analyse: csit project (catch malfunctions/degradations)
 285               </ul>
 286       </ul>
 287
 288     project-type: freestyle
 289     node: '{executor}'
 290     concurrent: false
 291     archive-artifacts: 'csit/archive/*.*'
 292     latest-only: false
 293
 294     build-discarder:
 295       daysToKeep: '{build-days-to-keep}'
 296       numToKeep: '{build-num-to-keep}'
 297       artifactDaysToKeep: '{build-artifact-days-to-keep}'
 298       artifactNumToKeep: '{build-artifact-num-to-keep}'
 299
 300     parameters:
 301       - project-parameter:
 302           project: '{project}'
 303       - gerrit-parameter:
 304           branch: '{branch}'
 305       - string:
 306           name: ARCHIVE_ARTIFACTS
 307           default: '{archive-artifacts}'
 308           description: Artifacts to archive to the logs server.
 309
 310     wrappers:
 311       - fdio-infra-wrappers-non-activity-timeout:
 312           build-timeout: '{build-timeout}'
 313
 314     triggers:
 315       - timed: '0 0 * * 6'
 316
 317     builders:
 318       - shell: 'echo "BRANCH_ID={branch-id}" > $WORKSPACE/variables.prop'
 319       - inject:
 320           properties-file: variables.prop
 321       - shell:
 322           !include-raw-escape: jjb/csit/include-raw-csit-perf-timed.sh
 323
 324     publishers:
 325       - robot:
 326           output-path: 'csit/archive'
 327           other-files:
 328             - '*.*'
 329
 330       - fdio-infra-shiplogs:
 331           maven-version: 'mvn33-new'
 332
 333 - job-template:
 334     name: 'csit-vpp-perf-ndrpdr-weekly-master-{node-arch}'
 335
 336     description: |
 337       <ul>
 338           <li>objective
 339               <ul>
 340                   <li>check vpp perf ndrpdr weekly on {node-arch}
 341               </ul>
 342           <li>perf test-cases selection
 343               <ul>
 344                   <li>pre-selected CSIT VPP NDRPDR perf suites
 345               </ul>
 346           <li>results
 347               <ul>
 348                   <li>jjb verify: PASS/FAIL
 349                   <li>present: no
 350                   <li>analyse: csit project (catch malfunctions/degradations)
 351               </ul>
 352       </ul>
 353     project-type: freestyle
 354     node: '{executor}'
 355     concurrent: false
 356     archive-artifacts: 'csit/archive/*.*'
 357     latest-only: false
 358
 359     build-discarder:
 360       daysToKeep: '{build-days-to-keep}'
 361       numToKeep: '{build-num-to-keep}'
 362       artifactDaysToKeep: '{build-artifact-days-to-keep}'
 363       artifactNumToKeep: '{build-artifact-num-to-keep}'
 364
 365     parameters:
 366       - project-parameter:
 367           project: '{project}'
 368       - gerrit-parameter:
 369           branch: '{branch}'
 370       - string:
 371           name: ARCHIVE_ARTIFACTS
 372           default: '{archive-artifacts}'
 373           description: Artifacts to archive to the logs server.
 374
 375     wrappers:
 376       - fdio-infra-wrappers-non-activity-timeout:
 377           build-timeout: '{build-timeout}'
 378
 379     triggers:
 380       - timed: '0 0 * * 6'
 381
 382     builders:
 383       - shell: 'echo "BRANCH_ID={branch-id}" > $WORKSPACE/variables.prop'
 384       - inject:
 385           properties-file: variables.prop
 386       - shell:
 387           !include-raw-escape: jjb/csit/include-raw-csit-perf-timed.sh
 388
 389     publishers:
 390       - robot:
 391           output-path: 'csit/archive'
 392           other-files:
 393             - '*.*'
 394
 395       - fdio-infra-shiplogs:
 396           maven-version: 'mvn33-new'
 397
 398 - job-template:
 399     name: 'csit-dpdk-perf-verify-{stream}-{node-arch}'
 400
 401     description: |
 402       <ul>
 403           <li>objective
 404               <ul>
 405                   <li>perf verify - per csit gerrit patch
 406               </ul>
 407           <li>job
 408               <ul>
 409                   <li>name: <b>csit-dpdk-perf-verify-{stream}-{node-arch}</b>
 410                   <li>trigger: keyword in comment to csit patch:
 411                       <b>csit-dpdk-{node-arch}-perftest tag_list</b>
 412               </ul>
 413           <li>perf test-cases selection
 414               <ul>
 415                   <li>test-cases selected using trigger keyword followed by tags
 416                   <li>all test-cases are labelled with CSIT RF topic tags
 417                       defined in <i>csit/docs/tag_documentation.rst</i>
 418               </ul>
 419           <li>default tags for dpdk perf test type
 420               <ul>
 421                   <li>IP4BASE, L2XCBASE
 422                   <li>64B: 64B packet size
 423                   <li>1C: one core
 424                   <li>X710: x710 Fortville NIC
 425                   <li>MRR: MRR throughput
 426               </ul>
 427           <li>results
 428               <ul>
 429                   <li>jjb verify: PASS/FAIL
 430                   <li>present: no
 431                   <li>analyse: no
 432               </ul>
 433       </ul>
 434
 435     project-type: freestyle
 436     node: '{executor}'
 437     concurrent: true
 438     archive-artifacts: 'archive/*.*'
 439     latest-only: false
 440
 441     build-discarder:
 442       daysToKeep: '{build-days-to-keep}'
 443       numToKeep: '{build-num-to-keep}'
 444       artifactDaysToKeep: '{build-artifact-days-to-keep}'
 445       artifactNumToKeep: '{build-artifact-num-to-keep}'
 446
 447     parameters:
 448       - project-parameter:
 449           project: '{project}'
 450       - gerrit-parameter:
 451           branch: '{branch}'
 452       - string:
 453           name: ARCHIVE_ARTIFACTS
 454           default: '{archive-artifacts}'
 455           description: Artifacts to archive to the logs server.
 456
 457     scm:
 458       - gerrit-trigger-scm:
 459           credentials-id: 'jenkins-gerrit-credentials'
 460           refspec: '$GERRIT_REFSPEC'
 461           choosing-strategy: 'gerrit'
 462
 463     wrappers:
 464       - fdio-infra-wrappers-non-activity-timeout:
 465           build-timeout: '{build-timeout}'
 466
 467     triggers:
 468       - gerrit:
 469           server-name: 'Primary'
 470           trigger-on:
 471             - comment-added-contains-event:
 472                 comment-contains-value: 'csit-dpdk-{node-arch}-perftest'
 473           projects:
 474             - project-compare-type: 'ANT'
 475               project-pattern: '{project}'
 476               branches:
 477                 - branch-compare-type: 'ANT'
 478                   branch-pattern: '**/{branch}'
 479           skip-vote:
 480             successful: false
 481             failed: false
 482             unstable: false
 483             notbuilt: false
 484
 485     builders:
 486       - shell:
 487           !include-raw-escape: jjb/csit/include-raw-csit-perf-verify.sh
 488
 489     publishers:
 490       - robot:
 491           output-path: archive
 492           other-files:
 493             - '*.*'
 494
 495       - fdio-infra-shiplogs:
 496           maven-version: 'mvn33-new'
 497
 498 - job-template:
 499     name: 'csit-dpdk-perf-mrr-weekly-master{node-arch}'
 500
 501     description: |
 502       <ul>
 503           <li>objective
 504               <ul>
 505                   <li>check dpdk perf mrr weekly
 506               </ul>
 507           <li>perf test-cases selection
 508               <ul>
 509                   <li>all CSIT DPDK MRR perf suites
 510               </ul>
 511           <li>results
 512               <ul>
 513                   <li>jjb verify: PASS/FAIL
 514                   <li>present: no
 515                   <li>analyse: csit project (catch malfunctions/degradations)
 516               </ul>
 517       </ul>
 518
 519     project-type: freestyle
 520     node: '{executor}'
 521     concurrent: false
 522     archive-artifacts: 'csit/archive/*.*'
 523     latest-only: false
 524
 525     build-discarder:
 526       daysToKeep: '{build-days-to-keep}'
 527       numToKeep: '{build-num-to-keep}'
 528       artifactDaysToKeep: '{build-artifact-days-to-keep}'
 529       artifactNumToKeep: '{build-artifact-num-to-keep}'
 530
 531     parameters:
 532       - project-parameter:
 533           project: '{project}'
 534       - gerrit-parameter:
 535           branch: '{branch}'
 536       - string:
 537           name: ARCHIVE_ARTIFACTS
 538           default: '{archive-artifacts}'
 539           description: Artifacts to archive to the logs server.
 540
 541     wrappers:
 542       - fdio-infra-wrappers-non-activity-timeout:
 543           build-timeout: '{build-timeout}'
 544
 545     triggers:
 546       - timed: '0 0 * * 6'
 547
 548     builders:
 549       - shell: 'echo "BRANCH_ID={branch-id}" > $WORKSPACE/variables.prop'
 550       - inject:
 551           properties-file: variables.prop
 552       - shell:
 553           !include-raw-escape: jjb/csit/include-raw-csit-perf-timed.sh
 554
 555     publishers:
 556       - robot:
 557           output-path: 'csit/archive'
 558           other-files:
 559             - '*.*'
 560
 561       - fdio-infra-shiplogs:
 562           maven-version: 'mvn33-new'
 563
 564 - job-template:
 565     name: 'csit-ligato-perf-verify-{stream}-{node-arch}'
 566
 567     # yamllint disable rule:line-length
 568     description: |
 569       <ul>
 570           <li>objective
 571               <ul>
 572                   <li>perf verify - per csit gerrit patch
 573               </ul>
 574           <li>job
 575               <ul>
 576                   <li>name: <b>csit-ligato-perf-verify-{stream}-{node-arch}</b>
 577                   <li>trigger: keyword in comment to csit patch:
 578                       <b>csit--ligato-{node-arch}-perftest tag_list</b>
 579               </ul>
 580           <li>perf test-cases selection
 581               <ul>
 582                   <li>test-cases selected using trigger keyword followed by tags
 583                   <li>all test-cases are labelled with CSIT RF topic tags
 584                       defined in <i>csit/docs/tag_documentation.rst</i>
 585               </ul>
 586           <li>proposed keywords
 587               <ul>
 588                   <li>focus on testing specified Kubernetes orchestrated topologies
 589                   <li>format: csit-ligato-{node-arch}-perftest tag_list
 590               </ul>
 591           <li>default tags for vpp startup config and perf test type
 592               <ul>
 593                   <li>IP4BASE, IP6BASE, L2BDBASE
 594                   <li>64B/78B: 64B/78B packet size
 595                   <li>1C: one core
 596                   <li>X710: x710 Fortville NIC
 597                   <li>MRR: MRR throughput
 598               </ul>
 599           <li>results
 600               <ul>
 601                   <li>jjb verify: PASS/FAIL
 602                   <li>present: no
 603                   <li>analyse: no
 604               </ul>
 605       </ul>
 606       # yamllint enable rule:line-length
 607
 608     project-type: freestyle
 609     # As the nomad executors are running docker containers we cannot run
 610     # ligato-perf job due to dependency on using docker inside.
 611     # TODO: Find the mechanics how to offload job logic from compiling docker
 612     #   image so we can move this executor to ubuntu1804-us
 613     node: '{ligato-executor}'
 614     concurrent: true
 615     archive-artifacts: 'archive/*.*'
 616     latest-only: false
 617
 618     build-discarder:
 619       daysToKeep: '{build-days-to-keep}'
 620       numToKeep: '{build-num-to-keep}'
 621       artifactDaysToKeep: '{build-artifact-days-to-keep}'
 622       artifactNumToKeep: '{build-artifact-num-to-keep}'
 623
 624     parameters:
 625       - project-parameter:
 626           project: '{project}'
 627       - gerrit-parameter:
 628           branch: '{branch}'
 629       - gerrit-refspec-parameter
 630       - gerrit-event-type-parameter
 631       - gerrit-event-comment-text-parameter
 632       - string:
 633           name: ARCHIVE_ARTIFACTS
 634           default: '{archive-artifacts}'
 635           description: Artifacts to archive to the logs server.
 636
 637     scm:
 638       - gerrit-trigger-scm:
 639           credentials-id: 'jenkins-gerrit-credentials'
 640           refspec: '$GERRIT_REFSPEC'
 641           choosing-strategy: 'gerrit'
 642
 643     wrappers:
 644       - fdio-infra-wrappers-non-activity-timeout:
 645           build-timeout: '{build-timeout}'
 646
 647     triggers:
 648       - gerrit:
 649           server-name: 'Primary'
 650           trigger-on:
 651             - comment-added-contains-event:
 652                 comment-contains-value: 'csit-ligato-{node-arch}-perftest'
 653           projects:
 654             - project-compare-type: 'ANT'
 655               project-pattern: '{project}'
 656               branches:
 657                 - branch-compare-type: 'ANT'
 658                   branch-pattern: '**/{branch}'
 659           skip-vote:
 660             successful: false
 661             failed: false
 662             unstable: false
 663             notbuilt: false
 664
 665     builders:
 666       - shell:
 667           !include-raw-escape: jjb/csit/include-raw-csit-perf-verify.sh
 668
 669     publishers:
 670       - robot:
 671           output-path: archive
 672           other-files:
 673             - '*.*'
 674
 675       - fdio-infra-shiplogs:
 676           maven-version: 'mvn33-new'