jjb/csit/csit-perf.yaml

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Copyright (c) 2019 PANTHEON.tech s.r.o. and/or its affiliates.
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at:
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 ---
  16 - project:
  17     name: csit-perf
  18     description: 'CSIT Jenkins jobs.'
  19
  20     jobs:
  21       - 'csit-vpp-perf-verify-{stream}-{node-arch}'
  22       - 'csit-vpp-perf-mrr-daily-master{node-arch}':
  23           node-arch:
  24             - ''
  25             - -2n-skx
  26             - -2n-dnv
  27             - -3n-skx
  28             - -3n-dnv
  29             - -3n-tsh:
  30                 executor: 'ubuntu1804arm-us'
  31                 vpp-mrr-daily-periodicity: '0 16 * * 6'
  32       - 'csit-vpp-perf-mrr-weekly-master-2n-skx'
  33       - 'csit-vpp-perf-ndrpdr-weekly-master-{node-arch}':
  34           node-arch:
  35             - 2n-skx
  36             - 3n-skx
  37             - 3n-hsw
  38       - 'csit-dpdk-perf-verify-{stream}-{node-arch}'
  39       - 'csit-dpdk-perf-mrr-weekly-master{node-arch}':
  40           node-arch:
  41             - ''
  42             - -2n-skx
  43             - -3n-skx
  44       - 'csit-ligato-perf-verify-{stream}-{node-arch}'
  45
  46     project: 'csit'
  47     executor: 'ubuntu1804-us'
  48     ligato-executor: 'ubuntu1804-basebuild-4c-4g'
  49     branch: 'master'
  50     branch-id: 'oper'
  51     vpp-mrr-daily-periodicity: '0 14 * * 1,4'
  52     stream:
  53       - master:
  54           branch: 'master'
  55           branch-id: 'oper'
  56           executor: 'ubuntu1804-us'
  57       - '1901':
  58           branch: 'rls1901'
  59           branch-id: 'oper-rls1901'
  60           executor: 'ubuntu1604-basebuild-8c-32g'
  61       - '1904':
  62           branch: 'rls1904'
  63           branch-id: 'oper-rls1904'
  64           executor: 'ubuntu1804-us'
  65       - '1908':
  66           branch: 'rls1908'
  67           branch-id: 'oper-rls1908'
  68           executor: 'ubuntu1804-us'
  69
  70     build-artifact-num-to-keep: 10
  71     node-arch:
  72       - 2n-skx
  73       - 2n-dnv
  74       - 3n-skx
  75       - 3n-hsw
  76       - 3n-dnv
  77       - 3n-tsh:
  78           executor: 'ubuntu1804arm-us'
  79           ligato-executor: 'ubuntu1804arm-us'
  80
  81 - job-template:
  82     name: 'csit-vpp-perf-verify-{stream}-{node-arch}'
  83
  84     description: |
  85       <ul>
  86           <li>objective
  87               <ul>
  88                   <li>perf verify - per csit gerrit patch
  89               </ul>
  90           <li>job
  91               <ul>
  92                   <li>name: <b>csit-vpp-perf-verify-{stream}-{node-arch}</b>
  93                   <li>trigger: keyword in comment to csit patch:
  94                       <b>csit-{node-arch}-perftest tag_list</b>
  95               </ul>
  96           <li>perf test-cases selection
  97               <ul>
  98                   <li>test-cases selected using trigger keyword followed by tags
  99                   <li>all test-cases are labelled with CSIT RF topic tags
 100                       defined in <i>csit/docs/tag_documentation.rst</i>
 101               </ul>
 102           <li>proposed keywords
 103               <ul>
 104                   <li>focus on testing specified vpp packet processing path
 105                   <li>format: csit-{node-arch}-perftest tag_list
 106               </ul>
 107           <li>default tags for vpp startup config and perf test type
 108               <ul>
 109                   <li>IP4BASE, IP6BASE, L2BDBASE
 110                   <li>64B/78B: 64B/78B packet size
 111                   <li>1C: one core
 112                   <li>X710: x710 Fortville NIC
 113                   <li>MRR: MRR throughput
 114               </ul>
 115           <li>results
 116               <ul>
 117                   <li>jjb verify: PASS/FAIL
 118                   <li>present: no
 119                   <li>analyse: no
 120               </ul>
 121       </ul>
 122
 123     project-type: freestyle
 124     node: '{executor}'
 125     concurrent: true
 126     archive-artifacts: 'archive/*.*'
 127     latest-only: false
 128
 129     build-discarder:
 130       daysToKeep: '{build-days-to-keep}'
 131       numToKeep: '{build-num-to-keep}'
 132       artifactDaysToKeep: '{build-artifact-days-to-keep}'
 133       artifactNumToKeep: '{build-artifact-num-to-keep}'
 134
 135     parameters:
 136       - project-parameter:
 137           project: '{project}'
 138       - gerrit-parameter:
 139           branch: '{branch}'
 140       - gerrit-refspec-parameter
 141       - gerrit-event-type-parameter
 142       - gerrit-event-comment-text-parameter
 143       - string:
 144           name: ARCHIVE_ARTIFACTS
 145           default: '{archive-artifacts}'
 146           description: Artifacts to archive to the logs server.
 147       # This is a manually-triggered verify job, part of API coverage.
 148       # CRC checking is needed to ensure vpp crc job breaks only when intended.
 149       # Can be removed when this job is no longer needed for full API coverage.
 150       - csit-fail-on-crc-mismatch-parameter:
 151           fail-on-crc-mismatch: 'True'
 152
 153     scm:
 154       - gerrit-trigger-scm:
 155           credentials-id: 'jenkins-gerrit-credentials'
 156           refspec: '$GERRIT_REFSPEC'
 157           choosing-strategy: 'gerrit'
 158
 159     wrappers:
 160       - fdio-infra-wrappers-non-activity-timeout:
 161           build-timeout: '{build-timeout}'
 162
 163     triggers:
 164       - gerrit:
 165           server-name: 'Primary'
 166           trigger-on:
 167             - comment-added-contains-event:
 168                 comment-contains-value: 'csit-{node-arch}-perftest'
 169           projects:
 170             - project-compare-type: 'ANT'
 171               project-pattern: '{project}'
 172               branches:
 173                 - branch-compare-type: 'ANT'
 174                   branch-pattern: '**/{branch}'
 175           skip-vote:
 176             successful: false
 177             failed: false
 178             unstable: false
 179             notbuilt: false
 180
 181     builders:
 182       - shell:
 183           !include-raw-escape: jjb/csit/include-raw-csit-perf-verify.sh
 184
 185     publishers:
 186       - robot:
 187           output-path: archive
 188           other-files:
 189             - '*.*'
 190
 191       - fdio-infra-shiplogs:
 192           maven-version: 'mvn33-new'
 193
 194 - job-template:
 195     name: 'csit-vpp-perf-mrr-daily-master{node-arch}'
 196
 197     description: |
 198       <ul>
 199           <li>objective
 200               <ul>
 201                   <li>check vpp perf mrr daily
 202               </ul>
 203           <li>perf test-cases selection
 204               <ul>
 205                   <li>all CSIT VPP MRR perf suites
 206               </ul>
 207           <li>results
 208               <ul>
 209                   <li>jjb verify: PASS/FAIL
 210                   <li>present: no
 211                   <li>analyse: csit project (catch malfunctions/degradations)
 212               </ul>
 213       </ul>
 214
 215     project-type: freestyle
 216     node: '{executor}'
 217     concurrent: false
 218     archive-artifacts: 'csit/archive/*.*'
 219     latest-only: false
 220
 221     build-discarder:
 222       daysToKeep: '{build-days-to-keep}'
 223       numToKeep: '{build-num-to-keep}'
 224       artifactDaysToKeep: '{build-artifact-days-to-keep}'
 225       artifactNumToKeep: '{build-artifact-num-to-keep}'
 226
 227     parameters:
 228       - project-parameter:
 229           project: '{project}'
 230       - gerrit-parameter:
 231           branch: '{branch}'
 232       - string:
 233           name: ARCHIVE_ARTIFACTS
 234           default: '{archive-artifacts}'
 235           description: Artifacts to archive to the logs server.
 236
 237     wrappers:
 238       - fdio-infra-wrappers-non-activity-timeout:
 239           build-timeout: '{build-timeout}'
 240
 241     triggers:
 242       - timed: '{vpp-mrr-daily-periodicity}'
 243
 244     builders:
 245       - shell: 'echo "BRANCH_ID={branch-id}" > $WORKSPACE/variables.prop'
 246       - inject:
 247           properties-file: variables.prop
 248       - shell:
 249           !include-raw-escape: jjb/csit/include-raw-csit-perf-timed.sh
 250
 251     publishers:
 252       - robot:
 253           output-path: 'csit/archive'
 254           other-files:
 255             - '*.*'
 256
 257       - fdio-infra-shiplogs:
 258           maven-version: 'mvn33-new'
 259
 260 - job-template:
 261     name: 'csit-vpp-perf-mrr-weekly-master-2n-skx'
 262
 263     description: |
 264       <ul>
 265           <li>objective
 266               <ul>
 267                   <li>check vpp perf mrr weekly on 2n-skx
 268               </ul>
 269           <li>perf test-cases selection
 270               <ul>
 271                   <li>NF Density CSIT VPP MRR perf suites
 272               </ul>
 273           <li>results
 274               <ul>
 275                   <li>jjb verify: PASS/FAIL
 276                   <li>present: no
 277                   <li>analyse: csit project (catch malfunctions/degradations)
 278               </ul>
 279       </ul>
 280
 281     project-type: freestyle
 282     node: '{executor}'
 283     concurrent: false
 284     archive-artifacts: 'csit/archive/*.*'
 285     latest-only: false
 286
 287     build-discarder:
 288       daysToKeep: '{build-days-to-keep}'
 289       numToKeep: '{build-num-to-keep}'
 290       artifactDaysToKeep: '{build-artifact-days-to-keep}'
 291       artifactNumToKeep: '{build-artifact-num-to-keep}'
 292
 293     parameters:
 294       - project-parameter:
 295           project: '{project}'
 296       - gerrit-parameter:
 297           branch: '{branch}'
 298       - string:
 299           name: ARCHIVE_ARTIFACTS
 300           default: '{archive-artifacts}'
 301           description: Artifacts to archive to the logs server.
 302
 303     wrappers:
 304       - fdio-infra-wrappers-non-activity-timeout:
 305           build-timeout: '{build-timeout}'
 306
 307     triggers:
 308       - timed: '0 0 * * 6'
 309
 310     builders:
 311       - shell: 'echo "BRANCH_ID={branch-id}" > $WORKSPACE/variables.prop'
 312       - inject:
 313           properties-file: variables.prop
 314       - shell:
 315           !include-raw-escape: jjb/csit/include-raw-csit-perf-timed.sh
 316
 317     publishers:
 318       - robot:
 319           output-path: 'csit/archive'
 320           other-files:
 321             - '*.*'
 322
 323       - fdio-infra-shiplogs:
 324           maven-version: 'mvn33-new'
 325
 326 - job-template:
 327     name: 'csit-vpp-perf-ndrpdr-weekly-master-{node-arch}'
 328
 329     description: |
 330       <ul>
 331           <li>objective
 332               <ul>
 333                   <li>check vpp perf ndrpdr weekly on {node-arch}
 334               </ul>
 335           <li>perf test-cases selection
 336               <ul>
 337                   <li>pre-selected CSIT VPP NDRPDR perf suites
 338               </ul>
 339           <li>results
 340               <ul>
 341                   <li>jjb verify: PASS/FAIL
 342                   <li>present: no
 343                   <li>analyse: csit project (catch malfunctions/degradations)
 344               </ul>
 345       </ul>
 346     project-type: freestyle
 347     node: '{executor}'
 348     concurrent: false
 349     archive-artifacts: 'csit/archive/*.*'
 350     latest-only: false
 351
 352     build-discarder:
 353       daysToKeep: '{build-days-to-keep}'
 354       numToKeep: '{build-num-to-keep}'
 355       artifactDaysToKeep: '{build-artifact-days-to-keep}'
 356       artifactNumToKeep: '{build-artifact-num-to-keep}'
 357
 358     parameters:
 359       - project-parameter:
 360           project: '{project}'
 361       - gerrit-parameter:
 362           branch: '{branch}'
 363       - string:
 364           name: ARCHIVE_ARTIFACTS
 365           default: '{archive-artifacts}'
 366           description: Artifacts to archive to the logs server.
 367
 368     wrappers:
 369       - fdio-infra-wrappers-non-activity-timeout:
 370           build-timeout: '{build-timeout}'
 371
 372     triggers:
 373       - timed: '0 0 * * 6'
 374
 375     builders:
 376       - shell: 'echo "BRANCH_ID={branch-id}" > $WORKSPACE/variables.prop'
 377       - inject:
 378           properties-file: variables.prop
 379       - shell:
 380           !include-raw-escape: jjb/csit/include-raw-csit-perf-timed.sh
 381
 382     publishers:
 383       - robot:
 384           output-path: 'csit/archive'
 385           other-files:
 386             - '*.*'
 387
 388       - fdio-infra-shiplogs:
 389           maven-version: 'mvn33-new'
 390
 391 - job-template:
 392     name: 'csit-dpdk-perf-verify-{stream}-{node-arch}'
 393
 394     description: |
 395       <ul>
 396           <li>objective
 397               <ul>
 398                   <li>perf verify - per csit gerrit patch
 399               </ul>
 400           <li>job
 401               <ul>
 402                   <li>name: <b>csit-dpdk-perf-verify-{stream}-{node-arch}</b>
 403                   <li>trigger: keyword in comment to csit patch:
 404                       <b>csit-dpdk-{node-arch}-perftest tag_list</b>
 405               </ul>
 406           <li>perf test-cases selection
 407               <ul>
 408                   <li>test-cases selected using trigger keyword followed by tags
 409                   <li>all test-cases are labelled with CSIT RF topic tags
 410                       defined in <i>csit/docs/tag_documentation.rst</i>
 411               </ul>
 412           <li>default tags for dpdk perf test type
 413               <ul>
 414                   <li>IP4BASE, L2XCBASE
 415                   <li>64B: 64B packet size
 416                   <li>1C: one core
 417                   <li>X710: x710 Fortville NIC
 418                   <li>MRR: MRR throughput
 419               </ul>
 420           <li>results
 421               <ul>
 422                   <li>jjb verify: PASS/FAIL
 423                   <li>present: no
 424                   <li>analyse: no
 425               </ul>
 426       </ul>
 427
 428     project-type: freestyle
 429     node: '{executor}'
 430     concurrent: true
 431     archive-artifacts: 'archive/*.*'
 432     latest-only: false
 433
 434     build-discarder:
 435       daysToKeep: '{build-days-to-keep}'
 436       numToKeep: '{build-num-to-keep}'
 437       artifactDaysToKeep: '{build-artifact-days-to-keep}'
 438       artifactNumToKeep: '{build-artifact-num-to-keep}'
 439
 440     parameters:
 441       - project-parameter:
 442           project: '{project}'
 443       - gerrit-parameter:
 444           branch: '{branch}'
 445       - string:
 446           name: ARCHIVE_ARTIFACTS
 447           default: '{archive-artifacts}'
 448           description: Artifacts to archive to the logs server.
 449
 450     scm:
 451       - gerrit-trigger-scm:
 452           credentials-id: 'jenkins-gerrit-credentials'
 453           refspec: '$GERRIT_REFSPEC'
 454           choosing-strategy: 'gerrit'
 455
 456     wrappers:
 457       - fdio-infra-wrappers-non-activity-timeout:
 458           build-timeout: '{build-timeout}'
 459
 460     triggers:
 461       - gerrit:
 462           server-name: 'Primary'
 463           trigger-on:
 464             - comment-added-contains-event:
 465                 comment-contains-value: 'csit-dpdk-{node-arch}-perftest'
 466           projects:
 467             - project-compare-type: 'ANT'
 468               project-pattern: '{project}'
 469               branches:
 470                 - branch-compare-type: 'ANT'
 471                   branch-pattern: '**/{branch}'
 472           skip-vote:
 473             successful: false
 474             failed: false
 475             unstable: false
 476             notbuilt: false
 477
 478     builders:
 479       - shell:
 480           !include-raw-escape: jjb/csit/include-raw-csit-perf-verify.sh
 481
 482     publishers:
 483       - robot:
 484           output-path: archive
 485           other-files:
 486             - '*.*'
 487
 488       - fdio-infra-shiplogs:
 489           maven-version: 'mvn33-new'
 490
 491 - job-template:
 492     name: 'csit-dpdk-perf-mrr-weekly-master{node-arch}'
 493
 494     description: |
 495       <ul>
 496           <li>objective
 497               <ul>
 498                   <li>check dpdk perf mrr weekly
 499               </ul>
 500           <li>perf test-cases selection
 501               <ul>
 502                   <li>all CSIT DPDK MRR perf suites
 503               </ul>
 504           <li>results
 505               <ul>
 506                   <li>jjb verify: PASS/FAIL
 507                   <li>present: no
 508                   <li>analyse: csit project (catch malfunctions/degradations)
 509               </ul>
 510       </ul>
 511
 512     project-type: freestyle
 513     node: '{executor}'
 514     concurrent: false
 515     archive-artifacts: 'csit/archive/*.*'
 516     latest-only: false
 517
 518     build-discarder:
 519       daysToKeep: '{build-days-to-keep}'
 520       numToKeep: '{build-num-to-keep}'
 521       artifactDaysToKeep: '{build-artifact-days-to-keep}'
 522       artifactNumToKeep: '{build-artifact-num-to-keep}'
 523
 524     parameters:
 525       - project-parameter:
 526           project: '{project}'
 527       - gerrit-parameter:
 528           branch: '{branch}'
 529       - string:
 530           name: ARCHIVE_ARTIFACTS
 531           default: '{archive-artifacts}'
 532           description: Artifacts to archive to the logs server.
 533
 534     wrappers:
 535       - fdio-infra-wrappers-non-activity-timeout:
 536           build-timeout: '{build-timeout}'
 537
 538     triggers:
 539       - timed: '0 0 * * 6'
 540
 541     builders:
 542       - shell: 'echo "BRANCH_ID={branch-id}" > $WORKSPACE/variables.prop'
 543       - inject:
 544           properties-file: variables.prop
 545       - shell:
 546           !include-raw-escape: jjb/csit/include-raw-csit-perf-timed.sh
 547
 548     publishers:
 549       - robot:
 550           output-path: 'csit/archive'
 551           other-files:
 552             - '*.*'
 553
 554       - fdio-infra-shiplogs:
 555           maven-version: 'mvn33-new'
 556
 557 - job-template:
 558     name: 'csit-ligato-perf-verify-{stream}-{node-arch}'
 559
 560     # yamllint disable rule:line-length
 561     description: |
 562       <ul>
 563           <li>objective
 564               <ul>
 565                   <li>perf verify - per csit gerrit patch
 566               </ul>
 567           <li>job
 568               <ul>
 569                   <li>name: <b>csit-ligato-perf-verify-{stream}-{node-arch}</b>
 570                   <li>trigger: keyword in comment to csit patch:
 571                       <b>csit--ligato-{node-arch}-perftest tag_list</b>
 572               </ul>
 573           <li>perf test-cases selection
 574               <ul>
 575                   <li>test-cases selected using trigger keyword followed by tags
 576                   <li>all test-cases are labelled with CSIT RF topic tags
 577                       defined in <i>csit/docs/tag_documentation.rst</i>
 578               </ul>
 579           <li>proposed keywords
 580               <ul>
 581                   <li>focus on testing specified Kubernetes orchestrated topologies
 582                   <li>format: csit-ligato-{node-arch}-perftest tag_list
 583               </ul>
 584           <li>default tags for vpp startup config and perf test type
 585               <ul>
 586                   <li>IP4BASE, IP6BASE, L2BDBASE
 587                   <li>64B/78B: 64B/78B packet size
 588                   <li>1C: one core
 589                   <li>X710: x710 Fortville NIC
 590                   <li>MRR: MRR throughput
 591               </ul>
 592           <li>results
 593               <ul>
 594                   <li>jjb verify: PASS/FAIL
 595                   <li>present: no
 596                   <li>analyse: no
 597               </ul>
 598       </ul>
 599       # yamllint enable rule:line-length
 600
 601     project-type: freestyle
 602     # As the nomad executors are running docker containers we cannot run
 603     # ligato-perf job due to dependency on using docker inside.
 604     # TODO: Find the mechanics how to offload job logic from compiling docker
 605     #   image so we can move this executor to ubuntu1804-us
 606     node: '{ligato-executor}'
 607     concurrent: true
 608     archive-artifacts: 'archive/*.*'
 609     latest-only: false
 610
 611     build-discarder:
 612       daysToKeep: '{build-days-to-keep}'
 613       numToKeep: '{build-num-to-keep}'
 614       artifactDaysToKeep: '{build-artifact-days-to-keep}'
 615       artifactNumToKeep: '{build-artifact-num-to-keep}'
 616
 617     parameters:
 618       - project-parameter:
 619           project: '{project}'
 620       - gerrit-parameter:
 621           branch: '{branch}'
 622       - gerrit-refspec-parameter
 623       - gerrit-event-type-parameter
 624       - gerrit-event-comment-text-parameter
 625       - string:
 626           name: ARCHIVE_ARTIFACTS
 627           default: '{archive-artifacts}'
 628           description: Artifacts to archive to the logs server.
 629
 630     scm:
 631       - gerrit-trigger-scm:
 632           credentials-id: 'jenkins-gerrit-credentials'
 633           refspec: '$GERRIT_REFSPEC'
 634           choosing-strategy: 'gerrit'
 635
 636     wrappers:
 637       - fdio-infra-wrappers-non-activity-timeout:
 638           build-timeout: '{build-timeout}'
 639
 640     triggers:
 641       - gerrit:
 642           server-name: 'Primary'
 643           trigger-on:
 644             - comment-added-contains-event:
 645                 comment-contains-value: 'csit-ligato-{node-arch}-perftest'
 646           projects:
 647             - project-compare-type: 'ANT'
 648               project-pattern: '{project}'
 649               branches:
 650                 - branch-compare-type: 'ANT'
 651                   branch-pattern: '**/{branch}'
 652           skip-vote:
 653             successful: false
 654             failed: false
 655             unstable: false
 656             notbuilt: false
 657
 658     builders:
 659       - shell:
 660           !include-raw-escape: jjb/csit/include-raw-csit-perf-verify.sh
 661
 662     publishers:
 663       - robot:
 664           output-path: archive
 665           other-files:
 666             - '*.*'
 667
 668       - fdio-infra-shiplogs:
 669           maven-version: 'mvn33-new'