X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;ds=sidebyside;f=jjb%2Fglobal-macros.yaml;h=1146607f1f2535c883788bf47ecc861b347edf04;hb=b9b2d29a77be841909d61ad2533a1c2d552d102e;hp=a953421966f5d2cd93d8e86435e133cbef50256d;hpb=6aad83f2d2f3d57129732153492ca8121f0dab0a;p=ci-management.git diff --git a/jjb/global-macros.yaml b/jjb/global-macros.yaml index a95342196..1146607f1 100644 --- a/jjb/global-macros.yaml +++ b/jjb/global-macros.yaml @@ -1,15 +1,110 @@ +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + --- -# Global macros +# Global macros. #### PARAMETERS +# Keep 'parameter' entries ordered alphabetically, please. - parameter: - name: project-parameter + name: arch-parameter parameters: - string: - name: PROJECT - default: '{project}' + name: ARCH + default: '{arch}' + description: "Architecture parameter" + +- parameter: + name: archive-artifacts-parameter + parameters: + - string: + name: ARCHIVE_ARTIFACTS + default: '{artifacts}' + description: "Artifacts to archive on the log server." + +- parameter: + name: compiler-parameter + parameters: + - string: + name: CC + default: '{cc}' + description: "CC parameter, can be gnu or clang" + +- parameter: + name: make-parallel-jobs-parameter + parameters: + - string: + name: MAKE_PARALLEL_JOBS + default: '{make-parallel-jobs}' + description: "The number of cores to use for builds." + +- parameter: + name: make-test-os-parameter + parameters: + - string: + name: MAKE_TEST_OS + default: '{make-test-os}' + description: "The OS on which 'make test' is run." + +- parameter: + name: csit-fail-on-crc-mismatch-parameter + parameters: + - string: + name: FDIO_CSIT_FAIL_ON_CRC_MISMATCH + default: '{fail-on-crc-mismatch}' + description: "If true, CRC is checked, else compatibility is tested." + +- parameter: + name: csit-perf-trial-duration-parameter + parameters: + - string: + name: CSIT_PERF_TRIAL_DURATION + default: "{csit-perf-trial-duration}" + description: "Duration of a trial measurement in BMRR tests." + +- parameter: + name: csit-perf-trial-multiplicity-parameter + parameters: + - string: + name: CSIT_PERF_TRIAL_MULTIPLICITY + default: "{csit-perf-trial-multiplicity}" + description: "Number of subsamples for BMRR tests." + +- parameter: + name: gerrit-csit-refspec-parameter + parameters: + - string: + name: CSIT_REF + default: '' # yamllint disable-line rule:line-length - description: "JJB configured PROJECT parameter to identify a Gerrit project" + description: "GERRIT_REFSPEC parameter to override separately cloned CSIT repository, leave empty for autodetection / no override." + +- parameter: + name: gerrit-event-comment-text-parameter + parameters: + - string: + name: GERRIT_EVENT_COMMENT_TEXT + default: '' + # yamllint disable-line rule:line-length + description: "GERRIT_EVENT_COMMENT_TEXT parameter not given by trigger" + +- parameter: + name: gerrit-event-type-parameter + parameters: + - string: + name: GERRIT_EVENT_TYPE + default: '' + description: "GERRIT_EVENT_TYPE parameter not given by trigger" - parameter: name: gerrit-parameter @@ -27,14 +122,6 @@ default: '{project}' description: "GERRIT_PROJECT parameter if not given by trigger" -- parameter: - name: os-parameter - parameters: - - string: - name: OS - default: '{os}' - description: "OS parameter" - - parameter: name: gerrit-refspec-parameter parameters: @@ -44,12 +131,12 @@ description: "GERRIT_REFSPEC parameter not given by trigger" - parameter: - name: maven-project-parameter + name: is-csit-vpp-job-parameter parameters: - string: - name: MAVEN_SELECTOR - default: '{maven}' - description: 'Maven selector to be used by shell scripts' + name: IS_CSIT_VPP_JOB + default: '{is-csit-vpp-job}' + description: "Parameter is True if this is a CSIT vpp job." - parameter: name: maven-exec @@ -61,102 +148,87 @@ description: 'Maven selector to be used by shell scripts' - parameter: - name: repo-name-parameter - parameters: - - string: - name: REPO_NAME - default: '{repo-name}' - description: 'Name of repo to which to publish packaging' - -- parameter: - name: compiler-parameter - parameters: - - string: - name: CC - default: '{cc}' - description: "CC parameter, can be gnu or clang" - -- parameter: - name: is-csit-vpp-job-parameter + name: maven-project-parameter parameters: - string: - name: IS_CSIT_VPP_JOB - default: '{is-csit-vpp-job}' - description: "Parameter is True if this is a CSIT vpp job." + name: MAVEN_SELECTOR + default: '{maven}' + description: 'Maven selector to be used by shell scripts' - parameter: - name: stream-parameter + name: mode-parameter parameters: - string: - name: STREAM - default: '{stream}' - description: "Stream job parameter to be used in shell scripts." + name: MODE + default: '{mode}' + description: "mode parameter to be used in shell scripts." - parameter: - name: topo-parameter + name: os-parameter parameters: - string: - name: TOPO - default: '{topo}' - description: "Topology parameter" + name: OS + default: '{os}' + description: "OS parameter" - parameter: - name: arch-parameter + name: project-parameter parameters: - string: - name: ARCH - default: '{arch}' - description: "Architecture parameter" + name: PROJECT + default: '{project}' + # yamllint disable-line rule:line-length + description: "JJB configured PROJECT parameter to identify a Gerrit project" - parameter: - name: csit-fail-on-crc-mismatch-parameter + name: repo-name-parameter parameters: - string: - name: FDIO_CSIT_FAIL_ON_CRC_MISMATCH - default: '{fail-on-crc-mismatch}' - description: "If true, CRC is checked, else compatibility is tested." + name: REPO_NAME + default: '{repo-name}' + description: 'Name of repo to which to publish packaging' - parameter: - name: gerrit-event-type-parameter + name: stream-parameter parameters: - string: - name: GERRIT_EVENT_TYPE - default: '' - description: "GERRIT_EVENT_TYPE parameter not given by trigger" + name: STREAM + default: '{stream}' + description: "Stream job parameter to be used in shell scripts." - parameter: - name: gerrit-event-comment-text-parameter + name: topo-parameter parameters: - string: - name: GERRIT_EVENT_COMMENT_TEXT - default: '' - # yamllint disable-line rule:line-length - description: "GERRIT_EVENT_COMMENT_TEXT parameter not given by trigger" + name: TOPO + default: '{topo}' + description: "Topology parameter" - parameter: - name: gerrit-csit-refspec-parameter + name: workspace-archive-artifacts-parameter parameters: - string: - name: CSIT_REF - default: '' - # yamllint disable-line rule:line-length - description: "GERRIT_REFSPEC parameter to override separately cloned CSIT repository, leave empty for autodetection / no override." + name: WS_ARCHIVE_ARTIFACTS + default: '{artifacts}' + description: "Artifacts to archive in $WORKSPACE/archives." +# [end] Keep 'parameter' entries ordered alphabetically, please. -##### SCMS +#### SCMS +# Keep 'scm' entries ordered alphabetically, please. - scm: - name: git-scm + name: gerrit-trigger-scm scm: - git: credentials-id: '{credentials-id}' url: '$GIT_BASE' - refspec: '' + refspec: '{refspec}' branches: - - 'origin/{branch}' + - 'origin/$GERRIT_BRANCH' skip-tag: true - wipe-workspace: true + choosing-strategy: '{choosing-strategy}' - scm: - name: gerrit-trigger-scm + name: gerrit-trigger-scm-recursive scm: - git: credentials-id: '{credentials-id}' @@ -166,22 +238,24 @@ - 'origin/$GERRIT_BRANCH' skip-tag: true choosing-strategy: '{choosing-strategy}' + submodule: + recursive: '{submodule-recursive}' - scm: - name: gerrit-trigger-scm-recursive + name: git-scm scm: - git: credentials-id: '{credentials-id}' url: '$GIT_BASE' - refspec: '{refspec}' + refspec: '' branches: - - 'origin/$GERRIT_BRANCH' + - 'origin/{branch}' skip-tag: true - choosing-strategy: '{choosing-strategy}' - submodule: - recursive: '{submodule-recursive}' + wipe-workspace: true +# [end] Keep 'scm' entries ordered alphabetically, please. -###### WRAPPERS +#### WRAPPERS +# Keep 'wrapper' entries ordered alphabetically, please. - wrapper: name: build-timeout wrappers: @@ -191,12 +265,14 @@ fail: true - wrapper: - # This wrapper is used for all jobs that require no-activity timeouts - name: fdio-infra-wrappers-non-activity-timeout + # This wrapper is required for all jobs as it configures the wrappers + # needed by the fd.io infra. + name: fdio-infra-wrappers wrappers: - timeout: - type: no-activity + type: absolute timeout: '{build-timeout}' + timeout-var: 'BUILD_TIMEOUT' fail: true - timestamps - ssh-agent-credentials: @@ -208,14 +284,12 @@ single-use: true - wrapper: - # This wrapper is required for all jobs as it configures the wrappers - # needed by the fd.io infra. - name: fdio-infra-wrappers + # This wrapper is used for all jobs that require no-activity timeouts + name: fdio-infra-wrappers-non-activity-timeout wrappers: - timeout: - type: absolute + type: no-activity timeout: '{build-timeout}' - timeout-var: 'BUILD_TIMEOUT' fail: true - timestamps - ssh-agent-credentials: @@ -225,23 +299,20 @@ single-use: true - openstack: single-use: true +# [end] Keep 'wrapper' entries ordered alphabetically, please. -###### TRIGGERS +#### TRIGGERS +# Keep 'trigger' entries ordered alphabetically, please. - trigger: - name: gerrit-trigger-patch-submitted + name: gerrit-trigger-checkstyle + # Comes from VPP, some verify jobs wait for checkstyle to pass. + # This macro is to be used for such verify jobs which are voting. triggers: - gerrit: server-name: 'Primary' trigger-on: - - patchset-created-event: - exclude-drafts: 'true' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'false' - - draft-published-event - comment-added-contains-event: - comment-contains-value: 'recheck' - - comment-added-contains-event: - comment-contains-value: 'reverify' + comment-contains-value: 'checkstyle_success' projects: - project-compare-type: 'ANT' project-pattern: '{name}' @@ -250,70 +321,102 @@ branch-pattern: '**/{branch}' - trigger: - name: gerrit-trigger-trivial-patch-submitted + name: gerrit-trigger-checkstyle-nonvoting + # Comes from VPP, some verify jobs wait for checkstyle to pass. + # This macro is to be used for such verify jobs which are non-voting. triggers: - gerrit: server-name: 'Primary' trigger-on: - - patchset-created-event: - exclude-drafts: 'true' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'true' - - draft-published-event - - comment-added-contains-event: - comment-contains-value: 'recheck' - comment-added-contains-event: - comment-contains-value: 'reverify' + comment-contains-value: 'checkstyle_success' projects: - project-compare-type: 'ANT' project-pattern: '{name}' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' + skip-vote: + successful: true + failed: true + unstable: true + notbuilt: true + override-votes: true + gerrit-build-successful-verified-value: 1 + gerrit-build-failed-verified-value: 1 + gerrit-build-unstable-verified-value: 1 + gerrit-build-notbuilt-verified-value: 1 - trigger: - name: gerrit-trigger-os-trivial-patch-submitted + name: gerrit-trigger-csit-perftest + # Manual trigger common for csit-vpp and vpp-csit perf jobs. triggers: - gerrit: server-name: 'Primary' trigger-on: - comment-added-contains-event: - comment-contains-value: 'os-check' - - comment-added-contains-event: - comment-contains-value: 'os-verify' + comment-contains-value: '{trigger-word}' projects: - project-compare-type: 'ANT' - project-pattern: '{name}' + project-pattern: '{project}' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' + skip-vote: + successful: '{skip-vote}' + failed: '{skip-vote}' + unstable: '{skip-vote}' + notbuilt: '{skip-vote}' - trigger: - name: gerrit-trigger-trivial-patch-submitted-skip-vote + name: gerrit-trigger-csit-vpp-devicetest-full + # Comes from CSIT, for automatic jobs running device tests with VPP builds. + # The job acts as an automatic verify job for CSIT contributions. triggers: - gerrit: server-name: 'Primary' trigger-on: - - patchset-created-event: - exclude-drafts: 'true' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'true' - - draft-published-event + # The main way the job is triggered automatically on patch upload. + # Gated by tox job, so we do not spend resources on WiP changes. - comment-added-contains-event: - comment-contains-value: 'recheck' + comment-contains-value: 'tox_success' + # Manual trigger to allow testing WiP changes with selected tags. - comment-added-contains-event: - comment-contains-value: 'reverify' + comment-contains-value: 'csit-devicetest' + # A comment with "recheck" restarts tox job, so no condition here. projects: - project-compare-type: 'ANT' - project-pattern: '{name}' + project-pattern: '{project}' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' skip-vote: - successful: true - failed: true - unstable: true - notbuilt: true + successful: '{skip-vote}' + failed: '{skip-vote}' + unstable: '{skip-vote}' + notbuilt: '{skip-vote}' + +- trigger: + name: gerrit-trigger-csit-vpp-devicetest-manual + # Comes from CSIT, for manual jobs running "device" tests with VPP builds. + # The job acts as a manual (optional) verify job for CSIT contributions. + triggers: + - gerrit: + server-name: 'Primary' + trigger-on: + - comment-added-contains-event: + comment-contains-value: 'csit-devicetest' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + skip-vote: + successful: '{skip-vote}' + failed: '{skip-vote}' + unstable: '{skip-vote}' + notbuilt: '{skip-vote}' - trigger: name: gerrit-trigger-manually-triggered @@ -359,6 +462,28 @@ gerrit-build-unstable-codereview-value: 0 gerrit-build-notbuilt-codereview-value: 0 +- trigger: + name: gerrit-trigger-patch-submitted + triggers: + - gerrit: + server-name: 'Primary' + trigger-on: + - patchset-created-event: + exclude-drafts: 'true' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{name}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + - trigger: name: gerrit-trigger-os-patch-merged triggers: @@ -387,7 +512,121 @@ gerrit-build-unstable-codereview-value: 0 gerrit-build-notbuilt-codereview-value: 0 -# Publishers +- trigger: + name: gerrit-trigger-os-trivial-patch-submitted + triggers: + - gerrit: + server-name: 'Primary' + trigger-on: + - comment-added-contains-event: + comment-contains-value: 'os-check' + - comment-added-contains-event: + comment-contains-value: 'os-verify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{name}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + +- trigger: + name: gerrit-trigger-trivial-patch-submitted + triggers: + - gerrit: + server-name: 'Primary' + trigger-on: + - patchset-created-event: + exclude-drafts: 'true' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'true' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{name}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + +- trigger: + name: gerrit-trigger-trivial-patch-submitted-skip-vote + triggers: + - gerrit: + server-name: 'Primary' + trigger-on: + - patchset-created-event: + exclude-drafts: 'true' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'true' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{name}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + skip-vote: + successful: true + failed: true + unstable: true + notbuilt: true + +- trigger: + name: gerrit-trigger-vpp-csit-devicetest-full + # Comes from VPP, for automated jobs running "device" tests with CSIT code. + # The job acts as an automatic verify job for VPP contributions. + triggers: + - gerrit: + server-name: 'Primary' + trigger-on: + - comment-added-contains-event: + comment-contains-value: 'checkstyle_success' + - comment-added-contains-event: + comment-contains-value: 'devicetest' + projects: + - project-compare-type: 'ANT' + project-pattern: '{name}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + skip-vote: + successful: '{skip-vote}' + failed: '{skip-vote}' + unstable: '{skip-vote}' + notbuilt: '{skip-vote}' + +- trigger: + name: gerrit-trigger-vpp-csit-devicetest-manual + # Comes from VPP, for manual jobs running "device" tests with CSIT code. + # The job acts as a manual (optional) verify job for VPP contributions. + triggers: + - gerrit: + server-name: 'Primary' + trigger-on: + - comment-added-contains-event: + comment-contains-value: 'devicetest' + projects: + - project-compare-type: 'ANT' + project-pattern: '{name}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + skip-vote: + successful: '{skip-vote}' + failed: '{skip-vote}' + unstable: '{skip-vote}' + notbuilt: '{skip-vote}' +# [end] Keep 'trigger' entries ordered alphabetically, please. + +#### PUBLISHERS +# Keep 'publisher' entries ordered alphabetically, please. - publisher: name: archive-artifacts publishers: @@ -417,29 +656,33 @@ - recipients - publisher: - name: jacoco-report - publishers: - - jacoco: - exec-pattern: '**/**.exec' - class-pattern: '**/classes' - source-pattern: '**/src/main/java' - # yamllint disable-line rule:line-length - exclusion-pattern: '**/gen**,**/generated-sources/**,**/yang-gen**,**/pax/**' - status-update: true - targets: - - branch: - healthy: 10 - unhealthy: 20 - - method: - healthy: 50 - unhealthy: 40 - -- publisher: - name: robot-report + name: fdio-infra-publish + # macro to finish up a build. + # + # Handles the following: + # - Shipping logs to Nexus or S3 logs repository + # - Cleanup workspace publishers: - - robot: - output-path: '{output-path}' - other-files: '' + - postbuildscript: + builders: + - role: BOTH + build-on: + - ABORTED + - FAILURE + - NOT_BUILT + - SUCCESS + - UNSTABLE + build-steps: + - shell: !include-raw: scripts/post_build_deploy_archives.sh + - shell: !include-raw: scripts/backup_upload_archives.sh + - fdio-infra-ship-logs + mark-unstable-if-failed: true + - workspace-cleanup: + exclude: + # Do not clean up *.jenkins-trigger files for jobs that use a + # properties file as input for triggering another build. + - "**/*.jenkins-trigger" + fail-build: false - publisher: name: fdio-infra-shiplogs @@ -464,6 +707,7 @@ - UNSTABLE build-steps: - shell: !include-raw: include-raw-deploy-archives.sh + - shell: !include-raw: scripts/backup_upload_archives.sh - maven-target: maven-version: '{maven-version}' pom: '.archives/deploy-archives.xml' @@ -478,29 +722,46 @@ - workspace-cleanup: fail-build: false +- publisher: + name: jacoco-report + publishers: + - jacoco: + exec-pattern: '**/**.exec' + class-pattern: '**/classes' + source-pattern: '**/src/main/java' + # yamllint disable-line rule:line-length + exclusion-pattern: '**/gen**,**/generated-sources/**,**/yang-gen**,**/pax/**' + status-update: true + targets: + - branch: + healthy: 10 + unhealthy: 20 + - method: + healthy: 50 + unhealthy: 40 + - publisher: name: retry-build-on-builder-error publishers: - naginator: max-failed-builds: 1 -###### BUILDERS +- publisher: + name: robot-report + publishers: + - robot: + output-path: '{output-path}' + other-files: '' +# [end] Keep 'publisher' entries ordered alphabetically, please. + +#### BUILDERS +# Keep 'builder' entries ordered alphabetically, please. - builder: name: ci-management-check-unicode builders: - shell: | $WORKSPACE/scripts/check-unicode.sh jjb/ -- builder: - name: provide-maven-settings - builders: - - config-file-provider: - files: - - file-id: '{global-settings-file}' - variable: 'GLOBAL_SETTINGS_FILE' - - file-id: '{settings-file}' - variable: 'SETTINGS_FILE' - - property: name: fdio-infra-properties properties: @@ -508,6 +769,33 @@ days-to-keep: '{build-days-to-keep}' num-to-keep: '{build-num-to-keep}' +- builder: + name: fdio-infra-ship-logs + builders: + # Ensure no pre-existing .netrc files are overriding logs config + - lf-provide-maven-settings-cleanup + - config-file-provider: + files: + - file-id: "jenkins-log-archives-settings" + variable: "SETTINGS_FILE" + - conditional-step: + condition-kind: regex-match + regex: "^.*logs-s3.*" + label: $S3_BUCKET + on-evaluation-failure: dont-run + steps: + - config-file-provider: + files: + - file-id: "jenkins-s3-log-ship" + target: $HOME/.aws/credentials + - lf-infra-create-netrc: + server-id: logs + - shell: !include-raw: + - ../global-jjb/shell/logs-deploy.sh + - shell: !include-raw: + - ../global-jjb/shell/logs-clear-credentials.sh + - description-setter: + regexp: "(^(Nexus|S3) build logs: .*)" - builder: name: packer-validate @@ -558,3 +846,14 @@ packer.io build -var-file=$CLOUDENV \ -var-file=../packer/vars/{platform}.json \ ../packer/templates/{template}.json + +- builder: + name: provide-maven-settings + builders: + - config-file-provider: + files: + - file-id: '{global-settings-file}' + variable: 'GLOBAL_SETTINGS_FILE' + - file-id: '{settings-file}' + variable: 'SETTINGS_FILE' +# [end] Keep 'builder' entries ordered alphabetically, please.