X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=jjb%2Fglobal-macros.yaml;h=03b99f5ecda091f9ee0897f1c6cce8145814f041;hb=bad9b7fbbd14af0aeb6c035f2f11d316945d52eb;hp=3a9d906214e7c84312a819c226fb5a1d6776f5e2;hpb=56276ce25c32ae0516b37598c33116768dfbca96;p=ci-management.git diff --git a/jjb/global-macros.yaml b/jjb/global-macros.yaml index 3a9d90621..03b99f5ec 100644 --- a/jjb/global-macros.yaml +++ b/jjb/global-macros.yaml @@ -1,80 +1,126 @@ +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + --- -# Global macros +# Global macros. #### PARAMETERS +# Keep 'parameter' entries ordered alphabetically, please. - parameter: - name: project-parameter + name: arch-parameter parameters: - string: - name: PROJECT - default: '{project}' - # yamllint disable-line rule:line-length - description: "JJB configured PROJECT parameter to identify a Gerrit project" + name: ARCH + default: '{arch}' + description: "Architecture parameter" - parameter: - name: gerrit-parameter + name: archive-artifacts-parameter parameters: - string: - name: GERRIT_BRANCH - default: '{branch}' - description: "JJB configured GERRIT_BRANCH parameter" + name: ARCHIVE_ARTIFACTS + default: '{artifacts}' + description: "Artifacts to archive on the log server." - parameter: - name: gerrit-project-parameter + name: compiler-parameter parameters: - string: - name: GERRIT_PROJECT - default: '{project}' - description: "GERRIT_PROJECT parameter if not given by trigger" + name: CC + default: '{cc}' + description: "CC parameter, can be gnu or clang" - parameter: - name: os-parameter + name: make-parallel-jobs-parameter parameters: - string: - name: OS - default: '{os}' - description: "OS parameter" + name: MAKE_PARALLEL_JOBS + default: '{make-parallel-jobs}' + description: "The number of cores to use for builds." - parameter: - name: gerrit-refspec-parameter + name: csit-fail-on-crc-mismatch-parameter parameters: - string: - name: GERRIT_REFSPEC - default: '{refspec}' - description: "GERRIT_REFSPEC parameter not given by trigger" + name: FDIO_CSIT_FAIL_ON_CRC_MISMATCH + default: '{fail-on-crc-mismatch}' + description: "If true, CRC is checked, else compatibility is tested." - parameter: - name: maven-project-parameter + name: csit-perf-trial-duration-parameter parameters: - string: - name: MAVEN_SELECTOR - default: '{maven}' - description: 'Maven selector to be used by shell scripts' + name: CSIT_PERF_TRIAL_DURATION + default: "{csit-perf-trial-duration}" + description: "Duration of a trial measurement in BMRR tests." - parameter: - name: maven-exec + name: csit-perf-trial-multiplicity-parameter parameters: - string: - name: MVN + name: CSIT_PERF_TRIAL_MULTIPLICITY + default: "{csit-perf-trial-multiplicity}" + description: "Number of subsamples for BMRR tests." + +- parameter: + name: gerrit-csit-refspec-parameter + parameters: + - string: + name: CSIT_REF + default: '' # yamllint disable-line rule:line-length - default: '/w/tools/hudson.tasks.Maven_MavenInstallation/{maven-version}/bin/mvn' - description: 'Maven selector to be used by shell scripts' + description: "GERRIT_REFSPEC parameter to override separately cloned CSIT repository, leave empty for autodetection / no override." - parameter: - name: repo-name-parameter + name: gerrit-event-comment-text-parameter parameters: - string: - name: REPO_NAME - default: '{repo-name}' - description: 'Name of repo to which to publish packaging' + name: GERRIT_EVENT_COMMENT_TEXT + default: '' + # yamllint disable-line rule:line-length + description: "GERRIT_EVENT_COMMENT_TEXT parameter not given by trigger" - parameter: - name: compiler-parameter + name: gerrit-event-type-parameter parameters: - string: - name: CC - default: '{cc}' - description: "CC parameter, can be gnu or clang" + name: GERRIT_EVENT_TYPE + default: '' + description: "GERRIT_EVENT_TYPE parameter not given by trigger" + +- parameter: + name: gerrit-parameter + parameters: + - string: + name: GERRIT_BRANCH + default: '{branch}' + description: "JJB configured GERRIT_BRANCH parameter" + +- parameter: + name: gerrit-project-parameter + parameters: + - string: + name: GERRIT_PROJECT + default: '{project}' + description: "GERRIT_PROJECT parameter if not given by trigger" + +- parameter: + name: gerrit-refspec-parameter + parameters: + - string: + name: GERRIT_REFSPEC + default: '{refspec}' + description: "GERRIT_REFSPEC parameter not given by trigger" - parameter: name: is-csit-vpp-job-parameter @@ -85,94 +131,96 @@ description: "Parameter is True if this is a CSIT vpp job." - parameter: - name: stream-parameter + name: maven-exec parameters: - string: - name: STREAM - default: '{stream}' - description: "Stream job parameter to be used in shell scripts." + name: MVN + # yamllint disable-line rule:line-length + default: '/w/tools/hudson.tasks.Maven_MavenInstallation/{maven-version}/bin/mvn' + description: 'Maven selector to be used by shell scripts' - parameter: - name: topo-parameter + name: maven-project-parameter parameters: - string: - name: TOPO - default: '{topo}' - description: "Topology parameter" + name: MAVEN_SELECTOR + default: '{maven}' + description: 'Maven selector to be used by shell scripts' - parameter: - name: arch-parameter + name: mode-parameter parameters: - string: - name: ARCH - default: '{arch}' - description: "Architecture parameter" + name: MODE + default: '{mode}' + description: "mode parameter to be used in shell scripts." - parameter: - name: csit-fail-on-crc-mismatch-parameter + name: os-parameter parameters: - string: - name: FDIO_CSIT_FAIL_ON_CRC_MISMATCH - default: '{fail-on-crc-mismatch}' - description: "If true, CRC is checked, else compatibility is tested." + name: OS + default: '{os}' + description: "OS parameter" - parameter: - name: gerrit-event-type-parameter + name: project-parameter parameters: - string: - name: GERRIT_EVENT_TYPE - default: '' - description: "GERRIT_EVENT_TYPE parameter not given by trigger" + name: PROJECT + default: '{project}' + # yamllint disable-line rule:line-length + description: "JJB configured PROJECT parameter to identify a Gerrit project" - parameter: - name: gerrit-event-comment-text-parameter + name: repo-name-parameter parameters: - string: - name: GERRIT_EVENT_COMMENT_TEXT - default: '' - # yamllint disable-line rule:line-length - description: "GERRIT_EVENT_COMMENT_TEXT parameter not given by trigger" + name: REPO_NAME + default: '{repo-name}' + description: 'Name of repo to which to publish packaging' - parameter: - name: gerrit-csit-refspec-parameter + name: stream-parameter parameters: - string: - name: CSIT_REF - default: '' - # yamllint disable-line rule:line-length - description: "GERRIT_REFSPEC parameter to override separately cloned CSIT repository, leave empty for autodetection / no override." + name: STREAM + default: '{stream}' + description: "Stream job parameter to be used in shell scripts." - parameter: - name: csit-perf-trial-duration-parameter + name: topo-parameter parameters: - string: - name: CSIT_PERF_TRIAL_DURATION - default: "{csit-perf-trial-duration}" - description: "Duration of a trial measurement in BMRR tests." + name: TOPO + default: '{topo}' + description: "Topology parameter" - parameter: - name: csit-perf-trial-multiplicity-parameter + name: workspace-archive-artifacts-parameter parameters: - string: - name: CSIT_PERF_TRIAL_MULTIPLICITY - default: "{csit-perf-trial-multiplicity}" - description: "Number of subsamples for BMRR tests." + name: WS_ARCHIVE_ARTIFACTS + default: '{artifacts}' + description: "Artifacts to archive in $WORKSPACE/archives." +# [end] Keep 'parameter' entries ordered alphabetically, please. -##### SCMS +#### SCMS +# Keep 'scm' entries ordered alphabetically, please. - scm: - name: git-scm + name: gerrit-trigger-scm scm: - git: credentials-id: '{credentials-id}' url: '$GIT_BASE' - refspec: '' + refspec: '{refspec}' branches: - - 'origin/{branch}' + - 'origin/$GERRIT_BRANCH' skip-tag: true - wipe-workspace: true + choosing-strategy: '{choosing-strategy}' - scm: - name: gerrit-trigger-scm + name: gerrit-trigger-scm-recursive scm: - git: credentials-id: '{credentials-id}' @@ -182,22 +230,24 @@ - 'origin/$GERRIT_BRANCH' skip-tag: true choosing-strategy: '{choosing-strategy}' + submodule: + recursive: '{submodule-recursive}' - scm: - name: gerrit-trigger-scm-recursive + name: git-scm scm: - git: credentials-id: '{credentials-id}' url: '$GIT_BASE' - refspec: '{refspec}' + refspec: '' branches: - - 'origin/$GERRIT_BRANCH' + - 'origin/{branch}' skip-tag: true - choosing-strategy: '{choosing-strategy}' - submodule: - recursive: '{submodule-recursive}' + wipe-workspace: true +# [end] Keep 'scm' entries ordered alphabetically, please. -###### WRAPPERS +#### WRAPPERS +# Keep 'wrapper' entries ordered alphabetically, please. - wrapper: name: build-timeout wrappers: @@ -207,12 +257,14 @@ fail: true - wrapper: - # This wrapper is used for all jobs that require no-activity timeouts - name: fdio-infra-wrappers-non-activity-timeout + # This wrapper is required for all jobs as it configures the wrappers + # needed by the fd.io infra. + name: fdio-infra-wrappers wrappers: - timeout: - type: no-activity + type: absolute timeout: '{build-timeout}' + timeout-var: 'BUILD_TIMEOUT' fail: true - timestamps - ssh-agent-credentials: @@ -224,14 +276,12 @@ single-use: true - wrapper: - # This wrapper is required for all jobs as it configures the wrappers - # needed by the fd.io infra. - name: fdio-infra-wrappers + # This wrapper is used for all jobs that require no-activity timeouts + name: fdio-infra-wrappers-non-activity-timeout wrappers: - timeout: - type: absolute + type: no-activity timeout: '{build-timeout}' - timeout-var: 'BUILD_TIMEOUT' fail: true - timestamps - ssh-agent-credentials: @@ -241,23 +291,20 @@ single-use: true - openstack: single-use: true +# [end] Keep 'wrapper' entries ordered alphabetically, please. -###### TRIGGERS +#### TRIGGERS +# Keep 'trigger' entries ordered alphabetically, please. - trigger: - name: gerrit-trigger-patch-submitted + name: gerrit-trigger-checkstyle + # Comes from VPP, some verify jobs wait for checkstyle to pass. + # This macro is to be used for such verify jobs which are voting. triggers: - gerrit: server-name: 'Primary' trigger-on: - - patchset-created-event: - exclude-drafts: 'true' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'false' - - draft-published-event - comment-added-contains-event: - comment-contains-value: 'recheck' - - comment-added-contains-event: - comment-contains-value: 'reverify' + comment-contains-value: 'checkstyle_success' projects: - project-compare-type: 'ANT' project-pattern: '{name}' @@ -266,70 +313,102 @@ branch-pattern: '**/{branch}' - trigger: - name: gerrit-trigger-trivial-patch-submitted + name: gerrit-trigger-checkstyle-nonvoting + # Comes from VPP, some verify jobs wait for checkstyle to pass. + # This macro is to be used for such verify jobs which are non-voting. triggers: - gerrit: server-name: 'Primary' trigger-on: - - patchset-created-event: - exclude-drafts: 'true' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'true' - - draft-published-event - - comment-added-contains-event: - comment-contains-value: 'recheck' - comment-added-contains-event: - comment-contains-value: 'reverify' + comment-contains-value: 'checkstyle_success' projects: - project-compare-type: 'ANT' project-pattern: '{name}' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' + skip-vote: + successful: true + failed: true + unstable: true + notbuilt: true + override-votes: true + gerrit-build-successful-verified-value: 1 + gerrit-build-failed-verified-value: 1 + gerrit-build-unstable-verified-value: 1 + gerrit-build-notbuilt-verified-value: 1 + +- trigger: + name: gerrit-trigger-csit-perftest + # Manual trigger common for csit-vpp and vpp-csit perf jobs. + triggers: + - gerrit: + server-name: 'Primary' + trigger-on: + - comment-added-contains-event: + comment-contains-value: '{trigger-word}' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + skip-vote: + successful: '{skip-vote}' + failed: '{skip-vote}' + unstable: '{skip-vote}' + notbuilt: '{skip-vote}' - trigger: - name: gerrit-trigger-os-trivial-patch-submitted + name: gerrit-trigger-csit-vpp-devicetest-full + # Comes from CSIT, for automatic jobs running device tests with VPP builds. + # The job acts as an automatic verify job for CSIT contributions. triggers: - gerrit: server-name: 'Primary' trigger-on: + # The main way the job is triggered automatically on patch upload. + # Gated by tox job, so we do not spend resources on WiP changes. - comment-added-contains-event: - comment-contains-value: 'os-check' + comment-contains-value: 'tox_success' + # Manual trigger to allow testing WiP changes with selected tags. - comment-added-contains-event: - comment-contains-value: 'os-verify' + comment-contains-value: 'csit-devicetest' + # A comment with "recheck" restarts tox job, so no condition here. projects: - project-compare-type: 'ANT' - project-pattern: '{name}' + project-pattern: '{project}' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' + skip-vote: + successful: '{skip-vote}' + failed: '{skip-vote}' + unstable: '{skip-vote}' + notbuilt: '{skip-vote}' - trigger: - name: gerrit-trigger-trivial-patch-submitted-skip-vote + name: gerrit-trigger-csit-vpp-devicetest-manual + # Comes from CSIT, for manual jobs running "device" tests with VPP builds. + # The job acts as a manual (optional) verify job for CSIT contributions. triggers: - gerrit: server-name: 'Primary' trigger-on: - - patchset-created-event: - exclude-drafts: 'true' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'true' - - draft-published-event - - comment-added-contains-event: - comment-contains-value: 'recheck' - comment-added-contains-event: - comment-contains-value: 'reverify' + comment-contains-value: 'csit-devicetest' projects: - project-compare-type: 'ANT' - project-pattern: '{name}' + project-pattern: '{project}' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' skip-vote: - successful: true - failed: true - unstable: true - notbuilt: true + successful: '{skip-vote}' + failed: '{skip-vote}' + unstable: '{skip-vote}' + notbuilt: '{skip-vote}' - trigger: name: gerrit-trigger-manually-triggered @@ -375,6 +454,28 @@ gerrit-build-unstable-codereview-value: 0 gerrit-build-notbuilt-codereview-value: 0 +- trigger: + name: gerrit-trigger-patch-submitted + triggers: + - gerrit: + server-name: 'Primary' + trigger-on: + - patchset-created-event: + exclude-drafts: 'true' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{name}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + - trigger: name: gerrit-trigger-os-patch-merged triggers: @@ -403,7 +504,121 @@ gerrit-build-unstable-codereview-value: 0 gerrit-build-notbuilt-codereview-value: 0 -# Publishers +- trigger: + name: gerrit-trigger-os-trivial-patch-submitted + triggers: + - gerrit: + server-name: 'Primary' + trigger-on: + - comment-added-contains-event: + comment-contains-value: 'os-check' + - comment-added-contains-event: + comment-contains-value: 'os-verify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{name}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + +- trigger: + name: gerrit-trigger-trivial-patch-submitted + triggers: + - gerrit: + server-name: 'Primary' + trigger-on: + - patchset-created-event: + exclude-drafts: 'true' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'true' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{name}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + +- trigger: + name: gerrit-trigger-trivial-patch-submitted-skip-vote + triggers: + - gerrit: + server-name: 'Primary' + trigger-on: + - patchset-created-event: + exclude-drafts: 'true' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'true' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{name}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + skip-vote: + successful: true + failed: true + unstable: true + notbuilt: true + +- trigger: + name: gerrit-trigger-vpp-csit-devicetest-full + # Comes from VPP, for automated jobs running "device" tests with CSIT code. + # The job acts as an automatic verify job for VPP contributions. + triggers: + - gerrit: + server-name: 'Primary' + trigger-on: + - comment-added-contains-event: + comment-contains-value: 'checkstyle_success' + - comment-added-contains-event: + comment-contains-value: 'devicetest' + projects: + - project-compare-type: 'ANT' + project-pattern: '{name}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + skip-vote: + successful: '{skip-vote}' + failed: '{skip-vote}' + unstable: '{skip-vote}' + notbuilt: '{skip-vote}' + +- trigger: + name: gerrit-trigger-vpp-csit-devicetest-manual + # Comes from VPP, for manual jobs running "device" tests with CSIT code. + # The job acts as a manual (optional) verify job for VPP contributions. + triggers: + - gerrit: + server-name: 'Primary' + trigger-on: + - comment-added-contains-event: + comment-contains-value: 'devicetest' + projects: + - project-compare-type: 'ANT' + project-pattern: '{name}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + skip-vote: + successful: '{skip-vote}' + failed: '{skip-vote}' + unstable: '{skip-vote}' + notbuilt: '{skip-vote}' +# [end] Keep 'trigger' entries ordered alphabetically, please. + +#### PUBLISHERS +# Keep 'publisher' entries ordered alphabetically, please. - publisher: name: archive-artifacts publishers: @@ -433,29 +648,33 @@ - recipients - publisher: - name: jacoco-report - publishers: - - jacoco: - exec-pattern: '**/**.exec' - class-pattern: '**/classes' - source-pattern: '**/src/main/java' - # yamllint disable-line rule:line-length - exclusion-pattern: '**/gen**,**/generated-sources/**,**/yang-gen**,**/pax/**' - status-update: true - targets: - - branch: - healthy: 10 - unhealthy: 20 - - method: - healthy: 50 - unhealthy: 40 - -- publisher: - name: robot-report + name: fdio-infra-publish + # macro to finish up a build. + # + # Handles the following: + # - Shipping logs to Nexus or S3 logs repository + # - Cleanup workspace publishers: - - robot: - output-path: '{output-path}' - other-files: '' + - postbuildscript: + builders: + - role: BOTH + build-on: + - ABORTED + - FAILURE + - NOT_BUILT + - SUCCESS + - UNSTABLE + build-steps: + - shell: !include-raw: scripts/post_build_deploy_archives.sh + - shell: !include-raw: scripts/backup_upload_archives.sh + - fdio-infra-ship-logs + mark-unstable-if-failed: true + - workspace-cleanup: + exclude: + # Do not clean up *.jenkins-trigger files for jobs that use a + # properties file as input for triggering another build. + - "**/*.jenkins-trigger" + fail-build: false - publisher: name: fdio-infra-shiplogs @@ -480,6 +699,7 @@ - UNSTABLE build-steps: - shell: !include-raw: include-raw-deploy-archives.sh + - shell: !include-raw: scripts/backup_upload_archives.sh - maven-target: maven-version: '{maven-version}' pom: '.archives/deploy-archives.xml' @@ -494,29 +714,46 @@ - workspace-cleanup: fail-build: false +- publisher: + name: jacoco-report + publishers: + - jacoco: + exec-pattern: '**/**.exec' + class-pattern: '**/classes' + source-pattern: '**/src/main/java' + # yamllint disable-line rule:line-length + exclusion-pattern: '**/gen**,**/generated-sources/**,**/yang-gen**,**/pax/**' + status-update: true + targets: + - branch: + healthy: 10 + unhealthy: 20 + - method: + healthy: 50 + unhealthy: 40 + - publisher: name: retry-build-on-builder-error publishers: - naginator: max-failed-builds: 1 -###### BUILDERS +- publisher: + name: robot-report + publishers: + - robot: + output-path: '{output-path}' + other-files: '' +# [end] Keep 'publisher' entries ordered alphabetically, please. + +#### BUILDERS +# Keep 'builder' entries ordered alphabetically, please. - builder: name: ci-management-check-unicode builders: - shell: | $WORKSPACE/scripts/check-unicode.sh jjb/ -- builder: - name: provide-maven-settings - builders: - - config-file-provider: - files: - - file-id: '{global-settings-file}' - variable: 'GLOBAL_SETTINGS_FILE' - - file-id: '{settings-file}' - variable: 'SETTINGS_FILE' - - property: name: fdio-infra-properties properties: @@ -524,6 +761,33 @@ days-to-keep: '{build-days-to-keep}' num-to-keep: '{build-num-to-keep}' +- builder: + name: fdio-infra-ship-logs + builders: + # Ensure no pre-existing .netrc files are overriding logs config + - lf-provide-maven-settings-cleanup + - config-file-provider: + files: + - file-id: "jenkins-log-archives-settings" + variable: "SETTINGS_FILE" + - conditional-step: + condition-kind: regex-match + regex: "^.*logs-s3.*" + label: $S3_BUCKET + on-evaluation-failure: dont-run + steps: + - config-file-provider: + files: + - file-id: "jenkins-s3-log-ship" + target: $HOME/.aws/credentials + - lf-infra-create-netrc: + server-id: logs + - shell: !include-raw: + - ../global-jjb/shell/logs-deploy.sh + - shell: !include-raw: + - ../global-jjb/shell/logs-clear-credentials.sh + - description-setter: + regexp: "(^(Nexus|S3) build logs: .*)" - builder: name: packer-validate @@ -574,3 +838,14 @@ packer.io build -var-file=$CLOUDENV \ -var-file=../packer/vars/{platform}.json \ ../packer/templates/{template}.json + +- builder: + name: provide-maven-settings + builders: + - config-file-provider: + files: + - file-id: '{global-settings-file}' + variable: 'GLOBAL_SETTINGS_FILE' + - file-id: '{settings-file}' + variable: 'SETTINGS_FILE' +# [end] Keep 'builder' entries ordered alphabetically, please.