X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=jjb%2Fglobal-macros.yaml;h=9c0eeacbb2e8fb12c3e222b7e02d8115de957d60;hb=HEAD;hp=f4b009afe30a965b657e88821ae34a6f6c0810d8;hpb=e9604519cc808fe304c59e804b4fb6287bbc4ac0;p=ci-management.git

diff --git a/jjb/global-macros.yaml b/jjb/global-macros.yaml
index f4b009afe..2de57c726 100644
--- a/jjb/global-macros.yaml
+++ b/jjb/global-macros.yaml
@@ -1,4 +1,4 @@
-# Copyright (c) 2020 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -24,14 +24,6 @@
           default: '{arch}'
           description: "Architecture parameter"
 
-- parameter:
-    name: archive-artifacts-parameter
-    parameters:
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: '{artifacts}'
-          description: "Artifacts to archive on the log server."
-
 - parameter:
     name: compiler-parameter
     parameters:
@@ -48,6 +40,22 @@
           default: '{make-parallel-jobs}'
           description: "The number of cores to use for builds."
 
+- parameter:
+    name: make-test-os-parameter
+    parameters:
+      - string:
+          name: MAKE_TEST_OS
+          default: '{make-test-os}'
+          description: "The OS on which 'make test' is run."
+
+- parameter:
+    name: make-test-multiworker-os-parameter
+    parameters:
+      - string:
+          name: MAKE_TEST_MULTIWORKER_OS
+          default: '{make-test-multiworker-os}'
+          description: "The OS on which 'make test' is run with multiple VPP workers."
+
 - parameter:
     name: csit-fail-on-crc-mismatch-parameter
     parameters:
@@ -172,14 +180,6 @@
           # yamllint disable-line rule:line-length
           description: "JJB configured PROJECT parameter to identify a Gerrit project"
 
-- parameter:
-    name: repo-name-parameter
-    parameters:
-      - string:
-          name: REPO_NAME
-          default: '{repo-name}'
-          description: 'Name of repo to which to publish packaging'
-
 - parameter:
     name: stream-parameter
     parameters:
@@ -195,14 +195,6 @@
           name: TOPO
           default: '{topo}'
           description: "Topology parameter"
-
-- parameter:
-    name: workspace-archive-artifacts-parameter
-    parameters:
-      - string:
-          name: WS_ARCHIVE_ARTIFACTS
-          default: '{artifacts}'
-          description: "Artifacts to archive in $WORKSPACE/archives."
 # [end] Keep 'parameter' entries ordered alphabetically, please.
 
 #### SCMS
@@ -295,6 +287,22 @@
 
 #### TRIGGERS
 # Keep 'trigger' entries ordered alphabetically, please.
+- trigger:
+    name: gerrit-trigger-build
+    # Comes from hicn, some verify jobs wait for build to pass.
+    triggers:
+      - gerrit:
+          server-name: 'Primary'
+          trigger-on:
+            - comment-added-contains-event:
+                comment-contains-value: 'build_success-{stream}-{os}-{executor-arch}'
+          projects:
+            - project-compare-type: 'ANT'
+              project-pattern: '{name}'
+              branches:
+                - branch-compare-type: 'ANT'
+                  branch-pattern: '**/{branch}'
+
 - trigger:
     name: gerrit-trigger-checkstyle
     # Comes from VPP, some verify jobs wait for checkstyle to pass.
@@ -360,25 +368,43 @@
             unstable: '{skip-vote}'
             notbuilt: '{skip-vote}'
 
+- trigger:
+    name: gerrit-trigger-csit-terraform
+    # Manual trigger common for csit-terraform jobs.
+    triggers:
+      - gerrit:
+          server-name: 'Primary'
+          trigger-on:
+            - comment-added-contains-event:
+                comment-contains-value: '{trigger-word}'
+          projects:
+            - project-compare-type: 'ANT'
+              project-pattern: '{project}'
+              branches:
+                - branch-compare-type: 'ANT'
+                  branch-pattern: '**/{branch}'
+          skip-vote:
+            successful: '{skip-vote}'
+            failed: '{skip-vote}'
+            unstable: '{skip-vote}'
+            notbuilt: '{skip-vote}'
+
 - trigger:
     name: gerrit-trigger-csit-vpp-devicetest-full
-    # Comes from CSIT, for automatic jobs running "device" tests with VPP builds.
+    # Comes from CSIT, for automatic jobs running device tests with VPP builds.
     # The job acts as an automatic verify job for CSIT contributions.
     triggers:
       - gerrit:
           server-name: 'Primary'
           trigger-on:
-            - patchset-created-event:
-                exclude-drafts: 'true'
-                exclude-trivial-rebase: 'false'
-                exclude-no-code-change: 'false'
-            - draft-published-event
+            # The main way the job is triggered automatically on patch upload.
+            # Gated by tox job, so we do not spend resources on WiP changes.
             - comment-added-contains-event:
-                comment-contains-value: 'recheck'
-            - comment-added-contains-event:
-                comment-contains-value: 'reverify'
+                comment-contains-value: 'tox_success'
+            # Manual trigger to allow testing WiP changes with selected tags.
             - comment-added-contains-event:
                 comment-contains-value: 'csit-devicetest'
+            # A comment with "recheck" restarts tox job, so no condition here.
           projects:
             - project-compare-type: 'ANT'
               project-pattern: '{project}'
@@ -631,6 +657,103 @@
           fingerprint: true
           latest-only: true
 
+- publisher:
+    name: csit-trending-failed
+    publishers:
+      - email-ext:
+          # yamllint disable-line rule:line-length
+          recipients: 'tifrank@cisco.com, csit-report+int+616+181919729795594561@lists.fd.io'
+          reply-to: ''
+          content-type: 'text'
+          subject: '{branch}: Failed tests as of $BUILD_TIMESTAMP'
+          # yamllint disable-line rule:line-length
+          body: |
+              Following tests failed in the last trending job runs, listed per testbed type.
+
+              ${{FILE,path="resources/tools/presentation/_build/_static/vpp/trending-failed-tests.txt"}}
+          failure: false
+          success: true
+
+- publisher:
+    name: csit-trending-failed-job
+    publishers:
+      - email-ext:
+          # yamllint disable-line rule:line-length
+          recipients: 'tifrank@cisco.com, csit-report+int+616+181919729795594561@lists.fd.io'
+          reply-to: ''
+          content-type: 'text'
+          subject: '{branch}: The build number $BUILD_NUMBER of the job $JOB_NAME failed'
+          # yamllint disable-line rule:line-length
+          body: |
+              For more information see $BUILD_URL.
+          failure: true
+          success: false
+
+- publisher:
+    name: csit-trending-progressions
+    publishers:
+      - email-ext:
+          # yamllint disable-line rule:line-length
+          recipients: 'tifrank@cisco.com, csit-report+int+616+181919729795594561@lists.fd.io'
+          reply-to: ''
+          content-type: 'text'
+          subject: '{branch}: Progressions as of $BUILD_TIMESTAMP'
+          # yamllint disable-line rule:line-length
+          body: |
+              Following progressions occured in the last trending job runs, listed per testbed type.
+
+              ${{FILE,path="resources/tools/presentation/_build/_static/vpp/trending-progressions.txt"}}
+          failure: false
+          success: true
+
+- publisher:
+    name: csit-trending-regressions
+    publishers:
+      - email-ext:
+          # yamllint disable-line rule:line-length
+          recipients: 'tifrank@cisco.com, csit-report+int+616+181919729795594561@lists.fd.io'
+          reply-to: ''
+          content-type: 'text'
+          subject: '{branch}: Regressions as of $BUILD_TIMESTAMP'
+          # yamllint disable-line rule:line-length
+          body: |
+              Following regressions occured in the last trending job runs, listed per testbed type.
+
+              ${{FILE,path="resources/tools/presentation/_build/_static/vpp/trending-regressions.txt"}}
+          failure: false
+          success: true
+
+- publisher:
+    name: vpp-dpdk-rdma-version-mismatch
+    publishers:
+      - email-ext:
+          # NOTE: noreply@jenkins.fd.io must be a member of vpp-build@lists.fd.io
+          #       for the email to be distributed to the group members.
+          # yamllint disable-line rule:line-length
+          recipients: 'vpp-build@lists.fd.io'
+          reply-to: ''
+          content-type: 'text'
+          subject: '{branch}: vpp-dpdk-rdma-version-mismatch job failed!'
+          # yamllint disable-line rule:line-length
+          body: |
+              Build $BUILD_NUMBER of $JOB_NAME failed.
+              Check $BUILD_URL
+              console log to see if the job failed due to a package version mismatch or
+              a CI infra issue. If the failure was due to a CI infra issue, then
+              there is nothing else to be done.
+
+              However, if there was a package version mismatch, then the MLX driver
+              is NOT being built in the VPP DPDK library. Please push a gerrit change
+              with the updated package version(s) in the version matrix file, resolve
+              any build issues, and merge the change to re-enable the building of the
+              MLX DPDK driver in the VPP DPDK library.
+
+              Thanks,
+              The VPP Build Maintainers
+
+          failure: true
+          success: false
+
 - publisher:
     name: email-notification
     publishers:
@@ -651,11 +774,11 @@
             - recipients
 
 - publisher:
-    name: fdio-infra-publish
-    # macro to finish up a build.
+    name: fdio-infra-publish-docs
+    # macro to finish up a docs build.
     #
     # Handles the following:
-    #   - Shipping logs to Nexus or S3 logs repository
+    #   - Mapping docs S3 bucket credentials for merge job docs upload
     #   - Cleanup workspace
     publishers:
       - postbuildscript:
@@ -668,8 +791,34 @@
                 - SUCCESS
                 - UNSTABLE
               build-steps:
-                - shell: !include-raw: scripts/post_build_deploy_archives.sh
-                - fdio-infra-ship-logs
+                - fdio-infra-ship-docs
+          mark-unstable-if-failed: true
+      - workspace-cleanup:
+          exclude:
+            # Do not clean up *.jenkins-trigger files for jobs that use a
+            # properties file as input for triggering another build.
+            - "**/*.jenkins-trigger"
+          fail-build: false
+
+- publisher:
+    name: fdio-infra-publish-cov-7day
+    # macro to finish up a verify docs build.
+    #
+    # Handles the following:
+    #   - Mapping 7-day retention S3 bucket credentials for verify job docs upload
+    #   - Cleanup workspace
+    publishers:
+      - postbuildscript:
+          builders:
+            - role: BOTH
+              build-on:
+                - ABORTED
+                - FAILURE
+                - NOT_BUILT
+                - SUCCESS
+                - UNSTABLE
+              build-steps:
+                - fdio-infra-ship-cov-7day
           mark-unstable-if-failed: true
       - workspace-cleanup:
           exclude:
@@ -679,16 +828,39 @@
           fail-build: false
 
 - publisher:
-    name: fdio-infra-shiplogs
-    # To archive things, the job will need to create an "archives" directory in
-    # the workspace and this macro will handle copying the contents of the
-    # archives directory.
+    name: fdio-infra-publish-docs-7day
+    # macro to finish up a verify docs build.
     #
-    # Uses the build parameter ARCHIVE_ARTIFACTS if not empty to find files to
-    # archive. You can pass globstar patterns for example "**/*.xml **/*.log" as
-    # the archive pattern. This is a space separated list of files to archive.
+    # Handles the following:
+    #   - Mapping 7-day retention S3 bucket credentials for verify job docs upload
+    #   - Cleanup workspace
+    publishers:
+      - postbuildscript:
+          builders:
+            - role: BOTH
+              build-on:
+                - ABORTED
+                - FAILURE
+                - NOT_BUILT
+                - SUCCESS
+                - UNSTABLE
+              build-steps:
+                - fdio-infra-ship-docs-7day
+          mark-unstable-if-failed: true
+      - workspace-cleanup:
+          exclude:
+            # Do not clean up *.jenkins-trigger files for jobs that use a
+            # properties file as input for triggering another build.
+            - "**/*.jenkins-trigger"
+          fail-build: false
+
+- publisher:
+    name: fdio-infra-publish
+    # macro to finish up a build.
     #
-    # Also ensure that the workspace is cleaned up at the end of the build.
+    # Handles the following:
+    #   - Shipping logs to S3 logs repository
+    #   - Cleanup workspace
     publishers:
       - postbuildscript:
           builders:
@@ -700,19 +872,15 @@
                 - SUCCESS
                 - UNSTABLE
               build-steps:
-                - shell: !include-raw: include-raw-deploy-archives.sh
-                - maven-target:
-                    maven-version: '{maven-version}'
-                    pom: '.archives/deploy-archives.xml'
-                    goals: 'clean deploy -V -B -q -Dmaven.repo.local=/tmp/r'
-                    settings: 'jenkins-log-archives-settings'
-                    settings-type: cfp
-                    global-settings: 'global-settings'
-                    global-settings-type: cfp
-                - description-setter:
-                    regexp: '^Build logs: .*'
+                - shell: !include-raw-escape: scripts/post_build_executor_info.sh
+                - shell: !include-raw-escape: scripts/post_build_deploy_archives.sh
+                - fdio-infra-ship-logs
           mark-unstable-if-failed: true
       - workspace-cleanup:
+          exclude:
+            # Do not clean up *.jenkins-trigger files for jobs that use a
+            # properties file as input for triggering another build.
+            - "**/*.jenkins-trigger"
           fail-build: false
 
 - publisher:
@@ -745,6 +913,7 @@
       - robot:
           output-path: '{output-path}'
           other-files: ''
+
 # [end] Keep 'publisher' entries ordered alphabetically, please.
 
 #### BUILDERS
@@ -755,40 +924,63 @@
       - shell: |
           $WORKSPACE/scripts/check-unicode.sh jjb/
 
-- property:
-    name: fdio-infra-properties
-    properties:
-      - build-discarder:
-          days-to-keep: '{build-days-to-keep}'
-          num-to-keep: '{build-num-to-keep}'
+- builder:
+    name: fdio-infra-ship-docs
+    builders:
+      - config-file-provider:
+          files:
+            - file-id: "jenkins-s3-docs-ship"
+              target: $HOME/.aws/credentials
+      - shell: !include-raw-escape:
+          - scripts/terraform_s3_docs_ship.sh
+      - shell: !include-raw-escape:
+          - scripts/publish_docs.sh
+      - shell: !include-raw:
+          - ../global-jjb/shell/logs-clear-credentials.sh
+
+- builder:
+    name: fdio-infra-ship-cov-7day
+    builders:
+      - config-file-provider:
+          files:
+            - file-id: "jenkins-s3-vpp-docs-ship"
+              target: $HOME/.aws/credentials
+      - shell: !include-raw-escape:
+          - scripts/terraform_s3_docs_ship.sh
+      - shell: !include-raw-escape:
+          - scripts/publish_cov.sh
+      - shell: !include-raw:
+          - ../global-jjb/shell/logs-clear-credentials.sh
+
+- builder:
+    name: fdio-infra-ship-docs-7day
+    builders:
+      - config-file-provider:
+          files:
+            - file-id: "jenkins-s3-vpp-docs-ship"
+              target: $HOME/.aws/credentials
+      - shell: !include-raw-escape:
+          - scripts/terraform_s3_docs_ship.sh
+      - shell: !include-raw-escape:
+          - scripts/publish_docs.sh
+      - shell: !include-raw:
+          - ../global-jjb/shell/logs-clear-credentials.sh
 
 - builder:
     name: fdio-infra-ship-logs
     builders:
-      # Ensure no pre-existing .netrc files are overriding logs config
-      - lf-provide-maven-settings-cleanup
       - config-file-provider:
           files:
-            - file-id: "jenkins-log-archives-settings"
-              variable: "SETTINGS_FILE"
-      - conditional-step:
-          condition-kind: regex-match
-          regex: "^.*logs-s3.*"
-          label: $S3_BUCKET
-          on-evaluation-failure: dont-run
-          steps:
-            - config-file-provider:
-                files:
-                  - file-id: "jenkins-s3-log-ship"
-                    target: $HOME/.aws/credentials
-      - lf-infra-create-netrc:
-          server-id: logs
+            - file-id: "jenkins-s3-log-ship"
+              target: $HOME/.aws/credentials
+      - shell: !include-raw-escape:
+          - scripts/publish_library_py.sh
       - shell: !include-raw:
-          - ../global-jjb/shell/logs-deploy.sh
+          - scripts/publish_logs.sh
       - shell: !include-raw:
           - ../global-jjb/shell/logs-clear-credentials.sh
       - description-setter:
-          regexp: "(^(Nexus|S3) build logs: .*)"
+          regexp: "(^S3 build logs: .*)"
 
 - builder:
     name: packer-validate