Use 'lftools deploy archives' for log upload

- This is a pre-requisite for using amazon S3
  for log storage.
- Requires docker images built by updated
  docker scripts to include packages required
  by 'lftools' or other project specific
  docker images to be upgraded with same
  packages. This change only updates jobs
  which are using fdiotools docker images.
- Add FD.io specific global macros to avoid
  inflating docker images with packages
  required by unneeded global-jjb scripts.
- Sort VPP job parameters alphabetically
- Dump installed python3 packages in
  setup_executor_env.sh

Change-Id: Ifddd8f2b70b10acf834ffc837b27ef5453213a86
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>

diff --git a/docker/scripts/dbld_lfit_requirements.sh b/docker/scripts/dbld_lfit_requirements.sh
index 7e58ac8..918a6ff 100755 (executable)
--- a/docker/scripts/dbld_lfit_requirements.sh
+++ b/docker/scripts/dbld_lfit_requirements.sh
@@ -60,11 +60,5 @@ perl -i -p -e "s/$jenkins_uid\:/0\:/g" /etc/passwd
 cp $DOCKER_CIMAN_ROOT/global-jjb/jenkins-init-scripts/lf-env.sh /root
 chmod 644 /root/lf-env.sh
 
-# Install lftools[openstack] -- from global-jjb/shell/python-tools-install.sh
-pinned_version=""
-if [ "$OS_NAME" = "debian-9" } ; then
-    # debian-9 does not have osc-lib==2.2.0 available breaking docker image
-    # build so pin the version of lftools which does not pin osc-lib==2.2.0
-    pinned_version="==0.34.1"
-fi
-python3 -m pip install --no-deps lftools[openstack]$pinned_version
+# Install lftools for log / artifact upload.
+python3 -m pip install lftools

diff --git a/docker/scripts/lib_apt.sh b/docker/scripts/lib_apt.sh
index d2aa70d..0c37115 100644 (file)
--- a/docker/scripts/lib_apt.sh
+++ b/docker/scripts/lib_apt.sh
@@ -112,6 +112,8 @@ ENV PATH=$PATH:$DOCKER_CIMAN_ROOT/docker/scripts
 #       facter
 #       python3-pip
 #       python3-venv
+#   for lftools:
+#       xmlstarlet
 #   from global-jjb/packer/provision/baseline.sh:
 #       unzip
 #       xz-utils
@@ -147,6 +149,7 @@ RUN apt-get update -q \\
         enchant \\
         emacs \\
         facter \\
+        gawk \\
         gfortran \\
         git \\
         git-review \\
@@ -173,6 +176,7 @@ RUN apt-get update -q \\
         tree \\
         vim \\
         wget \\
+        xmlstarlet \\
         xz-utils \\
     && rm -r /var/lib/apt/lists/*
 

diff --git a/docker/scripts/lib_common.sh b/docker/scripts/lib_common.sh
index 424aa78..17dcc14 100644 (file)
--- a/docker/scripts/lib_common.sh
+++ b/docker/scripts/lib_common.sh
@@ -179,7 +179,7 @@ export EXECUTOR_CLASS_ARCH="$EXECUTOR_DEFAULT_CLASS-$OS_ARCH"
 export EXECUTOR_CLASSES="$EXECUTOR_DEFAULT_CLASS csit csit_dut csit_shim"
 export EXECUTOR_ARCHS="aarch64 x86_64"
 declare -A EXECUTOR_CLASS_ARCH_OS_NAMES
-EXECUTOR_CLASS_ARCH_OS_NAMES["builder-aarch64"]="centos-7 centos-8 ubuntu-18.04 ubuntu-20.04"
+EXECUTOR_CLASS_ARCH_OS_NAMES["builder-aarch64"]="centos-8 ubuntu-18.04 ubuntu-20.04"
 EXECUTOR_CLASS_ARCH_OS_NAMES["builder-x86_64"]="centos-7 centos-8 debian-9 debian-10 ubuntu-18.04 ubuntu-20.04"
 EXECUTOR_CLASS_ARCH_OS_NAMES["csit-aarch64"]="ubuntu-18.04"
 EXECUTOR_CLASS_ARCH_OS_NAMES["csit-x86_64"]="ubuntu-18.04"

diff --git a/docker/scripts/lib_dnf.sh b/docker/scripts/lib_dnf.sh
index 41915e1..639e7f0 100644 (file)
--- a/docker/scripts/lib_dnf.sh
+++ b/docker/scripts/lib_dnf.sh
@@ -77,6 +77,9 @@ COPY . .
 # Install baseline packages (minimum build & utils).
 #
 # ci-management global-jjb requirements:
+#   for lftools:
+#       libxml2-devel
+#       xmlstarlet
 #   for lf-env.sh:
 #       facter
 #   from global-jjb/packer/provision/baseline.sh:
@@ -101,13 +104,16 @@ COPY . .
 #   lapack-devel    for python numpy/scipy (CSIT/aarch64)
 #   openblas-devel  for python numpy/scipy (CSIT/aarch64)
 #
-RUN dnf update -y \\
+RUN export LC_ALL=C.UTF8 \\
+    && dnf update -y \\
     && dnf install -y \\
         dnf-plugins-core \\
         epel-release \\
-    && dnf config-manager --set-enabled PowerTools --set-enabled epel \\
+    && dnf config-manager --set-enabled \$(dnf repolist all 2> /dev/null | grep -i powertools | cut -d' ' -f1) --set-enabled epel \\
+    && dnf repolist all \\
     && dnf clean all
-RUN dnf update -y \\
+RUN export LC_ALL=C.UTF8 \\
+    && dnf update -y \\
     && dnf install -y \\
         dnf-utils \\
         doxygen \\
@@ -124,6 +130,7 @@ RUN dnf update -y \\
         lapack-devel \\
         libffi-devel \\
         libpcap-devel \\
+        libxml2-devel \\
         make \\
         mawk \\
         mock \\
@@ -140,6 +147,7 @@ RUN dnf update -y \\
         unzip \\
         vim \\
         wget \\
+        xmlstarlet \\
         xz \\
     && dnf clean all
 

diff --git a/docker/scripts/lib_vpp.sh b/docker/scripts/lib_vpp.sh
index 38245c7..72bec64 100644 (file)
--- a/docker/scripts/lib_vpp.sh
+++ b/docker/scripts/lib_vpp.sh
@@ -81,7 +81,7 @@ docker_build_setup_vpp() {
 # Branches must be listed in chronological order -- oldest stable branch
 # first and master last.
 declare -A VPP_BRANCHES
-VPP_BRANCHES["centos-7"]="stable/1908 stable/2001 stable/2005 stable/2009 master"
+VPP_BRANCHES["centos-7"]="stable/1908 stable/2001 stable/2005 stable/2009"
 VPP_BRANCHES["centos-8"]="stable/2009 master"
 VPP_BRANCHES["debian-9"]="stable/2009 master"
 VPP_BRANCHES["debian-10"]="stable/2009 master"

diff --git a/docker/scripts/lib_yum.sh b/docker/scripts/lib_yum.sh
index 95a728e..89ac3a6 100644 (file)
--- a/docker/scripts/lib_yum.sh
+++ b/docker/scripts/lib_yum.sh
@@ -70,6 +70,9 @@ COPY . .
 # Install baseline packages (minimum build & utils).
 #
 # ci-management global-jjb requirements:
+#   for lftools:
+#       libxml2-devel
+#       xmlstarlet
 #   for lf-env.sh:
 #       facter
 #   from global-jjb/packer/provision/baseline.sh:
@@ -117,6 +120,7 @@ RUN yum update -y \\
         liblapack-devel \\
         libopenblas-devel \\
         libpcap-devel \\
+        libxml2-devel \\
         make \\
         mawk \\
         mock \\
@@ -131,6 +135,7 @@ RUN yum update -y \\
         unzip \\
         vim \\
         wget \\
+        xmlstarlet \\
         xz \\
     && yum clean all
 

diff --git a/jjb/cicn/cicn.yaml b/jjb/cicn/cicn.yaml
index d526ea6..5f618c6 100644 (file)
--- a/jjb/cicn/cicn.yaml
+++ b/jjb/cicn/cicn.yaml
@@ -1,3 +1,16 @@
+# Copyright (c) 2020 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 ---
 - project:
     name: cicn-libparc-x86_64
@@ -96,10 +109,6 @@
           repo-name: "{repo-stream-part}.{repo-os-part}"
       - stream-parameter:
           stream: "hicn"
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: "{archive-artifacts}"
-          description: Artifacts to archive to the logs server.
 
     scm:
       - gerrit-trigger-scm:
@@ -144,8 +153,7 @@
           - ../scripts/packagecloud_push.sh
 
     publishers:
-      - fdio-infra-shiplogs:
-          maven-version: "mvn33-new"
+      - fdio-infra-publish
 
 - job-template:
     name: "cicn-libparc-verify-{stream}-{os}-{executor-arch}"
@@ -167,10 +175,6 @@
           branch: "{branch}"
       - os-parameter:
           os: "{os}"
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: "{archive-artifacts}"
-          description: Artifacts to archive to the logs server.
 
     scm:
       - gerrit-trigger-scm:
@@ -192,8 +196,7 @@
           - include-raw-cicn-libparc-build.sh
 
     publishers:
-      - fdio-infra-shiplogs:
-          maven-version: "mvn33-new"
+      - fdio-infra-publish
 
 - job-template:
     name: "cicn-viper-merge-{stream}-{os}-{executor-arch}"
@@ -222,10 +225,6 @@
           repo-name: "{repo-stream-part}.{repo-os-part}"
       - stream-parameter:
           stream: "hicn"
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: "{archive-artifacts}"
-          description: Artifacts to archive to the logs server.
 
     scm:
       - gerrit-trigger-scm:
@@ -273,8 +272,7 @@
           - ../scripts/packagecloud_push.sh
 
     publishers:
-      - fdio-infra-shiplogs:
-          maven-version: "mvn33-new"
+      - fdio-infra-publish
 
 - job-template:
     name: "cicn-viper-verify-{stream}-{os}-{executor-arch}"
@@ -296,10 +294,6 @@
           branch: "{branch}"
       - os-parameter:
           os: "{os}"
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: "{archive-artifacts}"
-          description: Artifacts to archive to the logs server.
 
     scm:
       - gerrit-trigger-scm:
@@ -321,8 +315,7 @@
           - include-raw-cicn-viper-build.sh
 
     publishers:
-      - fdio-infra-shiplogs:
-          maven-version: "mvn33-new"
+      - fdio-infra-publish
 
 - project:
     name: cicn-info

diff --git a/jjb/global-macros.yaml b/jjb/global-macros.yaml
index 66a70ba..f4b009a 100644 (file)
--- a/jjb/global-macros.yaml
+++ b/jjb/global-macros.yaml
@@ -24,6 +24,14 @@
           default: '{arch}'
           description: "Architecture parameter"
 
+- parameter:
+    name: archive-artifacts-parameter
+    parameters:
+      - string:
+          name: ARCHIVE_ARTIFACTS
+          default: '{artifacts}'
+          description: "Artifacts to archive on the log server."
+
 - parameter:
     name: compiler-parameter
     parameters:
@@ -187,6 +195,14 @@
           name: TOPO
           default: '{topo}'
           description: "Topology parameter"
+
+- parameter:
+    name: workspace-archive-artifacts-parameter
+    parameters:
+      - string:
+          name: WS_ARCHIVE_ARTIFACTS
+          default: '{artifacts}'
+          description: "Artifacts to archive in $WORKSPACE/archives."
 # [end] Keep 'parameter' entries ordered alphabetically, please.
 
 #### SCMS
@@ -634,6 +650,34 @@
             - developers
             - recipients
 
+- publisher:
+    name: fdio-infra-publish
+    # macro to finish up a build.
+    #
+    # Handles the following:
+    #   - Shipping logs to Nexus or S3 logs repository
+    #   - Cleanup workspace
+    publishers:
+      - postbuildscript:
+          builders:
+            - role: BOTH
+              build-on:
+                - ABORTED
+                - FAILURE
+                - NOT_BUILT
+                - SUCCESS
+                - UNSTABLE
+              build-steps:
+                - shell: !include-raw: scripts/post_build_deploy_archives.sh
+                - fdio-infra-ship-logs
+          mark-unstable-if-failed: true
+      - workspace-cleanup:
+          exclude:
+            # Do not clean up *.jenkins-trigger files for jobs that use a
+            # properties file as input for triggering another build.
+            - "**/*.jenkins-trigger"
+          fail-build: false
+
 - publisher:
     name: fdio-infra-shiplogs
     # To archive things, the job will need to create an "archives" directory in
@@ -718,6 +762,34 @@
           days-to-keep: '{build-days-to-keep}'
           num-to-keep: '{build-num-to-keep}'
 
+- builder:
+    name: fdio-infra-ship-logs
+    builders:
+      # Ensure no pre-existing .netrc files are overriding logs config
+      - lf-provide-maven-settings-cleanup
+      - config-file-provider:
+          files:
+            - file-id: "jenkins-log-archives-settings"
+              variable: "SETTINGS_FILE"
+      - conditional-step:
+          condition-kind: regex-match
+          regex: "^.*logs-s3.*"
+          label: $S3_BUCKET
+          on-evaluation-failure: dont-run
+          steps:
+            - config-file-provider:
+                files:
+                  - file-id: "jenkins-s3-log-ship"
+                    target: $HOME/.aws/credentials
+      - lf-infra-create-netrc:
+          server-id: logs
+      - shell: !include-raw:
+          - ../global-jjb/shell/logs-deploy.sh
+      - shell: !include-raw:
+          - ../global-jjb/shell/logs-clear-credentials.sh
+      - description-setter:
+          regexp: "(^(Nexus|S3) build logs: .*)"
+
 - builder:
     name: packer-validate
     builders:

diff --git a/jjb/hicn/hicn.yaml b/jjb/hicn/hicn.yaml
index 61bb68b..665ae22 100644 (file)
--- a/jjb/hicn/hicn.yaml
+++ b/jjb/hicn/hicn.yaml
@@ -1,3 +1,16 @@
+# Copyright (c) 2020 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 ---
 - project:
     name: hicn-x86_64
@@ -197,10 +210,8 @@
           os: "{os}"
       - repo-name-parameter:
           repo-name: "{repo-stream-part}.ubuntu.bionic.main"
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: "{archive-artifacts}"
-          description: Artifacts to archive to the logs server.
+      - workspace-archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
 
     scm:
       - gerrit-trigger-scm:
@@ -258,8 +269,7 @@
             include-raw-hicn-checkstyle.sh
 
     publishers:
-      - fdio-infra-shiplogs:
-          maven-version: "mvn36"
+      - fdio-infra-publish
       - naginator:
           rerun-unstable-builds: true
           max-failed-builds: 1
@@ -289,10 +299,8 @@
           repo-name: "{repo-stream-part}.{repo-os-part}"
       - stream-parameter:
           stream: "{stream}"
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: "{archive-artifacts}"
-          description: Artifacts to archive to the logs server.
+      - workspace-archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
 
     scm:
       - gerrit-trigger-scm:
@@ -314,8 +322,7 @@
           - include-raw-hicn-build.sh
 
     publishers:
-      - fdio-infra-shiplogs:
-          maven-version: "mvn36"
+      - fdio-infra-publish
       - naginator:
           rerun-unstable-builds: false
           max-failed-builds: 2
@@ -348,10 +355,8 @@
           stream: "{repository-name}"
       - repo-name-parameter:
           repo-name: "{repo-stream-part}.{repo-os-part}"
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: "{archive-artifacts}"
-          description: Artifacts to archive to the logs server.
+      - workspace-archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
 
     scm:
       - gerrit-trigger-scm:
@@ -398,8 +403,7 @@
           - include-raw-hicn-maven-push.sh
 
     publishers:
-      - fdio-infra-shiplogs:
-          maven-version: "mvn36"
+      - fdio-infra-publish
       - naginator:
           rerun-unstable-builds: false
           max-failed-builds: 1
@@ -443,10 +447,8 @@
           repo-name: "{repo-stream-part}.{repo-os-part}"
       - stream-parameter:
           stream: "{stream}"
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: "{archive-artifacts}"
-          description: Artifacts to archive to the logs server.
+      - workspace-archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
 
     scm:
       - gerrit-trigger-scm:
@@ -476,8 +478,7 @@
           - include-raw-hicn-extras-build.sh
 
     publishers:
-      - fdio-infra-shiplogs:
-          maven-version: "mvn36"
+      - fdio-infra-publish
       - naginator:
           rerun-unstable-builds: false
           max-failed-builds: 2
@@ -510,10 +511,8 @@
           stream: "hicn"
       - repo-name-parameter:
           repo-name: "{repo-stream-part}.{repo-os-part}"
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: "{archive-artifacts}"
-          description: Artifacts to archive to the logs server.
+      - workspace-archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
 
     scm:
       - gerrit-trigger-scm:
@@ -566,8 +565,7 @@
           - include-raw-hicn-maven-push.sh
 
     publishers:
-      - fdio-infra-shiplogs:
-          maven-version: "mvn36"
+      - fdio-infra-publish
       - naginator:
           rerun-unstable-builds: false
           max-failed-builds: 1

diff --git a/jjb/scripts/post_build_deploy_archives.sh b/jjb/scripts/post_build_deploy_archives.sh
new file mode 100644 (file)
index 0000000..3f68e84
--- /dev/null
+++ b/jjb/scripts/post_build_deploy_archives.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# Copyright (c) 2020 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "---> jjb/scripts/post_build_deploy_archives.sh"
+
+set +e  # Do not affect the build result if some part of archiving fails.
+WS_ARCHIVES_DIR="$WORKSPACE/archives"
+BUILD_ENV_LOG="$WS_ARCHIVES_DIR/_build-enviroment-variables.log"
+
+# Log the build environment variables
+echo "Logging build environment variables in '$BUILD_ENV_LOG'..."
+mkdir -p $WS_ARCHIVES_DIR
+env > $BUILD_ENV_LOG
+
+echo "WS_ARCHIVE_ARTIFACTS = '$WS_ARCHIVE_ARTIFACTS'"
+if [ ! -z "${WS_ARCHIVE_ARTIFACTS}" ]; then
+    pushd $WORKSPACE
+    shopt -s globstar  # Enable globstar to copy archives
+    archive_artifacts=$(echo ${WS_ARCHIVE_ARTIFACTS})
+    for file in $archive_artifacts; do
+        if [ -f "$file" ] ; then
+            echo "Archiving '$file'..."
+            mkdir -p $WS_ARCHIVES_DIR/$(dirname $file)
+            mv $file $WS_ARCHIVES_DIR/$file
+        else
+            echo "Skipping '$file' which is not a file:"
+            ls -ld $file
+        fi
+    done
+    shopt -u globstar  # Disable globstar once archives are copied
+    popd
+    # Clean up failed 'make test' archive directories for better
+    # navigation and legibility of log directories.
+    if [ -d "$WS_ARCHIVES_DIR/tmp" ] ; then
+        mv $WS_ARCHIVES_DIR/tmp/* $WS_ARCHIVES_DIR
+        rmdir $WS_ARCHIVES_DIR/tmp
+    fi
+fi
+
+# find and gzip any 'text' files
+find $WS_ARCHIVES_DIR -type f -print0 \
+                | xargs -0r file \
+                | egrep -e ':.*text.*' \
+                | cut -d: -f1 \
+                | xargs -d'\n' -r gzip

diff --git a/jjb/scripts/setup_executor_env.sh b/jjb/scripts/setup_executor_env.sh
index 08b3787..2bfbe8b 100644 (file)
--- a/jjb/scripts/setup_executor_env.sh
+++ b/jjb/scripts/setup_executor_env.sh
@@ -48,6 +48,10 @@ elif [ "$OS_ID" == "centos" ] ; then
     yum list installed || true
 fi
 
+echo "$long_line"
+echo "Python3 package list:"
+pip3 list 2>/dev/null | column -t || true
+
 echo "$long_line"
 echo "Executor Downloads cache '$downloads_cache':"
 ls -lh "$downloads_cache" || true

diff --git a/jjb/vpp/vpp.yaml b/jjb/vpp/vpp.yaml
index 900bf55..4d31e89 100644 (file)
--- a/jjb/vpp/vpp.yaml
+++ b/jjb/vpp/vpp.yaml
@@ -1,3 +1,16 @@
+# Copyright (c) 2020 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 ---
 # PROJECT DEFINITIONS
 #
@@ -52,6 +65,7 @@
       - 'vpp-merge-{stream}-{os}-{executor-arch}'
     project: 'vpp'
     make-parallel-jobs: '4'
+    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
     stream:
       - master:
           branch: 'master'
@@ -83,6 +97,7 @@
       - 'vpp-merge-{stream}-{os}-{executor-arch}'
     project: 'vpp'
     make-parallel-jobs: '4'
+    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
     stream:
       - '1908':
           branch: 'stable/1908'
@@ -125,6 +140,7 @@
       - 'vpp-merge-{stream}-{os}-{executor-arch}'
     project: 'vpp'
     make-parallel-jobs: '4'
+    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
     stream:
       - master:
           branch: 'master'
@@ -158,6 +174,7 @@
       - 'vpp-debug-verify-{stream}-{os}-{executor-arch}'
     project: 'vpp'
     make-parallel-jobs: '4'
+    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
     stream:
       - master:
           branch: 'master'
@@ -194,6 +211,7 @@
 #          comment-trigger-value: 'vpp-beta-merge'
 #    project: 'vpp'
 #    make-parallel-jobs: '4'
+#    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
 #    stream:
 #      - master:
 #          branch: 'master'
@@ -234,6 +252,7 @@
       - 'vpp-merge-{stream}-{os}-{executor-arch}'
     project: 'vpp'
     make-parallel-jobs: '16'
+    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
     stream:
       - master:
           branch: 'master'
@@ -281,6 +300,7 @@
 #
 #    project: 'vpp'
 #    make-parallel-jobs: '16'
+#    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
 #    stream:
 #      - master:
 #          branch: 'master'
@@ -317,6 +337,7 @@
           comment-trigger-value: 'beta-merge'
     project: 'vpp'
     make-parallel-jobs: '16'
+    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
     stream:
       - master:
           branch: 'master'
@@ -434,7 +455,6 @@
     project-type: freestyle
     node: 'builder-{os}-prod-{executor-arch}'
     concurrent: true
-    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
 
     build-discarder:
       daysToKeep: '{build-days-to-keep}'
@@ -442,17 +462,20 @@
       artifactDaysToKeep: '{build-artifact-days-to-keep}'
       artifactNumToKeep: '{build-artifact-num-to-keep}'
 
+    # Please keep parameters in alphabetical order
     parameters:
-      - project-parameter:
-          project: '{project}'
       - gerrit-parameter:
           branch: '{branch}'
+      - gerrit-refspec-parameter:
+          refspec: '{branch-head-refspec}'
       - os-parameter:
           os: '{os}'
+      - project-parameter:
+          project: '{project}'
       - repo-name-parameter:
           repo-name: '{repo-stream-part}.ubuntu.bionic.main'
-      - gerrit-refspec-parameter:
-          refspec: '{branch-head-refspec}'
+      - workspace-archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
 
     scm:
       - gerrit-trigger-scm:
@@ -538,8 +561,8 @@
           - ../scripts/vpp/commitmsg.sh
 
     publishers:
-      - fdio-infra-shiplogs:
-          maven-version: 'mvn36'
+      - fdio-infra-publish
+
 # [end] VPP-CHECKSTYLE-VERIFY JOB TEMPLATE
 
 # VPP-VERIFY JOB TEMPLATE
@@ -549,7 +572,6 @@
     project-type: freestyle
     node: 'builder-{os}-prod-{executor-arch}'
     concurrent: true
-    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
 
     build-discarder:
       daysToKeep: '{build-days-to-keep}'
@@ -557,25 +579,24 @@
       artifactDaysToKeep: '{build-artifact-days-to-keep}'
       artifactNumToKeep: '{build-artifact-num-to-keep}'
 
+    # Please keep parameters in alphabetical order
     parameters:
-      - project-parameter:
-          project: '{project}'
       - gerrit-parameter:
           branch: '{branch}'
+      - gerrit-refspec-parameter:
+          refspec: '{branch-head-refspec}'
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
       - os-parameter:
           os: '{os}'
+      - project-parameter:
+          project: '{project}'
       - repo-name-parameter:
           repo-name: '{repo-stream-part}.{repo-os-part}'
       - stream-parameter:
           stream: '{stream}'
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: '{archive-artifacts}'
-          description: Artifacts to archive to the logs server.
-      - make-parallel-jobs-parameter:
-          make-parallel-jobs: '{make-parallel-jobs}'
-      - gerrit-refspec-parameter:
-          refspec: '{branch-head-refspec}'
+      - workspace-archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
 
     scm:
       - gerrit-trigger-scm:
@@ -607,8 +628,7 @@
           - ../scripts/vpp/build.sh
 
     publishers:
-      - fdio-infra-shiplogs:
-          maven-version: 'mvn36'
+      - fdio-infra-publish
 # [end] VPP-VERIFY JOB TEMPLATE
 
 # VPP-DEBUG-VERIFY JOB TEMPLATE
@@ -618,7 +638,6 @@
     project-type: freestyle
     node: 'builder-{os}-prod-{executor-arch}'
     concurrent: true
-    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
 
     build-discarder:
       daysToKeep: '{build-days-to-keep}'
@@ -626,21 +645,24 @@
       artifactDaysToKeep: '{build-artifact-days-to-keep}'
       artifactNumToKeep: '{build-artifact-num-to-keep}'
 
+    # Please keep parameters in alphabetical order
     parameters:
-      - project-parameter:
-          project: '{project}'
       - gerrit-parameter:
           branch: '{branch}'
+      - gerrit-refspec-parameter:
+          refspec: '{branch-head-refspec}'
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
       - os-parameter:
           os: 'ubuntu1804'
+      - project-parameter:
+          project: '{project}'
       - repo-name-parameter:
           repo-name: '{repo-stream-part}.ubuntu.bionic.main'
       - stream-parameter:
           stream: '{stream}'
-      - make-parallel-jobs-parameter:
-          make-parallel-jobs: '{make-parallel-jobs}'
-      - gerrit-refspec-parameter:
-          refspec: '{branch-head-refspec}'
+      - workspace-archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
 
     scm:
       - gerrit-trigger-scm:
@@ -674,8 +696,7 @@
           - ../scripts/vpp/debug-build.sh
 
     publishers:
-      - fdio-infra-shiplogs:
-          maven-version: 'mvn36'
+      - fdio-infra-publish
 # [end] VPP-DEBUG-VERIFY JOB TEMPLATE
 
 # VPP-MERGE JOB TEMPLATE
@@ -691,27 +712,22 @@
       artifactDaysToKeep: '{build-artifact-days-to-keep}'
       artifactNumToKeep: '{build-artifact-num-to-keep}'
 
+    # Please keep parameters in alphabetical order
     parameters:
-      - project-parameter:
-          project: '{project}'
       - gerrit-parameter:
           branch: '{branch}'
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
       - os-parameter:
           os: '{os}'
-      - maven-project-parameter:
-          maven: 'mvn33-new'
-      - maven-exec:
-          maven-version: 'mvn33-new'
-      - stream-parameter:
-          stream: '{stream}'
+      - project-parameter:
+          project: '{project}'
       - repo-name-parameter:
           repo-name: '{repo-stream-part}.{repo-os-part}'
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: '{archive-artifacts}'
-          description: Artifacts to archive to the logs server.
-      - make-parallel-jobs-parameter:
-          make-parallel-jobs: '{make-parallel-jobs}'
+      - stream-parameter:
+          stream: '{stream}'
+      - workspace-archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
 
     scm:
       - gerrit-trigger-scm:
@@ -758,8 +774,7 @@
           - ../scripts/packagecloud_push.sh
 
     publishers:
-      - fdio-infra-shiplogs:
-          maven-version: 'mvn36'
+      - fdio-infra-publish
 # [end] VPP-MERGE JOB TEMPLATE
 
 # VPP-BETA-VERIFY JOB TEMPLATE
@@ -769,7 +784,6 @@
     project-type: freestyle
     node: 'builder-{os}-prod-{executor-arch}'
     concurrent: true
-    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
 
     build-discarder:
       daysToKeep: '{build-days-to-keep}'
@@ -777,25 +791,24 @@
       artifactDaysToKeep: '{build-artifact-days-to-keep}'
       artifactNumToKeep: '{build-artifact-num-to-keep}'
 
+    # Please keep parameters in alphabetical order
     parameters:
-      - project-parameter:
-          project: '{project}'
       - gerrit-parameter:
           branch: '{branch}'
+      - gerrit-refspec-parameter:
+          refspec: '{branch-head-refspec}'
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
       - os-parameter:
           os: '{os}'
+      - project-parameter:
+          project: '{project}'
       - repo-name-parameter:
           repo-name: '{repo-stream-part}.{repo-os-part}'
       - stream-parameter:
           stream: '{stream}'
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: '{archive-artifacts}'
-          description: Artifacts to archive to the logs server.
-      - make-parallel-jobs-parameter:
-          make-parallel-jobs: '{make-parallel-jobs}'
-      - gerrit-refspec-parameter:
-          refspec: '{branch-head-refspec}'
+      - workspace-archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
 
     scm:
       - gerrit-trigger-scm:
@@ -828,8 +841,7 @@
           - ../scripts/vpp/build.sh
 
     publishers:
-      - fdio-infra-shiplogs:
-          maven-version: 'mvn36'
+      - fdio-infra-publish
 # [end] VPP-BETA-VERIFY JOB TEMPLATE
 
 # VPP-BETA-MERGE JOB TEMPLATE
@@ -845,27 +857,22 @@
       artifactDaysToKeep: '{build-artifact-days-to-keep}'
       artifactNumToKeep: '{build-artifact-num-to-keep}'
 
+    # Please keep parameters in alphabetical order
     parameters:
-      - project-parameter:
-          project: '{project}'
       - gerrit-parameter:
           branch: '{branch}'
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
       - os-parameter:
           os: '{os}'
-      - maven-project-parameter:
-          maven: 'mvn33-new'
-      - maven-exec:
-          maven-version: 'mvn33-new'
-      - stream-parameter:
-          stream: '{stream}'
+      - project-parameter:
+          project: '{project}'
       - repo-name-parameter:
           repo-name: '{repo-stream-part}.{repo-os-part}'
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: '{archive-artifacts}'
-          description: Artifacts to archive to the logs server.
-      - make-parallel-jobs-parameter:
-          make-parallel-jobs: '{make-parallel-jobs}'
+      - stream-parameter:
+          stream: '{stream}'
+      - workspace-archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
 
     scm:
       - gerrit-trigger-scm:
@@ -907,16 +914,12 @@
       - shell:
           !include-raw-escape:
           - ../scripts/vpp/build.sh
-      - provide-maven-settings:
-          settings-file: 'vpp-settings'
-          global-settings-file: 'global-settings'
       - shell:
           !include-raw-escape:
           - ../scripts/packagecloud_push.sh
 
     publishers:
-      - fdio-infra-shiplogs:
-          maven-version: 'mvn36'
+      - fdio-infra-publish
 # [end] VPP-BETA-MERGE JOB TEMPLATE
 
 # VPP-GCC-VERIFY JOB TEMPLATE
@@ -926,7 +929,6 @@
     project-type: freestyle
     node: 'builder-{os}-prod-{executor-arch}'
     concurrent: true
-    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
 
     build-discarder:
       daysToKeep: '{build-days-to-keep}'
@@ -934,25 +936,24 @@
       artifactDaysToKeep: '{build-artifact-days-to-keep}'
       artifactNumToKeep: '{build-artifact-num-to-keep}'
 
+    # Please keep parameters in alphabetical order
     parameters:
-      - project-parameter:
-          project: '{project}'
       - gerrit-parameter:
           branch: '{branch}'
+      - gerrit-refspec-parameter:
+          refspec: '{branch-head-refspec}'
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
       - os-parameter:
           os: '{os}'
+      - project-parameter:
+          project: '{project}'
       - repo-name-parameter:
           repo-name: '{repo-stream-part}.{repo-os-part}'
       - stream-parameter:
           stream: '{stream}'
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: '{archive-artifacts}'
-          description: Artifacts to archive to the logs server.
-      - make-parallel-jobs-parameter:
-          make-parallel-jobs: '{make-parallel-jobs}'
-      - gerrit-refspec-parameter:
-          refspec: '{branch-head-refspec}'
+      - workspace-archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
 
     scm:
       - gerrit-trigger-scm:
@@ -984,8 +985,7 @@
           - ../scripts/vpp/gcc-build.sh
 
     publishers:
-      - fdio-infra-shiplogs:
-          maven-version: 'mvn36'
+      - fdio-infra-publish
 # [end] VPP-GCC-VERIFY JOB TEMPLATE
 
 # VPP-CSIT-VERIFY-DEVICE JOB TEMPLATE
@@ -1011,27 +1011,26 @@
       artifactDaysToKeep: '{build-artifact-days-to-keep}'
       artifactNumToKeep: '{build-artifact-num-to-keep}'
 
+    # Please keep parameters in alphabetical order
     parameters:
-      - project-parameter:
-          project: '{project}'
+      - archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
+      - gerrit-csit-refspec-parameter
+      - gerrit-event-comment-text-parameter
+      - gerrit-event-type-parameter
       - gerrit-parameter:
           branch: '{branch}'
+      - gerrit-refspec-parameter
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
       - os-parameter:
           os: '{os}'
+      - project-parameter:
+          project: '{project}'
       - repo-name-parameter:
           repo-name: '{repo-stream-part}.ubuntu.bionic.main'
-      - gerrit-refspec-parameter
-      - gerrit-event-type-parameter
-      - gerrit-event-comment-text-parameter
-      - gerrit-csit-refspec-parameter
       - stream-parameter:
           stream: '{stream}'
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: '{archive-artifacts}'
-          description: Artifacts to archive to the logs server.
-      - make-parallel-jobs-parameter:
-          make-parallel-jobs: '{make-parallel-jobs}'
 
     scm:
       - gerrit-trigger-scm:
@@ -1100,31 +1099,30 @@
       artifactDaysToKeep: '{build-artifact-days-to-keep}'
       artifactNumToKeep: '{build-artifact-num-to-keep}'
 
+    # Please keep parameters in alphabetical order
     parameters:
-      - project-parameter:
-          project: '{project}'
+      - archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
+      - csit-perf-trial-duration-parameter:
+          csit-perf-trial-duration: "10.0"
+      - csit-perf-trial-multiplicity-parameter:
+          csit-perf-trial-multiplicity: "5"
+      - gerrit-csit-refspec-parameter
+      - gerrit-event-comment-text-parameter
+      - gerrit-event-type-parameter
+      - gerrit-refspec-parameter
       - gerrit-parameter:
           branch: '{branch}'
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
       - os-parameter:
           os: '{os}'
+      - project-parameter:
+          project: '{project}'
       - repo-name-parameter:
           repo-name: '{repo-stream-part}.ubuntu.bionic.main'
-      - gerrit-refspec-parameter
-      - gerrit-event-type-parameter
-      - gerrit-event-comment-text-parameter
-      - gerrit-csit-refspec-parameter
-      - csit-perf-trial-duration-parameter:
-          csit-perf-trial-duration: "10.0"
-      - csit-perf-trial-multiplicity-parameter:
-          csit-perf-trial-multiplicity: "5"
       - stream-parameter:
           stream: '{stream}'
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: '{archive-artifacts}'
-          description: Artifacts to archive to the logs server.
-      - make-parallel-jobs-parameter:
-          make-parallel-jobs: '{make-parallel-jobs}'
 
     scm:
       - gerrit-trigger-scm:
@@ -1161,7 +1159,8 @@
       - robot-report:
           output-path: 'csit_current/0/'
 
-      - lf-infra-publish
+      - fdio-infra-shiplogs:
+          maven-version: 'mvn36'
 # [end] VPP-CSIT-VERIFY-PERF JOB TEMPLATE
 
 # VPP-CSIT-VERIFY-API-CRC JOB TEMPLATE
@@ -1171,7 +1170,6 @@
     project-type: freestyle
     node: '{os}-us'
     concurrent: true
-    archive-artifacts: ''
     latest-only: false
 
     build-discarder:
@@ -1180,23 +1178,22 @@
       artifactDaysToKeep: '{build-artifact-days-to-keep}'
       artifactNumToKeep: '{build-artifact-num-to-keep}'
 
+    # Please keep parameters in alphabetical order
     parameters:
-      - project-parameter:
-          project: '{project}'
+      - archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
+      # Not sure whether not failing has any useful usage,
+      # but it does not hurt to have some flexibility for future.
+      - csit-fail-on-crc-mismatch-parameter:
+          fail-on-crc-mismatch: 'True'
+      - gerrit-csit-refspec-parameter
       - gerrit-parameter:
           branch: '{branch}'
       - gerrit-refspec-parameter
-      - gerrit-csit-refspec-parameter
+      - project-parameter:
+          project: '{project}'
       - stream-parameter:
           stream: '{stream}'
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: '{archive-artifacts}'
-          description: Artifacts to archive to the logs server.
-      # Not sure whether not failing has any useful usage,
-      # but it does not hurt to have some flexibility for future.
-      - csit-fail-on-crc-mismatch-parameter:
-          fail-on-crc-mismatch: 'True'
 
     scm:
       - gerrit-trigger-scm: