HC2VPP-372: Update hc2vpp integration and hc2vpp/csit jobs

 - add support for ubuntu bionic
 - update jobs to support package cloud download script
   (csit patch: https://gerrit.fd.io/r/#/c/17779/)
 - cleanup

Change-Id: I60d8af98c98612ff82ca782274d317a1726cfc7e
Signed-off-by: Michal Cmarada <mcmarada@cisco.com>

diff --git a/jjb/csit/csit-hc2vpp.yaml b/jjb/csit/csit-hc2vpp.yaml
index d230d18..e23c694 100644 (file)
--- a/jjb/csit/csit-hc2vpp.yaml
+++ b/jjb/csit/csit-hc2vpp.yaml
@@ -4,12 +4,15 @@
     description: 'jenkins jobs verify Hc2vpp functional and performance tests'
     jobs:
       - 'csit-hc2vpp-verify-func-{stream}-{os}'
-      - 'csit-hc2vpp-verify-perf-{stream}'
+      - 'csit-hc2vpp-verify-perf-{stream}-{os}'
     project: 'csit'
     branch: 'master'
     build-artifact-num-to-keep: 10
-    os: 'ubuntu1604'
-    repo-os-part: 'ubuntu.xenial.main'
+    os:
+      - ubuntu1804:
+          repo-os-part: 'ubuntu.bionic.main'
+      - ubuntu1604:
+          repo-os-part: 'ubuntu.xenial.main'
 
     stream:
       - master:
@@ -81,10 +84,10 @@
           maven-version: 'mvn33-new'
 
 - job-template:
-    name: 'csit-hc2vpp-verify-perf-{stream}'
+    name: 'csit-hc2vpp-verify-perf-{stream}-{os}'
 
     project-type: freestyle
-    node: 'ubuntu1604-basebuild-4c-4g'
+    node: '{os}-basebuild-4c-4g'
     concurrent: false
 
     build-discarder:

diff --git a/jjb/csit/include-raw-csit-hc2vpp-verify-func.sh b/jjb/csit/include-raw-csit-hc2vpp-verify-func.sh
index d81c09c..5237e98 100644 (file)
--- a/jjb/csit/include-raw-csit-hc2vpp-verify-func.sh
+++ b/jjb/csit/include-raw-csit-hc2vpp-verify-func.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # execute csit bootstrap script if it exists
-if [ ! -e bootstrap-hc2vpp-integration.sh ]
+if [[ ! -e bootstrap-hc2vpp-integration.sh ]]
 then
     echo 'ERROR: No bootstrap-hc2vpp-integration.sh found'
     exit 1
@@ -9,11 +9,7 @@ else
     # make sure that bootstrap.sh is executable
     chmod +x bootstrap-hc2vpp-integration.sh
     # run the script
-    if [ ${STREAM} == 'master' ]; then
-        ./bootstrap-hc2vpp-integration.sh ${STREAM} ${OS}
-    else
-        ./bootstrap-hc2vpp-integration.sh 'stable.'${STREAM} ${OS}
-    fi
+    ./bootstrap-hc2vpp-integration.sh ${STREAM} ${OS}
 fi
 
 # vim: ts=4 ts=4 sts=4 et :

diff --git a/jjb/csit/include-raw-csit-hc2vpp-verify-perf.sh b/jjb/csit/include-raw-csit-hc2vpp-verify-perf.sh
index d9a5fb1..fdfdb2a 100644 (file)
--- a/jjb/csit/include-raw-csit-hc2vpp-verify-perf.sh
+++ b/jjb/csit/include-raw-csit-hc2vpp-verify-perf.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # execute csit bootstrap script if it exists
-if [ ! -e bootstrap-hc2vpp-perf.sh ]
+if [[ ! -e bootstrap-hc2vpp-perf.sh ]]
 then
     echo 'ERROR: No bootstrap-hc2vpp-perf.sh found'
     exit 1
@@ -9,11 +9,7 @@ else
     # make sure that bootstrap.sh is executable
     chmod +x bootstrap-hc2vpp-perf.sh
     # run the script
-    if [ ${STREAM} == 'master' ]; then
-        ./bootstrap-hc2vpp-perf.sh ${STREAM} ${OS}
-    else
-        ./bootstrap-hc2vpp-perf.sh 'stable.'${STREAM} ${OS}
-    fi
+    ./bootstrap-hc2vpp-perf.sh ${STREAM} ${OS}
 fi
 
 # vim: ts=4 ts=4 sts=4 et :

diff --git a/jjb/hc2vpp/hc2vpp-csit.yaml b/jjb/hc2vpp/hc2vpp-csit.yaml
index 4fe39cb..c03aab3 100644 (file)
--- a/jjb/hc2vpp/hc2vpp-csit.yaml
+++ b/jjb/hc2vpp/hc2vpp-csit.yaml
@@ -23,8 +23,11 @@
           branch: 'stable/1901'
           repo-stream-part: 'stable.1901'
 
-    os: 'ubuntu1604'
-    repo-os-part: 'ubuntu.xenial.main'
+    os:
+      - ubuntu1804:
+          repo-os-part: 'ubuntu.bionic.main'
+      - ubuntu1604:
+          repo-os-part: 'ubuntu.xenial.main'
 
     odl: 'fluorine'
 
@@ -40,10 +43,13 @@
           branch: 'master'
           repo-stream-part: 'master'
 
-    os: 'ubuntu1604'
-    repo-os-part: 'ubuntu.xenial.main'
+    os:
+      - ubuntu1804:
+          repo-os-part: 'ubuntu.bionic.main'
+      - ubuntu1604:
+          repo-os-part: 'ubuntu.xenial.main'
 
-    odl: 'boron'
+    odl: 'fluorine'
 
 - job-template:
     name: 'hc2vpp-csit-integration-{stream}-{os}'

diff --git a/jjb/hc2vpp/include-raw-hc2vpp-csit-integration-odl.sh b/jjb/hc2vpp/include-raw-hc2vpp-csit-integration-odl.sh
index 5cd8b3e..d21d193 100644 (file)
--- a/jjb/hc2vpp/include-raw-hc2vpp-csit-integration-odl.sh
+++ b/jjb/hc2vpp/include-raw-hc2vpp-csit-integration-odl.sh
@@ -2,7 +2,7 @@
 set -xeu -o pipefail
 
 # Get CSIT branch
-if [ -f csit-test-branch ]; then
+if [[ -f csit-test-branch ]]; then
     chmod +x csit-test-branch
     CSIT_BRANCH=`./csit-test-branch`
 else
@@ -13,14 +13,14 @@ fi
 git clone https://gerrit.fd.io/r/csit --branch ${CSIT_BRANCH}
 
 # If the git clone fails, complain clearly and exit
-if [ $? != 0 ]; then
+if [[ $? != 0 ]]; then
     echo "Failed to run: git clone https://gerrit.fd.io/r/csit --branch ${CSIT_BRANCH}"
-    exit
+    exit 1
 fi
 
 cd csit
 # execute csit bootstrap script if it exists
-if [ ! -e bootstrap-hc2vpp-integration-odl.sh ]
+if [[ ! -e bootstrap-hc2vpp-integration-odl.sh ]]
 then
     echo 'ERROR: No bootstrap-hc2vpp-integration.sh found'
     exit 1
@@ -28,11 +28,7 @@ else
     # make sure that bootstrap.sh is executable
     chmod +x bootstrap-hc2vpp-integration-odl.sh
     # run the script
-    if [ ${STREAM} == 'master' ]; then
-        ./bootstrap-hc2vpp-integration-odl.sh ${STREAM} ${OS} ${ODL}
-    else
-        ./bootstrap-hc2vpp-integration-odl.sh 'stable.'${STREAM} ${OS} ${ODL}
-    fi
+    ./bootstrap-hc2vpp-integration-odl.sh ${STREAM} ${OS} ${ODL}
 fi
 
 # vim: ts=4 ts=4 sts=4 et :

diff --git a/jjb/hc2vpp/include-raw-hc2vpp-csit-integration.sh b/jjb/hc2vpp/include-raw-hc2vpp-csit-integration.sh
index e4bbd3f..84a3818 100644 (file)
--- a/jjb/hc2vpp/include-raw-hc2vpp-csit-integration.sh
+++ b/jjb/hc2vpp/include-raw-hc2vpp-csit-integration.sh
@@ -2,7 +2,7 @@
 set -xeu -o pipefail
 
 # Get CSIT branch
-if [ -f csit-test-branch ]; then
+if [[ -f csit-test-branch ]]; then
     chmod +x csit-test-branch
     CSIT_BRANCH=`./csit-test-branch`
 else
@@ -13,14 +13,14 @@ fi
 git clone https://gerrit.fd.io/r/csit --branch ${CSIT_BRANCH}
 
 # If the git clone fails, complain clearly and exit
-if [ $? != 0 ]; then
+if [[ $? != 0 ]]; then
     echo "Failed to run: git clone https://gerrit.fd.io/r/csit --branch ${CSIT_BRANCH}"
-    exit
+    exit 1
 fi
 
 cd csit
 # execute csit bootstrap script if it exists
-if [ ! -e bootstrap-hc2vpp-integration.sh ]
+if [[ ! -e bootstrap-hc2vpp-integration.sh ]]
 then
     echo 'ERROR: No bootstrap-hc2vpp-integration.sh found'
     exit 1
@@ -28,11 +28,7 @@ else
     # make sure that bootstrap.sh is executable
     chmod +x bootstrap-hc2vpp-integration.sh
     # run the script
-    if [ ${STREAM} == 'master' ]; then
-        ./bootstrap-hc2vpp-integration.sh ${STREAM} ${OS}
-    else
-        ./bootstrap-hc2vpp-integration.sh 'stable.'${STREAM} ${OS}
-    fi
+    ./bootstrap-hc2vpp-integration.sh ${STREAM} ${OS}
 fi
 
 # vim: ts=4 ts=4 sts=4 et :

diff --git a/jjb/hc2vpp/include-raw-hc2vpp-csit-perf.sh b/jjb/hc2vpp/include-raw-hc2vpp-csit-perf.sh
index 266c9ed..bf46af5 100644 (file)
--- a/jjb/hc2vpp/include-raw-hc2vpp-csit-perf.sh
+++ b/jjb/hc2vpp/include-raw-hc2vpp-csit-perf.sh
@@ -2,7 +2,7 @@
 set -xeu -o pipefail
 
 # Get CSIT branch
-if [ -f csit-test-branch ]; then
+if [[ -f csit-test-branch ]]; then
     chmod +x csit-test-branch
     CSIT_BRANCH=`./csit-test-branch`
 else
@@ -13,14 +13,14 @@ fi
 git clone https://gerrit.fd.io/r/csit --branch ${CSIT_BRANCH}
 
 # If the git clone fails, complain clearly and exit
-if [ $? != 0 ]; then
+if [[ $? != 0 ]]; then
     echo "Failed to run: git clone https://gerrit.fd.io/r/csit --branch ${CSIT_BRANCH}"
-    exit
+    exit 1
 fi
 
 cd csit
 # execute csit bootstrap script if it exists
-if [ ! -e bootstrap-hc2vpp-perf.sh ]
+if [[ ! -e bootstrap-hc2vpp-perf.sh ]]
 then
     echo 'ERROR: No bootstrap-hc2vpp-perf.sh found'
     exit 1
@@ -28,11 +28,7 @@ else
     # make sure that bootstrap.sh is executable
     chmod +x bootstrap-hc2vpp-perf.sh
     # run the script
-    if [ ${STREAM} == 'master' ]; then
-        ./bootstrap-hc2vpp-perf.sh ${STREAM} ${OS} ${ODL}
-    else
-        ./bootstrap-hc2vpp-perf.sh 'stable.'${STREAM} ${OS} ${ODL}
-    fi
+    ./bootstrap-hc2vpp-perf.sh ${STREAM} ${OS} ${ODL}
 fi
 
 # vim: ts=4 ts=4 sts=4 et :

diff --git a/jjb/hc2vpp/include-raw-hc2vpp-csit-verify-odl.sh b/jjb/hc2vpp/include-raw-hc2vpp-csit-verify-odl.sh
index 3938dac..271668d 100644 (file)
--- a/jjb/hc2vpp/include-raw-hc2vpp-csit-verify-odl.sh
+++ b/jjb/hc2vpp/include-raw-hc2vpp-csit-verify-odl.sh
@@ -1,13 +1,34 @@
 #!/bin/bash
 set -xeu -o pipefail
 
+# Figure out what system we are running on
+if [[ -f /etc/lsb-release ]];then
+    . /etc/lsb-release
+elif [[ -f /etc/redhat-release ]];then
+    sudo yum install -y redhat-lsb
+    DISTRIB_ID=`lsb_release -si`
+    DISTRIB_RELEASE=`lsb_release -sr`
+    DISTRIB_CODENAME=`lsb_release -sc`
+    DISTRIB_DESCRIPTION=`lsb_release -sd`
+fi
+echo "----- OS INFO -----"
+echo DISTRIB_ID: ${DISTRIB_ID}
+echo DISTRIB_RELEASE: ${DISTRIB_RELEASE}
+echo DISTRIB_CODENAME: ${DISTRIB_CODENAME}
+echo DISTRIB_DESCRIPTION: ${DISTRIB_DESCRIPTION}
+DISTRIB_ID="Ubuntu"
+if [[ "$DISTRIB_ID" != "Ubuntu" ]]; then
+    echo 'ERROR: Only Ubuntu is supported currently.'
+    exit 2
+fi
+
 # create HC .deb packages
-./packaging/deb/xenial/debuild.sh
-cp ./packaging/deb/xenial/*.deb ${WORKSPACE}/csit
+./packaging/deb/${DISTRIB_CODENAME}/debuild.sh
+cp ./packaging/deb/${DISTRIB_CODENAME}/*.deb ${WORKSPACE}/csit
 
 cd ${WORKSPACE}/csit
 # execute csit bootstrap script if it exists
-if [ ! -e bootstrap-hc2vpp-verify.sh ]
+if [[ ! -e bootstrap-hc2vpp-verify.sh ]]
 then
     echo 'ERROR: No bootstrap-hc2vpp-verify.sh found'
     exit 1

diff --git a/jjb/hc2vpp/include-raw-hc2vpp-csit-verify-prebuild.sh b/jjb/hc2vpp/include-raw-hc2vpp-csit-verify-prebuild.sh
index ac33627..9e95f4a 100644 (file)
--- a/jjb/hc2vpp/include-raw-hc2vpp-csit-verify-prebuild.sh
+++ b/jjb/hc2vpp/include-raw-hc2vpp-csit-verify-prebuild.sh
@@ -23,13 +23,13 @@ done
 
 # If HC variable is set, clone and build Honeycomb infra from the specified commit
 # Otherwise skip this step, hc2vpp will use Honeycomb snapshots from Nexus
-if [ -n "${hc_commit_id}" ]; then
+if [[ -n "${hc_commit_id}" ]]; then
     git clone https://gerrit.fd.io/r/honeycomb
     cd honeycomb
     ref=`git ls-remote -q | grep ${hc_commit_id} | awk '{print $2}'`
     git fetch origin ${ref} && git checkout FETCH_HEAD
     mvn clean install -DskipTests -Dcheckstyle.skip -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r -gs "${GLOBAL_SETTINGS_FILE}" -s "${SETTINGS_FILE}"
-    if [ $? != 0 ]; then
+    if [[ $? != 0 ]]; then
         echo "Honeycomb infra build failed."
         exit 1
     fi
@@ -41,7 +41,7 @@ fi
 # TODO: Add option to build custom VPP and NSH packages
 
 # Get CSIT branch from which to test from
-if [ -f csit-test-branch ]; then
+if [[ -f csit-test-branch ]]; then
     chmod +x csit-test-branch
     CSIT_BRANCH=`./csit-test-branch`
 else
@@ -52,7 +52,7 @@ fi
 git clone https://gerrit.fd.io/r/csit --branch ${CSIT_BRANCH}
 
 # If the git clone fails, complain clearly and exit
-if [ $? != 0 ]; then
+if [[ $? != 0 ]]; then
     echo "Failed to run: git clone https://gerrit.fd.io/r/csit --branch ${CSIT_BRANCH}"
     exit 1
 fi
@@ -60,7 +60,7 @@ fi
 cd csit
 
 # If CSIT commit ID is given, checkout the specified commit
-if [ -n "${csit_commit_id}" ]; then
+if [[ -n "${csit_commit_id}" ]]; then
     # Example:
     # ...
     # e8f326efebb58e28dacb9ebb653baf95aad1448c refs/changes/08/11808/1
@@ -70,10 +70,7 @@ if [ -n "${csit_commit_id}" ]; then
 fi
 
 # Download VPP packages
-if [ ${STREAM} == 'master' ]; then
-    ./resources/tools/scripts/download_hc_build_pkgs.sh ${STREAM} ${OS}
-else
-    ./resources/tools/scripts/download_hc_build_pkgs.sh 'stable.'${STREAM} ${OS}
-fi
+./resources/tools/scripts/download_hc_build_pkgs.sh ${STREAM} ${OS}l
+
 
 cd ${WORKSPACE}

diff --git a/jjb/hc2vpp/include-raw-hc2vpp-csit-verify.sh b/jjb/hc2vpp/include-raw-hc2vpp-csit-verify.sh
index 227df6f..21dc80c 100644 (file)
--- a/jjb/hc2vpp/include-raw-hc2vpp-csit-verify.sh
+++ b/jjb/hc2vpp/include-raw-hc2vpp-csit-verify.sh
@@ -1,13 +1,34 @@
 #!/bin/bash
 set -xeu -o pipefail
 
+# Figure out what system we are running on
+if [[ -f /etc/lsb-release ]];then
+    . /etc/lsb-release
+elif [[ -f /etc/redhat-release ]];then
+    sudo yum install -y redhat-lsb
+    DISTRIB_ID=`lsb_release -si`
+    DISTRIB_RELEASE=`lsb_release -sr`
+    DISTRIB_CODENAME=`lsb_release -sc`
+    DISTRIB_DESCRIPTION=`lsb_release -sd`
+fi
+echo "----- OS INFO -----"
+echo DISTRIB_ID: ${DISTRIB_ID}
+echo DISTRIB_RELEASE: ${DISTRIB_RELEASE}
+echo DISTRIB_CODENAME: ${DISTRIB_CODENAME}
+echo DISTRIB_DESCRIPTION: ${DISTRIB_DESCRIPTION}
+
+if [[ "$DISTRIB_ID" != "Ubuntu" ]]; then
+    echo 'ERROR: Only Ubuntu is supported currently.'
+    exit 2
+fi
+
 # create HC .deb packages
-./packaging/deb/xenial/debuild.sh
-cp ./packaging/deb/xenial/*.deb ${WORKSPACE}/csit
+./packaging/deb/${DISTRIB_CODENAME}/debuild.sh
+cp ./packaging/deb/${DISTRIB_CODENAME}/*.deb ${WORKSPACE}/csit
 
 cd ${WORKSPACE}/csit
 # execute csit bootstrap script if it exists
-if [ ! -e bootstrap-hc2vpp-verify.sh ]
+if [[ ! -e bootstrap-hc2vpp-verify.sh ]]
 then
     echo 'ERROR: No bootstrap-hc2vpp-verify.sh found'
     exit 1

diff --git a/jjb/hc2vpp/include-raw-hc2vpp-integration-prebuild.sh b/jjb/hc2vpp/include-raw-hc2vpp-integration-prebuild.sh
index 2d80524..b8f88bc 100644 (file)
--- a/jjb/hc2vpp/include-raw-hc2vpp-integration-prebuild.sh
+++ b/jjb/hc2vpp/include-raw-hc2vpp-integration-prebuild.sh
@@ -77,7 +77,7 @@ for item in jvpp*.jar; do
     # Version = 17.01
     basefile=$(basename -s .jar "$item")
     artifactId=$(echo "$basefile" | cut -d '-' -f 1-2)
-    mvn install:install-file -Dfile=${item} -DgroupId=io.fd.vpp -DartifactId=${artifactId} -Dversion=${version} -Dpackaging=jar -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r
+    mvn install:install-file -Dfile=${item} -DgroupId=io.fd.jvpp -DartifactId=${artifactId} -Dversion=${version} -Dpackaging=jar -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r
 done
 
 cd ${current_dir}

diff --git a/jjb/hc2vpp/include-raw-hc2vpp-pkg-push.sh b/jjb/hc2vpp/include-raw-hc2vpp-pkg-push.sh
index 99a911c..d32cfee 100644 (file)
--- a/jjb/hc2vpp/include-raw-hc2vpp-pkg-push.sh
+++ b/jjb/hc2vpp/include-raw-hc2vpp-pkg-push.sh
@@ -7,8 +7,8 @@ set -x
 # In order to have control of package dependencies in the release artifacts (HC2VPP-282),
 # the vpp_dependencies file is not modified in case of stable branch
 # (after VPP API freeze, Java API mismatches occur very rarely).
-if [ "${STREAM}" == "master" ]; then
-    if [ "${OS}" == "centos7" ]; then
+if [[ "${STREAM}" == "master" ]]; then
+    if [[ "${OS}" == "centos7" ]]; then
         # Determine VPP Java API version used in maven build
         JVPP_VERSION=`yum list installed vpp-api-java | grep vpp-api-java | awk '{ printf $2; }'`
         VERSION=`yum deplist vpp-api-java |grep vpp-lib |head -1 | awk '{ print $3}'`
@@ -36,44 +36,44 @@ if [ "${STREAM}" == "master" ]; then
 fi
 
 # Build package
-if [ "${OS}" == "centos7" ]; then
+if [[ "${OS}" == "centos7" ]]; then
 
     # Build the rpms
     ./packaging/rpm/rpmbuild.sh
 
     # Find the files
-    RPMS=$(find . -type f -iname '*.rpm')
-    SRPMS=$(find . -type f -iname '*.srpm')
-    SRCRPMS=$(find . -type f -name '*.src.rpm')
+    RPMS=$(find ./packaging/ -type f -iname '*.rpm')
+    SRPMS=$(find ./packaging/ -type f -iname '*.srpm')
+    SRCRPMS=$(find ./packaging/ -type f -name '*.src.rpm')
 
     # Publish hc2vpp packages
-    for i in $RPMS $SRPMS $SRCRPMS
+    for i in ${RPMS} ${SRPMS} ${SRCRPMS}
     do
         push_rpm "$i"
     done
-elif [ "${OS}" == "ubuntu1604" ]; then
+elif [[ "${OS}" == "ubuntu1604" ]]; then
 
     # Build the debs
     ./packaging/deb/xenial/debuild.sh
 
     # Find the files
-    DEBS=$(find . -type f -iname '*.deb')
+    DEBS=$(find ./packaging/ -type f -iname '*.deb')
 
     # Publish hc2vpp packages
-    for i in $DEBS
+    for i in ${DEBS}
     do
         push_deb "$i"
     done
-elif [ "${OS}" == "ubuntu1804" ]; then
+elif [[ "${OS}" == "ubuntu1804" ]]; then
 
     # Build the debs
     ./packaging/deb/bionic/debuild.sh
 
     # Find the files
-    DEBS=$(find . -type f -iname '*.deb')
+    DEBS=$(find ./packaging/ -type f -iname '*.deb')
 
     # Publish hc2vpp packages
-    for i in $DEBS
+    for i in ${DEBS}
     do
         push_deb "$i"
     done