From: Ed Warnicke <hagbard@gmail.com>
Date: Wed, 21 Sep 2016 17:16:53 +0000 (+0000)
Subject: Merge "Add a vpp-docs-verify job and move vpp-docs to vpp-merge-docs"
X-Git-Url: https://gerrit.fd.io/r/gitweb?a=commitdiff_plain;h=f25093d5a40a5e16b4f559bb3804c25ee8620c72;hp=fb764d5c558945041a034cd4d5d2526ec3ee32af;p=ci-management.git

Merge "Add a vpp-docs-verify job and move vpp-docs to vpp-merge-docs"
---

diff --git a/jjb/deb_dpdk/deb_dpdk.yaml b/jjb/deb_dpdk/deb_dpdk.yaml
new file mode 100644
index 000000000..d7f9384ee
--- /dev/null
+++ b/jjb/deb_dpdk/deb_dpdk.yaml
@@ -0,0 +1,182 @@
+- project:
+    name: deb_dpdk
+    jobs:
+        - 'deb_dpdk-verify-{stream}-{os}'
+        - 'deb_dpdk-merge-{stream}-{os}'
+        - 'deb_dpdk-verify-image-{stream}-{os}'
+
+    project: 'deb_dpdk'
+    os:
+        - ubuntu1404:
+            repo-os-part: 'ubuntu.trusty.main'
+        - ubuntu1604:
+            repo-os-part: 'ubuntu.xenial.main'
+    stream:
+        - master:
+            branch: 'master'
+            repo-stream-part: 'master'
+        - 1607:
+            branch: 'deb_dpdk_16.07'
+            repo-stream-part: 'deb_dpdk_1607'
+
+- job-template:
+    name: 'deb_dpdk-verify-{stream}-{os}'
+
+    project-type: freestyle
+    node: 'verify-{os}'
+    concurrent: true
+
+    logrotate:
+        daysToKeep: '{build-days-to-keep}'
+        numToKeep: '{build-num-to-keep}'
+        artifactDaysToKeep: '{build-artifact-days-to-keep}'
+        artifactNumToKeep: '{build-artifact-num-to-keep}'
+
+    parameters:
+        - project-parameter:
+            project: '{project}'
+        - gerrit-parameter:
+            branch: '{branch}'
+        - os-parameter:
+            os: '{os}'
+        - repo-name-parameter:
+            repo-name: '{repo-stream-part}.{repo-os-part}'
+
+    scm:
+        - gerrit-trigger-scm:
+            credentials-id: '{ssh-credentials}'
+            refspec: '$GERRIT_REFSPEC'
+            choosing-strategy: 'gerrit'
+
+    wrappers:
+        - ssh-agent-credentials:
+            users:
+                - '{ssh-credentials}'
+        - fdio-infra-wrappers:
+            build-timeout: '{build-timeout}'
+
+    triggers:
+        - gerrit-trigger-patch-submitted:
+            name: '{project}'
+            branch: '{branch}'
+
+    builders:
+        - shell:
+            !include-raw-escape:
+                - include-raw-deb_dpdk-build.sh
+
+- job-template:
+    name: 'deb_dpdk-verify-image-{stream}-{os}'
+
+    project-type: freestyle
+    node: 'verify-image-{os}'
+    concurrent: true
+
+    logrotate:
+        daysToKeep: '{build-days-to-keep}'
+        numToKeep: '{build-num-to-keep}'
+        artifactDaysToKeep: '{build-artifact-days-to-keep}'
+        artifactNumToKeep: '{build-artifact-num-to-keep}'
+
+    parameters:
+        - project-parameter:
+            project: '{project}'
+        - gerrit-parameter:
+            branch: '{branch}'
+        - os-parameter:
+            os: '{os}'
+        - repo-name-parameter:
+            repo-name: '{repo-stream-part}.{repo-os-part}'
+
+    scm:
+        - gerrit-trigger-scm:
+            credentials-id: '{ssh-credentials}'
+            refspec: '$GERRIT_REFSPEC'
+            choosing-strategy: 'gerrit'
+
+    wrappers:
+        - ssh-agent-credentials:
+            users:
+                - '{ssh-credentials}'
+        - fdio-infra-wrappers:
+            build-timeout: '{build-timeout}'
+
+    triggers:
+        - gerrit:
+            server-name: 'Primary'
+            trigger-on:
+                - comment-added-contains-event:
+                    comment-contains-value: 'verify-images'
+            projects:
+              - project-compare-type: 'ANT'
+                project-pattern: '{project}'
+                branches:
+                  - branch-compare-type: 'ANT'
+                    branch-pattern: '**/{branch}'
+            skip-vote:
+                successful: true
+                failed: true
+                unstable: true
+                notbuilt: true
+
+    builders:
+        - shell:
+            !include-raw-escape:
+                - include-raw-deb_dpdk-build.sh
+
+- job-template:
+    name: 'deb_dpdk-merge-{stream}-{os}'
+
+    project-type: freestyle
+    node: 'merge-{os}'
+
+    logrotate:
+        daysToKeep: '{build-days-to-keep}'
+        numToKeep: '{build-num-to-keep}'
+        artifactDaysToKeep: '{build-artifact-days-to-keep}'
+        artifactNumToKeep: '{build-artifact-num-to-keep}'
+
+    parameters:
+        - project-parameter:
+            project: '{project}'
+        - gerrit-parameter:
+            branch: '{branch}'
+        - os-parameter:
+            os: '{os}'
+        - repo-name-parameter:
+            repo-name: '{repo-stream-part}.{repo-os-part}'
+
+    scm:
+        - gerrit-trigger-scm:
+            credentials-id: '{ssh-credentials}'
+            refspec: ''
+            choosing-strategy: 'default'
+
+    wrappers:
+        - ssh-agent-credentials:
+            users:
+                - '{ssh-credentials}'
+        - fdio-infra-wrappers:
+            build-timeout: '{build-timeout}'
+
+    triggers:
+        - gerrit-trigger-patch-merged:
+            name: '{project}'
+            branch: '{branch}'
+
+    builders:
+        - config-file-provider:
+            files:
+              - file-id: 'deb_dpdk-settings'
+                variable: 'SETTINGS_FILE'
+              - file-id: 'global-settings'
+                variable: 'GLOBAL_SETTINGS_FILE'
+        - shell:
+            !include-raw-escape: ../scripts/create_maven_env.sh
+        - inject:
+            properties-file: maven_env.txt
+        - shell:
+            !include-raw-escape:
+                - include-raw-deb_dpdk-build.sh
+                - ../scripts/maven_push_functions.sh
+                - include-raw-deb_dpdk-pkg-push.sh
diff --git a/jjb/deb_dpdk/include-raw-deb_dpdk-build.sh b/jjb/deb_dpdk/include-raw-deb_dpdk-build.sh
new file mode 100644
index 000000000..23b56dabc
--- /dev/null
+++ b/jjb/deb_dpdk/include-raw-deb_dpdk-build.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+# basic build script example
+set -e -o pipefail
+# do nothing but print the current slave hostname
+hostname
+
+echo "cat /etc/bootstrap.sha"
+if [ -f /etc/bootstrap.sha ];then
+    cat /etc/bootstrap.sha
+else
+    echo "Cannot find /etc/bootstrap.sha"
+fi
+
+echo "cat /etc/bootstrap-functions.sha"
+if [ -f /etc/bootstrap-functions.sha ];then
+    cat /etc/bootstrap-functions.sha
+else
+    echo "Cannot find /etc/bootstrap-functions.sha"
+fi
+
+echo "sha1sum of this script: ${0}"
+sha1sum $0
+
+MISSING_PKGS=$(dpkg-checkbuilddeps |& perl -pe 's/^.+://g; s/\(.*?\)//g')
+sudo apt-get install -y ${MISSING_PKGS}
+debuild -uc -us -j4
+
+echo "*******************************************************************"
+echo "* DEB_DPDK BUILD SUCCESSFULLY COMPLETED"
+echo "*******************************************************************"
diff --git a/jjb/deb_dpdk/include-raw-deb_dpdk-pkg-push.sh b/jjb/deb_dpdk/include-raw-deb_dpdk-pkg-push.sh
new file mode 100644
index 000000000..8e03405f2
--- /dev/null
+++ b/jjb/deb_dpdk/include-raw-deb_dpdk-pkg-push.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+DEBS=$(find . -type f -iname '*.deb')
+echo "Found DEBS=${DEBS}"
+for i in $DEBS
+do
+    push_deb "$i"
+done
\ No newline at end of file
diff --git a/jjb/nsh_sfc/include-raw-nsh_sfc-docs.sh b/jjb/nsh_sfc/include-raw-nsh_sfc-docs.sh
new file mode 100644
index 000000000..119a41d41
--- /dev/null
+++ b/jjb/nsh_sfc/include-raw-nsh_sfc-docs.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+set -xe -o pipefail
+[ "$PROJECT_NAME" ] || PROJECT_NAME="nsh_sfc"
+[ "$DOCS_REPO_URL" ] || DOCS_REPO_URL="https://nexus.fd.io/content/sites/site"
+[ "$PROJECT_PATH" ] || PROJECT_PATH=io/fd/${PROJECT_NAME}
+[ "$DOC_DIR" ] || DOC_DIR=build-root/docs/html
+[ "$SITE_DIR" ] || SITE_DIR=build-root/docs/deploy-site/
+[ "$RESOURCES_DIR" ] || RESOURCES_DIR=${SITE_DIR}/src/site/resources
+[ "$MVN" ] || MVN="/opt/apache/maven/bin/mvn"
+[ "$VERSION" ] || VERSION=$(./nsh-plugin/packaging/scripts/version)
+
+make doxygen
+if [[ ${JOB_NAME} == *merge* ]]; then
+  mkdir -p $(dirname ${RESOURCES_DIR})
+  mv -f ${DOC_DIR} ${RESOURCES_DIR}
+  cd ${SITE_DIR}
+  find . -type f '(' -name '*.md5' -o -name '*.dot' -o -name '*.map' ')' -delete
+  cat > pom.xml << EOF
+  <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>io.fd.${PROJECT_NAME}</groupId>
+    <artifactId>docs</artifactId>
+    <version>1.0.0</version>
+    <packaging>pom</packaging>
+
+    <properties>
+      <generateReports>false</generateReports>
+    </properties>
+
+    <build>
+      <extensions>
+        <extension>
+          <groupId>org.apache.maven.wagon</groupId>
+           <artifactId>wagon-webdav-jackrabbit</artifactId>
+           <version>2.9</version>
+        </extension>
+      </extensions>
+    </build>
+    <distributionManagement>
+      <site>
+        <id>fdio-site</id>
+        <url>dav:${DOCS_REPO_URL}/${PROJECT_PATH}/${VERSION}</url>
+      </site>
+    </distributionManagement>
+  </project>
+EOF
+  ${MVN} site:site site:deploy -gs "${GLOBAL_SETTINGS_FILE}" -s "${SETTINGS_FILE}" -T 4C
+  cd -
+fi
diff --git a/jjb/nsh_sfc/include-raw-nsh_sfc-pkg-push.sh b/jjb/nsh_sfc/include-raw-nsh_sfc-pkg-push.sh
index c4f4885b0..4cfd30007 100644
--- a/jjb/nsh_sfc/include-raw-nsh_sfc-pkg-push.sh
+++ b/jjb/nsh_sfc/include-raw-nsh_sfc-pkg-push.sh
@@ -1,9 +1,11 @@
 #!/bin/bash
 # basic build script example
 set -e -o pipefail
+echo "Looking for jars/debs/rpms to publish"
 if [ "${OS}" == "ubuntu1404" ]; then
     # Find the files
     DEBS=$(find . -type f -iname '*.deb')
+    echo "Found DEBS=${DEBS}"
     for i in $DEBS
     do
         push_deb "$i"
@@ -21,6 +23,7 @@ elif [ "${OS}" == "ubuntu1604" ]; then
 
     # Find the files
     DEBS=$(find . -type f -iname '*.deb')
+    echo "Found DEBS=${DEBS}"
     for i in $DEBS
     do
         push_deb "$i"
@@ -31,6 +34,9 @@ elif [ "${OS}" == "centos7" ]; then
     RPMS=$(find . -type f -iname '*.rpm')
     SRPMS=$(find . -type f -iname '*.srpm')
     SRCRPMS=$(find . -type f -name '*.src.rpm')
+    echo "Found RPMS=${RPMS}"
+    echo "Found SRPMS=${SRPMS}"
+    echo "Found SRCRPMS=${SRCRPMS}"
     for i in $RPMS $SRPMS $SRCRPMS
     do
         push_rpm "$i"
diff --git a/jjb/nsh_sfc/nsh_sfc.yaml b/jjb/nsh_sfc/nsh_sfc.yaml
index 5fad549be..eb800088d 100644
--- a/jjb/nsh_sfc/nsh_sfc.yaml
+++ b/jjb/nsh_sfc/nsh_sfc.yaml
@@ -5,6 +5,8 @@
         - 'nsh_sfc-merge-{stream}-{os}'
         - 'nsh_sfc-integration-{stream}-{os}'
         - 'nsh_sfc-verify-image-{stream}-{os}'
+        - 'nsh_sfc-verify-docs-{stream}'
+        - 'nsh_sfc-merge-docs-{stream}'
 
     project: 'nsh_sfc'
     os:
@@ -282,3 +284,126 @@
                 - ../scripts/cleanup_vpp_plugin_dev_env.sh
                 - ../scripts/maven_push_functions.sh
                 - include-raw-nsh_sfc-pkg-push.sh
+
+- job-template:
+    name: 'nsh_sfc-verify-docs-{stream}'
+
+    project-type: freestyle
+    node: 'verify-ubuntu1604'
+    concurrent: true
+
+    logrotate:
+        daysToKeep: '{build-days-to-keep}'
+        numToKeep: '{build-num-to-keep}'
+        artifactDaysToKeep: '{build-artifact-days-to-keep}'
+        artifactNumToKeep: '{build-artifact-num-to-keep}'
+
+    parameters:
+        - project-parameter:
+            project: '{project}'
+        - gerrit-parameter:
+            branch: '{branch}'
+        - os-parameter:
+            os: 'ubuntu1604'
+
+    scm:
+        - gerrit-trigger-scm:
+            credentials-id: '{ssh-credentials}'
+            refspec: ''
+            choosing-strategy: 'default'
+
+    wrappers:
+        - ssh-agent-credentials:
+            users:
+                - '{ssh-credentials}'
+
+    triggers:
+        - gerrit-trigger-patch-submitted:
+            name: '{project}'
+            branch: '{branch}'
+
+    builders:
+        - maven-target:
+            maven-version: '{mvn33}'
+            goals: '--version'
+            settings: 'vpp-settings'
+            settings-type: cfp
+            global-settings: 'global-settings'
+            global-settings-type: cfp
+        - provide-maven-settings:
+            settings-file: 'vpp-settings'
+            global-settings-file: 'global-settings'
+        - shell:
+            !include-raw-escape: include-raw-nsh_sfc-docs.sh
+
+
+
+
+- job-template:
+    name: 'nsh_sfc-merge-docs-{stream}'
+
+    project-type: freestyle
+    node: 'verify-ubuntu1604'
+    concurrent: true
+
+    logrotate:
+        daysToKeep: '{build-days-to-keep}'
+        numToKeep: '{build-num-to-keep}'
+        artifactDaysToKeep: '{build-artifact-days-to-keep}'
+        artifactNumToKeep: '{build-artifact-num-to-keep}'
+
+    parameters:
+        - project-parameter:
+            project: '{project}'
+        - gerrit-parameter:
+            branch: '{branch}'
+        - os-parameter:
+            os: 'ubuntu1604'
+
+    scm:
+        - gerrit-trigger-scm:
+            credentials-id: '{ssh-credentials}'
+            refspec: ''
+            choosing-strategy: 'default'
+
+    wrappers:
+        - ssh-agent-credentials:
+            users:
+                - '{ssh-credentials}'
+
+    triggers:
+        - reverse:
+            jobs: 'nsh_sfc-merge-{stream}-ubuntu1404'
+            result: 'success'
+        - gerrit:
+            server-name: 'Primary'
+            trigger-on:
+                - comment-added-contains-event:
+                    comment-contains-value: 'run-docs'
+            projects:
+              - project-compare-type: 'ANT'
+                project-pattern: '{project}'
+                branches:
+                  - branch-compare-type: 'ANT'
+                    branch-pattern: '**/{branch}'
+            skip-vote:
+                successful: true
+                failed: true
+                unstable: true
+                notbuilt: true
+
+    builders:
+        - maven-target:
+            maven-version: '{mvn33}'
+            goals: '--version'
+            settings: 'vpp-settings'
+            settings-type: cfp
+            global-settings: 'global-settings'
+            global-settings-type: cfp
+        - provide-maven-settings:
+            settings-file: 'vpp-settings'
+            global-settings-file: 'global-settings'
+        - shell:
+            !include-raw-escape: include-raw-nsh_sfc-docs.sh
+
+
diff --git a/jjb/scripts/maven_push_functions.sh b/jjb/scripts/maven_push_functions.sh
index 22a22cbf8..5687a562e 100644
--- a/jjb/scripts/maven_push_functions.sh
+++ b/jjb/scripts/maven_push_functions.sh
@@ -70,8 +70,10 @@ function push_deb ()
     basefile=$(basename -s .deb "$debfile")
     artifactId=$(echo "$basefile" | cut -f 1 -d '_')
     version=$(echo "$basefile" | cut -f 2- -d '_')
+    file_type=deb
+    classifier=deb
 
-    push_file "$debfile" "$repoId" "$url" "$version" "$artifactId" deb deb
+    push_file "$debfile" "$repoId" "$url" "$version" "$artifactId" "$file_type" "$classifier"
 }
 
 function push_rpm ()
diff --git a/jjb/vpp/include-raw-vpp-maven-push.sh b/jjb/vpp/include-raw-vpp-maven-push.sh
index ce07fee65..24bd5c776 100644
--- a/jjb/vpp/include-raw-vpp-maven-push.sh
+++ b/jjb/vpp/include-raw-vpp-maven-push.sh
@@ -10,79 +10,6 @@ GROUP_ID="io.fd.${PROJECT}"
 BASEURL="${NEXUSPROXY}/content/repositories/fd.io."
 BASEREPOID='fdio-'
 
-function push_file ()
-{
-    push_file=$1
-    repoId=$2
-    url=$3
-    version=$4
-    artifactId=$5
-    file_type=$6
-    classifier=$7
-
-    if [ "$classifier" ]; then
-        d_classifier="-Dclassifier=$7"
-    fi
-
-    # Disable checks for doublequote to prevent glob / splitting
-    # shellcheck disable=SC2086
-    $MVN org.apache.maven.plugins:maven-deploy-plugin:deploy-file \
-        -Dfile=$push_file -DrepositoryId=$repoId \
-        -Durl=$url -DgroupId=$GROUP_ID \
-        -Dversion=$version -DartifactId=$artifactId \
-        -Dtype=$file_type $d_classifier\
-        -gs $GLOBAL_SETTINGS_FILE -s $SETTINGS_FILE
-
-    # make sure the script bombs if we fail an upload
-    if [ "$?" != '0' ]; then
-        echo "ERROR: There was an error with the upload"
-        exit 1
-    fi
-}
-
-function push_jar ()
-{
-    jarfile=$1
-    repoId="${BASEREPOID}snapshot"
-    url="${BASEURL}snapshot"
-
-    basefile=$(basename -s .jar "$jarfile")
-    artifactId=$(echo "$basefile" | rev | cut -d '-' -f 2-  | rev)
-    version=$(echo "$basefile" | rev | cut -d '-' -f 1  | rev)
-
-    push_file "$jarfile" "$repoId" "$url" "${version}-SNAPSHOT" "$artifactId" jar
-}
-
-function push_deb ()
-{
-    debfile=$1
-    repoId="fd.io.${REPO_NAME}"
-    url="${BASEURL}${REPO_NAME}"
-
-    basefile=$(basename -s .deb "$debfile")
-    artifactId=$(echo "$basefile" | cut -f 1 -d '_')
-    version=$(echo "$basefile" | cut -f 2- -d '_')
-
-    push_file "$debfile" "$repoId" "$url" "$version" "$artifactId" deb deb
-}
-
-function push_rpm ()
-{
-    rpmfile=$1
-    repoId="fd.io.${REPO_NAME}"
-    url="${BASEURL}${REPO_NAME}"
-
-    if grep -qE '\.s(rc\.)?rpm' <<<"$rpmfile"
-    then
-        rpmrelease=$(rpm -qp --queryformat="%{release}.src" "$rpmfile")
-    else
-        rpmrelease=$(rpm -qp --queryformat="%{release}.%{arch}" "$rpmfile")
-    fi
-    artifactId=$(rpm -qp --queryformat="%{name}" "$rpmfile")
-    version=$(rpm -qp --queryformat="%{version}" "$rpmfile")
-    push_file "$rpmfile" "$repoId" "$url" "${version}-${rpmrelease}" "$artifactId" rpm
-}
-
 if [ "${OS}" == "ubuntu1404" ]; then
     # Find the files
     JARS=$(find . -type f -iname '*.jar')
diff --git a/jjb/vpp/vpp.yaml b/jjb/vpp/vpp.yaml
index 5df99688f..ff131b2f7 100644
--- a/jjb/vpp/vpp.yaml
+++ b/jjb/vpp/vpp.yaml
@@ -101,6 +101,13 @@
             !include-raw-escape: include-raw-vpp-checkstyle.sh
         - shell:
             !include-raw-escape: include-raw-vpp-build.sh
+    publishers:
+        - archive:
+            artifacts: 'build-root/*.rpm,build-root/*.deb'
+            allow-empty: 'true'
+            fingerprint: false
+            only-if-success: true
+            default-excludes: false
 
 - job-template:
     name: 'vpp-verify-image-{stream}-{os}'
@@ -218,7 +225,9 @@
             settings-file: 'vpp-settings'
             global-settings-file: 'global-settings'
         - shell:
-            !include-raw-escape: include-raw-vpp-maven-push.sh
+            !include-raw-escape:
+                - ../scripts/maven_push_functions.sh
+                - include-raw-vpp-maven-push.sh
     publishers:
         - archive:
             artifacts: 'build-root/*.rpm,build-root/*.deb'
diff --git a/vagrant/lib/bootstrap-functions.sh b/vagrant/lib/bootstrap-functions.sh
index 71fa0625d..c3e1c8584 100644
--- a/vagrant/lib/bootstrap-functions.sh
+++ b/vagrant/lib/bootstrap-functions.sh
@@ -2,6 +2,11 @@
 
 do_setup() {
     echo "127.0.1.1 $(hostname) # temporary" >> /etc/hosts
+
+    # Dead peer detection
+    echo "TCPKeepAlive        true" >> /etc/ssh/ssh_config
+    echo "ServerAliveCountMax 30"   >> /etc/ssh/ssh_config
+    echo "ServerAliveInterval 10"   >> /etc/ssh/ssh_config
 }
 
 do_mvn_install() {
@@ -142,6 +147,11 @@ deb_install_pkgs() {
     # Install to allow the vpp-docs job to zip up docs to push them
     PACKAGES="$PACKAGES zip"
 
+    # Install for deb_dpdk debian package buiding
+    PACKAGES="$PACKAGES dpkg-dev dh-python inkscape libcap-dev libpcap-dev"
+    PACKAGES="$PACKAGES libxen-dev libxenstore3.0 python-sphinx python-sphinx-rtd-theme"
+    PACKAGES="$PACKAGES texlive-fonts-recommended texlive-latex-extra"
+
     echo '---> Installing packages'
     # disable double quoting check
     # shellcheck disable=SC2086