Add MAKE_PARALLEL_JOBS parameter

VPP uses the MAKE_PARALLEL_JOBS variable to limit the number of cpus
used for build. Make that varialbe configurable as job paramter instead
of setting it in script for easier job management.

Change-Id: Iaaafdb216c3af9244827b270c540a70b5844021f
Signed-off-by: Juraj Linkeš <[email protected]>

diff --git a/jjb/global-defaults.yaml b/jjb/global-defaults.yaml
index cba706d..d77b53d 100644 (file)
--- a/jjb/global-defaults.yaml
+++ b/jjb/global-defaults.yaml
@@ -24,6 +24,9 @@
     # voting
     skip-vote: false
 
+    # build
+    make-parallel-jobs: 4
+
     # CSIT
     csit-perf-trial-duration: 1.0
     csit-perf-trial-multiplicity: 1

diff --git a/jjb/global-macros.yaml b/jjb/global-macros.yaml
index 5247bfc..8354f5c 100644 (file)
--- a/jjb/global-macros.yaml
+++ b/jjb/global-macros.yaml
@@ -32,6 +32,14 @@
           default: '{cc}'
           description: "CC parameter, can be gnu or clang"
 
+- parameter:
+    name: make-parallel-jobs-parameter
+    parameters:
+      - string:
+          name: MAKE_PARALLEL_JOBS
+          default: '{make-parallel-jobs}'
+          description: "The number of cores to use for builds."
+
 - parameter:
     name: csit-fail-on-crc-mismatch-parameter
     parameters:

diff --git a/jjb/scripts/vpp/arm-build.sh b/jjb/scripts/vpp/arm-build.sh
index 2abe610..798e90e 100644 (file)
--- a/jjb/scripts/vpp/arm-build.sh
+++ b/jjb/scripts/vpp/arm-build.sh
@@ -46,11 +46,17 @@ echo "IS_CSIT_VPP_JOB=${IS_CSIT_VPP_JOB}"
 # If and only if we are doing verify *after* make verify was made to work
 # and we are not a CSIT job just building packages, then use make verify,
 # else use the old build-root/vagrant/build.sh
+if [ "x${MAKE_PARALLEL_JOBS}" != "x" ]
+then
+  echo "Building with MAKE_PARALLEL_JOBS=${MAKE_PARALLEL_JOBS}"
+  TEST_JOBS="TEST_JOBS=${MAKE_PARALLEL_JOBS}"
+fi
+
 if (git log --oneline | grep 37682e1 > /dev/null 2>&1) && \
         [ "x${IS_CSIT_VPP_JOB}" != "xTrue" ]
 then
     echo "Building using \"make verify\""
-    [ "x${DRYRUN}" == "xTrue" ] || make UNATTENDED=yes verify SKIP_AARCH64=yes
+    [ "x${DRYRUN}" == "xTrue" ] || make UNATTENDED=yes SKIP_AARCH64=yes ${TEST_JOBS} verify
 else
     echo "Building using \"make build-root/vagrant/build.sh\""
     [ "x${DRYRUN}" == "xTrue" ] || make UNATTENDED=yes install-dep

diff --git a/jjb/scripts/vpp/build.sh b/jjb/scripts/vpp/build.sh
index 6fdaec1..9fd989f 100644 (file)
--- a/jjb/scripts/vpp/build.sh
+++ b/jjb/scripts/vpp/build.sh
@@ -40,6 +40,11 @@ echo "IS_CSIT_VPP_JOB=${IS_CSIT_VPP_JOB}"
 # If and only if we are doing verify *after* make verify was made to work
 # and we are not a CSIT job just building packages, then use make verify,
 # else use make pkg-verify.
+if [ "x${MAKE_PARALLEL_JOBS}" != "x" ]
+then
+  echo "Building with MAKE_PARALLEL_JOBS=${MAKE_PARALLEL_JOBS}"
+fi
+
 if (git log --oneline | grep 37682e1 > /dev/null 2>&1) && \
         [ "x${IS_CSIT_VPP_JOB}" != "xTrue" ]
 then

diff --git a/jjb/scripts/vpp/parallel-vars.sh b/jjb/scripts/vpp/parallel-vars.sh
deleted file mode 100644 (file)
index 1a5de14..0000000
--- a/jjb/scripts/vpp/parallel-vars.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-CORES=4
-if [[ $(uname -m) == 'aarch64' ]]
-then
-    CORES=16
-fi
-
-echo "Using MAKE_PARALLEL_FLAGS='-j $CORES' TEST_JOBS=$CORES for parallel build/test"
-
-export MAKE_PARALLEL_FLAGS="-j $CORES"
-export TEST_JOBS=$CORES

diff --git a/jjb/vpp/vpp.yaml b/jjb/vpp/vpp.yaml
index 70c0dda..216ec40 100644 (file)
--- a/jjb/vpp/vpp.yaml
+++ b/jjb/vpp/vpp.yaml
@@ -95,7 +95,8 @@
 - project:
     name: vpp-perpatch
     jobs:
-      - 'vpp-csit-verify-device-{stream}-{device-node-arch}'
+      - 'vpp-csit-verify-device-{stream}-{device-node-arch}':
+          make-parallel-jobs: '32'
       - 'vpp-csit-verify-perf-{stream}-{node-arch}'
     project: 'vpp'
     os: ubuntu1804
@@ -133,6 +134,7 @@
       - 3n-dnv
       - 3n-tsh:
           executor: 'ubuntu1804arm-s'
+          make-parallel-jobs: '16'
     exclude:
       - stream: '1904'
         os: 'ubuntu1804'
@@ -189,6 +191,7 @@
     os:
       - ubuntu1804:
           repo-os-part: 'ubuntu-arm.bionic.main'
+    make-parallel-jobs: '16'
 
 # TODO: Merge with (e.g.) vpp-perpatch or similar,
 # when number of stable branches supported matches.
@@ -257,6 +260,8 @@
           name: ARCHIVE_ARTIFACTS
           default: '{archive-artifacts}'
           description: Artifacts to archive to the logs server.
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
 
     scm:
       - gerrit-trigger-scm:
@@ -314,6 +319,8 @@
           name: ARCHIVE_ARTIFACTS
           default: '{archive-artifacts}'
           description: Artifacts to archive to the logs server.
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
 
     scm:
       - gerrit-trigger-scm:
@@ -388,6 +395,9 @@
           name: ARCHIVE_ARTIFACTS
           default: '{archive-artifacts}'
           description: Artifacts to archive to the logs server.
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
+      - gerrit-refspec-parameter
 
     scm:
       - gerrit-trigger-scm:
@@ -413,8 +423,6 @@
           - ../scripts/setup_vpp_dpdk_dev_env.sh
       - shell:
           !include-raw-escape:
-          - ../scripts/vpp/parallel-vars.sh
-          # Do not split, the line below wants env vars from the line above.
           - ../scripts/vpp/arm-build.sh
 
     publishers:
@@ -452,6 +460,8 @@
           name: ARCHIVE_ARTIFACTS
           default: '{archive-artifacts}'
           description: Artifacts to archive to the logs server.
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
 
     scm:
       - gerrit-trigger-scm:
@@ -485,8 +495,6 @@
           - ../scripts/setup_vpp_dpdk_dev_env.sh
       - shell:
           !include-raw-escape:
-          - ../scripts/vpp/parallel-vars.sh
-          # Do not split, the line below wants env vars from the line above.
           - ../scripts/vpp/arm-build-no-test.sh
 
       - provide-maven-settings:
@@ -629,6 +637,8 @@
           name: ARCHIVE_ARTIFACTS
           default: '{archive-artifacts}'
           description: Artifacts to archive to the logs server.
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
 
     scm:
       - gerrit-trigger-scm:
@@ -690,6 +700,9 @@
           name: ARCHIVE_ARTIFACTS
           default: '{archive-artifacts}'
           description: Artifacts to archive to the logs server.
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
+      - gerrit-refspec-parameter
 
     scm:
       - gerrit-trigger-scm:
@@ -746,6 +759,8 @@
           repo-name: '{repo-stream-part}.ubuntu.bionic.main'
       - stream-parameter:
           stream: '{stream}'
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
 
     scm:
       - gerrit-trigger-scm:
@@ -907,6 +922,8 @@
           name: ARCHIVE_ARTIFACTS
           default: '{archive-artifacts}'
           description: Artifacts to archive to the logs server.
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
 
     scm:
       - gerrit-trigger-scm:
@@ -986,6 +1003,8 @@
           name: ARCHIVE_ARTIFACTS
           default: '{archive-artifacts}'
           description: Artifacts to archive to the logs server.
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
 
     scm:
       - gerrit-trigger-scm:
@@ -1076,6 +1095,8 @@
           name: ARCHIVE_ARTIFACTS
           default: '{archive-artifacts}'
           description: Artifacts to archive to the logs server.
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
 
     scm:
       - gerrit-trigger-scm:
@@ -1162,6 +1183,8 @@
           name: ARCHIVE_ARTIFACTS
           default: '{archive-artifacts}'
           description: Artifacts to archive to the logs server.
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
 
     scm:
       - gerrit-trigger-scm: