JJB: Include perf analysis per job

+ Useful for performance analysis before run.
  This should help to analyze machine load before starting
  payload by comparing conditions from running jobs.
- Requirements: sysstat (apt install sysstat)
- Remove extraneous python import from
  backup_upload_archives.sh
- Fix sar issue on centos-7
- Fix debian9 lftools install / upgrade issue

Signed-off-by: pmikus <[email protected]>
Signed-off-by: Dave Wallace <[email protected]>
Change-Id: I6b61f99b9b2cde2bac3dacdefaa926e9afbb157e

diff --git a/docker/scripts/dbld_lfit_requirements.sh b/docker/scripts/dbld_lfit_requirements.sh
index 7067d99..278b78e 100755 (executable)
--- a/docker/scripts/dbld_lfit_requirements.sh
+++ b/docker/scripts/dbld_lfit_requirements.sh
@@ -54,5 +54,10 @@ OLD_PATH="$PATH"
 python3 -m venv "$LF_VENV"
 PATH="$LF_VENV/bin:$PATH"
 python3 -m pip install --upgrade pip
+# on Debian-9 lftools needs to be installed before it is upgraded
+# in order to get the latest version.
+if [ "$OS_ID" = "debian"  ] && [ "$OS_VERSION_ID" = "9" ] ; then
+    python3 -m pip install lftools
+fi
 python3 -m pip install --upgrade --upgrade-strategy eager lftools
 PATH="$OLD_PATH"

diff --git a/docker/scripts/lib_apt.sh b/docker/scripts/lib_apt.sh
index 725aedc..099a85a 100644 (file)
--- a/docker/scripts/lib_apt.sh
+++ b/docker/scripts/lib_apt.sh
@@ -178,6 +178,7 @@ RUN apt-get update -qq \\
              ruby-dev \\
              software-properties-common \\
              sshpass \\
+             sysstat \\
              sudo \\
              texlive-fonts-extra \\
              texlive-fonts-recommended \\

diff --git a/docker/scripts/lib_dnf.sh b/docker/scripts/lib_dnf.sh
index 4520421..c606216 100644 (file)
--- a/docker/scripts/lib_dnf.sh
+++ b/docker/scripts/lib_dnf.sh
@@ -126,6 +126,7 @@ RUN dnf install -y \\
         rsync \\
         ruby-devel \\
         sshpass \\
+        sysstat \\
         sudo \\
         tree \\
         unzip \\

diff --git a/docker/scripts/lib_yum.sh b/docker/scripts/lib_yum.sh
index 27c297a..fe0bc53 100644 (file)
--- a/docker/scripts/lib_yum.sh
+++ b/docker/scripts/lib_yum.sh
@@ -120,6 +120,7 @@ RUN yum install -y \\
         rake \\
         ruby-devel \\
         sshpass \\
+        sysstat \\
         sudo \\
         tree \\
         unzip \\

diff --git a/jjb/scripts/backup_upload_archives.sh b/jjb/scripts/backup_upload_archives.sh
index b496749..b660f86 100755 (executable)
--- a/jjb/scripts/backup_upload_archives.sh
+++ b/jjb/scripts/backup_upload_archives.sh
@@ -35,7 +35,6 @@ cat >$PYTHON_SCRIPT <<'END_OF_PYTHON_SCRIPT'
 import argparse
 import gzip
 import os
-import requests
 from mimetypes import MimeTypes
 
 from boto3 import resource

diff --git a/jjb/scripts/post_build_executor_info.sh b/jjb/scripts/post_build_executor_info.sh
index f986666..fef7981 100755 (executable)
--- a/jjb/scripts/post_build_executor_info.sh
+++ b/jjb/scripts/post_build_executor_info.sh
@@ -45,3 +45,14 @@ if [ -n "$(which ccache)" ] ; then
 fi
 
 echo "$long_line"
+
+# Avoid sar anomaly on centos-7 in global-jjb/shell/logs-deploy.sh
+#
+# Note: VPP 20.09 will be removed in the next release cycle (21.10),
+#       therefore this hack is better than polluting the docker image
+#       build scripts with code to avoid installing sysstat on centos-7.
+#
+# TODO: Remove when vpp-*-2009-centos7-x86_64 jobs are removed
+if [ "$OS_ID" = "centos" ] && [ "$OS_VERSION_ID" = "7"  ] ; then
+    sudo yum remove -y sysstat >& /dev/null || true
+fi

diff --git a/jjb/scripts/setup_executor_env.sh b/jjb/scripts/setup_executor_env.sh
index a6b8216..4d3cb6e 100755 (executable)
--- a/jjb/scripts/setup_executor_env.sh
+++ b/jjb/scripts/setup_executor_env.sh
@@ -25,6 +25,7 @@ file_delimiter="----- %< -----"
 long_line="************************************************************************"
 downloads_cache="/root/Downloads"
 
+# Node info
 echo "$long_line"
 echo "Executor Runtime Attributes:"
 echo "OS: $OS_ID-$OS_VERSION_ID"
@@ -41,6 +42,35 @@ else
     echo "Unknown Executor: '$dockerfile' not found!"
 fi
 
+# Performance analysis
+perf_trials=2
+perf_interval=1
+if [ "$OS_ID" == "ubuntu" ] || [ "$OS_ID" = "debian" ] ; then
+    SYSSTAT_PATH="/var/log/sysstat"
+elif [ "$OS_ID" == "centos" ] ; then
+    if [ "$OS_VERSION_ID" = "7" ] ; then
+        SYSSTAT_PATH="/var/log/sa/sa02"
+    else
+        SYSSTAT_PATH="/var/log/sa"
+    fi
+fi
+echo "$long_line"
+echo "Virtual memory stat"
+vmstat ${perf_interval} ${perf_trials}
+echo "CPU time breakdowns per CPU"
+mpstat -P ALL ${perf_interval}  ${perf_trials}
+echo "Per-process summary"
+pidstat ${perf_interval} ${perf_trials}
+echo "Block device stats"
+iostat -xz ${perf_interval} ${perf_trials}
+echo "Memory utilization"
+free -m
+echo "Network interface throughput"
+sar -n DEV -o ${SYSSTAT_PATH} ${perf_interval} ${perf_trials}
+echo "TCP metrics"
+sar -n TCP,ETCP -o ${SYSSTAT_PATH} ${perf_interval} ${perf_trials}
+
+# SW stack
 echo "$long_line"
 echo "Executor package list:"
 if [ "$OS_ID" == "ubuntu" ] || [ "$OS_ID" = "debian" ] ; then