make test: add scripts for easy test looping

Allows easy running of test(s) in a loop with configurable
action (e.g. git pull) run between test runs and possible
email notification on failure.

Usage:

test-loop.sh [-p <pre-exec-cmd>] [-m <email>] -- <make test options>

Example:

Run 'make test-debug' in a loop until a failure is encountered,
upon which an email is fired to ksekera@cisco.com. In between test
runs, update the workspace using via 'git pull' and if anything changed,
perform 'git clean' before running another 'make test-debug':

test/scripts/test-loop.sh -p test/scripts/git_pull_or_clean.sh \
-m ksekera@cisco.com -- test-debug

Change-Id: I114321c6c152d2c7e181e915fc8c51aab1ff3693
Signed-off-by: Klement Sekera <ksekera@cisco.com>

diff --git a/test/scripts/git_pull_or_clean.sh b/test/scripts/git_pull_or_clean.sh
new file mode 100755 (executable)
index 0000000..489091f
--- /dev/null
+++ b/test/scripts/git_pull_or_clean.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+git pull | grep -q -v 'Already up-to-date.' || git clean -dfX */

diff --git a/test/scripts/test-loop.sh b/test/scripts/test-loop.sh
new file mode 100755 (executable)
index 0000000..9f22883
--- /dev/null
+++ b/test/scripts/test-loop.sh
@@ -0,0 +1,119 @@
+#!/bin/bash
+
+function usage() {
+ echo "$0" 1>&2
+ echo "" 1>&2
+ echo "Usage: $0 [-p <pre-exec-cmd>] [-m <email>] -- <make test options>" 1>&2
+ echo "" 1>&2
+ echo "Parameters:" 1>&2
+ echo "    -p <pre-exec-cmd> - run a command before each test loop (e.g. 'git pull')" 1>&2
+ echo "    -m <email>        - if set, email is sent to this address on failure" 1>&2
+ echo "" 1>&2
+ echo "Example:" 1>&2
+ echo "    $0 -m <somebody@cisco.com> -- test-debug TEST=l2bd"
+ exit 1;
+}
+
+PRE_EXEC_CMD=""
+EMAIL=""
+
+while getopts "p:m:h" o; do
+       case "${o}" in
+       p)
+               PRE_EXEC_CMD=${OPTARG}
+               ;;
+       m)
+               regex="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$"
+               m=${OPTARG}
+               if [[ ! $m =~ $regex ]]
+               then
+                       echo "Invalid -m parameter value: \`$m'" >&2
+                       usage
+               fi
+               EMAIL="$m"
+               ;;
+       h)
+               usage
+               ;;
+       ?)
+               usage
+               ;;
+esac
+       done
+shift $((OPTIND-1))
+
+if ! echo $* | grep test >/dev/null
+then
+        echo "Error: command line doesn't look right - should contain \`test' token..." >&2
+       usage
+fi
+
+function finish {
+       NOW=`date +%s`
+       RUNTIME=$((NOW - START))
+       AVG=$(echo "scale=2; $RUNTIME/$COUNT" | bc)
+       OUT="*********************************************************************"
+       OUT="$OUT\n* tail -n 30 $TMP:"
+       OUT="$OUT\n*********************************************************************"
+       OUT="$OUT\n`tail -n 30 $TMP`"
+       OUT="$OUT\n*********************************************************************"
+       OUT="$OUT\n* Total runtime: ${RUNTIME}s"
+       OUT="$OUT\n* Iterations:    ${COUNT}"
+       OUT="$OUT\n* Average time:  ${AVG}s"
+       OUT="$OUT\n* Log file:      ${TMP}"
+       OUT="$OUT\n*********************************************************************"
+       echo -e "$OUT"
+       if [[ "$EMAIL" != "" && "$REASON" != "" ]]
+       then
+               SUBJECT="test loop finished ($REASON)"
+               echo -e "$OUT" | mail -s "$SUBJECT" $EMAIL
+       fi
+}
+
+trap "echo Caught signal, exiting...; REASON=\"received signal\"; finish; exit -1" SIGINT SIGTERM
+
+TMP=`mktemp`
+START=`date +%s`
+COUNT=0
+
+if ! test -f "$TMP"
+then
+       echo "Couldn't create temporary file!"
+       exit -1
+fi
+
+echo "Temporary file is $TMP"
+CMD="make $*"
+echo "Command line is \`$CMD'"
+
+REASON=""
+while true
+do
+       COUNT=$((COUNT+1))
+       BEFORE=`date +%s`
+       if [[ "$PRE_EXEC_CMD" != "" ]]
+       then
+               echo "Executing \`$PRE_EXEC_CMD' before test.."
+               if ! ($PRE_EXEC_CMD 2>$TMP 1>$TMP)
+               then
+                       echo "\`$PRE_EXEC_CMD' failed!" >&2
+                       REASON="$PRE_EXEC_CMD failed"
+                       break
+               fi
+       fi
+       echo -n "Running test iteration #$COUNT..."
+       if ! ($CMD 2>$TMP 1>$TMP)
+       then
+               AFTER=`date +%s`
+               RUNTIME=$((AFTER-BEFORE))
+               echo "FAILED! (after ${RUNTIME}s)"
+               REASON="test failed"
+               break
+       fi
+       AFTER=`date +%s`
+       RUNTIME=$((AFTER-BEFORE))
+       echo "PASSED (after ${RUNTIME}s)"
+done
+
+finish
+exit 1