l4p/tcp_ofo: Tests for ofo and loss of pkts 83/22983/6
authorMariusz Drost <mariuszx.drost@intel.com>
Fri, 25 Oct 2019 12:55:38 +0000 (13:55 +0100)
committerMariusz Drost <mariuszx.drost@intel.com>
Thu, 12 Dec 2019 09:44:29 +0000 (09:44 +0000)
Tests for packets that are out of order and lost. They validate data
send over netcat (TCP) to TLDK/l4fwd app in echo mode. Data is set to
be reordered or loss some percentage of packets.

Signed-off-by: Mariusz Drost <mariuszx.drost@intel.com>
Change-Id: I68f80d1fb75d5e9e5ed6d052ddcdc60588284f78

README
examples/l4fwd/test/config.sh [new file with mode: 0644]
examples/l4fwd/test/example_env_vars [new file with mode: 0644]
examples/l4fwd/test/nctxrx.sh [new file with mode: 0644]
examples/l4fwd/test/run_test.sh [new file with mode: 0644]

diff --git a/README b/README
index 2ca150b..18cb877 100644 (file)
--- a/README
+++ b/README
@@ -78,7 +78,9 @@
    +----examples
    |      |
    |      +--l4fwd - sample app to demonstrate and test libtle_l4p TCP/UDP
-   |            usage (refer to examples/l4fwd/README for more information)
+   |           | usage (refer to examples/l4fwd/README for more information)
+   |           |
+   |           +--test - function test scripts for l4lib using l4fwd
    |
    +----test - unit-tests
    |      |
diff --git a/examples/l4fwd/test/config.sh b/examples/l4fwd/test/config.sh
new file mode 100644 (file)
index 0000000..5932936
--- /dev/null
@@ -0,0 +1,444 @@
+#! /bin/bash
+
+# hardcoded variables which can be changed by the user if needed---------------
+
+# DPDK port to be used
+DPDK_PORT=0
+
+# TCP port to be used
+TCP_PORT=6000
+
+# local interface addresses to set
+LOCAL_IPV4=192.168.1.60
+LOCAL_IPV6=fd12:3456:789a:0001:0000:0000:0000:0060
+
+# remote interface addresses to set
+REMOTE_IPV4=192.168.1.64
+REMOTE_IPV6=fd12:3456:789a:0001:0000:0000:0000:0064
+
+# mask length for addresses of each IP version
+MASK_IPV4=24
+MASK_IPV6=64
+
+# name of the config files for backend and frontend of l4fwd app
+L4FWD_BE_CFG_FILE=$(mktemp)
+L4FWD_FE_CFG_FILE=$(mktemp)
+
+# directory on remote to store tmp files - default /tmp/
+REMOTE_DIR=/tmp/l4fwd_test
+# directory on remote to store output files
+REMOTE_OUTDIR=${REMOTE_DIR}/out
+# directory on remote to store results
+REMOTE_RESDIR=${REMOTE_DIR}/results
+
+# checks done on environment variables-----------------------------------------
+
+# check ETH_DEV
+if [[ -z "${ETH_DEV}" ]]
+then
+       echo "ETH_DEV is invalid"
+       exit 127
+fi
+
+# check if L4FWD_PATH points to an executable
+if [[ ! -x ${L4FWD_PATH} ]]
+then
+       echo "${L4FWD_PATH} is not executable"
+       exit 127
+fi
+
+# check if REMOTE_HOST is reachable
+ssh ${REMOTE_HOST} echo
+st=$?
+if [[ $st -ne 0 ]]
+then
+       echo "host ${REMOTE_HOST} is not reachable"
+       exit $st
+fi
+
+# get ethernet address of REMOTE_HOST
+REMOTE_MAC=$(ssh ${REMOTE_HOST} ip addr show dev ${REMOTE_IFACE})
+st=$?
+REMOTE_MAC=$(echo ${REMOTE_MAC} | sed -e 's/^.*ether //' -e 's/ brd.*$//')
+if [[ $st -ne 0 || -z "${REMOTE_MAC}" ]]
+then
+       echo "could not retrive ethernet address from ${REMOTE_IFACE}"
+       exit 127
+fi
+
+# check if FECORE is set - default 0
+L4FWD_FECORE=${L4FWD_FECORE:-0}
+
+# check if BECORE is set - default FECORE
+L4FWD_BECORE=${L4FWD_BECORE:-${L4FWD_FECORE}}
+
+# l4fwd app settings-----------------------------------------------------------
+
+# set file for l4fwd app output
+L4FWD_OUT_FILE=./l4fwd.out
+# set rbufs/sbufs/streams to open for l4fwd
+L4FWD_STREAMS='--rbufs 0x100 --sbufs 0x100 --streams 0x100'
+
+# set lcores for DPDK to start
+if [[ ${L4FWD_FECORE} -ne ${L4FWD_BECORE} ]]
+then
+       L4FWD_LCORE="${L4FWD_FECORE},${L4FWD_BECORE}"
+else
+       L4FWD_LCORE="${L4FWD_FECORE}"
+fi
+
+# set EAL parameters
+L4FWD_CMD_EAL_PRM="--lcores='${L4FWD_LCORE}' -n 4 ${ETH_DEV}"
+
+# l4fwd parameters (listen, TCP only, enable arp, promiscuous)
+L4FWD_CMD_PRM="--listen --tcp --enable-arp --promisc ${L4FWD_STREAMS}"
+
+# l4fwd config files
+L4FWD_CONFIG="--fecfg ${L4FWD_FE_CFG_FILE} --becfg ${L4FWD_BE_CFG_FILE}"
+
+# port parameters
+if [[ ${ipv4} -eq 1 ]]
+then
+       L4FWD_PORT_PRM="port=${DPDK_PORT},lcore=${L4FWD_BECORE},rx_offload=0x0\
+,tx_offload=0x0,ipv4=${LOCAL_IPV4}"
+elif [[ ${ipv6} -eq 1 ]]
+then
+       L4FWD_PORT_PRM="port=${DPDK_PORT},lcore=${L4FWD_BECORE},rx_offload=0x0\
+,tx_offload=0x0,ipv6=${LOCAL_IPV6}"
+fi
+
+# other variables--------------------------------------------------------------
+
+# check if directories on remote are set, if not make one
+ssh ${REMOTE_HOST} mkdir -p {${REMOTE_OUTDIR},${REMOTE_RESDIR}}
+
+# <tc qdisc ... netem ...> instruction to set
+netem="ssh ${REMOTE_HOST} tc qdisc add dev ${REMOTE_IFACE} \
+root netem limit 100000"
+
+# setting for scp which suppresses output of scp when not in verbose mode
+if [[ ${verbose} -eq 1 ]]
+then
+       scp_suppress=""
+else
+       scp_suppress="-q"
+fi
+
+# setting for dd which suppresses output of dd when not in verbose mode
+if [[ ${verbose} -eq 1 ]]
+then
+       dd_suppress=""
+else
+       dd_suppress="status=none"
+fi
+
+# set address to use by netcat
+if [[ ${ipv4} -eq 1 ]]
+then
+       nc_addr=${LOCAL_IPV4}
+elif [[ ${ipv6} -eq 1 ]]
+then
+       nc_addr=${LOCAL_IPV6}
+fi
+
+let "ipv4_elem=(${MASK_IPV4}/8)"
+let "ipv6_elem=(${MASK_IPV6}/16)"
+let "ipv4_elem_rev=4-${ipv4_elem}"
+
+ipv4_append=""
+while [[ ${ipv4_elem_rev} -ne 0 ]]; do
+       ipv4_append="${ipv4_append}.0"
+       let "ipv4_elem_rev=${ipv4_elem_rev}-1"
+done
+
+ipv4_network=$(echo ${REMOTE_IPV4} | cut -d. -f-${ipv4_elem} | \
+       sed 's#.*#&'"${ipv4_append}"'#')
+ipv6_network=$(echo ${REMOTE_IPV6} | cut -d: -f-${ipv6_elem} | sed 's#.*#&::#')
+
+# helper functions-------------------------------------------------------------
+
+# function to check if verbose is set and run command if yes
+if_verbose()
+{
+       if [[ ${verbose} -eq 1 ]]
+       then
+               $@
+       fi
+}
+
+# update results file
+update_results()
+{
+       file=$1
+       status=$2
+       it=$3
+
+       # get only 'real' time in results file
+       $(ssh ${REMOTE_HOST} "awk '/real/{print \$2}' \
+               ${REMOTE_RESDIR}/${file}.result.${it} \
+               >> ${REMOTE_RESDIR}/results.out")
+
+       # add file and status of test to results
+       if [[ ${status} -ne 0 ]]
+       then
+               $(ssh ${REMOTE_HOST} "sed -i '$ s_.*_[FAIL]\t&_' \
+                       ${REMOTE_RESDIR}/results.out")
+       else
+               $(ssh ${REMOTE_HOST} "sed -i '$ s_.*_[OK]\t&_' \
+                       ${REMOTE_RESDIR}/results.out")
+       fi
+
+       length=$(expr length "${file}")
+       if [[ ${length} -lt 16 ]]
+       then
+               tab="\t\t"
+       else
+               tab="\t"
+       fi
+
+       $(ssh ${REMOTE_HOST} "sed -i '$ s_.*_${file}${tab}&_' \
+               ${REMOTE_RESDIR}/results.out")
+}
+
+# start l4fwd app
+l4fwd_start()
+{
+       # create temporary file for command running l4fwd
+       L4FWD_EXEC_FILE=$(mktemp)
+
+       # store run command
+       cat << EOF > ${L4FWD_EXEC_FILE}
+stdbuf -o0 ${L4FWD_PATH} ${L4FWD_CMD_EAL_PRM} -- ${L4FWD_CMD_PRM} \
+${L4FWD_CONFIG} ${L4FWD_PORT_PRM} > ${L4FWD_OUT_FILE} 2>&1 &
+echo \$!
+EOF
+
+       # visual break
+       if_verbose echo -e "\nApp l4fwd started with command:"
+       if_verbose cat ${L4FWD_EXEC_FILE}
+       if_verbose echo ""
+
+       # run l4fwd app and get process ID of it
+       L4FWD_PID=$(/bin/bash ${L4FWD_EXEC_FILE})
+
+       # wait 2s and check if l4fwd is still running (parsing and init OK)
+       sleep 2
+       if [[ ${L4FWD_PID} -ne $(pgrep -o l4fwd) ]]
+       then
+               echo "ERROR: l4fwd app have crashed during initialization"
+               rm -f ${L4FWD_EXEC_FILE}
+               exit 127
+       fi
+}
+
+# stop l4fwd app
+l4fwd_stop()
+{
+       # kill runnning l4fwd app
+       kill ${L4FWD_PID}
+
+       # remove temporary files
+       rm -f ${L4FWD_EXEC_FILE}
+       rm -f ${L4FWD_FE_CFG_FILE}
+       rm -f ${L4FWD_BE_CFG_FILE}
+}
+
+# helper function to set netem on remote
+setup_netem()
+{
+       # remove netem settings from remote interface if any
+       check_netem=$(ssh ${REMOTE_HOST} "tc qdisc show dev \
+               ${REMOTE_IFACE} | grep netem")
+       if [[ -n ${check_netem} ]]
+       then
+               ssh ${REMOTE_HOST} tc qdisc del dev ${REMOTE_IFACE} root
+       fi
+
+       # set default delay for reorder
+       if [[ ${reorder} -ne 0 && ${delay} -eq 0 ]]
+       then
+               delay=20
+       fi
+
+       # set appropriate delay/loss/reorder if specified
+       if [[ ${delay} -ne 0 ]]
+       then
+               netem="${netem} delay ${delay}ms"
+       fi
+
+       if [[ ${loss} -ne 0 ]]
+       then
+               # calculate parameters for Simplified Gilbert model
+               loss_to_set=$(( $(( ${loss} * ${loss_burst} )) \
+/ $(( 100 - ${loss} )) ))
+
+               if [[ ${loss_to_set} -gt 100 ]]
+               then
+                       loss_to_set=100
+               fi
+               netem="${netem} loss gemodel ${loss_to_set}% ${loss_burst}%"
+       fi
+
+       if [[ ${reorder} -ne 0 ]]
+       then
+               netem="${netem} reorder 100% gap ${reorder}"
+       fi
+
+       # set netem on remote
+       ${netem}
+
+       # visual break of the output
+       if_verbose echo -e "\nNetwork rules on remote set to:"
+
+       # print current netem settings
+       if_verbose ssh ${REMOTE_HOST} tc qdisc show dev ${REMOTE_IFACE}
+}
+
+# configure IPv4 remote machine
+configure_ip4_remote()
+{
+       # visual break of the output
+       if_verbose echo "Setting interface on remote"
+
+       # set remote interface with correct IP address
+       ssh ${REMOTE_HOST} ip link set ${REMOTE_IFACE} down
+       ssh ${REMOTE_HOST} ip addr flush dev ${REMOTE_IFACE}
+       ssh ${REMOTE_HOST} ip addr add ${REMOTE_IPV4}/${MASK_IPV4} \
+               dev ${REMOTE_IFACE}
+       ssh ${REMOTE_HOST} ip link set ${REMOTE_IFACE} up
+       if_verbose ssh ${REMOTE_HOST} ip addr show dev ${REMOTE_IFACE}
+
+       ssh ${REMOTE_HOST} ip neigh flush dev ${REMOTE_IFACE}
+       ssh ${REMOTE_HOST} iptables --flush
+
+       ssh ${REMOTE_HOST} ip route change ${ipv4_network}/${MASK_IPV4} dev \
+               ${REMOTE_IFACE} rto_min 30ms
+
+       # construct <tc qdisc ... nete ...> instruction
+       if [[ set_netem -eq 1 ]]
+       then
+               setup_netem
+       fi
+
+       # give linux 1 sec to handle all network settings
+       sleep 1
+}
+
+# configure IPv6 remote machine
+configure_ip6_remote()
+{
+       # visual break of the output
+       if_verbose echo "Setting interface on remote"
+
+       # set remote interface with correct IP address
+       ssh ${REMOTE_HOST} ip link set ${REMOTE_IFACE} down
+       ssh ${REMOTE_HOST} sysctl -q -w \
+               net.ipv6.conf.${REMOTE_IFACE}.disable_ipv6=0
+       ssh ${REMOTE_HOST} ip addr flush dev ${REMOTE_IFACE}
+       ssh ${REMOTE_HOST} ip -6 addr add ${REMOTE_IPV6}/${MASK_IPV6} \
+               dev ${REMOTE_IFACE}
+       ssh ${REMOTE_HOST} ip -6 link set ${REMOTE_IFACE} up
+       if_verbose ssh ${REMOTE_HOST} ip addr show dev ${REMOTE_IFACE}
+
+       ssh ${REMOTE_HOST} ip neigh flush dev ${REMOTE_IFACE}
+       ssh ${REMOTE_HOST} ip -6 neigh add ${LOCAL_IPV6} dev ${REMOTE_IFACE} \
+               lladdr ${LOCAL_MAC}
+       ssh ${REMOTE_HOST} iptables --flush
+       ssh ${REMOTE_HOST} ip6tables --flush
+
+       ssh ${REMOTE_HOST} ip route change ${ipv6_network}/${MASK_IPV6} dev \
+               ${REMOTE_IFACE} proto kernel metric 256 rto_min 30ms
+
+       ssh ${REMOTE_HOST} ip -6 route show
+
+       # construct <tc qdisc ... nete ...> instruction
+       if [[ set_netem -eq 1 ]]
+       then
+               setup_netem
+       fi
+
+       # give linux 1 sec to handle all network settings
+       sleep 1
+}
+
+# configure remote
+configure_remote()
+{
+       # call proper configuration
+       if [[ ${ipv4} -eq 1 ]]
+       then
+               configure_ip4_remote
+
+               if_verbose echo -e "\nBE configuration:"
+               config4_be
+
+               if_verbose echo -e "\nFE configuration:"
+               config4_fe
+       elif [[ ${ipv6} -eq 1 ]]
+       then
+               configure_ip6_remote
+
+               if_verbose echo -e "\nBE configuration:"
+               config6_be
+
+               if_verbose echo -e "\nFE configuration:"
+               config6_fe
+       fi
+
+       # create empty results file on remote
+       $(ssh ${REMOTE_HOST} "> ${REMOTE_RESDIR}/results.out")
+}
+
+# restore netem settings to default
+restore_netem()
+{
+       if [[ ${set_netem} -eq 1 ]]
+       then
+               ssh ${REMOTE_HOST} tc qdisc del dev ${REMOTE_IFACE} root
+       fi
+}
+
+# remove created directories after test is done
+remove_directories()
+{
+       ssh ${REMOTE_HOST} rm -fr ${REMOTE_DIR}
+}
+
+# configuration of be/fe config------------------------------------------------
+config4_be()
+{
+       cat <<EOF > ${L4FWD_BE_CFG_FILE}
+port=${DPDK_PORT},masklen=${MASK_IPV4},addr=${REMOTE_IPV4},mac=${REMOTE_MAC}
+EOF
+
+       if_verbose cat ${L4FWD_BE_CFG_FILE}
+}
+
+config6_be()
+{
+       cat <<EOF > ${L4FWD_BE_CFG_FILE}
+port=${DPDK_PORT},masklen=${MASK_IPV6},addr=${REMOTE_IPV6},mac=${REMOTE_MAC}
+EOF
+
+       if_verbose cat ${L4FWD_BE_CFG_FILE}
+}
+
+config4_fe()
+{
+       cat <<EOF > ${L4FWD_FE_CFG_FILE}
+lcore=${L4FWD_FECORE},belcore=${L4FWD_BECORE},op=echo,laddr=${LOCAL_IPV4}\
+,lport=${TCP_PORT},raddr=${REMOTE_IPV4},rport=0
+EOF
+
+       if_verbose cat ${L4FWD_FE_CFG_FILE}
+}
+
+config6_fe()
+{
+       cat <<EOF > ${L4FWD_FE_CFG_FILE}
+lcore=${L4FWD_FECORE},belcore=${L4FWD_BECORE},op=echo,laddr=${LOCAL_IPV6}\
+,lport=${TCP_PORT},raddr=${REMOTE_IPV6},rport=0
+EOF
+
+       if_verbose cat ${L4FWD_FE_CFG_FILE}
+}
diff --git a/examples/l4fwd/test/example_env_vars b/examples/l4fwd/test/example_env_vars
new file mode 100644 (file)
index 0000000..9877db8
--- /dev/null
@@ -0,0 +1,13 @@
+#! /bin/bash
+
+# ENV VARIABLES
+
+export REMOTE_HOST=root@10.237.214.104
+export REMOTE_IFACE=enp138s0f0
+export LOCAL_MAC="68:05:ca:04:47:02"
+export L4FWD_PATH=/opt/home/md/Projects/tldk/BuildForTLDK/app/l4fwd
+export L4FWD_FECORE=5 #optional
+export L4FWD_BECORE=6 #optional
+export ETH_DEV="-w 8a:00.0"
+
+# ENV VARIABLES end
diff --git a/examples/l4fwd/test/nctxrx.sh b/examples/l4fwd/test/nctxrx.sh
new file mode 100644 (file)
index 0000000..6a016e0
--- /dev/null
@@ -0,0 +1,363 @@
+#! /bin/bash
+
+# readme section---------------------------------------------------------------
+
+# usage: /bin/bash nctxrx.sh [-ifnpalrdovh]
+#
+# Run specific test setup based on options. For details about options run
+# script with -h (help)
+#
+# User needs to specify following environment variables:
+#  ETH_DEV     - ethernet device to be used on SUT by DPDK
+#  REMOTE_HOST - ip/hostname of DUT
+#  REMOTE_IFACE        - interface name for the test-port on DUT
+#  LOCAL_MAC   - MAC address used by DPDK
+#  L4FWD_PATH  - path to l4fwd app binary
+# Optional envirenment variables:
+#  L4FWD_FECORE        - core on which l4fwd frontend should run
+#  L4FWD_BECORE        - core on which l4fwd backend should run
+#
+# The purpose of the script is to automate validation tests for l4fwd app
+# where packets are out of order/lost. It expects l4fwd application being
+# run on local linux system (SUT). Script is operating on remote linux
+# machine (DUT) with use of ssh. SUT and DUT are connected via NIC. On SUT
+# network traffic is managed by DPDK and on DUT by linux. On DUT netcat is
+# used to send test data via TCP to TLDK on SUT, which is set to echo mode
+# (sends back the same data). Depending on test specified, TCP segments are
+# artificially changed in sending buffer of DUT, so they are lost in some
+# percentage or sent out of order. If specified, report is sent from DUT
+# to SUT after all tests were performed.
+#
+# Example traffic visualisation:
+# DUT --(TCP out of order)--> SUT --(TCP with correct order)--> DUT(validation)
+
+# options which can be changed by the user if needed---------------------------
+
+# timeout in [s] for calling nc (in case traffic stuck)
+timeout=600
+
+# delay for netem (50 [ms] is default value when reorder option used)
+delay=0
+
+# default loss of packets [%] value
+loss=0
+
+# default probability [%] of not losing burst of packets
+loss_burst=80
+
+# variables used by script-----------------------------------------------------
+
+# temp files to remove at the end
+rmxf=""
+rmresults=""
+
+# specify if <tc qdisc ... netem ...> instruction should be invoked
+set_netem=0
+
+# flag to check if default files should to be used (default 1)
+# default files are generated with urandom (couple of sizes)
+default_file=1
+
+# IP protocol version
+ipv4=0
+ipv6=0
+
+# default result file
+local_result_file=$(dirname $0)/results.out
+
+# should verbose mode be used
+verbose=0
+
+# netcat option for using IPv6, initially empty
+nc_ipv6=""
+
+# functions--------------------------------------------------------------------
+
+usage_internal()
+{
+       echo -e "Usage:"
+       echo -e "\t$0 [-vh] [-p protocol] [-f test_file] [-n number] \
+[-l loss] [-r gap] [-d delay] [-o result_file]"
+       echo -e "Options:"
+       echo -e "\t-p <protocol>\t\tSet IP protocol to use."
+       echo -e "\t\t\t\tAcceptable values: ipv4/ipv6."
+       echo -e "\n\t-f <test_file>\t\tChoose a file to be sent during tests \
+(full path to file on remote machine)."
+       echo -e "\t\t\t\tNot specified will perform tests on default files."
+       echo -e "\n\t-n <number>\t\tChoose how many times send the test file."
+       echo -e "\t\t\t\tFiles will be send simultaneously by opening \
+new netcat connection."
+       echo -e "\n\t-l <loss>\t\tSet average loss of packets in %."
+       echo -e "\t\t\t\tEg. loss=10 means 10% of packets will be lost."
+       echo -e "\n\t-r <gap>\t\tSet gap for packets to be reordered."
+       echo -e "\t\t\t\tEg. gap=5 means every 5'th packet will be reordered."
+       echo -e "\t\t\t\tIf delay is not set as well, default value of 10ms \
+will be used."
+       echo -e "\n\t-d <delay>\t\tSet delay for packet sending in ms."
+       echo -e "\n\t-o <result_file>\tUser specified file to which results \
+should be stored."
+       echo -e "\t\t\t\tDefault file is ${local_result_file}"
+       echo -e "\n\t-v\t\t\tVerbose mode - prints additional output."
+       echo -e "\n\t-h\t\t\tDisplay this help."
+}
+
+# parse options and arguments
+while getopts ":f:n:p:l:r:d:o:vh" opt
+do
+       case $opt in
+               p)
+                       ipv=$OPTARG
+                       if [[ ${ipv} == "ipv4" ]]
+                       then
+                               ipv4=1
+                       elif [[ ${ipv} == "ipv6" ]]
+                       then
+                               ipv6=1
+                               nc_ipv6="-6"
+                       else
+                               echo "No IP protocol specified"
+                               usage_internal
+                               exit 127
+                       fi
+                       ;;
+               f)
+                       file=$OPTARG
+                       default_file=0
+                       ;;
+               n)
+                       num=$OPTARG
+                       ;;
+               l)
+                       set_netem=1
+                       loss=$OPTARG
+                       ;;
+               r)
+                       set_netem=1
+                       reorder=$OPTARG
+                       ;;
+               d)
+                       set_netem=1
+                       delay=$OPTARG
+                       ;;
+               o)
+                       local_result_file=$OPTARG
+                       ;;
+               v)
+                       verbose=1
+                       ;;
+               h)
+                       usage_internal
+                       exit 0
+                       ;;
+               ?)
+                       echo "Invalid option"
+                       usage_internal
+                       exit 127
+                       ;;
+       esac
+done
+
+# load configuration
+. $(dirname $0)/config.sh
+
+# send file with results to local machine
+send_results()
+{
+       if_verbose echo -e "Sending result file to local"
+       scp ${scp_suppress} ${REMOTE_HOST}:${REMOTE_RESDIR}/results.out \
+               ${local_result_file}
+       ssh ${REMOTE_HOST} rm -f ${REMOTE_RESDIR}/results.out
+}
+
+# test setup
+run_test()
+{
+       of=$1
+       # visual break of the output
+       if_verbose echo -e "\nRunning netcat"
+
+       pids=""
+       i=0
+       while [ $i -lt $num ]
+       do
+               # save command for nc in 'cmd'
+               # time -> meassure time of execution for netcat
+               # -q 0 -> wait 0 seconds after EOF and quit
+               # timeout to deal with hanging connection when sth went wrong
+               # feed netcat with {of} file to send
+               # receiving end is redirected to out/...out files
+               # 'exec' for redirecting nc err output to not mess result
+               cmd="exec 4>&2
+\$({ time timeout ${timeout} nc ${nc_ipv6} -q 0 ${nc_addr} ${TCP_PORT} \
+       < ${REMOTE_DIR}/${of} \
+       > ${REMOTE_OUTDIR}/${of}.out.${i} 2>&4; } \
+       2>${REMOTE_RESDIR}/${of}.result.${i} )
+exec 4>&-"
+
+               # create temporary file for nc command to execute
+               xf=$(ssh ${REMOTE_HOST} mktemp -p ${REMOTE_DIR})
+
+               # store command from {cmd} into temporaty file
+               echo "${cmd}" | ssh ${REMOTE_HOST} "cat > ${xf}"
+
+               # execute nc command in the background
+               ssh ${REMOTE_HOST} /bin/bash ${xf} &
+
+               pids="${pids} $!"
+
+               # adds tempfiles to list to remove later
+               rmxf="${rmxf} ${xf}"
+               rmresults="${rmresults} ${REMOTE_RESDIR}/${of}.result.${i}"
+
+               i=$(expr $i + 1)
+       done
+
+       # sleep for 1 sec
+       sleep 1
+
+       # wait until previous commands finish (nc commands)
+       wait ${pids}
+
+       # remove temporary files
+       ssh ${REMOTE_HOST} rm -f ${rmxf}
+
+       # visual break
+       if_verbose echo -e "\nNetstat:"
+
+       # prints network information for given {TCP_PORT} number
+       # -n -> show numeric addresses
+       # -a -> show all (both listening and non-listening sockets)
+       if_verbose ssh ${REMOTE_HOST} netstat -na | grep ${TCP_PORT}
+
+       # visual break
+       if_verbose echo -e "\nJobs:"
+
+       # display status of jobs in the current session (this bash script)
+       if_verbose ssh ${REMOTE_HOST} jobs -l
+
+       # visual break
+       if_verbose echo -e "\nNetcat processes:"
+
+       # display current processes for netcat
+       # -e -> show all processes
+       # -f -> do full format listing (more info)
+       # grep -v -> get rid of the following word match from grep output
+       if_verbose ssh ${REMOTE_HOST} ps -ef | grep "nc " | grep -v grep
+
+       # visual break
+       if_verbose echo -e "\nRunning validation"
+
+       flag_error=0
+       i=0
+       while [[ ${i} -lt ${num} ]]
+       do
+               # prints checksum of sent and received file
+               if_verbose ssh ${REMOTE_HOST} cksum ${REMOTE_DIR}/${of} \
+                       ${REMOTE_OUTDIR}/${of}.out.${i}
+
+               # compares sent and received files if they match
+               # compare {of} and {out/of.out.i} line by line
+               ssh ${REMOTE_HOST} diff ${REMOTE_DIR}/${of} \
+                       ${REMOTE_OUTDIR}/${of}.out.${i}
+
+               # capture the result of diff command above
+               rc=$?
+
+               # update results file
+               update_results ${of} ${rc} ${i}
+
+               # check if result of diff is 0
+               # equals 0 -> files are the same
+               # not 0 -> files differ in some way -> report Error and exit
+               #               with no execution of the rest of the script
+               if [ ${rc} -ne 0 ]
+               then
+                       echo -e "TEST FAILED - ${of}"
+                       echo "ERROR: files ${of} ${of}.out.${i} differ"
+
+                       # mark that there was an error
+                       flag_error=${rc}
+               fi
+
+               # remove received file from out/ directory
+               ssh ${REMOTE_HOST} rm -f ${REMOTE_OUTDIR}/${of}.out.${i}
+
+               i=$(expr $i + 1)
+       done
+
+       # remove temporary results
+       ssh ${REMOTE_HOST} rm -f ${rmresults}
+
+       if [[ flag_error -eq 1 ]]
+       then
+               return ${flag_error}
+       fi
+
+       if_verbose echo ""
+       echo -e "TEST SUCCESSFUL - ${of}"
+       if_verbose echo ""
+       return 0
+}
+
+# clean up after error or end of tests
+cleanup()
+{
+       send_results
+       restore_netem
+       l4fwd_stop
+       remove_directories
+}
+
+# script start-----------------------------------------------------------------
+
+#configure remote machine
+configure_remote
+
+# start l4fwd app
+l4fwd_start
+
+# check if default files should be used
+if [[ ${default_file} -eq 0 ]]
+then
+       if_verbose echo -e "Sending test file to remote"
+       scp ${scp_suppress} ${file} ${REMOTE_HOST}:${REMOTE_DIR}
+       run_test ${file}
+
+       # check test outcome
+       ret=$?
+       if [[ ${ret} -ne 0 ]]
+       then
+               cleanup
+               exit ${ret}
+       fi
+       ssh ${REMOTE_HOST} rm -f ${REMOTE_DIR}/${file}
+else
+       # use default files with size 16MB
+       for size in 16
+       do
+               # generate file
+               if_verbose echo -e "Generating ${size}MB file for test"
+               x=$(ssh ${REMOTE_HOST} mktemp $(basename $0).${size}MB.XXX \
+                       -p ${REMOTE_DIR})
+
+                       ssh ${REMOTE_HOST} dd if=/dev/urandom of=${x} bs=1M \
+                               count=${size} ${dd_suppress}
+
+               # run test over generated file
+               run_test $(basename ${x})
+
+               # check test outcome
+               ret=$?
+               if [[ ${ret} -ne 0 ]]
+               then
+                       cleanup
+                       exit ${ret}
+               fi
+
+               # remove generated file only if test successful
+               ssh ${REMOTE_HOST} rm -f ${x}
+       done
+fi
+
+cleanup
+exit 0
diff --git a/examples/l4fwd/test/run_test.sh b/examples/l4fwd/test/run_test.sh
new file mode 100644 (file)
index 0000000..690651a
--- /dev/null
@@ -0,0 +1,278 @@
+#! /bin/bash
+
+# readme section---------------------------------------------------------------
+
+# usage: /bin/bash run_test.sh [-46lrh]
+#
+# Run all tests using nctxrx.sh. Report stored and printed
+# after tests were done. For details about options run
+# script with -h (help)
+#
+# User needs to specify following environment variables:
+#  ETH_DEV     - ethernet device to be used on SUT by DPDK
+#  REMOTE_HOST - ip/hostname of DUT
+#  REMOTE_IFACE        - interface name for the test-port on DUT
+#  LOCAL_MAC   - MAC address used by DPDK
+#  L4FWD_PATH  - path to l4fwd app binary
+# Optional envirenment variables:
+#  L4FWD_FECORE        - core on which l4fwd frontend should run
+#  L4FWD_BECORE        - core on which l4fwd backend should run
+
+# options which can be changed by user-----------------------------------------
+
+# reorder settings
+reorder_min=4
+reorder_max=9
+reorder_step=5
+
+# loss settings
+loss_min=0
+loss_max=20
+loss_step=20
+
+# file for results storage
+DIR=$(dirname $0)
+result=${DIR}/result.out
+echo -e "Test\t\tProtocol\tFile\t\t\tStatus\tTime" > ${result}
+
+# how many times test file should be send during tests
+nb=3
+
+# variables used by script-----------------------------------------------------
+
+# option parsing variables
+run_loss=0
+run_reorder=0
+use_ip4=0
+use_ip6=0
+
+# track number of tests which have failed
+error_count=0
+
+SECONDS=0
+
+# functions and calls----------------------------------------------------------
+
+usage()
+{
+       echo -e "Usage:"
+       echo -e "\t$0 [-alr46h]"
+       echo -e "Options:"
+       echo -e "\t-a Run all tests"
+       echo -e "\t-l Perform loss tests"
+       echo -e "\t-r Perform reorder tests"
+       echo -e "\t-4 Use IPv4/TCP"
+       echo -e "\t-6 Use IPv6/TCP"
+       echo -e "\t-h Display this help"
+       echo -e "Info:"
+       echo -e "\tOptions [4/6] may be used together."
+}
+
+while getopts ":alr46h" opt
+do
+       case $opt in
+               a)
+                       run_loss=1
+                       run_reorder=1
+                       ;;
+               l)
+                       run_loss=1
+                       ;;
+               r)
+                       run_reorder=1
+                       ;;
+               4)
+                       use_ip4=1
+                       ;;
+               6)
+                       use_ip6=1
+                       ;;
+               h)
+                       usage
+                       exit 0
+                       ;;
+               ?)
+                       echo "Invalid option"
+                       usage
+                       exit 127
+                       ;;
+       esac
+done
+
+# check if tests to perform are specified
+if [[ ${run_loss} -eq 0 && ${run_reorder} -eq 0 ]]
+then
+       echo -e "Error: No tests specified\n"
+       usage
+       exit 127
+fi
+
+# check if IP protocol was specified
+if [[ ${use_ip4} -eq 0 && ${use_ip6} -eq 0 ]]
+then
+       echo -e "Error: No IP protocol specified\n"
+       usage
+       exit 127
+fi
+
+# get number of tests to perform
+if [[ ${run_reorder} -eq 1 ]]
+then
+       nb_of_reorder=$(( $(( ${reorder_max} - ${reorder_min} )) \
+               / ${reorder_step} + 1 ))
+else
+       nb_of_reorder=0
+fi
+
+if [[ ${run_loss} -eq 1 ]]
+then
+       nb_of_loss=$(( $(( ${loss_max} - ${loss_min} )) / ${loss_step} + 1 ))
+else
+       nb_of_loss=0
+fi
+
+if [[ ${use_ip4} -eq 1 && ${use_ip6} -eq 1 ]]
+then
+       multiply=2
+else
+       multiply=1
+fi
+
+nb_of_tests=$(( $(( ${nb_of_loss} + ${nb_of_reorder} )) * ${multiply} ))
+tests_performed=0
+
+echo "Number of tests to run: ${nb_of_tests}"
+
+# add intermediary data into result file
+gather_data()
+{
+       test_case=$1
+       test_value=$2
+       protocol=$3
+
+       length=$(expr length "${test_case} ${test_value}")
+       if [[ ${length} -lt 8 ]]
+       then
+               tab="\t\t"
+       else
+               tab="\t"
+       fi
+
+       # add protocol used in test case which was invoked
+       sed -i "s_.*_${protocol}\t\t&_" ${result}.tmp
+       # add description of test case which was invoked (in first line)
+       sed -i "1 s_.*_${test_case} ${test_value}${tab}&_" ${result}.tmp
+       # add blank space to be aligned with first row
+       sed -i "1 ! s_.*_\t\t&_" ${result}.tmp
+       # add empty line befor each major test case
+       sed -i "1 s_.*_\n&_" ${result}.tmp
+       cat ${result}.tmp >> ${result}
+       rm -f ${result}.tmp
+}
+
+# run all tests
+while [[ ${use_ip4} -ne 0 || ${use_ip6} -ne 0 ]]
+do
+       #set protocol to be used in this round of tests
+       if [[ ${use_ip4} -eq 1 ]]
+       then
+               proto="ipv4"
+       elif [[ ${use_ip6} -eq 1 ]]
+       then
+               proto="ipv6"
+       fi
+
+       # check if reorder tests should be run
+       if [[ ${run_reorder} -eq 1 ]]
+       then
+               # run test for all specified reorder values
+               for reorder in $(seq ${reorder_min} \
+                               ${reorder_step} \
+                               ${reorder_max})
+               do
+                       /bin/bash ${DIR}/nctxrx.sh \
+                               -p ${proto} \
+                               -n ${nb} \
+                               -r ${reorder} \
+                               -o ${result}.tmp \
+                               -v
+
+                       # check test status
+                       st=$?
+                       if [[ ${st} -eq 0 ]]
+                       then
+                               echo -e "\nTest for reorder: ${reorder}\t[OK]"
+                       else
+                               echo -e "\nTest for reorder: $reorder}\t[FAIL]"
+                               error_count=$(expr ${error_count} + 1)
+                       fi
+
+                       # gather results
+                       gather_data "Reorder" ${reorder} ${proto}
+                       tests_performed=$(( ${tests_performed} + 1 ))
+                       echo -e "\n[PROGRESS] ${tests_performed} out of \
+${nb_of_tests} done\n"
+               done
+       fi
+
+       # check if loss tests should be run
+       if [[ ${run_loss} -eq 1 ]]
+       then
+               # run test for all specified reorder values
+               for loss in $(seq ${loss_min} ${loss_step} ${loss_max})
+               do
+                       /bin/bash ${DIR}/nctxrx.sh \
+                               -p ${proto} \
+                               -n ${nb} \
+                               -l ${loss} \
+                               -o ${result}.tmp \
+                               -v
+
+                       # check test status
+                       st=$?
+                       if [[ ${st} -eq 0 ]]
+                       then
+                               echo -e "\nTest for loss: ${loss}\t[OK]"
+                       else
+                               echo -e "\nTest for loss: ${loss}\t[FAIL]"
+                               error_count=$(expr ${error_count} + 1)
+                       fi
+
+                       # gather results
+                       gather_data "Loss" ${loss} ${proto}
+                       tests_performed=$(( ${tests_performed} + 1 ))
+                       echo -e "\n[PROGRESS] ${tests_performed} out of \
+${nb_of_tests} done\n"
+               done
+       fi
+
+       # mark that tests were done for one of the protocols
+       if [[ ${use_ip4} -eq 1 ]]
+       then
+               use_ip4=0
+       elif [[ ${use_ip6} -eq 1 ]]
+       then
+               use_ip6=0
+       fi
+done
+
+if [[ ${error_count} -eq 0 ]]
+then
+       echo -e "\nAll tests have ended successfully" >> ${result}
+else
+       echo -e "\n${error_count} tests have failed" >> ${result}
+fi
+
+if [[ $SECONDS -gt 60 ]]
+then
+       let "minutes=SECONDS/60"
+       let "seconds=SECONDS%60"
+       echo "All tests completed in $minutes minute(s) and $seconds second(s)"\
+               >> ${result}
+else
+       echo "All tests completed in $SECONDS second(s)" >> ${result}
+fi
+
+# print report after all tests were done
+echo -e "Report\n"
+cat ${result}