test/scripts/socket_test.sh

   1 #! /bin/bash
   2 #
   3 # socket_test.sh -- script to run socket tests.
   4 #
   5 script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
   6 vpp_dir="$WS_ROOT/build-root/install-vpp-native/vpp/bin/"
   7 vpp_debug_dir="$WS_ROOT/build-root/install-vpp_debug-native/vpp/bin/"
   8 vpp_shm_dir="/dev/shm/"
   9 vpp_run_dir="/run/vpp"
  10 lib64_dir="$WS_ROOT/build-root/install-vpp-native/vpp/lib64/"
  11 lib64_debug_dir="$WS_ROOT/build-root/install-vpp_debug-native/vpp/lib64/"
  12 dpdk_devbind="/usr/share/dpdk/usertools/dpdk-devbind.py"
  13 docker_vpp_dir="/vpp/"
  14 docker_app_dir="/vpp/"
  15 docker_lib64_dir="/vpp-lib64/"
  16 docker_os="ubuntu"
  17 vcl_ldpreload_lib="libvcl_ldpreload.so.0.0.0"
  18 user_gid="$(id -g)"
  19 vpp_app="vpp"
  20 sock_srvr_app="sock_test_server"
  21 sock_clnt_app="sock_test_client"
  22 sock_srvr_addr="127.0.0.1"
  23 sock_srvr_port="22000"
  24 iperf_srvr_app="iperf3 -V4d -s"
  25 iperf_clnt_app="iperf3 -V4d -c \$srvr_addr"
  26 gdb_in_emacs="gdb_in_emacs"
  27 vcl_config="vcl.conf"
  28 vcl_config_dir="$WS_ROOT/src/vcl/"
  29 docker_vcl_config_dir="/etc/vpp/"
  30 xterm_geom="100x60"
  31 bash_header="#! /bin/bash"
  32 tmp_cmdfile_prefix="/tmp/socket_test_cmd"
  33 cmd1_file="${tmp_cmdfile_prefix}1.$$"
  34 cmd2_file="${tmp_cmdfile_prefix}2.$$"
  35 cmd3_file="${tmp_cmdfile_prefix}3.$$"
  36 vpp_eth_name="enp0s8"
  37 tmp_vpp_exec_file="/tmp/vpp_config.$$"
  38 tmp_gdb_cmdfile_prefix="/tmp/gdb_cmdfile"
  39 def_gdb_cmdfile_prefix="$WS_ROOT/extras/gdb/gdb_cmdfile"
  40 tmp_gdb_cmdfile_vpp="${tmp_gdb_cmdfile_prefix}_vpp.$$"
  41 tmp_gdb_cmdfile_client="${tmp_gdb_cmdfile_prefix}_vcl_client.$$"
  42 tmp_gdb_cmdfile_server="${tmp_gdb_cmdfile_prefix}_vcl_server.$$"
  43 get_docker_server_ip4addr='srvr_addr=$(docker network inspect bridge | grep IPv4Address | awk -e '\''{print $2}'\'' | sed -e '\''s,/16,,'\'' -e '\''s,",,g'\'' -e '\''s/,//'\'')'
  44 #' single quote to fix the confused emacs colorizer.
  45 trap_signals="SIGINT SIGTERM EXIT"
  46
  47 # Set default values for imported environment variables if they don't exist.
  48 #
  49 VPP_GDB_CMDFILE="${VPP_GDB_CMDFILE:-${def_gdb_cmdfile_prefix}.vpp}"
  50 VPPCOM_CLIENT_GDB_CMDFILE="${VPPCOM_CLIENT_GDB_CMDFILE:-${def_gdb_cmdfile_prefix}.vppcom_client}"
  51 VPPCOM_SERVER_GDB_CMDFILE="${VPPCOM_SERVER_GDB_CMDFILE:-${def_gdb_cmdfile_prefix}.vppcom_server}"
  52
  53 usage() {
  54     cat <<EOF
  55 Usage: socket_test.sh OPTIONS TEST
  56 TESTS:
  57   nk, native-kernel   Run server & client on host using kernel.
  58   nv, native-vcl      Run vpp, server & client on host using VppComLib.
  59   np, native-preload  Run vpp, server & client on host using LD_PRELOAD.
  60   dk, docker-kernel   Run server & client in docker using kernel stack.
  61   dv, docker-vcl      Run vpp on host, server & client in docker using VppComLib.
  62   dp, docker-preload  Run vpp on host, server & client in docker using LD_PRELOAD.
  63
  64 OPTIONS:
  65   -h                  Print this usage text.
  66   -l                  Leave ${tmp_cmdfile_prefix}* files after test run.
  67   -b                  Run bash after application exit.
  68   -d                  Run the vpp_debug version of all apps.
  69   -c                  Set VCL_CONFIG to use the vcl_test.conf file.
  70   -i                  Run iperf3 for client/server app in native tests.
  71   -n                  Name of ethernet for VPP to use in multi-host cfg.
  72   -6                  Use ipv6 addressing.
  73   -f                  Full thru host stack vpp configuration.
  74   -m c[lient]         Run client in multi-host cfg (server on remote host)
  75      s[erver]         Run server in multi-host cfg (client on remote host)
  76   -e a[ll]            Run all in emacs+gdb.
  77      c[lient]         Run client in emacs+gdb.
  78      s[erver]         Run server in emacs+gdb.
  79      v[pp]            Run vpp in emacs+gdb.
  80   -g a[ll]            Run all in gdb.
  81      c[lient]         Run client in gdb.
  82      s[erver]         Run server in gdb.
  83      v[pp]            Run vpp in gdb.
  84   -t                  Use tabs in one xterm if available (e.g. xfce4-terminal).
  85
  86 OPTIONS passed to client/server:
  87   -S <ip address>     Server IP address.
  88   -P <server port>    Server Port number.
  89   -E <data>           Run Echo test.
  90   -N <num-writes>     Test Cfg: number of writes.
  91   -R <rxbuf-size>     Test Cfg: rx buffer size.
  92   -T <txbuf-size>     Test Cfg: tx buffer size.
  93   -U                  Run Uni-directional test.
  94   -B                  Run Bi-directional test.
  95   -I <num-tst-socks>  Send data over multiple test sockets in parallel.
  96   -V                  Test Cfg: Verbose mode.
  97   -X                  Exit client/server after running test.
  98
  99 Environment variables:
 100   VCL_CONFIG                Pathname of vppcom configuration file.
 101   VPP_GDB_CMDFILE            Pathname of gdb command file for vpp.
 102   VPPCOM_CLIENT_GDB_CMDFILE  Pathname of gdb command file for client.
 103   VPPCOM_SERVER_GDB_CMDFILE  Pathname of gdb command file for server.
 104 EOF
 105     exit 1
 106 }
 107
 108 declare -i emacs_vpp=0
 109 declare -i emacs_client=0
 110 declare -i emacs_server=0
 111 declare -i gdb_vpp=0
 112 declare -i gdb_client=0
 113 declare -i gdb_server=0
 114 declare -i perf_vpp=0
 115 declare -i perf_client=0
 116 declare -i perf_server=0
 117 declare -i leave_tmp_files=0
 118 declare -i bash_after_exit=0
 119 declare -i iperf3=0
 120 declare -i use_ipv6=0
 121
 122 while getopts ":hitlbcd6fn:m:e:g:p:E:I:N:P:R:S:T:UBVX" opt; do
 123     case $opt in
 124         h) usage ;;
 125         l) leave_tmp_files=1
 126            ;;
 127         b) bash_after_exit=1
 128            ;;
 129         i) iperf3=1
 130            ;;
 131         6) use_ipv6=1
 132            ;;
 133         f) full_thru_host_stack_vpp_cfg=1
 134            ;;
 135         t) xterm_geom="180x40"
 136            use_tabs="true"
 137            ;;
 138         c) VCL_CONFIG="${vcl_config_dir}vcl_test.conf"
 139            ;;
 140         d) title_dbg="-DEBUG"
 141            vpp_dir=$vpp_debug_dir
 142            lib64_dir=$lib64_debug_dir
 143            ;;
 144         e) if [ $OPTARG = "a" ] || [ $OPTARG = "all" ] ; then
 145                emacs_client=1
 146                emacs_server=1
 147                emacs_vpp=1
 148            elif [ $OPTARG = "c" ] || [ $OPTARG = "client" ] ; then
 149                emacs_client=1
 150            elif [ $OPTARG = "s" ] || [ $OPTARG = "server" ] ; then
 151                emacs_server=1
 152            elif [ $OPTARG = "v" ] || [ $OPTARG = "vpp" ] ; then
 153                emacs_vpp=1
 154            else
 155                echo "ERROR: Option -e unknown argument \'$OPTARG\'" >&2
 156                usage
 157            fi
 158            title_dbg="-DEBUG"
 159            vpp_dir=$vpp_debug_dir
 160            lib64_dir=$lib64_debug_dir
 161            ;;
 162         n) vpp_eth_name="$OPTARG"
 163            ;;
 164         m) if [ $OPTARG = "c" ] || [ $OPTARG = "client" ] ; then
 165                multi_host="client"
 166            elif [ $OPTARG = "s" ] || [ $OPTARG = "server" ] ; then
 167                multi_host="server"
 168            else
 169                echo "ERROR: Option -e unknown argument \'$OPTARG\'" >&2
 170                usage
 171            fi
 172            ;;
 173         g) if [ $OPTARG = "a" ] || [ $OPTARG = "all" ] ; then
 174                gdb_client=1
 175                gdb_server=1
 176                gdb_vpp=1
 177            elif [ $OPTARG = "c" ] || [ $OPTARG = "client" ] ; then
 178                gdb_client=1
 179            elif [ $OPTARG = "s" ] || [ $OPTARG = "server" ] ; then
 180                gdb_server=1
 181            elif [ $OPTARG = "v" ] || [ $OPTARG = "vpp" ] ; then
 182                gdb_vpp=1
 183            else
 184                echo "ERROR: Option -g unknown argument \'$OPTARG\'" >&2
 185                usage
 186            fi
 187            title_dbg="-DEBUG"
 188            vpp_dir=$vpp_debug_dir
 189            lib64_dir=$lib64_debug_dir
 190            ;;
 191         p) if [ $OPTARG = "a" ] || [ $OPTARG = "all" ] ; then
 192                perf_client=1
 193                perf_server=1
 194                perf_vpp=1
 195            elif [ $OPTARG = "c" ] || [ $OPTARG = "client" ] ; then
 196                perf_client=1
 197            elif [ $OPTARG = "s" ] || [ $OPTARG = "server" ] ; then
 198                perf_server=1
 199            elif [ $OPTARG = "v" ] || [ $OPTARG = "vpp" ] ; then
 200                perf_vpp=1
 201            else
 202                echo "ERROR: Option -p unknown argument \'$OPTARG\'" >&2
 203                usage
 204            fi
 205            echo "WARNING: -p options TBD"
 206            ;;
 207         S) sock_srvr_addr="$OPTARG"
 208            ;;
 209         P) sock_srvr_port="$OPTARG"
 210            ;;
 211 E|I|N|R|T) sock_clnt_options="$sock_clnt_options -$opt \"$OPTARG\""
 212            ;;
 213   U|B|V|X) sock_clnt_options="$sock_clnt_options -$opt"
 214            ;;
 215        \?)
 216            echo "ERROR: Invalid option: -$OPTARG" >&2
 217            usage
 218            ;;
 219         :)
 220            echo "ERROR: Option -$OPTARG requires an argument." >&2
 221            usage
 222            ;;
 223     esac
 224 done
 225
 226 shift $(( $OPTIND-1 ))
 227 while ! [[ $run_test ]] && (( $# > 0 )) ; do
 228     case $1 in
 229         "nk" | "native-kernel")
 230             run_test="native_kernel" ;;
 231         "np" | "native-preload")
 232             run_test="native_preload" ;;
 233         "nv" | "native-vcl")
 234             sock_srvr_app="vcl_test_server"
 235             sock_clnt_app="vcl_test_client"
 236             run_test="native_vcl" ;;
 237         "dk" | "docker-kernel")
 238             run_test="docker_kernel" ;;
 239         "dp" | "docker-preload")
 240             run_test="docker_preload" ;;
 241         "dv" | "docker-vcl")
 242             sock_srvr_app="vcl_test_server"
 243             sock_clnt_app="vcl_test_client"
 244             run_test="docker_vcl" ;;
 245         *)
 246             echo "ERROR: Unknown option '$1'!" >&2
 247             usage ;;
 248     esac
 249     shift
 250 done
 251
 252 if [ -z "$VCL_DEBUG" ] ; then
 253     if [ "$title_dbg" = "-DEBUG" ] ; then
 254         VCL_DEBUG=1
 255     else
 256         VCL_DEBUG=0
 257     fi
 258 fi
 259
 260 VCL_LDPRELOAD_LIB_DIR="${VCL_LDPRELOAD_LIB_DIR:-$lib64_dir}"
 261
 262 if [ -z "$WS_ROOT" ] ; then
 263     echo "ERROR: WS_ROOT environment variable not set!" >&2
 264     echo "       Please set WS_ROOT to VPP workspace root directory." >&2
 265     exit 1
 266 fi
 267
 268 if [[ "$(grep bin_PROGRAMS $WS_ROOT/src/vcl.am)" = "" ]] ; then
 269     $WS_ROOT/extras/vagrant/vcl_test.sh $WS_ROOT $USER
 270     (cd $WS_ROOT; make build)
 271 fi
 272
 273 if [ ! -d $vpp_dir ] ; then
 274     echo "ERROR: Missing VPP$title_dbg bin directory!" >&2
 275     echo "       $vpp_dir" >&2
 276     env_test_failed="true"
 277 fi
 278
 279 if [[ $run_test =~ .*"_preload" ]] ; then
 280    if [ ! -d $lib64_dir ] ; then
 281        echo "ERROR: Missing VPP$title_dbg lib64 directory!" >&2
 282        echo "       $lib64_dir" >&2
 283    elif [ ! -d $VCL_LDPRELOAD_LIB_DIR ] ; then
 284        echo "ERROR: Missing VCL LD_PRELOAD Library directory!" >&2
 285        echo "       $VCL_LDPRELOAD_LIB_DIR" >&2
 286        env_test_failed="true"
 287    elif [ ! -f $VCL_LDPRELOAD_LIB_DIR/$vcl_ldpreload_lib ] ; then
 288        echo "ERROR: Missing VCL LD_PRELOAD library!" >&2
 289        echo "       $VCL_LDPRELOAD_LIB_DIR/$vcl_ldpreload_lib" >&2
 290        env_test_failed="true"
 291    fi
 292 fi
 293
 294 if [ ! -f $vpp_dir$vpp_app ] ; then
 295     echo "ERROR: Missing VPP$title_dbg Application!" >&2
 296     echo "       $vpp_dir$vpp_app" >&2
 297     env_test_failed="true"
 298 fi
 299
 300 if [ ! -f $vpp_dir$sock_srvr_app ] && [ ! $iperf3 -eq 1 ] ; then
 301     echo "ERROR: Missing$title_dbg Socket Server Application!" >&2
 302     echo "       $vpp_dir$sock_srvr_app" >&2
 303     env_test_failed="true"
 304 fi
 305
 306 if [ ! -f $vpp_dir$sock_clnt_app ] && [ ! $iperf3 -eq 1 ] ; then
 307     echo "ERROR: Missing$title_dbg Socket Client Application!" >&2
 308     echo "       $vpp_dir$sock_clnt_app" >&2
 309     env_test_failed="true"
 310 fi
 311
 312 if [[ $run_test =~ "docker_".* ]] ; then
 313     if [ $emacs_client -eq 1 ] || [ $emacs_server -eq 1 ] || [ $gdb_client -eq 1 ] || [ $gdb_server -eq 1 ] ; then
 314
 315         echo "WARNING: gdb is not currently supported in docker."
 316         echo "         Ignoring client/server gdb options."
 317         emacs_client=0
 318         emacs_server=0
 319         gdb_client=0
 320         gdb_server=0
 321     fi
 322 fi
 323
 324 if [[ $run_test =~ .*"_vcl" ]] && [ $iperf3 -eq 1 ] ; then
 325     echo "ERROR: Invalid option 'i' for test $run_test!"
 326     echo "       iperf3 is not compiled with the VCL library."
 327     env_test_failed="true"
 328 fi
 329
 330 if [ -n "$multi_host"] && [ ! -f "$dpdk_devbind" ] ; then
 331     echo "ERROR: Can't find dpdk-devbind.py!"
 332     echo "       Run \"cd \$WS_ROOT; make dpdk-install-dev\" to install it."
 333     echo
 334     env_test_failed="true"
 335 fi
 336
 337 if [ -n "$full_thru_host_stack_vpp_cfg" ] && [ -n "$multi_host" ] ; then
 338     echo "ERROR: Invalid options, cannot specify both \"-f\" and \"-m $multi_host\"!"
 339     echo
 340     env_test_failed="true"
 341 fi
 342
 343 if [ -n "$env_test_failed" ] ; then
 344     exit 1
 345 fi
 346
 347 if [ -f "$VCL_CONFIG" ] ; then
 348     vcl_config="$(basename $VCL_CONFIG)"
 349     vcl_config_dir="$(dirname $VCL_CONFIG)/"
 350     api_prefix="$(egrep -s '^\s*api-prefix \w+' $VCL_CONFIG | tail -1 | awk -e '{print $2}')"
 351     if [ -n "$api_prefix" ] ; then
 352         api_segment=" api-segment { gid $user_gid prefix $api_prefix }"
 353     fi
 354 fi
 355 if [ -n "$VCL_APP_NAMESPACE_ID" ] && [ -n "$VCL_APP_NAMESPACE_SECRET" ] ; then
 356     namespace_id="$VCL_APP_NAMESPACE_ID"
 357     namespace_secret="$VCL_APP_NAMESPACE_SECRET"
 358 fi
 359
 360 if [ -z "$api_segment" ] ; then
 361     api_segment=" api-segment { gid $user_gid }"
 362 fi
 363 vpp_args="unix { interactive full-coredump coredump-size unlimited exec $tmp_vpp_exec_file}${api_segment}"
 364
 365 if [ $iperf3 -eq 1 ] ; then
 366     app_dir="$(dirname $(which iperf3))/"
 367     srvr_app=$iperf_srvr_app
 368     clnt_app=$iperf_clnt_app
 369     if [[ $run_test =~ "docker_".* ]] ; then
 370         unset -v app_dir
 371         sock_srvr_port=5201
 372         docker_app_dir="networkstatic/"
 373         unset -v docker_os
 374     fi
 375 else
 376     app_dir="$vpp_dir"
 377     srvr_app="$sock_srvr_app $sock_srvr_port"
 378     clnt_app="$sock_clnt_app${sock_clnt_options} \$srvr_addr $sock_srvr_port"
 379 fi
 380
 381
 382 verify_no_vpp() {
 383     local grep_for_vpp="ps -eaf|grep -v grep|grep \"bin/vpp\""
 384
 385     if [ -n "$api_prefix" ] ; then
 386         grep_for_vpp="$grep_for_vpp|grep \"prefix $api_prefix\""
 387     fi
 388     local running_vpp="$(eval $grep_for_vpp)"
 389     if [ -n "$running_vpp" ] ; then
 390         echo "ERROR: Please kill the following vpp instance(s):"
 391         echo
 392         echo $running_vpp
 393         echo
 394         exit 1
 395     fi
 396     clean_devshm="$vpp_shm_dir*db $vpp_shm_dir*global_vm $vpp_shm_dir*vpe-api $vpp_shm_dir[0-9]*-[0-9]* $vpp_shm_dir*:segment[0-9]*"
 397     sudo rm -f $clean_devshm
 398     devshm_files="$(ls -l $clean_devshm 2>/dev/null | grep $(whoami))"
 399     if [ "$devshm_files" != "" ] ; then
 400         echo "ERROR: Please remove the following $vpp_shm_dir files:"
 401         for file in "$devshm_files" ; do
 402             echo "  $file"
 403         done
 404         exit 1
 405     fi
 406     if [ ! -d "$vpp_run_dir" ] ; then
 407         sudo mkdir $vpp_run_dir
 408         sudo chown root:$USER $vpp_run_dir
 409     fi
 410     if [ -n "$full_thru_host_stack_vpp_cfg" ] ; then
 411         sock_srvr_addr=172.16.1.1
 412         client_namespace_id="1"
 413         client_namespace_secret="5678"
 414         server_namespace_id="0"
 415         server_namespace_secret="1234"
 416         cat <<EOF >> $tmp_vpp_exec_file
 417 session enable
 418 create loop inter
 419 create loop inter
 420 set inter state loop0 up
 421 set inter ip table loop0 0
 422 set inter ip address loop0 172.16.1.1/24
 423 set inter state loop1 up
 424 set inter ip table loop1 1
 425 set inter ip address loop1 172.16.2.1/24
 426 app ns add id 0 secret 1234 sw_if_index 1
 427 app ns add id 1 secret 5678 sw_if_index 2
 428 ip route add 172.16.1.1/32 table 1 via lookup in table 0
 429 ip route add 172.16.2.1/32 table 0 via lookup in table 1
 430 EOF
 431
 432     elif [ -n "$multi_host" ] ; then
 433         vpp_eth_pci_id="$(ls -ld /sys/class/net/$vpp_eth_name/device | awk '{print $11}' | cut -d/ -f4)"
 434         if [ -z "$vpp_eth_pci_id" ] ; then
 435             echo "ERROR: Missing ethernet interface $vpp_eth_name!"
 436             usage
 437         fi
 438         printf -v bus "%x" "0x$(echo $vpp_eth_pci_id | cut -d: -f2)"
 439         printf -v slot "%x" "0x$(echo $vpp_eth_pci_id | cut -d: -f3 | cut -d. -f1)"
 440         printf -v func "%x" "0x$(echo $vpp_eth_pci_id | cut -d. -f2)"
 441
 442         vpp_eth_kernel_driver="$(basename $(ls -l /sys/bus/pci/devices/$vpp_eth_pci_id/driver | awk '{print $11}'))"
 443         if [ -z "$vpp_eth_kernel_driver" ] ; then
 444             echo "ERROR: Missing kernel driver for $vpp_eth_name!"
 445             usage
 446         fi
 447         case $vpp_eth_kernel_driver in
 448             e1000)
 449                 vpp_eth_ifname="GigabitEthernet$bus/$slot/$func" ;;
 450             ixgbe)
 451                 vpp_eth_ifname="TenGigabitEthernet$bus/$slot/$func" ;;
 452             i40e)
 453                 vpp_eth_ifname="FortyGigabitEthernet$bus/$slot/$func" ;;
 454             *)
 455                 echo "ERROR: Unknown ethernet kernel driver $vpp_eth_kernel_driver!"
 456                 usage ;;
 457         esac
 458
 459         vpp_eth_ip4_addr="$(ip -4 -br addr show $vpp_eth_name | awk '{print $3}')"
 460         if [ -z "$vpp_eth_ip4_addr" ] ; then
 461             if [ "$multi_host" = "server" ] ; then
 462                 vpp_eth_ip4_addr="10.10.10.10/24"
 463             else
 464                 vpp_eth_ip4_addr="10.10.10.11/24"
 465             fi
 466         fi
 467         if [ $use_ipv6 -eq 1 ] && [ -z "$vpp_eth_ip6_addr" ] ; then
 468             echo "ERROR: No inet6 address configured for $vpp_eth_name!"
 469             usage
 470         fi
 471         vpp_args="$vpp_args plugins { path ${lib64_dir}vpp_plugins } dpdk { dev $vpp_eth_pci_id }"
 472
 473         sudo ifconfig $vpp_eth_name down 2> /dev/null
 474         echo "Configuring VPP to use $vpp_eth_name ($vpp_eth_pci_id), inet addr $vpp_eth_ip4_addr"
 475
 476         cat <<EOF >> $tmp_vpp_exec_file
 477 set int state $vpp_eth_ifname up
 478 set int ip addr $vpp_eth_ifname $vpp_eth_ip4_addr
 479 EOF
 480
 481     fi
 482
 483     if [ -z "$full_thru_host_stack_vpp_cfg" ] && [ -n "$namespace_id" ] ; then
 484         cat <<EOF >> $tmp_vpp_exec_file
 485 session enable
 486 app ns add id $namespace_id secret $namespace_secret sw_if_index 0
 487 EOF
 488     fi
 489
 490     cat <<EOF >> $tmp_vpp_exec_file
 491 show version
 492 show version verbose
 493 show cpu
 494 show int
 495 EOF
 496 }
 497
 498 verify_no_docker_containers() {
 499     if (( $(which docker | wc -l) < 1 )) ; then
 500         echo "ERROR: docker is not installed!"
 501         echo "See https://docs.docker.com/engine/installation/linux/ubuntu/"
 502         echo " or https://docs.docker.com/engine/installation/linux/centos/"
 503         exit 1
 504     fi
 505     if (( $(docker ps | wc -l) > 1 )) ; then
 506         echo "ERROR: Run the following to kill all docker containers:"
 507         echo "docker kill \$(docker ps -q)"
 508         echo
 509         docker ps
 510         exit 1
 511     fi
 512 }
 513
 514 set_pre_cmd() {
 515     # arguments
 516     #   $1 : emacs flag
 517     #   $2 : gdb flag
 518     #   $3 : optional LD_PRELOAD library pathname
 519     local -i emacs=$1
 520     local -i gdb=$2
 521
 522     if [ $emacs -eq 1 ] ; then
 523         write_gdb_cmdfile $tmp_gdb_cmdfile $gdb_cmdfile $emacs $3
 524         pre_cmd="$gdb_in_emacs "
 525     elif [ $gdb -eq 1 ] ; then
 526         write_gdb_cmdfile $tmp_gdb_cmdfile $gdb_cmdfile $emacs $3
 527         pre_cmd="gdb -x $tmp_gdb_cmdfile --args "
 528     elif [ -z $3 ] ; then
 529         unset -v pre_cmd
 530     else
 531         docker_ld_preload="-e LD_PRELOAD=$3 "
 532         pre_cmd="LD_PRELOAD=$3 "
 533     fi
 534 }
 535
 536 write_script_header() {
 537     # arguments
 538     #   $1 : command script file
 539     #   $2 : gdb command file
 540     #   $3 : title
 541     #   $4 : optional command string (typically "sleep 2")
 542     echo "$bash_header" > $1
 543     echo -e "#\n# $1 generated on $(date)\n#" >> $1
 544     if [ $leave_tmp_files -eq 0 ] ; then
 545         if [ -n "$multi_host" ] ; then
 546             echo "trap \"rm -f $1 $2 $tmp_vpp_exec_file; sudo $dpdk_devbind -b $vpp_eth_kernel_driver $vpp_eth_pci_id; sudo ifconfig $vpp_eth_name up\" $trap_signals" >> $1
 547         else
 548             echo "trap \"rm -f $1 $2 $tmp_vpp_exec_file\" $trap_signals" >> $1
 549         fi
 550     fi
 551     if [ -n "$VCL_CONFIG" ] ; then
 552         echo "export VCL_CONFIG=${vcl_config_dir}${vcl_config}" >> $1
 553     fi
 554     if [ -n "$VCL_API_PREFIX" ] ; then
 555         echo "export VCL_API_PREFIX=$VCL_API_PREFIX" >> $1
 556     fi
 557     if [ -n "$VCL_DEBUG" ] ; then
 558         echo "export VCL_DEBUG=$VCL_DEBUG" >> $1
 559     fi
 560     if [ -n "$LDP_DEBUG" ] ; then
 561         echo "export LDP_DEBUG=$LDP_DEBUG" >> $1
 562     fi
 563     if [ -n "$VCOM_APP_NAME" ] ; then
 564         echo "export VCOM_APP_NAME=$VCOM_APP_NAME" >> $1
 565     fi
 566     if [ -n "$VCOM_SID_BIT" ] ; then
 567         echo "export VCOM_SID_BIT=$VCOM_SID_BIT" >> $1
 568     fi
 569     if [ -n "$namespace_id" ] ; then
 570         echo "export VCL_APP_NAMESPACE_ID=\"$namespace_id\"" >> $1
 571         echo "export VCL_APP_NAMESPACE_SECRET=\"$namespace_secret\"" >> $1
 572     fi
 573     if [ -n "$VCL_APP_SCOPE_LOCAL" ] || [ -z "$multi_host" ] &&
 574            [ -z "$full_thru_host_stack_vpp_cfg" ] ; then
 575         echo "export VCL_APP_SCOPE_LOCAL=true" >> $1
 576     fi
 577     if [ -n "$VCL_APP_SCOPE_GLOBAL" ] || [ -n "$multi_host" ] ||
 578            [ -n "$full_thru_host_stack_vpp_cfg" ] ; then
 579         echo "export VCL_APP_SCOPE_GLOBAL=true" >> $1
 580     fi
 581     if [ -n "$VCL_APP_PROXY_TRANSPORT_TCP" ] ; then
 582         echo "export VCL_APP_PROXY_TRANSPORT_TCP=true" >> $1
 583     fi
 584     if [ -n "$VCL_APP_PROXY_TRANSPORT_UDP" ] ; then
 585         echo "export VCL_APP_PROXY_TRANSPORT_UDP=true" >> $1
 586     fi
 587     if [ "$pre_cmd" = "$gdb_in_emacs " ] ; then
 588         if [ -n "$multi_host" ] && [[ $3 =~ "VPP".* ]] ; then
 589             cat <<EOF >> $1
 590 $gdb_in_emacs() {
 591     sudo emacs --eval "(gdb \"gdb -x $2 -i=mi --args \$*\")" --eval "(setq frame-title-format \"$3\")"
 592 }
 593 EOF
 594         else
 595             cat <<EOF >> $1
 596 $gdb_in_emacs() {
 597     emacs --eval "(gdb \"gdb -x $2 -i=mi --args \$*\")" --eval "(setq frame-title-format \"$3\")"
 598 }
 599 EOF
 600         fi
 601     fi
 602     if [ -n "$4" ] ; then
 603         echo "$4" >> $1
 604     fi
 605 }
 606
 607 write_script_footer() {
 608     # arguments
 609     #   $1 : command script file
 610     #   $2 : perf flag indicating to run bash before exit
 611     local -i perf=$2
 612     if [ $bash_after_exit -eq 1 ] || [ $perf -eq 1 ] ; then
 613         echo "bash" >> $1
 614     fi
 615 }
 616
 617 write_gdb_cmdfile() {
 618     # arguments
 619     #   $1 : gdb command file
 620     #   $2 : User specified gdb cmdfile
 621     #   $3 : emacs flag
 622     #   $4 : optional LD_PRELOAD library pathname.
 623     local -i emacs=$3
 624
 625     echo "# $1 generated on $(date)" > $1
 626     echo "#" >> $1
 627     echo "set confirm off" >> $1
 628     if [ -n "$4" ] ; then
 629         echo "set exec-wrapper env LD_PRELOAD=$4" >> $1
 630         # echo "start" >> $1
 631     fi
 632
 633     if [ ! -f $2 ] ; then
 634         echo -n "# " >> $1
 635     fi
 636     echo "source $2" >> $1
 637     if [ $emacs -eq 0 ] ; then
 638         echo "run" >> $1
 639     fi
 640 }
 641
 642 native_kernel() {
 643     banner="Running NATIVE-KERNEL socket test"
 644     if [ -z "$multi_host" ] || [ "$multi_host" = "server" ] ; then
 645         title1="SERVER$title_dbg (Native-Kernel Socket Test)"
 646         tmp_gdb_cmdfile=$tmp_gdb_cmdfile_server
 647         gdb_cmdfile=$VPPCOM_SERVER_GDB_CMDFILE
 648         set_pre_cmd $emacs_server $gdb_server
 649         write_script_header $cmd1_file $tmp_gdb_cmdfile "$title1"
 650         echo "${pre_cmd}${app_dir}${srvr_app}" >> $cmd1_file
 651         write_script_footer $cmd1_file $perf_server
 652         chmod +x $cmd1_file
 653     fi
 654
 655     if [ -z "$multi_host" ] || [ "$multi_host" = "client" ] ; then
 656         title2="CLIENT$title_dbg (Native-Kernel Socket Test)"
 657         tmp_gdb_cmdfile=$tmp_gdb_cmdfile_client
 658         gdb_cmdfile=$VPPCOM_CLIENT_GDB_CMDFILE
 659         set_pre_cmd $emacs_client $gdb_client
 660         write_script_header $cmd2_file $tmp_gdb_cmdfile "$title2" "sleep 2"
 661         echo "srvr_addr=\"$sock_srvr_addr\"" >> $cmd2_file
 662         echo "${pre_cmd}${app_dir}${clnt_app}" >> $cmd2_file
 663         write_script_footer $cmd2_file $perf_client
 664         chmod +x $cmd2_file
 665
 666     fi
 667 }
 668
 669 native_preload() {
 670     verify_no_vpp
 671     banner="Running NATIVE-PRELOAD socket test"
 672     ld_preload="$VCL_LDPRELOAD_LIB_DIR/$vcl_ldpreload_lib "
 673
 674     title1="VPP$title_dbg (Native-Preload Socket Test)"
 675     tmp_gdb_cmdfile=$tmp_gdb_cmdfile_vpp
 676     gdb_cmdfile=$VPP_GDB_CMDFILE
 677     set_pre_cmd $emacs_vpp $gdb_vpp
 678     write_script_header $cmd1_file $tmp_gdb_cmdfile "$title1"
 679     if [ -n "$multi_host" ] && [ $emacs_vpp -eq 0 ] ; then
 680         echo -n "sudo " >> $cmd1_file
 681     fi
 682     echo "${pre_cmd}$vpp_dir$vpp_app $vpp_args " >> $cmd1_file
 683     write_script_footer $cmd1_file $perf_vpp
 684     chmod +x $cmd1_file
 685
 686     if [ -z "$multi_host" ] || [ "$multi_host" = "server" ] ; then
 687         title2="SERVER$title_dbg (Native-Preload Socket Test)"
 688         tmp_gdb_cmdfile=$tmp_gdb_cmdfile_server
 689         gdb_cmdfile=$VPPCOM_SERVER_GDB_CMDFILE
 690         set_pre_cmd $emacs_server $gdb_server $ld_preload
 691         if [ -n "$full_thru_host_stack_vpp_cfg" ] ; then
 692             namespace_id="$server_namespace_id"
 693             namespace_secret="$server_namespace_secret"
 694         fi
 695         write_script_header $cmd2_file $tmp_gdb_cmdfile "$title2" "sleep 2"
 696         echo "export LD_LIBRARY_PATH=\"$lib64_dir:$VCL_LDPRELOAD_LIB_DIR:$LD_LIBRARY_PATH\"" >> $cmd2_file
 697         echo "${pre_cmd}${app_dir}${srvr_app}" >> $cmd2_file
 698         write_script_footer $cmd2_file $perf_server
 699         chmod +x $cmd2_file
 700     fi
 701
 702     if [ -z "$multi_host" ] || [ "$multi_host" = "client" ] ; then
 703         title3="CLIENT$title_dbg (Native-Preload Socket Test)"
 704         tmp_gdb_cmdfile=$tmp_gdb_cmdfile_client
 705         gdb_cmdfile=$VPPCOM_CLIENT_GDB_CMDFILE
 706         set_pre_cmd $emacs_client $gdb_client $ld_preload
 707         if [ -n "$full_thru_host_stack_vpp_cfg" ] ; then
 708             namespace_id="$client_namespace_id"
 709             namespace_secret="$client_namespace_secret"
 710         fi
 711         write_script_header $cmd3_file $tmp_gdb_cmdfile "$title3" "sleep 4"
 712         echo "export LD_LIBRARY_PATH=\"$lib64_dir:$VCL_LDPRELOAD_LIB_DIR:$LD_LIBRARY_PATH\"" >> $cmd3_file
 713         echo "srvr_addr=\"$sock_srvr_addr\"" >> $cmd3_file
 714         echo "${pre_cmd}${app_dir}${clnt_app}" >> $cmd3_file
 715         write_script_footer $cmd3_file $perf_client
 716         chmod +x $cmd3_file
 717     fi
 718 }
 719
 720 native_vcl() {
 721     verify_no_vpp
 722     banner="Running NATIVE-VCL socket test"
 723
 724     title1="VPP$title_dbg (Native-VCL Socket Test)"
 725     tmp_gdb_cmdfile=$tmp_gdb_cmdfile_vpp
 726     gdb_cmdfile=$VPP_GDB_CMDFILE
 727     set_pre_cmd $emacs_vpp $gdb_vpp
 728     write_script_header $cmd1_file $tmp_gdb_cmdfile "$title1"
 729     if [ -n "$multi_host" ] && [ $emacs_vpp -eq 0 ] ; then
 730         echo -n "sudo " >> $cmd1_file
 731     fi
 732     echo "${pre_cmd}$vpp_dir$vpp_app $vpp_args " >> $cmd1_file
 733     write_script_footer $cmd1_file $perf_vpp
 734     chmod +x $cmd1_file
 735
 736     if [ -z "$multi_host" ] || [ "$multi_host" = "server" ] ; then
 737         title2="SERVER$title_dbg (Native-VCL Socket Test)"
 738         tmp_gdb_cmdfile=$tmp_gdb_cmdfile_server
 739         gdb_cmdfile=$VPPCOM_SERVER_GDB_CMDFILE
 740         set_pre_cmd $emacs_server $gdb_server
 741         if [ "$multi_host" = "server" ] ; then
 742             delay="sleep 10"
 743         else
 744             delay="sleep 3"
 745         fi
 746         if [ -n "$full_thru_host_stack_vpp_cfg" ] ; then
 747             namespace_id="$server_namespace_id"
 748             namespace_secret="$server_namespace_secret"
 749         fi
 750         write_script_header $cmd2_file $tmp_gdb_cmdfile "$title2" "$delay"
 751         echo "export LD_LIBRARY_PATH=\"$lib64_dir:$LD_LIBRARY_PATH\"" >> $cmd2_file
 752         echo "${pre_cmd}${app_dir}${srvr_app}" >> $cmd2_file
 753         write_script_footer $cmd2_file $perf_server
 754         chmod +x $cmd2_file
 755     fi
 756
 757     if [ -z "$multi_host" ] || [ "$multi_host" = "client" ] ; then
 758         title3="CLIENT$title_dbg (Native-VCL Socket Test)"
 759         tmp_gdb_cmdfile=$tmp_gdb_cmdfile_client
 760         gdb_cmdfile=$VPPCOM_CLIENT_GDB_CMDFILE
 761         set_pre_cmd $emacs_client $gdb_client
 762         if [ "$multi_host" = "client" ] ; then
 763             delay="sleep 10"
 764         else
 765             delay="sleep 4"
 766         fi
 767         if [ -n "$full_thru_host_stack_vpp_cfg" ] ; then
 768             namespace_id="$client_namespace_id"
 769             namespace_secret="$client_namespace_secret"
 770         fi
 771         write_script_header $cmd3_file $tmp_gdb_cmdfile "$title3" "$delay"
 772         echo "export LD_LIBRARY_PATH=\"$lib64_dir:$LD_LIBRARY_PATH\"" >> $cmd3_file
 773         echo "srvr_addr=\"$sock_srvr_addr\"" >> $cmd3_file
 774         echo "${pre_cmd}${app_dir}${clnt_app}" >> $cmd3_file
 775         write_script_footer $cmd3_file $perf_client
 776         chmod +x $cmd3_file
 777     fi
 778 }
 779
 780 docker_kernel() {
 781     verify_no_docker_containers
 782     banner="Running DOCKER-KERNEL socket test"
 783
 784     if [ -z "$multi_host" ] || [ "$multi_host" = "server" ] ; then
 785         title1="SERVER$title_dbg (Docker-Native Socket Test)"
 786         tmp_gdb_cmdfile=$tmp_gdb_cmdfile_server
 787         gdb_cmdfile=$VPPCOM_SERVER_GDB_CMDFILE
 788         set_pre_cmd $emacs_server $gdb_server
 789         write_script_header $cmd1_file $tmp_gdb_cmdfile "$title1"
 790         echo "docker run -it --cpuset-cpus='4-7' --cpuset-cpus='4-7' -v $vpp_dir:$docker_vpp_dir -p $sock_srvr_port:$sock_srvr_port $docker_os ${docker_app_dir}${srvr_app}" >> $cmd1_file
 791         write_script_footer $cmd1_file $perf_server
 792         chmod +x $cmd1_file
 793     fi
 794
 795     if [ -z "$multi_host" ] || [ "$multi_host" = "client" ] ; then
 796         title2="CLIENT$title_dbg (Docker-Native Socket Test)"
 797         tmp_gdb_cmdfile=$tmp_gdb_cmdfile_client
 798         gdb_cmdfile=$VPPCOM_CLIENT_GDB_CMDFILE
 799         set_pre_cmd $emacs_client $gdb_client
 800         write_script_header $cmd2_file $tmp_gdb_cmdfile "$title2" "sleep 2"
 801         echo "$get_docker_server_ip4addr" >> $cmd2_file
 802         echo "docker run -it --cpuset-cpus='4-7' -v $vpp_dir:$docker_vpp_dir $docker_os ${docker_app_dir}${clnt_app}" >> $cmd2_file
 803         write_script_footer $cmd2_file $perf_client
 804         chmod +x $cmd2_file
 805     fi
 806 }
 807
 808 docker_preload() {
 809     verify_no_vpp
 810     verify_no_docker_containers
 811     banner="Running DOCKER-PRELOAD socket test"
 812     docker_ld_preload_dir="/vcl-ldpreload/"
 813     ld_preload_dir="$VCL_LDPRELOAD_LIB_DIR"
 814     ld_preload="$docker_ld_preload_dir$vcl_ldpreload_lib "
 815     docker_ld_preload_lib="$docker_ld_preload_dir$vcl_ldpreload_lib "
 816
 817     title1="VPP$title_dbg (Docker-Preload Socket Test)"
 818     tmp_gdb_cmdfile=$tmp_gdb_cmdfile_vpp
 819     gdb_cmdfile=$VPP_GDB_CMDFILE
 820     set_pre_cmd $emacs_vpp $gdb_vpp
 821     write_script_header $cmd1_file $tmp_gdb_cmdfile "$title1"
 822     if [ -n "$multi_host" ] ; then
 823         echo -n "sudo " >> $cmd1_file
 824     fi
 825     echo "${pre_cmd}$vpp_dir$vpp_app $vpp_args" >> $cmd1_file
 826     write_script_footer $cmd1_file $perf_vpp
 827     chmod +x $cmd1_file
 828
 829     if [ -z "$multi_host" ] || [ "$multi_host" = "server" ] ; then
 830         title2="SERVER$title_dbg (Docker-Preload Socket Test)"
 831         tmp_gdb_cmdfile=$tmp_gdb_cmdfile_server
 832         gdb_cmdfile=$VPPCOM_SERVER_GDB_CMDFILE
 833         set_pre_cmd $emacs_server $gdb_server $docker_ld_preload_lib
 834         if [ -n "$full_thru_host_stack_vpp_cfg" ] ; then
 835             namespace_id="$server_namespace_id"
 836             namespace_secret="$server_namespace_secret"
 837         fi
 838         write_script_header $cmd2_file $tmp_gdb_cmdfile "$title2" "sleep 2"
 839         echo "docker run -it -v $vpp_shm_dir:$vpp_shm_dir -v $vpp_dir:$docker_vpp_dir -v $lib64_dir:$docker_lib64_dir -v $ld_preload_dir:$docker_ld_preload_dir -v $vcl_config_dir:$docker_vcl_config_dir -p $sock_srvr_port:$sock_srvr_port -e VCL_DEBUG=$VCL_DEBUG -e VCL_CONFIG=${docker_vcl_config_dir}$vcl_config -e LD_LIBRARY_PATH=$docker_lib64_dir:$docker_ld_preload_dir ${docker_ld_preload}$docker_os ${docker_app_dir}${srvr_app}" >> $cmd2_file
 840         write_script_footer $cmd2_file $perf_server
 841         chmod +x $cmd2_file
 842     fi
 843
 844     if [ -z "$multi_host" ] || [ "$multi_host" = "client" ] ; then
 845         title3="CLIENT$title_dbg (Docker-Preload Socket Test)"
 846         tmp_gdb_cmdfile=$tmp_gdb_cmdfile_client
 847         gdb_cmdfile=$VPPCOM_CLIENT_GDB_CMDFILE
 848         set_pre_cmd $emacs_client $gdb_client $docker_ld_preload_lib
 849         if [ -n "$full_thru_host_stack_vpp_cfg" ] ; then
 850             namespace_id="$client_namespace_id"
 851             namespace_secret="$client_namespace_secret"
 852         fi
 853         write_script_header $cmd3_file $tmp_gdb_cmdfile "$title3" "sleep 4"
 854         echo "$get_docker_server_ip4addr" >> $cmd3_file
 855         echo "docker run -it --cpuset-cpus='4-7' -v $vpp_shm_dir:$vpp_shm_dir -v $vpp_dir:$docker_vpp_dir -v $lib64_dir:$docker_lib64_dir  -v $ld_preload_dir:$docker_ld_preload_dir -v $vcl_config_dir:$docker_vcl_config_dir -e VCL_DEBUG=$VCL_DEBUG -e VCL_CONFIG=${docker_vcl_config_dir}$vcl_config -e LD_LIBRARY_PATH=$docker_lib64_dir ${docker_ld_preload}$docker_os ${docker_app_dir}${clnt_app}" >> $cmd3_file
 856         write_script_footer $cmd3_file $perf_client
 857         chmod +x $cmd3_file
 858     fi
 859 }
 860
 861 docker_vcl() {
 862     verify_no_vpp
 863     verify_no_docker_containers
 864     banner="Running DOCKER-VCL socket test"
 865
 866     title1="VPP$title_dbg (Docker-VCL Socket Test)"
 867     tmp_gdb_cmdfile=$tmp_gdb_cmdfile_vpp
 868     gdb_cmdfile=$VPP_GDB_CMDFILE
 869     set_pre_cmd $emacs_vpp $gdb_vpp
 870     write_script_header $cmd1_file $tmp_gdb_cmdfile "$title1"
 871     if [ -n "$multi_host" ] ; then
 872         echo -n "sudo " >> $cmd1_file
 873     fi
 874     echo "${pre_cmd}$vpp_dir$vpp_app $vpp_args" >> $cmd1_file
 875     write_script_footer $cmd1_file $perf_vpp
 876     chmod +x $cmd1_file
 877
 878     if [ -z "$multi_host" ] || [ "$multi_host" = "server" ] ; then
 879         title2="SERVER$title_dbg (Docker-VCL Socket Test)"
 880         tmp_gdb_cmdfile=$tmp_gdb_cmdfile_server
 881         gdb_cmdfile=$VPPCOM_SERVER_GDB_CMDFILE
 882         set_pre_cmd $emacs_server $gdb_server
 883         if [ -n "$full_thru_host_stack_vpp_cfg" ] ; then
 884             namespace_id="$server_namespace_id"
 885             namespace_secret="$server_namespace_secret"
 886         fi
 887         write_script_header $cmd2_file $tmp_gdb_cmdfile "$title2" "sleep 2"
 888         echo "docker run -it --cpuset-cpus='4-7' -v $vpp_shm_dir:$vpp_shm_dir -v $vpp_dir:$docker_vpp_dir -v $lib64_dir:$docker_lib64_dir -v $vcl_config_dir:$docker_vcl_config_dir -p $sock_srvr_port:$sock_srvr_port -e VCL_CONFIG=${docker_vcl_config_dir}/$vcl_config -e LD_LIBRARY_PATH=$docker_lib64_dir $docker_os ${docker_app_dir}${srvr_app}" >> $cmd2_file
 889         write_script_footer $cmd2_file $perf_server
 890         chmod +x $cmd2_file
 891     fi
 892
 893     if [ -z "$multi_host" ] || [ "$multi_host" = "client" ] ; then
 894         title3="CLIENT$title_dbg (Docker-VCL Socket Test)"
 895         tmp_gdb_cmdfile=$tmp_gdb_cmdfile_client
 896         gdb_cmdfile=$VPPCOM_CLIENT_GDB_CMDFILE
 897         set_pre_cmd $emacs_client $gdb_client
 898         if [ -n "$full_thru_host_stack_vpp_cfg" ] ; then
 899             namespace_id="$client_namespace_id"
 900             namespace_secret="$client_namespace_secret"
 901         fi
 902         write_script_header $cmd3_file $tmp_gdb_cmdfile "$title3" "sleep 3"
 903         echo "$get_docker_server_ip4addr" >> $cmd3_file
 904         echo "docker run -it --cpuset-cpus='4-7' -v $vpp_shm_dir:$vpp_shm_dir -v $vpp_dir:$docker_vpp_dir -v $lib64_dir:$docker_lib64_dir -v $vcl_config_dir:$docker_vcl_config_dir -e VCL_CONFIG=${docker_vcl_config_dir}/$vcl_config -e LD_LIBRARY_PATH=$docker_lib64_dir $docker_os ${docker_app_dir}${clnt_app}" >> $cmd3_file
 905         write_script_footer $cmd3_file $perf_client
 906         chmod +x $cmd3_file
 907     fi
 908 }
 909
 910 if [[ $run_test ]] ; then
 911     eval $run_test
 912 else
 913     echo "ERROR: Please specify a test to run!" >&2
 914     usage;
 915 fi
 916
 917 if (( $(which xfce4-terminal | wc -l) > 0 )) ; then
 918     xterm_cmd="xfce4-terminal --geometry $xterm_geom"
 919     if [[ $use_tabs ]] ; then
 920         declare -a tab_cmd_files
 921         declare -a tab_titles
 922         declare -i i=0
 923
 924         if [ -x "$cmd1_file" ] ; then
 925             tab_cmd_files[$i]="$cmd1_file"
 926             tab_titles[$i]="$title1"
 927             (( i++ ))
 928         fi
 929         if [ -x "$cmd2_file" ] ; then
 930             tab_cmd_files[$i]="$cmd2_file"
 931             tab_titles[$i]="$title2"
 932             (( i++ ))
 933         fi
 934         if [ -x "$cmd3_file" ] ; then
 935             tab_cmd_files[$i]="$cmd3_file"
 936             tab_titles[$i]="$title3"
 937         fi
 938
 939         if [ -n "${tab_cmd_files[2]}" ] ; then
 940             $xterm_cmd  --title "${tab_titles[0]}" --command "${tab_cmd_files[0]}" --tab --title "${tab_titles[1]}" --command "${tab_cmd_files[1]}" --tab --title "${tab_titles[2]}" --command "${tab_cmd_files[2]}"
 941         elif [ -n "${tab_cmd_files[1]}" ] ; then
 942             $xterm_cmd --title "${tab_titles[0]}" --command "${tab_cmd_files[0]}" --tab --title "${tab_titles[1]}" --command "${tab_cmd_files[1]}"
 943
 944         else
 945             $xterm_cmd --title "${tab_titles[0]}" --command "${tab_cmd_files[0]}"
 946         fi
 947
 948     else
 949         if [ -x "$cmd1_file" ] ; then
 950             ($xterm_cmd --title "$title1" --command "$cmd1_file" &)
 951         fi
 952         if [ -x "$cmd2_file" ] ; then
 953             ($xterm_cmd --title "$title2" --command "$cmd2_file" &)
 954         fi
 955         if [ -x "$cmd3_file" ] ; then
 956             ($xterm_cmd --title "$title3" --command "$cmd3_file" &)
 957         fi
 958     fi
 959
 960 else
 961     if [[ $use_tabs ]] ; then
 962         echo "Sorry, plain ol' xterm doesn't support tabs."
 963     fi
 964     xterm_cmd="xterm -fs 10 -geometry $xterm_geom"
 965     if [ -x "$cmd1_file" ] ; then
 966         ($xterm_cmd -title "$title1" -e "$cmd1_file" &)
 967     fi
 968     if [ -x "$cmd2_file" ] ; then
 969         ($xterm_cmd -title "$title2" -e "$cmd2_file" &)
 970     fi
 971     if [ -x "$cmd3_file" ] ; then
 972         ($xterm_cmd -title "$title3" -e "$cmd3_file" &)
 973     fi
 974 fi
 975
 976 sleep 1