session: support multiple worker binds
[vpp.git] / test / scripts / socket_test.sh
1 #! /bin/bash
2 #
3 # socket_test.sh -- script to run socket tests.
4 #
5 script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
6 vpp_dir="$WS_ROOT/build-root/install-vpp-native/vpp/bin/"
7 vpp_debug_dir="$WS_ROOT/build-root/install-vpp_debug-native/vpp/bin/"
8 vpp_shm_dir="/dev/shm/"
9 vpp_run_dir="/run/vpp"
10 lib64_dir="$WS_ROOT/build-root/install-vpp-native/vpp/lib64/"
11 lib64_debug_dir="$WS_ROOT/build-root/install-vpp_debug-native/vpp/lib64/"
12 dpdk_devbind="/usr/share/dpdk/usertools/dpdk-devbind.py"
13 docker_vpp_dir="/vpp/"
14 docker_app_dir="/vpp/"
15 docker_lib64_dir="/vpp-lib64/"
16 docker_os="ubuntu"
17 vcl_ldpreload_lib="libvcl_ldpreload.so.0.0.0"
18 user_gid="$(id -g)"
19 vpp_app="vpp"
20 sock_srvr_app="sock_test_server"
21 sock_clnt_app="sock_test_client"
22 sock_srvr_addr="127.0.0.1"
23 sock_srvr_port="22000"
24 iperf_srvr_app="iperf3 -V4d -s"
25 iperf_clnt_app="iperf3 -V4d -c \$srvr_addr"
26 gdb_in_emacs="gdb_in_emacs"
27 vcl_config="vcl.conf"
28 vcl_config_dir="$WS_ROOT/src/vcl/"
29 docker_vcl_config_dir="/etc/vpp/"
30 xterm_geom="100x60"
31 bash_header="#! /bin/bash"
32 tmp_cmdfile_prefix="/tmp/socket_test_cmd"
33 cmd1_file="${tmp_cmdfile_prefix}1.$$"
34 cmd2_file="${tmp_cmdfile_prefix}2.$$"
35 cmd3_file="${tmp_cmdfile_prefix}3.$$"
36 vpp_eth_name="enp0s8"
37 tmp_vpp_exec_file="/tmp/vpp_config.$$"
38 tmp_gdb_cmdfile_prefix="/tmp/gdb_cmdfile"
39 def_gdb_cmdfile_prefix="$WS_ROOT/extras/gdb/gdb_cmdfile"
40 tmp_gdb_cmdfile_vpp="${tmp_gdb_cmdfile_prefix}_vpp.$$"
41 tmp_gdb_cmdfile_client="${tmp_gdb_cmdfile_prefix}_vcl_client.$$"
42 tmp_gdb_cmdfile_server="${tmp_gdb_cmdfile_prefix}_vcl_server.$$"
43 get_docker_server_ip4addr='srvr_addr=$(docker network inspect bridge | grep IPv4Address | awk -e '\''{print $2}'\'' | sed -e '\''s,/16,,'\'' -e '\''s,",,g'\'' -e '\''s/,//'\'')'
44 #' single quote to fix the confused emacs colorizer.
45 trap_signals="SIGINT SIGTERM EXIT"
46
47 # Set default values for imported environment variables if they don't exist.
48 #
49 VPP_GDB_CMDFILE="${VPP_GDB_CMDFILE:-${def_gdb_cmdfile_prefix}.vpp}"
50 VPPCOM_CLIENT_GDB_CMDFILE="${VPPCOM_CLIENT_GDB_CMDFILE:-${def_gdb_cmdfile_prefix}.vppcom_client}"
51 VPPCOM_SERVER_GDB_CMDFILE="${VPPCOM_SERVER_GDB_CMDFILE:-${def_gdb_cmdfile_prefix}.vppcom_server}"
52
53 usage() {
54     cat <<EOF
55 Usage: socket_test.sh OPTIONS TEST
56 TESTS:
57   nk, native-kernel   Run server & client on host using kernel.
58   nv, native-vcl      Run vpp, server & client on host using VppComLib.
59   np, native-preload  Run vpp, server & client on host using LD_PRELOAD.
60   dk, docker-kernel   Run server & client in docker using kernel stack.
61   dv, docker-vcl      Run vpp on host, server & client in docker using VppComLib.
62   dp, docker-preload  Run vpp on host, server & client in docker using LD_PRELOAD.
63
64 OPTIONS:
65   -h                  Print this usage text.
66   -l                  Leave ${tmp_cmdfile_prefix}* files after test run.
67   -b                  Run bash after application exit.
68   -d                  Run the vpp_debug version of all apps.
69   -c                  Set VCL_CONFIG to use the vcl_test.conf file.
70   -i                  Run iperf3 for client/server app in native tests.
71   -n                  Name of ethernet for VPP to use in multi-host cfg.
72   -f                  Full thru host stack vpp configuration. 
73   -m c[lient]         Run client in multi-host cfg (server on remote host)
74      s[erver]         Run server in multi-host cfg (client on remote host)
75   -e a[ll]            Run all in emacs+gdb.
76      c[lient]         Run client in emacs+gdb.
77      s[erver]         Run server in emacs+gdb.
78      v[pp]            Run vpp in emacs+gdb.
79   -g a[ll]            Run all in gdb.
80      c[lient]         Run client in gdb.
81      s[erver]         Run server in gdb.
82      v[pp]            Run vpp in gdb.
83   -t                  Use tabs in one xterm if available (e.g. xfce4-terminal).
84
85 OPTIONS passed to client/server:
86   -6                  Use IPv6.
87   -D                  Use UDP as the transport.
88   -S <ip address>     Server IP address.
89   -P <server port>    Server Port number.
90   -E <data>           Run Echo test.
91   -N <num-writes>     Test Cfg: number of writes.
92   -R <rxbuf-size>     Test Cfg: rx buffer size.
93   -T <txbuf-size>     Test Cfg: tx buffer size.
94   -U                  Run Uni-directional test.
95   -B                  Run Bi-directional test.
96   -I <num-tst-socks>  Send data over multiple test sockets in parallel.
97   -V                  Test Cfg: Verbose mode.
98   -X                  Exit client/server after running test.
99
100 Environment variables:
101   VCL_CONFIG                Pathname of vppcom configuration file.
102   VPP_GDB_CMDFILE            Pathname of gdb command file for vpp.
103   VPPCOM_CLIENT_GDB_CMDFILE  Pathname of gdb command file for client.
104   VPPCOM_SERVER_GDB_CMDFILE  Pathname of gdb command file for server.
105 EOF
106     exit 1
107 }
108
109 declare -i emacs_vpp=0
110 declare -i emacs_client=0
111 declare -i emacs_server=0
112 declare -i gdb_vpp=0
113 declare -i gdb_client=0
114 declare -i gdb_server=0
115 declare -i perf_vpp=0
116 declare -i perf_client=0
117 declare -i perf_server=0
118 declare -i leave_tmp_files=0
119 declare -i bash_after_exit=0
120 declare -i iperf3=0
121 declare -i use_ipv6=0
122 declare -i transport_udp=0
123
124 while getopts ":hitlbcd6fn:m:e:g:p:E:I:N:P:R:S:T:UBVXD" opt; do
125     case $opt in
126         h) usage ;;
127         l) leave_tmp_files=1
128            ;;
129         b) bash_after_exit=1
130            ;;
131         i) iperf3=1
132            ;;
133         6) use_ipv6=1
134            sock_srvr_addr="::1"
135            sock_clnt_options="$sock_clnt_options -$opt"
136            sock_srvr_options="$sock_srvr_options -$opt"
137            ;;
138         f) full_thru_host_stack_vpp_cfg=1
139            ;;
140         t) xterm_geom="180x40"
141            use_tabs="true"
142            ;;
143         c) VCL_CONFIG="${vcl_config_dir}vcl_test.conf"
144            ;;
145         d) title_dbg="-DEBUG"
146            vpp_dir=$vpp_debug_dir
147            lib64_dir=$lib64_debug_dir
148            ;;
149         e) if [ $OPTARG = "a" ] || [ $OPTARG = "all" ] ; then
150                emacs_client=1
151                emacs_server=1
152                emacs_vpp=1
153            elif [ $OPTARG = "c" ] || [ $OPTARG = "client" ] ; then
154                emacs_client=1
155            elif [ $OPTARG = "s" ] || [ $OPTARG = "server" ] ; then
156                emacs_server=1
157            elif [ $OPTARG = "v" ] || [ $OPTARG = "vpp" ] ; then
158                emacs_vpp=1
159            else
160                echo "ERROR: Option -e unknown argument \'$OPTARG\'" >&2
161                usage
162            fi
163            title_dbg="-DEBUG"
164            vpp_dir=$vpp_debug_dir
165            lib64_dir=$lib64_debug_dir
166            ;;
167         n) vpp_eth_name="$OPTARG"
168            ;;
169         m) if [ $OPTARG = "c" ] || [ $OPTARG = "client" ] ; then
170                multi_host="client"
171            elif [ $OPTARG = "s" ] || [ $OPTARG = "server" ] ; then
172                multi_host="server"
173            else
174                echo "ERROR: Option -e unknown argument \'$OPTARG\'" >&2
175                usage
176            fi
177            ;;
178         g) if [ $OPTARG = "a" ] || [ $OPTARG = "all" ] ; then
179                gdb_client=1
180                gdb_server=1
181                gdb_vpp=1
182            elif [ $OPTARG = "c" ] || [ $OPTARG = "client" ] ; then
183                gdb_client=1
184            elif [ $OPTARG = "s" ] || [ $OPTARG = "server" ] ; then
185                gdb_server=1
186            elif [ $OPTARG = "v" ] || [ $OPTARG = "vpp" ] ; then
187                gdb_vpp=1
188            else
189                echo "ERROR: Option -g unknown argument \'$OPTARG\'" >&2
190                usage
191            fi
192            ;;
193         p) if [ $OPTARG = "a" ] || [ $OPTARG = "all" ] ; then
194                perf_client=1
195                perf_server=1
196                perf_vpp=1
197            elif [ $OPTARG = "c" ] || [ $OPTARG = "client" ] ; then
198                perf_client=1
199            elif [ $OPTARG = "s" ] || [ $OPTARG = "server" ] ; then
200                perf_server=1
201            elif [ $OPTARG = "v" ] || [ $OPTARG = "vpp" ] ; then
202                perf_vpp=1
203            else
204                echo "ERROR: Option -p unknown argument \'$OPTARG\'" >&2
205                usage
206            fi
207            echo "WARNING: -p options TBD"
208            ;;
209         S) sock_srvr_addr="$OPTARG"
210            ;;
211         P) sock_srvr_port="$OPTARG"
212            ;;
213         D) sock_clnt_options="$sock_clnt_options -$opt"
214            sock_srvr_options="$sock_srvr_options -$opt"
215            ;;
216 E|I|N|R|T) sock_clnt_options="$sock_clnt_options -$opt \"$OPTARG\""
217            ;;
218   U|B|V|X) sock_clnt_options="$sock_clnt_options -$opt"
219            ;;
220        \?)
221            echo "ERROR: Invalid option: -$OPTARG" >&2
222            usage
223            ;;
224         :)
225            echo "ERROR: Option -$OPTARG requires an argument." >&2
226            usage
227            ;;
228     esac
229 done
230
231 shift $(( $OPTIND-1 ))
232 while ! [[ $run_test ]] && (( $# > 0 )) ; do
233     case $1 in
234         "nk" | "native-kernel")
235             run_test="native_kernel" ;;
236         "np" | "native-preload")
237             run_test="native_preload" ;;
238         "nv" | "native-vcl")
239             sock_srvr_app="bin/vcl_test_server"
240             sock_clnt_app="bin/vcl_test_client"
241             run_test="native_vcl" ;;
242         "dk" | "docker-kernel")
243             run_test="docker_kernel" ;;
244         "dp" | "docker-preload")
245             run_test="docker_preload" ;;
246         "dv" | "docker-vcl")
247             sock_srvr_app="bin/vcl_test_server"
248             sock_clnt_app="bin/vcl_test_client"
249             run_test="docker_vcl" ;;
250         *)
251             echo "ERROR: Unknown option '$1'!" >&2
252             usage ;;
253     esac
254     shift
255 done
256
257 if [ -z "$VCL_DEBUG" ] ; then
258     if [ "$title_dbg" = "-DEBUG" ] ; then
259         VCL_DEBUG=1
260     else
261         VCL_DEBUG=0
262     fi
263 fi
264
265 VCL_LDPRELOAD_LIB_DIR="${VCL_LDPRELOAD_LIB_DIR:-$lib64_dir}"
266
267 if [ -z "$WS_ROOT" ] ; then
268     echo "ERROR: WS_ROOT environment variable not set!" >&2
269     echo "       Please set WS_ROOT to VPP workspace root directory." >&2
270     exit 1
271 fi
272
273 if [[ "$(grep bin_PROGRAMS $WS_ROOT/src/vcl.am)" = "" ]] ; then
274     $WS_ROOT/extras/vagrant/vcl_test.sh $WS_ROOT $USER
275     (cd $WS_ROOT; make build)
276 fi
277
278 if [ ! -d $vpp_dir ] ; then
279     echo "ERROR: Missing VPP$title_dbg bin directory!" >&2
280     echo "       $vpp_dir" >&2
281     env_test_failed="true"
282 fi
283
284 if [[ $run_test =~ .*"_preload" ]] ; then
285    if [ ! -d $lib64_dir ] ; then
286        echo "ERROR: Missing VPP$title_dbg lib64 directory!" >&2
287        echo "       $lib64_dir" >&2
288    elif [ ! -d $VCL_LDPRELOAD_LIB_DIR ] ; then
289        echo "ERROR: Missing VCL LD_PRELOAD Library directory!" >&2
290        echo "       $VCL_LDPRELOAD_LIB_DIR" >&2
291        env_test_failed="true"
292    elif [ ! -f $VCL_LDPRELOAD_LIB_DIR/$vcl_ldpreload_lib ] ; then
293        echo "ERROR: Missing VCL LD_PRELOAD library!" >&2
294        echo "       $VCL_LDPRELOAD_LIB_DIR/$vcl_ldpreload_lib" >&2
295        env_test_failed="true"
296    fi
297 fi
298
299 if [ ! -f $vpp_dir$vpp_app ] ; then
300     echo "ERROR: Missing VPP$title_dbg Application!" >&2
301     echo "       $vpp_dir$vpp_app" >&2
302     env_test_failed="true"
303 fi
304
305 if [ ! -f $vpp_dir$sock_srvr_app ] && [ ! $iperf3 -eq 1 ] ; then
306     echo "ERROR: Missing$title_dbg Socket Server Application!" >&2
307     echo "       $vpp_dir$sock_srvr_app" >&2
308     env_test_failed="true"
309 fi
310
311 if [ ! -f $vpp_dir$sock_clnt_app ] && [ ! $iperf3 -eq 1 ] ; then
312     echo "ERROR: Missing$title_dbg Socket Client Application!" >&2
313     echo "       $vpp_dir$sock_clnt_app" >&2
314     env_test_failed="true"
315 fi
316
317 if [[ $run_test =~ "docker_".* ]] ; then
318     if [ $emacs_client -eq 1 ] || [ $emacs_server -eq 1 ] || [ $gdb_client -eq 1 ] || [ $gdb_server -eq 1 ] ; then
319         
320         echo "WARNING: gdb is not currently supported in docker."
321         echo "         Ignoring client/server gdb options."
322         emacs_client=0
323         emacs_server=0
324         gdb_client=0
325         gdb_server=0
326     fi
327 fi
328
329 if [[ $run_test =~ .*"_vcl" ]] && [ $iperf3 -eq 1 ] ; then
330     echo "ERROR: Invalid option 'i' for test $run_test!"
331     echo "       iperf3 is not compiled with the VCL library."
332     env_test_failed="true"
333 fi
334
335 if [ -n "$multi_host"] && [ ! -f "$dpdk_devbind" ] ; then
336     echo "ERROR: Can't find dpdk-devbind.py!"
337     echo "       Run \"cd \$WS_ROOT; make dpdk-install-dev\" to install it."
338     echo
339     env_test_failed="true"
340 fi
341
342 if [ -n "$full_thru_host_stack_vpp_cfg" ] && [ -n "$multi_host" ] ; then
343     echo "ERROR: Invalid options, cannot specify both \"-f\" and \"-m $multi_host\"!"
344     echo
345     env_test_failed="true"
346 fi
347
348 if [ -n "$env_test_failed" ] ; then
349     exit 1
350 fi
351
352 if [ -f "$VCL_CONFIG" ] ; then
353     vcl_config="$(basename $VCL_CONFIG)"
354     vcl_config_dir="$(dirname $VCL_CONFIG)/"
355     api_prefix="$(egrep -s '^\s*api-prefix \w+' $VCL_CONFIG | tail -1 | awk -e '{print $2}')"
356     if [ -n "$api_prefix" ] ; then
357         api_segment=" api-segment { gid $user_gid prefix $api_prefix }"
358     fi
359 fi
360 if [ -n "$VCL_APP_NAMESPACE_ID" ] && [ -n "$VCL_APP_NAMESPACE_SECRET" ] ; then
361     namespace_id="$VCL_APP_NAMESPACE_ID"
362     namespace_secret="$VCL_APP_NAMESPACE_SECRET"
363 fi
364     
365 if [ -z "$api_segment" ] ; then
366     api_segment=" api-segment { gid $user_gid }"
367 fi
368 vpp_args="unix { interactive full-coredump coredump-size unlimited exec $tmp_vpp_exec_file}${api_segment}"
369
370 if [ $iperf3 -eq 1 ] ; then
371     app_dir="$(dirname $(which iperf3))/"
372     srvr_app=$iperf_srvr_app
373     clnt_app=$iperf_clnt_app
374     if [[ $run_test =~ "docker_".* ]] ; then
375         unset -v app_dir
376         sock_srvr_port=5201
377         docker_app_dir="networkstatic/"
378         unset -v docker_os
379     fi
380 else
381     app_dir="$vpp_dir"
382     srvr_app="$sock_srvr_app${sock_srvr_options} $sock_srvr_port"
383     clnt_app="$sock_clnt_app${sock_clnt_options} \$srvr_addr $sock_srvr_port"
384 fi
385
386
387 verify_no_vpp() {
388     local grep_for_vpp="ps -eaf|grep -v grep|grep \"bin/vpp\""
389     
390     if [ -n "$api_prefix" ] ; then
391         grep_for_vpp="$grep_for_vpp|grep \"prefix $api_prefix\""
392     fi
393     local running_vpp="$(eval $grep_for_vpp)"
394     if [ -n "$running_vpp" ] ; then
395         echo "ERROR: Please kill the following vpp instance(s):"
396         echo
397         echo $running_vpp
398         echo
399         exit 1
400     fi
401     clean_devshm="$vpp_shm_dir*db $vpp_shm_dir*global_vm $vpp_shm_dir*vpe-api $vpp_shm_dir[0-9]*-[0-9]* $vpp_shm_dir*:segment[0-9]*"
402     sudo rm -f $clean_devshm
403     devshm_files="$(ls -l $clean_devshm 2>/dev/null | grep $(whoami))"
404     if [ "$devshm_files" != "" ] ; then
405         echo "ERROR: Please remove the following $vpp_shm_dir files:"
406         for file in "$devshm_files" ; do
407             echo "  $file"
408         done
409         exit 1
410     fi
411     if [ ! -d "$vpp_run_dir" ] ; then
412         sudo mkdir $vpp_run_dir
413         sudo chown root:$USER $vpp_run_dir
414     fi
415     if [ $use_ipv6 -eq 0 ] && [ -n "$full_thru_host_stack_vpp_cfg" ] ; then
416         sock_srvr_table=0
417         sock_srvr_addr=172.16.1.1
418         sock_client_table=1
419         sock_client_addr=172.16.2.1
420         client_namespace_id="1"
421         client_namespace_secret="5678"
422         server_namespace_id="0"
423         server_namespace_secret="1234"
424         cat <<EOF >> $tmp_vpp_exec_file
425 session enable
426 create loop inter
427 create loop inter
428 set inter state loop0 up 
429 set inter ip table loop0 $sock_srvr_table
430 set inter ip address loop0 $sock_srvr_addr/24
431 set inter state loop1 up
432 set inter ip table loop1 $sock_client_table
433 set inter ip address loop1 $sock_client_addr/24
434 app ns add id 0 secret 1234 sw_if_index 1
435 app ns add id 1 secret 5678 sw_if_index 2
436 ip route add $sock_srvr_addr/32 table $sock_client_table via lookup in table $sock_srvr_table
437 ip route add $sock_client_addr/32 table $sock_srvr_table via lookup in table $sock_client_table
438 EOF
439     elif [ $use_ipv6 -eq 1 ] && [ -n "$full_thru_host_stack_vpp_cfg" ] ; then
440         sock_srvr_table=1
441         sock_srvr_addr=fd01:1::1
442         sock_client_table=2
443         sock_client_addr=fd01:2::1
444         client_namespace_id="1"
445         client_namespace_secret="5678"
446         server_namespace_id="0"
447         server_namespace_secret="1234"
448         cat <<EOF >> $tmp_vpp_exec_file
449 session enable
450 create loop inter
451 create loop inter
452 set inter state loop0 up 
453 set inter ip6 table loop0 $sock_srvr_table
454 set inter ip address loop0 $sock_srvr_addr/64
455 set inter state loop1 up
456 set inter ip6 table loop1 $sock_client_table
457 set inter ip address loop1 $sock_client_addr/64
458 app ns add id 0 secret 1234 sw_if_index 1
459 app ns add id 1 secret 5678 sw_if_index 2
460 ip route add $sock_srvr_addr/128 table $sock_client_table via lookup in table $sock_srvr_table
461 ip route add $sock_client_addr/128 table $sock_srvr_table via lookup in table $sock_client_table
462 EOF
463     elif [ -n "$multi_host" ] ; then
464         vpp_eth_pci_id="$(ls -ld /sys/class/net/$vpp_eth_name/device | awk '{print $11}' | cut -d/ -f4)"
465         if [ -z "$vpp_eth_pci_id" ] ; then
466             echo "ERROR: Missing ethernet interface $vpp_eth_name!"
467             usage
468         fi
469         printf -v bus "%x" "0x$(echo $vpp_eth_pci_id | cut -d: -f2)"
470         printf -v slot "%x" "0x$(echo $vpp_eth_pci_id | cut -d: -f3 | cut -d. -f1)"
471         printf -v func "%x" "0x$(echo $vpp_eth_pci_id | cut -d. -f2)"
472
473         vpp_eth_kernel_driver="$(basename $(ls -l /sys/bus/pci/devices/$vpp_eth_pci_id/driver | awk '{print $11}'))"
474         if [ -z "$vpp_eth_kernel_driver" ] ; then
475             echo "ERROR: Missing kernel driver for $vpp_eth_name!"
476             usage
477         fi
478         case $vpp_eth_kernel_driver in
479             e1000)
480                 vpp_eth_ifname="GigabitEthernet$bus/$slot/$func" ;;
481             ixgbe)
482                 vpp_eth_ifname="TenGigabitEthernet$bus/$slot/$func" ;;
483             i40e)
484                 vpp_eth_ifname="FortyGigabitEthernet$bus/$slot/$func" ;;
485             *)
486                 echo "ERROR: Unknown ethernet kernel driver $vpp_eth_kernel_driver!"
487                 usage ;;
488         esac
489         
490         vpp_eth_ip4_addr="$(ip -4 -br addr show $vpp_eth_name | awk '{print $3}')"
491         if [ -z "$vpp_eth_ip4_addr" ] ; then
492             if [ "$multi_host" = "server" ] ; then
493                 vpp_eth_ip4_addr="10.10.10.10/24"
494             else
495                 vpp_eth_ip4_addr="10.10.10.11/24"
496             fi
497         fi
498         if [ $use_ipv6 -eq 1 ] && [ -z "$vpp_eth_ip6_addr" ] ; then
499             echo "ERROR: No inet6 address configured for $vpp_eth_name!"
500             usage
501         fi
502         vpp_args="$vpp_args plugins { path ${lib64_dir}vpp_plugins } dpdk { dev $vpp_eth_pci_id }"
503                 
504         sudo ifconfig $vpp_eth_name down 2> /dev/null
505         echo "Configuring VPP to use $vpp_eth_name ($vpp_eth_pci_id), inet addr $vpp_eth_ip4_addr"
506
507         cat <<EOF >> $tmp_vpp_exec_file
508 set int state $vpp_eth_ifname up
509 set int ip addr $vpp_eth_ifname $vpp_eth_ip4_addr
510 EOF
511
512     fi
513
514     if [ -z "$full_thru_host_stack_vpp_cfg" ] && [ -n "$namespace_id" ] ; then
515         cat <<EOF >> $tmp_vpp_exec_file
516 session enable
517 app ns add id $namespace_id secret $namespace_secret sw_if_index 0
518 EOF
519     fi
520     
521     cat <<EOF >> $tmp_vpp_exec_file
522 show version
523 show version verbose
524 show cpu
525 show int
526 EOF
527 }
528
529 verify_no_docker_containers() {
530     if (( $(which docker | wc -l) < 1 )) ; then
531         echo "ERROR: docker is not installed!"
532         echo "See https://docs.docker.com/engine/installation/linux/ubuntu/"
533         echo " or https://docs.docker.com/engine/installation/linux/centos/"
534         exit 1
535     fi
536     if (( $(docker ps | wc -l) > 1 )) ; then
537         echo "ERROR: Run the following to kill all docker containers:"
538         echo "docker kill \$(docker ps -q)"
539         echo
540         docker ps
541         exit 1
542     fi
543 }
544
545 set_pre_cmd() {
546     # arguments
547     #   $1 : emacs flag
548     #   $2 : gdb flag
549     #   $3 : optional LD_PRELOAD library pathname
550     local -i emacs=$1
551     local -i gdb=$2
552
553     if [ $emacs -eq 1 ] ; then
554         write_gdb_cmdfile $tmp_gdb_cmdfile $gdb_cmdfile $emacs $3
555         pre_cmd="$gdb_in_emacs "
556     elif [ $gdb -eq 1 ] ; then
557         write_gdb_cmdfile $tmp_gdb_cmdfile $gdb_cmdfile $emacs $3
558         pre_cmd="gdb -x $tmp_gdb_cmdfile --args "
559     elif [ -z $3 ] ; then
560         unset -v pre_cmd
561     else
562         docker_ld_preload="-e LD_PRELOAD=$3 "
563         pre_cmd="LD_PRELOAD=$3 "
564     fi
565 }
566
567 write_script_header() {
568     # arguments
569     #   $1 : command script file
570     #   $2 : gdb command file
571     #   $3 : title
572     #   $4 : optional command string (typically "sleep 2")
573     echo "$bash_header" > $1
574     echo -e "#\n# $1 generated on $(date)\n#" >> $1
575     if [ $leave_tmp_files -eq 0 ] ; then
576         if [ -n "$multi_host" ] ; then
577             echo "trap \"rm -f $1 $2 $tmp_vpp_exec_file; sudo $dpdk_devbind -b $vpp_eth_kernel_driver $vpp_eth_pci_id; sudo ifconfig $vpp_eth_name up\" $trap_signals" >> $1
578         else
579             echo "trap \"rm -f $1 $2 $tmp_vpp_exec_file\" $trap_signals" >> $1
580         fi
581     fi
582     if [ -n "$VCL_CONFIG" ] ; then
583         echo "export VCL_CONFIG=${vcl_config_dir}${vcl_config}" >> $1
584     fi
585     if [ -n "$VCL_API_PREFIX" ] ; then
586         echo "export VCL_API_PREFIX=$VCL_API_PREFIX" >> $1
587     fi
588     if [ -n "$VCL_DEBUG" ] ; then
589         echo "export VCL_DEBUG=$VCL_DEBUG" >> $1
590     fi
591     if [ -n "$LDP_DEBUG" ] ; then
592         echo "export LDP_DEBUG=$LDP_DEBUG" >> $1
593     fi
594     if [ -n "$VCOM_APP_NAME" ] ; then
595         echo "export VCOM_APP_NAME=$VCOM_APP_NAME" >> $1
596     fi
597     if [ -n "$VCOM_SID_BIT" ] ; then
598         echo "export VCOM_SID_BIT=$VCOM_SID_BIT" >> $1
599     fi
600     if [ -n "$namespace_id" ] ; then
601         echo "export VCL_APP_NAMESPACE_ID=\"$namespace_id\"" >> $1
602         echo "export VCL_APP_NAMESPACE_SECRET=\"$namespace_secret\"" >> $1
603     fi
604     if [ -n "$VCL_APP_SCOPE_LOCAL" ] || [ -z "$multi_host" ] &&
605            [ -z "$full_thru_host_stack_vpp_cfg" ] ; then
606         echo "export VCL_APP_SCOPE_LOCAL=true" >> $1
607     fi
608     if [ -n "$VCL_APP_SCOPE_GLOBAL" ] || [ -n "$multi_host" ] ||
609            [ -n "$full_thru_host_stack_vpp_cfg" ] ; then
610         echo "export VCL_APP_SCOPE_GLOBAL=true" >> $1
611     fi
612     if [ -n "$VCL_APP_PROXY_TRANSPORT_TCP" ] ; then
613         echo "export VCL_APP_PROXY_TRANSPORT_TCP=true" >> $1
614     fi
615     if [ -n "$VCL_APP_PROXY_TRANSPORT_UDP" ] ; then
616         echo "export VCL_APP_PROXY_TRANSPORT_UDP=true" >> $1
617     fi
618     if [ "$pre_cmd" = "$gdb_in_emacs " ] ; then
619         if [ -n "$multi_host" ] && [[ $3 =~ "VPP".* ]] ; then
620             cat <<EOF >> $1
621 $gdb_in_emacs() {
622     sudo emacs --eval "(gdb \"gdb -x $2 -i=mi --args \$*\")" --eval "(setq frame-title-format \"$3\")"
623 }
624 EOF
625         else
626             cat <<EOF >> $1
627 $gdb_in_emacs() {
628     emacs --eval "(gdb \"gdb -x $2 -i=mi --args \$*\")" --eval "(setq frame-title-format \"$3\")"
629 }
630 EOF
631         fi
632     fi
633     if [ -n "$4" ] ; then
634         echo "$4" >> $1
635     fi
636 }
637
638 write_script_footer() {
639     # arguments
640     #   $1 : command script file
641     #   $2 : perf flag indicating to run bash before exit
642     local -i perf=$2
643     if [ $bash_after_exit -eq 1 ] || [ $perf -eq 1 ] ; then
644         echo "bash" >> $1
645     fi
646 }
647
648 write_gdb_cmdfile() {
649     # arguments
650     #   $1 : gdb command file
651     #   $2 : User specified gdb cmdfile
652     #   $3 : emacs flag
653     #   $4 : optional LD_PRELOAD library pathname.
654     local -i emacs=$3
655     
656     echo "# $1 generated on $(date)" > $1
657     echo "#" >> $1
658     echo "set confirm off" >> $1
659     if [ -n "$4" ] ; then
660         echo "set exec-wrapper env LD_PRELOAD=$4" >> $1
661         # echo "start" >> $1
662     fi
663
664     if [ ! -f $2 ] ; then
665         echo -n "# " >> $1
666     fi
667     echo "source $2" >> $1
668     if [ $emacs -eq 0 ] ; then
669         echo "run" >> $1
670     fi
671 }
672
673 native_kernel() {
674     banner="Running NATIVE-KERNEL socket test"
675     if [ -z "$multi_host" ] || [ "$multi_host" = "server" ] ; then
676         title1="SERVER$title_dbg (Native-Kernel Socket Test)"
677         tmp_gdb_cmdfile=$tmp_gdb_cmdfile_server
678         gdb_cmdfile=$VPPCOM_SERVER_GDB_CMDFILE
679         set_pre_cmd $emacs_server $gdb_server
680         write_script_header $cmd1_file $tmp_gdb_cmdfile "$title1"
681         echo "${pre_cmd}${app_dir}${srvr_app}" >> $cmd1_file
682         write_script_footer $cmd1_file $perf_server
683         chmod +x $cmd1_file
684     fi
685     
686     if [ -z "$multi_host" ] || [ "$multi_host" = "client" ] ; then
687         title2="CLIENT$title_dbg (Native-Kernel Socket Test)"
688         tmp_gdb_cmdfile=$tmp_gdb_cmdfile_client
689         gdb_cmdfile=$VPPCOM_CLIENT_GDB_CMDFILE
690         set_pre_cmd $emacs_client $gdb_client
691         write_script_header $cmd2_file $tmp_gdb_cmdfile "$title2" "sleep 2"
692         echo "srvr_addr=\"$sock_srvr_addr\"" >> $cmd2_file
693         echo "${pre_cmd}${app_dir}${clnt_app}" >> $cmd2_file
694         write_script_footer $cmd2_file $perf_client
695         chmod +x $cmd2_file
696
697     fi
698 }
699
700 native_preload() {
701     verify_no_vpp
702     banner="Running NATIVE-PRELOAD socket test"
703     ld_preload="$VCL_LDPRELOAD_LIB_DIR/$vcl_ldpreload_lib "
704
705     title1="VPP$title_dbg (Native-Preload Socket Test)"
706     tmp_gdb_cmdfile=$tmp_gdb_cmdfile_vpp
707     gdb_cmdfile=$VPP_GDB_CMDFILE
708     set_pre_cmd $emacs_vpp $gdb_vpp
709     write_script_header $cmd1_file $tmp_gdb_cmdfile "$title1"
710     if [ -n "$multi_host" ] && [ $emacs_vpp -eq 0 ] ; then
711         echo -n "sudo " >> $cmd1_file
712     fi
713     echo "${pre_cmd}$vpp_dir$vpp_app $vpp_args " >> $cmd1_file
714     write_script_footer $cmd1_file $perf_vpp
715     chmod +x $cmd1_file
716
717     if [ -z "$multi_host" ] || [ "$multi_host" = "server" ] ; then
718         title2="SERVER$title_dbg (Native-Preload Socket Test)"
719         tmp_gdb_cmdfile=$tmp_gdb_cmdfile_server
720         gdb_cmdfile=$VPPCOM_SERVER_GDB_CMDFILE
721         set_pre_cmd $emacs_server $gdb_server $ld_preload
722         if [ -n "$full_thru_host_stack_vpp_cfg" ] ; then
723             namespace_id="$server_namespace_id"
724             namespace_secret="$server_namespace_secret"
725         fi
726         write_script_header $cmd2_file $tmp_gdb_cmdfile "$title2" "sleep 3"
727         echo "export LD_LIBRARY_PATH=\"$lib64_dir:$VCL_LDPRELOAD_LIB_DIR:$LD_LIBRARY_PATH\"" >> $cmd2_file
728         echo "${pre_cmd}${app_dir}${srvr_app}" >> $cmd2_file
729         write_script_footer $cmd2_file $perf_server
730         chmod +x $cmd2_file
731     fi
732
733     if [ -z "$multi_host" ] || [ "$multi_host" = "client" ] ; then
734         title3="CLIENT$title_dbg (Native-Preload Socket Test)"
735         tmp_gdb_cmdfile=$tmp_gdb_cmdfile_client
736         gdb_cmdfile=$VPPCOM_CLIENT_GDB_CMDFILE
737         set_pre_cmd $emacs_client $gdb_client $ld_preload
738         if [ -n "$full_thru_host_stack_vpp_cfg" ] ; then
739             namespace_id="$client_namespace_id"
740             namespace_secret="$client_namespace_secret"
741         fi
742         write_script_header $cmd3_file $tmp_gdb_cmdfile "$title3" "sleep 4"
743         echo "export LD_LIBRARY_PATH=\"$lib64_dir:$VCL_LDPRELOAD_LIB_DIR:$LD_LIBRARY_PATH\"" >> $cmd3_file
744         echo "srvr_addr=\"$sock_srvr_addr\"" >> $cmd3_file
745         echo "${pre_cmd}${app_dir}${clnt_app}" >> $cmd3_file
746         write_script_footer $cmd3_file $perf_client
747         chmod +x $cmd3_file
748     fi
749 }
750
751 native_vcl() {
752     verify_no_vpp
753     banner="Running NATIVE-VCL socket test"
754
755     title1="VPP$title_dbg (Native-VCL Socket Test)"
756     tmp_gdb_cmdfile=$tmp_gdb_cmdfile_vpp
757     gdb_cmdfile=$VPP_GDB_CMDFILE
758     set_pre_cmd $emacs_vpp $gdb_vpp
759     write_script_header $cmd1_file $tmp_gdb_cmdfile "$title1"
760     if [ -n "$multi_host" ] && [ $emacs_vpp -eq 0 ] ; then
761         echo -n "sudo " >> $cmd1_file
762     fi
763     echo "${pre_cmd}$vpp_dir$vpp_app $vpp_args " >> $cmd1_file
764     write_script_footer $cmd1_file $perf_vpp
765     chmod +x $cmd1_file
766
767     if [ -z "$multi_host" ] || [ "$multi_host" = "server" ] ; then
768         title2="SERVER$title_dbg (Native-VCL Socket Test)"
769         tmp_gdb_cmdfile=$tmp_gdb_cmdfile_server
770         gdb_cmdfile=$VPPCOM_SERVER_GDB_CMDFILE
771         set_pre_cmd $emacs_server $gdb_server
772         if [ "$multi_host" = "server" ] ; then
773             delay="sleep 10"
774         else
775             delay="sleep 3"
776         fi
777         if [ -n "$full_thru_host_stack_vpp_cfg" ] ; then
778             namespace_id="$server_namespace_id"
779             namespace_secret="$server_namespace_secret"
780         fi
781         write_script_header $cmd2_file $tmp_gdb_cmdfile "$title2" "$delay"
782         echo "export LD_LIBRARY_PATH=\"$lib64_dir:$LD_LIBRARY_PATH\"" >> $cmd2_file
783         echo "${pre_cmd}${app_dir}${srvr_app}" >> $cmd2_file
784         write_script_footer $cmd2_file $perf_server
785         chmod +x $cmd2_file
786     fi
787
788     if [ -z "$multi_host" ] || [ "$multi_host" = "client" ] ; then
789         title3="CLIENT$title_dbg (Native-VCL Socket Test)"
790         tmp_gdb_cmdfile=$tmp_gdb_cmdfile_client
791         gdb_cmdfile=$VPPCOM_CLIENT_GDB_CMDFILE
792         set_pre_cmd $emacs_client $gdb_client
793         if [ "$multi_host" = "client" ] ; then
794             delay="sleep 10"
795         else
796             delay="sleep 4"
797         fi
798         if [ -n "$full_thru_host_stack_vpp_cfg" ] ; then
799             namespace_id="$client_namespace_id"
800             namespace_secret="$client_namespace_secret"
801         fi
802         write_script_header $cmd3_file $tmp_gdb_cmdfile "$title3" "$delay"
803         echo "export LD_LIBRARY_PATH=\"$lib64_dir:$LD_LIBRARY_PATH\"" >> $cmd3_file
804         echo "srvr_addr=\"$sock_srvr_addr\"" >> $cmd3_file
805         echo "${pre_cmd}${app_dir}${clnt_app}" >> $cmd3_file
806         write_script_footer $cmd3_file $perf_client
807         chmod +x $cmd3_file
808     fi
809 }
810
811 docker_kernel() {
812     verify_no_docker_containers
813     banner="Running DOCKER-KERNEL socket test"
814     
815     if [ -z "$multi_host" ] || [ "$multi_host" = "server" ] ; then
816         title1="SERVER$title_dbg (Docker-Native Socket Test)"
817         tmp_gdb_cmdfile=$tmp_gdb_cmdfile_server
818         gdb_cmdfile=$VPPCOM_SERVER_GDB_CMDFILE
819         set_pre_cmd $emacs_server $gdb_server
820         write_script_header $cmd1_file $tmp_gdb_cmdfile "$title1"
821         echo "docker run -it --cpuset-cpus='4-7' --cpuset-cpus='4-7' -v $vpp_dir:$docker_vpp_dir -p $sock_srvr_port:$sock_srvr_port $docker_os ${docker_app_dir}${srvr_app}" >> $cmd1_file
822         write_script_footer $cmd1_file $perf_server
823         chmod +x $cmd1_file
824     fi
825     
826     if [ -z "$multi_host" ] || [ "$multi_host" = "client" ] ; then
827         title2="CLIENT$title_dbg (Docker-Native Socket Test)"
828         tmp_gdb_cmdfile=$tmp_gdb_cmdfile_client
829         gdb_cmdfile=$VPPCOM_CLIENT_GDB_CMDFILE
830         set_pre_cmd $emacs_client $gdb_client
831         write_script_header $cmd2_file $tmp_gdb_cmdfile "$title2" "sleep 2"
832         echo "$get_docker_server_ip4addr" >> $cmd2_file
833         echo "docker run -it --cpuset-cpus='4-7' -v $vpp_dir:$docker_vpp_dir $docker_os ${docker_app_dir}${clnt_app}" >> $cmd2_file
834         write_script_footer $cmd2_file $perf_client
835         chmod +x $cmd2_file
836     fi
837 }
838
839 docker_preload() {
840     verify_no_vpp
841     verify_no_docker_containers
842     banner="Running DOCKER-PRELOAD socket test"
843     docker_ld_preload_dir="/vcl-ldpreload/"
844     ld_preload_dir="$VCL_LDPRELOAD_LIB_DIR"
845     ld_preload="$docker_ld_preload_dir$vcl_ldpreload_lib "
846     docker_ld_preload_lib="$docker_ld_preload_dir$vcl_ldpreload_lib "
847     
848     title1="VPP$title_dbg (Docker-Preload Socket Test)"
849     tmp_gdb_cmdfile=$tmp_gdb_cmdfile_vpp
850     gdb_cmdfile=$VPP_GDB_CMDFILE
851     set_pre_cmd $emacs_vpp $gdb_vpp
852     write_script_header $cmd1_file $tmp_gdb_cmdfile "$title1"
853     if [ -n "$multi_host" ] ; then
854         echo -n "sudo " >> $cmd1_file
855     fi
856     echo "${pre_cmd}$vpp_dir$vpp_app $vpp_args" >> $cmd1_file
857     write_script_footer $cmd1_file $perf_vpp
858     chmod +x $cmd1_file
859
860     if [ -z "$multi_host" ] || [ "$multi_host" = "server" ] ; then
861         title2="SERVER$title_dbg (Docker-Preload Socket Test)"
862         tmp_gdb_cmdfile=$tmp_gdb_cmdfile_server
863         gdb_cmdfile=$VPPCOM_SERVER_GDB_CMDFILE
864         set_pre_cmd $emacs_server $gdb_server $docker_ld_preload_lib
865         if [ -n "$full_thru_host_stack_vpp_cfg" ] ; then
866             namespace_id="$server_namespace_id"
867             namespace_secret="$server_namespace_secret"
868         fi
869         write_script_header $cmd2_file $tmp_gdb_cmdfile "$title2" "sleep 2"
870         echo "docker run -it -v $vpp_shm_dir:$vpp_shm_dir -v $vpp_dir:$docker_vpp_dir -v $lib64_dir:$docker_lib64_dir -v $ld_preload_dir:$docker_ld_preload_dir -v $vcl_config_dir:$docker_vcl_config_dir -p $sock_srvr_port:$sock_srvr_port -e VCL_DEBUG=$VCL_DEBUG -e VCL_CONFIG=${docker_vcl_config_dir}$vcl_config -e LD_LIBRARY_PATH=$docker_lib64_dir:$docker_ld_preload_dir ${docker_ld_preload}$docker_os ${docker_app_dir}${srvr_app}" >> $cmd2_file
871         write_script_footer $cmd2_file $perf_server
872         chmod +x $cmd2_file
873     fi
874
875     if [ -z "$multi_host" ] || [ "$multi_host" = "client" ] ; then
876         title3="CLIENT$title_dbg (Docker-Preload Socket Test)"
877         tmp_gdb_cmdfile=$tmp_gdb_cmdfile_client
878         gdb_cmdfile=$VPPCOM_CLIENT_GDB_CMDFILE
879         set_pre_cmd $emacs_client $gdb_client $docker_ld_preload_lib
880         if [ -n "$full_thru_host_stack_vpp_cfg" ] ; then
881             namespace_id="$client_namespace_id"
882             namespace_secret="$client_namespace_secret"
883         fi
884         write_script_header $cmd3_file $tmp_gdb_cmdfile "$title3" "sleep 4"
885         echo "$get_docker_server_ip4addr" >> $cmd3_file
886         echo "docker run -it --cpuset-cpus='4-7' -v $vpp_shm_dir:$vpp_shm_dir -v $vpp_dir:$docker_vpp_dir -v $lib64_dir:$docker_lib64_dir  -v $ld_preload_dir:$docker_ld_preload_dir -v $vcl_config_dir:$docker_vcl_config_dir -e VCL_DEBUG=$VCL_DEBUG -e VCL_CONFIG=${docker_vcl_config_dir}$vcl_config -e LD_LIBRARY_PATH=$docker_lib64_dir ${docker_ld_preload}$docker_os ${docker_app_dir}${clnt_app}" >> $cmd3_file
887         write_script_footer $cmd3_file $perf_client
888         chmod +x $cmd3_file
889     fi
890 }
891
892 docker_vcl() {
893     verify_no_vpp
894     verify_no_docker_containers
895     banner="Running DOCKER-VCL socket test"
896     
897     title1="VPP$title_dbg (Docker-VCL Socket Test)"
898     tmp_gdb_cmdfile=$tmp_gdb_cmdfile_vpp
899     gdb_cmdfile=$VPP_GDB_CMDFILE
900     set_pre_cmd $emacs_vpp $gdb_vpp
901     write_script_header $cmd1_file $tmp_gdb_cmdfile "$title1"
902     if [ -n "$multi_host" ] ; then
903         echo -n "sudo " >> $cmd1_file
904     fi
905     echo "${pre_cmd}$vpp_dir$vpp_app $vpp_args" >> $cmd1_file
906     write_script_footer $cmd1_file $perf_vpp
907     chmod +x $cmd1_file
908
909     if [ -z "$multi_host" ] || [ "$multi_host" = "server" ] ; then
910         title2="SERVER$title_dbg (Docker-VCL Socket Test)"
911         tmp_gdb_cmdfile=$tmp_gdb_cmdfile_server
912         gdb_cmdfile=$VPPCOM_SERVER_GDB_CMDFILE
913         set_pre_cmd $emacs_server $gdb_server
914         if [ -n "$full_thru_host_stack_vpp_cfg" ] ; then
915             namespace_id="$server_namespace_id"
916             namespace_secret="$server_namespace_secret"
917         fi
918         write_script_header $cmd2_file $tmp_gdb_cmdfile "$title2" "sleep 2"
919         echo "docker run -it --cpuset-cpus='4-7' -v $vpp_shm_dir:$vpp_shm_dir -v $vpp_dir:$docker_vpp_dir -v $lib64_dir:$docker_lib64_dir -v $vcl_config_dir:$docker_vcl_config_dir -p $sock_srvr_port:$sock_srvr_port -e VCL_CONFIG=${docker_vcl_config_dir}/$vcl_config -e LD_LIBRARY_PATH=$docker_lib64_dir $docker_os ${docker_app_dir}${srvr_app}" >> $cmd2_file
920         write_script_footer $cmd2_file $perf_server
921         chmod +x $cmd2_file
922     fi
923
924     if [ -z "$multi_host" ] || [ "$multi_host" = "client" ] ; then
925         title3="CLIENT$title_dbg (Docker-VCL Socket Test)"
926         tmp_gdb_cmdfile=$tmp_gdb_cmdfile_client
927         gdb_cmdfile=$VPPCOM_CLIENT_GDB_CMDFILE
928         set_pre_cmd $emacs_client $gdb_client
929         if [ -n "$full_thru_host_stack_vpp_cfg" ] ; then
930             namespace_id="$client_namespace_id"
931             namespace_secret="$client_namespace_secret"
932         fi
933         write_script_header $cmd3_file $tmp_gdb_cmdfile "$title3" "sleep 3"
934         echo "$get_docker_server_ip4addr" >> $cmd3_file
935         echo "docker run -it --cpuset-cpus='4-7' -v $vpp_shm_dir:$vpp_shm_dir -v $vpp_dir:$docker_vpp_dir -v $lib64_dir:$docker_lib64_dir -v $vcl_config_dir:$docker_vcl_config_dir -e VCL_CONFIG=${docker_vcl_config_dir}/$vcl_config -e LD_LIBRARY_PATH=$docker_lib64_dir $docker_os ${docker_app_dir}${clnt_app}" >> $cmd3_file
936         write_script_footer $cmd3_file $perf_client
937         chmod +x $cmd3_file
938     fi
939 }
940
941 if [[ $run_test ]] ; then
942     eval $run_test
943 else
944     echo "ERROR: Please specify a test to run!" >&2
945     usage;
946 fi
947
948 if (( $(which xfce4-terminal | wc -l) > 0 )) ; then
949     xterm_cmd="xfce4-terminal --geometry $xterm_geom"
950     if [[ $use_tabs ]] ; then
951         declare -a tab_cmd_files
952         declare -a tab_titles
953         declare -i i=0
954
955         if [ -x "$cmd1_file" ] ; then
956             tab_cmd_files[$i]="$cmd1_file"
957             tab_titles[$i]="$title1"
958             (( i++ ))
959         fi
960         if [ -x "$cmd2_file" ] ; then
961             tab_cmd_files[$i]="$cmd2_file"
962             tab_titles[$i]="$title2"
963             (( i++ ))
964         fi
965         if [ -x "$cmd3_file" ] ; then
966             tab_cmd_files[$i]="$cmd3_file"
967             tab_titles[$i]="$title3"
968         fi
969
970         if [ -n "${tab_cmd_files[2]}" ] ; then
971             $xterm_cmd  --title "${tab_titles[0]}" --command "${tab_cmd_files[0]}" --tab --title "${tab_titles[1]}" --command "${tab_cmd_files[1]}" --tab --title "${tab_titles[2]}" --command "${tab_cmd_files[2]}"
972         elif [ -n "${tab_cmd_files[1]}" ] ; then
973             $xterm_cmd --title "${tab_titles[0]}" --command "${tab_cmd_files[0]}" --tab --title "${tab_titles[1]}" --command "${tab_cmd_files[1]}"
974
975         else
976             $xterm_cmd --title "${tab_titles[0]}" --command "${tab_cmd_files[0]}"
977         fi
978         
979     else
980         if [ -x "$cmd1_file" ] ; then
981             ($xterm_cmd --title "$title1" --command "$cmd1_file" &)
982         fi
983         if [ -x "$cmd2_file" ] ; then
984             ($xterm_cmd --title "$title2" --command "$cmd2_file" &)
985         fi
986         if [ -x "$cmd3_file" ] ; then
987             ($xterm_cmd --title "$title3" --command "$cmd3_file" &)
988         fi
989     fi
990         
991 else
992     if [[ $use_tabs ]] ; then
993         echo "Sorry, plain ol' xterm doesn't support tabs."
994     fi
995     xterm_cmd="xterm -fs 10 -geometry $xterm_geom"
996     if [ -x "$cmd1_file" ] ; then
997         ($xterm_cmd -title "$title1" -e "$cmd1_file" &)
998     fi
999     if [ -x "$cmd2_file" ] ; then
1000         ($xterm_cmd -title "$title2" -e "$cmd2_file" &)
1001     fi
1002     if [ -x "$cmd3_file" ] ; then
1003         ($xterm_cmd -title "$title3" -e "$cmd3_file" &)
1004     fi
1005 fi
1006
1007 sleep 1