test/scripts/core_pinning.sh

   1 #!/bin/bash
   2 #
   3 # core_pinning_auto.sh -- script to test vpp (debug-build) core-pinning
   4 #                      -- in bare-metal, containers (docker, lxc)
   5 #
   6 DOCKER_CONTAINER_NAME="vpp_core_pinning"
   7 VPP_SOCK_PATH=/run/vpp
   8 CONTAINER_CPU_RANGE="4-7"
   9 TEST_SUCCESS=0
  10 TEST_FAIL=0
  11 if [ ! $WS_ROOT ]
  12 then
  13         if [ ! -d "../../../vpp" ]; then
  14                 echo "VPP workspace path invalid"
  15                 echo "Please execute script from vpp/test/scripts folder.."
  16                 exit 1
  17         fi
  18         WS_ROOT="$(dirname $(readlink -e "../../../vpp"))/$(basename "../../../vpp")"
  19 fi
  20 # Get available CPU count on host machine
  21 host_cpulist=$(cat /sys/devices/system/cpu/online)
  22 startcpu="${host_cpulist%-*}"
  23 endcpu="${host_cpulist#*\-}"
  24 cpucount="$(($endcpu - $startcpu + 1))"
  25 if [ $cpucount -lt 8 ]
  26 then
  27         echo "Current host machine has $cpucount CPUs"
  28     echo "A minimum of 8 CPUs is required to run testcases, exiting.."
  29     exit 1
  30 fi
  31 # Check that container 'vpp_core_pinning' does not already exist
  32 count=$(docker ps -a | grep -c "$DOCKER_CONTAINER_NAME")
  33 if [ $count -ne 0 ]
  34 then
  35         echo "Error: docker container $DOCKER_CONTAINER_NAME already exists"
  36         echo "Remove it using 'docker stop/docker rm', then re-run test"
  37         exit 1
  38 fi
  39 # Check that there is no vpp instance currently running on the machine
  40 count=$(pgrep vpp | wc -l)
  41 if [ $count -ne 0 ]
  42 then
  43         echo "Error: a vpp instance is currently running on this machine"
  44         echo "Please stop the running instance, then re-run test"
  45         exit 1
  46 fi
  47 mkdir -p $VPP_SOCK_PATH
  48
  49 # Function to parse main core
  50 parse_maincore () {
  51     main_core_args=$1
  52     main_core_parsed=$main_core_args
  53     if [ $main_core_args = "auto" ];
  54     then
  55         main_core_parsed="0"
  56         if [ -n "$SKIP_CORE" ]
  57         then
  58             main_core_parsed=$(($main_core_parsed + $SKIP_CORE))
  59         fi
  60         if [ -n "$CONTAINER_RESTRAIN_CPUSET" ]
  61         then
  62             main_core_parsed=(${container_cpus[ $main_core_parsed ]})
  63         fi
  64     fi
  65     echo $main_core_parsed
  66 }
  67
  68 # Function to parse n workers range to an array
  69 # e.g. "4" is parsed to ('0','1','2','3')
  70 parse_workers_n () {
  71     workers_n_args=$1
  72     workers_n_parsed=()
  73     main_core_increment="0"
  74     skip_core_increment="0"
  75     if [ -n "$SKIP_CORE" ]
  76         then
  77             skip_core_increment=$(($SKIP_CORE))
  78         fi
  79
  80     for ((i=0;i<$workers_n_args;i++)); do
  81
  82     if [ -n "$CONTAINER_RESTRAIN_CPUSET" ]
  83         then
  84         if [ $(( ${container_cpus[ $(($i + $skip_core_increment)) ]})) -eq $(("$parsed_main_core")) ]
  85         then
  86             main_core_increment=$(($main_core_increment + 1))
  87         fi
  88             workers_n_parsed+=" ${container_cpus[ $(($i + $main_core_increment + $skip_core_increment)) ]}"
  89         else
  90         if [ $(( $skip_core_increment + $i)) -eq $(("$parsed_main_core")) ]
  91         then
  92             main_core_increment=$(($main_core_increment + 1))
  93         fi
  94             workers_n_parsed+=" $(($i + $main_core_increment + $skip_core_increment))"
  95         fi
  96     done
  97     echo $workers_n_parsed
  98 }
  99
 100 # Function to parse corelist range to an array
 101 # e.g. "0,3-5,7" is parsed to ('0','3','4','5','7')
 102 parse_corelist () {
 103     corelist_args=$1
 104     corelist_args=$(echo $corelist_args | grep -Po '[0-9]+-[0-9]+|[0-9]+')
 105     corelist_parsed=()
 106     for corelist_elt in ${corelist_args[@]};do
 107         if [ $(echo $corelist_elt | grep -Po '[0-9]+-[0-9]+') ]
 108         then
 109             startcpu="${corelist_elt%-*}"
 110             endcpu="${corelist_elt#*\-}"
 111             cpucount="$(($endcpu - $startcpu))"
 112             for ((i=0;i<=$cpucount;i++)); do
 113               corelist_parsed+=" $(($i+$startcpu))"
 114             done
 115         elif [ $(echo $corelist_elt | grep -Po '[0-9]+') ]
 116         then
 117             corelist_parsed+=" ${corelist_elt}"
 118         fi
 119     done
 120     echo $corelist_parsed
 121 }
 122 # Test VPP core pinning configuration
 123 test_pinning_conf () {
 124     VPP_CPU_EXTRA_OPTIONS=""
 125         if [ -n "$CORELIST_WORKERS" ];
 126         then
 127                 VPP_CPU_EXTRA_OPTIONS=" corelist-workers ${CORELIST_WORKERS}"
 128         fi
 129     if [ -n "$WORKERS_AUTO" ];
 130         then
 131                 VPP_CPU_EXTRA_OPTIONS=" workers ${WORKERS_AUTO}"
 132         fi
 133     if [ -n "$SKIP_CORE" ];
 134         then
 135                 VPP_CPU_EXTRA_OPTIONS="${VPP_CPU_EXTRA_OPTIONS} skip-cores ${SKIP_CORE}"
 136         fi
 137         echo "TEST - conf 'cpu {main-core ${MAIN_CORE} ${VPP_CPU_EXTRA_OPTIONS}}'"
 138         if [ -z "$CONTAINER_RESTRAIN_CPUSET" ];
 139         then
 140                 VPP_CONTAINER_CPUSET=""
 141                 echo "(Running vpp in container with full host cpuset $host_cpulist)"
 142         else
 143                 VPP_CONTAINER_CPUSET="--cpuset-cpus $CONTAINER_CPU_RANGE"
 144                 echo "(Running vpp in container with limited cpuset $CONTAINER_CPU_RANGE)"
 145         fi
 146         (docker run -d ${VPP_CONTAINER_CPUSET} --name="$DOCKER_CONTAINER_NAME" \
 147         -e LD_LIBRARY_PATH="/vpp/build-root/build-vpp_debug-native/vpp/lib/x86_64-linux-gnu/" -v $VPP_SOCK_PATH:$VPP_SOCK_PATH \
 148         -v $WS_ROOT:/vpp  ubuntu:22.04 sh -c "/vpp/build-root/build-vpp_debug-native/vpp/bin/vpp unix {interactive \
 149         nodaemon cli-listen $VPP_SOCK_PATH/cli.sock} cpu {main-core ${MAIN_CORE} ${VPP_CPU_EXTRA_OPTIONS} } plugins \
 150         { plugin dpdk_plugin.so {disable } }" > /dev/null )
 151         sleep 3 # wait for VPP to initialize socket
 152         # Change access permissions on vpp cli socket
 153         # docker exec -it "$DOCKER_CONTAINER_NAME" /bin/bash -c "chmod 777  $VPP_SOCK_PATH/cli.sock"  > /dev/null
 154         # check if vppctl can connect to vpp container instance
 155         $WS_ROOT/build-root/build-vpp_debug-native/vpp/bin/vppctl -s $VPP_SOCK_PATH/cli.sock show threads  1> /dev/null
 156         # get CPUs vpp instance in container is running on
 157         taskset_vpp_cpus=($( taskset --all-tasks -pc $(pgrep vpp) | grep -e ".$" -o))
 158         rc=$?
 159         # parse list of user requested CPUs for vpp
 160         requested_cpus=()
 161     parsed_main_core=$(parse_maincore ${MAIN_CORE})
 162         requested_cpus+=($parsed_main_core)
 163     if [ -n "$CORELIST_WORKERS" ];
 164     then
 165             requested_cpus+=($(parse_corelist ${CORELIST_WORKERS}))
 166     fi
 167     if [ -n "$WORKERS_AUTO" ];
 168     then
 169             requested_cpus+=($(parse_workers_n ${WORKERS_AUTO}))
 170     fi
 171
 172         # parse list of expected CPUs used by vpp
 173         expected_cpu_mapping=()
 174     expected_cpu_mapping=("${requested_cpus[@]}")
 175         echo "CPUs requested by user:      [${requested_cpus[@]}]"
 176         echo "--------------------"
 177         echo "Expected CPU Mapping:        [${expected_cpu_mapping[@]}]"
 178         echo "VPP pinning (taskset):       [${taskset_vpp_cpus[@]}]"
 179         #check if expected CPU mapping matches CPUs vpp instance in container is running on
 180         failure_cond=""
 181         for index in ${!taskset_vpp_cpus[@]}; do
 182                 if [ ${taskset_vpp_cpus[$index]} -ne  ${expected_cpu_mapping[ $index ]} ]
 183                 then
 184                         failure_cond="t"
 185                 fi
 186         done
 187         if [ $rc -eq 0 ] && [ -z "$failure_cond" ]
 188         then
 189                 echo "Test Successful"
 190                 TEST_SUCCESS=$(($TEST_SUCCESS+1))
 191         else
 192                 echo "Test Failed"
 193                 TEST_FAIL=$(($TEST_FAIL+1))
 194         fi
 195         echo "=============================================="
 196         echo " "
 197         # Stop & destroy container instance
 198         docker stop $DOCKER_CONTAINER_NAME  &> /dev/null
 199         docker rm -f $DOCKER_CONTAINER_NAME &> /dev/null
 200 }
 201 test_invalid_conf () {
 202         if [ -n "$CORELIST_WORKERS" ];
 203         then
 204                 VPP_CPU_EXTRA_OPTIONS=" corelist-workers ${CORELIST_WORKERS}"
 205         fi
 206     if [ -n "$WORKERS_AUTO" ];
 207         then
 208                 VPP_CPU_EXTRA_OPTIONS=" workers ${WORKERS_AUTO}"
 209         fi
 210     if [ -n "$SKIP_CORE" ];
 211         then
 212                 VPP_CPU_EXTRA_OPTIONS="${VPP_CPU_EXTRA_OPTIONS} skip-cores ${SKIP_CORE}"
 213         fi
 214         echo "TEST - conf 'cpu {main-core ${MAIN_CORE} ${VPP_CPU_EXTRA_OPTIONS}}'"
 215         if [ -z "$CONTAINER_RESTRAIN_CPUSET" ];
 216         then
 217                 VPP_CONTAINER_CPUSET=""
 218                 echo "(Running vpp in container with full host cpuset $host_cpulist)"
 219         else
 220                 VPP_CONTAINER_CPUSET="--cpuset-cpus $CONTAINER_CPU_RANGE"
 221                 echo "(Running vpp in container with limited cpuset $CONTAINER_CPU_RANGE)"
 222         fi
 223         (docker run -d --cpuset-cpus $CONTAINER_CPU_RANGE --name="$DOCKER_CONTAINER_NAME" \
 224         -e LD_LIBRARY_PATH="/vpp/build-root/build-vpp_debug-native/vpp/lib/x86_64-linux-gnu/" -v $VPP_SOCK_PATH:$VPP_SOCK_PATH \
 225         -v $WS_ROOT:/vpp  ubuntu:22.04 sh -c "/vpp/build-root/build-vpp_debug-native/vpp/bin/vpp unix {interactive \
 226         nodaemon cli-listen $VPP_SOCK_PATH/cli.sock} cpu {main-core ${MAIN_CORE} ${VPP_CPU_EXTRA_OPTIONS}} plugins \
 227         { plugin dpdk_plugin.so {disable } }"  > /dev/null)
 228         sleep 3 # wait for vpp to initialize socket
 229         # check if vpp launched with invalid configuration
 230         taskset --all-tasks -pc $(pgrep vpp) &> /dev/null
 231         rc=$?
 232         if [ $rc -eq 1 ]
 233         then
 234         echo " "
 235                 echo "OK... VPP did not launch with invalid configuration"
 236                 TEST_SUCCESS=$(($TEST_SUCCESS+1))
 237         else
 238         echo " "
 239                 echo "Failure... VPP launched with wrong configuration"
 240                 TEST_FAIL=$(($TEST_FAIL+1))
 241         fi
 242         echo "=============================================="
 243         echo " "
 244         # Stop & destroy container instance
 245         docker stop $DOCKER_CONTAINER_NAME  &> /dev/null
 246         docker rm -f $DOCKER_CONTAINER_NAME &> /dev/null
 247 }
 248 run_tests () {
 249         container_cpus=($(parse_corelist ${CONTAINER_CPU_RANGE}))
 250         echo "TESTING VALID CORE PINNING CONFIGURATIONS"
 251         echo " "
 252     WORKERS_AUTO=""
 253     SKIP_CORE=""
 254         CONTAINER_RESTRAIN_CPUSET=""
 255         CORELIST_WORKERS="1-3"
 256         MAIN_CORE="0"
 257         test_pinning_conf
 258     WORKERS_AUTO=""
 259     SKIP_CORE=""
 260         CONTAINER_RESTRAIN_CPUSET=""
 261         CORELIST_WORKERS="0,2-3"
 262         MAIN_CORE="1"
 263         test_pinning_conf
 264     WORKERS_AUTO=""
 265     SKIP_CORE=""
 266         CONTAINER_RESTRAIN_CPUSET=""
 267         CORELIST_WORKERS="0-2"
 268         MAIN_CORE="3"
 269         test_pinning_conf
 270     WORKERS_AUTO="2"
 271     SKIP_CORE=""
 272         CONTAINER_RESTRAIN_CPUSET=""
 273     CORELIST_WORKERS=""
 274         MAIN_CORE="auto"
 275         test_pinning_conf
 276     WORKERS_AUTO="3"
 277     SKIP_CORE=""
 278         CONTAINER_RESTRAIN_CPUSET="t"
 279     CORELIST_WORKERS=""
 280         MAIN_CORE="auto"
 281         test_pinning_conf
 282     WORKERS_AUTO="2"
 283     SKIP_CORE="1"
 284         CONTAINER_RESTRAIN_CPUSET="t"
 285     CORELIST_WORKERS=""
 286         MAIN_CORE="auto"
 287         test_pinning_conf
 288     WORKERS_AUTO="2"
 289     SKIP_CORE=""
 290         CONTAINER_RESTRAIN_CPUSET="t"
 291     CORELIST_WORKERS=""
 292         MAIN_CORE="5"
 293         test_pinning_conf
 294         echo "TESTING NON-VALID CORE PINNING CONFIGURATIONS"
 295         echo " "
 296     WORKERS_AUTO=""
 297     SKIP_CORE=""
 298         CONTAINER_RESTRAIN_CPUSET="t"
 299         CORELIST_WORKERS="1-3"
 300         MAIN_CORE="0"
 301         test_invalid_conf
 302         WORKERS_AUTO="3"
 303     SKIP_CORE="1"
 304         CONTAINER_RESTRAIN_CPUSET="t"
 305         CORELIST_WORKERS=""
 306         MAIN_CORE="auto"
 307         test_invalid_conf
 308         WORKERS_AUTO="5"
 309     SKIP_CORE=""
 310         CONTAINER_RESTRAIN_CPUSET="t"
 311         CORELIST_WORKERS=""
 312         MAIN_CORE="auto"
 313         test_invalid_conf
 314         WORKERS_AUTO=""
 315     SKIP_CORE="4"
 316         CONTAINER_RESTRAIN_CPUSET="t"
 317         CORELIST_WORKERS=""
 318         MAIN_CORE="auto"
 319         test_invalid_conf
 320         echo " "
 321         echo "========================"
 322         echo "RESULTS:"
 323         echo "SUCCESS: $TEST_SUCCESS"
 324         echo "FAILURE: $TEST_FAIL"
 325         echo "========================"
 326         echo " "
 327 }
 328 run_tests