docker/scripts/lib_csit.sh

   1 # lib_csit.sh - Docker build script CSIT library.
   2 #               For import only.
   3
   4 # Copyright (c) 2020 Cisco and/or its affiliates.
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at:
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 # Don't import more than once.
  18 if [ -n "$(alias lib_csit_imported 2> /dev/null)" ] ; then
  19     return 0
  20 fi
  21 alias lib_csit_imported=true
  22
  23 export CIMAN_DOCKER_SCRIPTS="${CIMAN_DOCKER_SCRIPTS:-$(dirname $BASH_SOURCE)}"
  24 . $CIMAN_DOCKER_SCRIPTS/lib_common.sh
  25 . $CIMAN_DOCKER_SCRIPTS/lib_apt.sh
  26 . $CIMAN_DOCKER_SCRIPTS/lib_yum.sh
  27 . $CIMAN_DOCKER_SCRIPTS/lib_dnf.sh
  28
  29 csit_checkout_branch_for_vpp() {
  30     local vpp_branch=$1
  31     local csit_dir="$DOCKER_CSIT_DIR"
  32     local csit_bash_function_dir="$csit_dir/resources/libraries/bash/function"
  33
  34     # import checkout_csit_for_vpp() if not defined
  35     set +e && [ -z "$(declare -f checkout_csit_for_vpp)" ] \
  36         && source $csit_bash_function_dir/branch.sh
  37     CSIT_DIR=$(csit_dir) checkout_csit_for_vpp $vpp_branch
  38
  39     csit_branch="$(git branch | grep -e '^*' | mawk '{print $2}')"
  40 }
  41
  42 csit_install_packages() {
  43     local branch="$1"
  44     local branchname="$(echo $branch | sed -e 's,/,_,')"
  45     local csit_dir="$DOCKER_CSIT_DIR"
  46     local csit_ansible_dir="$csit_dir/resources/tools/testbed-setup/ansible"
  47     local bld_log="$DOCKER_BUILD_LOG_DIR/$FDIOTOOLS_IMAGENAME"
  48     bld_log="${bld_log}-$branchname-csit_install_packages-bld.log"
  49
  50     git clean -qfdx
  51
  52     # Install PyYAML required by dbld_csit_find_ansible_packages.py
  53     #
  54     # Note: Conditional install due to Bug 1696324 -
  55     #       Update to python3.6 breaks PyYAML dependencies
  56     # Status:       CLOSED CANTFIX
  57     # https://bugzilla.redhat.com/show_bug.cgi?id=1696324
  58     if [ "$OS_NAME" = "centos-8" ] ; then
  59         dnf_install_packages python3-pyyaml
  60     else
  61         python3 -m pip install pyyaml
  62     fi
  63
  64     # NOTE: the CSIT baseline os is implicitly pinned to ubuntu 18.04
  65     #       so only gather packages from ansible for that OS.
  66     if [ "$OS_NAME" = "ubuntu-18.04" ] ; then
  67         # Not in double quotes to let bash remove newline characters
  68         local exclude_roles="-e calibration -e kernel -e mellanox -e nomad"
  69         [ "$OS_ARCH" = "aarch64" ] && exclude_roles="$exclude_roles -e iperf"
  70         local yaml_files=$(grep -r packages_by $csit_ansible_dir | cut -d: -f1 | sort -u | grep -v $exclude_roles)
  71         packages=$(dbld_csit_find_ansible_packages.py --$OS_ID --$OS_ARCH $yaml_files)
  72
  73         if [ -n "$packages" ] ; then
  74             case "$OS_NAME" in
  75                 ubuntu*)
  76                     apt_install_packages $packages
  77                     ;;
  78                 debian*)
  79                     apt_install_packages $packages
  80                     ;;
  81                 centos-7)
  82                     yum_install_packages $packages
  83                     ;;
  84                 centos-8)
  85                     dnf_install_packages $packages
  86                     ;;
  87                 *)
  88                     echo "Unsupported OS ($OS_ID): CSIT packages NOT INSTALLED!"
  89                     ;;
  90             esac
  91         fi
  92     fi
  93 }
  94
  95 csit_pip_cache() {
  96     local branch="$1"
  97     local VENV_OPTS=""
  98     # ensure PS1 is defined (used by virtualenv activate script)
  99     PS1=${PS1:-"#"}
 100     local csit_dir="$DOCKER_CSIT_DIR"
 101     local csit_bash_function_dir="$csit_dir/resources/libraries/bash/function"
 102
 103     if [ -f "$csit_dir/VPP_REPO_URL" ] \
 104            && [ -f "$csit_dir/requirements.txt" ]; then
 105
 106         local branchname="$(echo $branch | sed -e 's,/,_,')"
 107         local bld_log="$DOCKER_BUILD_LOG_DIR"
 108         bld_log="${bld_log}/$FDIOTOOLS_IMAGENAME-$branchname-csit_pip_cache-bld.log"
 109
 110         export PYTHONPATH=$csit_dir
 111         git clean -qfdx
 112
 113         description="Install CSIT python packages from $branch branch"
 114         echo_log "    Starting  $description..."
 115         [ -n "$(declare -f deactivate)" ] && deactivate
 116         local PIP=pip
 117         local setup_framework=$csit_dir/resources/libraries/python/SetupFramework.py
 118         if [ -n "$(grep pip3 $setup_framework)" ]; then
 119             PIP=pip3
 120             VENV_OPTS="-p python3"
 121         fi
 122         rm -rf $PYTHONPATH/env
 123         virtualenv $VENV_OPTS $PYTHONPATH/env
 124         . $PYTHONPATH/env/bin/activate
 125         if [ "$OS_ARCH" = "aarch64" ] ; then
 126             local numpy_ver="$(grep numpy $PYTHONPATH/requirements.txt)"
 127             [ -n "$numpy_ver" ] && $PIP install --upgrade $numpy_ver 2>&1 \
 128                 | tee -a $bld_log
 129         fi
 130         $PIP install --upgrade -r $PYTHONPATH/requirements.txt 2>&1 \
 131             | tee -a $bld_log
 132         $PIP install --upgrade -r $PYTHONPATH/tox-requirements.txt 2>&1 \
 133             | tee -a $bld_log
 134         if [ "$OS_ARCH" = "x86_64" ] ; then
 135             local PRESENTATION_DIR="$PYTHONPATH/resources/tools/presentation"
 136             # TODO: Remove condition when 19.08 is deprecated.
 137             if [ -n "$(grep -r python3 $PRESENTATION_DIR)" ] && [ "$PIP" = "pip3" ] ; then
 138                 $PIP install --upgrade -r $PRESENTATION_DIR/requirements.txt 2>&1 \
 139                     | tee -a $bld_log
 140             else
 141                 echo_log "Skipping 'pip install $PRESENTATION_DIR/requirements.txt' in branch $branch!"
 142             fi
 143         fi
 144
 145         deactivate
 146         rm -rf $PYTHONPATH/env
 147
 148         # Virtualenv version is pinned in common.sh in newer csit branches.
 149         # (note: xargs removes leading/trailing spaces)
 150         install_virtualenv="$(grep 'virtualenv' $csit_bash_function_dir/common.sh | grep pip | grep install | cut -d'|' -f1 | xargs)"
 151         $install_virtualenv
 152
 153         git checkout -q -- .
 154         echo_log "    Completed $description!"
 155     else
 156         echo_log "ERROR: Missing or invalid CSIT_DIR: '$csit_dir'!"
 157         return 1
 158     fi
 159 }
 160
 161 docker_build_setup_csit() {
 162     if [ ! -d "$DOCKER_CSIT_DIR" ] ; then
 163         echo_log "Cloning CSIT into $DOCKER_CSIT_DIR..."
 164         git clone -q https://gerrit.fd.io/r/csit $DOCKER_CSIT_DIR
 165     fi
 166     clean_git_repo $DOCKER_CSIT_DIR
 167 }