# lib_csit.sh - Docker build script CSIT library.
# For import only.
-# Copyright (c) 2021 Cisco and/or its affiliates.
+# Copyright (c) 2023 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
fi
alias lib_csit_imported=true
-export CIMAN_DOCKER_SCRIPTS="${CIMAN_DOCKER_SCRIPTS:-$(dirname $BASH_SOURCE)}"
+export CIMAN_DOCKER_SCRIPTS="${CIMAN_DOCKER_SCRIPTS:-$(dirname ${BASH_SOURCE[0]})}"
. "$CIMAN_DOCKER_SCRIPTS/lib_common.sh"
. "$CIMAN_DOCKER_SCRIPTS/lib_apt.sh"
-. "$CIMAN_DOCKER_SCRIPTS/lib_yum.sh"
-. "$CIMAN_DOCKER_SCRIPTS/lib_dnf.sh"
CSIT_SUPPORTED_EXECUTOR_CLASSES="builder csit_dut"
csit_supported_executor_class() {
csit_supported_os() {
case "$1" in
- # TODO: Remove ubuntu-18.04 once CSIT has completed transition
- # to ubuntu-20.04
- ubuntu-18.04) return 0 ;;
- ubuntu-20.04) return 0 ;;
+ ubuntu-22.04) return 0 ;;
*) ;;
esac
return 1
&& source "$csit_bash_function_dir/branch.sh"
CSIT_DIR="$csit_dir" checkout_csit_for_vpp "$vpp_branch"
+ # shellcheck disable=SC2034,SC2063
csit_branch="$(git branch | grep -e '^*' | mawk '{print $2}')"
}
csit_install_packages() {
local branch="$1"
- local branchname="$(echo $branch | sed -e 's,/,_,')"
+ local branchname
+ branchname="$(echo $branch | sed -e 's,/,_,')"
local csit_dir="$DOCKER_CSIT_DIR"
local csit_ansible_dir="$csit_dir/fdio.infra.ansible"
+ if [ ! -d "$csit_ansible_dir" ] ; then
+ csit_ansible_dir="$csit_dir/resources/tools/testbed-setup/ansible"
+ fi
local bld_log="$DOCKER_BUILD_LOG_DIR/$FDIOTOOLS_IMAGENAME"
bld_log="${bld_log}-$branchname-csit_install_packages-bld.log"
git clean -qfdx
+ python3 -m pip install pyyaml
- # Install PyYAML required by dbld_csit_find_ansible_packages.py
- #
- # Note: Conditional install due to Bug 1696324 -
- # Update to python3.6 breaks PyYAML dependencies
- # Status: CLOSED CANTFIX
- # https://bugzilla.redhat.com/show_bug.cgi?id=1696324
- if [ "$OS_NAME" = "centos-8" ] ; then
- dnf_install_packages python3-pyyaml
- else
- python3 -m pip install pyyaml
- fi
-
- local exclude_roles="-e calibration -e kernel -e mellanox -e nomad -e consul"
+ local exclude_roles="-e calibration -e kernel -e mellanox -e nomad -e consul -e aws -e vpp"
[ "$OS_ARCH" = "aarch64" ] && exclude_roles="$exclude_roles -e iperf"
# Not in double quotes to let bash remove newline characters
- local yaml_files="$(grep -r packages_by $csit_ansible_dir | cut -d: -f1 | sort -u | grep -v $exclude_roles)"
+ local yaml_files
+ yaml_files="$(grep -r packages_by $csit_ansible_dir | cut -d: -f1 | sort -u | grep -v $exclude_roles)"
packages="$(dbld_csit_find_ansible_packages.py --$OS_ID --$OS_ARCH $yaml_files)"
- packages="${packages/bionic /}"
+ packages="${packages/jammy /}"
packages="${packages/focal /}"
+ packages="${packages/libmbedcrypto1/libmbedcrypto3}"
+ packages="${packages/libmbedtls10/libmbedtls12}"
+ packages="$(echo ${packages//python\-/python3\-} | tr ' ' '\n' | sort -u | xargs)"
- # TODO: Fix Ubuntu-18.04 specific package names that fail on Ubuntu-20.04
- # (remove when CSIT is updated)
- if [ "$OS_NAME" = "ubuntu-20.04" ] ; then
- packages="${packages/libmbedcrypto1/libmbedcrypto3}"
- packages="${packages/libmbedtls10/libmbedtls12}"
- packages="$(echo ${packages//python\-/python3\-} | tr ' ' '\n' | sort -u | xargs)"
- fi
if [ -n "$packages" ] ; then
case "$OS_NAME" in
ubuntu*)
- apt_install_packages $packages
+ apt_install_packages "$packages" 2>&1 | tee -a "$bld_log"
;;
debian*)
- apt_install_packages $packages
- ;;
- centos-7)
- yum_install_packages $packages
- ;;
- centos-8)
- dnf_install_packages $packages
+ apt_install_packages "$packages" 2>&1 | tee -a "$bld_log"
;;
*)
echo "Unsupported OS ($OS_ID): CSIT packages NOT INSTALLED!"
fi
}
+csit_install_hugo() {
+ local branch="$1"
+ CSIT_DIR="$DOCKER_CSIT_DIR"
+
+ if [ -f "$CSIT_DIR/VPP_REPO_URL" ] \
+ && [ -f "$CSIT_DIR/requirements.txt" ]; then
+
+ local branchname
+ # use bash variable substitution to replace '/' with '_' to convert from
+ # vpp to csit branch name nomenclature
+ branchname="${branch////_}"
+ local csit_bash_function_dir="$CSIT_DIR/resources/libraries/bash/function"
+ local bld_log="$DOCKER_BUILD_LOG_DIR"
+ bld_log="${bld_log}/$FDIOTOOLS_IMAGENAME-$branchname-csit_install_hugo-bld.log"
+
+ description="Install CSIT hugo packages from $branch branch"
+ echo_log " Starting $description..."
+ git clean -qfdx
+
+ source "$csit_bash_function_dir"/hugo.sh
+ go_install 2>&1 | tee -a "$bld_log"
+ hugo_install 2>&1 | tee -a "$bld_log"
+
+ else
+ echo_log "ERROR: Missing or invalid CSIT_DIR: '$CSIT_DIR'!"
+ return 1
+ fi
+}
+
csit_pip_cache() {
local branch="$1"
- local VENV_OPTS=""
# ensure PS1 is defined (used by virtualenv activate script)
PS1=${PS1:-"#"}
CSIT_DIR="$DOCKER_CSIT_DIR"
if [ -f "$CSIT_DIR/VPP_REPO_URL" ] \
&& [ -f "$CSIT_DIR/requirements.txt" ]; then
+ local branchname
+ # use bash variable substitution to replace '/' with '_' to convert from
+ # vpp to csit branch name nomenclature
+ branchname="${branch////_}"
local csit_bash_function_dir="$CSIT_DIR/resources/libraries/bash/function"
- local branchname="$(echo $branch | sed -e 's,/,_,')"
local bld_log="$DOCKER_BUILD_LOG_DIR"
bld_log="${bld_log}/$FDIOTOOLS_IMAGENAME-$branchname-csit_pip_cache-bld.log"
- local pip_cmd="python3 -m pip --disable-pip-version-check"
export PYTHONPATH=$CSIT_DIR
description="Install CSIT python packages from $branch branch"
- echo_log " Starting $description..."
- git clean -qfdx
+ echo_log " Starting $description..." 2>&1 | tee -a "$bld_log"
+ git clean -qfdx 2>&1 | tee -a "$bld_log"
rm -rf "$PYTHONPATH/env"
- # TODO: Update CSIT release branches to avoid build breakage
- # Fixes https://github.com/pypa/pip/issues/8260
- $pip_cmd install pip==21.0.1
- # rls2009_lts-* branches missing cherry-pick of
- # https://gerrit.fd.io/r/c/csit/+/31338
- sed -i 's/scipy==1.1.0/scipy==1.5.4/' "$PYTHONPATH/requirements.txt"
-
- # Virtualenv version is pinned in common.sh in newer csit branches.
- # (note: xargs removes leading/trailing spaces)
+ # Activate / install CSIT python virtualenv ($CSIT_DIR/requirements.txt)
local common_sh="$csit_bash_function_dir/common.sh"
- install_virtualenv="$(grep 'virtualenv' $common_sh | grep pip | grep install | cut -d'|' -f1 | xargs)"
- $install_virtualenv
- virtualenv --no-download --python=$(which python3) "$CSIT_DIR/env"
- source "$CSIT_DIR/env/bin/activate"
-
- if [ "$OS_ARCH" = "aarch64" ] ; then
- local numpy_ver="$(grep numpy $PYTHONPATH/requirements.txt)"
- [ -n "$numpy_ver" ] && $pip_cmd install $numpy_ver 2>&1 | \
- tee -a $bld_log
- fi
+ # shellcheck disable=1090
+ source "$common_sh"
+ activate_virtualenv "${CSIT_DIR}" "${CSIT_DIR}/requirements.txt" 2>&1 | tee -a "$bld_log"
- # Install csit python requirements
- $pip_cmd install -r "$CSIT_DIR/requirements.txt" 2>&1 | \
- tee -a "$bld_log"
# Install tox python requirements
- $pip_cmd install -r "$CSIT_DIR/tox-requirements.txt" 2>&1 | \
+ activate_virtualenv "${CSIT_DIR}" "${CSIT_DIR}/tox-requirements.txt" 2>&1 |\
tee -a "$bld_log"
- # Run tox which installs pylint requirments
- pushd $CSIT_DIR >& /dev/null
- tox || true
- popd >& /dev/null
# Clean up virtualenv directories
- deactivate
git checkout -q -- .
git clean -qfdx
- echo_log " Completed $description!"
+ echo_log " Completed $description!" 2>&1 | tee -a "$bld_log"
else
echo_log "ERROR: Missing or invalid CSIT_DIR: '$CSIT_DIR'!"
return 1
Code Review / ci-management.git / blobdiff