X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Flibraries%2Fbash%2Ffunction%2Fgather.sh;h=e3a6a9d1509744167e9da9fab77480a506790fb5;hp=25df7347901bbd782ee57a799b16b80fcbb02d0f;hb=6da5a6920171682bd5bf6a77517bedfef91cbd0e;hpb=cebbe0a29e2c6329c54d09a77a64e874703c7d4f

diff --git a/resources/libraries/bash/function/gather.sh b/resources/libraries/bash/function/gather.sh
index 25df734790..e3a6a9d150 100644
--- a/resources/libraries/bash/function/gather.sh
+++ b/resources/libraries/bash/function/gather.sh
@@ -1,4 +1,5 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.
+# Copyright (c) 2021 PANTHEON.tech and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -23,8 +24,6 @@ set -exuo pipefail
 
 function gather_build () {
 
-    set -exuo pipefail
-
     # Variables read:
     # - TEST_CODE - String affecting test selection, usually jenkins job name.
     # - DOWNLOAD_DIR - Path to directory pybot takes the build to test from.
@@ -34,13 +33,16 @@ function gather_build () {
     # - ${DOWNLOAD_DIR} - Files needed by tests are gathered here.
     # Functions called:
     # - die - Print to stderr and exit, defined in common.sh
-    # - gather_dpdk, gather_vpp, gather_ligato - See their definitions.
+    # - gather_os - Parse os parameter for OS/distro name.
+    # - gather_dpdk, gather_vpp - See their definitions.
     # Multiple other side effects are possible,
     # see functions called from here for their current description.
 
     # TODO: Separate DUT-from-TEST_CODE from gather-for-DUT,
     #   when the first one becomes relevant for per_patch.
 
+    set -exuo pipefail
+
     pushd "${DOWNLOAD_DIR}" || die "Pushd failed."
     case "${TEST_CODE}" in
         *"hc2vpp"*)
@@ -51,10 +53,6 @@ function gather_build () {
             DUT="vpp"
             gather_vpp || die "The function should have died on error."
             ;;
-        *"ligato"*)
-            DUT="kubernetes"
-            gather_ligato || die "The function should have died on error."
-            ;;
         *"dpdk"*)
             DUT="dpdk"
             gather_dpdk || die "The function should have died on error."
@@ -69,8 +67,6 @@ function gather_build () {
 
 function gather_dpdk () {
 
-    set -exuo pipefail
-
     # Ensure latest DPDK archive is downloaded.
     #
     # Variables read:
@@ -82,6 +78,8 @@ function gather_dpdk () {
     # Functions called:
     # - die - Print to stderr and exit, defined in common.sh
 
+    set -exuo pipefail
+
     dpdk_repo="https://fast.dpdk.org/rel"
     # Use downloaded packages with specific version
     if [[ "${TEST_CODE}" == *"daily"* ]] || \
@@ -97,9 +95,12 @@ function gather_dpdk () {
             die "Composite piped command failed."
         }
     else
-        echo "Downloading DPDK packages of specific version from repo..."
-        # TODO: Can we autodetect this based on what CSIT-stable VPP uses?
-        dpdk_stable_ver="dpdk-18.08.tar.xz"
+        echo "Downloading DPDK package of specific version from repo ..."
+        # Downloading DPDK version based on what VPP is using. Currently
+        # it is not easy way to detect from VPP version automatically.
+        dpdk_stable_ver="$(< "${CSIT_DIR}/DPDK_VPP_VER")".tar.xz || {
+            die "Failed to read DPDK VPP version!"
+        }
     fi
     # TODO: Use "wget -N" instead checking for file presence?
     if [[ ! -f "${dpdk_stable_ver}" ]]; then
@@ -110,150 +111,24 @@ function gather_dpdk () {
 }
 
 
-function gather_ligato () {
-
-    set -exuo pipefail
-
-    # Build docker image (with vpp, ligato and vpp-agent),
-    # and put it to ${DOWNLOAD_DIR}/.
-    #
-    # Access rights needed for:
-    # - "wget", "git clone", "dpdk -x", "cd" above ${CSIT_DIR}.
-    # - "sudo" without password.
-    # - With sudo:
-    #   - "dpdk -i" is allowed.
-    #   - "docker" commands have everything they needs.
-    # Variables read:
-    # - DOWNLOAD_DIR - Path to directory pybot takes the build to test from.
-    # - CSIT_DIR - Path to existing root of local CSIT git repository.
-    # Files read:
-    # - ${CSIT_DIR}/VPP_AGENT_STABLE_VER - Vpp agent version to use.
-    # Directories updated:
-    # - ${DOWNLOAD_DIR} - Docker image stored, VPP *.deb stored and deleted.
-    # - /tmp/vpp - VPP is unpacked there, not cleaned afterwards.
-    # - ${CSIT_DIR}/vpp-agent - Created, vpp-agent git repo si cloned there.
-    #   - Also, various temporary files are stored there.
-    # System consequences:
-    # - Docker package is installed.
-    # - Presumably dockerd process is started.
-    # - The ligato/dev-vpp-agent docker image is downloaded.
-    # - Results of subsequent image manipulation are probably left lingering.
-    # Other hardcoded values:
-    # - Docker .deb file name to download and install.
-    # Functions called:
-    # - die - Print to stderr and exit, defined in common_functions.sh
-    # - gather_vpp - See eponymous fragment file assumend to be sourced already.
-    # TODO: What is the best order of description items?
-
-    # TODO: Many of the following comments act as abstraction.
-    #   But the abstracted blocks are mostly one-liners (plus "|| die"),
-    #   so maybe it is not worth introducing fragments/functions for the blocks.
-    # TODO: This fragment is too long anyway, split it up.
-
-    gather_vpp || die "The function should have died on error."
-
-    mkdir -p /tmp/vpp && rm -f /tmp/vpp/* || {
-        die "Failed to create temporary directory!"
-    }
-    dpkg -x "${DOWNLOAD_DIR}/vpp_"*".deb" "/tmp/vpp" || {
-        die "Failed to extract VPP packages for kubernetes!"
-    }
-
-    ligato_repo_url="https://github.com/ligato/"
-    vpp_agent_stable_ver="$(< "${CSIT_DIR}/VPP_AGENT_STABLE_VER")" || {
-        die "Failed to read vpp-agent stable version!"
-    }
-
-    # Clone & checkout stable vpp-agent.
-    cd "${CSIT_DIR}" || die "Change directory failed!"
-    git clone -b master --single-branch \
-        "${ligato_repo_url}/vpp-agent" "vpp-agent" || {
-        die "Failed to run: git clone ${ligato_repo_url}/vpp-agent!"
-    }
-    cd "vpp-agent" || die "Change directory failed!"
-
-    # Install Docker.
-    curl -fsSL https://get.docker.com | sudo bash || {
-        die "Failed to install Docker package!"
-    }
-
-    # Pull ligato/dev_vpp_agent docker image and re-tag as local.
-    sudo docker pull "ligato/dev-vpp-agent:${vpp_agent_stable_ver}" || {
-        die "Failed to pull Docker image!"
-    }
-    params=(ligato/dev-vpp-agent:${vpp_agent_stable_ver} dev_vpp_agent:latest)
-    sudo docker tag "${params[@]}" || {
-        die "Failed to tag Docker image!"
-    }
-
-    # Start dev_vpp_agent container as daemon.
-    sudo docker run --rm -itd --name "agentcnt" "dev_vpp_agent" bash || {
-        die "Failed to run Docker image!"
-    }
-
-    # Copy latest vpp api into running container.
-    sudo docker exec agentcnt rm -rf "agentcnt:/usr/share/vpp/api" || {
-        die "Failed to remove previous API!"
-    }
-    sudo docker cp "/tmp/vpp/usr/share/vpp/api" "agentcnt:/usr/share/vpp" || {
-        die "Failed to copy files Docker image!"
-    }
-
-    # Recompile vpp-agent.
-    script_arg=". ~/.bashrc; cd /go/src/github.com/ligato/vpp-agent"
-    script_arg+=" && make generate && make install"
-    sudo docker exec -i agentcnt script -qec "${script_arg}" || {
-        die "Failed to recompile vpp-agent in Docker image!"
-    }
-    # Make sure .deb files of other version are not present.
-    rm_cmd="rm -vf /opt/vpp-agent/dev/vpp/build-root/vpp*.deb /opt/vpp/*.deb"
-    sudo docker exec agentcnt bash -c "${rm_cmd}" || {
-        die "Failed to remove VPP debian packages!"
-    }
-    for f in "${DOWNLOAD_DIR}"/*; do
-        sudo docker cp "$f" "agentcnt:/opt/vpp-agent/dev/vpp/build-root"/ || {
-            die "Failed to copy files to Docker image!"
-        }
-    done
-    # Save container state.
-    sudo docker commit "$(sudo docker ps -q)" "dev_vpp_agent:latest" || {
-        die "Failed to commit state of Docker image!"
-    }
-
-    # Build prod_vpp_agent docker image.
-    cd "docker/prod" || die "Change directory failed."
-    sudo docker build --tag "prod_vpp_agent" --no-cache "." || {
-        die "Failed to build Docker image!"
-    }
-    # Export Docker image.
-    sudo docker save "prod_vpp_agent" | gzip > "prod_vpp_agent.tar.gz" || {
-        die "Failed to save Docker image!"
-    }
-    docker_image="$(readlink -e "prod_vpp_agent.tar.gz")" || {
-        die "Failed to get Docker image path!"
-    }
-    rm -r "${DOWNLOAD_DIR}/vpp"* || die "Failed to remove VPP packages!"
-    mv "${docker_image}" "${DOWNLOAD_DIR}"/ || die "Failed to move image!"
-}
-
-
 function gather_vpp () {
 
-    set -exuo pipefail
-
     # Variables read:
     # - BASH_FUNCTION_DIR - Bash directory with functions.
     # - TEST_CODE - The test selection string from environment or argument.
     # - DOWNLOAD_DIR - Path to directory pybot takes the build to test from.
     # - CSIT_DIR - Path to existing root of local CSIT git repository.
+    # Variables set:
+    # - VPP_VERSION - VPP stable version under test.
     # Files read:
     # - ${CSIT_DIR}/DPDK_STABLE_VER - DPDK version to use
     #   by csit-vpp not-timed jobs.
-    # - ${CSIT_DIR}/VPP_STABLE_VER_UBUNTU - VPP version to use by those.
-    # - ../vpp*.deb - Relative to ${DOWNLOAD_DIR}, copied for vpp-csit jobs.
+    # - ${CSIT_DIR}/${VPP_VER_FILE} - Ubuntu VPP version to use.
+    # - ../*vpp*.deb|rpm - Relative to ${DOWNLOAD_DIR},
+    #   copied for vpp-csit jobs.
     # Directories updated:
     # - ${DOWNLOAD_DIR}, vpp-*.deb files are copied here for vpp-csit jobs.
-    # - ./ - Assumed ${DOWNLOAD_DIR}, vpp-*.deb files
+    # - ./ - Assumed ${DOWNLOAD_DIR}, *vpp*.deb|rpm files
     #   are downloaded here for csit-vpp.
     # Functions called:
     # - die - Print to stderr and exit, defined in common_functions.sh
@@ -261,35 +136,32 @@ function gather_vpp () {
     # - ${CSIT_DIR}/resources/tools/scripts/download_install_vpp_pkgs.sh
     #   - Should download and extract requested files to ./.
 
+    set -exuo pipefail
+
     case "${TEST_CODE}" in
-        # Not csit-vpp as this code is re-used by ligato gathering.
         "csit-"*)
             # Use downloaded packages with specific version.
             if [[ "${TEST_CODE}" == *"daily"* ]] || \
-               ([[ "${TEST_CODE}" == *"weekly"* ]] && \
-                [[ "${TEST_CODE}" != *"device"* ]]) || \
-               [[ "${TEST_CODE}" == *"semiweekly"* ]];
+               { [[ "${TEST_CODE}" == *"weekly"* ]] && \
+                 [[ "${TEST_CODE}" != *"device"* ]]; } || \
+               [[ "${TEST_CODE}" == *"semiweekly"* ]] || \
+               [[ "${TEST_CODE}" == *"hourly"* ]];
             then
                 warn "Downloading latest VPP packages from Packagecloud."
             else
                 warn "Downloading stable VPP packages from Packagecloud."
-                if [[ "${TEST_CODE}" == *"device"* ]];
-                then
-                    VPP_VERSION="$(<"${CSIT_DIR}/VPP_STABLE_VER_UBUNTU_BIONIC")" || {
-                        die "Read VPP stable version failed."
-                    }
-                else
-                    VPP_VERSION="$(<"${CSIT_DIR}/VPP_STABLE_VER_UBUNTU")" || {
-                        die "Read VPP stable version failed."
-                    }
-                fi
+                VPP_VERSION="$(<"${CSIT_DIR}/${VPP_VER_FILE}")" || {
+                    die "Read VPP stable version failed."
+                }
             fi
             source "${BASH_FUNCTION_DIR}/artifacts.sh" || die "Source failed."
             download_artifacts || die
             ;;
         "vpp-csit-"*)
+            # Shorten line.
+            pgks="${PKG_SUFFIX}"
             # Use locally built packages.
-            mv "${DOWNLOAD_DIR}"/../"vpp"*".deb" "${DOWNLOAD_DIR}"/ || {
+            mv "${DOWNLOAD_DIR}"/../*vpp*."${pkgs}" "${DOWNLOAD_DIR}"/ || {
                 die "Move command failed."
             }
             ;;