fix(docker/scripts): clean up docker executor build scripts

- post-release clean up after VPP 24.06 release
- init vpp external/downloads from cache
- clean up shellcheck warnings

Change-Id: I8123e29d76f2034324a8bed1db16fd435f2bc04e
Signed-off-by: Dave Wallace <[email protected]>

diff --git a/docker/scripts/dbld_csit_find_ansible_packages.py b/docker/scripts/dbld_csit_find_ansible_packages.py
index 52ed4c3..50b7d84 100755 (executable)
--- a/docker/scripts/dbld_csit_find_ansible_packages.py
+++ b/docker/scripts/dbld_csit_find_ansible_packages.py
@@ -37,6 +37,7 @@ class CsitAnsibleYamlStruct:
         self.__dict__.update(entries)
 
 def packages_in_csit_ansible_yaml_file(yamlfile: str, distro, arch) -> list:
+    codename = os.getenv("OS_CODENAME")
     with open(yamlfile) as yf:
         csit_ansible_yaml = yaml.safe_load(yf)
         if csit_ansible_yaml is None:
@@ -50,7 +51,11 @@ def packages_in_csit_ansible_yaml_file(yamlfile: str, distro, arch) -> list:
             packages += [pkg for pkg in cays.packages_by_arch[arch]
                          if type(pkg) is str]
         if distro in [*cays.packages_by_distro]:
-            packages += [pkg for pkg in cays.packages_by_distro[distro]
+            if codename in [*cays.packages_by_distro[distro]]:
+                packages += [pkg for pkg in cays.packages_by_distro[distro][codename]
+                         if type(pkg) is str]
+            else:
+                packages += [pkg for pkg in cays.packages_by_distro[distro]
                          if type(pkg) is str]
         return packages
 

diff --git a/docker/scripts/dbld_vpp_install_packages.sh b/docker/scripts/dbld_vpp_install_packages.sh
index fecd823..be8b34c 100755 (executable)
--- a/docker/scripts/dbld_vpp_install_packages.sh
+++ b/docker/scripts/dbld_vpp_install_packages.sh
@@ -33,11 +33,6 @@ do_git_config vpp
 for branch in ${VPP_BRANCHES[$OS_NAME]} ; do
     do_git_branch "$branch"
 
-    # Install hs-test depndencies
-    if [ "$OS_NAME" = "ubuntu-22.04" ] && [ "$branch" = "master" ] ; then
-        install_hst_deps "$branch"
-    fi
-
     # Install OS packages
     make_vpp "install-deps" "$branch"
 
@@ -57,6 +52,16 @@ for branch in ${VPP_BRANCHES[$OS_NAME]} ; do
         echo "ERROR: Missing VPP external deps package: '$vpp_ext_deps_pkg'"
         exit 1
     fi
+
+    # Install hs-test depndencies
+    if [ "$OS_NAME" = "ubuntu-22.04" ] && [ "$branch" = "master" ] ; then
+        make_vpp build "$branch" "false"
+        make_vpp build-release "$branch" "false"
+        make_vpp build-vpp-gcov "$branch" "false"
+        install_hst_deps "$branch"
+        git clean -qfdx
+    fi
+
     # TODO: remove this after all supported VPP branches have removed
     # python3-virtualenv & virtualenv from install-deps which are no longer
     # used in vpp repo.  These packages can mess up csit virtualenv

diff --git a/docker/scripts/lib_apt.sh b/docker/scripts/lib_apt.sh
index a1ee6db..f8a6f67 100644 (file)
--- a/docker/scripts/lib_apt.sh
+++ b/docker/scripts/lib_apt.sh
@@ -20,7 +20,7 @@ if [ -n "$(alias lib_apt_imported 2> /dev/null)" ] ; then
 fi
 alias lib_apt_imported=true
 
-DIND_FROM_IMAGE="cruizba/ubuntu-dind:latest"
+DIND_FROM_IMAGE="cruizba/ubuntu-dind:jammy-26.1.3-r2"
 HST_FROM_IMAGE="ubuntu:22.04"
 
 export CIMAN_DOCKER_SCRIPTS=${CIMAN_DOCKER_SCRIPTS:-"$(dirname $BASH_SOURCE)"}
@@ -41,6 +41,7 @@ apt_install_packages() {
 generate_apt_dockerfile_common() {
     local executor_class="$1"
     local executor_image="$2"
+    local install_golang="$3"
     local dpkg_arch="$(dpkg --print-architecture)"
 
     cat <<EOF >>"$DOCKERFILE"
@@ -160,6 +161,13 @@ RUN wget https://releases.hashicorp.com/terraform/1.7.3/terraform_1.7.3_linux_$d
   && unzip terraform_1.7.3_linux_$dpkg_arch.zip \\
   && mv terraform /usr/bin \\
   && rm -f terraform_1.7.3_linux_$dpkg_arch.zip
+EOF
+
+    if [ "$install_golang" = "true" ] ; then
+        generate_apt_dockerfile_install_golang "1.21.11"
+    fi
+
+    cat <<EOF >>"$DOCKERFILE"
 
 # Install packages for all project branches
 #
@@ -182,7 +190,12 @@ EOF
 }
 
 generate_apt_dockerfile_install_golang() {
-    local GO_VERSION="$1"
+    local go_version="$1"
+    local go_tarball_arch="amd64"
+
+    if [ "$OS_ARCH" = "aarch64" ] ; then
+        go_tarball_arch="arm64"
+    fi
 
     cat <<EOF >>"$DOCKERFILE"
 
@@ -192,9 +205,9 @@ ENV GOPATH /go
 ENV GOROOT /usr/local/go
 ENV PATH \$GOPATH/bin:/usr/local/go/bin:\$PATH
 RUN rm -rf /usr/local/go /usr/bin/go \\
-    && wget -P /tmp "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" \\
-    && tar -C /usr/local -xzf "/tmp/go${GO_VERSION}.linux-amd64.tar.gz" \\
-    && rm "/tmp/go${GO_VERSION}.linux-amd64.tar.gz" \\
+    && wget -P /tmp "https://go.dev/dl/go${go_version}.linux-${go_tarball_arch}.tar.gz" \\
+    && tar -C /usr/local -xzf "/tmp/go${go_version}.linux-${go_tarball_arch}.tar.gz" \\
+    && rm "/tmp/go${go_version}.linux-${go_tarball_arch}.tar.gz" \\
     && ln -s /usr/local/go/bin/go /usr/bin/go \\
     && echo -n "\nGOPATH=\$GOPATH\nGOROOT=\$GOROOT" | tee -a /etc/environment \\
     && mkdir -p "\$GOPATH/src" "\$GOPATH/bin" && chmod -R 777 "\$GOPATH"
@@ -209,11 +222,8 @@ builder_generate_apt_dockerfile() {
     local install_golang="$4"
     local vpp_install_skip_sysctl_envvar="";
 
-    generate_apt_dockerfile_common "$executor_class" "$executor_image"
+    generate_apt_dockerfile_common "$executor_class" "$executor_image" "$install_golang"
     csit_builder_generate_docker_build_files
-    if [ "$install_golang" = "true" ] ; then
-        generate_apt_dockerfile_install_golang "1.21.9"
-    fi
     cat <<EOF >>"$DOCKERFILE"
 
 # Install LF-IT requirements
@@ -368,6 +378,7 @@ generate_apt_dockerfile() {
     local install_golang="false"
     local is_dind_image="false"
 
+    # TODO: Enable HST on AARCH64 when supported in vpp/extras/hs-test
     if [ "$from_image" = "$HST_FROM_IMAGE" ] ; then
         from_image="$DIND_FROM_IMAGE"
         install_golang="true"

diff --git a/docker/scripts/lib_common.sh b/docker/scripts/lib_common.sh
index 150441b..166b06f 100644 (file)
--- a/docker/scripts/lib_common.sh
+++ b/docker/scripts/lib_common.sh
@@ -232,6 +232,7 @@ export DOCKERIGNOREFILE="$DOCKER_BUILD_DIR/.dockerignore"
 export DOCKERFILE_FROM=${DOCKERFILE_FROM:="${OS_ID}:${OS_VERSION_ID}"}
 export DOCKER_TAG="$DOCKER_DATE-$OS_ARCH"
 export DOCKER_VPP_DIR="$DOCKER_BUILD_DIR/vpp"
+export DOCKER_VPP_DL_CACHE_DIR="$DOCKER_BUILD_DIR"/vpp_ext_deps_downloads
 export DOCKER_CSIT_DIR="$DOCKER_BUILD_DIR/csit"
 export DOCKER_DOWNLOADS_DIR="/root/Downloads"
 export DOCKER_BUILD_FILES_DIR="$DOCKER_BUILD_DIR/files"

diff --git a/docker/scripts/lib_csit.sh b/docker/scripts/lib_csit.sh
index 2e877f8..0499cbf 100644 (file)
--- a/docker/scripts/lib_csit.sh
+++ b/docker/scripts/lib_csit.sh
@@ -75,7 +75,7 @@ csit_install_packages() {
     # Not in double quotes to let bash remove newline characters
     local yaml_files
     yaml_files="$(grep -r packages_by $csit_ansible_dir | cut -d: -f1 | sort -u | grep -v $exclude_roles)"
-    packages="$(dbld_csit_find_ansible_packages.py --$OS_ID --$OS_ARCH $yaml_files)"
+    packages="$(dbld_csit_find_ansible_packages.py --$OS_ID --$OS_ARCH $yaml_files) | grep -v "$OS_CODENAME")"
     packages="${packages/jammy /}"
     packages="${packages/focal /}"
     packages="${packages/libmbedcrypto1/libmbedcrypto3}"

diff --git a/docker/scripts/lib_vpp.sh b/docker/scripts/lib_vpp.sh
index a4efbd1..4bf8dab 100644 (file)
--- a/docker/scripts/lib_vpp.sh
+++ b/docker/scripts/lib_vpp.sh
@@ -48,17 +48,26 @@ install_hst_deps() {
 make_vpp() {
     local target=$1
     local branch=${2:-"master"}
+    local clean=${3:-"true"}
     local branchname=${branch/\//_}
     local bld_log="$DOCKER_BUILD_LOG_DIR"
+    if [ "$target" = "install-ext-deps" ] ; then
+        if [ -d "$DOCKER_VPP_DL_CACHE_DIR" ] ; then
+            mkdir -p "$DOCKER_DOWNLOADS_DIR"
+            cp -a "$DOCKER_VPP_DL_CACHE_DIR"/* "$DOCKER_DOWNLOADS_DIR"
+        fi
+    fi
     bld_log="${bld_log}/$FDIOTOOLS_IMAGENAME-$branchname"
     bld_log="${bld_log}-make_vpp_${target}-bld.log"
 
     makefile_target="^${target}:"
-    if [ -z "$(grep $makefile_target Makefile)" ] ; then
+    if ! grep "$makefile_target" Makefile ; then
         echo "Make target '$target' does not exist in VPP branch '$branch'!"
         return
     fi
-    git clean -qfdx
+    if [ "$clean" = "true" ] ; then
+        git clean -qfdx
+    fi
     description="'make UNATTENDED=yes $target' in $(pwd) ($branch)"
     echo_log -e "    Starting  $description..."
     make UNATTENDED=yes "$target" 2>&1 | tee -a "$bld_log"
@@ -75,7 +84,7 @@ make_vpp_test() {
     bld_log="${bld_log}-make_vpp_test_${target}-bld.log"
 
     makefile_target="^${target}:"
-    if [ -z "$(grep -e $makefile_target test/Makefile)" ] ; then
+    if ! grep -e "$makefile_target" test/Makefile ; then
         echo "Make test target '$target' does not exist in VPP branch '$branch'!"
         return
     fi
@@ -83,7 +92,7 @@ make_vpp_test() {
     description="'make -C test $target' in $(pwd) ($branch)"
     echo_log "    Starting  $description..."
     make WS_ROOT="$DOCKER_VPP_DIR" BR="$DOCKER_VPP_DIR/build-root" \
-         TEST_DIR="$DOCKER_VPP_DIR/test" -C test $target 2>&1 | tee -a $bld_log
+         TEST_DIR="$DOCKER_VPP_DIR"/test -C test "$target" 2>&1 | tee -a "$bld_log"
     remove_pyc_files_and_pycache_dirs
     git checkout -q -- .
     echo_log "    Completed $description!"
@@ -94,6 +103,10 @@ docker_build_setup_vpp() {
         if [ ! -d "$DOCKER_VPP_DIR" ] ; then
             echo_log "Cloning VPP into $DOCKER_VPP_DIR..."
             git clone -q https://gerrit.fd.io/r/vpp $DOCKER_VPP_DIR
+            if [ -d "$DOCKER_DOWNLOADS_DIR" ] ; then
+                mkdir -p "$DOCKER_VPP_DL_CACHE_DIR"
+                cp -a "$DOCKER_DOWNLOADS_DIR"/* "$DOCKER_VPP_DL_CACHE_DIR"
+            fi
         fi
         clean_git_repo $DOCKER_VPP_DIR
     fi
@@ -109,7 +122,7 @@ docker_build_setup_vpp() {
 #       to create an enumerated set of jobs jobs that match the
 #       definitions here.
 declare -A VPP_BRANCHES
-VPP_BRANCHES["debian-11"]="stable/2310 stable/2402 master"
-VPP_BRANCHES["ubuntu-20.04"]="stable/2310 stable/2402 master"
-VPP_BRANCHES["ubuntu-22.04"]="stable/2310 stable/2402 master"
+VPP_BRANCHES["debian-11"]="stable/2402 stable/2406 master"
+VPP_BRANCHES["ubuntu-20.04"]="stable/2402 stable/2406 master"
+VPP_BRANCHES["ubuntu-22.04"]="stable/2402 stable/2406 master"
 export VPP_BRANCHES

diff --git a/jjb/vpp/docs.yaml b/jjb/vpp/docs.yaml
index 1ee98ba..545789e 100644 (file)
--- a/jjb/vpp/docs.yaml
+++ b/jjb/vpp/docs.yaml
@@ -15,9 +15,6 @@
       - 'master':
           branch: 'master'
           branch-refspec: ''
-      - '2310':
-          branch: 'stable/2310'
-          branch-refspec: ''
       - '2402':
           branch: 'stable/2402'
           branch-refspec: ''

diff --git a/jjb/vpp/vpp.yaml b/jjb/vpp/vpp.yaml
index c421b29..49c6ce8 100644 (file)
--- a/jjb/vpp/vpp.yaml
+++ b/jjb/vpp/vpp.yaml
@@ -39,9 +39,6 @@
       - 'master':
           branch: 'master'
           branch-refspec: ''
-      - '2310':
-          branch: 'stable/2310'
-          branch-refspec: ''
       - '2402':
           branch: 'stable/2402'
           branch-refspec: ''
@@ -106,11 +103,6 @@
           branch-refspec: ''
           make-test-os: 'ubuntu-20.04 ubuntu-22.04'
           make-test-multiworker-os: 'debian-11'
-      - '2310':
-          branch: 'stable/2310'
-          branch-refspec: ''
-          make-test-os: 'ubuntu-20.04 ubuntu-22.04'
-          make-test-multiworker-os: 'debian-11'
       - '2402':
           branch: 'stable/2402'
           branch-refspec: ''
@@ -142,9 +134,6 @@
       - 'master':
           branch: 'master'
           branch-refspec: ''
-      - '2310':
-          branch: 'stable/2310'
-          branch-refspec: ''
       - '2402':
           branch: 'stable/2402'
           branch-refspec: ''
@@ -249,11 +238,6 @@
           branch-refspec: ''
           make-test-os: 'ubuntu-20.04 ubuntu-22.04'
           make-test-multiworker-os: ''
-      - '2310':
-          branch: 'stable/2310'
-          branch-refspec: ''
-          make-test-os: 'ubuntu-20.04 ubuntu-22.04'
-          make-test-multiworker-os: ''
       - '2402':
           branch: 'stable/2402'
           branch-refspec: ''
@@ -371,9 +355,6 @@
       - 'master':
           branch: 'master'
           branch-refspec: ''
-      - '2310':
-          branch: 'stable/2310'
-          branch-refspec: ''
       - '2402':
           branch: 'stable/2402'
           branch-refspec: ''