From 13b49bb3be30d95b39e04c080b51fc5bc5004b7c Mon Sep 17 00:00:00 2001 From: Adrian Villin Date: Mon, 14 Jul 2025 12:49:59 +0200 Subject: [PATCH] hs-test: KinD cluster build script improvements - new makefile targets, cleanup - cluster can be rebuilt without shutting down Type: improvement Change-Id: I020e396a6410786ae29b9146a358d60503ac5662 Signed-off-by: Adrian Villin --- extras/hs-test/Makefile | 18 +++- extras/hs-test/kubernetes/setupCluster.sh | 172 ++++++++++++++++++------------ 2 files changed, 118 insertions(+), 72 deletions(-) diff --git a/extras/hs-test/Makefile b/extras/hs-test/Makefile index 39c5f08e502..6b94fe682c2 100644 --- a/extras/hs-test/Makefile +++ b/extras/hs-test/Makefile @@ -87,7 +87,9 @@ help: @echo " build-cov - coverage build of VPP and Docker images" @echo " build-debug - build test infra (vpp debug image)" @echo " build-go - just build golang files" - @echo " setup-cluster - setup KinD cluster for performance testing" + @echo " master-cluster - setup KinD cluster for performance testing (master CalicoVPP + VPP)" + @echo " rebuild-master-cluster - rebuild VPP and update related pods without shutting down the cluster" + @echo " release-cluster - setup KinD cluster for performance testing (latest CalicoVPP release)" @echo " checkstyle-go - check style of .go source files" @echo " fixstyle-go - format .go source files" @echo " cleanup-hst - removes all docker containers and namespaces from last test run" @@ -203,9 +205,17 @@ test-perf: .deps.ok .build.ok --ginkgo_timeout=$(GINKGO_TIMEOUT); \ ./script/compress.sh $$? -.PHONY: setup-cluster -setup-cluster: .kind_deps.ok - @bash ./kubernetes/setupCluster.sh $(BASE) +.PHONY: release-cluster +release-cluster: .kind_deps.ok + @bash ./kubernetes/setupCluster.sh setup_release + +.PHONY: master-cluster +master-cluster: .kind_deps.ok + @bash ./kubernetes/setupCluster.sh setup_master + +.PHONY: rebuild-master-cluster +rebuild-cluster: .kind_deps.ok + @bash ./kubernetes/setupCluster.sh rebuild_master .PHONY: build-go build-go: diff --git a/extras/hs-test/kubernetes/setupCluster.sh b/extras/hs-test/kubernetes/setupCluster.sh index 131248ca272..3c176f6e98b 100755 --- a/extras/hs-test/kubernetes/setupCluster.sh +++ b/extras/hs-test/kubernetes/setupCluster.sh @@ -1,83 +1,119 @@ #!/usr/bin/env bash set -e -MASTER_OR_LATEST=${1-"latest"} +COMMAND=$1 CALICOVPP_DIR="$HOME/vpp-dataplane" VPP_DIR=$(pwd) VPP_DIR=${VPP_DIR%extras*} STASH_SAVED=0 -if [ $MASTER_OR_LATEST = "master" ]; then - if [ ! -d "$CALICOVPP_DIR" ]; then - git clone https://github.com/projectcalico/vpp-dataplane.git $CALICOVPP_DIR - fi - cd $CALICOVPP_DIR - git pull - cd $VPP_DIR +# ---------------- images ---------------- +export CALICO_AGENT_IMAGE=localhost:5000/calicovpp/agent:latest +export CALICO_VPP_IMAGE=localhost:5000/calicovpp/vpp:latest +export MULTINET_MONITOR_IMAGE=localhost:5000/calicovpp/multinet-monitor:latest +export IMAGE_PULL_POLICY=Always - # ---------------- images ---------------- - export CALICO_AGENT_IMAGE=localhost:5000/calicovpp/agent:latest - export CALICO_VPP_IMAGE=localhost:5000/calicovpp/vpp:latest - export MULTINET_MONITOR_IMAGE=localhost:5000/calicovpp/multinet-monitor:latest - export IMAGE_PULL_POLICY=Always +# ---------------- interfaces ---------------- +export CALICOVPP_INTERFACES='{ + "uplinkInterfaces": [ + { + "interfaceName": "eth0", + "vppDriver": "af_packet" + } + ] + }' +export CALICOVPP_DISABLE_HUGEPAGES=true +export CALICOVPP_CONFIG_TEMPLATE=" + unix { + nodaemon + full-coredump + log /var/run/vpp/vpp.log + cli-listen /var/run/vpp/cli.sock + pidfile /run/vpp/vpp.pid + } + buffers { + buffers-per-numa 131072 + } + socksvr { socket-name /var/run/vpp/vpp-api.sock } + plugins { + plugin default { enable } + plugin calico_plugin.so { enable } + plugin dpdk_plugin.so { disable } + }" +export CALICOVPP_ENABLE_VCL=true - # ---------------- interfaces ---------------- - export CALICOVPP_INTERFACES='{ - "uplinkInterfaces": [ - { - "interfaceName": "eth0", - "vppDriver": "af_packet" - } - ] - }' - export CALICOVPP_DISABLE_HUGEPAGES=true - export CALICOVPP_CONFIG_TEMPLATE=" - unix { - nodaemon - full-coredump - log /var/run/vpp/vpp.log - cli-listen /var/run/vpp/cli.sock - pidfile /run/vpp/vpp.pid - } - buffers { - buffers-per-numa 131072 - } - socksvr { socket-name /var/run/vpp/vpp-api.sock } - plugins { - plugin default { enable } - plugin calico_plugin.so { enable } - plugin dpdk_plugin.so { disable } - }" - export CALICOVPP_ENABLE_VCL=true +help() { + echo "Usage:" + echo " make master-cluster | rebuild-master-cluster | release-cluster" + echo "or" + echo " ./kubernetes/setupCluster.sh [master-cluster | rebuild-master-cluster | release-cluster]" + echo "" + echo "'master-cluster' pulls CalicoVPP and builds VPP from this directory, then brings up a KinD cluster." + echo "'rebuild-master-cluster' stops CalicoVPP pods, rebuilds VPP and restarts CalicoVPP pods. Cluster keeps running." + echo "'release-cluster' starts up a KinD cluster and uses latest CalicoVPP release (e.g. v3.29)" + echo "" + echo "To shut down the cluster, use 'kind delete cluster'" +} - make -C $CALICOVPP_DIR kind-new-cluster N_KIND_WORKERS=2 - kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.28.3/manifests/tigera-operator.yaml - make -C $CALICOVPP_DIR cherry-vpp FORCE=y BASE=origin/master VPP_DIR=$VPP_DIR - make build - make -C $CALICOVPP_DIR dev-kind - make -C $CALICOVPP_DIR load-kind - $CALICOVPP_DIR/yaml/overlays/dev/kustomize.sh up - if ! git diff-index --quiet HEAD --; then - echo "Saving stash" - git stash save "HST: temp stash" - git reset --hard origin/master - git stash pop - fi - else - echo "********" - echo "Performance tests only work on Ubuntu 22.04 for now." - echo "********" +setup_master() { + if [ ! -d "$CALICOVPP_DIR" ]; then + git clone https://github.com/projectcalico/vpp-dataplane.git $CALICOVPP_DIR + else + cd $CALICOVPP_DIR + git pull + cd $VPP_DIR + fi - kind create cluster --config kubernetes/kind-config.yaml - kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.28.3/manifests/tigera-operator.yaml + make -C $CALICOVPP_DIR kind-new-cluster N_KIND_WORKERS=2 + kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.28.3/manifests/tigera-operator.yaml + make -C $CALICOVPP_DIR cherry-vpp FORCE=y BASE=origin/master VPP_DIR=$VPP_DIR + make build-vpp-release + make -C $CALICOVPP_DIR dev-kind + make -C $CALICOVPP_DIR load-kind + $CALICOVPP_DIR/yaml/overlays/dev/kustomize.sh up + if ! git diff-index --quiet HEAD --; then + echo "Saving stash" + git stash save "HST: temp stash" + git reset --hard origin/master + git stash pop + fi +} - echo "Sleeping for 10s, waiting for tigera operator to start up." - sleep 10 +rebuild_master() { + echo "Shutting down pods may take some time, timeout is set to 1m." + timeout 1m $CALICOVPP_DIR/yaml/overlays/dev/kustomize.sh dn || true + make build-vpp-release + make -C $CALICOVPP_DIR dev-kind + make -C $CALICOVPP_DIR load-kind + $CALICOVPP_DIR/yaml/overlays/dev/kustomize.sh up +} - kubectl create -f https://raw.githubusercontent.com/projectcalico/vpp-dataplane/master/yaml/calico/installation-default.yaml - kubectl create -f kubernetes/calico-config.yaml +setup_release() { + kind create cluster --config kubernetes/kind-config.yaml + kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.28.3/manifests/tigera-operator.yaml - echo "Done. Please wait for the cluster to come fully online before running tests." - echo "Use 'watch kubectl get pods -A' to monitor cluster status." - echo "To delete the cluster, use 'kind delete cluster'" - fi + echo "Sleeping for 10s, waiting for tigera operator to start up." + sleep 10 + + kubectl create -f https://raw.githubusercontent.com/projectcalico/vpp-dataplane/master/yaml/calico/installation-default.yaml + kubectl create -f kubernetes/calico-config.yaml + + echo "Done. Please wait for the cluster to come fully online before running tests." + echo "Use 'watch kubectl get pods -A' to monitor cluster status." + echo "To delete the cluster, use 'kind delete cluster'" +} + +case "$COMMAND" in + master-cluster) + setup_master + ;; + rebuild-master-cluster) + rebuild_master + ;; + release-cluster) + setup_release + ;; +*) + help + ;; +esac -- 2.16.6