hs-test: KinD cluster setup improvements

- script cleanup
- clarified Makefile build-msg
- release-cluster: versions can be specified
- master-cluster: local repo gets properly reset to its
  original state, fixed rebuild-master-cluster

Type: improvement

Change-Id: I81f813d2cd0227eaac3b996590152ba7e1173e38
Signed-off-by: Adrian Villin <[email protected]>

diff --git a/.gitignore b/.gitignore
index 67415d8..c8e3edf 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -147,6 +147,7 @@ compile_commands.json
 /extras/hs-test/.last_state_hash
 /extras/hs-test/.kind_deps.ok
 /extras/hs-test/.go_cache/
+/extras/hs-test/kubernetes/calico-config.yaml
 
 # ./configure
 /CMakeFiles

diff --git a/extras/hs-test/Makefile b/extras/hs-test/Makefile
index 5406b69..c88d5a3 100644 (file)
--- a/extras/hs-test/Makefile
+++ b/extras/hs-test/Makefile
@@ -107,6 +107,7 @@ help:
        @echo " build-cov                - coverage build of VPP and Docker images"
        @echo " build-debug              - build test infra (vpp debug image)"
        @echo " build-go                 - just build golang files"
+       @echo " cluster-help             - print info about KinD cluster setup (perf testing)"
        @echo " master-cluster           - setup KinD cluster for performance testing (master CalicoVPP + VPP)"
        @echo " rebuild-master-cluster   - rebuild VPP and update related pods without shutting down the cluster"
        @echo " release-cluster          - setup KinD cluster for performance testing (latest CalicoVPP release)"
@@ -152,7 +153,9 @@ list-tests:
 build-msg:
        @echo "Building VPP as '$(BUILD_AS)'"
        @echo "****************************************************************"
-       @echo "If you wish to build VPP as root, use 'SUDO_USER=root make ...'"
+       @echo "If you wish to build VPP as root, use 'make ... BUILD_AS=root'"
+       @echo "or"
+       @echo "'[sudo] SUDO_USER=root make ...'"
        @echo "****************************************************************"
 
 .PHONY: build-vpp-release
@@ -233,17 +236,21 @@ test-perf: .deps.ok .build.ok
                --ginkgo_timeout=$(GINKGO_TIMEOUT); \
                ./script/compress.sh $$?
 
+.PHONY: cluster-help
+cluster-help:
+       @bash ./kubernetes/setup-cluster.sh help
+
 .PHONY: release-cluster
 release-cluster: .kind_deps.ok
-       @bash ./kubernetes/setupCluster.sh release-cluster
+       @bash ./kubernetes/setup-cluster.sh release-cluster
 
 .PHONY: master-cluster
 master-cluster: .kind_deps.ok
-       @bash ./kubernetes/setupCluster.sh master-cluster
+       @bash ./kubernetes/setup-cluster.sh master-cluster
 
 .PHONY: rebuild-master-cluster
-rebuild-cluster: .kind_deps.ok
-       @bash ./kubernetes/setupCluster.sh rebuild-master-cluster
+rebuild-master-cluster: .kind_deps.ok
+       @bash ./kubernetes/setup-cluster.sh rebuild-master-cluster
 
 # this is executed in a container by hs-test.sh
 .PHONY: build-go

diff --git a/extras/hs-test/kubernetes/calico-config.yaml b/extras/hs-test/kubernetes/calico-config-template.yaml
similarity index 98%
rename from extras/hs-test/kubernetes/calico-config.yaml
rename to extras/hs-test/kubernetes/calico-config-template.yaml
index ea6fad1..395e404 100644 (file)
--- a/extras/hs-test/kubernetes/calico-config.yaml
+++ b/extras/hs-test/kubernetes/calico-config-template.yaml
@@ -230,7 +230,7 @@ spec:
         envFrom:
         - configMapRef:
             name: calico-vpp-config
-        image: docker.io/calicovpp/agent:latest
+        image: docker.io/calicovpp/agent:${CALICOVPP_VERSION}
         imagePullPolicy: IfNotPresent
         name: agent
         resources:
@@ -262,7 +262,7 @@ spec:
         envFrom:
         - configMapRef:
             name: calico-vpp-config
-        image: docker.io/calicovpp/vpp:latest
+        image: docker.io/calicovpp/vpp:${CALICOVPP_VERSION}
         imagePullPolicy: IfNotPresent
         name: vpp
         resources:

diff --git a/extras/hs-test/kubernetes/setupCluster.sh b/extras/hs-test/kubernetes/setup-cluster.sh
similarity index 64%
rename from extras/hs-test/kubernetes/setupCluster.sh
rename to extras/hs-test/kubernetes/setup-cluster.sh
index 4eba189..95ef3eb 100755 (executable)
--- a/extras/hs-test/kubernetes/setupCluster.sh
+++ b/extras/hs-test/kubernetes/setup-cluster.sh
@@ -5,8 +5,9 @@ COMMAND=$1
 CALICOVPP_DIR="$HOME/vpp-dataplane"
 VPP_DIR=$(pwd)
 VPP_DIR=${VPP_DIR%extras*}
-STASH_SAVED=0
+COMMIT_HASH=$(git rev-parse HEAD)
 
+export DOCKER_BUILD_PROXY=$HTTP_PROXY
 # ---------------- images ----------------
 export CALICO_AGENT_IMAGE=localhost:5000/calicovpp/agent:latest
 export CALICO_VPP_IMAGE=localhost:5000/calicovpp/vpp:latest
@@ -44,15 +45,52 @@ export CALICOVPP_ENABLE_VCL=true
 
 help() {
   echo "Usage:"
-  echo "  make master-cluster | rebuild-master-cluster | release-cluster"
-  echo "or"
-  echo "  ./kubernetes/setupCluster.sh [master-cluster | rebuild-master-cluster | release-cluster]"
-  echo ""
+  echo -e "  make master-cluster | rebuild-master-cluster | release-cluster\n"
+
   echo "'master-cluster' pulls CalicoVPP and builds VPP from this directory, then brings up a KinD cluster."
   echo "'rebuild-master-cluster' stops CalicoVPP pods, rebuilds VPP and restarts CalicoVPP pods. Cluster keeps running."
-  echo "'release-cluster' starts up a KinD cluster and uses latest CalicoVPP release (e.g. v3.29)"
-  echo ""
-  echo "To shut down the cluster, use 'kind delete cluster'"
+  echo "'release-cluster' starts up a KinD cluster and uses latest CalicoVPP release (e.g. v3.29),
+    or you can override versions by using env variables 'CALICOVPP_VERSION' and 'TIGERA_VERSION':
+    CALICOVPP_VERSION: latest | v[x].[y].[z] (default=latest)
+    TIGERA_VERSION:    master | v[x].[y].[z] (default=v3.28.3)"
+
+  echo -e "\nTo shut down the cluster, use 'kind delete cluster'"
+}
+
+cherry_pick() {
+  STASHED_CHANGES=0
+  echo "checkpoint: $COMMIT_HASH"
+  # chery-vpp hard resets the repo to a commit - we want to keep our changes
+  if ! git diff-index --quiet HEAD --; then
+           echo "Saving stash"
+      git stash save "HST: temp stash"
+      STASHED_CHANGES=1
+       fi
+  make -C $CALICOVPP_DIR cherry-vpp FORCE=y BASE=origin/master VPP_DIR=$VPP_DIR
+
+  # pop the stash to build VPP with CalicoVPP's patches + our changes
+  if [ $STASHED_CHANGES -eq 1 ]; then
+           git stash pop
+       fi
+}
+
+build_load_start_cni() {
+  make -C $VPP_DIR/extras/hs-test build-vpp-release
+  make -C $CALICOVPP_DIR dev-kind
+  make -C $CALICOVPP_DIR load-kind
+  $CALICOVPP_DIR/yaml/overlays/dev/kustomize.sh up
+}
+
+restore_repo() {
+  # stash changes, reset local repo to the original state and unstash changes (removes CalicoVPP's patches)
+  if ! git diff-index --quiet HEAD --; then
+           echo "Saving stash"
+      git stash save "HST: temp stash"
+      git reset --hard $COMMIT_HASH
+      git stash pop
+       else
+    git reset --hard $COMMIT_HASH
+  fi
 }
 
 setup_master() {
@@ -66,34 +104,32 @@ setup_master() {
 
   make -C $CALICOVPP_DIR kind-new-cluster N_KIND_WORKERS=2
   kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.28.3/manifests/tigera-operator.yaml
-  make -C $CALICOVPP_DIR cherry-vpp FORCE=y BASE=origin/master VPP_DIR=$VPP_DIR
-  make -C $VPP_DIR/extras/hs-test build-vpp-release
-  make -C $CALICOVPP_DIR dev-kind
-  make -C $CALICOVPP_DIR load-kind
-  $CALICOVPP_DIR/yaml/overlays/dev/kustomize.sh up
-  if ! git diff-index --quiet HEAD --; then
-           echo "Saving stash"
-      git stash save "HST: temp stash"
-      git reset --hard origin/master
-      git stash pop
-       fi
+
+  cherry_pick
+  build_load_start_cni
+  restore_repo
 }
 
 rebuild_master() {
   echo "Shutting down pods may take some time, timeout is set to 1m."
   timeout 1m $CALICOVPP_DIR/yaml/overlays/dev/kustomize.sh dn || true
-  make build-vpp-release
-  make -C $CALICOVPP_DIR dev-kind
-  make -C $CALICOVPP_DIR load-kind
-  $CALICOVPP_DIR/yaml/overlays/dev/kustomize.sh up
+  cherry_pick
+  build_load_start_cni
+  restore_repo
 }
 
 setup_release() {
+  CALICOVPP_VERSION="${CALICOVPP_VERSION:-latest}"
+  TIGERA_VERSION="${TIGERA_VERSION:-v3.28.3}"
+  echo "CALICOVPP_VERSION=$CALICOVPP_VERSION"
+  echo "TIGERA_VERSION=$TIGERA_VERSION"
+  envsubst < kubernetes/calico-config-template.yaml > kubernetes/calico-config.yaml
+
   kind create cluster --config kubernetes/kind-config.yaml
-  kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.28.3/manifests/tigera-operator.yaml
+  kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/$TIGERA_VERSION/manifests/tigera-operator.yaml
 
-  echo "Sleeping for 10s, waiting for tigera operator to start up."
-  sleep 10
+  echo "Waiting for tigera-operator pod to start up."
+  kubectl -n tigera-operator wait --for=condition=Ready pod --all --timeout=1m
 
   kubectl create -f https://raw.githubusercontent.com/projectcalico/vpp-dataplane/master/yaml/calico/installation-default.yaml
   kubectl create -f kubernetes/calico-config.yaml