From: Adrian Villin <avillin@cisco.com>
Date: Mon, 21 Jul 2025 12:08:14 +0000 (+0200)
Subject: hs-test: containerize ginkgo
X-Git-Tag: v26.02-rc0~143
X-Git-Url: https://gerrit.fd.io/r/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F85%2F43385%2F7;p=vpp.git

hs-test: containerize ginkgo

- ginkgo will run in a container
- replaced '--network=host' with '--network=container:ginkgo'
- removed --privileged flags
- removed network namespaces from HST
- updated goimports
- fixed state hashes

Type: improvement

Change-Id: I5f4c5aa93437f40b36a8eb2ba3d6486cdfe23e09
Signed-off-by: Adrian Villin <avillin@cisco.com>
---

diff --git a/.gitignore b/.gitignore
index 17b48e307cc..67415d84410 100644
--- a/.gitignore
+++ b/.gitignore
@@ -143,10 +143,10 @@ compile_commands.json
 /extras/hs-test/.build.ok
 /extras/hs-test/.build.cov.ok
 /extras/hs-test/.last_hst_ppid
-/extras/hs-test/.goimports.ok
 /extras/hs-test/summary/
 /extras/hs-test/.last_state_hash
 /extras/hs-test/.kind_deps.ok
+/extras/hs-test/.go_cache/
 
 # ./configure
 /CMakeFiles
diff --git a/extras/hs-test/Makefile b/extras/hs-test/Makefile
index 6b94fe682c2..4d33529959f 100644
--- a/extras/hs-test/Makefile
+++ b/extras/hs-test/Makefile
@@ -73,9 +73,27 @@ ifeq ($(GINKGO_TIMEOUT),)
 GINKGO_TIMEOUT=3h
 endif
 
+CORE_PATTERN := $(shell cat /proc/sys/kernel/core_pattern)
+CORE_VOLUME:=
+DOCKER_TTY:=
+
+ifeq ($(shell expr "$(CORE_PATTERN)" : '^/'), 1)
+CORE_VOLUME := -v $(CORE_PATTERN):$(CORE_PATTERN)
+endif
+
+ifeq ($(shell tty -s && echo $$?), 0)
+DOCKER_TTY := -it
+endif
+
 FORCE_BUILD?=true
 BUILD_AS:=$(strip $(shell echo $${SUDO_USER:-$${USER:-root}}))
 
+DOCKER_CAPABILITIES:=--cap-add=NET_ADMIN --cap-add=SYS_RESOURCE --cap-add=IPC_LOCK
+DOCKER_DEVICES:=--device /dev/vhost-net:/dev/vhost-net --device /dev/net/tun:/dev/net/tun
+DOCKER_VOLUMES:=-v $(WS_ROOT):$(WS_ROOT) -v /var/run/docker.sock:/var/run/docker.sock -v /tmp/hs-test:/tmp/hs-test \
+	-v /etc/localtime:/etc/localtime:ro $(CORE_VOLUME) -v $(HS_ROOT)/.go_cache/mod:/root/go/pkg/mod \
+	-v $(HS_ROOT)/.go_cache/build:/root/.cache/go-build
+
 .PHONY: help
 help:
 	@echo "Make targets:"
@@ -159,21 +177,25 @@ build-vpp-gcov: build-msg
 .PHONY: test
 test: FORCE_BUILD=false
 test: .deps.ok .build.ok
-	@bash ./hs_test.sh --persist=$(PERSIST) --verbose=$(VERBOSE) \
+	docker run $(DOCKER_TTY) --rm $(DOCKER_CAPABILITIES) $(DOCKER_DEVICES) \
+		-e BUILD_NUMBER=$(BUILD_NUMBER) $(DOCKER_VOLUMES) --name ginkgo hs-test/ginkgo \
+		.$(HS_ROOT)/hs_test.sh --persist=$(PERSIST) --verbose=$(VERBOSE) \
 		--unconfigure=$(UNCONFIGURE) --debug=$(DEBUG) --test=$(TEST) --cpus=$(CPUS) \
 		--vppsrc=$(VPPSRC) --parallel=$(PARALLEL) --repeat=$(REPEAT) --cpu0=$(CPU0) \
 		--dryrun=$(DRYRUN) --skip=$(SKIP) --no_color=$(NO_COLOR) --timeout=$(TIMEOUT) \
-		--ginkgo_timeout=$(GINKGO_TIMEOUT) --vpp_cpus=$(VPP_CPUS); \
+		--ginkgo_timeout=$(GINKGO_TIMEOUT) --vpp_cpus=$(VPP_CPUS) --hs_root=$(HS_ROOT); \
 		./script/compress.sh $$?
 
 .PHONY: test-debug
 test-debug: FORCE_BUILD=false
 test-debug: .deps.ok .build_debug.ok
-	@bash ./hs_test.sh --persist=$(PERSIST) --verbose=$(VERBOSE) \
+	docker run $(DOCKER_TTY) --rm $(DOCKER_CAPABILITIES) $(DOCKER_DEVICES) \
+		-e BUILD_NUMBER=$(BUILD_NUMBER) $(DOCKER_VOLUMES) --name ginkgo hs-test/ginkgo \
+		.$(HS_ROOT)/hs_test.sh --persist=$(PERSIST) --verbose=$(VERBOSE) \
 		--unconfigure=$(UNCONFIGURE) --debug=$(DEBUG) --test=$(TEST) --cpus=$(CPUS) \
 		--vppsrc=$(VPPSRC) --parallel=$(PARALLEL) --repeat=$(REPEAT) --debug_build=true \
 		--cpu0=$(CPU0) --dryrun=$(DRYRUN) --skip=$(SKIP) --no_color=$(NO_COLOR) --timeout=$(TIMEOUT) \
-		--ginkgo_timeout=$(GINKGO_TIMEOUT) --vpp_cpus=$(VPP_CPUS); \
+		--ginkgo_timeout=$(GINKGO_TIMEOUT) --vpp_cpus=$(VPP_CPUS) --hs_root=$(HS_ROOT); \
 		./script/compress.sh $$?
 
 .PHONY: wipe-lcov
@@ -183,18 +205,22 @@ wipe-lcov:
 .PHONY: test-cov
 test-cov: FORCE_BUILD=false
 test-cov: .deps.ok .build.cov.ok wipe-lcov
-	-@bash ./hs_test.sh --coverage=true --persist=$(PERSIST) --verbose=$(VERBOSE) \
+	-docker run $(DOCKER_TTY) --rm $(DOCKER_CAPABILITIES) $(DOCKER_DEVICES) \
+		-e BUILD_NUMBER=$(BUILD_NUMBER) $(DOCKER_VOLUMES) --name ginkgo hs-test/ginkgo \
+		.$(HS_ROOT)/hs_test.sh --coverage=true --persist=$(PERSIST) --verbose=$(VERBOSE) \
 		--unconfigure=$(UNCONFIGURE) --debug=$(DEBUG) --test=$(TEST-HS) --cpus=$(CPUS) \
 		--vppsrc=$(VPPSRC) --cpu0=$(CPU0) --dryrun=$(DRYRUN) --skip=$(SKIP) --no_color=$(NO_COLOR) \
-		--timeout=$(TIMEOUT) --ginkgo_timeout=$(GINKGO_TIMEOUT) --vpp_cpus=$(VPP_CPUS); \
+		--timeout=$(TIMEOUT) --ginkgo_timeout=$(GINKGO_TIMEOUT) --vpp_cpus=$(VPP_CPUS) --hs_root=$(HS_ROOT); \
 		./script/compress.sh $$?
 	$(MAKE) -C ../.. test-cov-post-standalone HS_TEST=1
 
 .PHONY: test-leak
 test-leak: FORCE_BUILD=false
 test-leak: .deps.ok .build_debug.ok
-	@bash ./hs_test.sh --test=$(TEST) --debug_build=true --leak_check=true --vppsrc=$(VPPSRC) --timeout=$(TIMEOUT) \
-	--ginkgo_timeout=$(GINKGO_TIMEOUT) --vpp_cpus=$(VPP_CPUS);
+	docker run $(DOCKER_TTY) --rm $(DOCKER_CAPABILITIES) $(DOCKER_DEVICES) \
+		-e BUILD_NUMBER=$(BUILD_NUMBER) $(DOCKER_VOLUMES) --name ginkgo hs-test/ginkgo \
+		.$(HS_ROOT)/hs_test.sh --test=$(TEST) --debug_build=true --leak_check=true --vppsrc=$(VPPSRC) --timeout=$(TIMEOUT) \
+		--ginkgo_timeout=$(GINKGO_TIMEOUT) --vpp_cpus=$(VPP_CPUS) --hs_root=$(HS_ROOT);
 
 .PHONY: test-perf
 test-perf: FORCE_BUILD=false
@@ -207,34 +233,35 @@ test-perf: .deps.ok .build.ok
 
 .PHONY: release-cluster
 release-cluster: .kind_deps.ok
-	@bash ./kubernetes/setupCluster.sh setup_release
+	@bash ./kubernetes/setupCluster.sh release-cluster
 
 .PHONY: master-cluster
 master-cluster: .kind_deps.ok
-	@bash ./kubernetes/setupCluster.sh setup_master
+	@bash ./kubernetes/setupCluster.sh master-cluster
 
 .PHONY: rebuild-master-cluster
 rebuild-cluster: .kind_deps.ok
-	@bash ./kubernetes/setupCluster.sh rebuild_master
+	@bash ./kubernetes/setupCluster.sh rebuild-master-cluster
 
+# this is executed in a container by hs-test.sh
 .PHONY: build-go
 build-go:
-	go build ./tools/http_server
+	go build --buildvcs=false ./tools/http_server
 
 .PHONY: build
-build: .deps.ok build-vpp-release build-go
+build: .deps.ok build-vpp-release
 	@rm -f .build.ok
 	bash ./script/build_hst.sh release $(FORCE_BUILD) $(BUILD_AS)
 	@touch .build.ok
 
 .PHONY: build-cov
-build-cov: .deps.ok build-vpp-gcov build-go
+build-cov: .deps.ok build-vpp-gcov
 	@rm -f .build.cov.ok
 	bash ./script/build_hst.sh gcov $(FORCE_BUILD) $(BUILD_AS)
 	@touch .build.cov.ok
 
 .PHONY: build-debug
-build-debug: .deps.ok build-vpp-debug build-go
+build-debug: .deps.ok build-vpp-debug
 	@rm -f .build.ok
 	bash ./script/build_hst.sh debug $(FORCE_BUILD) $(BUILD_AS)
 	@touch .build.ok
@@ -291,15 +318,9 @@ install-deps:
 	@sudo -E apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
 	@touch .deps.ok
 
-.goimports.ok:
-	@rm -f .goimports.ok
-	go install golang.org/x/tools/cmd/goimports@v0.25.0
-	@touch .goimports.ok
-
 .PHONY: checkstyle-go
-checkstyle-go: .goimports.ok
-	$(eval GOPATH := $(shell go env GOPATH))
-	@output=$$($(GOPATH)/bin/goimports -d $${WS_ROOT}); \
+checkstyle-go:
+	@output=$$(find . -type f -name '*.go' -not -path './.go_cache/*' -exec go run golang.org/x/tools/cmd/goimports@v0.35.0 -d {} +); \
 	status=$$?; \
 	if [ $$status -ne 0 ]; then \
 		exit $$status; \
@@ -316,10 +337,9 @@ checkstyle-go: .goimports.ok
     fi
 
 .PHONY: fixstyle-go
-fixstyle-go: .goimports.ok
-	$(eval GOPATH := $(shell go env GOPATH))
+fixstyle-go:
 	@echo "Modified files:"
-	@$(GOPATH)/bin/goimports -w -l $(WS_ROOT)
+	@find . -type f -name '*.go' -not -path './.go_cache/*' -exec go run golang.org/x/tools/cmd/goimports@v0.35.0 -w -l {} +
 	@go mod tidy
 	@echo "*******************************************************************"
 	@echo "Fixstyle done."
diff --git a/extras/hs-test/docker/Dockerfile.base b/extras/hs-test/docker/Dockerfile.base
index 12b7c0588a8..e6e19f9f393 100644
--- a/extras/hs-test/docker/Dockerfile.base
+++ b/extras/hs-test/docker/Dockerfile.base
@@ -37,7 +37,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     # Tools moved from derived images
     apache2-utils \
     nghttp2 \
-    wrk
+    wrk \
+    # Utilities needed for running tests
+    sudo \
+    make \
+    ethtool \
+    bridge-utils \
+    jq
 
 # Because of http/3 we can't use stock curl in ubuntu 24.04
 ARG TARGETARCH
diff --git a/extras/hs-test/docker/Dockerfile.ginkgo b/extras/hs-test/docker/Dockerfile.ginkgo
new file mode 100644
index 00000000000..2ec1e107ab9
--- /dev/null
+++ b/extras/hs-test/docker/Dockerfile.ginkgo
@@ -0,0 +1,19 @@
+FROM localhost:5001/vpp-test-base:latest
+
+# We don't need to install these packages as they're in the base image
+# Just install anything specific to VPP that isn't in the base
+
+ARG UBUNTU_VERSION
+ARG TARGETARCH
+ARG CODENAME
+ARG GO_VERSION=1.23.10
+RUN echo "I'm building for ${TARGETARCH}" \
+&& wget "https://go.dev/dl/go${GO_VERSION}.linux-${TARGETARCH}.tar.gz" -O /tmp/go.tar.gz \
+&& tar -xzf /tmp/go.tar.gz -C /usr/local \
+&& rm /tmp/go.tar.gz
+
+RUN wget "https://download.docker.com/linux/ubuntu/dists/${CODENAME}/pool/stable/${TARGETARCH}/docker-ce-cli_28.3.2-1~ubuntu.${UBUNTU_VERSION}~${CODENAME}_${TARGETARCH}.deb" \
+-O /tmp/docker-ce-cli.deb
+RUN dpkg -i /tmp/docker-ce-cli.deb
+
+RUN ln -s /usr/local/go/bin/go /usr/bin/go
diff --git a/extras/hs-test/hs_test.sh b/extras/hs-test/hs_test.sh
old mode 100644
new mode 100755
index a6c70a2bd39..c30a75966ec
--- a/extras/hs-test/hs_test.sh
+++ b/extras/hs-test/hs_test.sh
@@ -17,6 +17,7 @@ skip_names=()
 dryrun=
 no_color=
 perf=
+hs_root=
 
 for i in "$@"
 do
@@ -125,6 +126,9 @@ case "${i}" in
     --timeout=*)
         args="$args -timeout ${i#*=}"
         ;;
+    --hs_root=*)
+        hs_root="${i#*=}"
+        cd $hs_root
 esac
 done
 
@@ -183,9 +187,12 @@ if [ $leak_check_set -eq 1 ]; then
 fi
 
 if [ -n "${BUILD_NUMBER}" ]; then
-       ginkgo_args="$ginkgo_args --no-color"
+        ginkgo_args="$ginkgo_args --no-color"
 fi
 
+mkdir -p .go_cache
+make build-go
+
 mkdir -p summary
 # shellcheck disable=SC2086
 CMD="sudo -E go run github.com/onsi/ginkgo/v2/ginkgo --json-report=summary/report.json $ginkgo_args -- $args"
diff --git a/extras/hs-test/infra/container.go b/extras/hs-test/infra/container.go
index 0426516ae1c..5a3f9f3aa2f 100644
--- a/extras/hs-test/infra/container.go
+++ b/extras/hs-test/infra/container.go
@@ -129,7 +129,8 @@ func (c *Container) GetContainerWorkDir() (res string) {
 }
 
 func (c *Container) getContainerArguments() string {
-	args := "--ulimit nofile=90000:90000 --cap-add=all --privileged --network host"
+	args := "--ulimit nofile=90000:90000 --cap-add=NET_ADMIN --cap-add=SYS_RESOURCE " +
+		"--cap-add=IPC_LOCK --device /dev/net/tun:/dev/net/tun --device /dev/vhost-net:/dev/vhost-net"
 	args += c.getVolumesAsCliOption()
 	args += c.getEnvVarsAsCliOption()
 	if *VppSourceFileDir != "" {
@@ -170,10 +171,9 @@ func (c *Container) Create() error {
 	resp, err := c.Suite.Docker.ContainerCreate(
 		c.ctx,
 		&containerTypes.Config{
-			Hostname: c.Name,
-			Image:    c.Image,
-			Env:      c.getEnvVars(),
-			Cmd:      strings.Split(c.ExtraRunningArgs, " "),
+			Image: c.Image,
+			Env:   c.getEnvVars(),
+			Cmd:   strings.Split(c.ExtraRunningArgs, " "),
 		},
 		&containerTypes.HostConfig{
 			Resources: containerTypes.Resources{
@@ -185,10 +185,21 @@ func (c *Container) Create() error {
 					},
 				},
 				CpusetCpus: cpuSet,
+				Devices: []containerTypes.DeviceMapping{
+					{
+						PathOnHost:        "/dev/net/tun",
+						PathInContainer:   "/dev/net/tun",
+						CgroupPermissions: "rwm",
+					},
+					{
+						PathOnHost:        "/dev/vhost-net",
+						PathInContainer:   "dev/vhost-net",
+						CgroupPermissions: "rwm",
+					},
+				},
 			},
-			CapAdd:      []string{"ALL"},
-			Privileged:  true,
-			NetworkMode: "host",
+			CapAdd:      []string{"NET_ADMIN", "SYS_RESOURCE", "IPC_LOCK"},
+			NetworkMode: "container:ginkgo",
 			Binds:       c.getVolumesAsSlice(),
 		},
 		nil,
diff --git a/extras/hs-test/infra/suite_iperf_linux.go b/extras/hs-test/infra/suite_iperf_linux.go
index be41e5082a3..0e31c9c17af 100644
--- a/extras/hs-test/infra/suite_iperf_linux.go
+++ b/extras/hs-test/infra/suite_iperf_linux.go
@@ -4,7 +4,6 @@ import (
 	"reflect"
 	"runtime"
 	"strings"
-	"time"
 
 	. "fd.io/hs-test/infra/common"
 	. "github.com/onsi/ginkgo/v2"
@@ -36,7 +35,6 @@ func RegisterIperfSoloTests(tests ...func(s *IperfSuite)) {
 }
 
 func (s *IperfSuite) SetupSuite() {
-	time.Sleep(1 * time.Second)
 	s.HstSuite.SetupSuite()
 	s.ConfigureNetworkTopology("2taps")
 	s.LoadContainerTopology("2containers")
diff --git a/extras/hs-test/ldp_test.go b/extras/hs-test/ldp_test.go
index 73f10ed4227..1c057b081b3 100644
--- a/extras/hs-test/ldp_test.go
+++ b/extras/hs-test/ldp_test.go
@@ -28,16 +28,14 @@ func LdpIperfUdpVppInterruptModeTest(s *LdpSuite) {
 }
 
 func ldpIperfTcpReorder(s *LdpSuite, netInterface *NetInterface, extraIperfArgs string) {
-	cmd := exec.Command("ip", "netns", "exec", netInterface.Peer.NetworkNamespace,
-		"tc", "qdisc", "del", "dev", netInterface.Peer.Name(),
+	cmd := exec.Command("tc", "qdisc", "del", "dev", netInterface.Peer.Name(),
 		"root")
 	s.Log("defer '%s'", cmd.String())
 	defer cmd.Run()
 
 	// "10% of packets (with a correlation of 50%) will get sent immediately, others will be delayed by 10ms"
 	// https://www.man7.org/linux/man-pages/man8/tc-netem.8.html
-	cmd = exec.Command("ip", "netns", "exec", netInterface.Peer.NetworkNamespace,
-		"tc", "qdisc", "add", "dev", netInterface.Peer.Name(),
+	cmd = exec.Command("tc", "qdisc", "add", "dev", netInterface.Peer.Name(),
 		"root", "netem", "delay", "10ms", "reorder", "10%", "50%")
 	s.Log(cmd.String())
 	o, err := cmd.CombinedOutput()
diff --git a/extras/hs-test/script/build-images.sh b/extras/hs-test/script/build-images.sh
index 2ec7de3746f..a989b2fa758 100755
--- a/extras/hs-test/script/build-images.sh
+++ b/extras/hs-test/script/build-images.sh
@@ -5,6 +5,7 @@ set -e
 
 # Get default architecture for multi-arch builds
 ARCH=${OS_ARCH:-$(dpkg --print-architecture)}
+CODENAME=$(lsb_release -cs)
 
 # Set up buildx configuration
 DOCKER_BUILD_DIR="/scratch/docker-build"
@@ -84,6 +85,7 @@ build_image() {
     docker build \
         --build-arg UBUNTU_VERSION="${UBUNTU_VERSION:-22.04}" \
         --build-arg OS_ARCH="$ARCH" \
+        --build-arg CODENAME="$CODENAME" \
         --build-arg http_proxy="$HTTP_PROXY" \
         --build-arg https_proxy="$HTTP_PROXY" \
         --build-arg HTTP_PROXY="$HTTP_PROXY" \
@@ -107,6 +109,7 @@ build_image "Dockerfile.h2load" "hs-test/h2load"
 build_image "Dockerfile.curl" "hs-test/curl"
 build_image "Dockerfile.ab" "hs-test/ab"
 build_image "Dockerfile.wrk" "hs-test/wrk"
+build_image "Dockerfile.ginkgo" "hs-test/ginkgo"
 
 # Build HTTP/3 nginx if available
 echo "=== Building HTTP/3 nginx image ==="
diff --git a/extras/hs-test/script/build_hst.sh b/extras/hs-test/script/build_hst.sh
index 77d9dc7fef1..f0ad9ae7f97 100755
--- a/extras/hs-test/script/build_hst.sh
+++ b/extras/hs-test/script/build_hst.sh
@@ -22,8 +22,9 @@ fi
 LAST_STATE_FILE=".last_state_hash"
 
 # get current state hash and ubuntu version
-current_state_hash=$(ls -l "$VPP_BUILD_ROOT"/.mu_build_install_timestamp; ls -l docker | sha1sum | awk '{print $1}')
-current_state_hash=$current_state_hash$UBUNTU_VERSION$1
+ctime_hash1=$(stat -c %Z "$VPP_BUILD_ROOT"/.mu_build_install_timestamp | sha1sum | awk '{print $1}')
+ctime_hash2=$(stat -c %Z docker/* | sha1sum | awk '{print $1}')
+current_state_hash=$ctime_hash1-$ctime_hash2-$UBUNTU_VERSION$1
 
 if [ -f "$LAST_STATE_FILE" ]; then
     last_state_hash=$(cat "$LAST_STATE_FILE")
diff --git a/extras/hs-test/topo-network/2peerVeth.yaml b/extras/hs-test/topo-network/2peerVeth.yaml
index f991d8b3701..7e5f60b3dba 100644
--- a/extras/hs-test/topo-network/2peerVeth.yaml
+++ b/extras/hs-test/topo-network/2peerVeth.yaml
@@ -1,24 +1,18 @@
 ---
 devices:
-  - name: "hsns"
-    type: "netns"
-
   - name: "srv"
     type: "veth"
     preset-hw-address: "00:00:5e:00:53:01"
     peer:
       name: "srv_veth"
-      netns: "hsns"
 
   - name: "cln"
     type: "veth"
     peer:
       name: "cln_veth"
-      netns: "hsns"
 
   - name: "br"
     type: "bridge"
-    netns: "hsns"
     interfaces:
       - srv_veth
       - cln_veth
diff --git a/extras/hs-test/topo-network/2peerVeth6.yaml b/extras/hs-test/topo-network/2peerVeth6.yaml
index 139d9964d08..f95183152f4 100644
--- a/extras/hs-test/topo-network/2peerVeth6.yaml
+++ b/extras/hs-test/topo-network/2peerVeth6.yaml
@@ -1,26 +1,20 @@
 ---
 devices:
-  - name: "hsns"
-    type: "netns"
-
   - name: "srv"
     ipv6: true
     type: "veth"
     preset-hw-address: "00:00:5e:00:53:01"
     peer:
       name: "srv_veth"
-      netns: "hsns"
 
   - name: "cln"
     ipv6: true
     type: "veth"
     peer:
       name: "cln_veth"
-      netns: "hsns"
 
   - name: "br"
     type: "bridge"
-    netns: "hsns"
     interfaces:
       - srv_veth
       - cln_veth