From 608d0069d98579b0635be978dea8e316f77a8841 Mon Sep 17 00:00:00 2001 From: Filip Tehlar Date: Fri, 28 Apr 2023 10:29:47 +0200 Subject: [PATCH] hs-test: support for multiple workers Type: test Signed-off-by: Filip Tehlar Change-Id: Ie90e4b02c268bc3ca40171b03829f5686fb83162 --- extras/hs-test/Makefile | 7 +++- extras/hs-test/container.go | 10 ++---- extras/hs-test/cpu.go | 69 ++++++++++++++++++++++++++++++++++++ extras/hs-test/hst_suite.go | 37 +++++++++++++++++-- extras/hs-test/netconfig.go | 2 +- extras/hs-test/suite_nginx_test.go | 14 ++++---- extras/hs-test/suite_no_topo_test.go | 13 ++++--- extras/hs-test/suite_ns_test.go | 13 ++++--- extras/hs-test/suite_tap_test.go | 2 +- extras/hs-test/suite_veth_test.go | 19 +++++----- extras/hs-test/test | 3 ++ extras/hs-test/utils.go | 2 +- extras/hs-test/vppinstance.go | 30 ++++++++++++++-- 13 files changed, 173 insertions(+), 48 deletions(-) create mode 100644 extras/hs-test/cpu.go diff --git a/extras/hs-test/Makefile b/extras/hs-test/Makefile index 14d95fc060a..3d7673a7267 100644 --- a/extras/hs-test/Makefile +++ b/extras/hs-test/Makefile @@ -19,6 +19,10 @@ ifeq ($(DEBUG),) DEBUG=false endif +ifeq ($(CPUS),) +CPUS=1 +endif + ifeq ($(UBUNTU_CODENAME),) UBUNTU_CODENAME=$(shell grep '^UBUNTU_CODENAME=' /etc/os-release | cut -f2- -d=) endif @@ -47,6 +51,7 @@ help: @echo " UNCONFIGURE=[true|false] - unconfigure selected test" @echo " DEBUG=[true|false] - attach VPP to GDB" @echo " TEST=[test-name] - specific test to run" + @echo " CPUS=[n-cpus] - number of cpus to run with vpp" @echo @echo "List of all tests:" $(call list_tests) @@ -64,7 +69,7 @@ build-vpp-debug: .PHONY: test test: .deps.ok .build.vpp @bash ./test --persist=$(PERSIST) --verbose=$(VERBOSE) \ - --unconfigure=$(UNCONFIGURE) --debug=$(DEBUG) --test=$(TEST) + --unconfigure=$(UNCONFIGURE) --debug=$(DEBUG) --test=$(TEST) --cpus=$(CPUS) build-go: go build ./tools/http_server diff --git a/extras/hs-test/container.go b/extras/hs-test/container.go index 1dc49b763a5..fd3aa47b21b 100644 --- a/extras/hs-test/container.go +++ b/extras/hs-test/container.go @@ -216,16 +216,12 @@ func (c *Container) getEnvVarsAsCliOption() string { return cliOption } -func (c *Container) newVppInstance(additionalConfig ...Stanza) (*VppInstance, error) { +func (c *Container) newVppInstance(cpus []int, additionalConfigs ...Stanza) (*VppInstance, error) { vpp := new(VppInstance) vpp.container = c - - if len(additionalConfig) > 0 { - vpp.additionalConfig = additionalConfig[0] - } - + vpp.cpus = cpus + vpp.additionalConfig = append(vpp.additionalConfig, additionalConfigs...) c.vppInstance = vpp - return vpp, nil } diff --git a/extras/hs-test/cpu.go b/extras/hs-test/cpu.go new file mode 100644 index 00000000000..e17bc11fbe0 --- /dev/null +++ b/extras/hs-test/cpu.go @@ -0,0 +1,69 @@ +package main + +import ( + "bufio" + "fmt" + "os" +) + +var CPU_PATH = "/sys/fs/cgroup/cpuset.cpus.effective" + +type CpuContext struct { + cpuAllocator *CpuAllocatorT + cpus []int +} + +func (c *CpuContext) Release() { + c.cpuAllocator.cpus = append(c.cpuAllocator.cpus, c.cpus...) + c.cpus = c.cpus[:0] // empty the list +} + +type CpuAllocatorT struct { + cpus []int +} + +var cpuAllocator *CpuAllocatorT = nil + +func (c *CpuAllocatorT) Allocate(nCpus int) (*CpuContext, error) { + var cpuCtx CpuContext + + if len(c.cpus) < nCpus { + return nil, fmt.Errorf("could not allocate %d CPUs; available: %d", nCpus, len(c.cpus)) + } + cpuCtx.cpus = c.cpus[0:nCpus] + cpuCtx.cpuAllocator = c + c.cpus = c.cpus[nCpus:] + return &cpuCtx, nil +} + +func (c *CpuAllocatorT) readCpus(fname string) error { + var first, last int + file, err := os.Open(CPU_PATH) + if err != nil { + return err + } + defer file.Close() + + sc := bufio.NewScanner(file) + sc.Scan() + line := sc.Text() + _, err = fmt.Sscanf(line, "%d-%d", &first, &last) + if err != nil { + return err + } + for i := first; i <= last; i++ { + c.cpus = append(c.cpus, i) + } + return nil +} + +func CpuAllocator() (*CpuAllocatorT, error) { + if cpuAllocator == nil { + cpuAllocator = new(CpuAllocatorT) + err := cpuAllocator.readCpus(CPU_PATH) + if err != nil { + return nil, err + } + } + return cpuAllocator, nil +} diff --git a/extras/hs-test/hst_suite.go b/extras/hs-test/hst_suite.go index 042b4fe6b24..1fcffa42e00 100644 --- a/extras/hs-test/hst_suite.go +++ b/extras/hs-test/hst_suite.go @@ -14,13 +14,14 @@ import ( ) const ( - defaultNetworkNumber int = 1 + DEFAULT_NETWORK_NUM int = 1 ) var isPersistent = flag.Bool("persist", false, "persists topology config") var isVerbose = flag.Bool("verbose", false, "verbose test output") var isUnconfiguring = flag.Bool("unconfigure", false, "remove topology") var isVppDebug = flag.Bool("debug", false, "attach gdb to vpp") +var nConfiguredCpus = flag.Int("cpus", 1, "number of CPUs assigned to vpp") type HstSuite struct { suite.Suite @@ -30,6 +31,29 @@ type HstSuite struct { netInterfaces map[string]*NetInterface addresser *Addresser testIds map[string]string + cpuAllocator *CpuAllocatorT + cpuContexts []*CpuContext + cpuPerVpp int +} + +func (s *HstSuite) SetupSuite() { + var err error + s.cpuAllocator, err = CpuAllocator() + if err != nil { + s.FailNow("failed to init cpu allocator: %v", err) + } + s.cpuPerVpp = *nConfiguredCpus +} + +func (s *HstSuite) AllocateCpus() []int { + cpuCtx, err := s.cpuAllocator.Allocate(s.cpuPerVpp) + s.assertNil(err) + s.AddCpuContext(cpuCtx) + return cpuCtx.cpus +} + +func (s *HstSuite) AddCpuContext(cpuCtx *CpuContext) { + s.cpuContexts = append(s.cpuContexts, cpuCtx) } func (s *HstSuite) TearDownSuite() { @@ -40,6 +64,9 @@ func (s *HstSuite) TearDownTest() { if *isPersistent { return } + for _, c := range s.cpuContexts { + c.Release() + } s.resetContainers() s.removeVolumes() } @@ -66,7 +93,7 @@ func (s *HstSuite) setupVolumes() { func (s *HstSuite) setupContainers() { for _, container := range s.containers { - if container.isOptional == false { + if !container.isOptional { container.run() } } @@ -130,6 +157,12 @@ func (s *HstSuite) skip(args ...any) { s.T().SkipNow() } +func (s *HstSuite) SkipIfMultiWorker(args ...any) { + if *nConfiguredCpus > 1 { + s.skip("test case not supported with multiple vpp workers") + } +} + func (s *HstSuite) resetContainers() { for _, container := range s.containers { container.stop() diff --git a/extras/hs-test/netconfig.go b/extras/hs-test/netconfig.go index 9e259ab1c06..6059b7b8567 100644 --- a/extras/hs-test/netconfig.go +++ b/extras/hs-test/netconfig.go @@ -77,7 +77,7 @@ func newNetworkInterface(cfg NetDevConfig, a *Addresser) (*NetInterface, error) var err error newInterface.addresser = a newInterface.name = cfg["name"].(string) - newInterface.networkNumber = defaultNetworkNumber + newInterface.networkNumber = DEFAULT_NETWORK_NUM if interfaceType, ok := cfg["type"]; ok { newInterface.category = interfaceType.(string) diff --git a/extras/hs-test/suite_nginx_test.go b/extras/hs-test/suite_nginx_test.go index b61ea6c8b0d..c250ed7843e 100644 --- a/extras/hs-test/suite_nginx_test.go +++ b/extras/hs-test/suite_nginx_test.go @@ -14,27 +14,25 @@ type NginxSuite struct { } func (s *NginxSuite) SetupSuite() { + s.HstSuite.SetupSuite() s.loadNetworkTopology("2taps") - s.loadContainerTopology("nginxProxyAndServer") } func (s *NginxSuite) SetupTest() { - s.skipIfUnconfiguring() - - s.setupVolumes() - s.setupContainers() + s.HstSuite.SetupTest() // Setup test conditions - var startupConfig Stanza - startupConfig. + var sessionConfig Stanza + sessionConfig. newStanza("session"). append("enable"). append("use-app-socket-api").close() + cpus := s.AllocateCpus() // ... for proxy vppProxyContainer := s.getContainerByName(vppProxyContainerName) - proxyVpp, _ := vppProxyContainer.newVppInstance(startupConfig) + proxyVpp, _ := vppProxyContainer.newVppInstance(cpus, sessionConfig) proxyVpp.start() clientInterface := s.netInterfaces[mirroringClientInterfaceName] diff --git a/extras/hs-test/suite_no_topo_test.go b/extras/hs-test/suite_no_topo_test.go index 8ef56b24d8b..8f7c87620ce 100644 --- a/extras/hs-test/suite_no_topo_test.go +++ b/extras/hs-test/suite_no_topo_test.go @@ -12,25 +12,24 @@ type NoTopoSuite struct { } func (s *NoTopoSuite) SetupSuite() { + s.HstSuite.SetupSuite() s.loadNetworkTopology("tap") - s.loadContainerTopology("single") } func (s *NoTopoSuite) SetupTest() { - s.skipIfUnconfiguring() - s.setupVolumes() - s.setupContainers() + s.HstSuite.SetupTest() // Setup test conditions - var startupConfig Stanza - startupConfig. + var sessionConfig Stanza + sessionConfig. newStanza("session"). append("enable"). append("use-app-socket-api").close() + cpus := s.AllocateCpus() container := s.getContainerByName(singleTopoContainerVpp) - vpp, _ := container.newVppInstance(startupConfig) + vpp, _ := container.newVppInstance(cpus, sessionConfig) vpp.start() tapInterface := s.netInterfaces[tapInterfaceName] diff --git a/extras/hs-test/suite_ns_test.go b/extras/hs-test/suite_ns_test.go index 34fc9ec9723..3bf3cc7e3ed 100644 --- a/extras/hs-test/suite_ns_test.go +++ b/extras/hs-test/suite_ns_test.go @@ -11,27 +11,26 @@ type NsSuite struct { } func (s *NsSuite) SetupSuite() { + s.HstSuite.SetupSuite() s.configureNetworkTopology("ns") - s.loadContainerTopology("ns") } func (s *NsSuite) SetupTest() { - s.skipIfUnconfiguring() - s.setupVolumes() - s.setupContainers() + s.HstSuite.SetupTest() // Setup test conditions - var startupConfig Stanza - startupConfig. + var sessionConfig Stanza + sessionConfig. newStanza("session"). append("enable"). append("use-app-socket-api"). append("evt_qs_memfd_seg"). append("event-queue-length 100000").close() + cpus := s.AllocateCpus() container := s.getContainerByName("vpp") - vpp, _ := container.newVppInstance(startupConfig) + vpp, _ := container.newVppInstance(cpus, sessionConfig) vpp.start() idx, err := vpp.createAfPacket(s.netInterfaces[serverInterface]) diff --git a/extras/hs-test/suite_tap_test.go b/extras/hs-test/suite_tap_test.go index 96f475c4c5a..8b0950a797e 100644 --- a/extras/hs-test/suite_tap_test.go +++ b/extras/hs-test/suite_tap_test.go @@ -10,6 +10,6 @@ type TapSuite struct { func (s *TapSuite) SetupSuite() { time.Sleep(1 * time.Second) - + s.HstSuite.SetupSuite() s.configureNetworkTopology("tap") } diff --git a/extras/hs-test/suite_veth_test.go b/extras/hs-test/suite_veth_test.go index be79ce250fb..bb703df6981 100644 --- a/extras/hs-test/suite_veth_test.go +++ b/extras/hs-test/suite_veth_test.go @@ -16,22 +16,18 @@ type VethsSuite struct { func (s *VethsSuite) SetupSuite() { time.Sleep(1 * time.Second) - + s.HstSuite.SetupSuite() s.configureNetworkTopology("2peerVeth") - s.loadContainerTopology("2peerVeth") } func (s *VethsSuite) SetupTest() { - s.skipIfUnconfiguring() - - s.setupVolumes() - s.setupContainers() + s.HstSuite.SetupTest() // Setup test conditions - var startupConfig Stanza - startupConfig. + var sessionConfig Stanza + sessionConfig. newStanza("session"). append("enable"). append("use-app-socket-api").close() @@ -39,7 +35,8 @@ func (s *VethsSuite) SetupTest() { // ... For server serverContainer := s.getContainerByName("server-vpp") - serverVpp, _ := serverContainer.newVppInstance(startupConfig) + cpus := s.AllocateCpus() + serverVpp, _ := serverContainer.newVppInstance(cpus, sessionConfig) s.assertNotNil(serverVpp) s.setupServerVpp() @@ -47,7 +44,8 @@ func (s *VethsSuite) SetupTest() { // ... For client clientContainer := s.getContainerByName("client-vpp") - clientVpp, _ := clientContainer.newVppInstance(startupConfig) + cpus = s.AllocateCpus() + clientVpp, _ := clientContainer.newVppInstance(cpus, sessionConfig) s.assertNotNil(clientVpp) s.setupClientVpp() @@ -67,7 +65,6 @@ func (s *VethsSuite) setupServerVpp() { namespaceSecret := "1" err = serverVpp.addAppNamespace(1, idx, namespaceSecret) s.assertNil(err) - } func (s *VethsSuite) setupClientVpp() { diff --git a/extras/hs-test/test b/extras/hs-test/test index a8866524b58..db53d5a447e 100755 --- a/extras/hs-test/test +++ b/extras/hs-test/test @@ -38,6 +38,9 @@ case "${i}" in unconfigure_set=1 fi ;; + --cpus=*) + args="$args -cpus ${i#*=}" + ;; --test=*) tc_name="${i#*=}" if [ $tc_name != "all" ]; then diff --git a/extras/hs-test/utils.go b/extras/hs-test/utils.go index 151567c3fed..4261f4dc2d1 100644 --- a/extras/hs-test/utils.go +++ b/extras/hs-test/utils.go @@ -126,7 +126,7 @@ func startWget(finished chan error, server_ip, port, query, netNs string) { if err != nil { finished <- fmt.Errorf("wget error: '%v\n\n%s'", err, o) return - } else if strings.Contains(string(o), "200 OK") == false { + } else if !strings.Contains(string(o), "200 OK") { finished <- fmt.Errorf("wget error: response not 200 OK") return } diff --git a/extras/hs-test/vppinstance.go b/extras/hs-test/vppinstance.go index c08514e22dc..a9b97bcaa0b 100644 --- a/extras/hs-test/vppinstance.go +++ b/extras/hs-test/vppinstance.go @@ -72,9 +72,10 @@ const ( type VppInstance struct { container *Container - additionalConfig Stanza + additionalConfig []Stanza connection *core.Connection apiChannel api.Channel + cpus []int } func (vpp *VppInstance) getSuite() *HstSuite { @@ -113,7 +114,10 @@ func (vpp *VppInstance) start() error { defaultApiSocketFilePath, defaultLogFilePath, ) - configContent += vpp.additionalConfig.toString() + configContent += vpp.generateCpuConfig() + for _, c := range vpp.additionalConfig { + configContent += c.toString() + } startupFileName := vpp.getEtcDir() + "/startup.conf" vpp.container.createFile(startupFileName, configContent) @@ -341,3 +345,25 @@ func (vpp *VppInstance) disconnect() { vpp.connection.Disconnect() vpp.apiChannel.Close() } + +func (vpp *VppInstance) generateCpuConfig() string { + var c Stanza + var s string + if len(vpp.cpus) < 1 { + return "" + } + c.newStanza("cpu"). + append(fmt.Sprintf("main-core %d", vpp.cpus[0])) + workers := vpp.cpus[1:] + + if len(workers) > 0 { + for i := 0; i < len(workers); i++ { + if i != 0 { + s = s + ", " + } + s = s + fmt.Sprintf("%d", workers[i]) + } + c.append(fmt.Sprintf("corelist-workers %s", s)) + } + return c.close().toString() +} -- 2.16.6