From 9530f52b44df0c8673f9e2070b3e4591a51b77d3 Mon Sep 17 00:00:00 2001 From: Adrian Villin Date: Wed, 20 Aug 2025 12:46:12 +0200 Subject: [PATCH] hs-test: monitor performance of some loss tests + updated asserts Type: test Change-Id: I4aec44dde3e6c0472fca9fe6a7f80c068070f92e Signed-off-by: Adrian Villin --- extras/hs-test/echo_test.go | 105 +++++++++++++++++++++++----- extras/hs-test/http1_test.go | 4 +- extras/hs-test/http2_test.go | 4 +- extras/hs-test/infra/common/suite_common.go | 50 ++++++++----- extras/hs-test/infra/suite_veth.go | 38 +++++++++- extras/hs-test/infra/suite_veth6.go | 4 +- extras/hs-test/infra/utils.go | 11 +++ extras/hs-test/nsim_test.go | 16 ++++- extras/hs-test/proxy_test.go | 4 +- 9 files changed, 188 insertions(+), 48 deletions(-) diff --git a/extras/hs-test/echo_test.go b/extras/hs-test/echo_test.go index 65bd49f825b..0d6c5eb13a3 100644 --- a/extras/hs-test/echo_test.go +++ b/extras/hs-test/echo_test.go @@ -9,8 +9,8 @@ import ( func init() { RegisterVethTests(EchoBuiltinTest, EchoBuiltinBandwidthTest, EchoBuiltinEchobytesTest, EchoBuiltinRoundtripTest, EchoBuiltinTestbytesTest) - RegisterSoloVethTests(TcpWithLossTest) - RegisterVeth6Tests(TcpWithLoss6Test) + RegisterVethMWTests(TcpWithLossTest) + RegisterSoloVeth6Tests(TcpWithLoss6Test) } func EchoBuiltinTest(s *VethsSuite) { @@ -118,7 +118,7 @@ func EchoBuiltinTestbytesTest(s *VethsSuite) { s.AssertContains(o, " bytes out of 32768 sent (32768 target)") } -func TcpWithLossTest(s *VethsSuite) { +func tcpWithoutLoss(s *VethsSuite) string { serverVpp := s.Containers.ServerVpp.VppInstance serverVpp.Vppctl("test echo server uri tcp://%s/"+s.Ports.Port1, @@ -126,38 +126,107 @@ func TcpWithLossTest(s *VethsSuite) { clientVpp := s.Containers.ClientVpp.VppInstance - // Add loss of packets with Network Delay Simulator - s.Log(clientVpp.Vppctl("set nsim poll-main-thread delay 0.01 ms bandwidth 40 gbit" + - " packet-size 1400 packets-per-drop 1000")) - - s.Log(clientVpp.Vppctl("nsim output-feature enable-disable host-" + s.Interfaces.Client.Name())) - // Do echo test from client-vpp container - output := clientVpp.Vppctl("test echo client uri tcp://%s/%s verbose echo-bytes bytes 50m", + output := clientVpp.Vppctl("test echo client uri tcp://%s/%s verbose echo-bytes run-time 10 test-timeout 20", s.Interfaces.Server.Ip4AddressString(), s.Ports.Port1) s.Log(output) s.AssertNotEqual(len(output), 0) s.AssertNotContains(output, "failed", output) + + return output } -func TcpWithLoss6Test(s *Veths6Suite) { +func TcpWithLossTest(s *VethsSuite) { + s.CpusPerVppContainer = 2 + s.CpusPerContainer = 1 + s.SetupTest() + clientVpp := s.Containers.ClientVpp.VppInstance serverVpp := s.Containers.ServerVpp.VppInstance - serverVpp.Vppctl("test echo server uri tcp://%s/%s", - s.Interfaces.Server.Ip6AddressString(), s.Ports.Port1) + s.Log(clientVpp.Vppctl("set nsim poll-main-thread delay 10 ms bandwidth 40 gbit")) + s.Log(clientVpp.Vppctl("nsim output-feature enable-disable host-" + s.Interfaces.Client.Name())) - clientVpp := s.Containers.ClientVpp.VppInstance + s.Log(" * running TcpWithoutLoss") + output := tcpWithoutLoss(s) + baseline, err := s.ParseEchoClientTransfer(output) + s.AssertNil(err) - // Add loss of packets with Network Delay Simulator - s.Log(clientVpp.Vppctl("set nsim poll-main-thread delay 0.01 ms bandwidth 40 gbit" + - " packet-size 1400 packets-per-drop 1000")) + clientVpp.Disconnect() + clientVpp.Stop() + s.SetupClientVpp() + serverVpp.Disconnect() + serverVpp.Stop() + s.SetupServerVpp() + // Add loss of packets with Network Delay Simulator + s.Log(clientVpp.Vppctl("set nsim poll-main-thread delay 10 ms bandwidth 40 gbit" + + " packet-size 1400 drop-fraction 0.033")) s.Log(clientVpp.Vppctl("nsim output-feature enable-disable host-" + s.Interfaces.Client.Name())) + s.Log(" * running TcpWithLoss") + output = tcpWithoutLoss(s) + + withLoss, err := s.ParseEchoClientTransfer(output) + s.AssertNil(err) + + if !s.CoverageRun { + s.Log("\nBaseline: %v gbit/s\nWith loss: %v gbit/s", baseline, withLoss) + s.AssertGreaterEqual(baseline, withLoss) + s.AssertGreaterEqual(withLoss, baseline*0.2) + } +} + +func tcpWithoutLoss6(s *Veths6Suite) string { + serverVpp := s.Containers.ServerVpp.VppInstance + + serverVpp.Vppctl("test echo server uri tcp://%s/"+s.Ports.Port1, + s.Interfaces.Server.Ip6AddressString()) + + clientVpp := s.Containers.ClientVpp.VppInstance + // Do echo test from client-vpp container - output := clientVpp.Vppctl("test echo client uri tcp://%s/%s verbose echo-bytes bytes 50m", + output := clientVpp.Vppctl("test echo client uri tcp://%s/%s verbose echo-bytes run-time 10 test-timeout 20", s.Interfaces.Server.Ip6AddressString(), s.Ports.Port1) s.Log(output) s.AssertNotEqual(len(output), 0) s.AssertNotContains(output, "failed", output) + + return output +} + +func TcpWithLoss6Test(s *Veths6Suite) { + clientVpp := s.Containers.ClientVpp.VppInstance + serverVpp := s.Containers.ServerVpp.VppInstance + s.Log(clientVpp.Vppctl("set nsim poll-main-thread delay 10 ms bandwidth 40 gbit")) + s.Log(clientVpp.Vppctl("nsim output-feature enable-disable host-" + s.Interfaces.Client.Name())) + + s.Log(" * running TcpWithoutLoss") + output := tcpWithoutLoss6(s) + baseline, err := s.ParseEchoClientTransfer(output) + s.AssertNil(err) + + clientVpp.Disconnect() + clientVpp.Stop() + s.SetupClientVpp() + serverVpp.Disconnect() + serverVpp.Stop() + s.SetupServerVpp() + + // Add loss of packets with Network Delay Simulator + s.Log(clientVpp.Vppctl("set nsim poll-main-thread delay 10 ms bandwidth 40 gbit" + + " packet-size 1400 drop-fraction 0.033")) + + s.Log(clientVpp.Vppctl("nsim output-feature enable-disable host-" + s.Interfaces.Client.Name())) + + s.Log(" * running TcpWithLoss") + output = tcpWithoutLoss6(s) + + withLoss, err := s.ParseEchoClientTransfer(output) + s.AssertNil(err) + + if !s.CoverageRun { + s.Log("Baseline: %v gbit/s\nWith loss: %v gbit/s", baseline, withLoss) + s.AssertGreaterEqual(baseline, withLoss) + s.AssertGreaterEqual(withLoss, baseline*0.2) + } } diff --git a/extras/hs-test/http1_test.go b/extras/hs-test/http1_test.go index 6e0f9d7a800..dbbd5d54b8f 100644 --- a/extras/hs-test/http1_test.go +++ b/extras/hs-test/http1_test.go @@ -698,7 +698,7 @@ func httpClientRepeat(s *Http1Suite, requestMethod string, clientArgs string) { s.Log("Server response count: %d", replyCountInt) s.AssertNotNil(o) s.AssertNotContains(o, "error") - s.AssertGreaterThan(replyCountInt, 15000) + s.AssertGreaterEqual(replyCountInt, 15000) replyCount = "" cmd = fmt.Sprintf("http client %s %s repeat %d header Hello:World uri %s", @@ -835,7 +835,7 @@ func HttpStaticPromTest(s *Http1Suite) { s.Log(DumpHttpResp(resp, false)) s.AssertHttpStatus(resp, 200) s.AssertHttpHeaderWithValue(resp, "Content-Type", "text/plain") - s.AssertGreaterThan(resp.ContentLength, 0) + s.AssertGreaterEqual(resp.ContentLength, 0) _, err = io.ReadAll(resp.Body) s.AssertNil(err, fmt.Sprint(err)) } diff --git a/extras/hs-test/http2_test.go b/extras/hs-test/http2_test.go index 82230a0643e..eaf18a91a1a 100644 --- a/extras/hs-test/http2_test.go +++ b/extras/hs-test/http2_test.go @@ -125,7 +125,7 @@ func Http2ContinuationTxTest(s *Http2Suite) { s.AssertContains(log, "[64 bytes data]") sizeHeader, err := strconv.Atoi(strings.ReplaceAll(writeOut, "\x00", "")) s.AssertNil(err, fmt.Sprint(err)) - s.AssertGreaterThan(sizeHeader, 32768) + s.AssertGreaterEqual(sizeHeader, 32768) } func Http2ServerMemLeakTest(s *Http2Suite) { @@ -339,7 +339,7 @@ func Http2ClientContinuationTest(s *VethsSuite) { o := s.Containers.ClientVpp.VppInstance.Vppctl("http client fifo-size 64k verbose save-to response.txt uri " + uri) s.Log(o) s.AssertContains(o, "HTTP/2 200 OK") - s.AssertGreaterThan(strings.Count(o, "x"), 32768) + s.AssertGreaterEqual(strings.Count(o, "x"), 32768) } func Http2ClientMemLeakTest(s *Http2Suite) { diff --git a/extras/hs-test/infra/common/suite_common.go b/extras/hs-test/infra/common/suite_common.go index 2b7560daccb..be9a0a6ee37 100644 --- a/extras/hs-test/infra/common/suite_common.go +++ b/extras/hs-test/infra/common/suite_common.go @@ -105,71 +105,83 @@ func (s *HstCommon) Log(log any, arg ...any) { } } -func (s *HstCommon) AssertNil(object interface{}, msgAndArgs ...interface{}) { +func (s *HstCommon) AssertNil(object any, msgAndArgs ...any) { ExpectWithOffset(2, object).To(BeNil(), msgAndArgs...) } -func (s *HstCommon) AssertNotNil(object interface{}, msgAndArgs ...interface{}) { +func (s *HstCommon) AssertNotNil(object any, msgAndArgs ...any) { ExpectWithOffset(2, object).ToNot(BeNil(), msgAndArgs...) } -func (s *HstCommon) AssertEqual(expected, actual interface{}, msgAndArgs ...interface{}) { +func (s *HstCommon) AssertEqual(expected, actual any, msgAndArgs ...any) { ExpectWithOffset(2, actual).To(Equal(expected), msgAndArgs...) } -func (s *HstCommon) AssertNotEqual(expected, actual interface{}, msgAndArgs ...interface{}) { +func (s *HstCommon) AssertNotEqual(expected, actual any, msgAndArgs ...any) { ExpectWithOffset(2, actual).ToNot(Equal(expected), msgAndArgs...) } -func (s *HstCommon) AssertContains(testString, contains interface{}, msgAndArgs ...interface{}) { +func (s *HstCommon) AssertContains(testString, contains any, msgAndArgs ...any) { ExpectWithOffset(2, strings.ToLower(fmt.Sprint(testString))).To(ContainSubstring(strings.ToLower(fmt.Sprint(contains))), msgAndArgs...) } -func (s *HstCommon) AssertNotContains(testString, contains interface{}, msgAndArgs ...interface{}) { +func (s *HstCommon) AssertNotContains(testString, contains any, msgAndArgs ...any) { ExpectWithOffset(2, strings.ToLower(fmt.Sprint(testString))).ToNot(ContainSubstring(strings.ToLower(fmt.Sprint(contains))), msgAndArgs...) } -func (s *HstCommon) AssertEmpty(object interface{}, msgAndArgs ...interface{}) { +func (s *HstCommon) AssertEmpty(object any, msgAndArgs ...any) { ExpectWithOffset(2, object).To(BeEmpty(), msgAndArgs...) } -func (s *HstCommon) AssertNotEmpty(object interface{}, msgAndArgs ...interface{}) { +func (s *HstCommon) AssertNotEmpty(object any, msgAndArgs ...any) { ExpectWithOffset(2, object).ToNot(BeEmpty(), msgAndArgs...) } -func (s *HstCommon) AssertMatchError(actual, expected error, msgAndArgs ...interface{}) { +func (s *HstCommon) AssertMatchError(actual, expected error, msgAndArgs ...any) { ExpectWithOffset(2, actual).To(MatchError(expected), msgAndArgs...) } -func (s *HstCommon) AssertGreaterThan(actual, expected interface{}, msgAndArgs ...interface{}) { +func (s *HstCommon) AssertGreaterEqual(actual, expected any, msgAndArgs ...any) { ExpectWithOffset(2, actual).Should(BeNumerically(">=", expected), msgAndArgs...) } -func (s *HstCommon) AssertEqualWithinThreshold(actual, expected, threshold interface{}, msgAndArgs ...interface{}) { +func (s *HstCommon) AssertGreaterThan(actual, expected any, msgAndArgs ...any) { + ExpectWithOffset(2, actual).Should(BeNumerically(">", expected), msgAndArgs...) +} + +func (s *HstCommon) AssertLessEqual(actual, expected any, msgAndArgs ...any) { + ExpectWithOffset(2, actual).Should(BeNumerically("<=", expected), msgAndArgs...) +} + +func (s *HstCommon) AssertLessThan(actual, expected any, msgAndArgs ...any) { + ExpectWithOffset(2, actual).Should(BeNumerically("<", expected), msgAndArgs...) +} + +func (s *HstCommon) AssertEqualWithinThreshold(actual, expected, threshold any, msgAndArgs ...any) { ExpectWithOffset(2, actual).Should(BeNumerically("~", expected, threshold), msgAndArgs...) } -func (s *HstCommon) AssertTimeEqualWithinThreshold(actual, expected time.Time, threshold time.Duration, msgAndArgs ...interface{}) { +func (s *HstCommon) AssertTimeEqualWithinThreshold(actual, expected time.Time, threshold time.Duration, msgAndArgs ...any) { ExpectWithOffset(2, actual).Should(BeTemporally("~", expected, threshold), msgAndArgs...) } -func (s *HstCommon) AssertHttpStatus(resp *http.Response, expectedStatus int, msgAndArgs ...interface{}) { +func (s *HstCommon) AssertHttpStatus(resp *http.Response, expectedStatus int, msgAndArgs ...any) { ExpectWithOffset(2, resp).To(HaveHTTPStatus(expectedStatus), msgAndArgs...) } -func (s *HstCommon) AssertHttpHeaderWithValue(resp *http.Response, key string, value interface{}, msgAndArgs ...interface{}) { +func (s *HstCommon) AssertHttpHeaderWithValue(resp *http.Response, key string, value any, msgAndArgs ...any) { ExpectWithOffset(2, resp).To(HaveHTTPHeaderWithValue(key, value), msgAndArgs...) } -func (s *HstCommon) AssertHttpHeaderNotPresent(resp *http.Response, key string, msgAndArgs ...interface{}) { +func (s *HstCommon) AssertHttpHeaderNotPresent(resp *http.Response, key string, msgAndArgs ...any) { ExpectWithOffset(2, resp.Header.Get(key)).To(BeEmpty(), msgAndArgs...) } -func (s *HstCommon) AssertHttpContentLength(resp *http.Response, expectedContentLen int64, msgAndArgs ...interface{}) { +func (s *HstCommon) AssertHttpContentLength(resp *http.Response, expectedContentLen int64, msgAndArgs ...any) { ExpectWithOffset(2, resp).To(HaveHTTPHeaderWithValue("Content-Length", strconv.FormatInt(expectedContentLen, 10)), msgAndArgs...) } -func (s *HstCommon) AssertHttpBody(resp *http.Response, expectedBody string, msgAndArgs ...interface{}) { +func (s *HstCommon) AssertHttpBody(resp *http.Response, expectedBody string, msgAndArgs ...any) { ExpectWithOffset(2, resp).To(HaveHTTPBody(expectedBody), msgAndArgs...) } @@ -190,8 +202,8 @@ func (s *HstCommon) AssertIperfMinTransfer(result IPerfResult, minTransferred in return } if result.Start.Details.Protocol == "TCP" { - s.AssertGreaterThan(result.End.TcpReceived.MBytes, minTransferred) + s.AssertGreaterEqual(result.End.TcpReceived.MBytes, minTransferred) } else { - s.AssertGreaterThan(result.End.Udp.MBytes, minTransferred) + s.AssertGreaterEqual(result.End.Udp.MBytes, minTransferred) } } diff --git a/extras/hs-test/infra/suite_veth.go b/extras/hs-test/infra/suite_veth.go index 65506c448b3..37907f1fcb2 100644 --- a/extras/hs-test/infra/suite_veth.go +++ b/extras/hs-test/infra/suite_veth.go @@ -13,6 +13,7 @@ import ( var vethTests = map[string][]func(s *VethsSuite){} var vethSoloTests = map[string][]func(s *VethsSuite){} +var vethMWTests = map[string][]func(s *VethsSuite){} type VethsSuite struct { HstSuite @@ -38,6 +39,9 @@ func RegisterVethTests(tests ...func(s *VethsSuite)) { func RegisterSoloVethTests(tests ...func(s *VethsSuite)) { vethSoloTests[GetTestFilename()] = tests } +func RegisterVethMWTests(tests ...func(s *VethsSuite)) { + vethMWTests[GetTestFilename()] = tests +} func (s *VethsSuite) SetupSuite() { time.Sleep(1 * time.Second) @@ -83,7 +87,7 @@ func (s *VethsSuite) SetupTest() { s.AssertNotNil(clientVpp, fmt.Sprint(err)) s.SetupServerVpp() - s.setupClientVpp() + s.SetupClientVpp() if *DryRun { s.LogStartedContainers() s.Skip("Dry run mode = true") @@ -99,7 +103,7 @@ func (s *VethsSuite) SetupServerVpp() { s.AssertNotEqual(0, idx) } -func (s *VethsSuite) setupClientVpp() { +func (s *VethsSuite) SetupClientVpp() { clientVpp := s.GetContainerByName("client-vpp").VppInstance s.AssertNil(clientVpp.Start()) @@ -168,3 +172,33 @@ var _ = Describe("VethsSuiteSolo", Ordered, ContinueOnFailure, Serial, func() { } } }) + +var _ = Describe("VethsSuiteMW", Ordered, ContinueOnFailure, Serial, func() { + var s VethsSuite + BeforeAll(func() { + s.SetupSuite() + }) + BeforeEach(func() { + s.SkipIfNotEnoguhCpus = true + }) + AfterAll(func() { + s.TeardownSuite() + }) + AfterEach(func() { + s.TeardownTest() + }) + + // https://onsi.github.io/ginkgo/#dynamically-generating-specs + for filename, tests := range vethMWTests { + for _, test := range tests { + test := test + pc := reflect.ValueOf(test).Pointer() + funcValue := runtime.FuncForPC(pc) + testName := filename + "/" + strings.Split(funcValue.Name(), ".")[2] + It(testName, Label("SOLO", "VPP Multi-Worker"), func(ctx SpecContext) { + s.Log(testName + ": BEGIN") + test(&s) + }, SpecTimeout(TestTimeout)) + } + } +}) diff --git a/extras/hs-test/infra/suite_veth6.go b/extras/hs-test/infra/suite_veth6.go index 2e3fe98c3b2..a57a7526da8 100644 --- a/extras/hs-test/infra/suite_veth6.go +++ b/extras/hs-test/infra/suite_veth6.go @@ -78,7 +78,7 @@ func (s *Veths6Suite) SetupTest() { s.AssertNotNil(clientVpp, fmt.Sprint(err)) s.SetupServerVpp() - s.setupClientVpp() + s.SetupClientVpp() if *DryRun { s.LogStartedContainers() s.Skip("Dry run mode = true") @@ -94,7 +94,7 @@ func (s *Veths6Suite) SetupServerVpp() { s.AssertNotEqual(0, idx) } -func (s *Veths6Suite) setupClientVpp() { +func (s *Veths6Suite) SetupClientVpp() { clientVpp := s.GetContainerByName("client-vpp").VppInstance s.AssertNil(clientVpp.Start()) diff --git a/extras/hs-test/infra/utils.go b/extras/hs-test/infra/utils.go index 1d84a3f61ea..c00cb9bc401 100644 --- a/extras/hs-test/infra/utils.go +++ b/extras/hs-test/infra/utils.go @@ -367,3 +367,14 @@ func (s *HstSuite) StartUdpEchoServer(addr string, port int) *net.UDPConn { s.Log("* started udp echo server " + addr + ":" + strconv.Itoa(port)) return conn } + +// Parses transfer speed from the last line ("N gbit/second full-duplex") +func (s *HstSuite) ParseEchoClientTransfer(stats string) (float64, error) { + lines := strings.Split(strings.TrimSpace(stats), "\n") + parts := strings.Fields(lines[len(lines)-1]) + if len(parts) == 0 { + return 0, errors.New("check format of stats") + } + number, err := strconv.ParseFloat(parts[0], 64) + return number, err +} diff --git a/extras/hs-test/nsim_test.go b/extras/hs-test/nsim_test.go index c47e83455e7..ed9aac585a6 100644 --- a/extras/hs-test/nsim_test.go +++ b/extras/hs-test/nsim_test.go @@ -1,6 +1,9 @@ package main import ( + "errors" + "regexp" + "strconv" "strings" . "fd.io/hs-test/infra" @@ -20,5 +23,16 @@ func NsimLossTest(s *VethsSuite) { lines := strings.Split(o, "\n") stats := lines[len(lines)-2] s.Log(stats) - s.AssertContains(stats, "10% packet loss") + + re := regexp.MustCompile(`(\d+\.?\d*)\s*%\s*packet loss`) + matches := re.FindStringSubmatch(stats) + if len(matches) < 2 { + s.AssertNil(errors.New("Error when parsing stats.")) + } + packetLossStr := matches[1] + packetLoss, err := strconv.ParseFloat(packetLossStr, 64) + s.AssertNil(err) + if !s.CoverageRun { + s.AssertEqual(packetLoss, float64(10), "Packet loss != 10%%") + } } diff --git a/extras/hs-test/proxy_test.go b/extras/hs-test/proxy_test.go index 183cca72523..21681345360 100644 --- a/extras/hs-test/proxy_test.go +++ b/extras/hs-test/proxy_test.go @@ -323,7 +323,7 @@ func vppConnectProxyStressLoad(s *VppProxySuite, proxyPort string) { report += fmt.Sprintf("Errors: timeout %d, read %d, write %d, invalid data received %d, connection %d\n", timeout.Load(), readError.Load(), writeError.Load(), invalidData.Load(), connectError.Load()) report += fmt.Sprintf("Successes ratio: %.2f%%\n", successRatio) AddReportEntry(summary, report) - s.AssertGreaterThan(successRatio, 90.0) + s.AssertGreaterEqual(successRatio, 90.0) } func VppConnectProxyStressTest(s *VppProxySuite) { @@ -627,7 +627,7 @@ func vppConnectUdpStressLoad(s *VppUdpProxySuite) { report += fmt.Sprintf("Errors: timeout %d, read %d, write %d, invalid data received %d, connection %d\n", timeout.Load(), readError.Load(), writeError.Load(), invalidData.Load(), connectError.Load()) report += fmt.Sprintf("Successes ratio: %.2f%%\n", successRatio) AddReportEntry(summary, report) - s.AssertGreaterThan(successRatio, 90.0) + s.AssertGreaterEqual(successRatio, 90.0) } func VppConnectUdpStressTest(s *VppUdpProxySuite) { -- 2.16.6