hs-test: various improvements 44/41644/3
authorAdrian Villin <[email protected]>
Tue, 15 Oct 2024 12:56:16 +0000 (14:56 +0200)
committerFlorin Coras <[email protected]>
Tue, 15 Oct 2024 16:49:40 +0000 (16:49 +0000)
- fixed timed out/panicked tests not copying logs to CI archives
- fixed log formatting
- renamed SuiteTimeout to TestTimeout
- fixed ginkgo node leak on test timeout -> added AssertChannelClosed
- updated docs

Type: test

Change-Id: Ia71d765bd61576230a4cfd26d4b14fd1be1692c7
Signed-off-by: Adrian Villin <[email protected]>
15 files changed:
extras/hs-test/README.rst
extras/hs-test/framework_test.go
extras/hs-test/infra/container.go
extras/hs-test/infra/hst_suite.go
extras/hs-test/infra/suite_cpu_pinning.go
extras/hs-test/infra/suite_envoy_proxy.go
extras/hs-test/infra/suite_iperf_linux.go
extras/hs-test/infra/suite_ldp.go
extras/hs-test/infra/suite_nginx_proxy.go
extras/hs-test/infra/suite_no_topo.go
extras/hs-test/infra/suite_veth.go
extras/hs-test/infra/suite_vpp_proxy.go
extras/hs-test/iperf_linux_test.go
extras/hs-test/ldp_test.go
extras/hs-test/script/compress.sh

index 159f679..c62be5a 100644 (file)
@@ -167,8 +167,8 @@ Modifying the framework
 
                         // Add custom setup code here
 
-                        s.ConfigureNetworkTopology("myTopology")
-                        s.LoadContainerTopology("2peerVeth")
+                        s.ConfigureNetworkTopology("myNetworkTopology")
+                        s.LoadContainerTopology("myContainerTopology")
                 }
 
 #. In suite file, implement ``SetupTest`` method which gets executed before each test. Starting containers and
@@ -216,7 +216,7 @@ Modifying the framework
                                It(testName, func(ctx SpecContext) {
                                        s.Log(testName + ": BEGIN")
                                        test(&s)
-                               }, SpecTimeout(SuiteTimeout))
+                               }, SpecTimeout(TestTimeout))
                        }
                }
                 })
@@ -237,7 +237,7 @@ Modifying the framework
                         It(testName, Label("SOLO"), func(ctx SpecContext) {
                                 s.Log(testName + ": BEGIN")
                                test(&s)
-                       }, SpecTimeout(time.Minute*5))
+                       }, SpecTimeout(TestTimeout))
                 })
 
 #. Next step is to add test cases to the suite. For that, see section `Adding a test case`_ above
index 0671d75..a578fb1 100644 (file)
@@ -15,9 +15,9 @@ import (
 func TestHst(t *testing.T) {
        if *IsVppDebug {
                // 30 minute timeout so that the framework won't timeout while debugging
-               SuiteTimeout = time.Minute * 30
+               TestTimeout = time.Minute * 30
        } else {
-               SuiteTimeout = time.Minute * 5
+               TestTimeout = time.Minute * 5
        }
 
        output, err := os.ReadFile("/sys/devices/system/node/online")
index d4ef693..cc79a5c 100644 (file)
@@ -328,7 +328,6 @@ func (c *Container) getVolumesAsSlice() []string {
        }
 
        core_pattern, err := sysctl.Read("kernel.core_pattern")
-       core_pattern = strings.ReplaceAll(core_pattern, "%", "%%")
        if err == nil {
                if len(core_pattern) > 0 && core_pattern[0] != '|' {
                        index := strings.LastIndex(core_pattern, "/")
index 435001a..3aeeb24 100644 (file)
@@ -43,7 +43,7 @@ var IsLeakCheck = flag.Bool("leak_check", false, "run leak-check tests")
 var ParallelTotal = flag.Lookup("ginkgo.parallel.total")
 var DryRun = flag.Bool("dryrun", false, "set up containers but don't run tests")
 var NumaAwareCpuAlloc bool
-var SuiteTimeout time.Duration
+var TestTimeout time.Duration
 
 type HstSuite struct {
        Containers        map[string]*Container
@@ -310,6 +310,10 @@ func (s *HstSuite) AssertHttpBody(resp *http.Response, expectedBody string, msgA
        ExpectWithOffset(2, resp).To(HaveHTTPBody(expectedBody), msgAndArgs...)
 }
 
+func (s *HstSuite) AssertChannelClosed(timeout time.Duration, channel chan error) {
+       EventuallyWithOffset(2, channel).WithTimeout(timeout).Should(BeClosed())
+}
+
 func (s *HstSuite) CreateLogger() {
        suiteName := s.GetCurrentSuiteName()
        var err error
@@ -323,12 +327,19 @@ func (s *HstSuite) CreateLogger() {
 // Logs to files by default, logs to stdout when VERBOSE=true with GinkgoWriter
 // to keep console tidy
 func (s *HstSuite) Log(log any, arg ...any) {
-       logs := strings.Split(fmt.Sprintf(fmt.Sprint(log), arg...), "\n")
+       var logStr string
+       if len(arg) == 0 {
+               logStr = fmt.Sprint(log)
+       } else {
+               logStr = fmt.Sprintf(fmt.Sprint(log), arg...)
+       }
+       logs := strings.Split(logStr, "\n")
+
        for _, line := range logs {
                s.Logger.Println(line)
        }
        if *IsVerbose {
-               GinkgoWriter.Println(fmt.Sprintf(fmt.Sprint(log), arg...))
+               GinkgoWriter.Println(logStr)
        }
 }
 
index 4adfe16..57efbe7 100644 (file)
@@ -83,7 +83,7 @@ var _ = Describe("CpuPinningSuite", Ordered, ContinueOnFailure, func() {
                        It(testName, func(ctx SpecContext) {
                                s.Log(testName + ": BEGIN")
                                test(&s)
-                       }, SpecTimeout(SuiteTimeout))
+                       }, SpecTimeout(TestTimeout))
                }
        }
 })
@@ -112,7 +112,7 @@ var _ = Describe("CpuPinningSuiteSolo", Ordered, ContinueOnFailure, Serial, func
                        It(testName, Label("SOLO"), func(ctx SpecContext) {
                                s.Log(testName + ": BEGIN")
                                test(&s)
-                       }, SpecTimeout(SuiteTimeout))
+                       }, SpecTimeout(TestTimeout))
                }
        }
 })
index 9b1ae1f..50ffd11 100644 (file)
@@ -191,7 +191,7 @@ var _ = Describe("EnvoyProxySuite", Ordered, ContinueOnFailure, func() {
                        It(testName, func(ctx SpecContext) {
                                s.Log(testName + ": BEGIN")
                                test(&s)
-                       }, SpecTimeout(SuiteTimeout))
+                       }, SpecTimeout(TestTimeout))
                }
        }
 })
@@ -220,7 +220,7 @@ var _ = Describe("EnvoyProxySuiteSolo", Ordered, ContinueOnFailure, func() {
                        It(testName, Label("SOLO"), func(ctx SpecContext) {
                                s.Log(testName + ": BEGIN")
                                test(&s)
-                       }, SpecTimeout(SuiteTimeout))
+                       }, SpecTimeout(TestTimeout))
                }
        }
 })
index 728429b..fb685f9 100644 (file)
@@ -61,7 +61,7 @@ var _ = Describe("IperfSuite", Ordered, ContinueOnFailure, func() {
                        It(testName, func(ctx SpecContext) {
                                s.Log(testName + ": BEGIN")
                                test(&s)
-                       }, SpecTimeout(SuiteTimeout))
+                       }, SpecTimeout(TestTimeout))
                }
        }
 })
@@ -90,7 +90,7 @@ var _ = Describe("IperfSuiteSolo", Ordered, ContinueOnFailure, Serial, func() {
                        It(testName, Label("SOLO"), func(ctx SpecContext) {
                                s.Log(testName + ": BEGIN")
                                test(&s)
-                       }, SpecTimeout(SuiteTimeout))
+                       }, SpecTimeout(TestTimeout))
                }
        }
 })
index 45f55ce..6294fab 100644 (file)
@@ -163,7 +163,7 @@ var _ = Describe("LdpSuite", Ordered, ContinueOnFailure, func() {
                        It(testName, func(ctx SpecContext) {
                                s.Log(testName + ": BEGIN")
                                test(&s)
-                       }, SpecTimeout(SuiteTimeout))
+                       }, SpecTimeout(TestTimeout))
                }
        }
 })
@@ -193,7 +193,7 @@ var _ = Describe("LdpSuiteSolo", Ordered, ContinueOnFailure, Serial, func() {
                        It(testName, Label("SOLO"), func(ctx SpecContext) {
                                s.Log(testName + ": BEGIN")
                                test(&s)
-                       }, SpecTimeout(SuiteTimeout))
+                       }, SpecTimeout(TestTimeout))
                }
        }
 })
index 476631e..9d81f70 100644 (file)
@@ -200,7 +200,7 @@ var _ = Describe("NginxProxySuite", Ordered, ContinueOnFailure, func() {
                        It(testName, func(ctx SpecContext) {
                                s.Log(testName + ": BEGIN")
                                test(&s)
-                       }, SpecTimeout(SuiteTimeout))
+                       }, SpecTimeout(TestTimeout))
                }
        }
 })
@@ -229,7 +229,7 @@ var _ = Describe("NginxProxySuiteSolo", Ordered, ContinueOnFailure, Serial, func
                        It(testName, Label("SOLO"), func(ctx SpecContext) {
                                s.Log(testName + ": BEGIN")
                                test(&s)
-                       }, SpecTimeout(SuiteTimeout))
+                       }, SpecTimeout(TestTimeout))
                }
        }
 })
index 590bd2b..068c43b 100644 (file)
@@ -166,7 +166,7 @@ var _ = Describe("NoTopoSuite", Ordered, ContinueOnFailure, func() {
                        It(testName, func(ctx SpecContext) {
                                s.Log(testName + ": BEGIN")
                                test(&s)
-                       }, SpecTimeout(SuiteTimeout))
+                       }, SpecTimeout(TestTimeout))
                }
        }
 })
@@ -195,7 +195,7 @@ var _ = Describe("NoTopoSuiteSolo", Ordered, ContinueOnFailure, Serial, func() {
                        It(testName, Label("SOLO"), func(ctx SpecContext) {
                                s.Log(testName + ": BEGIN")
                                test(&s)
-                       }, SpecTimeout(SuiteTimeout))
+                       }, SpecTimeout(TestTimeout))
                }
        }
 })
index 03640c4..cbbd7ee 100644 (file)
@@ -120,7 +120,7 @@ var _ = Describe("VethsSuite", Ordered, ContinueOnFailure, func() {
                        It(testName, func(ctx SpecContext) {
                                s.Log(testName + ": BEGIN")
                                test(&s)
-                       }, SpecTimeout(SuiteTimeout))
+                       }, SpecTimeout(TestTimeout))
                }
        }
 })
@@ -150,7 +150,7 @@ var _ = Describe("VethsSuiteSolo", Ordered, ContinueOnFailure, Serial, func() {
                        It(testName, Label("SOLO"), func(ctx SpecContext) {
                                s.Log(testName + ": BEGIN")
                                test(&s)
-                       }, SpecTimeout(SuiteTimeout))
+                       }, SpecTimeout(TestTimeout))
                }
        }
 })
index 25bc6a6..16c6115 100644 (file)
@@ -184,7 +184,7 @@ var _ = Describe("VppProxySuite", Ordered, ContinueOnFailure, func() {
                        It(testName, func(ctx SpecContext) {
                                s.Log(testName + ": BEGIN")
                                test(&s)
-                       }, SpecTimeout(SuiteTimeout))
+                       }, SpecTimeout(TestTimeout))
                }
        }
 })
@@ -213,7 +213,7 @@ var _ = Describe("VppProxySuiteSolo", Ordered, ContinueOnFailure, func() {
                        It(testName, Label("SOLO"), func(ctx SpecContext) {
                                s.Log(testName + ": BEGIN")
                                test(&s)
-                       }, SpecTimeout(SuiteTimeout))
+                       }, SpecTimeout(TestTimeout))
                }
        }
 })
index 14422fe..b2b52a7 100644 (file)
@@ -2,6 +2,7 @@ package main
 
 import (
        "fmt"
+       "time"
 
        . "fd.io/hs-test/infra"
        . "github.com/onsi/ginkgo/v2"
@@ -41,8 +42,6 @@ func IperfLinuxTest(s *IperfSuite) {
                        " -u -l 1460 -b 10g -p " + s.GetPortFromPpid()
                s.StartClientApp(clientContainer, cmd, clnCh, clnRes)
        }()
-
+       s.AssertChannelClosed(time.Minute*3, clnCh)
        s.Log(<-clnRes)
-       err = <-clnCh
-       s.AssertNil(err, "err: '%s'", err)
 }
index 03636b1..e72fac4 100644 (file)
@@ -2,6 +2,7 @@ package main
 
 import (
        "fmt"
+       "time"
 
        . "fd.io/hs-test/infra"
        . "github.com/onsi/ginkgo/v2"
@@ -24,6 +25,10 @@ func LDPreloadIperfVppTest(s *LdpSuite) {
        clnCh := make(chan error)
        clnRes := make(chan string, 1)
 
+       defer func() {
+               stopServerCh <- struct{}{}
+       }()
+
        go func() {
                defer GinkgoRecover()
                cmd := "iperf3 -4 -s -p " + s.GetPortFromPpid()
@@ -39,14 +44,9 @@ func LDPreloadIperfVppTest(s *LdpSuite) {
                cmd := "iperf3 -c " + serverVethAddress + " -u -l 1460 -b 10g -p " + s.GetPortFromPpid()
                s.StartClientApp(clientContainer, cmd, clnCh, clnRes)
        }()
-       s.Log(<-clnRes)
-
-       // wait for client's result
-       err = <-clnCh
-       s.AssertNil(err, fmt.Sprint(err))
 
-       // stop server
-       stopServerCh <- struct{}{}
+       s.AssertChannelClosed(time.Minute*3, clnCh)
+       s.Log(<-clnRes)
 }
 
 func RedisBenchmarkTest(s *LdpSuite) {
@@ -61,6 +61,10 @@ func RedisBenchmarkTest(s *LdpSuite) {
        clnCh := make(chan error)
        clnRes := make(chan string, 1)
 
+       defer func() {
+               doneSrv <- struct{}{}
+       }()
+
        go func() {
                defer GinkgoRecover()
                cmd := "redis-server --daemonize yes --protected-mode no --bind " + serverVethAddress
@@ -79,12 +83,10 @@ func RedisBenchmarkTest(s *LdpSuite) {
                        cmd = "redis-benchmark --threads " + fmt.Sprint(*NConfiguredCpus) + "-h " + serverVethAddress
                }
                s.StartClientApp(clientContainer, cmd, clnCh, clnRes)
+
        }()
 
+       // 4.5 minutes
+       s.AssertChannelClosed(time.Second*270, clnCh)
        s.Log(<-clnRes)
-       // wait for client's result
-       err = <-clnCh
-       s.AssertNil(err, fmt.Sprint(err))
-       // stop server
-       doneSrv <- struct{}{}
 }
index 3a4642e..ebc60d2 100755 (executable)
@@ -11,7 +11,7 @@ then
     if [ -n "${WORKSPACE}" ]
     then
         echo -n "Copying docker logs..."
-        dirs=$(jq -r '.[0] | .SpecReports[] | select(.State == "failed") | .LeafNodeText | split("/")[1]' ${HS_ROOT}/summary/report.json)
+        dirs=$(jq -r '.[0] | .SpecReports[] | select((.State == "failed") or (.State == "timedout") or (.State == "panicked")) | .LeafNodeText | split("/")[1]' ${HS_ROOT}/summary/report.json)
         for dirName in $dirs; do
             logDir=/tmp/hs-test/$dirName
             if [ -d "$logDir" ]; then