From 8941f9ade958d10cdfe0d94de84fd215ebc3968d Mon Sep 17 00:00:00 2001 From: Adrian Villin Date: Mon, 4 Aug 2025 14:02:56 +0200 Subject: [PATCH] hs-test: load k8s pod definitions from a file - improved pod teardown, fixed obtaining IP addresses from pods - added contexts for better goroutine management Type: improvement Change-Id: Iab8bd76598465c44626188d47b57d561469e279b Signed-off-by: Adrian Villin --- extras/hs-test/infra/kind/deployment.go | 5 +- extras/hs-test/infra/kind/pod.go | 119 +++++++++++++++---------- extras/hs-test/infra/kind/suite_kind.go | 22 +++-- extras/hs-test/kind_test.go | 46 ++++++---- extras/hs-test/kubernetes/pod-definitions.yaml | 75 ++++++++++++++++ 5 files changed, 195 insertions(+), 72 deletions(-) create mode 100644 extras/hs-test/kubernetes/pod-definitions.yaml diff --git a/extras/hs-test/infra/kind/deployment.go b/extras/hs-test/infra/kind/deployment.go index 20fc6c80c3e..847e9605761 100644 --- a/extras/hs-test/infra/kind/deployment.go +++ b/extras/hs-test/infra/kind/deployment.go @@ -41,6 +41,7 @@ func (s *KindSuite) createNamespace(name string) { } func (s *KindSuite) deletePod(namespace string, podName string) error { + delete(s.CurrentlyRunning, podName) return s.ClientSet.CoreV1().Pods(namespace).Delete(context.TODO(), podName, metav1.DeleteOptions{GracePeriodSeconds: int64Ptr(0)}) } @@ -49,8 +50,6 @@ func (s *KindSuite) deleteNamespace(namespace string) error { } func (s *KindSuite) DeployPod(pod *Pod) { - pod.suite = s - s.CurrentlyRunning = append(s.CurrentlyRunning, pod.Name) pod.CreatedPod = &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: s.Namespace, @@ -86,10 +85,12 @@ func (s *KindSuite) DeployPod(pod *Pod) { // Create the Pod _, err := s.ClientSet.CoreV1().Pods(s.Namespace).Create(context.TODO(), pod.CreatedPod, metav1.CreateOptions{}) s.AssertNil(err) + s.CurrentlyRunning[pod.Name] = pod s.Log("Pod '%s' created", pod.Name) // Get IP s.Log("Obtaining IP from '%s'", pod.Name) + pod.IpAddress = "" counter := 1 for pod.IpAddress == "" { pod.CreatedPod, err = s.ClientSet.CoreV1().Pods(s.Namespace).Get(context.TODO(), pod.Name, metav1.GetOptions{}) diff --git a/extras/hs-test/infra/kind/pod.go b/extras/hs-test/infra/kind/pod.go index f323130af21..4a8d5011332 100644 --- a/extras/hs-test/infra/kind/pod.go +++ b/extras/hs-test/infra/kind/pod.go @@ -5,10 +5,11 @@ import ( "context" "os" "os/exec" + "slices" "text/template" - "time" . "fd.io/hs-test/infra/common" + "gopkg.in/yaml.v3" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/remotecommand" ) @@ -19,53 +20,79 @@ type Pod struct { Image string ContainerName string Worker string + Namespace string IpAddress string CreatedPod *corev1.Pod } -// Sets pod names, image names, namespace name +type Image struct { + Name string `yaml:"name"` +} +type Container struct { + Name string `yaml:"name"` +} +type Worker struct { + Name string `yaml:"name"` +} +type Namespace struct { + Name string `yaml:"name"` +} +type PodYaml struct { + Name string `yaml:"name"` + Image []Image `yaml:"image"` + Container []Container `yaml:"container"` + Worker []Worker `yaml:"worker"` + Namespace []Namespace `yaml:"namespace"` +} +type Config struct { + Pods []PodYaml `yaml:"pods"` +} + +func (s *KindSuite) LoadPodConfigs() { + data, err := os.ReadFile("kubernetes/pod-definitions.yaml") + s.AssertNil(err) + + var config Config + err = yaml.Unmarshal(data, &config) + s.AssertNil(err) + + for _, podData := range config.Pods { + newPod(s, podData) + } +} + +func newPod(suite *KindSuite, input PodYaml) (*Pod, error) { + var pod = new(Pod) + pod.suite = suite + pod.Name = input.Name + suite.Ppid + pod.Image = input.Image[0].Name + pod.ContainerName = input.Container[0].Name + pod.Worker = input.Worker[0].Name + pod.Namespace = input.Namespace[0].Name + suite.Ppid + + if suite.AllPods == nil { + suite.AllPods = make(map[string]*Pod) + suite.Namespace = pod.Namespace + } + + suite.AllPods[pod.Name] = pod + if !slices.Contains(suite.images, pod.Image) { + suite.images = append(suite.images, pod.Image) + } + + return pod, nil +} + func (s *KindSuite) initPods() { - wrk1 := "kind-worker" - wrk2 := "kind-worker2" - vppImg := "hs-test/vpp:latest" - nginxLdpImg := "hs-test/nginx-ldp:latest" - abImg := "hs-test/ab:latest" - clientCont := "client" - serverCont := "server" - - // TODO: load from file - s.images = append(s.images, vppImg, nginxLdpImg, abImg) - s.Namespace = "namespace" + s.Ppid - - s.Pods.ClientGeneric = new(Pod) - s.Pods.ClientGeneric.Name = "client" + s.Ppid - s.Pods.ClientGeneric.Image = vppImg - s.Pods.ClientGeneric.ContainerName = clientCont - s.Pods.ClientGeneric.Worker = wrk1 - - s.Pods.ServerGeneric = new(Pod) - s.Pods.ServerGeneric.Name = "server" + s.Ppid - s.Pods.ServerGeneric.Image = vppImg - s.Pods.ServerGeneric.ContainerName = serverCont - s.Pods.ServerGeneric.Worker = wrk2 - - s.Pods.Ab = new(Pod) - s.Pods.Ab.Name = "ab" + s.Ppid - s.Pods.Ab.Image = abImg - s.Pods.Ab.ContainerName = clientCont - s.Pods.Ab.Worker = wrk1 - - s.Pods.Nginx = new(Pod) - s.Pods.Nginx.Name = "nginx-ldp" + s.Ppid - s.Pods.Nginx.Image = nginxLdpImg - s.Pods.Nginx.ContainerName = serverCont - s.Pods.Nginx.Worker = wrk2 - - s.Pods.NginxProxy = new(Pod) - s.Pods.NginxProxy.Name = "nginx-proxy" + s.Ppid - s.Pods.NginxProxy.Image = nginxLdpImg - s.Pods.NginxProxy.ContainerName = serverCont - s.Pods.NginxProxy.Worker = wrk2 + s.Pods.Ab = s.getPodsByName("ab") + s.Pods.ClientGeneric = s.getPodsByName("client-generic") + s.Pods.ServerGeneric = s.getPodsByName("server-generic") + s.Pods.Nginx = s.getPodsByName("nginx-ldp") + s.Pods.NginxProxy = s.getPodsByName("nginx-proxy") +} + +func (s *KindSuite) getPodsByName(podName string) *Pod { + return s.AllPods[podName+s.Ppid] } func (pod *Pod) CopyToPod(namespace string, src string, dst string) { @@ -74,7 +101,7 @@ func (pod *Pod) CopyToPod(namespace string, src string, dst string) { pod.suite.AssertNil(err, string(out)) } -func (pod *Pod) Exec(command []string) (string, error) { +func (pod *Pod) Exec(ctx context.Context, command []string) (string, error) { var stdout, stderr bytes.Buffer // Prepare the request @@ -98,9 +125,6 @@ func (pod *Pod) Exec(command []string) (string, error) { pod.suite.Log("Error creating executor: %s", err.Error()) } - ctx, cancel := context.WithTimeout(context.Background(), 500*time.Second) - defer cancel() - err = executor.StreamWithContext(ctx, remotecommand.StreamOptions{ Stdout: &stdout, Stderr: &stderr, @@ -125,7 +149,6 @@ func (pod *Pod) CreateConfigFromTemplate(targetConfigName string, templateName s err = template.Execute(f, values) pod.suite.AssertNil(err, err) - err = f.Close() pod.suite.AssertNil(err, err) diff --git a/extras/hs-test/infra/kind/suite_kind.go b/extras/hs-test/infra/kind/suite_kind.go index 1754853f789..ecbbe1dc35e 100644 --- a/extras/hs-test/infra/kind/suite_kind.go +++ b/extras/hs-test/infra/kind/suite_kind.go @@ -1,6 +1,7 @@ package hst_kind import ( + "context" "fmt" "reflect" "regexp" @@ -22,8 +23,9 @@ type KindSuite struct { Config *rest.Config Namespace string KubeconfigPath string - CurrentlyRunning []string + CurrentlyRunning map[string]*Pod images []string + AllPods map[string]*Pod Pods struct { ServerGeneric *Pod ClientGeneric *Pod @@ -31,6 +33,7 @@ type KindSuite struct { NginxProxy *Pod Ab *Pod } + MainContext context.Context } var kindTests = map[string][]func(s *KindSuite){} @@ -59,6 +62,7 @@ func RegisterKindTests(tests ...func(s *KindSuite)) { } func (s *KindSuite) SetupTest() { + s.MainContext = context.Background() s.HstCommon.SetupTest() } @@ -68,9 +72,10 @@ func (s *KindSuite) SetupSuite() { Fail(message, callerSkip...) }) + s.CurrentlyRunning = make(map[string]*Pod) + s.LoadPodConfigs() s.initPods() s.loadDockerImages() - var err error if *SudoUser == "root" { s.KubeconfigPath = "/.kube/config" @@ -94,14 +99,17 @@ func (s *KindSuite) TeardownTest() { if len(s.CurrentlyRunning) != 0 { s.Log("Removing:") for _, pod := range s.CurrentlyRunning { - s.Log(" %s", pod) - s.deletePod(s.Namespace, pod) + s.Log(" %s", pod.Name) + s.deletePod(s.Namespace, pod.Name) } } } func (s *KindSuite) TeardownSuite() { s.HstCommon.TeardownSuite() + if len(s.CurrentlyRunning) == 0 { + return + } s.Log("Removing:\n %s", s.Namespace) s.AssertNil(s.deleteNamespace(s.Namespace)) } @@ -110,7 +118,7 @@ func (s *KindSuite) TeardownSuite() { // and searches for the first version string, then creates symlinks. func (s *KindSuite) FixVersionNumber(pods ...*Pod) { regex := regexp.MustCompile(`lib.*\.so\.([0-9]+\.[0-9]+)`) - o, _ := s.Pods.ServerGeneric.Exec([]string{"/bin/bash", "-c", + o, _ := s.Pods.ServerGeneric.Exec(context.TODO(), []string{"/bin/bash", "-c", "ldd /usr/lib/libvcl_ldpreload.so"}) match := regex.FindStringSubmatch(o) @@ -125,7 +133,7 @@ func (s *KindSuite) FixVersionNumber(pods ...*Pod) { "fi\n"+ "done", version) for _, pod := range pods { - pod.Exec([]string{"/bin/bash", "-c", cmd}) + pod.Exec(context.TODO(), []string{"/bin/bash", "-c", cmd}) } } else { @@ -149,7 +157,7 @@ func (s *KindSuite) CreateNginxConfig(pod *Pod) { } func (s *KindSuite) CreateNginxProxyConfig(pod *Pod) { - pod.Exec([]string{"/bin/bash", "-c", "mkdir -p /tmp/nginx"}) + pod.Exec(context.TODO(), []string{"/bin/bash", "-c", "mkdir -p /tmp/nginx"}) values := struct { Workers uint8 LogPrefix string diff --git a/extras/hs-test/kind_test.go b/extras/hs-test/kind_test.go index bc09fb7c3e4..12f6b3972f3 100644 --- a/extras/hs-test/kind_test.go +++ b/extras/hs-test/kind_test.go @@ -1,6 +1,8 @@ package main import ( + "context" + "errors" "time" . "fd.io/hs-test/infra/kind" @@ -15,73 +17,87 @@ const vcl string = "VCL_CONFIG=/vcl.conf" const ldp string = "LD_PRELOAD=/usr/lib/libvcl_ldpreload.so" func KindIperfVclTest(s *KindSuite) { + ctx, cancel := context.WithTimeout(s.MainContext, time.Second*30) + defer cancel() s.DeployPod(s.Pods.ClientGeneric) s.DeployPod(s.Pods.ServerGeneric) - _, err := s.Pods.ClientGeneric.Exec([]string{"/bin/bash", "-c", VclConfIperf}) + _, err := s.Pods.ClientGeneric.Exec(ctx, []string{"/bin/bash", "-c", VclConfIperf}) s.AssertNil(err) - _, err = s.Pods.ServerGeneric.Exec([]string{"/bin/bash", "-c", VclConfIperf}) + _, err = s.Pods.ServerGeneric.Exec(ctx, []string{"/bin/bash", "-c", VclConfIperf}) s.AssertNil(err) s.FixVersionNumber(s.Pods.ClientGeneric, s.Pods.ServerGeneric) - o, err := s.Pods.ServerGeneric.Exec([]string{"/bin/bash", "-c", + o, err := s.Pods.ServerGeneric.Exec(ctx, []string{"/bin/bash", "-c", vcl + " " + ldp + " iperf3 -s -D -4"}) s.AssertNil(err, o) - o, err = s.Pods.ClientGeneric.Exec([]string{"/bin/bash", "-c", + o, err = s.Pods.ClientGeneric.Exec(ctx, []string{"/bin/bash", "-c", vcl + " " + ldp + " iperf3 -l 1460 -b 10g -c " + s.Pods.ServerGeneric.IpAddress}) s.Log(o) s.AssertNil(err) } func NginxRpsTest(s *KindSuite) { + ctx, cancel := context.WithCancel(s.MainContext) + defer cancel() + s.DeployPod(s.Pods.Nginx) s.DeployPod(s.Pods.Ab) s.CreateNginxConfig(s.Pods.Nginx) - out, err := s.Pods.Nginx.Exec([]string{"/bin/bash", "-c", VclConfNginx}) + out, err := s.Pods.Nginx.Exec(ctx, []string{"/bin/bash", "-c", VclConfNginx}) s.AssertNil(err, out) go func() { defer GinkgoRecover() - out, err := s.Pods.Nginx.Exec([]string{"/bin/bash", "-c", ldp + " " + vcl + " nginx -c /nginx.conf"}) - s.AssertNil(err, out) + out, err := s.Pods.Nginx.Exec(ctx, []string{"/bin/bash", "-c", "nginx -c /nginx.conf"}) + if !errors.Is(err, context.Canceled) { + s.AssertNil(err, out) + } }() // wait for nginx to start up time.Sleep(time.Second * 2) - out, err = s.Pods.Ab.Exec([]string{"ab", "-k", "-r", "-n", "1000000", "-c", "1000", "http://" + s.Pods.Nginx.IpAddress + ":8081/64B.json"}) + out, err = s.Pods.Ab.Exec(ctx, []string{"ab", "-k", "-r", "-n", "1000000", "-c", "1000", "http://" + s.Pods.Nginx.IpAddress + ":8081/64B.json"}) s.Log(out) s.AssertNil(err) } func NginxProxyMirroringTest(s *KindSuite) { + ctx, cancel := context.WithCancel(s.MainContext) + defer cancel() + s.DeployPod(s.Pods.Nginx) s.DeployPod(s.Pods.NginxProxy) s.DeployPod(s.Pods.ClientGeneric) s.CreateNginxConfig(s.Pods.Nginx) s.CreateNginxProxyConfig(s.Pods.NginxProxy) - out, err := s.Pods.Nginx.Exec([]string{"/bin/bash", "-c", VclConfNginx}) + out, err := s.Pods.Nginx.Exec(ctx, []string{"/bin/bash", "-c", VclConfNginx}) s.AssertNil(err, out) - out, err = s.Pods.NginxProxy.Exec([]string{"/bin/bash", "-c", VclConfNginx}) + out, err = s.Pods.NginxProxy.Exec(ctx, []string{"/bin/bash", "-c", VclConfNginx}) s.AssertNil(err, out) go func() { defer GinkgoRecover() - out, err := s.Pods.Nginx.Exec([]string{"/bin/bash", "-c", ldp + " " + vcl + " nginx -c /nginx.conf"}) - s.AssertNil(err, out) + out, err := s.Pods.Nginx.Exec(ctx, []string{"/bin/bash", "-c", ldp + " " + vcl + " nginx -c /nginx.conf"}) + if !errors.Is(err, context.Canceled) { + s.AssertNil(err, out) + } }() go func() { defer GinkgoRecover() - out, err := s.Pods.NginxProxy.Exec([]string{"/bin/bash", "-c", "nginx -c /nginx.conf"}) - s.AssertNil(err, out) + out, err := s.Pods.NginxProxy.Exec(ctx, []string{"/bin/bash", "-c", "nginx -c /nginx.conf"}) + if !errors.Is(err, context.Canceled) { + s.AssertNil(err, out) + } }() // wait for nginx to start up time.Sleep(time.Second * 2) - out, err = s.Pods.ClientGeneric.Exec([]string{"curl", "-v", "--noproxy", "'*'", "--insecure", "http://" + s.Pods.NginxProxy.IpAddress + ":8080/64B.json"}) + out, err = s.Pods.ClientGeneric.Exec(ctx, []string{"curl", "-v", "--noproxy", "'*'", "--insecure", "http://" + s.Pods.NginxProxy.IpAddress + ":8080/64B.json"}) s.Log(out) s.AssertNil(err) } diff --git a/extras/hs-test/kubernetes/pod-definitions.yaml b/extras/hs-test/kubernetes/pod-definitions.yaml new file mode 100644 index 00000000000..f15e552fbe3 --- /dev/null +++ b/extras/hs-test/kubernetes/pod-definitions.yaml @@ -0,0 +1,75 @@ +definitions: + image-names: + - image: &hs-test + name: "hs-test/vpp:latest" + - image: &nginx-ldp + name: "hs-test/nginx-ldp:latest" + - image: &ab + name: "hs-test/ab:latest" + + container-names: + - container: &client + name: "client" + - container: &server + name: "server" + + namespace-names: + - namespace: &defaultNs + name: "hs-test" + + worker-names: + - worker: &worker1 + name: "kind-worker" + - worker: &worker2 + name: "kind-worker2" + +pods: + - name: "client-generic" + image: + - <<: *hs-test + container: + - <<: *client + worker: + - <<: *worker1 + namespace: + - <<: *defaultNs + + - name: "server-generic" + image: + - <<: *hs-test + container: + - <<: *server + worker: + - <<: *worker2 + namespace: + - <<: *defaultNs + + - name: "ab" + image: + - <<: *ab + container: + - <<: *client + worker: + - <<: *worker1 + namespace: + - <<: *defaultNs + + - name: "nginx-ldp" + image: + - <<: *nginx-ldp + container: + - <<: *server + worker: + - <<: *worker2 + namespace: + - <<: *defaultNs + + - name: "nginx-proxy" + image: + - <<: *nginx-ldp + container: + - <<: *server + worker: + - <<: *worker2 + namespace: + - <<: *defaultNs -- 2.16.6