From dd02eb893b485a290a47ddca7d76ecedb45a5063 Mon Sep 17 00:00:00 2001 From: Adrian Villin Date: Fri, 29 Nov 2024 16:33:14 +0100 Subject: [PATCH] hs-test: added multi-threaded proxy tests - TCP and UDP iperf proxy tests added Type: test Change-Id: Ic6f429cc6d48388ce9a17f8b9cd7c4b54b9a7e4d Signed-off-by: Adrian Villin --- extras/hs-test/infra/suite_envoy_proxy.go | 4 +- extras/hs-test/infra/suite_nginx_proxy.go | 4 +- extras/hs-test/infra/suite_no_topo.go | 2 +- extras/hs-test/infra/suite_vpp_proxy.go | 72 +++++++++------- extras/hs-test/infra/suite_vpp_udp_proxy.go | 4 +- extras/hs-test/infra/vppinstance.go | 46 ++++++++-- extras/hs-test/proxy_test.go | 90 ++++++++++++++++++-- extras/hs-test/topo-containers/vppProxy.yaml | 12 +++ 8 files changed, 179 insertions(+), 55 deletions(-) diff --git a/extras/hs-test/infra/suite_envoy_proxy.go b/extras/hs-test/infra/suite_envoy_proxy.go index 50ffd11898e..754705d24e0 100644 --- a/extras/hs-test/infra/suite_envoy_proxy.go +++ b/extras/hs-test/infra/suite_envoy_proxy.go @@ -114,8 +114,8 @@ func (s *EnvoyProxySuite) SetupTest() { s.AssertNil(vpp.Start()) // wait for VPP to start time.Sleep(time.Second * 1) - s.AssertNil(vpp.createTap(clientInterface, 1)) - s.AssertNil(vpp.createTap(serverInterface, 2)) + s.AssertNil(vpp.CreateTap(clientInterface, 1, 1)) + s.AssertNil(vpp.CreateTap(serverInterface, 1, 2)) vppContainer.Exec(false, "chmod 777 -R %s", vppContainer.GetContainerWorkDir()) // Add Ipv4 ARP entry for nginx HTTP server, otherwise first request fail (HTTP error 503) diff --git a/extras/hs-test/infra/suite_nginx_proxy.go b/extras/hs-test/infra/suite_nginx_proxy.go index 9d81f70720c..f0c178fc2f8 100644 --- a/extras/hs-test/infra/suite_nginx_proxy.go +++ b/extras/hs-test/infra/suite_nginx_proxy.go @@ -85,8 +85,8 @@ func (s *NginxProxySuite) SetupTest() { ) s.AssertNil(vpp.Start()) - s.AssertNil(vpp.createTap(clientInterface, 1)) - s.AssertNil(vpp.createTap(serverInterface, 2)) + s.AssertNil(vpp.CreateTap(clientInterface, 1, 1)) + s.AssertNil(vpp.CreateTap(serverInterface, 1, 2)) if *DryRun { s.LogStartedContainers() diff --git a/extras/hs-test/infra/suite_no_topo.go b/extras/hs-test/infra/suite_no_topo.go index 068c43b14d0..727789b4deb 100644 --- a/extras/hs-test/infra/suite_no_topo.go +++ b/extras/hs-test/infra/suite_no_topo.go @@ -58,7 +58,7 @@ func (s *NoTopoSuite) SetupTest() { s.AssertNil(vpp.Start()) tapInterface := s.GetInterfaceByName(TapInterfaceName) - s.AssertNil(vpp.createTap(tapInterface), "failed to create tap interface") + s.AssertNil(vpp.CreateTap(tapInterface, 1, 1), "failed to create tap interface") if *DryRun { s.LogStartedContainers() diff --git a/extras/hs-test/infra/suite_vpp_proxy.go b/extras/hs-test/infra/suite_vpp_proxy.go index 57308dfe076..51beade2d9b 100644 --- a/extras/hs-test/infra/suite_vpp_proxy.go +++ b/extras/hs-test/infra/suite_vpp_proxy.go @@ -16,15 +16,17 @@ import ( // These correspond to names used in yaml config const ( - VppProxyContainerName = "vpp-proxy" - ClientTapInterfaceName = "hstcln" - ServerTapInterfaceName = "hstsrv" - CurlContainerTestFile = "/tmp/testFile" + VppProxyContainerName = "vpp-proxy" + ClientTapInterfaceName = "hstcln" + ServerTapInterfaceName = "hstsrv" + IperfServerContainerName = "iperfA" + IperfClientContainerName = "iperfB" + CurlContainerTestFile = "/tmp/testFile" ) type VppProxySuite struct { HstSuite - nginxPort uint16 + serverPort uint16 maxTimeout int } @@ -44,6 +46,7 @@ func (s *VppProxySuite) SetupSuite() { s.LoadNetworkTopology("2taps") s.LoadContainerTopology("vppProxy") + s.serverPort = 80 if *IsVppDebug { s.maxTimeout = 600 } else { @@ -62,31 +65,9 @@ func (s *VppProxySuite) SetupTest() { clientInterface := s.GetInterfaceByName(ClientTapInterfaceName) serverInterface := s.GetInterfaceByName(ServerTapInterfaceName) - // nginx HTTP server - nginxContainer := s.GetTransientContainerByName(NginxServerContainerName) - s.AssertNil(nginxContainer.Create()) - s.nginxPort = 80 - nginxSettings := struct { - LogPrefix string - Address string - Port uint16 - Timeout int - }{ - LogPrefix: nginxContainer.Name, - Address: serverInterface.Ip4AddressString(), - Port: s.nginxPort, - Timeout: s.maxTimeout, - } - nginxContainer.CreateConfigFromTemplate( - "/nginx.conf", - "./resources/nginx/nginx_server.conf", - nginxSettings, - ) - s.AssertNil(nginxContainer.Start()) - s.AssertNil(vpp.Start()) - s.AssertNil(vpp.createTap(clientInterface, 1)) - s.AssertNil(vpp.createTap(serverInterface, 2)) + s.AssertNil(vpp.CreateTap(clientInterface, 1, 1)) + s.AssertNil(vpp.CreateTap(serverInterface, 1, 2)) if *DryRun { s.LogStartedContainers() @@ -104,11 +85,34 @@ func (s *VppProxySuite) TearDownTest() { s.HstSuite.TearDownTest() } -func (s *VppProxySuite) NginxPort() uint16 { - return s.nginxPort +func (s *VppProxySuite) SetupNginxServer() { + nginxContainer := s.GetTransientContainerByName(NginxServerContainerName) + serverInterface := s.GetInterfaceByName(ServerTapInterfaceName) + s.AssertNil(nginxContainer.Create()) + nginxSettings := struct { + LogPrefix string + Address string + Port uint16 + Timeout int + }{ + LogPrefix: nginxContainer.Name, + Address: serverInterface.Ip4AddressString(), + Port: s.serverPort, + Timeout: s.maxTimeout, + } + nginxContainer.CreateConfigFromTemplate( + "/nginx.conf", + "./resources/nginx/nginx_server.conf", + nginxSettings, + ) + s.AssertNil(nginxContainer.Start()) } -func (s *VppProxySuite) NginxAddr() string { +func (s *VppProxySuite) ServerPort() uint16 { + return s.serverPort +} + +func (s *VppProxySuite) ServerAddr() string { return s.GetInterfaceByName(ServerTapInterfaceName).Ip4AddressString() } @@ -116,6 +120,10 @@ func (s *VppProxySuite) VppProxyAddr() string { return s.GetInterfaceByName(ClientTapInterfaceName).Peer.Ip4AddressString() } +func (s *VppProxySuite) ClientAddr() string { + return s.GetInterfaceByName(ClientTapInterfaceName).Ip4AddressString() +} + func (s *VppProxySuite) CurlRequest(targetUri string) (string, string) { args := fmt.Sprintf("--insecure --noproxy '*' %s", targetUri) body, log := s.RunCurlContainer(args) diff --git a/extras/hs-test/infra/suite_vpp_udp_proxy.go b/extras/hs-test/infra/suite_vpp_udp_proxy.go index 35c9cd561df..84e76d6643a 100644 --- a/extras/hs-test/infra/suite_vpp_udp_proxy.go +++ b/extras/hs-test/infra/suite_vpp_udp_proxy.go @@ -48,8 +48,8 @@ func (s *VppUdpProxySuite) SetupTest() { serverInterface := s.GetInterfaceByName(ServerTapInterfaceName) s.AssertNil(vpp.Start()) - s.AssertNil(vpp.createTap(clientInterface, 1)) - s.AssertNil(vpp.createTap(serverInterface, 2)) + s.AssertNil(vpp.CreateTap(clientInterface, 1, 1)) + s.AssertNil(vpp.CreateTap(serverInterface, 1, 2)) s.proxyPort = 8080 s.serverPort = 80 diff --git a/extras/hs-test/infra/vppinstance.go b/extras/hs-test/infra/vppinstance.go index 96e162cc395..8d4d694cbb9 100644 --- a/extras/hs-test/infra/vppinstance.go +++ b/extras/hs-test/infra/vppinstance.go @@ -81,6 +81,7 @@ const ( defaultCliSocketFilePath = "/var/run/vpp/cli.sock" defaultApiSocketFilePath = "/var/run/vpp/api.sock" defaultLogFilePath = "/var/log/vpp/vpp.log" + Consistent_qp = 256 ) type VppInstance struct { @@ -441,22 +442,28 @@ func (vpp *VppInstance) addAppNamespace( return nil } -func (vpp *VppInstance) createTap(tap *NetInterface, tapId ...uint32) error { - var id uint32 = 1 - if len(tapId) > 0 { - id = tapId[0] +func (vpp *VppInstance) CreateTap(tap *NetInterface, numRxQueues uint16, tapId uint32, flags ...uint32) error { + var tapFlags uint32 = 0 + if len(flags) > 0 { + tapFlags = flags[0] } if *DryRun { - vppCliConfig := fmt.Sprintf("create tap id %d host-if-name %s host-ip4-addr %s\n"+ + flagsCli := "" + if tapFlags == Consistent_qp { + flagsCli = "consistent-qp" + } + vppCliConfig := fmt.Sprintf("create tap id %d host-if-name %s host-ip4-addr %s num-rx-queues %d %s\n"+ "set int ip addr tap%d %s\n"+ "set int state tap%d up\n", - id, + tapId, tap.name, tap.Ip4Address, - id, + numRxQueues, + flagsCli, + tapId, tap.Peer.Ip4Address, - id, + tapId, ) vpp.AppendToCliConfig(vppCliConfig) vpp.getSuite().Log("%s* Interface added:\n%s%s", Colors.grn, vppCliConfig, Colors.rst) @@ -464,11 +471,13 @@ func (vpp *VppInstance) createTap(tap *NetInterface, tapId ...uint32) error { } createTapReq := &tapv2.TapCreateV3{ - ID: id, + ID: tapId, HostIfNameSet: true, HostIfName: tap.Name(), HostIP4PrefixSet: true, HostIP4Prefix: tap.Ip4AddressWithPrefix(), + NumRxQueues: numRxQueues, + TapFlags: tapv2.TapFlags(tapFlags), } vpp.getSuite().Log("create tap interface " + tap.Name()) @@ -548,6 +557,25 @@ func (vpp *VppInstance) createTap(tap *NetInterface, tapId ...uint32) error { return nil } +func (vpp *VppInstance) DeleteTap(tapInterface *NetInterface) error { + deleteReq := &tapv2.TapDeleteV2{ + SwIfIndex: tapInterface.Peer.Index, + } + vpp.getSuite().Log("delete tap interface " + tapInterface.Name()) + if err := vpp.ApiStream.SendMsg(deleteReq); err != nil { + return err + } + replymsg, err := vpp.ApiStream.RecvMsg() + if err != nil { + return err + } + reply := replymsg.(*tapv2.TapDeleteV2Reply) + if err = api.RetvalToVPPApiError(reply.Retval); err != nil { + return err + } + return nil +} + func (vpp *VppInstance) saveLogs() { logTarget := vpp.getSuite().getLogDirPath() + "vppinstance-" + vpp.Container.Name + ".log" logSource := vpp.Container.GetHostWorkDir() + defaultLogFilePath diff --git a/extras/hs-test/proxy_test.go b/extras/hs-test/proxy_test.go index 367818925d2..a7f83ab7430 100644 --- a/extras/hs-test/proxy_test.go +++ b/extras/hs-test/proxy_test.go @@ -2,14 +2,17 @@ package main import ( "fmt" + "strconv" + "time" . "fd.io/hs-test/infra" + . "github.com/onsi/ginkgo/v2" ) func init() { RegisterVppProxyTests(VppProxyHttpGetTcpTest, VppProxyHttpGetTlsTest, VppProxyHttpPutTcpTest, VppProxyHttpPutTlsTest, VppConnectProxyGetTest, VppConnectProxyPutTest) - RegisterVppProxySoloTests(VppProxyHttpGetTcpMTTest, VppProxyHttpPutTcpMTTest) + RegisterVppProxySoloTests(VppProxyHttpGetTcpMTTest, VppProxyHttpPutTcpMTTest, VppProxyTcpIperfMTTest, VppProxyUdpIperfMTTest) RegisterVppUdpProxyTests(VppProxyUdpTest) RegisterEnvoyProxyTests(EnvoyProxyHttpGetTcpTest, EnvoyProxyHttpPutTcpTest) RegisterNginxProxyTests(NginxMirroringTest) @@ -19,9 +22,13 @@ func init() { func configureVppProxy(s *VppProxySuite, proto string, proxyPort uint16) { vppProxy := s.GetContainerByName(VppProxyContainerName).VppInstance cmd := fmt.Sprintf("test proxy server fifo-size 512k server-uri %s://%s/%d", proto, s.VppProxyAddr(), proxyPort) - if proto != "http" { - cmd += fmt.Sprintf(" client-uri tcp://%s/%d", s.NginxAddr(), s.NginxPort()) + if proto != "http" && proto != "udp" { + proto = "tcp" } + if proto != "http" { + cmd += fmt.Sprintf(" client-uri %s://%s/%d", proto, s.ServerAddr(), s.ServerPort()) + } + output := vppProxy.Vppctl(cmd) s.Log("proxy configured: " + output) } @@ -30,8 +37,74 @@ func VppProxyHttpGetTcpMTTest(s *VppProxySuite) { VppProxyHttpGetTcpTest(s) } +func VppProxyTcpIperfMTTest(s *VppProxySuite) { + vppProxyIperfMTTest(s, "tcp") +} + +func VppProxyUdpIperfMTTest(s *VppProxySuite) { + vppProxyIperfMTTest(s, "udp") +} + +func vppProxyIperfMTTest(s *VppProxySuite, proto string) { + iperfServer := s.GetContainerByName(IperfServerContainerName) + iperfClient := s.GetContainerByName(IperfClientContainerName) + iperfServer.Run() + iperfClient.Run() + serverInterface := s.GetInterfaceByName(ServerTapInterfaceName) + clientInterface := s.GetInterfaceByName(ClientTapInterfaceName) + vppProxy := s.GetContainerByName(VppProxyContainerName).VppInstance + proxyPort, err := strconv.Atoi(s.GetPortFromPpid()) + s.AssertNil(err) + + // tap interfaces are created on test setup with 1 rx-queue, + // need to recreate them with 2 + consistent-qp + s.AssertNil(vppProxy.DeleteTap(serverInterface)) + s.AssertNil(vppProxy.CreateTap(serverInterface, 2, uint32(serverInterface.Peer.Index), Consistent_qp)) + + s.AssertNil(vppProxy.DeleteTap(clientInterface)) + s.AssertNil(vppProxy.CreateTap(clientInterface, 2, uint32(clientInterface.Peer.Index), Consistent_qp)) + + configureVppProxy(s, "tcp", uint16(proxyPort)) + if proto == "udp" { + configureVppProxy(s, "udp", uint16(proxyPort)) + proto = "-u" + } else { + proto = "" + } + + stopServerCh := make(chan struct{}, 1) + srvCh := make(chan error, 1) + clnCh := make(chan error) + clnRes := make(chan []byte, 1) + + defer func() { + stopServerCh <- struct{}{} + }() + + go func() { + defer GinkgoRecover() + cmd := fmt.Sprintf("iperf3 -4 -s -B %s -p %s", s.ServerAddr(), fmt.Sprint(s.ServerPort())) + s.StartServerApp(iperfServer, "iperf3", cmd, srvCh, stopServerCh) + }() + + err = <-srvCh + s.AssertNil(err, fmt.Sprint(err)) + + go func() { + defer GinkgoRecover() + cmd := fmt.Sprintf("iperf3 -c %s -P 4 -l 1460 -b 10g -J -p %d -B %s %s", s.VppProxyAddr(), proxyPort, s.ClientAddr(), proto) + s.StartClientApp(iperfClient, cmd, clnCh, clnRes) + }() + + s.AssertChannelClosed(time.Minute*4, clnCh) + result := s.ParseJsonIperfOutput(<-clnRes) + s.LogJsonIperfOutput(result) + s.AssertIperfMinTransfer(result, 400) +} + func VppProxyHttpGetTcpTest(s *VppProxySuite) { var proxyPort uint16 = 8080 + s.SetupNginxServer() configureVppProxy(s, "tcp", proxyPort) uri := fmt.Sprintf("http://%s:%d/httpTestFile", s.VppProxyAddr(), proxyPort) s.CurlDownloadResource(uri) @@ -39,6 +112,7 @@ func VppProxyHttpGetTcpTest(s *VppProxySuite) { func VppProxyHttpGetTlsTest(s *VppProxySuite) { var proxyPort uint16 = 8080 + s.SetupNginxServer() configureVppProxy(s, "tls", proxyPort) uri := fmt.Sprintf("https://%s:%d/httpTestFile", s.VppProxyAddr(), proxyPort) s.CurlDownloadResource(uri) @@ -50,6 +124,7 @@ func VppProxyHttpPutTcpMTTest(s *VppProxySuite) { func VppProxyHttpPutTcpTest(s *VppProxySuite) { var proxyPort uint16 = 8080 + s.SetupNginxServer() configureVppProxy(s, "tcp", proxyPort) uri := fmt.Sprintf("http://%s:%d/upload/testFile", s.VppProxyAddr(), proxyPort) s.CurlUploadResource(uri, CurlContainerTestFile) @@ -57,6 +132,7 @@ func VppProxyHttpPutTcpTest(s *VppProxySuite) { func VppProxyHttpPutTlsTest(s *VppProxySuite) { var proxyPort uint16 = 8080 + s.SetupNginxServer() configureVppProxy(s, "tls", proxyPort) uri := fmt.Sprintf("https://%s:%d/upload/testFile", s.VppProxyAddr(), proxyPort) s.CurlUploadResource(uri, CurlContainerTestFile) @@ -94,21 +170,21 @@ func nginxMirroring(s *NginxProxySuite, multiThreadWorkers bool) { func VppConnectProxyGetTest(s *VppProxySuite) { var proxyPort uint16 = 8080 - + s.SetupNginxServer() configureVppProxy(s, "http", proxyPort) - targetUri := fmt.Sprintf("http://%s:%d/httpTestFile", s.NginxAddr(), s.NginxPort()) + targetUri := fmt.Sprintf("http://%s:%d/httpTestFile", s.ServerAddr(), s.ServerPort()) proxyUri := fmt.Sprintf("http://%s:%d", s.VppProxyAddr(), proxyPort) s.CurlDownloadResourceViaTunnel(targetUri, proxyUri) } func VppConnectProxyPutTest(s *VppProxySuite) { var proxyPort uint16 = 8080 - + s.SetupNginxServer() configureVppProxy(s, "http", proxyPort) proxyUri := fmt.Sprintf("http://%s:%d", s.VppProxyAddr(), proxyPort) - targetUri := fmt.Sprintf("http://%s:%d/upload/testFile", s.NginxAddr(), s.NginxPort()) + targetUri := fmt.Sprintf("http://%s:%d/upload/testFile", s.ServerAddr(), s.ServerPort()) s.CurlUploadResourceViaTunnel(targetUri, proxyUri, CurlContainerTestFile) } diff --git a/extras/hs-test/topo-containers/vppProxy.yaml b/extras/hs-test/topo-containers/vppProxy.yaml index a1f24bbc187..73a02b0031b 100644 --- a/extras/hs-test/topo-containers/vppProxy.yaml +++ b/extras/hs-test/topo-containers/vppProxy.yaml @@ -9,6 +9,18 @@ containers: - <<: *shared-vol container-dir: "/tmp/vpp" is-default-work-dir: true + - name: "iperfB" + volumes: + - <<: *shared-vol + container-dir: "/tmp/vpp" + is-default-work-dir: true + is-optional: true + - name: "iperfA" + volumes: + - <<: *shared-vol + container-dir: "/tmp/vpp" + is-default-work-dir: true + is-optional: true - name: "nginx-server" volumes: - <<: *shared-vol