ligang/vpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

hs-test: added nginx multi-thread tests

Browse Source 
- added Dockerfile.envoy
- removed nginx vcl.conf file as it's created by
  the framework now

Type: test

Change-Id: I5f2be015c864c8d2aa938a22b1abece64989999b
Signed-off-by: Adrian Villin <avillin@cisco.com>
This commit is contained in:
Adrian Villin
2024-09-19 17:19:39 +02:00
committed by Florin Coras
parent fae41c6783
commit 46ab0b22bb
16 changed files with 168 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
extras/hs-test/docker/Dockerfile.envoy Normal file
View File

@ -0,0 +1,6 @@
FROM envoyproxy/envoy-contrib:v1.31-latest
RUN chmod go+r /etc/envoy/envoy.yaml
RUN chown envoy:envoy /dev/stdout /dev/stderr
ENTRYPOINT ["/bin/sh", "-c", "envoy --log-format [%t][%l][%g:%#]%_ --concurrency 2 -c /etc/envoy/envoy.yaml"]

3
extras/hs-test/docker/Dockerfile.nginx
View File

@ -7,11 +7,10 @@ RUN apt-get update \
&& rm -rf /var/lib/apt/lists/*
COPY vpp-data/lib/* /usr/lib/
COPY resources/nginx/vcl.conf /vcl.conf
COPY resources/nginx/nginx.conf /nginx.conf
COPY script/nginx_ldp.sh /usr/bin/nginx_ldp.sh
ENV VCL_CONFIG=/vcl.conf
ENV VCL_CONFIG=/tmp/nginx/vcl.conf
ENV LDP=/usr/lib/libvcl_ldpreload.so
ENV LDP_DEBUG=0
ENV VCL_DEBUG=0

21
extras/hs-test/infra/hst_suite.go
View File

@ -322,20 +322,25 @@ func (s *HstSuite) SkipIfMultiWorker(args ...any) {
}
}
func (s *HstSuite) SkipIfNotEnoughAvailableCpus() bool {
var MaxRequestedCpu int
func (s *HstSuite) SkipIfNotEnoughAvailableCpus() {
var maxRequestedCpu int
availableCpus := len(s.CpuAllocator.cpus) - 1
if *UseCpu0 {
availableCpus++
}
if s.CpuAllocator.runningInCi {
MaxRequestedCpu = ((s.CpuAllocator.buildNumber + 1) * s.CpuAllocator.maxContainerCount * s.CpuCount)
maxRequestedCpu = ((s.CpuAllocator.buildNumber + 1) * s.CpuAllocator.maxContainerCount * s.CpuCount)
} else {
MaxRequestedCpu = (GinkgoParallelProcess() * s.CpuAllocator.maxContainerCount * s.CpuCount)
maxRequestedCpu = (GinkgoParallelProcess() * s.CpuAllocator.maxContainerCount * s.CpuCount)
}
if len(s.CpuAllocator.cpus)-1 < MaxRequestedCpu {
s.Skip(fmt.Sprintf("test case cannot allocate requested cpus (%d cpus * %d containers)", s.CpuCount, s.CpuAllocator.maxContainerCount))
if availableCpus < maxRequestedCpu {
s.Skip(fmt.Sprintf("Test case cannot allocate requested cpus "+
"(%d cpus * %d containers, %d available). Try using 'CPU0=true'",
s.CpuCount, s.CpuAllocator.maxContainerCount, availableCpus))
}
return true
}
func (s *HstSuite) SkipUnlessLeakCheck() {

1
extras/hs-test/infra/suite_envoy_proxy.go
View File

@ -96,6 +96,7 @@ func (s *EnvoyProxySuite) SetupTest() {
// Envoy
envoyContainer := s.GetContainerByName(EnvoyProxyContainerName)
s.AssertNil(envoyContainer.Create())
s.proxyPort = 8080
envoySettings := struct {
LogPrefix string

75
extras/hs-test/infra/suite_nginx_proxy.go
View File

@ -65,26 +65,9 @@ func (s *NginxProxySuite) SetupTest() {
s.AssertNil(vpp.createTap(serverInterface, 2))
// nginx proxy
nginxProxyContainer := s.GetTransientContainerByName(NginxProxyContainerName)
nginxProxyContainer := s.GetContainerByName(NginxProxyContainerName)
s.AssertNil(nginxProxyContainer.Create())
s.proxyPort = 80
values := struct {
LogPrefix string
Proxy string
Server string
Port uint16
}{
LogPrefix: nginxProxyContainer.Name,
Proxy: clientInterface.Peer.Ip4AddressString(),
Server: serverInterface.Ip4AddressString(),
Port: s.proxyPort,
}
nginxProxyContainer.CreateConfig(
"/nginx.conf",
"./resources/nginx/nginx_proxy_mirroring.conf",
values,
)
s.AssertNil(nginxProxyContainer.Start())
// nginx HTTP server
nginxServerContainer := s.GetTransientContainerByName(NginxServerContainerName)
@ -104,8 +87,6 @@ func (s *NginxProxySuite) SetupTest() {
nginxSettings,
)
s.AssertNil(nginxServerContainer.Start())
vpp.WaitForApp("nginx-", 5)
}
func (s *NginxProxySuite) TearDownTest() {
@ -116,6 +97,35 @@ func (s *NginxProxySuite) TearDownTest() {
s.HstSuite.TearDownTest()
}
func (s *NginxProxySuite) CreateNginxProxyConfig(container *Container, multiThreadWorkers bool) {
clientInterface := s.GetInterfaceByName(MirroringClientInterfaceName)
serverInterface := s.GetInterfaceByName(MirroringServerInterfaceName)
var workers uint8
if multiThreadWorkers {
workers = 2
} else {
workers = 1
}
values := struct {
Workers uint8
LogPrefix string
Proxy string
Server string
Port uint16
}{
Workers: workers,
LogPrefix: container.Name,
Proxy: clientInterface.Peer.Ip4AddressString(),
Server: serverInterface.Ip4AddressString(),
Port: s.proxyPort,
}
container.CreateConfig(
"/nginx.conf",
"./resources/nginx/nginx_proxy_mirroring.conf",
values,
)
}
func (s *NginxProxySuite) ProxyPort() uint16 {
return s.proxyPort
}
@ -132,6 +142,31 @@ func (s *NginxProxySuite) CurlDownloadResource(uri string) {
s.AssertNotContains(log, "Operation timed out")
}
func (s *NginxProxySuite) AddVclConfig(container *Container, multiThreadWorkers bool) {
var vclConf Stanza
vclFileName := container.GetHostWorkDir() + "/vcl.conf"
appSocketApi := fmt.Sprintf("app-socket-api %s/var/run/app_ns_sockets/default",
container.GetContainerWorkDir())
vclConf.
NewStanza("vcl").
Append("heapsize 64M").
Append("rx-fifo-size 4000000").
Append("tx-fifo-size 4000000").
Append("segment-size 4000000000").
Append("add-segment-size 4000000000").
Append("event-queue-size 100000").
Append("use-mq-eventfd").
Append(appSocketApi)
if multiThreadWorkers {
vclConf.Append("multi-thread-workers")
}
err := vclConf.Close().SaveToFile(vclFileName)
s.AssertNil(err, fmt.Sprint(err))
}
var _ = Describe("NginxProxySuite", Ordered, ContinueOnFailure, func() {
var s NginxProxySuite
BeforeAll(func() {

47
extras/hs-test/infra/suite_no_topo.go
View File

@ -1,6 +1,7 @@
package hst
import (
"fmt"
"reflect"
"runtime"
"strings"
@ -68,6 +69,50 @@ func (s *NoTopoSuite) TearDownTest() {
s.HstSuite.TearDownTest()
}
func (s *NoTopoSuite) CreateNginxConfig(container *Container, multiThreadWorkers bool) {
var workers uint8
if multiThreadWorkers {
workers = 2
} else {
workers = 1
}
values := struct {
Workers uint8
}{
Workers: workers,
}
container.CreateConfig(
"/nginx.conf",
"./resources/nginx/nginx.conf",
values,
)
}
func (s *NoTopoSuite) AddNginxVclConfig(multiThreadWorkers bool) {
nginxCont := s.GetContainerByName(SingleTopoContainerNginx)
vclFileName := nginxCont.GetHostWorkDir() + "/vcl.conf"
appSocketApi := fmt.Sprintf("app-socket-api %s/var/run/app_ns_sockets/default",
nginxCont.GetContainerWorkDir())
var vclConf Stanza
vclConf.
NewStanza("vcl").
Append("heapsize 64M").
Append("rx-fifo-size 4000000").
Append("tx-fifo-size 4000000").
Append("segment-size 4000000000").
Append("add-segment-size 4000000000").
Append("event-queue-size 100000").
Append("use-mq-eventfd").
Append(appSocketApi)
if multiThreadWorkers {
vclConf.Append("multi-thread-workers")
}
err := vclConf.Close().SaveToFile(vclFileName)
s.AssertNil(err, fmt.Sprint(err))
}
func (s *NoTopoSuite) VppAddr() string {
return s.GetInterfaceByName(TapInterfaceName).Peer.Ip4AddressString()
}
@ -80,7 +125,7 @@ func (s *NoTopoSuite) HostAddr() string {
return s.GetInterfaceByName(TapInterfaceName).Ip4AddressString()
}
func (s *NoTopoSuite) CreateNginxConfig(container *Container) {
func (s *NoTopoSuite) CreateNginxHttp3Config(container *Container) {
nginxSettings := struct {
LogPrefix string
}{

127
extras/hs-test/infra/suite_ns.go
View File

@ -1,127 +0,0 @@
package hst
import (
"fmt"
"reflect"
"runtime"
"strings"
. "github.com/onsi/ginkgo/v2"
)
// These correspond to names used in yaml config
const (
ClientInterface = "hclnvpp"
ServerInterface = "hsrvvpp"
)
var nsTests = map[string][]func(s *NsSuite){}
var nsSoloTests = map[string][]func(s *NsSuite){}
type NsSuite struct {
HstSuite
}
func RegisterNsTests(tests ...func(s *NsSuite)) {
nsTests[getTestFilename()] = tests
}
func RegisterNsSoloTests(tests ...func(s *NsSuite)) {
nsSoloTests[getTestFilename()] = tests
}
func (s *NsSuite) SetupSuite() {
s.HstSuite.SetupSuite()
s.ConfigureNetworkTopology("ns")
s.LoadContainerTopology("ns")
}
func (s *NsSuite) SetupTest() {
s.HstSuite.SetupTest()
// Setup test conditions
var sessionConfig Stanza
sessionConfig.
NewStanza("session").
Append("enable").
Append("use-app-socket-api").
Append("evt_qs_memfd_seg").
Append("event-queue-length 100000")
if strings.Contains(CurrentSpecReport().LeafNodeText, "InterruptMode") {
sessionConfig.Append("use-private-rx-mqs").Close()
} else {
sessionConfig.Close()
}
container := s.GetContainerByName("vpp")
vpp, _ := container.newVppInstance(container.AllocatedCpus, sessionConfig)
s.AssertNil(vpp.Start())
idx, err := vpp.createAfPacket(s.GetInterfaceByName(ServerInterface))
s.AssertNil(err, fmt.Sprint(err))
s.AssertNotEqual(0, idx)
idx, err = vpp.createAfPacket(s.GetInterfaceByName(ClientInterface))
s.AssertNil(err, fmt.Sprint(err))
s.AssertNotEqual(0, idx)
container.Exec("chmod 777 -R %s", container.GetContainerWorkDir())
}
var _ = Describe("NsSuite", Ordered, ContinueOnFailure, func() {
var s NsSuite
BeforeAll(func() {
s.SetupSuite()
})
BeforeEach(func() {
s.SetupTest()
})
AfterAll(func() {
s.TearDownSuite()
})
AfterEach(func() {
s.TearDownTest()
})
for filename, tests := range nsTests {
for _, test := range tests {
test := test
pc := reflect.ValueOf(test).Pointer()
funcValue := runtime.FuncForPC(pc)
testName := filename + "/" + strings.Split(funcValue.Name(), ".")[2]
It(testName, func(ctx SpecContext) {
s.Log(testName + ": BEGIN")
test(&s)
}, SpecTimeout(SuiteTimeout))
}
}
})
var _ = Describe("NsSuiteSolo", Ordered, ContinueOnFailure, Serial, func() {
var s NsSuite
BeforeAll(func() {
s.SetupSuite()
})
BeforeEach(func() {
s.SetupTest()
})
AfterAll(func() {
s.TearDownSuite()
})
AfterEach(func() {
s.TearDownTest()
})
for filename, tests := range nsSoloTests {
for _, test := range tests {
test := test
pc := reflect.ValueOf(test).Pointer()
funcValue := runtime.FuncForPC(pc)
testName := filename + "/" + strings.Split(funcValue.Name(), ".")[2]
It(testName, Label("SOLO"), func(ctx SpecContext) {
s.Log(testName + ": BEGIN")
test(&s)
}, SpecTimeout(SuiteTimeout))
}
}
})

1
extras/hs-test/infra/utils.go
View File

@ -254,6 +254,7 @@ func (s *HstSuite) StartWget(finished chan error, server_ip, port, query, netNs
netNs)
s.Log(cmd)
o, err := cmd.CombinedOutput()
s.Log(string(o))
if err != nil {
finished <- fmt.Errorf("wget error: '%v\n\n%s'", err, o)
return

33
extras/hs-test/nginx_test.go
View File

@ -12,13 +12,15 @@ import (
func init() {
RegisterNoTopoTests(NginxHttp3Test, NginxAsServerTest, NginxPerfCpsTest, NginxPerfRpsTest, NginxPerfWrkTest,
NginxPerfCpsInterruptModeTest, NginxPerfRpsInterruptModeTest, NginxPerfWrkInterruptModeTest)
RegisterNoTopoSoloTests(NginxPerfRpsMultiThreadTest, NginxPerfCpsMultiThreadTest)
}
func NginxHttp3Test(s *NoTopoSuite) {
query := "index.html"
nginxCont := s.GetContainerByName(NginxHttp3ContainerName)
nginxCont.Create()
s.CreateNginxConfig(nginxCont)
s.CreateNginxHttp3Config(nginxCont)
nginxCont.Start()
vpp := s.GetContainerByName("vpp").VppInstance
@ -43,7 +45,10 @@ func NginxAsServerTest(s *NoTopoSuite) {
finished := make(chan error, 1)
nginxCont := s.GetContainerByName("nginx")
nginxCont.Run()
nginxCont.Create()
s.CreateNginxConfig(nginxCont, false)
s.AddNginxVclConfig(false)
nginxCont.Start()
vpp := s.GetContainerByName("vpp").VppInstance
vpp.WaitForApp("nginx-", 5)
@ -68,7 +73,7 @@ func parseString(s, pattern string) string {
return ""
}
func runNginxPerf(s *NoTopoSuite, mode, ab_or_wrk string) error {
func runNginxPerf(s *NoTopoSuite, mode, ab_or_wrk string, multiThreadWorkers bool) error {
nRequests := 1000000
nClients := 1000
@ -77,7 +82,10 @@ func runNginxPerf(s *NoTopoSuite, mode, ab_or_wrk string) error {
vpp := s.GetContainerByName("vpp").VppInstance
nginxCont := s.GetContainerByName(SingleTopoContainerNginx)
nginxCont.Run()
nginxCont.Create()
s.AddNginxVclConfig(multiThreadWorkers)
s.CreateNginxConfig(nginxCont, multiThreadWorkers)
nginxCont.Start()
vpp.WaitForApp("nginx-", 5)
if ab_or_wrk == "ab" {
@ -92,6 +100,7 @@ func runNginxPerf(s *NoTopoSuite, mode, ab_or_wrk string) error {
args += " -r"
args += " http://" + serverAddress + ":80/64B.json"
abCont.ExtraRunningArgs = args
s.Log("Test might take up to 2 minutes to finish. Please wait")
abCont.Run()
o, err := abCont.GetOutput()
rps := parseString(o, "Requests per second:")
@ -117,18 +126,24 @@ func NginxPerfCpsInterruptModeTest(s *NoTopoSuite) {
NginxPerfCpsTest(s)
}
// unstable with multiple workers
func NginxPerfCpsMultiThreadTest(s *NoTopoSuite) {
s.AssertNil(runNginxPerf(s, "cps", "ab", true))
}
func NginxPerfCpsTest(s *NoTopoSuite) {
s.SkipIfMultiWorker()
s.AssertNil(runNginxPerf(s, "cps", "ab"))
s.AssertNil(runNginxPerf(s, "cps", "ab", false))
}
func NginxPerfRpsInterruptModeTest(s *NoTopoSuite) {
NginxPerfRpsTest(s)
}
func NginxPerfRpsMultiThreadTest(s *NoTopoSuite) {
s.AssertNil(runNginxPerf(s, "rps", "ab", true))
}
func NginxPerfRpsTest(s *NoTopoSuite) {
s.AssertNil(runNginxPerf(s, "rps", "ab"))
s.AssertNil(runNginxPerf(s, "rps", "ab", false))
}
func NginxPerfWrkInterruptModeTest(s *NoTopoSuite) {
@ -136,5 +151,5 @@ func NginxPerfWrkInterruptModeTest(s *NoTopoSuite) {
}
func NginxPerfWrkTest(s *NoTopoSuite) {
s.AssertNil(runNginxPerf(s, "", "wrk"))
s.AssertNil(runNginxPerf(s, "", "wrk", false))
}

18
extras/hs-test/proxy_test.go
View File

@ -10,6 +10,7 @@ func init() {
RegisterVppProxyTests(VppProxyHttpGetTcpTest, VppProxyHttpGetTlsTest, VppProxyHttpPutTcpTest, VppProxyHttpPutTlsTest)
RegisterEnvoyProxyTests(EnvoyProxyHttpGetTcpTest, EnvoyProxyHttpPutTcpTest)
RegisterNginxProxyTests(NginxMirroringTest)
RegisterNginxProxySoloTests(MirrorMultiThreadTest)
}
func configureVppProxy(s *VppProxySuite, proto string, proxyPort uint16) {
@ -63,9 +64,22 @@ func EnvoyProxyHttpPutTcpTest(s *EnvoyProxySuite) {
s.CurlUploadResource(uri, CurlContainerTestFile)
}
// broken when CPUS > 1
func MirrorMultiThreadTest(s *NginxProxySuite) {
nginxMirroring(s, true)
}
func NginxMirroringTest(s *NginxProxySuite) {
s.SkipIfMultiWorker()
nginxMirroring(s, false)
}
func nginxMirroring(s *NginxProxySuite, multiThreadWorkers bool) {
nginxProxyContainer := s.GetContainerByName(NginxProxyContainerName)
vpp := s.GetContainerByName(VppContainerName).VppInstance
s.AddVclConfig(nginxProxyContainer, multiThreadWorkers)
s.CreateNginxProxyConfig(nginxProxyContainer, multiThreadWorkers)
nginxProxyContainer.Start()
vpp.WaitForApp("nginx-", 5)
uri := fmt.Sprintf("http://%s:%d/httpTestFile", s.ProxyAddr(), s.ProxyPort())
s.CurlDownloadResource(uri)
}

2
extras/hs-test/resources/nginx/nginx.conf
View File

@ -1,6 +1,6 @@
master_process on;
worker_rlimit_nofile 10240;
worker_processes 2;
worker_processes {{.Workers}};
daemon off;
events {

2
extras/hs-test/resources/nginx/nginx_proxy_mirroring.conf
View File

@ -1,5 +1,5 @@
master_process on;
worker_processes 4;
worker_processes {{.Workers}};
worker_rlimit_nofile 102400;
daemon off;

1
extras/hs-test/resources/nginx/vcl.conf
View File

@ -5,7 +5,6 @@ vcl {
rx-fifo-size 4000000
tx-fifo-size 4000000
event-queue-size 100000
use-mq-eventfd
app-socket-api /tmp/nginx/var/run/app_ns_sockets/default
}

1
extras/hs-test/script/build_hst.sh
View File

@ -85,6 +85,7 @@ docker_build hs-test/vpp vpp
docker_build hs-test/nginx-ldp nginx
docker_build hs-test/nginx-server nginx-server
docker_build hs-test/curl curl
docker_build hs-test/envoy envoy
docker_build hs-test/nginx-http3 nginx-http3
# cleanup detached images

5
extras/hs-test/topo-containers/envoyProxy.yaml
View File

@ -20,9 +20,8 @@ containers:
- name: "ENVOY_UID"
value: "0"
- name: "VCL_CONFIG"
value: "/tmp/vcl.conf"
image: "envoyproxy/envoy-contrib:v1.30-latest"
extra-args: "--log-format [%t][%l][%g:%#]%_ --concurrency 2 -c /etc/envoy/envoy.yaml"
value: /tmp/vcl.conf
image: "hs-test/envoy"
is-optional: true
- name: "nginx-server"
volumes:

27
extras/hs-test/topo-containers/ns.yaml
View File

@ -1,27 +0,0 @@
---
volumes:
- volume: &shared-vol
host-dir: "$HST_VOLUME_DIR/shared-vol"
# $HST_DIR will be replaced during runtime by path to hs-test directory
containers:
- name: "vpp"
volumes:
- <<: *shared-vol
container-dir: "/tmp/vpp"
is-default-work-dir: true
- name: "envoy"
volumes:
- <<: *shared-vol
container-dir: "/tmp/vpp-envoy"
is-default-work-dir: true
- host-dir: "$HST_DIR/resources/envoy"
container-dir: "/tmp"
vars:
- name: "ENVOY_UID"
value: "0"
- name: "VCL_CONFIG"
value: "/tmp/vcl.conf"
image: "envoyproxy/envoy-contrib:v1.30-latest"
extra-args: "--concurrency 2 -c /etc/envoy/envoy.yaml"
is-optional: true