hs-test: add test suite features

Test suite now supports assertions which on fail stop test case run,
also it allows to create docker containers which are going to be
stopped automatically after the test run is finished.

Type: improvement
Signed-off-by: Maros Ondrejicka <maros.ondrejicka@pantheon.tech>
Change-Id: I2834709b1efd17b8182d36cc0404b986b4ed595d
Signed-off-by: Filip Tehlar <ftehlar@cisco.com>

extras/hs-test/README.rst

@@ -45,10 +45,11 @@ For adding a new suite, please see `Modifying the framework`_ below.
 #. Implement test behaviour inside the test method. This typically includes the following:
 
   #. Start docker container(s) as needed. Function ``dockerRun(instance, args string)``
-     from ``utils.go`` serves this purpose. Alternatively use suite struct's ``NewContainer(name string)`` method
+     from ``utils.go`` serves this purpose. Alternatively use suite struct's ``NewContainer(name string)`` method to create
+     an object representing a container and start it with ``run()`` method
   #. Execute *hs-test* action(s) inside any of the running containers.
      Function ``hstExec`` from ``utils.go`` does this by using ``docker exec`` command to run ``hs-test`` executable.
-     For starting an VPP instance inside a container, the ``Vpp`` struct can be used as a forward-looking alternative
+     For starting an VPP instance inside a container, the ``VppInstance`` struct can be used as a forward-looking alternative
   #. Run arbitrary commands inside the containers with ``dockerExec(cmd string, instance string)``
   #. Run other external tool with one of the preexisting functions in the ``utils.go`` file.
      For example, use ``wget`` with ``startWget(..)`` function
@@ -124,14 +125,13 @@ Modifying the framework
 
 #. Adding a new suite takes place in ``framework_test.go``
 
-#. Make a ``struct`` with at least ``HstSuite`` struct and a ``teardownSuite`` function as its members.
+#. Make a ``struct`` with at least ``HstSuite`` struct as its member.
    HstSuite provides functionality that can be shared for all suites, like starting containers
 
         ::
 
                 type MySuite struct {
                         HstSuite
-                        teardownSuite func()
                 }
 
 #. Implement SetupSuite method which testify runs before running the tests.
@@ -147,17 +147,6 @@ Modifying the framework
                         s.teardownSuite = setupSuite(&s.Suite, "myTopology")
                 }
 
-#. Implement TearDownSuite method which testify runs after the tests, to clean-up.
-   It's good idea to add at least the suite's own ``teardownSuite()``
-   and HstSuite upper suite's ``stopContainers()`` methods
-
-        ::
-
-                func (s *MySuite) TearDownSuite() {
-                        s.teardownSuite()
-                        s.StopContainers()
-                }
-
 #. In order for ``go test`` to run this suite, we need to create a normal test function and pass our suite to ``suite.Run``
 
         ::

extras/hs-test/actions.go

@@ -236,20 +236,26 @@ func (a *Actions) Configure2Veths(args []string) *ActionResult {
 
 	ctx, cancel := newVppContext()
 	defer cancel()
+
+	vppConfig, err := DeserializeVppConfig(args[2])
+	if err != nil {
+		return NewActionResult(err, ActionResultWithDesc("deserializing configuration failed"))
+	}
+
 	con, vppErrCh := vpphelper.StartAndDialContext(ctx,
-		vpphelper.WithVppConfig(configTemplate+startup.ToString()),
+		vpphelper.WithVppConfig(vppConfig.getTemplate()+startup.ToString()),
 		vpphelper.WithRootDir(fmt.Sprintf("/tmp/%s", args[1])))
 	exitOnErrCh(ctx, cancel, vppErrCh)
 
 	var fn func(context.Context, api.Connection) error
-	if args[2] == "srv" {
+	if vppConfig.Variant == "srv" {
 		fn = configure2vethsTopo("vppsrv", "10.10.10.1/24", "1", 1)
-	} else if args[2] == "srv-with-preset-hw-addr" {
+	} else if vppConfig.Variant == "srv-with-preset-hw-addr" {
 		fn = configure2vethsTopo("vppsrv", "10.10.10.1/24", "1", 1, "00:00:5e:00:53:01")
 	} else {
 		fn = configure2vethsTopo("vppcln", "10.10.10.2/24", "2", 2)
 	}
-	err := fn(ctx, con)
+	err = fn(ctx, con)
 	if err != nil {
 		return NewActionResult(err, ActionResultWithDesc("configuration failed"))
 	}

extras/hs-test/container.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/edwarnicke/exechelper"
+)
+
+type Container struct {
+	name string
+}
+
+func (c *Container) run() error {
+	if c.name == "" {
+		return fmt.Errorf("create volume failed: container name is blank")
+	}
+
+	exechelper.Run(fmt.Sprintf("mkdir -p /tmp/%s/sync", c.name))
+	syncPath := fmt.Sprintf("-v /tmp/%s/sync:/tmp/sync", c.name)
+	cmd := "docker run --cap-add=all -d --privileged --network host --rm "
+	cmd += syncPath
+	cmd += " --name " + c.name + " hs-test/vpp"
+	fmt.Println(cmd)
+	err := exechelper.Run(cmd)
+	if err != nil {
+		return fmt.Errorf("create volume failed: %s", err)
+	}
+
+	return nil
+}
+

extras/hs-test/echo_test.go

@@ -6,46 +6,26 @@ import (
 	"github.com/edwarnicke/exechelper"
 )
 
-func (s *Veths2Suite) TestEchoBuiltin() {
-	t := s.T()
+func (s *VethsSuite) TestEchoBuiltin() {
 	srvInstance := "echo-srv-internal"
 	clnInstance := "echo-cln-internal"
-	err := dockerRun(srvInstance, "")
-	if err != nil {
-		t.Errorf("%v", err)
-		return
-	}
+
+	s.assertNil(dockerRun(srvInstance, ""), "failed to start docker (srv)")
 	defer func() { exechelper.Run("docker stop " + srvInstance) }()
 
-	err = dockerRun(clnInstance, "")
-	if err != nil {
-		t.Errorf("%v", err)
-		return
-	}
+	s.assertNil(dockerRun(clnInstance, ""), "failed to start docker (cln)")
 	defer func() { exechelper.Run("docker stop " + clnInstance) }()
 
-	_, err = hstExec("Configure2Veths srv", srvInstance)
-	if err != nil {
-		t.Errorf("%v", err)
-		return
-	}
+	_, err := hstExec("Configure2Veths srv", srvInstance)
+	s.assertNil(err)
 
 	_, err = hstExec("Configure2Veths cln", clnInstance)
-	if err != nil {
-		t.Errorf("%v", err)
-		return
-	}
+	s.assertNil(err)
 
 	_, err = hstExec("RunEchoSrvInternal private-segment-size 1g fifo-size 4 no-echo", srvInstance)
-	if err != nil {
-		t.Errorf("%v", err)
-		return
-	}
+	s.assertNil(err)
 
 	o, err := hstExec("RunEchoClnInternal nclients 10000 bytes 1 syn-timeout 100 test-timeout 100 no-return private-segment-size 1g fifo-size 4", clnInstance)
-	if err != nil {
-		t.Errorf("%v", err)
-		return
-	}
+	s.assertNil(err)
 	fmt.Println(o)
 }

extras/hs-test/framework_test.go

@@ -1,15 +1,82 @@
 package main
 
 import (
+	"fmt"
 	"testing"
 	"time"
 
+	"github.com/edwarnicke/exechelper"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
 )
 
-type TapSuite struct {
+type HstSuite struct {
 	suite.Suite
 	teardownSuite func()
+	containers    []string
+	volumes       []string
+}
+
+func (s *HstSuite) TearDownSuite() {
+	s.teardownSuite()
+	s.StopContainers()
+	s.RemoveVolumes()
+}
+
+func (s *HstSuite) hstFail() {
+	s.T().FailNow()
+}
+
+func (s *HstSuite) assertNil(object interface{}, msgAndArgs ...interface{}) {
+	if !assert.Nil(s.T(), object, msgAndArgs...) {
+		s.hstFail()
+	}
+}
+
+func (s *HstSuite) assertNotNil(object interface{}, msgAndArgs ...interface{}) {
+	if !assert.NotNil(s.T(), object, msgAndArgs...) {
+		s.hstFail()
+	}
+}
+
+func (s *HstSuite) assertEqual(expected, actual interface{}, msgAndArgs ...interface{}) {
+	if !assert.Equal(s.T(), expected, actual, msgAndArgs...) {
+		s.hstFail()
+	}
+}
+
+func (s *HstSuite) assertNotContains(testString, contains interface{}, msgAndArgs ...interface{}) {
+	if !assert.NotContains(s.T(), testString, contains, msgAndArgs...) {
+		s.hstFail()
+	}
+}
+
+func (s *HstSuite) NewContainer(name string) (*Container, error) {
+	if name == "" {
+		return nil, fmt.Errorf("creating container failed: name must not be blank")
+	}
+
+	s.containers = append(s.containers, name)
+
+	container := new(Container)
+	container.name = name
+	return container, nil
+}
+
+func (s *HstSuite) StopContainers() {
+	for _, containerName := range s.containers {
+		exechelper.Run("docker stop " + containerName)
+	}
+}
+
+func (s *HstSuite) RemoveVolumes() {
+	for _, volumeName := range s.volumes {
+		exechelper.Run("docker volume rm " + volumeName)
+	}
+}
+
+type TapSuite struct {
+	HstSuite
 }
 
 func (s *TapSuite) SetupSuite() {
@@ -17,37 +84,23 @@ func (s *TapSuite) SetupSuite() {
 	s.teardownSuite = setupSuite(&s.Suite, "tap")
 }
 
-func (s *TapSuite) TearDownSuite() {
-	s.teardownSuite()
+type VethsSuite struct {
+	HstSuite
 }
 
-type Veths2Suite struct {
-	suite.Suite
-	teardownSuite func()
-}
-
-func (s *Veths2Suite) SetupSuite() {
+func (s *VethsSuite) SetupSuite() {
 	time.Sleep(1 * time.Second)
 	s.teardownSuite = setupSuite(&s.Suite, "2peerVeth")
 }
 
-func (s *Veths2Suite) TearDownSuite() {
-	s.teardownSuite()
-}
-
 type NsSuite struct {
-	suite.Suite
-	teardownSuite func()
+	HstSuite
 }
 
 func (s *NsSuite) SetupSuite() {
 	s.teardownSuite = setupSuite(&s.Suite, "ns")
 }
 
-func (s *NsSuite) TearDownSuite() {
-	s.teardownSuite()
-}
-
 func setupSuite(s *suite.Suite, topologyName string) func() {
 	t := s.T()
 	topology, err := LoadTopology(TopologyDir, topologyName)
@@ -75,7 +128,7 @@ func TestNs(t *testing.T) {
 	suite.Run(t, &m)
 }
 
-func TestVeths2(t *testing.T) {
-	var m Veths2Suite
+func TestVeths(t *testing.T) {
+	var m VethsSuite
 	suite.Run(t, &m)
 }

extras/hs-test/http_test.go

@@ -37,7 +37,7 @@ func (s *NsSuite) TestHttpTps() {
 	}
 }
 
-func (s *Veths2Suite) TestHttpCli() {
+func (s *VethsSuite) TestHttpCli() {
 	t := s.T()
 
 	srvInstance := "http-cli-srv"

extras/hs-test/ldp_test.go

@@ -8,7 +8,7 @@ import (
 	"github.com/edwarnicke/exechelper"
 )
 
-func (s *Veths2Suite) TestLDPreloadIperfVpp() {
+func (s *VethsSuite) TestLDPreloadIperfVpp() {
 	t := s.T()
 	var clnVclConf, srvVclConf Stanza
 

extras/hs-test/utils.go

@@ -16,6 +16,7 @@ import (
 	"github.com/edwarnicke/exechelper"
 )
 
+// TODO remove `configTemplate` once its usage has been replaced everywhere with VppConfig
 const configTemplate = `unix {
   nodaemon
   log %[1]s/var/log/vpp/vpp.log

extras/hs-test/vcl_test.go

@@ -7,21 +7,21 @@ import (
 	"github.com/edwarnicke/exechelper"
 )
 
-func (s *Veths2Suite) TestVclEchoQuic() {
+func (s *VethsSuite) TestVclEchoQuic() {
 	s.T().Skip("quic test skipping..")
 	s.testVclEcho("quic")
 }
 
-func (s *Veths2Suite) TestVclEchoUdp() {
+func (s *VethsSuite) TestVclEchoUdp() {
 	s.T().Skip("udp echo currently broken in vpp, skipping..")
 	s.testVclEcho("udp")
 }
 
-func (s *Veths2Suite) TestVclEchoTcp() {
+func (s *VethsSuite) TestVclEchoTcp() {
 	s.testVclEcho("tcp")
 }
 
-func (s *Veths2Suite) testVclEcho(proto string) {
+func (s *VethsSuite) testVclEcho(proto string) {
 	t := s.T()
 
 	exechelper.Run("docker volume create --name=echo-srv-vol")
@@ -86,12 +86,12 @@ func (s *Veths2Suite) testVclEcho(proto string) {
 	fmt.Println(o)
 }
 
-func (s *Veths2Suite) TestVclRetryAttach() {
+func (s *VethsSuite) TestVclRetryAttach() {
 	s.T().Skip()
 	s.testRetryAttach("tcp")
 }
 
-func (s *Veths2Suite) testRetryAttach(proto string) {
+func (s *VethsSuite) testRetryAttach(proto string) {
 	t := s.T()
 
 	exechelper.Run("docker volume create --name=echo-srv-vol")

extras/hs-test/vppinstance.go

@@ -0,0 +1,124 @@
+package main
+
+import (
+	"fmt"
+	"encoding/json"
+	"github.com/edwarnicke/exechelper"
+)
+
+const vppConfigTemplate = `unix {
+  nodaemon
+  log %[1]s/var/log/vpp/vpp.log
+  full-coredump
+  cli-listen %[1]s%[2]s
+  runtime-dir %[1]s/var/run
+  gid vpp
+}
+
+api-trace {
+  on
+}
+
+api-segment {
+  gid vpp
+}
+
+socksvr {
+  socket-name %[1]s/var/run/vpp/api.sock
+}
+
+statseg {
+  socket-name %[1]s/var/run/vpp/stats.sock
+}
+
+plugins {
+  plugin unittest_plugin.so { enable }
+  plugin dpdk_plugin.so { disable }
+  plugin crypto_aesni_plugin.so { enable }
+  plugin quic_plugin.so { enable }
+}
+
+`
+
+type VppInstance struct {
+	container *Container
+	config VppConfig
+	actionFuncName string
+}
+
+type VppConfig struct {
+	Variant string
+	CliSocketFilePath string
+}
+
+func (vc *VppConfig) getTemplate() string {
+	return fmt.Sprintf(vppConfigTemplate, "%[1]s", vc.CliSocketFilePath)
+}
+
+func (vpp *VppInstance) set2VethsServer() {
+	vpp.actionFuncName = "Configure2Veths"
+	vpp.config.Variant = "srv"
+}
+
+func (vpp *VppInstance) set2VethsClient() {
+	vpp.actionFuncName = "Configure2Veths"
+	vpp.config.Variant = "cln"
+}
+
+func (vpp *VppInstance) setCliSocket(filePath string) {
+	vpp.config.CliSocketFilePath = filePath
+}
+
+func (vpp *VppInstance) getCliSocket() string {
+	return fmt.Sprintf("/tmp/%s/%s", vpp.actionFuncName, vpp.config.CliSocketFilePath)
+}
+
+func (vpp *VppInstance) start() error {
+	if vpp.config.Variant == "" {
+		return fmt.Errorf("vpp start failed: variant must not be blank")
+	}
+	if vpp.actionFuncName == "" {
+		return fmt.Errorf("vpp start failed: action function name must not be blank")
+	}
+
+	serializedConfig, err := json.Marshal(vpp.config)
+	if err != nil {
+		return fmt.Errorf("vpp start failed: serializing configuration failed: %s", err)
+	}
+	args := fmt.Sprintf("%s '%s'", vpp.actionFuncName, string(serializedConfig))
+	_, err = hstExec(args, vpp.container.name)
+	if err != nil {
+		return fmt.Errorf("vpp start failed: %s", err)
+	}
+
+	return nil
+}
+
+func (vpp *VppInstance) vppctl(command string) (string, error) {
+	dockerExecCommand := fmt.Sprintf("docker exec --detach=false %[1]s vppctl -s %[2]s %[3]s",
+		vpp.container.name, vpp.getCliSocket(), command)
+	output, err := exechelper.CombinedOutput(dockerExecCommand)
+	if err != nil {
+		return "", fmt.Errorf("vppctl failed: %s", err)
+	}
+
+	return string(output), nil
+}
+
+func NewVppInstance(c *Container) *VppInstance {
+	vpp := new(VppInstance)
+	vpp.container = c
+	return vpp
+}
+
+func DeserializeVppConfig(input string) (VppConfig, error) {
+	var vppConfig VppConfig
+	err := json.Unmarshal([]byte(input), &vppConfig)
+	if err != nil {
+		// Since input is not a  valid JSON it is going be used as variant value
+		// for compatibility reasons
+		vppConfig.Variant = input
+		vppConfig.CliSocketFilePath = "/var/run/vpp/cli.sock"
+	}
+	return vppConfig, nil
+}