hs-test: improve test infra

- add support for building/running debug/release images
- have one point of control (Makefile)
- list all test cases

Type: test

Signed-off-by: Filip Tehlar <ftehlar@cisco.com>
Change-Id: I97949abc2fff85d7a2b3784122be159aeec72b52

.gitignore

@@ -128,3 +128,4 @@ compile_commands.json
 /extras/hs-test/vpp-data
 /extras/hs-test/hs-test
 /extras/hs-test/http_server
+/extras/hs-test/.build.vpp

extras/hs-test/Makefile

@@ -1,16 +1,73 @@
-all: build docker
 
-install-dep:
-	apt update -y && apt install -y golang docker-ce apache2-utils wrk bridge-utils
+ifeq ($(VERBOSE),)
+VERBOSE=false
+endif
 
-build:
+ifeq ($(PERSIST),)
+PERSIST=false
+endif
+
+ifeq ($(TEST),)
+TEST=all
+endif
+
+list_tests = @(grep -r ') Test' *_test.go | cut -d '*' -f2 | cut -d '(' -f1 | \
+		tr -d ' ' | tr ')' '/' | sed 's/Suite//')
+
+.PHONY: help
+help:
+	@echo "Make targets:"
+	@echo " test                 - run tests"
+	@echo " build                - build test infra"
+	@echo " build-debug          - build test infra (vpp debug image)"
+	@echo " build-go             - just build golang files"
+	@echo " fixstyle             - format .go source files"
+	@echo " list-tests           - list all tests"
+	@echo
+	@echo "Make arguments:"
+	@echo " UBUNTU_VERSION           - ubuntu version for docker image"
+	@echo " PERSIST=[true|false]     - whether clean up topology and dockers after test"
+	@echo " VERBOSE=[true|false]     - verbose output"
+	@echo " TEST=[test-name]         - specific test to run"
+	@echo
+	@echo "List of all tests:"
+	$(call list_tests)
+
+.PHONY: list-tests
+list-tests:
+	$(call list_tests)
+
+build-vpp-release:
+	@make -C ../.. build-release
+
+build-vpp-debug:
+	@make -C ../.. build
+
+.PHONY: test
+test: .deps.ok .build.vpp
+	@bash ./test --persist=$(PERSIST) --verbose=$(VERBOSE) --test=$(TEST)
+
+build-go:
 	go build ./tools/http_server
 
-docker:
-	bash ./script/build.sh
+build: .deps.ok build-vpp-release build-go
+	@rm .build.vpp || exit 0
+	bash ./script/build.sh release
+	@touch .build.vpp
 
+build-debug: .deps.ok build-vpp-debug build-go
+	@rm .build.vpp || exit 0
+	bash ./script/build.sh debug
+	@touch .build.vpp
+
+.PHONY: install-deps
+install-deps:
+	@rm .deps.ok || exit 0
+	@apt update -y && apt install -y golang docker-ce apache2-utils wrk bridge-utils
+	@touch .deps.ok
+
+.PHONY: fixstyle
 fixstyle:
 	@gofmt -w .
 	@go mod tidy
 
-.PHONY: docker

extras/hs-test/README.rst

@@ -16,14 +16,16 @@ Anatomy of a test case
 
 **Prerequisites**:
 
-* Tests use *hs-test*'s own docker image, so building it before starting tests is a prerequisite. Run ``sudo make`` to do so
+* Install hs-test dependencies with ``make install-deps``
+* Tests use *hs-test*'s own docker image, so building it before starting tests is a prerequisite. Run ``make build[-debug]`` to do so
 * Docker has to be installed and Go has to be in path of both the running user and root
 * Root privileges are required to run tests as it uses Linux ``ip`` command for configuring topology
 
 **Action flow when running a test case**:
 
-#. It starts with running ``./test``. This script is basically a wrapper for ``go test`` and accepts its parameters,
-   for example following runs a specific test: ``./test -run TestNs/TestHttpTps``
+#. It starts with running ``make test``. Optional arguments are VERBOSE, PERSIST (topology configuration isn't cleaned up after test run),
+   and TEST=<test-name> to run specific test.
+#. ``make list-tests`` (or ``make help``) shows all test names.
 #. ``go test`` compiles package ``main`` along with any files with names matching the file pattern ``*_test.go``
    and then runs the resulting test binaries
 #. The go test framework runs each function matching :ref:`naming convention<test-convention>`. Each of these corresponds to a `test suite`_

extras/hs-test/hst_suite.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"flag"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -15,13 +16,8 @@ const (
 	defaultNetworkNumber int = 1
 )
 
-func IsPersistent() bool {
-	return os.Getenv("HST_PERSIST") == "1"
-}
-
-func IsVerbose() bool {
-	return os.Getenv("HST_VERBOSE") == "1"
-}
+var IsPersistent = flag.Bool("persist", false, "persists topology config")
+var IsVerbose = flag.Bool("verbose", false, "verbose test output")
 
 type HstSuite struct {
 	suite.Suite
@@ -37,7 +33,7 @@ func (s *HstSuite) TearDownSuite() {
 }
 
 func (s *HstSuite) TearDownTest() {
-	if IsPersistent() {
+	if *IsPersistent {
 		return
 	}
 	s.ResetContainers()
@@ -112,7 +108,7 @@ func (s *HstSuite) assertNotEmpty(object interface{}, msgAndArgs ...interface{})
 }
 
 func (s *HstSuite) log(args ...any) {
-	if IsVerbose() {
+	if *IsVerbose {
 		s.T().Log(args...)
 	}
 }
@@ -241,7 +237,7 @@ func (s *HstSuite) configureNetworkTopology(topologyName string) {
 }
 
 func (s *HstSuite) unconfigureNetworkTopology() {
-	if IsPersistent() {
+	if *IsPersistent {
 		return
 	}
 	for _, nc := range s.netConfigs {

extras/hs-test/script/build.sh

@@ -17,23 +17,47 @@ if [ -z $(which wrk) ]; then
 	exit 1
 fi
 
-source vars
+export VPP_WS=../..
+
+if [ "$1" == "debug" ]; then
+	VPP_BUILD_ROOT=${VPP_WS}/build-root/build-vpp_debug-native/vpp
+else
+	VPP_BUILD_ROOT=${VPP_WS}/build-root/build-vpp-native/vpp
+fi
+echo "Taking build objects from ${VPP_BUILD_ROOT}"
+
+if [ -z "$UBUNTU_VERSION" ] ; then
+	export UBUNTU_VERSION=$(lsb_release -rs)
+fi
+echo "Ubuntu version is set to ${UBUNTU_VERSION}"
+
+export HST_LDPRELOAD=${VPP_BUILD_ROOT}/lib/x86_64-linux-gnu/libvcl_ldpreload.so
+echo "HST_LDPRELOAD is set to ${HST_LDPRELOAD}"
+
+export PATH=${VPP_BUILD_ROOT}/bin:$PATH
 
 bin=vpp-data/bin
 lib=vpp-data/lib
 
 mkdir -p ${bin} ${lib} || true
+rm -rf vpp-data/bin/* || true
+rm -rf vpp-data/lib/* || true
 
-cp ${VPP_WS}/build-root/build-vpp_debug-native/vpp/bin/* ${bin}
+cp ${VPP_BUILD_ROOT}/bin/* ${bin}
 res+=$?
-cp -r ${VPP_WS}/build-root/build-vpp_debug-native/vpp/lib/x86_64-linux-gnu/* ${lib}
+cp -r ${VPP_BUILD_ROOT}/lib/x86_64-linux-gnu/* ${lib}
 res+=$?
 if [ $res -ne 0 ]; then
 	echo "Failed to copy VPP files. Is VPP built? Try running 'make build' in VPP directory."
 	exit 1
 fi
 
-docker build --build-arg UBUNTU_VERSION --build-arg http_proxy=$HTTP_PROXY \
-	-t hs-test/vpp -f docker/Dockerfile.vpp .
-docker build --build-arg UBUNTU_VERSION --build-arg http_proxy=$HTTP_PROXY \
-	-t hs-test/nginx-ldp -f docker/Dockerfile.nginx .
+docker_build () {
+    tag=$1
+    dockername=$2
+    docker build --build-arg UBUNTU_VERSION --build-arg http_proxy=$HTTP_PROXY \
+            -t $tag  -f docker/Dockerfile.$dockername .
+}
+
+docker_build hs-test/vpp vpp
+docker_build hs-test/nginx-ldp nginx

extras/hs-test/test

@@ -2,17 +2,38 @@
 
 source vars
 
-for ARG in "$@"
+args=
+single_test=0
+persist_set=0
+
+for i in "$@"
 do
-	if [[ "$ARG" = "-p" ]]
-	then
-		export HST_PERSIST=1
-		shift
-	elif [[ "$ARG" = "-v" ]]
-	then
-		export HST_VERBOSE=1
-		shift
-	fi
+case "${i}" in
+    --persist=*)
+        persist="${i#*=}"
+        if [ $persist = "true" ]; then
+            args="$args -persist"
+            persist_set=1
+        fi
+        ;;
+    --verbose=*)
+        verbose="${i#*=}"
+        if [ $verbose = "true" ]; then
+            args="$args -verbose"
+        fi
+        ;;
+    --test=*)
+        tc_name="${i#*=}"
+        if [ $tc_name != "all" ]; then
+            single_test=1
+            args="$args -run $tc_name"
+        fi
+esac
 done
 
-sudo -E go test -buildvcs=false -v $@
+if [ $single_test -eq 0 ] && [ $persist_set -eq 1 ]; then
+    echo "persist flag is not supperted while running all tests!"
+    exit 1
+fi
+
+sudo -E go test -buildvcs=false -v $args

extras/hs-test/vars

@@ -1,6 +1,6 @@
 export VPP_WS=../../
 
-export HST_LDPRELOAD=${VPP_WS}/build-root/build-vpp_debug-native/vpp/lib/x86_64-linux-gnu/libvcl_ldpreload.so
-export PATH=${VPP_WS}/build-root/build-vpp_debug-native/vpp/bin:$PATH
+export HST_LDPRELOAD=${VPP_WS}/build-root/build-vpp-native/vpp/lib/x86_64-linux-gnu/libvcl_ldpreload.so
+export PATH=${VPP_WS}/build-root/build-vpp-native/vpp/bin:$PATH
 
 export UBUNTU_VERSION=$(lsb_release -rs)