add vpp debugging support to test framework

improve test documentation

Change-Id: Ia9678aa2532ecb4cb33736aedb4a31aa3f2a3f93
Signed-off-by: Klement Sekera <ksekera@cisco.com>

.gitignore

@@ -13,6 +13,8 @@
 /build-root/*.deb
 /build-root/*.rpm
 /build-root/*.changes
+/build-root/test-doc/
+/build-roit/python/
 /build-config.mk
 /dpdk/*.tar.gz
 /dpdk/*.tar.xz

Makefile

@@ -33,7 +33,7 @@ endif
 DEB_DEPENDS  = curl build-essential autoconf automake bison libssl-dev ccache
 DEB_DEPENDS += debhelper dkms git libtool libganglia1-dev libapr1-dev dh-systemd
 DEB_DEPENDS += libconfuse-dev git-review exuberant-ctags cscope
-DEB_DEPENDS += python-dev
+DEB_DEPENDS += python-dev python-virtualenv
 ifeq ($(OS_VERSION_ID),14.04)
 	DEB_DEPENDS += openjdk-8-jdk-headless
 else
@@ -58,7 +58,7 @@ endif
 .PHONY: help bootstrap wipe wipe-release build build-release rebuild rebuild-release
 .PHONY: run run-release debug debug-release build-vat run-vat pkg-deb pkg-rpm
 .PHONY: ctags cscope plugins plugins-release build-vpp-api
-.PHONY: test test-debug retest retest-debug
+.PHONY: test test-debug retest retest-debug test-doc test-wipe-doc test-help test-wipe
 
 help:
 	@echo "Make Targets:"
@@ -78,8 +78,10 @@ help:
 	@echo " debug-release       - run release binary with debugger"
 	@echo " test                - build and run functional tests"
 	@echo " test-debug          - build and run functional tests (debug build)"
+	@echo " test-wipe           - wipe files generated by unit tests"
 	@echo " retest              - run functional tests"
 	@echo " retest-debug        - run functional tests (debug build)"
+	@echo " test-help           - show help on test framework"
 	@echo " build-vat           - build vpp-api-test tool"
 	@echo " build-vpp-api       - build vpp-api"
 	@echo " run-vat             - run vpp-api-test tool"
@@ -93,6 +95,8 @@ help:
 	@echo " doxygen             - (re)generate documentation"
 	@echo " bootstrap-doxygen   - setup Doxygen dependencies"
 	@echo " wipe-doxygen        - wipe all generated documentation"
+	@echo " test-doc            - generate documentation for test framework"
+	@echo " test-wipe-doc       - wipe documentation for test framework"
 	@echo ""
 	@echo "Make Arguments:"
 	@echo " V=[0|1]             - set build verbosity level"
@@ -105,7 +109,7 @@ help:
 	@echo " PLATFORM=<name>     - target platform. default is vpp"
 	@echo " TEST=<name>         - only run specific test"
 	@echo ""
-	@echo "Current Argumernt Values:"
+	@echo "Current Argument Values:"
 	@echo " V            = $(V)"
 	@echo " STARTUP_CONF = $(STARTUP_CONF)"
 	@echo " STARTUP_DIR  = $(STARTUP_DIR)"
@@ -188,7 +192,7 @@ plugins-release: $(BR)/.bootstrap.ok
 build-vpp-api: $(BR)/.bootstrap.ok
 	$(call make,$(PLATFORM)_debug,vpp-api-install)
 
-PYTHON_PATH=$(BR)/python
+VPP_PYTHON_PREFIX=$(BR)/python
 
 define test
 	$(if $(filter-out $(3),retest),make -C $(BR) PLATFORM=$(1) TAG=$(2) vpp-api-install plugins-install vpp-install vpp-api-test-install,)
@@ -197,7 +201,7 @@ define test
 	  VPP_TEST_API_TEST_BIN=$(BR)/install-$(2)-native/vpp-api-test/bin/vpp_api_test \
 	  VPP_TEST_PLUGIN_PATH=$(BR)/install-$(2)-native/plugins/lib64/vpp_plugins \
 	  LD_LIBRARY_PATH=$(BR)/install-$(2)-native/vpp-api/lib64/ \
-	  WS_ROOT=$(WS_ROOT) I=$(I) V=$(V) TEST=$(TEST) PYTHON_PATH=$(PYTHON_PATH) $(3)
+	  WS_ROOT=$(WS_ROOT) V=$(V) TEST=$(TEST) VPP_PYTHON_PREFIX=$(VPP_PYTHON_PREFIX) $(3)
 endef
 
 test: bootstrap
@@ -206,12 +210,17 @@ test: bootstrap
 test-debug: bootstrap
 	$(call test,vpp_lite,vpp_lite_debug,test)
 
-test-doc:
+test-help:
-	make -C $(BR) PLATFORM=vpp_lite TAG=vpp_lite vpp-api-install plugins-install vpp-install vpp-api-test-install
+	@make -C test help
-	make -C test PYTHON_PATH=$(PYTHON_PATH) LD_LIBRARY_PATH=$(BR)/install-vpp_lite-native/vpp-api/lib64/ doc
 
-test-clean:
+test-wipe:
-	make -C test clean
+	@make -C test wipe
+
+test-doc:
+	@make -C test WS_ROOT=$(WS_ROOT) BR=$(BR) VPP_PYTHON_PREFIX=$(VPP_PYTHON_PREFIX) doc
+
+test-wipe-doc:
+	@make -C test wipe-doc BR=$(BR)
 
 retest:
 	$(call test,vpp_lite,vpp_lite,retest)

test/Makefile

@@ -1,22 +1,58 @@
-PYTHON_VENV_PATH=$(PYTHON_PATH)/virtualenv
+.PHONY: verify-python-path
 
-test: clean
+verify-python-path:
+ifndef VPP_PYTHON_PREFIX
+	$(error VPP_PYTHON_PREFIX is not set)
+endif
+
+PYTHON_VENV_PATH=$(VPP_PYTHON_PREFIX)/virtualenv
+
+test: wipe verify-python-path
 	@virtualenv $(PYTHON_VENV_PATH)
 	@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && pip install scapy"
 	@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && pip install pexpect"
 	@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && cd $(WS_ROOT)/vpp-api/python && python setup.py install"
 	@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && python run_tests.py discover -p test_$(TEST)\"*.py\""
 
-retest: clean
+retest: wipe verify-python-path
 	@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && python run_tests.py discover -p test_$(TEST)\"*.py\""
 
-.PHONY: clean doc
+.PHONY: wipe doc
 
-clean:
+wipe: verify-python-path
 	@rm -f /dev/shm/vpp-unittest-*
 	@rm -rf /tmp/vpp-unittest-*
 
-doc:
+doc: verify-python-path
 	@virtualenv $(PYTHON_VENV_PATH)
 	@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && pip install sphinx"
-	@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && make -C doc html"
+	@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && make -C doc WS_ROOT=$(WS_ROOT) BR=$(BR) NO_VPP_PAPI=1 html"
+
+wipe-doc:
+	@make -C doc wipe BR=$(BR)
+
+help:
+	@echo "Running tests:"
+	@echo ""
+	@echo " test                - build and run functional tests"
+	@echo " test-debug          - build and run functional tests (debug build)"
+	@echo " retest              - run functional tests"
+	@echo " retest-debug        - run functional tests (debug build)"
+	@echo " wipe-test           - wipe (temporary) files generated by unit tests"
+	@echo ""
+	@echo "Arguments controlling test runs:"
+	@echo " V=[0|1|2]            - set test verbosity level"
+	@echo " DEBUG=<type>         - set VPP debugging kind"
+	@echo "    DEBUG=core        - detect coredump and load it in gdb on crash"
+	@echo "    DEBUG=gdb         - allow easy debugging by printing VPP PID "
+	@echo "                        and waiting for user input before running "
+	@echo "                        and tearing down a testcase"
+	@echo "    DEBUG=gdbserver   - run gdb inside a gdb server, otherwise "
+	@echo "                        same as above"
+	@echo " STEP=[yes|no]        - ease debugging by stepping through a testcase "
+	@echo " TEST=<name>          - only run specific test"
+	@echo ""
+	@echo "Creating test documentation"
+	@echo " test-doc            - generate documentation for test framework"
+	@echo " wipe-test-doc       - wipe documentation for test framework"
+	@echo ""

test/doc/conf.py

@@ -20,6 +20,7 @@ import os
 import sys
 sys.path.insert(0, os.path.abspath('..'))
 
+
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.

test/hook.py

@@ -1,7 +1,8 @@
 import signal
 import os
 import pexpect
-from logging import *
+import traceback
+from log import *
 
 
 class Hook(object):
@@ -9,6 +10,9 @@ class Hook(object):
     Generic hooks before/after API/CLI calls
     """
 
+    def __init__(self, logger):
+        self.logger = logger
+
     def before_api(self, api_name, api_args):
         """
         Function called before API call
@@ -17,7 +21,8 @@ class Hook(object):
         @param api_name: name of the API
         @param api_args: tuple containing the API arguments
         """
-        debug("API: %s (%s)" % (api_name, api_args))
+        self.logger.debug("API: %s (%s)" %
+                          (api_name, api_args), extra={'color': RED})
 
     def after_api(self, api_name, api_args):
         """
@@ -35,7 +40,7 @@ class Hook(object):
 
         @param cli: CLI string
         """
-        debug("CLI: %s" % (cli))
+        self.logger.debug("CLI: %s" % (cli), extra={'color': RED})
 
     def after_cli(self, cli):
         """
@@ -54,6 +59,7 @@ class PollHook(Hook):
     def __init__(self, testcase):
         self.vpp_dead = False
         self.testcase = testcase
+        self.logger = testcase.logger
 
     def spawn_gdb(self, gdb_path, core_path):
         gdb_cmdline = gdb_path + ' ' + self.testcase.vpp_bin + ' ' + core_path
@@ -74,11 +80,12 @@ class PollHook(Hook):
                 self.spawn_gdb(gdb_path, core_path)
                 return
             else:
-                error("Debugger '%s' does not exist or is not an executable.." %
+                self.logger.error(
-                      gdb_path)
+                    "Debugger '%s' does not exist or is not an executable.." %
+                    gdb_path)
 
-        critical('core file present, debug with: gdb ' +
+        self.logger.critical('core file present, debug with: gdb ' +
-                 self.testcase.vpp_bin + ' ' + core_path)
+                             self.testcase.vpp_bin + ' ' + core_path)
 
     def poll_vpp(self):
         """
@@ -97,7 +104,7 @@ class PollHook(Hook):
             msg = "VPP subprocess died unexpectedly with returncode %d [%s]" % (
                 self.testcase.vpp.returncode,
                 signaldict[abs(self.testcase.vpp.returncode)])
-            critical(msg)
+            self.logger.critical(msg)
             core_path = self.testcase.tempdir + '/core'
             if os.path.isfile(core_path):
                 self.on_crash(core_path)
@@ -126,3 +133,92 @@ class PollHook(Hook):
         """
         super(PollHook, self).after_cli(cli)
         self.poll_vpp()
+
+
+class StepHook(PollHook):
+    """ Hook which requires user to press ENTER before doing any API/CLI """
+
+    def __init__(self, testcase):
+        self.skip_stack = None
+        self.skip_num = None
+        self.skip_count = 0
+        super(StepHook, self).__init__(testcase)
+
+    def skip(self):
+        if self.skip_stack is None:
+            return False
+        stack = traceback.extract_stack()
+        counter = 0
+        skip = True
+        for e in stack:
+            if counter > self.skip_num:
+                break
+            if e[0] != self.skip_stack[counter][0]:
+                skip = False
+            if e[1] != self.skip_stack[counter][1]:
+                skip = False
+            counter += 1
+        if skip:
+            self.skip_count += 1
+            return True
+        else:
+            print("%d API/CLI calls skipped in specified stack "
+                  "frame" % self.skip_count)
+            self.skip_count = 0
+            self.skip_stack = None
+            self.skip_num = None
+            return False
+
+    def user_input(self):
+        print('number\tfunction\tfile\tcode')
+        counter = 0
+        stack = traceback.extract_stack()
+        for e in stack:
+            print('%02d.\t%s\t%s:%d\t[%s]' % (counter, e[2], e[0], e[1], e[3]))
+            counter += 1
+        print(single_line_delim)
+        print("You can enter a number of stack frame chosen from above")
+        print("Calls in/below that stack frame will be not be stepped anymore")
+        print(single_line_delim)
+        while True:
+            choice = raw_input("Enter your choice, if any, and press ENTER to "
+                               "continue running the testcase...")
+            if choice == "":
+                choice = None
+            try:
+                if choice is not None:
+                    num = int(choice)
+            except:
+                print("Invalid input")
+                continue
+            if choice is not None and (num < 0 or num >= len(stack)):
+                print("Invalid choice")
+                continue
+            break
+        if choice is not None:
+            self.skip_stack = stack
+            self.skip_num = num
+
+    def before_cli(self, cli):
+        """ Wait for ENTER before executing CLI """
+        if self.skip():
+            print("Skip pause before executing CLI: %s" % cli)
+        else:
+            print(double_line_delim)
+            print("Test paused before executing CLI: %s" % cli)
+            print(single_line_delim)
+            self.user_input()
+        super(StepHook, self).before_cli(cli)
+
+    def before_api(self, api_name, api_args):
+        """ Wait for ENTER before executing API """
+        if self.skip():
+            print("Skip pause before executing API: %s (%s)"
+                  % (api_name, api_args))
+        else:
+            print(double_line_delim)
+            print("Test paused before executing API: %s (%s)"
+                  % (api_name, api_args))
+            print(single_line_delim)
+            self.user_input()
+        super(StepHook, self).before_api(api_name, api_args)

test/log.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+
+import sys
+import os
+import logging
+
+""" @var formatting delimiter consisting of '=' characters """
+double_line_delim = '=' * 70
+""" @var formatting delimiter consisting of '-' characters """
+single_line_delim = '-' * 70
+
+
+def colorize(msg, color):
+    return color + msg + COLOR_RESET
+
+
+class ColorFormatter(logging.Formatter):
+
+    def init(self, fmt=None, datefmt=None):
+        super(ColorFormatter, self).__init__(fmt, datefmt)
+
+    def format(self, record):
+        message = super(ColorFormatter, self).format(record)
+        if hasattr(record, 'color'):
+            message = colorize(message, record.color)
+        return message
+
+handler = logging.StreamHandler(sys.stdout)
+handler.setFormatter(ColorFormatter())
+
+global_logger = logging.getLogger()
+global_logger.addHandler(handler)
+try:
+    verbose = int(os.getenv("V", 0))
+except:
+    verbose = 0
+
+# 40 = ERROR, 30 = WARNING, 20 = INFO, 10 = DEBUG, 0 = NOTSET (all messages)
+if verbose >= 2:
+    log_level = 10
+elif verbose == 1:
+    log_level = 20
+else:
+    log_level = 40
+
+scapy_logger = logging.getLogger("scapy.runtime")
+scapy_logger.setLevel(logging.ERROR)
+
+
+def getLogger(name):
+    logger = logging.getLogger(name)
+    logger.setLevel(log_level)
+    return logger
+
+# Static variables to store color formatting strings.
+#
+# These variables (RED, GREEN, YELLOW and LPURPLE) are used to configure
+# the color of the text to be printed in the terminal. Variable COLOR_RESET
+# is used to revert the text color to the default one.
+if hasattr(sys.stdout, 'isatty') and sys.stdout.isatty():
+    RED = '\033[91m'
+    GREEN = '\033[92m'
+    YELLOW = '\033[93m'
+    LPURPLE = '\033[94m'
+    COLOR_RESET = '\033[0m'
+else:
+    RED = ''
+    GREEN = ''
+    YELLOW = ''
+    LPURPLE = ''
+    COLOR_RESET = ''

test/test_l2bd.py

@@ -189,7 +189,7 @@ class TestL2bd(VppTestCase):
             MAC learning enabled
             learn 100 MAC enries
             3 interfaces: untagged, dot1q, dot1ad (dot1q used instead of dot1ad
-             in the first version)
+            in the first version)
 
         2.sending l2 eth pkts between 3 interface
             64B, 512B, 1518B, 9200B (ether_size)

test/test_l2xc.py

@@ -78,7 +78,6 @@ class TestL2xc(VppTestCase):
         interfaces. Create host IPv4 address for every host MAC address too.
 
         :param count: Number of hosts to create MAC and IPv4 addresses for.
-        Type: int
         """
         for pg_if in self.pg_interfaces:
             # self.MY_MACS[i.sw_if_index] = []
@@ -151,12 +150,11 @@ class TestL2xc(VppTestCase):
         """ L2XC test
 
         Test scenario:
-        1.config
+            1. config
-            2 pairs of 2 interfaces, l2xconnected
+               2 pairs of 2 interfaces, l2xconnected
-
+            2. sending l2 eth packets between 4 interfaces
-        2.sending l2 eth packets between 4 interfaces
+               64B, 512B, 1518B, 9018B (ether_size)
-            64B, 512B, 1518B, 9018B (ether_size)
+               burst of packets per interface
-            burst of packets per interface
         """
 
         # Create incoming packet streams for packet-generator interfaces

test/vpp_papi_provider.py

@@ -1,7 +1,19 @@
-import vpp_papi
+import os
 from logging import error
 from hook import Hook
 
+do_import = True
+try:
+    no_vpp_papi = os.getenv("NO_VPP_PAPI")
+    if no_vpp_papi == "1":
+        do_import = False
+except:
+    pass
+
+if do_import:
+    import vpp_papi
+
+
 # from vnet/vnet/mpls/mpls_types.h
 MPLS_IETF_MAX_LABEL = 0xfffff
 MPLS_LABEL_INVALID = MPLS_IETF_MAX_LABEL + 1
@@ -16,7 +28,7 @@ class VppPapiProvider(object):
     """
 
     def __init__(self, name, shm_prefix):
-        self.hook = Hook()
+        self.hook = Hook("vpp-papi-provider")
         self.name = name
         self.shm_prefix = shm_prefix
 
@@ -130,7 +142,6 @@ class VppPapiProvider(object):
                          default_router, max_interval, min_interval,
                          lifetime, initial_count, initial_interval, async))
 
-
     def vxlan_add_del_tunnel(
             self,
             src_addr,
@@ -177,7 +188,7 @@ class VppPapiProvider(object):
         :param rx_sw_if_index: Software interface index of Rx interface.
         :param tx_sw_if_index: Software interface index of Tx interface.
         :param enable: Create cross-connect if equal to 1, delete cross-connect
-        if equal to 0.
+                       if equal to 0.
         :type rx_sw_if_index: str or int
         :type rx_sw_if_index: str or int
         :type enable: int