CSIT-1139: Implement parallel test execution

The implementation of parallel test execution in VPP Test Framework.
- VPPTestCase test methods are grouped together
- tests are running in separate processes
- VPP instances spawned by tests are assigned to different cores
- output from these processes is redirected through pipes and printed
out testcase by testcase
- TEST_JOBS env var is used to specify the number of parallel processes
- improved test summary
- a bit of code cleanup

Change-Id: I9ca93904d9fe2c3daf980500c64a8611838ae28c
Signed-off-by: juraj.linkes <juraj.linkes@pantheon.tech>

src/vpp-api/python/vpp_papi/vpp_papi.py

@@ -63,7 +63,7 @@ vpp_api = ffi.dlopen('libvppapiclient.so')
 def vpp_atexit(vpp_weakref):
     """Clean up VPP connection on shutdown."""
     vpp_instance = vpp_weakref()
-    if vpp_instance.connected:
+    if vpp_instance and vpp_instance.connected:
         vpp_instance.logger.debug('Cleaning up VPP on exit')
         vpp_instance.disconnect()
 

test/Makefile

@@ -21,6 +21,12 @@ else ifeq ($(DEBUG),gdbserver)
 FORCE_FOREGROUND=1
 else ifeq ($(DEBUG),core)
 FORCE_FOREGROUND=1
+else ifeq ($(STEP),yes)
+FORCE_FOREGROUND=1
+else ifeq ($(STEP),y)
+FORCE_FOREGROUND=1
+else ifeq ($(STEP),1)
+FORCE_FOREGROUND=1
 else
 FORCE_FOREGROUND=0
 endif
@@ -121,8 +127,25 @@ SANITY_IMPORT_VPP_PAPI_CMD=source $(PYTHON_VENV_PATH)/bin/activate && python san
 SANITY_RUN_VPP_CMD=source $(PYTHON_VENV_PATH)/bin/activate && python sanity_run_vpp.py
 endif
 
+ifndef TEST_JOBS
+PARALLEL_ILLEGAL=0
+else ifeq ($(FORCE_FOREGROUND),0)
+PARALLEL_ILLEGAL=0
+else ifeq ($(TEST_JOBS),auto)
+PARALLEL_ILLEGAL=0
+else ifeq ($(TEST_JOBS),1)
+PARALLEL_ILLEGAL=0
+else
+PARALLEL_ILLEGAL=1
+endif
+
 sanity: verify-no-running-vpp
 	@sys_req/dev_shm_size.sh
+	@bash -c "test $(PARALLEL_ILLEGAL) -eq 0 ||\
+	    (echo \"*******************************************************************\" &&\
+		 echo \"* Sanity check failed, TEST_JOBS is not 1 or 'auto' and DEBUG, STEP or PROFILE is set\" &&\
+	         echo \"*******************************************************************\" &&\
+		 false)"
 	@bash -c "$(SANITY_IMPORT_VPP_PAPI_CMD) ||\
 		(echo \"*******************************************************************\" &&\
 		 echo \"* Sanity check failed, cannot import vpp_papi\" &&\
@@ -236,18 +259,19 @@ help:
 	@echo ""
 	@echo "Arguments controlling test runs:"
 	@echo " V=[0|1|2]            - set test verbosity level"
+	@echo " TEST_JOBS=[<n>|auto] - use <n> parallel processes for test execution or automatic discovery of maximum acceptable processes (default: 1)"
 	@echo " CACHE_OUTPUT=[0|1]   - cache VPP stdout/stderr and log as one block after test finishes (default: 1)"
 	@echo " FAILFAST=[0|1]       - fail fast if 1, complete all tests if 0"
-	@echo " TIMEOUT=<timeout>    - fail test suite if any single test takes longer than <timeout> (in seconds) to finish"
+	@echo " TIMEOUT=<timeout>    - fail test suite if any single test takes longer than <timeout> (in seconds) to finish (default: 600)"
 	@echo " RETRIES=<n>          - retry failed tests <n> times"
 	@echo " DEBUG=<type>         - set VPP debugging kind"
 	@echo "    DEBUG=core        - detect coredump and load it in gdb on crash"
-	@echo "    DEBUG=gdb         - allow easy debugging by printing VPP PID "
-	@echo "                        and waiting for user input before running "
+	@echo "    DEBUG=gdb         - allow easy debugging by printing VPP PID"
+	@echo "                        and waiting for user input before running"
 	@echo "                        and tearing down a testcase"
-	@echo "    DEBUG=gdbserver   - run gdb inside a gdb server, otherwise "
+	@echo "    DEBUG=gdbserver   - run gdb inside a gdb server, otherwise"
 	@echo "                        same as above"
-	@echo " STEP=[yes|no]        - ease debugging by stepping through a testcase "
+	@echo " STEP=[yes|no]        - ease debugging by stepping through a testcase"
 	@echo " SANITY=[yes|no]      - perform sanity import of vpp-api/sanity vpp run before running tests (default: yes)"
 	@echo " EXTENDED_TESTS=[1|y] - run extended tests"
 	@echo " TEST=<filter>        - filter the set of tests:"

test/hook.py

@@ -176,19 +176,19 @@ class StepHook(PollHook):
             print('%02d.\t%s\t%s:%d\t[%s]' % (counter, e[2], e[0], e[1], e[3]))
             counter += 1
         print(single_line_delim)
-        print("You can enter a number of stack frame chosen from above")
+        print("You may enter a number of stack frame chosen from above")
         print("Calls in/below that stack frame will be not be stepped anymore")
         print(single_line_delim)
         while True:
-            choice = sys.stdin.readline(
-                "Enter your choice, if any, and press ENTER to continue "
-                "running the testcase...")
+            print("Enter your choice, if any, and press ENTER to continue "
+                  "running the testcase...")
+            choice = sys.stdin.readline()
             if choice == "":
                 choice = None
             try:
                 if choice is not None:
                     num = int(choice)
-            except TypeError:
+            except ValueError:
                 print("Invalid input")
                 continue
             if choice is not None and (num < 0 or num >= len(stack)):

test/log.py

@@ -38,8 +38,9 @@ else:
     log_level = 40
 
 handler = logging.StreamHandler(sys.stdout)
-handler.setFormatter(ColorFormatter(fmt='%(asctime)s,%(msecs)03d %(message)s',
-                                    datefmt="%H:%M:%S"))
+color_formatter = ColorFormatter(fmt='%(asctime)s,%(msecs)03d %(message)s',
+                                 datefmt="%H:%M:%S")
+handler.setFormatter(color_formatter)
 handler.setLevel(log_level)
 
 global_logger = logging.getLogger()
@@ -54,6 +55,16 @@ def getLogger(name):
     logger.setLevel(logging.DEBUG)
     return logger
 
+
+def get_parallel_logger(stream):
+    logger = logging.getLogger('parallel_logger_{}'.format(stream))
+    logger.propagate = False
+    handler = logging.StreamHandler(stream)
+    handler.setFormatter(color_formatter)
+    handler.setLevel(log_level)
+    logger.addHandler(handler)
+    return logger
+
 # Static variables to store color formatting strings.
 #
 # These variables (RED, GREEN, YELLOW and LPURPLE) are used to configure