make test: support out-of-tree tests

env EXTERN_TESTS="/path/to/extra/tests" make test

causes to run the default test set and tests collected from
test_*.py files under subtree specified in EXTERN_TESTS.

Change-Id: I58c5471dd6010730278a5b47d4318737d920bc28
Signed-off-by: Klement Sekera <ksekera@cisco.com>

Makefile

@@ -222,6 +222,7 @@ export VPP_PYTHON_PREFIX=$(BR)/python
 define test
 	$(if $(filter-out $(3),retest),make -C $(BR) PLATFORM=$(1) TAG=$(2) vpp-install,)
 	make -C test \
+	  TEST_DIR=$(WS_ROOT)/test \
 	  VPP_TEST_BUILD_DIR=$(BR)/build-$(2)-native \
 	  VPP_TEST_BIN=$(BR)/install-$(2)-native/vpp/bin/vpp \
 	  VPP_TEST_PLUGIN_PATH=$(BR)/install-$(2)-native/vpp/lib64/vpp_plugins \
@@ -252,10 +253,10 @@ test-wipe:
 	@make -C test wipe
 
 test-shell: bootstrap
-	$(call test,vpp_lite,vpp_lite,shell)
+	$(call test,vpp,vpp,shell)
 
 test-shell-debug: bootstrap
-	$(call test,vpp_lite,vpp_lite_debug,shell)
+	$(call test,vpp,vpp_debug,shell)
 
 test-doc:
 	@make -C test doc

test/Makefile

@@ -23,10 +23,15 @@ verify-no-running-vpp:
 		false; \
 	fi
 
-UNITTEST_EXTRA_OPTS=""
+UNITTEST_EXTRA_OPTS=
+UNITTEST_FAILFAST_OPTS=
 
 ifeq ($(FAILFAST),1)
-UNITTEST_EXTRA_OPTS="-f"
+UNITTEST_EXTRA_OPTS=-f
+endif
+
+ifneq ($(EXTERN_TESTS),)
+UNITTEST_EXTRA_OPTS=$(UNITTEST_FAILFAST_OPTS) -d $(EXTERN_TESTS)
 endif
 
 PYTHON_VENV_PATH=$(VPP_PYTHON_PREFIX)/virtualenv
@@ -59,7 +64,7 @@ $(PAPI_INSTALL_DONE): $(PIP_PATCH_DONE)
 	@touch $@
 
 define retest-func
-	@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && python run_tests.py discover $(UNITTEST_EXTRA_OPTS) -p test_\"*.py\""
+	@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && python run_tests.py -d $(TEST_DIR) $(UNITTEST_EXTRA_OPTS)"
 endef
 
 .PHONY: sanity

test/run_tests.py

@@ -1,12 +1,61 @@
 #!/usr/bin/env python
 
+import sys
 import os
 import unittest
+import argparse
+import importlib
 from framework import VppTestRunner
 
+
+def add_from_dir(suite, directory):
+    do_insert = True
+    for _f in os.listdir(directory):
+        f = "%s/%s" % (directory, _f)
+        if os.path.isdir(f):
+            add_from_dir(suite, f)
+            continue
+        if not os.path.isfile(f):
+            continue
+        if do_insert:
+            sys.path.insert(0, directory)
+            do_insert = False
+        if not _f.startswith("test_") or not _f.endswith(".py"):
+            continue
+        name = "".join(f.split("/")[-1].split(".")[:-1])
+        if name in sys.modules:
+            raise Exception("Duplicate test module `%s' found!" % name)
+        module = importlib.import_module(name)
+        for name, cls in module.__dict__.items():
+            if not isinstance(cls, type):
+                continue
+            if not issubclass(cls, unittest.TestCase):
+                continue
+            if name == "VppTestCase":
+                continue
+            for method in dir(cls):
+                if not callable(getattr(cls, method)):
+                    continue
+                if method.startswith("test_"):
+                    suite.addTest(cls(method))
+
 if __name__ == '__main__':
     try:
         verbose = int(os.getenv("V", 0))
     except:
         verbose = 0
-    unittest.main(testRunner=VppTestRunner, module=None, verbosity=verbose)
+
+    parser = argparse.ArgumentParser(description="VPP unit tests")
+    parser.add_argument("-f", "--failfast", action='count',
+                        help="fast failure flag")
+    parser.add_argument("-d", "--dir", action='append', type=str,
+                        help="directory containing test files "
+                             "(may be specified multiple times)")
+    args = parser.parse_args()
+    failfast = True if args.failfast == 1 else False
+
+    suite = unittest.TestSuite()
+    for d in args.dir:
+        print("Adding tests from directory tree %s" % d)
+        add_from_dir(suite, d)
+    VppTestRunner(verbosity=verbose, failfast=failfast).run(suite)