Split non-parallel testsuite

Split one big suite into smaller suites when not running tests in
parallel. This results in all tests being executed in one iteration.

Change-Id: I0d3d357a95d9cc596b606d5911a5819e8ffdeee5
Signed-off-by: juraj.linkes <juraj.linkes@pantheon.tech>

test/framework.py

@@ -1020,7 +1020,7 @@ class VppTestResult(unittest.TestResult):
     core_crash_test_cases_info = set()
     current_test_case_info = None
 
-    def __init__(self, stream, descriptions, verbosity):
+    def __init__(self, stream, descriptions, verbosity, runner):
         """
         :param stream File descriptor to store where to report test results.
             Set to the standard error stream by default.
@@ -1033,6 +1033,7 @@ class VppTestResult(unittest.TestResult):
         self.descriptions = descriptions
         self.verbosity = verbosity
         self.result_string = None
+        self.runner = runner
 
     def addSuccess(self, test):
         """
@@ -1216,9 +1217,16 @@ class VppTestResult(unittest.TestResult):
         """
         Print errors from running the test case
         """
-        self.stream.writeln()
-        self.printErrorList('ERROR', self.errors)
-        self.printErrorList('FAIL', self.failures)
+        if len(self.errors) > 0 or len(self.failures) > 0:
+            self.stream.writeln()
+            self.printErrorList('ERROR', self.errors)
+            self.printErrorList('FAIL', self.failures)
+
+        # ^^ that is the last output from unittest before summary
+        if not self.runner.print_summary:
+            devnull = unittest.runner._WritelnDecorator(open(os.devnull, 'w'))
+            self.stream = devnull
+            self.runner.stream = devnull
 
     def printErrorList(self, flavour, errors):
         """
@@ -1248,7 +1256,7 @@ class VppTestRunner(unittest.TextTestRunner):
 
     def __init__(self, keep_alive_pipe=None, descriptions=True, verbosity=1,
                  result_pipe=None, failfast=False, buffer=False,
-                 resultclass=None):
+                 resultclass=None, print_summary=True):
 
         # ignore stream setting here, use hard-coded stdout to be in sync
         # with prints from VppTestCase methods ...
@@ -1257,7 +1265,16 @@ class VppTestRunner(unittest.TextTestRunner):
                                             resultclass)
         KeepAliveReporter.pipe = keep_alive_pipe
 
-        VppTestResult.test_framework_result_pipe = result_pipe
+        self.orig_stream = self.stream
+        self.resultclass.test_framework_result_pipe = result_pipe
+
+        self.print_summary = print_summary
+
+    def _makeResult(self):
+        return self.resultclass(self.stream,
+                                self.descriptions,
+                                self.verbosity,
+                                self)
 
     def run(self, test):
         """
@@ -1269,6 +1286,9 @@ class VppTestRunner(unittest.TextTestRunner):
         faulthandler.enable()  # emit stack trace to stderr if killed by signal
 
         result = super(VppTestRunner, self).run(test)
+        if not self.print_summary:
+            self.stream = self.orig_stream
+            result.stream = self.orig_stream
         return result
 
 

test/run_tests.py

@@ -125,7 +125,8 @@ def test_runner_wrapper(suite, keep_alive_pipe, stdouterr_queue,
                            descriptions=descriptions,
                            verbosity=verbose,
                            result_pipe=result_pipe,
-                           failfast=failfast).run(suite)
+                           failfast=failfast,
+                           print_summary=False).run(suite)
     finished_pipe.send(result.wasSuccessful())
     finished_pipe.close()
     keep_alive_pipe.close()
@@ -234,12 +235,9 @@ def handle_failed_suite(logger, last_test_temp_dir, vpp_pid):
         failed_dir = os.getenv('VPP_TEST_FAILED_DIR')
         link_path = '%s%s-FAILED' % (failed_dir, lttd)
         if not os.path.exists(link_path):
-            logger.error("Creating a link to the failed test: %s -> %s" %
-                         (link_path, lttd))
             os.symlink(last_test_temp_dir, link_path)
-        else:
-            logger.error("Link to the failed test already exists: %s -> %s" %
-                         (link_path, lttd))
+        logger.error("Symlink to failed testcase directory: %s -> %s"
+                     % (link_path, lttd))
 
         # Report core existence
         core_path = get_core_path(last_test_temp_dir)
@@ -575,10 +573,7 @@ class AllResults(dict):
             retval = 1
 
         if retval != 0:
-            if concurrent_tests == 1:
-                self.rerun.append(result.suite_from_failed())
-            else:
-                self.rerun.append(result.testcase_suite)
+            self.rerun.append(result.testcase_suite)
 
         return retval
 
@@ -767,13 +762,6 @@ if __name__ == '__main__':
         tests_amount += testcase_suite.countTestCases()
         suites.append(testcase_suite)
 
-    if concurrent_tests == 1:
-        new_suite = unittest.TestSuite()
-        for suite in suites:
-            new_suite.addTests(suite)
-
-        suites = [new_suite]
-
     print("%s out of %s tests match specified filters" % (
         tests_amount, tests_amount + cb.filtered.countTestCases()))
 
@@ -786,8 +774,9 @@ if __name__ == '__main__':
 
     if run_interactive:
         # don't fork if requiring interactive terminal
-        result = VppTestRunner(verbosity=verbose, failfast=failfast)\
-            .run(suites[0])
+        result = VppTestRunner(verbosity=verbose,
+                               failfast=failfast,
+                               print_summary=True).run(suites[0])
         was_successful = result.wasSuccessful()
         if not was_successful:
             for test_case_info in result.failed_test_cases_info:

test/sanity_run_vpp.py

@@ -8,7 +8,7 @@ from framework import VppTestCase, KeepAliveReporter
 
 
 class SanityTestCase(VppTestCase):
-    """ Sanity test case - verify if VPP is able to start """
+    """ Sanity test case - verify thether VPP is able to start """
     pass
 
 if __name__ == '__main__':
@@ -29,4 +29,9 @@ if __name__ == '__main__':
     x.close()
     y.close()
 
+    if rc == 0:
+        print('Sanity test case passed\n')
+    else:
+        print('Sanity test case failed\n')
+
     exit(rc)