diff --git a/src/plugins/dhcp/test/test_dhcp.py b/src/plugins/dhcp/test/test_dhcp.py index 065683fbcda..266932c3830 100644 --- a/src/plugins/dhcp/test/test_dhcp.py +++ b/src/plugins/dhcp/test/test_dhcp.py @@ -35,6 +35,10 @@ DHCP6_SERVER_PORT = 546 class TestDHCP(VppTestCase): """ DHCP Test Case """ + @classmethod + def force_solo(cls): + return True + @classmethod def setUpClass(cls): super(TestDHCP, cls).setUpClass() diff --git a/src/plugins/dhcp/test/test_dhcp6.py b/src/plugins/dhcp/test/test_dhcp6.py index 7254496f48f..ebd43729e4b 100644 --- a/src/plugins/dhcp/test/test_dhcp6.py +++ b/src/plugins/dhcp/test/test_dhcp6.py @@ -223,6 +223,10 @@ class TestDHCPv6DataPlane(VppTestCase): class TestDHCPv6IANAControlPlane(VppTestCase): """ DHCPv6 IA NA Control Plane Test Case """ + @classmethod + def force_solo(cls): + return True + @classmethod def setUpClass(cls): super(TestDHCPv6IANAControlPlane, cls).setUpClass() diff --git a/src/plugins/flowprobe/test/test_flowprobe.py b/src/plugins/flowprobe/test/test_flowprobe.py index 092e8d3951d..4cf019ad47e 100644 --- a/src/plugins/flowprobe/test/test_flowprobe.py +++ b/src/plugins/flowprobe/test/test_flowprobe.py @@ -346,6 +346,10 @@ class MethodHolder(VppTestCase): class Flowprobe(MethodHolder): """Template verification, timer tests""" + @classmethod + def force_solo(cls): + return True + @classmethod def setUpClass(cls): super(Flowprobe, cls).setUpClass() diff --git a/src/plugins/memif/test/test_memif.py b/src/plugins/memif/test/test_memif.py index f11dd89f975..244818cfc3a 100644 --- a/src/plugins/memif/test/test_memif.py +++ b/src/plugins/memif/test/test_memif.py @@ -16,6 +16,10 @@ from vpp_papi import VppEnum class TestMemif(VppTestCase): """ Memif Test Case """ + @classmethod + def force_solo(cls): + return True + @classmethod def setUpClass(cls): # fork new process before client connects to VPP diff --git a/test/framework.py b/test/framework.py index c9d72a768bb..ba5f4014798 100644 --- a/test/framework.py +++ b/test/framework.py @@ -298,6 +298,11 @@ class VppTestCase(unittest.TestCase): else: return 0 + @classmethod + def force_solo(cls): + """ if the test case class is timing-sensitive - return true """ + return False + @classmethod def instance(cls): """Return the instance of this testcase""" @@ -1412,9 +1417,19 @@ class VppTestResult(unittest.TestResult): """ def print_header(test): + test_doc = getdoc(test) + if not test_doc: + raise Exception("No doc string for test '%s'" % test.id()) + test_title = test_doc.splitlines()[0] + test_title_colored = colorize(test_title, GREEN) + if test.force_solo(): + # long live PEP-8 and 80 char width limitation... + c = YELLOW + test_title_colored = colorize("SOLO RUN: " + test_title, c) + if not hasattr(test.__class__, '_header_printed'): print(double_line_delim) - print(colorize(getdoc(test).splitlines()[0], GREEN)) + print(test_title_colored) print(double_line_delim) test.__class__._header_printed = True diff --git a/test/run_tests.py b/test/run_tests.py index 499d6df1290..66118ca6f41 100644 --- a/test/run_tests.py +++ b/test/run_tests.py @@ -324,6 +324,8 @@ def process_finished_testsuite(wrapped_testcase_suite, def run_forked(testcase_suites): wrapped_testcase_suites = set() + solo_testcase_suites = [] + total_test_runners = 0 # suites are unhashable, need to use list results = [] @@ -331,12 +333,29 @@ def run_forked(testcase_suites): finished_unread_testcases = set() manager = StreamQueueManager() manager.start() - for i in range(concurrent_tests): + total_test_runners = 0 + while total_test_runners < concurrent_tests: if testcase_suites: - wrapped_testcase_suite = TestCaseWrapper(testcase_suites.pop(0), + a_suite = testcase_suites.pop(0) + if a_suite.force_solo: + solo_testcase_suites.append(a_suite) + continue + wrapped_testcase_suite = TestCaseWrapper(a_suite, manager) wrapped_testcase_suites.add(wrapped_testcase_suite) unread_testcases.add(wrapped_testcase_suite) + total_test_runners = total_test_runners + 1 + else: + break + + while total_test_runners < 1 and solo_testcase_suites: + if solo_testcase_suites: + a_suite = solo_testcase_suites.pop(0) + wrapped_testcase_suite = TestCaseWrapper(a_suite, + manager) + wrapped_testcase_suites.add(wrapped_testcase_suite) + unread_testcases.add(wrapped_testcase_suite) + total_test_runners = total_test_runners + 1 else: break @@ -448,14 +467,32 @@ def run_forked(testcase_suites): wrapped_testcase_suites.remove(finished_testcase) finished_unread_testcases.add(finished_testcase) finished_testcase.stdouterr_queue.put(None) + total_test_runners = total_test_runners - 1 if stop_run: while testcase_suites: results.append(TestResult(testcase_suites.pop(0))) elif testcase_suites: - new_testcase = TestCaseWrapper(testcase_suites.pop(0), - manager) - wrapped_testcase_suites.add(new_testcase) - unread_testcases.add(new_testcase) + a_testcase = testcase_suites.pop(0) + while a_testcase and a_testcase.force_solo: + solo_testcase_suites.append(a_testcase) + if testcase_suites: + a_testcase = testcase_suites.pop(0) + else: + a_testcase = None + if a_testcase: + new_testcase = TestCaseWrapper(a_testcase, + manager) + wrapped_testcase_suites.add(new_testcase) + total_test_runners = total_test_runners + 1 + unread_testcases.add(new_testcase) + else: + if solo_testcase_suites and total_test_runners == 0: + a_testcase = solo_testcase_suites.pop(0) + new_testcase = TestCaseWrapper(a_testcase, + manager) + wrapped_testcase_suites.add(new_testcase) + total_test_runners = total_test_runners + 1 + unread_testcases.add(new_testcase) time.sleep(0.1) except Exception: for wrapped_testcase_suite in wrapped_testcase_suites: @@ -484,7 +521,10 @@ class SplitToSuitesCallback: self.suite_name = file_name + cls.__name__ if self.suite_name not in self.suites: self.suites[self.suite_name] = unittest.TestSuite() + self.suites[self.suite_name].force_solo = False self.suites[self.suite_name].addTest(test_method) + if test_method.force_solo(): + self.suites[self.suite_name].force_solo = True else: self.filtered.addTest(test_method) diff --git a/test/test_bfd.py b/test/test_bfd.py index 67b62766c21..f66f75a3408 100644 --- a/test/test_bfd.py +++ b/test/test_bfd.py @@ -685,6 +685,10 @@ class BFD4TestCase(VppTestCase): vpp_session = None test_session = None + @classmethod + def force_solo(cls): + return True + @classmethod def setUpClass(cls): super(BFD4TestCase, cls).setUpClass() @@ -1493,6 +1497,10 @@ class BFD6TestCase(VppTestCase): vpp_session = None test_session = None + @classmethod + def force_solo(cls): + return True + @classmethod def setUpClass(cls): super(BFD6TestCase, cls).setUpClass() @@ -1704,6 +1712,10 @@ class BFDFIBTestCase(VppTestCase): vpp_session = None test_session = None + @classmethod + def force_solo(cls): + return True + @classmethod def setUpClass(cls): super(BFDFIBTestCase, cls).setUpClass() @@ -1892,6 +1904,10 @@ class BFDSHA1TestCase(VppTestCase): vpp_session = None test_session = None + @classmethod + def force_solo(cls): + return True + @classmethod def setUpClass(cls): super(BFDSHA1TestCase, cls).setUpClass() @@ -2122,6 +2138,10 @@ class BFDAuthOnOffTestCase(VppTestCase): vpp_session = None test_session = None + @classmethod + def force_solo(cls): + return True + @classmethod def setUpClass(cls): super(BFDAuthOnOffTestCase, cls).setUpClass() @@ -2331,6 +2351,10 @@ class BFDCLITestCase(VppTestCase): """Bidirectional Forwarding Detection (BFD) (CLI) """ pg0 = None + @classmethod + def force_solo(cls): + return True + @classmethod def setUpClass(cls): super(BFDCLITestCase, cls).setUpClass() diff --git a/test/test_ip6.py b/test/test_ip6.py index f8a2dc24e42..36ef5777207 100644 --- a/test/test_ip6.py +++ b/test/test_ip6.py @@ -164,6 +164,10 @@ class TestIPv6ND(VppTestCase): class TestIPv6(TestIPv6ND): """ IPv6 Test Case """ + @classmethod + def force_solo(cls): + return True + @classmethod def setUpClass(cls): super(TestIPv6, cls).setUpClass() diff --git a/test/test_session.py b/test/test_session.py index 94218d08e8e..309bf6e32e4 100644 --- a/test/test_session.py +++ b/test/test_session.py @@ -120,6 +120,10 @@ class TestSessionUnitTests(VppTestCase): class TestSvmFifoUnitTests(VppTestCase): """ SVM Fifo Unit Tests Case """ + @classmethod + def force_solo(cls): + return True + @classmethod def setUpClass(cls): super(TestSvmFifoUnitTests, cls).setUpClass() diff --git a/test/test_util.py b/test/test_util.py index 51cc739ca8d..8501881a065 100755 --- a/test/test_util.py +++ b/test/test_util.py @@ -7,8 +7,15 @@ from vpp_papi import mac_pton, mac_ntop class TestUtil (unittest.TestCase): - """ MAC to binary and back """ + """ Test framework utility tests """ + + @classmethod + def force_solo(cls): + """ if the test case class is timing-sensitive - return true """ + return False + def test_mac_to_binary(self): + """ MAC to binary and back """ mac = 'aa:bb:cc:dd:ee:ff' b = mac_pton(mac) mac2 = mac_ntop(b)