tests: add generalized tags for tests, use them for run-solo tests

We have accumulated several scenarios in prod or wishlists
where it would be useful to have a general infra to say yes/no
about a certain test, and potentially make decisions based on that,
for example:

- runs solo (aka 'time-dependent')
- (wishlist) part of quick smoke-test set
- (wishlist) intermittent failure unrelated to timing
- (wishlist) test broken with a multi-worker config in vpp

Refactor the current "run-solo" code to allow for this extension.

Type: test

Change-Id: Ia5b3810e57c0543753c8e0dc4dc0cfb4a30b36ac
Signed-off-by: Andrew Yourtchenko <ayourtch@gmail.com>
Signed-off-by: Klement Sekera <ksekera@cisco.com>

src/plugins/dhcp/test/test_dhcp.py

@@ -6,6 +6,7 @@ import struct
 import six
 
 from framework import VppTestCase, VppTestRunner, running_extended_tests
+from framework import tag_run_solo
 from vpp_neighbor import VppNeighbor
 from vpp_ip_route import find_route, VppIpTable
 from util import mk_ll_addr
@@ -32,13 +33,10 @@ DHCP6_CLIENT_PORT = 547
 DHCP6_SERVER_PORT = 546
 
 
+@tag_run_solo
 class TestDHCP(VppTestCase):
     """ DHCP Test Case """
 
-    @classmethod
-    def force_solo(cls):
-        return True
-
     @classmethod
     def setUpClass(cls):
         super(TestDHCP, cls).setUpClass()

src/plugins/dhcp/test/test_dhcp6.py

@@ -9,6 +9,7 @@ from scapy.layers.inet6 import IPv6, Ether, UDP
 from scapy.utils6 import in6_mactoifaceid
 
 from framework import VppTestCase
+from framework import tag_run_solo
 from vpp_papi import VppEnum
 import util
 import os
@@ -220,13 +221,10 @@ class TestDHCPv6DataPlane(VppTestCase):
         self.vapi.dhcp6_clients_enable_disable(enable=0)
 
 
+@tag_run_solo
 class TestDHCPv6IANAControlPlane(VppTestCase):
     """ DHCPv6 IA NA Control Plane Test Case """
 
-    @classmethod
-    def force_solo(cls):
-        return True
-
     @classmethod
     def setUpClass(cls):
         super(TestDHCPv6IANAControlPlane, cls).setUpClass()

src/plugins/flowprobe/test/test_flowprobe.py

@@ -13,6 +13,7 @@ from scapy.layers.inet import IP, TCP, UDP
 from scapy.layers.inet6 import IPv6
 
 from framework import VppTestCase, VppTestRunner, running_extended_tests
+from framework import tag_run_solo
 from vpp_object import VppObject
 from vpp_pg_interface import CaptureTimeoutError
 from util import ppp
@@ -343,13 +344,10 @@ class MethodHolder(VppTestCase):
         return p
 
 
+@tag_run_solo
 class Flowprobe(MethodHolder):
     """Template verification, timer tests"""
 
-    @classmethod
-    def force_solo(cls):
-        return True
-
     @classmethod
     def setUpClass(cls):
         super(Flowprobe, cls).setUpClass()

src/plugins/memif/test/test_memif.py

@@ -5,6 +5,7 @@ from scapy.layers.l2 import Ether
 from scapy.layers.inet import IP, ICMP
 
 from framework import VppTestCase, VppTestRunner, running_extended_tests
+from framework import tag_run_solo
 from remote_test import RemoteClass, RemoteVppTestCase
 from vpp_memif import remove_all_memif_vpp_config, \
     VppSocketFilename, VppMemif
@@ -12,13 +13,10 @@ from vpp_ip_route import VppIpRoute, VppRoutePath
 from vpp_papi import VppEnum
 
 
+@tag_run_solo
 class TestMemif(VppTestCase):
     """ Memif Test Case """
 
-    @classmethod
-    def force_solo(cls):
-        return True
-
     @classmethod
     def setUpClass(cls):
         # fork new process before client connects to VPP

src/vnet/bfd/test/test_bfd.py

@@ -22,6 +22,7 @@ from scapy.packet import Raw
 from bfd import VppBFDAuthKey, BFD, BFDAuthType, VppBFDUDPSession, \
     BFDDiagCode, BFDState, BFD_vpp_echo
 from framework import VppTestCase, VppTestRunner, running_extended_tests
+from framework import tag_run_solo
 from util import ppp
 from vpp_ip import DpoProto
 from vpp_ip_route import VppIpRoute, VppRoutePath
@@ -677,6 +678,7 @@ def wait_for_bfd_packet(test, timeout=1, pcap_time_min=None, is_tunnel=False):
     return p
 
 
+@tag_run_solo
 class BFD4TestCase(VppTestCase):
     """Bidirectional Forwarding Detection (BFD)"""
 
@@ -685,10 +687,6 @@ class BFD4TestCase(VppTestCase):
     vpp_session = None
     test_session = None
 
-    @classmethod
-    def force_solo(cls):
-        return True
-
     @classmethod
     def setUpClass(cls):
         super(BFD4TestCase, cls).setUpClass()
@@ -1489,6 +1487,7 @@ class BFD4TestCase(VppTestCase):
         self.assertFalse(vpp_session.query_vpp_config())
 
 
+@tag_run_solo
 class BFD6TestCase(VppTestCase):
     """Bidirectional Forwarding Detection (BFD) (IPv6) """
 
@@ -1497,10 +1496,6 @@ class BFD6TestCase(VppTestCase):
     vpp_session = None
     test_session = None
 
-    @classmethod
-    def force_solo(cls):
-        return True
-
     @classmethod
     def setUpClass(cls):
         super(BFD6TestCase, cls).setUpClass()
@@ -1706,16 +1701,13 @@ class BFD6TestCase(VppTestCase):
         self.assertFalse(vpp_session.query_vpp_config())
 
 
+@tag_run_solo
 class BFDFIBTestCase(VppTestCase):
     """ BFD-FIB interactions (IPv6) """
 
     vpp_session = None
     test_session = None
 
-    @classmethod
-    def force_solo(cls):
-        return True
-
     @classmethod
     def setUpClass(cls):
         super(BFDFIBTestCase, cls).setUpClass()
@@ -1896,6 +1888,7 @@ class BFDTunTestCase(VppTestCase):
         bfd_session_down(self)
 
 
+@tag_run_solo
 class BFDSHA1TestCase(VppTestCase):
     """Bidirectional Forwarding Detection (BFD) (SHA1 auth) """
 
@@ -1904,10 +1897,6 @@ class BFDSHA1TestCase(VppTestCase):
     vpp_session = None
     test_session = None
 
-    @classmethod
-    def force_solo(cls):
-        return True
-
     @classmethod
     def setUpClass(cls):
         super(BFDSHA1TestCase, cls).setUpClass()
@@ -2131,6 +2120,7 @@ class BFDSHA1TestCase(VppTestCase):
         bfd_session_up(self)
 
 
+@tag_run_solo
 class BFDAuthOnOffTestCase(VppTestCase):
     """Bidirectional Forwarding Detection (BFD) (changing auth) """
 
@@ -2138,10 +2128,6 @@ class BFDAuthOnOffTestCase(VppTestCase):
     vpp_session = None
     test_session = None
 
-    @classmethod
-    def force_solo(cls):
-        return True
-
     @classmethod
     def setUpClass(cls):
         super(BFDAuthOnOffTestCase, cls).setUpClass()
@@ -2347,14 +2333,11 @@ class BFDAuthOnOffTestCase(VppTestCase):
                           "number of bfd events")
 
 
+@tag_run_solo
 class BFDCLITestCase(VppTestCase):
     """Bidirectional Forwarding Detection (BFD) (CLI) """
     pg0 = None
 
-    @classmethod
-    def force_solo(cls):
-        return True
-
     @classmethod
     def setUpClass(cls):
         super(BFDCLITestCase, cls).setUpClass()

test/framework.py

@@ -21,6 +21,7 @@ from threading import Thread, Event
 from inspect import getdoc, isclass
 from traceback import format_exception
 from logging import FileHandler, DEBUG, Formatter
+from enum import Enum
 
 import scapy.compat
 from scapy.packet import Raw
@@ -255,6 +256,22 @@ class KeepAliveReporter(object):
         self.pipe.send((desc, test.vpp_bin, test.tempdir, test.vpp.pid))
 
 
+class TestCaseTag(Enum):
+    RUN_SOLO = 1
+
+
+def create_tag_decorator(e):
+    def decorator(cls):
+        try:
+            cls.test_tags.append(e)
+        except AttributeError:
+            cls.test_tags = [e]
+        return cls
+    return decorator
+
+tag_run_solo = create_tag_decorator(TestCaseTag.RUN_SOLO)
+
+
 class VppTestCase(unittest.TestCase):
     """This subclass is a base class for VPP test cases that are implemented as
     classes. It provides methods to create and run test case.
@@ -279,10 +296,19 @@ class VppTestCase(unittest.TestCase):
             return 0
 
     @classmethod
-    def force_solo(cls):
-        """ if the test case class is timing-sensitive - return true """
+    def has_tag(cls, tag):
+        """ if the test case has a given tag - return true """
+        try:
+            return tag in cls.test_tags
+        except AttributeError:
+            pass
         return False
 
+    @classmethod
+    def is_tagged_run_solo(cls):
+        """ if the test case class is timing-sensitive - return true """
+        return cls.has_tag(TestCaseTag.RUN_SOLO)
+
     @classmethod
     def instance(cls):
         """Return the instance of this testcase"""
@@ -1404,7 +1430,7 @@ class VppTestResult(unittest.TestResult):
                 raise Exception("No doc string for test '%s'" % test.id())
             test_title = test_doc.splitlines()[0]
             test_title_colored = colorize(test_title, GREEN)
-            if test.force_solo():
+            if test.is_tagged_run_solo():
                 # long live PEP-8 and 80 char width limitation...
                 c = YELLOW
                 test_title_colored = colorize("SOLO RUN: " + test_title, c)

test/run_tests.py

@@ -337,7 +337,7 @@ def run_forked(testcase_suites):
     while total_test_runners < concurrent_tests:
         if testcase_suites:
             a_suite = testcase_suites.pop(0)
-            if a_suite.force_solo:
+            if a_suite.is_tagged_run_solo:
                 solo_testcase_suites.append(a_suite)
                 continue
             wrapped_testcase_suite = TestCaseWrapper(a_suite,
@@ -473,7 +473,7 @@ def run_forked(testcase_suites):
                         results.append(TestResult(testcase_suites.pop(0)))
                 elif testcase_suites:
                     a_testcase = testcase_suites.pop(0)
-                    while a_testcase and a_testcase.force_solo:
+                    while a_testcase and a_testcase.is_tagged_run_solo:
                         solo_testcase_suites.append(a_testcase)
                         if testcase_suites:
                             a_testcase = testcase_suites.pop(0)
@@ -520,10 +520,10 @@ class SplitToSuitesCallback:
             self.suite_name = file_name + cls.__name__
             if self.suite_name not in self.suites:
                 self.suites[self.suite_name] = unittest.TestSuite()
-                self.suites[self.suite_name].force_solo = False
+                self.suites[self.suite_name].is_tagged_run_solo = False
             self.suites[self.suite_name].addTest(test_method)
-            if test_method.force_solo():
-                self.suites[self.suite_name].force_solo = True
+            if test_method.is_tagged_run_solo():
+                self.suites[self.suite_name].is_tagged_run_solo = True
 
         else:
             self.filtered.addTest(test_method)

test/test_ip6.py

@@ -19,7 +19,7 @@ from scapy.utils6 import in6_getnsma, in6_getnsmac, in6_ptop, in6_islladdr, \
     in6_mactoifaceid
 from six import moves
 
-from framework import VppTestCase, VppTestRunner
+from framework import VppTestCase, VppTestRunner, tag_run_solo
 from util import ppp, ip6_normalize, mk_ll_addr
 from vpp_papi import VppEnum
 from vpp_ip import DpoProto, VppIpPuntPolicer, VppIpPuntRedirect
@@ -162,13 +162,10 @@ class TestIPv6ND(VppTestCase):
         self.assertEqual(ip.dst, dip)
 
 
+@tag_run_solo
 class TestIPv6(TestIPv6ND):
     """ IPv6 Test Case """
 
-    @classmethod
-    def force_solo(cls):
-        return True
-
     @classmethod
     def setUpClass(cls):
         super(TestIPv6, cls).setUpClass()

test/test_session.py

@@ -3,6 +3,7 @@
 import unittest
 
 from framework import VppTestCase, VppTestRunner
+from framework import tag_run_solo
 from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath
 
 
@@ -117,13 +118,10 @@ class TestSessionUnitTests(VppTestCase):
         self.vapi.session_enable_disable(is_enable=0)
 
 
+@tag_run_solo
 class TestSvmFifoUnitTests(VppTestCase):
     """ SVM Fifo Unit Tests Case """
 
-    @classmethod
-    def force_solo(cls):
-        return True
-
     @classmethod
     def setUpClass(cls):
         super(TestSvmFifoUnitTests, cls).setUpClass()

test/test_util.py

@@ -10,7 +10,7 @@ class TestUtil (unittest.TestCase):
     """ Test framework utility tests """
 
     @classmethod
-    def force_solo(cls):
+    def is_tagged_run_solo(cls):
         """ if the test case class is timing-sensitive - return true """
         return False