make test: split into basic and extended tests
Implement plumbing to allow decorating tests as extended, e.g.: @unittest.skipUnless(running_extended_tests(), "part of extended tests") both methods and classes can be decorated this way. Change make test and make test-debug to run only non-extended tests. Introduce make test-all and make test-all-debug to run the full suite. Run full suite as part of make verify. Decorate most BFD tests as extended. Change-Id: I3bc64f59e9fe238f7f767d7e043dc165d03e9dfa Signed-off-by: Klement Sekera <ksekera@cisco.com>
This commit is contained in:
Klement Sekera
committed by
Damjan Marion
Damjan Marion
parent
d96bad8ceb
commit
871349371a
@@ -82,8 +82,10 @@ help:
|
||||
@echo " run-release - run release binary"
|
||||
@echo " debug - run debug binary with debugger"
|
||||
@echo " debug-release - run release binary with debugger"
|
||||
@echo " test - build and run functional tests"
|
||||
@echo " test-debug - build and run functional tests (debug build)"
|
||||
@echo " test - build and run (basic) functional tests"
|
||||
@echo " test-debug - build and run (basic) functional tests (debug build)"
|
||||
@echo " test-all - build and run (all) functional tests"
|
||||
@echo " test-all-debug - build and run (all) functional tests (debug build)"
|
||||
@echo " test-wipe - wipe files generated by unit tests"
|
||||
@echo " retest - run functional tests"
|
||||
@echo " retest-debug - run functional tests (debug build)"
|
||||
@@ -221,6 +223,7 @@ define test
|
||||
VPP_TEST_PLUGIN_PATH=$(BR)/install-$(2)-native/vpp/lib64/vpp_plugins \
|
||||
VPP_TEST_INSTALL_PATH=$(BR)/install-$(2)-native/ \
|
||||
LD_LIBRARY_PATH=$(BR)/install-$(2)-native/vpp/lib64/ \
|
||||
EXTENDED_TESTS=$(EXTENDED_TESTS) \
|
||||
$(3)
|
||||
endef
|
||||
|
||||
@@ -230,6 +233,14 @@ test: bootstrap
|
||||
test-debug: bootstrap
|
||||
$(call test,vpp_lite,vpp_lite_debug,test)
|
||||
|
||||
test-all: bootstrap
|
||||
$(eval EXTENDED_TESTS=yes)
|
||||
$(call test,vpp_lite,vpp_lite,test)
|
||||
|
||||
test-all-debug: bootstrap
|
||||
$(eval EXTENDED_TESTS=yes)
|
||||
$(call test,vpp_lite,vpp_lite_debug,test)
|
||||
|
||||
test-help:
|
||||
@make -C test help
|
||||
|
||||
@@ -368,6 +379,6 @@ ifeq ($(OS_ID)-$(OS_VERSION_ID),ubuntu-16.04)
|
||||
endif
|
||||
$(call banner,"Building $(PKG) packages")
|
||||
@make pkg-$(PKG)
|
||||
@make test
|
||||
@make test-all
|
||||
|
||||
|
||||
|
||||
+4
-2
@@ -119,8 +119,10 @@ checkstyle: verify-python-path
|
||||
help:
|
||||
@echo "Running tests:"
|
||||
@echo ""
|
||||
@echo " test - build and run functional tests"
|
||||
@echo " test-debug - build and run functional tests (debug build)"
|
||||
@echo " test - build and run (basic) functional tests"
|
||||
@echo " test-debug - build and run (basic) functional tests (debug build)"
|
||||
@echo " test-all - build and run (all) functional tests"
|
||||
@echo " test-all-debug - build and run (all) functional tests (debug build)"
|
||||
@echo " retest - run functional tests"
|
||||
@echo " retest-debug - run functional tests (debug build)"
|
||||
@echo " test-wipe - wipe (temporary) files generated by unit tests"
|
||||
|
||||
+27
-3
@@ -78,6 +78,15 @@ def pump_output(testclass):
|
||||
# of properly terminating the loop
|
||||
|
||||
|
||||
def running_extended_tests():
|
||||
try:
|
||||
s = os.getenv("EXTENDED_TESTS")
|
||||
return True if s.lower() in ("y", "yes", "1") else False
|
||||
except:
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
class VppTestCase(unittest.TestCase):
|
||||
"""This subclass is a base class for VPP test cases that are implemented as
|
||||
classes. It provides methods to create and run test case.
|
||||
@@ -230,9 +239,6 @@ class VppTestCase(unittest.TestCase):
|
||||
cls.verbose = 0
|
||||
cls.vpp_dead = False
|
||||
cls.registry = VppObjectRegistry()
|
||||
print(double_line_delim)
|
||||
print(colorize(getdoc(cls).splitlines()[0], YELLOW))
|
||||
print(double_line_delim)
|
||||
# need to catch exceptions here because if we raise, then the cleanup
|
||||
# doesn't get called and we might end with a zombie vpp
|
||||
try:
|
||||
@@ -613,6 +619,22 @@ class VppTestCase(unittest.TestCase):
|
||||
time.sleep(timeout)
|
||||
|
||||
|
||||
class TestCasePrinter(object):
|
||||
_shared_state = {}
|
||||
|
||||
def __init__(self):
|
||||
self.__dict__ = self._shared_state
|
||||
if not hasattr(self, "_test_case_set"):
|
||||
self._test_case_set = set()
|
||||
|
||||
def print_test_case_heading_if_first_time(self, case):
|
||||
if case.__class__ not in self._test_case_set:
|
||||
print(double_line_delim)
|
||||
print(colorize(getdoc(case.__class__).splitlines()[0], YELLOW))
|
||||
print(double_line_delim)
|
||||
self._test_case_set.add(case.__class__)
|
||||
|
||||
|
||||
class VppTestResult(unittest.TestResult):
|
||||
"""
|
||||
@property result_string
|
||||
@@ -641,6 +663,7 @@ class VppTestResult(unittest.TestResult):
|
||||
self.descriptions = descriptions
|
||||
self.verbosity = verbosity
|
||||
self.result_string = None
|
||||
self.printer = TestCasePrinter()
|
||||
|
||||
def addSuccess(self, test):
|
||||
"""
|
||||
@@ -740,6 +763,7 @@ class VppTestResult(unittest.TestResult):
|
||||
:param test:
|
||||
|
||||
"""
|
||||
self.printer.print_test_case_heading_if_first_time(test)
|
||||
unittest.TestResult.startTest(self, test)
|
||||
if self.verbosity > 0:
|
||||
self.stream.writeln(
|
||||
|
||||
+33
-2
@@ -15,7 +15,7 @@ from scapy.layers.inet import UDP, IP
|
||||
from scapy.layers.inet6 import IPv6
|
||||
from bfd import VppBFDAuthKey, BFD, BFDAuthType, VppBFDUDPSession, \
|
||||
BFDDiagCode, BFDState, BFD_vpp_echo
|
||||
from framework import VppTestCase, VppTestRunner
|
||||
from framework import VppTestCase, VppTestRunner, running_extended_tests
|
||||
from vpp_pg_interface import CaptureTimeoutError
|
||||
from util import ppp
|
||||
from vpp_papi_provider import UnexpectedApiReturnValueError
|
||||
@@ -40,6 +40,7 @@ class AuthKeyFactory(object):
|
||||
conf_key_id=conf_key_id, key=key)
|
||||
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
class BFDAPITestCase(VppTestCase):
|
||||
"""Bidirectional Forwarding Detection (BFD) - API"""
|
||||
|
||||
@@ -692,6 +693,7 @@ class BFD4TestCase(VppTestCase):
|
||||
bfd_session_up(self)
|
||||
bfd_session_down(self)
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_hold_up(self):
|
||||
""" hold BFD session up """
|
||||
bfd_session_up(self)
|
||||
@@ -701,6 +703,7 @@ class BFD4TestCase(VppTestCase):
|
||||
self.assert_equal(len(self.vapi.collect_events()), 0,
|
||||
"number of bfd events")
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_slow_timer(self):
|
||||
""" verify slow periodic control frames while session down """
|
||||
packet_count = 3
|
||||
@@ -715,6 +718,7 @@ class BFD4TestCase(VppTestCase):
|
||||
time_diff, 0.70, 1.05, "time between slow packets")
|
||||
prev_packet = next_packet
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_zero_remote_min_rx(self):
|
||||
""" no packets when zero remote required min rx interval """
|
||||
bfd_session_up(self)
|
||||
@@ -739,6 +743,7 @@ class BFD4TestCase(VppTestCase):
|
||||
self.assert_equal(
|
||||
len(self.vapi.collect_events()), 0, "number of bfd events")
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_conn_down(self):
|
||||
""" verify session goes down after inactivity """
|
||||
bfd_session_up(self)
|
||||
@@ -748,6 +753,7 @@ class BFD4TestCase(VppTestCase):
|
||||
e = self.vapi.wait_for_event(1, "bfd_udp_session_details")
|
||||
verify_event(self, e, expected_state=BFDState.down)
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_large_required_min_rx(self):
|
||||
""" large remote required min rx interval """
|
||||
bfd_session_up(self)
|
||||
@@ -777,6 +783,7 @@ class BFD4TestCase(VppTestCase):
|
||||
break
|
||||
self.assert_equal(count, 0, "number of packets received")
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_immediate_remote_min_rx_reduction(self):
|
||||
""" immediately honor remote required min rx reduction """
|
||||
self.vpp_session.remove_vpp_config()
|
||||
@@ -809,6 +816,7 @@ class BFD4TestCase(VppTestCase):
|
||||
"time between BFD packets")
|
||||
reference_packet = p
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_modify_req_min_rx_double(self):
|
||||
""" modify session - double required min rx """
|
||||
bfd_session_up(self)
|
||||
@@ -838,6 +846,7 @@ class BFD4TestCase(VppTestCase):
|
||||
self.assert_in_range(time_to_event, .9 * timeout,
|
||||
1.1 * timeout, "session timeout")
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_modify_req_min_rx_halve(self):
|
||||
""" modify session - halve required min rx """
|
||||
self.vpp_session.modify_parameters(
|
||||
@@ -878,6 +887,7 @@ class BFD4TestCase(VppTestCase):
|
||||
"time before bfd session goes down")
|
||||
verify_event(self, e, expected_state=BFDState.down)
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_modify_detect_mult(self):
|
||||
""" modify detect multiplier """
|
||||
bfd_session_up(self)
|
||||
@@ -901,6 +911,7 @@ class BFD4TestCase(VppTestCase):
|
||||
self.assertNotIn("P", p.sprintf("%BFD.flags%"),
|
||||
"Poll bit not set in BFD packet")
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_queued_poll(self):
|
||||
""" test poll sequence queueing """
|
||||
bfd_session_up(self)
|
||||
@@ -966,6 +977,7 @@ class BFD4TestCase(VppTestCase):
|
||||
self.assertNotIn("P", p.sprintf("%BFD.flags%"),
|
||||
"Poll bit set in BFD packet")
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_poll_response(self):
|
||||
""" test correct response to control frame with poll bit set """
|
||||
bfd_session_up(self)
|
||||
@@ -976,6 +988,7 @@ class BFD4TestCase(VppTestCase):
|
||||
self, pcap_time_min=time.time() - self.vpp_clock_offset)
|
||||
self.assertIn("F", final.sprintf("%BFD.flags%"))
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_no_periodic_if_remote_demand(self):
|
||||
""" no periodic frames outside poll sequence if remote demand set """
|
||||
bfd_session_up(self)
|
||||
@@ -1093,6 +1106,7 @@ class BFD4TestCase(VppTestCase):
|
||||
"number of bfd events")
|
||||
self.test_session.send_packet()
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_echo_fail(self):
|
||||
""" session goes down if echo function fails """
|
||||
bfd_session_up(self)
|
||||
@@ -1132,6 +1146,7 @@ class BFD4TestCase(VppTestCase):
|
||||
self.assert_equal(events[0].state, BFDState.down, BFDState)
|
||||
self.assertTrue(verified_diag, "Incorrect diagnostics code received")
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_echo_stop(self):
|
||||
""" echo function stops if peer sets required min echo rx zero """
|
||||
bfd_session_up(self)
|
||||
@@ -1162,6 +1177,7 @@ class BFD4TestCase(VppTestCase):
|
||||
events = self.vapi.collect_events()
|
||||
self.assert_equal(len(events), 0, "number of bfd events")
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_echo_source_removed(self):
|
||||
""" echo function stops if echo source is removed """
|
||||
bfd_session_up(self)
|
||||
@@ -1192,6 +1208,7 @@ class BFD4TestCase(VppTestCase):
|
||||
events = self.vapi.collect_events()
|
||||
self.assert_equal(len(events), 0, "number of bfd events")
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_stale_echo(self):
|
||||
""" stale echo packets don't keep a session up """
|
||||
bfd_session_up(self)
|
||||
@@ -1243,6 +1260,7 @@ class BFD4TestCase(VppTestCase):
|
||||
self.test_session.send_packet()
|
||||
self.assertTrue(timeout_ok, "Expected timeout event didn't occur")
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_invalid_echo_checksum(self):
|
||||
""" echo packets with invalid checksum don't keep a session up """
|
||||
bfd_session_up(self)
|
||||
@@ -1291,6 +1309,7 @@ class BFD4TestCase(VppTestCase):
|
||||
self.test_session.send_packet()
|
||||
self.assertTrue(timeout_ok, "Expected timeout event didn't occur")
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_admin_up_down(self):
|
||||
""" put session admin-up and admin-down """
|
||||
bfd_session_up(self)
|
||||
@@ -1328,6 +1347,7 @@ class BFD4TestCase(VppTestCase):
|
||||
e = self.vapi.wait_for_event(1, "bfd_udp_session_details")
|
||||
verify_event(self, e, expected_state=BFDState.up)
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_config_change_remote_demand(self):
|
||||
""" configuration change while peer in demand mode """
|
||||
bfd_session_up(self)
|
||||
@@ -1445,6 +1465,7 @@ class BFD6TestCase(VppTestCase):
|
||||
self.test_session.send_packet()
|
||||
self.assert_equal(self.vpp_session.state, BFDState.up, BFDState)
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_hold_up(self):
|
||||
""" hold BFD session up """
|
||||
bfd_session_up(self)
|
||||
@@ -1502,6 +1523,7 @@ class BFD6TestCase(VppTestCase):
|
||||
self.assert_equal(udp_sport_tx, udp_sport_rx, "UDP source port (== "
|
||||
"ECHO packet identifier for test purposes)")
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_echo(self):
|
||||
""" echo function used """
|
||||
bfd_session_up(self)
|
||||
@@ -1596,6 +1618,7 @@ class BFDSHA1TestCase(VppTestCase):
|
||||
bfd_key_id=self.vpp_session.bfd_key_id)
|
||||
bfd_session_up(self)
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_hold_up(self):
|
||||
""" hold BFD session up """
|
||||
key = self.factory.create_random_key(self)
|
||||
@@ -1614,6 +1637,7 @@ class BFDSHA1TestCase(VppTestCase):
|
||||
self.test_session.send_packet()
|
||||
self.assert_equal(self.vpp_session.state, BFDState.up, BFDState)
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_hold_up_meticulous(self):
|
||||
""" hold BFD session up - meticulous auth """
|
||||
key = self.factory.create_random_key(
|
||||
@@ -1635,6 +1659,7 @@ class BFDSHA1TestCase(VppTestCase):
|
||||
self.test_session.send_packet()
|
||||
self.assert_equal(self.vpp_session.state, BFDState.up, BFDState)
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_send_bad_seq_number(self):
|
||||
""" session is not kept alive by msgs with bad sequence numbers"""
|
||||
key = self.factory.create_random_key(
|
||||
@@ -1697,6 +1722,7 @@ class BFDSHA1TestCase(VppTestCase):
|
||||
wait_for_bfd_packet(self)
|
||||
self.assert_equal(self.vpp_session.state, BFDState.up, BFDState)
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_mismatch_auth(self):
|
||||
""" session is not brought down by unauthenticated msg """
|
||||
key = self.factory.create_random_key(self)
|
||||
@@ -1711,6 +1737,7 @@ class BFDSHA1TestCase(VppTestCase):
|
||||
legitimate_test_session,
|
||||
rogue_test_session)
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_mismatch_bfd_key_id(self):
|
||||
""" session is not brought down by msg with non-existent key-id """
|
||||
key = self.factory.create_random_key(self)
|
||||
@@ -1730,6 +1757,7 @@ class BFDSHA1TestCase(VppTestCase):
|
||||
legitimate_test_session,
|
||||
rogue_test_session)
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_mismatched_auth_type(self):
|
||||
""" session is not brought down by msg with wrong auth type """
|
||||
key = self.factory.create_random_key(self)
|
||||
@@ -1746,6 +1774,7 @@ class BFDSHA1TestCase(VppTestCase):
|
||||
vpp_session, legitimate_test_session, rogue_test_session,
|
||||
{'auth_type': BFDAuthType.keyed_md5})
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
def test_restart(self):
|
||||
""" simulate remote peer restart and resynchronization """
|
||||
key = self.factory.create_random_key(
|
||||
@@ -1761,7 +1790,7 @@ class BFDSHA1TestCase(VppTestCase):
|
||||
# don't send any packets for 2*detection_time
|
||||
detection_time = self.test_session.detect_mult *\
|
||||
self.vpp_session.required_min_rx / USEC_IN_SEC
|
||||
self.sleep(2*detection_time, "simulating peer restart")
|
||||
self.sleep(2 * detection_time, "simulating peer restart")
|
||||
events = self.vapi.collect_events()
|
||||
self.assert_equal(len(events), 1, "number of bfd events")
|
||||
verify_event(self, events[0], expected_state=BFDState.down)
|
||||
@@ -1774,6 +1803,7 @@ class BFDSHA1TestCase(VppTestCase):
|
||||
bfd_session_up(self)
|
||||
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
class BFDAuthOnOffTestCase(VppTestCase):
|
||||
"""Bidirectional Forwarding Detection (BFD) (changing auth) """
|
||||
|
||||
@@ -1981,6 +2011,7 @@ class BFDAuthOnOffTestCase(VppTestCase):
|
||||
"number of bfd events")
|
||||
|
||||
|
||||
@unittest.skipUnless(running_extended_tests(), "part of extended tests")
|
||||
class BFDCLITestCase(VppTestCase):
|
||||
"""Bidirectional Forwarding Detection (BFD) (CLI) """
|
||||
pg0 = None
|
||||
|
||||
Reference in New Issue
Block a user