tests: support multiple filter expressions
Support multiple comma-delimited filter expressions, e.g. to run both bfd and ip4 tests, it's now possible to do: make test TEST=bfd,ip4 Same goes for wildcards, e.g.: make test TEST=bfd,..test_longest_prefix_match,..test_icmp_error Type: improvement Change-Id: I0cceaa443cb612dca955f301c7407959f9a71a6e Signed-off-by: Klement Sekera <klement.sekera@gmail.com>
This commit is contained in:
Klement Sekera
committed by
Dave Wallace
Dave Wallace
parent
39d7699c20
commit
08c50e3b7a
199
test/Makefile
199
test/Makefile
@ -404,52 +404,165 @@ help:
|
||||
@echo " test-shell-debug - enter shell with test environment (debug build)"
|
||||
@echo " test-refresh-deps - refresh the Python dependencies for the tests"
|
||||
@echo ""
|
||||
@echo "Arguments controlling test runs:"
|
||||
@echo "Environment variables controlling test runs:"
|
||||
@echo ""
|
||||
@echo " V=[0|1|2] - set test verbosity level"
|
||||
@echo " 0=ERROR, 1=INFO, 2=DEBUG"
|
||||
@echo " TEST_JOBS=[<n>|auto] - use at most <n> parallel python processes for test execution, if auto, set to number of available cpus (default: 1)"
|
||||
@echo " MAX_VPP_CPUS=[<n>|auto]- use at most <n> cpus for running vpp main and worker threads, if auto, set to number of available cpus (default: auto)"
|
||||
@echo " CACHE_OUTPUT=[0|n|no] - disable cache VPP stdout/stderr and log as one block after test finishes (default: yes)"
|
||||
@echo " FAILFAST=[1|y|yes] - fail fast if 1, otherwise complete all tests"
|
||||
@echo " TIMEOUT=<timeout> - fail test suite if any single test takes longer than <timeout> (in seconds) to finish (default: 600)"
|
||||
@echo " RETRIES=<n> - retry failed tests <n> times"
|
||||
@echo " DEBUG=<type> - set VPP debugging kind"
|
||||
@echo " DEBUG=core - detect coredump and load it in gdb on crash"
|
||||
@echo " DEBUG=gdb - allow easy debugging by printing VPP PID"
|
||||
@echo " and waiting for user input before running"
|
||||
@echo " and tearing down a testcase"
|
||||
@echo " DEBUG=gdbserver - run gdb inside a gdb server, otherwise"
|
||||
@echo " same as above"
|
||||
@echo " DEBUG=attach - attach test case to already running vpp in gdb (see test-start-vpp-in-gdb)"
|
||||
@echo " STEP=[1|y|yes] - enable stepping through a testcase (for testcase debugging)"
|
||||
@echo " SANITY=[0|n|no] - disable sanity import of vpp-api/sanity vpp run before running tests"
|
||||
@echo " EXTENDED_TESTS=[1|y|yes] - run extended tests"
|
||||
@echo " TEST=<filter> - filter the set of tests:"
|
||||
@echo " by file-name - only run tests from specified file, e.g. TEST=test_bfd selects all tests from test_bfd.py"
|
||||
@echo " by file-suffix - same as file-name, but 'test_' is omitted e.g. TEST=bfd selects all tests from test_bfd.py"
|
||||
@echo " by wildcard - wildcard filter is <file>.<class>.<test function>, each can be replaced by '*'"
|
||||
@echo " e.g. TEST='test_bfd.*.*' is equivalent to above example of filter by file-name"
|
||||
@echo " TEST='bfd.*.*' is equivalent to above example of filter by file-suffix"
|
||||
@echo " TEST='bfd.BFDAPITestCase.*' selects all tests from test_bfd.py which are part of BFDAPITestCase class"
|
||||
@echo " TEST='bfd.BFDAPITestCase.test_add_bfd' selects a single test named test_add_bfd from test_bfd.py/BFDAPITestCase"
|
||||
@echo " TEST='*.*.test_add_bfd' selects all test functions named test_add_bfd from all files/classes"
|
||||
@echo " VARIANT=<variant> - specify which march node variant to unit test"
|
||||
@echo " V=[0|1|2]"
|
||||
@echo " set test verbosity level: 0=ERROR, 1=INFO, 2=DEBUG"
|
||||
@echo ""
|
||||
@echo " TEST_JOBS=[<n>|auto]"
|
||||
@echo " use at most <n> parallel python processes for test"
|
||||
@echo " execution, if auto, set to number of available cpus"
|
||||
@echo " (default: 1)"
|
||||
@echo ""
|
||||
@echo " MAX_VPP_CPUS=[<n>|auto]"
|
||||
@echo " use at most <n> cpus for running vpp"
|
||||
@echo " 'auto' sets to number of available cpus"
|
||||
@echo " (default: auto)"
|
||||
@echo ""
|
||||
@echo " CACHE_OUTPUT=[0|n|no]"
|
||||
@echo " disable caching VPP stdout/stderr and logging it"
|
||||
@echo " as one block after test finishes"
|
||||
@echo " (default: yes)"
|
||||
@echo ""
|
||||
@echo " FAILFAST=[1|y|yes]"
|
||||
@echo " if enabled, stop running tests on first failure"
|
||||
@echo " otherwise finish running whole suite"
|
||||
@echo " (default: no)"
|
||||
@echo ""
|
||||
@echo " TIMEOUT=<timeout>"
|
||||
@echo " fail test suite if any single test takes longer"
|
||||
@echo " than <timeout> (in seconds) to finish"
|
||||
@echo " (default: 600)"
|
||||
@echo ""
|
||||
@echo " RETRIES=<n>"
|
||||
@echo " retry failed tests <n> times"
|
||||
@echo " (default: 0)"
|
||||
@echo ""
|
||||
@echo " DEBUG=<type>"
|
||||
@echo " configure VPP debugging:"
|
||||
@echo " DEBUG=core"
|
||||
@echo " detect coredump and load it in gdb on crash"
|
||||
@echo ""
|
||||
@echo " DEBUG=gdb"
|
||||
@echo " print VPP PID and wait for user input before"
|
||||
@echo " running and tearing down a testcase, allowing"
|
||||
@echo " easy gdb attach"
|
||||
@echo ""
|
||||
@echo " DEBUG=gdbserver"
|
||||
@echo " same as above, but run gdb inside a gdb server"
|
||||
@echo ""
|
||||
@echo " DEBUG=attach"
|
||||
@echo " attach to existing vpp in running in gdb"
|
||||
@echo " (see test-start-vpp-in-gdb)"
|
||||
@echo " (default: none)"
|
||||
@echo ""
|
||||
@echo " STEP=[1|y|yes]"
|
||||
@echo " enable stepping through a testcase"
|
||||
@echo " (default: no)"
|
||||
@echo ""
|
||||
@echo " SANITY=[0|n|no]"
|
||||
@echo " disable sanity import of vpp-api/vpp sanity"
|
||||
@echo " run before running tests"
|
||||
@echo " (default: yes)"
|
||||
@echo ""
|
||||
@echo " EXTENDED_TESTS=[1|y|yes]"
|
||||
@echo " run extended tests"
|
||||
@echo " (default: no)"
|
||||
@echo ""
|
||||
@echo " TEST=<filter>,[<filter>],..."
|
||||
@echo " only run tests matching one or more comma-delimited"
|
||||
@echo " filter expressions"
|
||||
@echo ""
|
||||
@echo " simple filter:"
|
||||
@echo " file name or file suffix select all tests from a file"
|
||||
@echo " examples:"
|
||||
@echo " TEST=test_bfd"
|
||||
@echo " TEST=bfd"
|
||||
@echo " equivalent expressions selecting all"
|
||||
@echo " tests defined in test_bfd.py"
|
||||
@echo ""
|
||||
@echo " wildcard filter:"
|
||||
@echo " advanced filtering based on test file, test class"
|
||||
@echo " and test function"
|
||||
@echo " each filter expression is in the form of"
|
||||
@echo " <file>.<class>.<test function>"
|
||||
@echo " each of the tokens can be left empty or replaced"
|
||||
@echo " with '*' to select all objects available"
|
||||
@echo " examples:"
|
||||
@echo " TEST=test_bfd.*.*"
|
||||
@echo " TEST=test_bfd.."
|
||||
@echo " TEST=bfd.*.*"
|
||||
@echo " TEST=bfd.."
|
||||
@echo " select all tests defined in test_bfd.py"
|
||||
@echo " TEST=bfd.BFDAPITestCase.*"
|
||||
@echo " TEST=bfd.BFDAPITestCase."
|
||||
@echo " select all tests from test_bfd.py"
|
||||
@echo " which are part of BFDAPITestCase class"
|
||||
@echo " TEST=bfd.BFDAPITestCase.test_add_bfd"
|
||||
@echo " select a single test named test_add_bfd"
|
||||
@echo " from test_bfd.py/BFDAPITestCase"
|
||||
@echo " TEST=..test_add_bfd"
|
||||
@echo " TEST=*.*.test_add_bfd"
|
||||
@echo " select all test functions named test_add_bfd"
|
||||
@echo " from all files/classes"
|
||||
@echo " TEST=bfd,ip4,..test_icmp_error"
|
||||
@echo " select all test functions in test_bfd.py,"
|
||||
@echo " test_ip4.py and all test functions named"
|
||||
@echo " 'test_icmp_error' in all files"
|
||||
@echo " (default: '')"
|
||||
@echo ""
|
||||
@echo " VARIANT=<variant>"
|
||||
@echo " specify which march node variant to unit test"
|
||||
@echo " e.g. VARIANT=skx test the skx march variants"
|
||||
@echo " e.g. VARIANT=icl test the icl march variants"
|
||||
@echo " COREDUMP_SIZE=<size> - pass <size> as unix { coredump-size <size> } argument to vpp"
|
||||
@echo " e.g. COREDUMP_SIZE=4g"
|
||||
@echo " COREDUMP_SIZE=unlimited"
|
||||
@echo " COREDUMP_COMPRESS=[1|y|yes] - compress core files if not debugging them"
|
||||
@echo " EXTERN_TESTS=<path> - path to out-of-tree test_<name>.py files containing test cases"
|
||||
@echo " EXTERN_PLUGINS=<path> - path to out-of-tree plugins to be loaded by vpp under test"
|
||||
@echo " EXTERN_COV_DIR=<path> - path to out-of-tree prefix, where source, object and .gcda files can be found for coverage report"
|
||||
@echo " PROFILE=[1|y|yes] - enable profiling of test framework via cProfile module"
|
||||
@echo " PROFILE_SORT_BY=opt - sort profiling report by opt - consult cProfile documentation for possible values (default: cumtime)"
|
||||
@echo " PROFILE_OUTPUT=file - output profiling info to file - use absolute path (default: stdout)"
|
||||
@echo " TEST_DEBUG=[1|y|yes] - enable debugging of the test framework itself (expert)"
|
||||
@echo " API_FUZZ=[1|y|yes] - enable VPP api fuzz testing"
|
||||
@echo " RND_SEED=<seed> - Seed RND with given seed"
|
||||
@echo " (default: '')"
|
||||
@echo ""
|
||||
@echo " COREDUMP_SIZE=<size>"
|
||||
@echo " pass <size> as unix { coredump-size <size> } argument"
|
||||
@echo " to vpp, e.g. COREDUMP_SIZE=4g or COREDUMP_SIZE=unlimited"
|
||||
@echo " (default: '')"
|
||||
@echo ""
|
||||
@echo " COREDUMP_COMPRESS=[1|y|yes]"
|
||||
@echo " if no debug option is set, compress any core files"
|
||||
@echo " (default: no)"
|
||||
@echo ""
|
||||
@echo " EXTERN_TESTS=<path>"
|
||||
@echo " include out-of-tree test_*.py files under <path>"
|
||||
@echo " (default: '')"
|
||||
@echo ""
|
||||
@echo " EXTERN_PLUGINS=<path>"
|
||||
@echo " load out-of-tree vpp plugins in <path>"
|
||||
@echo " (default: '')"
|
||||
@echo ""
|
||||
@echo " EXTERN_COV_DIR=<path>"
|
||||
@echo " path to out-of-tree prefix, where source, object"
|
||||
@echo " and .gcda files can be found for coverage report"
|
||||
@echo " (default: '')"
|
||||
@echo ""
|
||||
@echo " PROFILE=[1|y|yes]"
|
||||
@echo " enable profiling of test framework via cProfile module"
|
||||
@echo " (default: no)"
|
||||
@echo ""
|
||||
@echo " PROFILE_SORT_BY=opt"
|
||||
@echo " sort profiling report by opt - see cProfile documentation"
|
||||
@echo " for possible values"
|
||||
@echo " (default: cumtime)"
|
||||
@echo ""
|
||||
@echo " PROFILE_OUTPUT=file"
|
||||
@echo " output profiling info to file - use absolute path"
|
||||
@echo " (default: stdout)"
|
||||
@echo ""
|
||||
@echo " TEST_DEBUG=[1|y|yes]"
|
||||
@echo " enable debugging of the test framework itself (expert)"
|
||||
@echo " (default: no)"
|
||||
@echo ""
|
||||
@echo " API_FUZZ=[1|y|yes]"
|
||||
@echo " enable VPP api fuzz testing"
|
||||
@echo " (default: no)"
|
||||
@echo ""
|
||||
@echo " RND_SEED=<seed>"
|
||||
@echo " random seed used by test framework"
|
||||
@echo " (default: time.time())"
|
||||
@echo ""
|
||||
@echo "Starting VPP in GDB for use with DEBUG=attach:"
|
||||
@echo ""
|
||||
|
||||
@ -122,7 +122,8 @@ parser.add_argument(
|
||||
)
|
||||
|
||||
filter_help_string = """\
|
||||
expression consists of 3 string selectors separated by '.' separators:
|
||||
expression consists of one or more filters separated by commas (',')
|
||||
filter consists of 3 string selectors separated by dots ('.')
|
||||
|
||||
<file>.<class>.<function>
|
||||
|
||||
@ -142,6 +143,8 @@ examples:
|
||||
test_add_bfd from test_bfd.py/BFDAPITestCase
|
||||
4. '.*.test_add_bfd' selects all test functions named test_add_bfd
|
||||
from all files/classes
|
||||
5. 'bfd,ip4,..test_icmp_error' selects all test functions in test_bfd.py,
|
||||
test_ip4.py and all test functions named 'test_icmp_error' in all files
|
||||
"""
|
||||
parser.add_argument(
|
||||
"--filter", action="store", metavar="FILTER_EXPRESSION", help=filter_help_string
|
||||
|
||||
@ -631,7 +631,7 @@ def parse_test_filter(test_filter):
|
||||
if "." in f:
|
||||
parts = f.split(".")
|
||||
if len(parts) > 3:
|
||||
raise Exception("Unrecognized %s option: %s" % (test_option, f))
|
||||
raise Exception(f"Invalid test filter: {test_filter}")
|
||||
if len(parts) > 2:
|
||||
if parts[2] not in ("*", ""):
|
||||
filter_func_name = parts[2]
|
||||
@ -677,22 +677,41 @@ def filter_tests(tests, filter_cb):
|
||||
|
||||
|
||||
class FilterByTestOption:
|
||||
def __init__(self, filter_file_name, filter_class_name, filter_func_name):
|
||||
self.filter_file_name = filter_file_name
|
||||
self.filter_class_name = filter_class_name
|
||||
self.filter_func_name = filter_func_name
|
||||
def __init__(self, filters):
|
||||
self.filters = filters
|
||||
|
||||
def __call__(self, file_name, class_name, func_name):
|
||||
if self.filter_file_name:
|
||||
fn_match = fnmatch.fnmatch(file_name, self.filter_file_name)
|
||||
def test_one(
|
||||
filter_file_name,
|
||||
filter_class_name,
|
||||
filter_func_name,
|
||||
file_name,
|
||||
class_name,
|
||||
func_name,
|
||||
):
|
||||
if filter_file_name:
|
||||
fn_match = fnmatch.fnmatch(file_name, filter_file_name)
|
||||
if not fn_match:
|
||||
return False
|
||||
if self.filter_class_name and class_name != self.filter_class_name:
|
||||
if filter_class_name and class_name != filter_class_name:
|
||||
return False
|
||||
if self.filter_func_name and func_name != self.filter_func_name:
|
||||
if filter_func_name and func_name != filter_func_name:
|
||||
return False
|
||||
return True
|
||||
|
||||
for filter_file_name, filter_class_name, filter_func_name in self.filters:
|
||||
if test_one(
|
||||
filter_file_name,
|
||||
filter_class_name,
|
||||
filter_func_name,
|
||||
file_name,
|
||||
class_name,
|
||||
func_name,
|
||||
):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
class FilterByClassList:
|
||||
def __init__(self, classes_with_filenames):
|
||||
@ -938,14 +957,17 @@ if __name__ == "__main__":
|
||||
descriptions = True
|
||||
|
||||
print("Running tests using custom test runner.")
|
||||
filter_file, filter_class, filter_func = parse_test_filter(config.filter)
|
||||
filters = [(parse_test_filter(f)) for f in config.filter.split(",")]
|
||||
|
||||
print(
|
||||
"Selected filters: file=%s, class=%s, function=%s"
|
||||
% (filter_file, filter_class, filter_func)
|
||||
"Selected filters: ",
|
||||
"|".join(
|
||||
f"file={filter_file}, class={filter_class}, function={filter_func}"
|
||||
for filter_file, filter_class, filter_func in filters
|
||||
),
|
||||
)
|
||||
|
||||
filter_cb = FilterByTestOption(filter_file, filter_class, filter_func)
|
||||
filter_cb = FilterByTestOption(filters)
|
||||
|
||||
cb = SplitToSuitesCallback(filter_cb)
|
||||
for d in config.test_src_dir:
|
||||
|
||||
Reference in New Issue
Block a user