ligang/vpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

make test: detect hung tests

Browse Source 
Run tests in a forked process with a set of pipes to communicate
keep-alives and overall result. This allows us to detect when e.g.
vpp dies mid-API call causing the test to hang waiting for response
(which will never come since vpp died).

Support setting a (per test case) TIMEOUT make test option to set timeout,
with a default timeout of 120 seconds.

Example - fail the test suite if any test-case fails to finish within
300s:

make TIMEOUT=300 test

Change-Id: I0d04f26a7232968f4bf043adf5d5b508f5018717
Signed-off-by: Klement Sekera <ksekera@cisco.com>
This commit is contained in:
Klement Sekera
2017-08-08 04:33:53 +02:00
committed by Dave Wallace
parent 52851e6aa9
commit 909a6a1eb9
6 changed files with 148 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
test/run_tests.py
View File

@ -2,10 +2,14 @@
import sys
import os
import select
import unittest
import argparse
import importlib
from multiprocessing import Process, Pipe
from framework import VppTestRunner
from debug import spawn_gdb
from log import global_logger
def add_from_dir(suite, directory):
@ -39,12 +43,38 @@ def add_from_dir(suite, directory):
if method.startswith("test_"):
suite.addTest(cls(method))
def test_runner_wrapper(keep_alive_pipe, result_pipe):
result = not VppTestRunner(
pipe=keep_alive_pipe,
verbosity=verbose,
failfast=failfast).run(suite).wasSuccessful()
result_pipe.send(result)
result_pipe.close()
keep_alive_pipe.close()
def handle_core(vpp_binary, core_path):
try:
d = os.getenv("DEBUG")
except:
d = None
if d and d.lower() == "core":
spawn_gdb(vpp_binary, core_path, global_logger)
if __name__ == '__main__':
try:
verbose = int(os.getenv("V", 0))
except:
verbose = 0
default_test_timeout = 120
try:
test_timeout = int(os.getenv("TIMEOUT", default_test_timeout))
except:
test_timeout = default_test_timeout
parser = argparse.ArgumentParser(description="VPP unit tests")
parser.add_argument("-f", "--failfast", action='count',
help="fast failure flag")
@ -56,7 +86,43 @@ if __name__ == '__main__':
suite = unittest.TestSuite()
for d in args.dir:
print("Adding tests from directory tree %s" % d)
global_logger.info("Adding tests from directory tree %s" % d)
add_from_dir(suite, d)
sys.exit(not VppTestRunner(verbosity=verbose,
failfast=failfast).run(suite).wasSuccessful())
keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False)
result_parent_end, result_child_end = Pipe(duplex=False)
p = Process(target=test_runner_wrapper,
args=(keep_alive_child_end,
result_child_end))
p.start()
last_test_temp_dir = None
last_test_vpp_binary = None
last_test = None
result = None
while result is None:
readable = select.select([keep_alive_parent_end.fileno(),
result_parent_end.fileno(),
],
[], [], test_timeout)[0]
if result_parent_end.fileno() in readable:
result = result_parent_end.recv()
elif keep_alive_parent_end.fileno() in readable:
while keep_alive_parent_end.poll():
last_test, last_test_vpp_binary, last_test_temp_dir =\
keep_alive_parent_end.recv()
else:
global_logger.critical("Timeout while waiting for child test "
"runner process (last test running was "
"`%s' in `%s')!" %
(last_test, last_test_temp_dir))
if last_test_temp_dir and last_test_vpp_binary:
core_path = "%s/core" % last_test_temp_dir
if os.path.isfile(core_path):
global_logger.error("Core-file exists in test temporary "
"directory: %s!" % core_path)
handle_core(last_test_vpp_binary, core_path)
p.terminate()
result = -1
keep_alive_parent_end.close()
result_parent_end.close()
sys.exit(result)