bbfa5fdf52
Change-Id: Ib845578485f523b7f14e98c83d05f78db382ecde Signed-off-by: Klement Sekera <ksekera@cisco.com>
225 lines
8.5 KiB
Python
225 lines
8.5 KiB
Python
#!/usr/bin/env python
|
|
|
|
import sys
|
|
import shutil
|
|
import os
|
|
import select
|
|
import unittest
|
|
import argparse
|
|
import time
|
|
from multiprocessing import Process, Pipe
|
|
from framework import VppTestRunner
|
|
from debug import spawn_gdb
|
|
from log import global_logger
|
|
from discover_tests import discover_tests
|
|
from subprocess import check_output, CalledProcessError
|
|
from util import check_core_path
|
|
|
|
# timeout which controls how long the child has to finish after seeing
|
|
# a core dump in test temporary directory. If this is exceeded, parent assumes
|
|
# that child process is stuck (e.g. waiting for shm mutex, which will never
|
|
# get unlocked) and kill the child
|
|
core_timeout = 3
|
|
|
|
|
|
def test_runner_wrapper(suite, keep_alive_pipe, result_pipe, failed_pipe):
|
|
result = not VppTestRunner(
|
|
keep_alive_pipe=keep_alive_pipe,
|
|
failed_pipe=failed_pipe,
|
|
verbosity=verbose,
|
|
failfast=failfast).run(suite).wasSuccessful()
|
|
result_pipe.send(result)
|
|
result_pipe.close()
|
|
keep_alive_pipe.close()
|
|
failed_pipe.close()
|
|
|
|
|
|
class add_to_suite_callback:
|
|
def __init__(self, suite):
|
|
self.suite = suite
|
|
|
|
def __call__(self, file_name, cls, method):
|
|
suite.addTest(cls(method))
|
|
|
|
|
|
class Filter_by_class_list:
|
|
def __init__(self, class_list):
|
|
self.class_list = class_list
|
|
|
|
def __call__(self, file_name, class_name, func_name):
|
|
return class_name in self.class_list
|
|
|
|
|
|
def suite_from_failed(suite, failed):
|
|
filter_cb = Filter_by_class_list(failed)
|
|
suite = VppTestRunner.filter_tests(suite, filter_cb)
|
|
if 0 == suite.countTestCases():
|
|
raise Exception("Suite is empty after filtering out the failed tests!")
|
|
return suite
|
|
|
|
|
|
def run_forked(suite):
|
|
keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False)
|
|
result_parent_end, result_child_end = Pipe(duplex=False)
|
|
failed_parent_end, failed_child_end = Pipe(duplex=False)
|
|
|
|
child = Process(target=test_runner_wrapper,
|
|
args=(suite, keep_alive_child_end, result_child_end,
|
|
failed_child_end))
|
|
child.start()
|
|
last_test_temp_dir = None
|
|
last_test_vpp_binary = None
|
|
last_test = None
|
|
result = None
|
|
failed = set()
|
|
last_heard = time.time()
|
|
core_detected_at = None
|
|
debug_core = os.getenv("DEBUG", "").lower() == "core"
|
|
while True:
|
|
readable = select.select([keep_alive_parent_end.fileno(),
|
|
result_parent_end.fileno(),
|
|
failed_parent_end.fileno(),
|
|
],
|
|
[], [], 1)[0]
|
|
if result_parent_end.fileno() in readable:
|
|
result = result_parent_end.recv()
|
|
break
|
|
if keep_alive_parent_end.fileno() in readable:
|
|
while keep_alive_parent_end.poll():
|
|
last_test, last_test_vpp_binary,\
|
|
last_test_temp_dir, vpp_pid = keep_alive_parent_end.recv()
|
|
last_heard = time.time()
|
|
if failed_parent_end.fileno() in readable:
|
|
while failed_parent_end.poll():
|
|
failed_test = failed_parent_end.recv()
|
|
failed.add(failed_test.__name__)
|
|
last_heard = time.time()
|
|
fail = False
|
|
if last_heard + test_timeout < time.time() and \
|
|
not os.path.isfile("%s/_core_handled" % last_test_temp_dir):
|
|
fail = True
|
|
global_logger.critical("Timeout while waiting for child test "
|
|
"runner process (last test running was "
|
|
"`%s' in `%s')!" %
|
|
(last_test, last_test_temp_dir))
|
|
elif not child.is_alive():
|
|
fail = True
|
|
global_logger.critical("Child python process unexpectedly died "
|
|
"(last test running was `%s' in `%s')!" %
|
|
(last_test, last_test_temp_dir))
|
|
elif last_test_temp_dir and last_test_vpp_binary:
|
|
core_path = "%s/core" % last_test_temp_dir
|
|
if os.path.isfile(core_path):
|
|
if core_detected_at is None:
|
|
core_detected_at = time.time()
|
|
elif core_detected_at + core_timeout < time.time():
|
|
if not os.path.isfile(
|
|
"%s/_core_handled" % last_test_temp_dir):
|
|
global_logger.critical(
|
|
"Child python process unresponsive and core-file "
|
|
"exists in test temporary directory!")
|
|
fail = True
|
|
|
|
if fail:
|
|
failed_dir = os.getenv('VPP_TEST_FAILED_DIR')
|
|
lttd = last_test_temp_dir.split("/")[-1]
|
|
link_path = '%s%s-FAILED' % (failed_dir, lttd)
|
|
global_logger.error("Creating a link to the failed " +
|
|
"test: %s -> %s" % (link_path, lttd))
|
|
try:
|
|
os.symlink(last_test_temp_dir, link_path)
|
|
except Exception:
|
|
pass
|
|
api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid
|
|
if os.path.isfile(api_post_mortem_path):
|
|
global_logger.error("Copying api_post_mortem.%d to %s" %
|
|
(vpp_pid, last_test_temp_dir))
|
|
shutil.copy2(api_post_mortem_path, last_test_temp_dir)
|
|
if last_test_temp_dir and last_test_vpp_binary:
|
|
core_path = "%s/core" % last_test_temp_dir
|
|
if os.path.isfile(core_path):
|
|
global_logger.error("Core-file exists in test temporary "
|
|
"directory: %s!" % core_path)
|
|
check_core_path(global_logger, core_path)
|
|
global_logger.debug("Running `file %s':" % core_path)
|
|
try:
|
|
info = check_output(["file", core_path])
|
|
global_logger.debug(info)
|
|
except CalledProcessError as e:
|
|
global_logger.error(
|
|
"Could not run `file' utility on core-file, "
|
|
"rc=%s" % e.returncode)
|
|
pass
|
|
if debug_core:
|
|
spawn_gdb(last_test_vpp_binary, core_path,
|
|
global_logger)
|
|
child.terminate()
|
|
result = -1
|
|
break
|
|
keep_alive_parent_end.close()
|
|
result_parent_end.close()
|
|
failed_parent_end.close()
|
|
return result, failed
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
try:
|
|
verbose = int(os.getenv("V", 0))
|
|
except ValueError:
|
|
verbose = 0
|
|
|
|
default_test_timeout = 600 # 10 minutes
|
|
try:
|
|
test_timeout = int(os.getenv("TIMEOUT", default_test_timeout))
|
|
except ValueError:
|
|
test_timeout = default_test_timeout
|
|
|
|
debug = os.getenv("DEBUG")
|
|
|
|
s = os.getenv("STEP", "n")
|
|
step = True if s.lower() in ("y", "yes", "1") else False
|
|
|
|
parser = argparse.ArgumentParser(description="VPP unit tests")
|
|
parser.add_argument("-f", "--failfast", action='count',
|
|
help="fast failure flag")
|
|
parser.add_argument("-d", "--dir", action='append', type=str,
|
|
help="directory containing test files "
|
|
"(may be specified multiple times)")
|
|
args = parser.parse_args()
|
|
failfast = True if args.failfast == 1 else False
|
|
|
|
suite = unittest.TestSuite()
|
|
cb = add_to_suite_callback(suite)
|
|
for d in args.dir:
|
|
print("Adding tests from directory tree %s" % d)
|
|
discover_tests(d, cb)
|
|
|
|
try:
|
|
retries = int(os.getenv("RETRIES", 0))
|
|
except ValueError:
|
|
retries = 0
|
|
|
|
try:
|
|
force_foreground = int(os.getenv("FORCE_FOREGROUND", 0))
|
|
except ValueError:
|
|
force_foreground = 0
|
|
attempts = retries + 1
|
|
if attempts > 1:
|
|
print("Perform %s attempts to pass the suite..." % attempts)
|
|
if (debug is not None and debug.lower() in ["gdb", "gdbserver"]) or step\
|
|
or force_foreground:
|
|
# don't fork if requiring interactive terminal..
|
|
sys.exit(not VppTestRunner(
|
|
verbosity=verbose, failfast=failfast).run(suite).wasSuccessful())
|
|
else:
|
|
while True:
|
|
result, failed = run_forked(suite)
|
|
attempts = attempts - 1
|
|
print("%s test(s) failed, %s attempt(s) left" %
|
|
(len(failed), attempts))
|
|
if len(failed) > 0 and attempts > 0:
|
|
suite = suite_from_failed(suite, failed)
|
|
continue
|
|
sys.exit(result)
|