tests: make tests less make dependent
Implement command line argument parsing instead of passing arguments via environment variables. Add script for running tests without having to invoke make. Deprecate running tests via make. Type: improvement Change-Id: I2e3054a61a2ae25d460e9be00be7d7705fbf943e Signed-off-by: Klement Sekera <ksekera@cisco.com> Signed-off-by: Dave Wallace <dwallacelf@gmail.com>
This commit is contained in:
Klement Sekera
committed by
Andrew Yourtchenko
Andrew Yourtchenko
parent
8ccc6b3507
commit
b23ffd7ef2
@ -9,13 +9,13 @@ import select
|
||||
import signal
|
||||
import subprocess
|
||||
import unittest
|
||||
import tempfile
|
||||
import re
|
||||
import time
|
||||
import faulthandler
|
||||
import random
|
||||
import copy
|
||||
import psutil
|
||||
import platform
|
||||
import shutil
|
||||
from collections import deque
|
||||
from threading import Thread, Event
|
||||
from inspect import getdoc, isclass
|
||||
@ -27,6 +27,7 @@ from struct import pack, unpack
|
||||
|
||||
import scapy.compat
|
||||
from scapy.packet import Raw, Packet
|
||||
from config import config, available_cpus, num_cpus, max_vpp_cpus
|
||||
import hook as hookmodule
|
||||
from vpp_pg_interface import VppPGInterface
|
||||
from vpp_sub_interface import VppSubInterface
|
||||
@ -45,7 +46,6 @@ from scapy.layers.inet import IPerror, TCPerror, UDPerror, ICMPerror
|
||||
from scapy.layers.inet6 import ICMPv6DestUnreach, ICMPv6EchoRequest
|
||||
from scapy.layers.inet6 import ICMPv6EchoReply
|
||||
|
||||
from cpu_config import available_cpus, num_cpus, max_vpp_cpus
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@ -61,27 +61,7 @@ TEST_RUN = 4
|
||||
SKIP_CPU_SHORTAGE = 5
|
||||
|
||||
|
||||
class BoolEnvironmentVariable(object):
|
||||
|
||||
def __init__(self, env_var_name, default='n', true_values=None):
|
||||
self.name = env_var_name
|
||||
self.default = default
|
||||
self.true_values = true_values if true_values is not None else \
|
||||
("y", "yes", "1")
|
||||
|
||||
def __bool__(self):
|
||||
return os.getenv(self.name, self.default).lower() in self.true_values
|
||||
|
||||
if sys.version_info[0] == 2:
|
||||
__nonzero__ = __bool__
|
||||
|
||||
def __repr__(self):
|
||||
return 'BoolEnvironmentVariable(%r, default=%r, true_values=%r)' % \
|
||||
(self.name, self.default, self.true_values)
|
||||
|
||||
|
||||
debug_framework = BoolEnvironmentVariable('TEST_DEBUG')
|
||||
if debug_framework:
|
||||
if config.debug_framework:
|
||||
import debug_internal
|
||||
|
||||
"""
|
||||
@ -175,7 +155,7 @@ def pump_output(testclass):
|
||||
limit = -1
|
||||
stdout_fragment = split[-1]
|
||||
testclass.vpp_stdout_deque.extend(split[:limit])
|
||||
if not testclass.cache_vpp_output:
|
||||
if not config.cache_vpp_output:
|
||||
for line in split[:limit]:
|
||||
testclass.logger.info(
|
||||
"VPP STDOUT: %s" % line.rstrip("\n"))
|
||||
@ -193,7 +173,7 @@ def pump_output(testclass):
|
||||
stderr_fragment = split[-1]
|
||||
|
||||
testclass.vpp_stderr_deque.extend(split[:limit])
|
||||
if not testclass.cache_vpp_output:
|
||||
if not config.cache_vpp_output:
|
||||
for line in split[:limit]:
|
||||
testclass.logger.error(
|
||||
"VPP STDERR: %s" % line.rstrip("\n"))
|
||||
@ -201,13 +181,6 @@ def pump_output(testclass):
|
||||
# flag will take care of properly terminating the loop
|
||||
|
||||
|
||||
def _is_skip_aarch64_set():
|
||||
return BoolEnvironmentVariable('SKIP_AARCH64')
|
||||
|
||||
|
||||
is_skip_aarch64_set = _is_skip_aarch64_set()
|
||||
|
||||
|
||||
def _is_platform_aarch64():
|
||||
return platform.machine() == 'aarch64'
|
||||
|
||||
@ -215,35 +188,6 @@ def _is_platform_aarch64():
|
||||
is_platform_aarch64 = _is_platform_aarch64()
|
||||
|
||||
|
||||
def _running_extended_tests():
|
||||
return BoolEnvironmentVariable("EXTENDED_TESTS")
|
||||
|
||||
|
||||
running_extended_tests = _running_extended_tests()
|
||||
|
||||
|
||||
def _running_gcov_tests():
|
||||
return BoolEnvironmentVariable("GCOV_TESTS")
|
||||
|
||||
|
||||
running_gcov_tests = _running_gcov_tests()
|
||||
|
||||
|
||||
def get_environ_vpp_worker_count():
|
||||
worker_config = os.getenv("VPP_WORKER_CONFIG", None)
|
||||
if worker_config:
|
||||
elems = worker_config.split(" ")
|
||||
if elems[0] != "workers" or len(elems) != 2:
|
||||
raise ValueError("Wrong VPP_WORKER_CONFIG == '%s' value." %
|
||||
worker_config)
|
||||
return int(elems[1])
|
||||
else:
|
||||
return 0
|
||||
|
||||
|
||||
environ_vpp_worker_count = get_environ_vpp_worker_count()
|
||||
|
||||
|
||||
class KeepAliveReporter(object):
|
||||
"""
|
||||
Singleton object which reports test start to parent process
|
||||
@ -277,7 +221,7 @@ class KeepAliveReporter(object):
|
||||
else:
|
||||
desc = test.id()
|
||||
|
||||
self.pipe.send((desc, test.vpp_bin, test.tempdir, test.vpp.pid))
|
||||
self.pipe.send((desc, config.vpp, test.tempdir, test.vpp.pid))
|
||||
|
||||
|
||||
class TestCaseTag(Enum):
|
||||
@ -411,7 +355,7 @@ class VppTestCase(CPUInterface, unittest.TestCase):
|
||||
if cls.has_tag(TestCaseTag.FIXME_VPP_WORKERS):
|
||||
cls.vpp_worker_count = 0
|
||||
else:
|
||||
cls.vpp_worker_count = environ_vpp_worker_count
|
||||
cls.vpp_worker_count = config.vpp_worker_count
|
||||
return cls.vpp_worker_count
|
||||
|
||||
@classmethod
|
||||
@ -421,42 +365,38 @@ class VppTestCase(CPUInterface, unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpConstants(cls):
|
||||
""" Set-up the test case class based on environment variables """
|
||||
cls.step = BoolEnvironmentVariable('STEP')
|
||||
# inverted case to handle '' == True
|
||||
c = os.getenv("CACHE_OUTPUT", "1")
|
||||
cls.cache_vpp_output = False if c.lower() in ("n", "no", "0") else True
|
||||
cls.vpp_bin = os.getenv('VPP_BIN', "vpp")
|
||||
extern_plugin_path = os.getenv('EXTERN_PLUGINS')
|
||||
cls.step = config.step
|
||||
cls.plugin_path = ":".join(config.vpp_plugin_dir)
|
||||
cls.test_plugin_path = ":".join(config.vpp_test_plugin_dir)
|
||||
cls.extern_plugin_path = ":".join(config.extern_plugin_dir)
|
||||
debug_cli = ""
|
||||
if cls.step or cls.debug_gdb or cls.debug_gdbserver:
|
||||
debug_cli = "cli-listen localhost:5002"
|
||||
coredump_size = None
|
||||
size = os.getenv("COREDUMP_SIZE")
|
||||
if size is not None:
|
||||
coredump_size = "coredump-size %s" % size
|
||||
if coredump_size is None:
|
||||
size = re.search(r"\d+[gG]", config.coredump_size)
|
||||
if size:
|
||||
coredump_size = f"coredump-size {config.coredump_size}".lower()
|
||||
else:
|
||||
coredump_size = "coredump-size unlimited"
|
||||
|
||||
default_variant = os.getenv("VARIANT")
|
||||
default_variant = config.variant
|
||||
if default_variant is not None:
|
||||
default_variant = "defaults { %s 100 }" % default_variant
|
||||
else:
|
||||
default_variant = ""
|
||||
|
||||
api_fuzzing = os.getenv("API_FUZZ")
|
||||
api_fuzzing = config.api_fuzz
|
||||
if api_fuzzing is None:
|
||||
api_fuzzing = 'off'
|
||||
|
||||
cls.vpp_cmdline = [
|
||||
cls.vpp_bin,
|
||||
config.vpp,
|
||||
"unix", "{", "nodaemon", debug_cli, "full-coredump",
|
||||
coredump_size, "runtime-dir", cls.tempdir, "}",
|
||||
"api-trace", "{", "on", "}",
|
||||
"api-segment", "{", "prefix", cls.get_api_segment_prefix(), "}",
|
||||
"cpu", "{", "main-core", str(cls.cpus[0]), ]
|
||||
if extern_plugin_path is not None:
|
||||
if cls.extern_plugin_path not in (None, ""):
|
||||
cls.extra_vpp_plugin_config.append(
|
||||
"add-path %s" % extern_plugin_path)
|
||||
"add-path %s" % cls.extern_plugin_path)
|
||||
if cls.get_vpp_worker_count():
|
||||
cls.vpp_cmdline.extend([
|
||||
"corelist-workers", ",".join([str(x) for x in cls.cpus[1:]])])
|
||||
@ -495,15 +435,14 @@ class VppTestCase(CPUInterface, unittest.TestCase):
|
||||
print(single_line_delim)
|
||||
print("You can debug VPP using:")
|
||||
if cls.debug_gdbserver:
|
||||
print("sudo gdb " + cls.vpp_bin +
|
||||
" -ex 'target remote localhost:{port}'"
|
||||
.format(port=cls.gdbserver_port))
|
||||
print(f"sudo gdb {config.vpp} "
|
||||
f"-ex 'target remote localhost:{cls.gdbserver_port}'")
|
||||
print("Now is the time to attach gdb by running the above "
|
||||
"command, set up breakpoints etc., then resume VPP from "
|
||||
"within gdb by issuing the 'continue' command")
|
||||
cls.gdbserver_port += 1
|
||||
elif cls.debug_gdb:
|
||||
print("sudo gdb " + cls.vpp_bin + " -ex 'attach %s'" % cls.vpp.pid)
|
||||
print(f"sudo gdb {config.vpp} -ex 'attach {cls.vpp.pid}'")
|
||||
print("Now is the time to attach gdb by running the above "
|
||||
"command and set up breakpoints etc., then resume VPP from"
|
||||
" within gdb by issuing the 'continue' command")
|
||||
@ -585,11 +524,23 @@ class VppTestCase(CPUInterface, unittest.TestCase):
|
||||
|
||||
@classmethod
|
||||
def get_tempdir(cls):
|
||||
if cls.debug_attach:
|
||||
return os.getenv("VPP_IN_GDB_TMP_DIR",
|
||||
"/tmp/unittest-attach-gdb")
|
||||
else:
|
||||
return tempfile.mkdtemp(prefix='vpp-unittest-%s-' % cls.__name__)
|
||||
tmpdir = f"{config.tmp_dir}/vpp-unittest-{cls.__name__}"
|
||||
if config.wipe_tmp_dir:
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
os.mkdir(tmpdir)
|
||||
return tmpdir
|
||||
|
||||
@classmethod
|
||||
def create_file_handler(cls):
|
||||
if config.log_dir is None:
|
||||
cls.file_handler = FileHandler(f"{cls.tempdir}/log.txt")
|
||||
return
|
||||
|
||||
logdir = f"{config.log_dir}/vpp-unittest-{cls.__name__}"
|
||||
if config.wipe_tmp_dir:
|
||||
shutil.rmtree(logdir, ignore_errors=True)
|
||||
os.mkdir(logdir)
|
||||
cls.file_handler = FileHandler(f"{logdir}/log.txt")
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
@ -599,15 +550,13 @@ class VppTestCase(CPUInterface, unittest.TestCase):
|
||||
"""
|
||||
super(VppTestCase, cls).setUpClass()
|
||||
cls.logger = get_logger(cls.__name__)
|
||||
seed = os.environ["RND_SEED"]
|
||||
random.seed(seed)
|
||||
random.seed(config.rnd_seed)
|
||||
if hasattr(cls, 'parallel_handler'):
|
||||
cls.logger.addHandler(cls.parallel_handler)
|
||||
cls.logger.propagate = False
|
||||
d = os.getenv("DEBUG", None)
|
||||
cls.set_debug_flags(d)
|
||||
cls.set_debug_flags(config.debug)
|
||||
cls.tempdir = cls.get_tempdir()
|
||||
cls.file_handler = FileHandler("%s/log.txt" % cls.tempdir)
|
||||
cls.create_file_handler()
|
||||
cls.file_handler.setFormatter(
|
||||
Formatter(fmt='%(asctime)s,%(msecs)03d %(message)s',
|
||||
datefmt="%H:%M:%S"))
|
||||
@ -617,7 +566,7 @@ class VppTestCase(CPUInterface, unittest.TestCase):
|
||||
os.chdir(cls.tempdir)
|
||||
cls.logger.info("Temporary dir is %s, api socket is %s",
|
||||
cls.tempdir, cls.get_api_sock_path())
|
||||
cls.logger.debug("Random seed is %s", seed)
|
||||
cls.logger.debug("Random seed is %s", config.rnd_seed)
|
||||
cls.setUpConstants()
|
||||
cls.reset_packet_infos()
|
||||
cls._pcaps = []
|
||||
@ -636,7 +585,7 @@ class VppTestCase(CPUInterface, unittest.TestCase):
|
||||
cls.run_vpp()
|
||||
cls.reporter.send_keep_alive(cls, 'setUpClass')
|
||||
VppTestResult.current_test_case_info = TestCaseInfo(
|
||||
cls.logger, cls.tempdir, cls.vpp.pid, cls.vpp_bin)
|
||||
cls.logger, cls.tempdir, cls.vpp.pid, config.vpp)
|
||||
cls.vpp_stdout_deque = deque()
|
||||
cls.vpp_stderr_deque = deque()
|
||||
if not cls.debug_attach:
|
||||
@ -785,7 +734,7 @@ class VppTestCase(CPUInterface, unittest.TestCase):
|
||||
cls.quit()
|
||||
cls.file_handler.close()
|
||||
cls.reset_packet_infos()
|
||||
if debug_framework:
|
||||
if config.debug_framework:
|
||||
debug_internal.on_tear_down_class(cls)
|
||||
|
||||
def show_commands_at_teardown(self):
|
||||
@ -890,7 +839,7 @@ class VppTestCase(CPUInterface, unittest.TestCase):
|
||||
def pg_start(cls, trace=True):
|
||||
""" Enable the PG, wait till it is done, then clean up """
|
||||
for (intf, worker) in cls._old_pcaps:
|
||||
intf.rename_old_pcap_file(intf.get_in_path(worker),
|
||||
intf.handle_old_pcap_file(intf.get_in_path(worker),
|
||||
intf.in_history_counter)
|
||||
cls._old_pcaps = []
|
||||
if trace:
|
||||
@ -1439,7 +1388,7 @@ class VppTestResult(unittest.TestResult):
|
||||
def symlink_failed(self):
|
||||
if self.current_test_case_info:
|
||||
try:
|
||||
failed_dir = os.getenv('FAILED_DIR')
|
||||
failed_dir = config.failed_dir
|
||||
link_path = os.path.join(
|
||||
failed_dir,
|
||||
'%s-FAILED' %
|
||||
|
||||
Reference in New Issue
Block a user