e416893a59
Tests gso/gro-coalesce features on tapv2, tunv2 and af_packet interfaces to ensure that packet transmission is enabled correctly for various MTU sizes and interface combinations in bridged and routed topologies for IPv4 and IPv6. Interface tests are dynamically generated at run time from the config file vm_test_config.py. Type: test Change-Id: I5f9d8cc80d20b4e34011fc8a87e35659bd9613bc Signed-off-by: Naveen Joy <najoy@cisco.com>
489 lines
16 KiB
Python
Executable File
489 lines
16 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
#
|
|
# Copyright (c) 2022 Cisco and/or its affiliates.
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at:
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
# Build the Virtual Environment & run VPP unit tests
|
|
|
|
import argparse
|
|
import glob
|
|
import logging
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
from subprocess import Popen, PIPE, STDOUT, call
|
|
import sys
|
|
import time
|
|
import venv
|
|
import datetime
|
|
import re
|
|
|
|
|
|
# Required Std. Path Variables
|
|
test_dir = os.path.dirname(os.path.realpath(__file__))
|
|
ws_root = os.path.dirname(test_dir)
|
|
build_root = os.path.join(ws_root, "build-root")
|
|
venv_dir = os.path.join(build_root, "test", "venv")
|
|
venv_bin_dir = os.path.join(venv_dir, "bin")
|
|
venv_lib_dir = os.path.join(venv_dir, "lib")
|
|
venv_run_dir = os.path.join(venv_dir, "run")
|
|
venv_install_done = os.path.join(venv_run_dir, "venv_install.done")
|
|
papi_python_src_dir = os.path.join(ws_root, "src", "vpp-api", "python")
|
|
|
|
# Path Variables Set after VPP Build/Install
|
|
vpp_build_dir = vpp_install_path = vpp_bin = vpp_lib = vpp_lib64 = None
|
|
vpp_plugin_path = vpp_test_plugin_path = ld_library_path = None
|
|
|
|
# Pip version pinning
|
|
pip_version = "22.0.4"
|
|
pip_tools_version = "6.6.0"
|
|
|
|
# Test requirement files
|
|
test_requirements_file = os.path.join(test_dir, "requirements.txt")
|
|
# Auto-generated requirement file
|
|
pip_compiled_requirements_file = os.path.join(test_dir, "requirements-3.txt")
|
|
|
|
|
|
# Gracefully exit after executing cleanup scripts
|
|
# upon receiving a SIGINT or SIGTERM
|
|
def handler(signum, frame):
|
|
print("Received Signal {0}".format(signum))
|
|
post_vm_test_run()
|
|
|
|
|
|
signal.signal(signal.SIGINT, handler)
|
|
signal.signal(signal.SIGTERM, handler)
|
|
|
|
|
|
def show_progress(stream, exclude_pattern=None):
|
|
"""
|
|
Read lines from a subprocess stdout/stderr streams and write
|
|
to sys.stdout & the logfile
|
|
|
|
arguments:
|
|
stream - subprocess stdout or stderr data stream
|
|
exclude_pattern - lines matching this reg-ex will be excluded
|
|
from stdout.
|
|
"""
|
|
while True:
|
|
s = stream.readline()
|
|
if not s:
|
|
break
|
|
data = s.decode("utf-8")
|
|
# Filter the annoying SIGTERM signal from the output when VPP is
|
|
# terminated after a test run
|
|
if "SIGTERM" not in data:
|
|
if exclude_pattern is not None:
|
|
if bool(re.search(exclude_pattern, data)) is False:
|
|
sys.stdout.write(data)
|
|
else:
|
|
sys.stdout.write(data)
|
|
logging.debug(data)
|
|
sys.stdout.flush()
|
|
stream.close()
|
|
|
|
|
|
class ExtendedEnvBuilder(venv.EnvBuilder):
|
|
"""
|
|
1. Builds a Virtual Environment for running VPP unit tests
|
|
2. Installs all necessary scripts, pkgs & patches into the vEnv
|
|
- python3, pip, pip-tools, papi, scapy patches &
|
|
test-requirement pkgs
|
|
"""
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
|
|
def post_setup(self, context):
|
|
"""
|
|
Setup all packages that need to be pre-installed into the venv
|
|
prior to running VPP unit tests.
|
|
|
|
:param context: The context of the virtual environment creation
|
|
request being processed.
|
|
"""
|
|
os.environ["VIRTUAL_ENV"] = context.env_dir
|
|
os.environ[
|
|
"CUSTOM_COMPILE_COMMAND"
|
|
] = "make test-refresh-deps (or update requirements.txt)"
|
|
# Cleanup previously auto-generated pip req. file
|
|
try:
|
|
os.unlink(pip_compiled_requirements_file)
|
|
except OSError:
|
|
pass
|
|
# Set the venv python executable & binary install path
|
|
env_exe = context.env_exe
|
|
bin_path = context.bin_path
|
|
# Packages/requirements to be installed in the venv
|
|
# [python-module, cmdline-args, package-name_or_requirements-file-name]
|
|
test_req = [
|
|
["pip", "install", "pip===%s" % pip_version],
|
|
["pip", "install", "pip-tools===%s" % pip_tools_version],
|
|
[
|
|
"piptools",
|
|
"compile",
|
|
"-q",
|
|
"--generate-hashes",
|
|
test_requirements_file,
|
|
"--output-file",
|
|
pip_compiled_requirements_file,
|
|
],
|
|
["piptools", "sync", pip_compiled_requirements_file],
|
|
["pip", "install", "-e", papi_python_src_dir],
|
|
]
|
|
for req in test_req:
|
|
args = [env_exe, "-m"]
|
|
args.extend(req)
|
|
print(args)
|
|
p = Popen(args, stdout=PIPE, stderr=STDOUT, cwd=bin_path)
|
|
show_progress(p.stdout)
|
|
self.pip_patch()
|
|
|
|
def pip_patch(self):
|
|
"""
|
|
Apply scapy patch files
|
|
"""
|
|
scapy_patch_dir = Path(os.path.join(test_dir, "patches", "scapy-2.4.3"))
|
|
scapy_source_dir = glob.glob(
|
|
os.path.join(venv_lib_dir, "python3.*", "site-packages")
|
|
)[0]
|
|
for f in scapy_patch_dir.iterdir():
|
|
print("Applying patch: {}".format(os.path.basename(str(f))))
|
|
args = ["patch", "--forward", "-p1", "-d", scapy_source_dir, "-i", str(f)]
|
|
print(args)
|
|
p = Popen(args, stdout=PIPE, stderr=STDOUT)
|
|
show_progress(p.stdout)
|
|
|
|
|
|
# Build VPP Release/Debug binaries
|
|
def build_vpp(debug=True, release=False):
|
|
"""
|
|
Install VPP Release(if release=True) or Debug(if debug=True) Binaries.
|
|
|
|
Default is to build the debug binaries.
|
|
"""
|
|
global vpp_build_dir, vpp_install_path, vpp_bin, vpp_lib, vpp_lib64
|
|
global vpp_plugin_path, vpp_test_plugin_path, ld_library_path
|
|
if debug:
|
|
print("Building VPP debug binaries")
|
|
args = ["make", "build"]
|
|
build = "build-vpp_debug-native"
|
|
install = "install-vpp_debug-native"
|
|
elif release:
|
|
print("Building VPP release binaries")
|
|
args = ["make", "build-release"]
|
|
build = "build-vpp-native"
|
|
install = "install-vpp-native"
|
|
p = Popen(args, stdout=PIPE, stderr=STDOUT, cwd=ws_root)
|
|
show_progress(p.stdout)
|
|
vpp_build_dir = os.path.join(build_root, build)
|
|
vpp_install_path = os.path.join(build_root, install)
|
|
vpp_bin = os.path.join(vpp_install_path, "vpp", "bin", "vpp")
|
|
vpp_lib = os.path.join(vpp_install_path, "vpp", "lib")
|
|
vpp_lib64 = os.path.join(vpp_install_path, "vpp", "lib64")
|
|
vpp_plugin_path = (
|
|
os.path.join(vpp_lib, "vpp_plugins")
|
|
+ ":"
|
|
+ os.path.join(vpp_lib64, "vpp_plugins")
|
|
)
|
|
vpp_test_plugin_path = (
|
|
os.path.join(vpp_lib, "vpp_api_test_plugins")
|
|
+ ":"
|
|
+ os.path.join(vpp_lib64, "vpp_api_test_plugins")
|
|
)
|
|
ld_library_path = os.path.join(vpp_lib) + ":" + os.path.join(vpp_lib64)
|
|
|
|
|
|
# Environment Vars required by the test framework,
|
|
# papi_provider & unittests
|
|
def set_environ():
|
|
os.environ["WS_ROOT"] = ws_root
|
|
os.environ["BR"] = build_root
|
|
os.environ["VENV_PATH"] = venv_dir
|
|
os.environ["VENV_BIN"] = venv_bin_dir
|
|
os.environ["RND_SEED"] = str(time.time())
|
|
os.environ["VPP_BUILD_DIR"] = vpp_build_dir
|
|
os.environ["VPP_BIN"] = vpp_bin
|
|
os.environ["VPP_PLUGIN_PATH"] = vpp_plugin_path
|
|
os.environ["VPP_TEST_PLUGIN_PATH"] = vpp_test_plugin_path
|
|
os.environ["VPP_INSTALL_PATH"] = vpp_install_path
|
|
os.environ["LD_LIBRARY_PATH"] = ld_library_path
|
|
os.environ["FAILED_DIR"] = "/tmp/vpp-failed-unittests/"
|
|
if not os.environ.get("TEST_JOBS"):
|
|
os.environ["TEST_JOBS"] = "1"
|
|
|
|
|
|
# Runs a test inside a spawned QEMU VM
|
|
# If a kernel image is not provided, a linux-image-kvm image is
|
|
# downloaded to the test_data_dir
|
|
def vm_test_runner(test_name, kernel_image, test_data_dir, cpu_mask, mem, jobs="auto"):
|
|
script = os.path.join(test_dir, "scripts", "run_vpp_in_vm.sh")
|
|
os.environ["TEST_JOBS"] = str(jobs)
|
|
p = Popen(
|
|
[script, test_name, kernel_image, test_data_dir, cpu_mask, mem],
|
|
stdout=PIPE,
|
|
cwd=ws_root,
|
|
)
|
|
# Show only the test result without clobbering the stdout.
|
|
# The VM console displays VPP stderr & Linux IPv6 netdev change
|
|
# messages, which is logged by default and can be excluded.
|
|
exclude_pattern = r"vpp\[\d+\]:|ADDRCONF\(NETDEV_CHANGE\):"
|
|
show_progress(p.stdout, exclude_pattern)
|
|
post_vm_test_run()
|
|
|
|
|
|
def post_vm_test_run():
|
|
# Revert the ownership of certain directories from root to the
|
|
# original user after running in QEMU
|
|
print("Running post test cleanup tasks")
|
|
dirs = ["/tmp/vpp-failed-unittests", os.path.join(ws_root, "test", "__pycache__")]
|
|
dirs.extend(glob.glob("/tmp/vpp-unittest-*"))
|
|
dirs.extend(glob.glob("/tmp/api_post_mortem.*"))
|
|
user = os.getlogin()
|
|
for dir in dirs:
|
|
if os.path.exists(dir) and Path(dir).owner() != user:
|
|
cmd = ["sudo", "chown", "-R", "{0}:{0}".format(user), dir]
|
|
p = Popen(cmd, stdout=PIPE, stderr=STDOUT)
|
|
show_progress(p.stdout)
|
|
|
|
|
|
def build_venv():
|
|
# Builds a virtual env containing all the required packages and patches
|
|
# for running VPP unit tests
|
|
if not os.path.exists(venv_install_done):
|
|
env_builder = ExtendedEnvBuilder(clear=True, with_pip=True)
|
|
print("Creating a vEnv for running VPP unit tests in {}".format(venv_dir))
|
|
env_builder.create(venv_dir)
|
|
# Write state to the venv run dir
|
|
Path(venv_run_dir).mkdir(exist_ok=True)
|
|
Path(venv_install_done).touch()
|
|
|
|
|
|
def expand_mix_string(s):
|
|
# Returns an expanded string computed from a mixrange string (s)
|
|
# E.g: If param s = '5-8,10,11' returns '5,6,7,8,10,11'
|
|
result = []
|
|
for val in s.split(","):
|
|
if "-" in val:
|
|
start, end = val.split("-")
|
|
result.extend(list(range(int(start), int(end) + 1)))
|
|
else:
|
|
result.append(int(val))
|
|
return ",".join(str(i) for i in set(result))
|
|
|
|
|
|
def set_logging(test_data_dir, test_name):
|
|
Path(test_data_dir).mkdir(exist_ok=True)
|
|
log_file = "vm_{0}_{1}.log".format(test_name, str(time.time())[-5:])
|
|
filename = "{0}/{1}".format(test_data_dir, log_file)
|
|
Path(filename).touch()
|
|
logging.basicConfig(filename=filename, level=logging.DEBUG)
|
|
|
|
|
|
def run_tests_in_venv(
|
|
test,
|
|
jobs,
|
|
log_dir,
|
|
socket_dir="",
|
|
running_vpp=False,
|
|
extended=False,
|
|
):
|
|
"""Runs tests in the virtual environment set by venv_dir.
|
|
|
|
Arguments:
|
|
test: Name of the test to run
|
|
jobs: Maximum concurrent test jobs
|
|
log_dir: Directory location for storing log files
|
|
socket_dir: Use running VPP's socket files
|
|
running_vpp: True if tests are run against a running VPP
|
|
extended: Run extended tests
|
|
"""
|
|
script = os.path.join(test_dir, "scripts", "run.sh")
|
|
args = [
|
|
f"--venv-dir={venv_dir}",
|
|
f"--vpp-ws-dir={ws_root}",
|
|
f"--socket-dir={socket_dir}",
|
|
f"--filter={test}",
|
|
f"--jobs={jobs}",
|
|
f"--log-dir={log_dir}",
|
|
f"--tmp-dir={log_dir}",
|
|
f"--cache-vpp-output",
|
|
]
|
|
if running_vpp:
|
|
args = args + [f"--use-running-vpp"]
|
|
if extended:
|
|
args = args + [f"--extended"]
|
|
print(f"Running script: {script} " f"{' '.join(args)}")
|
|
process_args = [script] + args
|
|
call(process_args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# Build a Virtual Environment for running tests on host & QEMU
|
|
# (TODO): Create a single config object by merging the below args with
|
|
# config.py after gathering dev use-cases.
|
|
parser = argparse.ArgumentParser(
|
|
description="Run VPP Unit Tests", formatter_class=argparse.RawTextHelpFormatter
|
|
)
|
|
parser.add_argument(
|
|
"--vm",
|
|
dest="vm",
|
|
required=False,
|
|
action="store_true",
|
|
help="Run Test Inside a QEMU VM",
|
|
)
|
|
parser.add_argument(
|
|
"--debug",
|
|
dest="debug",
|
|
required=False,
|
|
default=True,
|
|
action="store_true",
|
|
help="Run Tests on Debug Build",
|
|
)
|
|
parser.add_argument(
|
|
"--release",
|
|
dest="release",
|
|
required=False,
|
|
default=False,
|
|
action="store_true",
|
|
help="Run Tests on release Build",
|
|
)
|
|
parser.add_argument(
|
|
"-t",
|
|
"--test",
|
|
dest="test_name",
|
|
required=False,
|
|
action="store",
|
|
default="",
|
|
help="Test Name or Test filter",
|
|
)
|
|
parser.add_argument(
|
|
"--vm-kernel-image",
|
|
dest="kernel_image",
|
|
required=False,
|
|
action="store",
|
|
default="",
|
|
help="Kernel Image Selection to Boot",
|
|
)
|
|
parser.add_argument(
|
|
"--vm-cpu-list",
|
|
dest="vm_cpu_list",
|
|
required=False,
|
|
action="store",
|
|
default="5-8",
|
|
help="Set CPU Affinity\n"
|
|
"E.g. 5-7,10 will schedule on processors "
|
|
"#5, #6, #7 and #10. (Default: 5-8)",
|
|
)
|
|
parser.add_argument(
|
|
"--vm-mem",
|
|
dest="vm_mem",
|
|
required=False,
|
|
action="store",
|
|
default="2",
|
|
help="Guest Memory in Gibibytes\n" "E.g. 4 (Default: 2)",
|
|
)
|
|
parser.add_argument(
|
|
"--log-dir",
|
|
action="store",
|
|
default=os.path.abspath(f"./test-run-{datetime.date.today()}"),
|
|
help="directory where to store directories "
|
|
"containing log files (default: ./test-run-YYYY-MM-DD)",
|
|
)
|
|
parser.add_argument(
|
|
"--jobs",
|
|
action="store",
|
|
default="auto",
|
|
help="maximum concurrent test jobs",
|
|
)
|
|
parser.add_argument(
|
|
"-r",
|
|
"--use-running-vpp",
|
|
dest="running_vpp",
|
|
required=False,
|
|
action="store_true",
|
|
default=False,
|
|
help="Runs tests against a running VPP.",
|
|
)
|
|
parser.add_argument(
|
|
"-d",
|
|
"--socket-dir",
|
|
dest="socket_dir",
|
|
required=False,
|
|
action="store",
|
|
default="",
|
|
help="Relative or absolute path of running VPP's socket directory "
|
|
"containing api.sock & stats.sock files.\n"
|
|
"Default: /var/run/vpp if VPP is started as the root user, else "
|
|
"/var/run/user/${uid}/vpp.",
|
|
)
|
|
parser.add_argument(
|
|
"-e",
|
|
"--extended",
|
|
dest="extended",
|
|
required=False,
|
|
action="store_true",
|
|
default=False,
|
|
help="Run extended tests.",
|
|
)
|
|
args = parser.parse_args()
|
|
vm_tests = False
|
|
# Enable VM tests
|
|
if args.vm and args.test_name:
|
|
test_data_dir = "/tmp/vpp-vm-tests"
|
|
set_logging(test_data_dir, args.test_name)
|
|
vm_tests = True
|
|
elif args.vm and not args.test_name:
|
|
print("Error: The --test argument must be set for running VM tests")
|
|
sys.exit(1)
|
|
build_venv()
|
|
# Build VPP release or debug binaries
|
|
debug = False if args.release else True
|
|
build_vpp(debug, args.release)
|
|
set_environ()
|
|
if args.running_vpp:
|
|
print("Tests will be run against a running VPP..")
|
|
elif not vm_tests:
|
|
print("Tests will be run by spawning a new VPP instance..")
|
|
# Run tests against a running VPP or a new instance of VPP
|
|
if not vm_tests:
|
|
run_tests_in_venv(
|
|
test=args.test_name,
|
|
jobs=args.jobs,
|
|
log_dir=args.log_dir,
|
|
socket_dir=args.socket_dir,
|
|
running_vpp=args.running_vpp,
|
|
extended=args.extended,
|
|
)
|
|
# Run tests against a VPP inside a VM
|
|
else:
|
|
print("Running VPP unit test(s):{0} inside a QEMU VM".format(args.test_name))
|
|
# Check Available CPUs & Usable Memory
|
|
cpus = expand_mix_string(args.vm_cpu_list)
|
|
num_cpus, usable_cpus = (len(cpus.split(",")), len(os.sched_getaffinity(0)))
|
|
if num_cpus > usable_cpus:
|
|
print(f"Error:# of CPUs:{num_cpus} > Avail CPUs:{usable_cpus}")
|
|
sys.exit(1)
|
|
avail_mem = int(os.popen("free -t -g").readlines()[-1].split()[-1])
|
|
if int(args.vm_mem) > avail_mem:
|
|
print(f"Error: Mem Size:{args.vm_mem}G > Avail Mem:{avail_mem}G")
|
|
sys.exit(1)
|
|
vm_test_runner(
|
|
args.test_name,
|
|
args.kernel_image,
|
|
test_data_dir,
|
|
cpus,
|
|
f"{args.vm_mem}G",
|
|
args.jobs,
|
|
)
|