Improve test framework documentation
Change-Id: I06f0cbbbdd29e04a07f1db6807b3e16f1d41e8d2 Signed-off-by: Klement Sekera <ksekera@cisco.com>
This commit is contained in:
Klement Sekera
committed by
Damjan Marion
Damjan Marion
parent
d72fdf5712
commit
778c2765c8
@@ -33,7 +33,7 @@ endif
|
||||
DEB_DEPENDS = curl build-essential autoconf automake bison libssl-dev ccache
|
||||
DEB_DEPENDS += debhelper dkms git libtool libganglia1-dev libapr1-dev dh-systemd
|
||||
DEB_DEPENDS += libconfuse-dev git-review exuberant-ctags cscope
|
||||
DEB_DEPENDS += python-dev python-virtualenv
|
||||
DEB_DEPENDS += python-dev python-virtualenv python-pip
|
||||
ifeq ($(OS_VERSION_ID),14.04)
|
||||
DEB_DEPENDS += openjdk-8-jdk-headless
|
||||
else
|
||||
|
||||
+4
-4
@@ -6,11 +6,11 @@ ifndef VPP_PYTHON_PREFIX
|
||||
endif
|
||||
|
||||
PYTHON_VENV_PATH=$(VPP_PYTHON_PREFIX)/virtualenv
|
||||
PYTHON_DEPENDS=scapy pexpect
|
||||
|
||||
test: wipe verify-python-path
|
||||
@virtualenv $(PYTHON_VENV_PATH)
|
||||
@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && pip install scapy"
|
||||
@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && pip install pexpect"
|
||||
@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && pip install $(PYTHON_DEPENDS)"
|
||||
@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && cd $(WS_ROOT)/vpp-api/python && python setup.py install"
|
||||
@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && python run_tests.py discover -p test_$(TEST)\"*.py\""
|
||||
|
||||
@@ -19,13 +19,13 @@ retest: wipe verify-python-path
|
||||
|
||||
.PHONY: wipe doc
|
||||
|
||||
wipe: verify-python-path
|
||||
wipe:
|
||||
@rm -f /dev/shm/vpp-unittest-*
|
||||
@rm -rf /tmp/vpp-unittest-*
|
||||
|
||||
doc: verify-python-path
|
||||
@virtualenv $(PYTHON_VENV_PATH)
|
||||
@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && pip install sphinx"
|
||||
@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && pip install $(PYTHON_DEPENDS) sphinx"
|
||||
@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && make -C doc WS_ROOT=$(WS_ROOT) BR=$(BR) NO_VPP_PAPI=1 html"
|
||||
|
||||
wipe-doc:
|
||||
|
||||
+26
-35
@@ -8,15 +8,13 @@ SPHINXBUILD = sphinx-build
|
||||
PAPER =
|
||||
BUILD_DOC_ROOT = $(BR)/test-doc
|
||||
BUILD_DOC_DIR = $(BUILD_DOC_ROOT)/build
|
||||
API_DOC_GEN_DIR = $(BUILD_DOC_ROOT)/apidoc
|
||||
|
||||
# Internal variables.
|
||||
PAPEROPT_a4 = -D latex_paper_size=a4
|
||||
PAPEROPT_letter = -D latex_paper_size=letter
|
||||
ALLSPHINXOPTS = -d $(BUILD_DOC_DIR)/.sphinx-cache $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) $(API_DOC_GEN_DIR) -c $(SRC_DOC_DIR)
|
||||
ALLSPHINXOPTS = -d $(BUILD_DOC_DIR)/.sphinx-cache $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) $(SRC_DOC_DIR)
|
||||
# the i18n builder cannot share the environment and doctrees with the others
|
||||
I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
|
||||
INDEX_REL_PATH:=$(shell realpath --relative-to=$(API_DOC_GEN_DIR) $(SRC_DOC_DIR)/index.rst)
|
||||
IN_VENV:=$(shell if pip -V | grep "virtualenv" 2>&1 > /dev/null; then echo 1; else echo 0; fi)
|
||||
|
||||
.PHONY: verify-virtualenv
|
||||
@@ -25,13 +23,6 @@ ifeq ($(IN_VENV),0)
|
||||
$(error "Not running inside virtualenv (are you running 'make test-doc' from root?)")
|
||||
endif
|
||||
|
||||
.PHONY: regen-api-doc
|
||||
regen-api-doc: verify-virtualenv
|
||||
@mkdir -p $(API_DOC_GEN_DIR)
|
||||
#@echo ".. include:: $(INDEX_REL_PATH)" > $(API_DOC_GEN_DIR)/index.rst
|
||||
@cp $(SRC_DOC_DIR)/index.rst $(API_DOC_GEN_DIR)
|
||||
sphinx-apidoc -o $(API_DOC_GEN_DIR) ..
|
||||
|
||||
.PHONY: help
|
||||
help:
|
||||
@echo "Please use \`make <target>' where <target> is one of"
|
||||
@@ -67,44 +58,44 @@ wipe:
|
||||
rm -rf $(BUILD_DOC_ROOT)
|
||||
|
||||
.PHONY: html
|
||||
html: regen-api-doc verify-virtualenv
|
||||
html: verify-virtualenv
|
||||
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILD_DOC_DIR)/html
|
||||
@echo
|
||||
@echo "Build finished. The HTML pages are in $(BUILD_DOC_DIR)/html."
|
||||
|
||||
.PHONY: dirhtml
|
||||
dirhtml: regen-api-doc verify-virtualenv
|
||||
dirhtml: verify-virtualenv
|
||||
$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILD_DOC_DIR)/dirhtml
|
||||
@echo
|
||||
@echo "Build finished. The HTML pages are in $(BUILD_DOC_DIR)/dirhtml."
|
||||
|
||||
.PHONY: singlehtml
|
||||
singlehtml: regen-api-doc verify-virtualenv
|
||||
singlehtml: verify-virtualenv
|
||||
$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILD_DOC_DIR)/singlehtml
|
||||
@echo
|
||||
@echo "Build finished. The HTML page is in $(BUILD_DOC_DIR)/singlehtml."
|
||||
|
||||
.PHONY: pickle
|
||||
pickle: regen-api-doc verify-virtualenv
|
||||
pickle: verify-virtualenv
|
||||
$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILD_DOC_DIR)/pickle
|
||||
@echo
|
||||
@echo "Build finished; now you can process the pickle files."
|
||||
|
||||
.PHONY: json
|
||||
json: regen-api-doc verify-virtualenv
|
||||
json: verify-virtualenv
|
||||
$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILD_DOC_DIR)/json
|
||||
@echo
|
||||
@echo "Build finished; now you can process the JSON files."
|
||||
|
||||
.PHONY: htmlhelp
|
||||
htmlhelp: regen-api-doc verify-virtualenv
|
||||
htmlhelp: verify-virtualenv
|
||||
$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILD_DOC_DIR)/htmlhelp
|
||||
@echo
|
||||
@echo "Build finished; now you can run HTML Help Workshop with the" \
|
||||
".hhp project file in $(BUILD_DOC_DIR)/htmlhelp."
|
||||
|
||||
.PHONY: qthelp
|
||||
qthelp: regen-api-doc verify-virtualenv
|
||||
qthelp: verify-virtualenv
|
||||
$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILD_DOC_DIR)/qthelp
|
||||
@echo
|
||||
@echo "Build finished; now you can run "qcollectiongenerator" with the" \
|
||||
@@ -114,7 +105,7 @@ qthelp: regen-api-doc verify-virtualenv
|
||||
@echo "# assistant -collectionFile $(BUILD_DOC_DIR)/qthelp/VPPtestframework.qhc"
|
||||
|
||||
.PHONY: applehelp
|
||||
applehelp: regen-api-doc verify-virtualenv
|
||||
applehelp: verify-virtualenv
|
||||
$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILD_DOC_DIR)/applehelp
|
||||
@echo
|
||||
@echo "Build finished. The help book is in $(BUILD_DOC_DIR)/applehelp."
|
||||
@@ -123,7 +114,7 @@ applehelp: regen-api-doc verify-virtualenv
|
||||
"bundle."
|
||||
|
||||
.PHONY: devhelp
|
||||
devhelp: regen-api-doc verify-virtualenv
|
||||
devhelp: verify-virtualenv
|
||||
$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILD_DOC_DIR)/devhelp
|
||||
@echo
|
||||
@echo "Build finished."
|
||||
@@ -133,7 +124,7 @@ devhelp: regen-api-doc verify-virtualenv
|
||||
@echo "# devhelp"
|
||||
|
||||
.PHONY: epub
|
||||
epub: regen-api-doc verify-virtualenv
|
||||
epub: verify-virtualenv
|
||||
$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILD_DOC_DIR)/epub
|
||||
@echo
|
||||
@echo "Build finished. The epub file is in $(BUILD_DOC_DIR)/epub."
|
||||
@@ -145,7 +136,7 @@ epub3:
|
||||
@echo "Build finished. The epub3 file is in $(BUILD_DOC_DIR)/epub3."
|
||||
|
||||
.PHONY: latex
|
||||
latex: regen-api-doc verify-virtualenv
|
||||
latex: verify-virtualenv
|
||||
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILD_DOC_DIR)/latex
|
||||
@echo
|
||||
@echo "Build finished; the LaTeX files are in $(BUILD_DOC_DIR)/latex."
|
||||
@@ -153,33 +144,33 @@ latex: regen-api-doc verify-virtualenv
|
||||
"(use \`make latexpdf' here to do that automatically)."
|
||||
|
||||
.PHONY: latexpdf
|
||||
latexpdf: regen-api-doc verify-virtualenv
|
||||
latexpdf: verify-virtualenv
|
||||
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILD_DOC_DIR)/latex
|
||||
@echo "Running LaTeX files through pdflatex..."
|
||||
$(MAKE) -C $(BUILD_DOC_DIR)/latex all-pdf
|
||||
@echo "pdflatex finished; the PDF files are in $(BUILD_DOC_DIR)/latex."
|
||||
|
||||
.PHONY: latexpdfja
|
||||
latexpdfja: regen-api-doc verify-virtualenv
|
||||
latexpdfja: verify-virtualenv
|
||||
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILD_DOC_DIR)/latex
|
||||
@echo "Running LaTeX files through platex and dvipdfmx..."
|
||||
$(MAKE) -C $(BUILD_DOC_DIR)/latex all-pdf-ja
|
||||
@echo "pdflatex finished; the PDF files are in $(BUILD_DOC_DIR)/latex."
|
||||
|
||||
.PHONY: text
|
||||
text: regen-api-doc verify-virtualenv
|
||||
text: verify-virtualenv
|
||||
$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILD_DOC_DIR)/text
|
||||
@echo
|
||||
@echo "Build finished. The text files are in $(BUILD_DOC_DIR)/text."
|
||||
|
||||
.PHONY: man
|
||||
man: regen-api-doc verify-virtualenv
|
||||
man: verify-virtualenv
|
||||
$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILD_DOC_DIR)/man
|
||||
@echo
|
||||
@echo "Build finished. The manual pages are in $(BUILD_DOC_DIR)/man."
|
||||
|
||||
.PHONY: texinfo
|
||||
texinfo: regen-api-doc verify-virtualenv
|
||||
texinfo: verify-virtualenv
|
||||
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILD_DOC_DIR)/texinfo
|
||||
@echo
|
||||
@echo "Build finished. The Texinfo files are in $(BUILD_DOC_DIR)/texinfo."
|
||||
@@ -187,57 +178,57 @@ texinfo: regen-api-doc verify-virtualenv
|
||||
"(use \`make info' here to do that automatically)."
|
||||
|
||||
.PHONY: info
|
||||
info: regen-api-doc verify-virtualenv
|
||||
info: verify-virtualenv
|
||||
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILD_DOC_DIR)/texinfo
|
||||
@echo "Running Texinfo files through makeinfo..."
|
||||
make -C $(BUILD_DOC_DIR)/texinfo info
|
||||
@echo "makeinfo finished; the Info files are in $(BUILD_DOC_DIR)/texinfo."
|
||||
|
||||
.PHONY: gettext
|
||||
gettext: regen-api-doc verify-virtualenv
|
||||
gettext: verify-virtualenv
|
||||
$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILD_DOC_DIR)/locale
|
||||
@echo
|
||||
@echo "Build finished. The message catalogs are in $(BUILD_DOC_DIR)/locale."
|
||||
|
||||
.PHONY: changes
|
||||
changes: regen-api-doc verify-virtualenv
|
||||
changes: verify-virtualenv
|
||||
$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILD_DOC_DIR)/changes
|
||||
@echo
|
||||
@echo "The overview file is in $(BUILD_DOC_DIR)/changes."
|
||||
|
||||
.PHONY: linkcheck
|
||||
linkcheck: regen-api-doc verify-virtualenv
|
||||
linkcheck: verify-virtualenv
|
||||
$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILD_DOC_DIR)/linkcheck
|
||||
@echo
|
||||
@echo "Link check complete; look for any errors in the above output " \
|
||||
"or in $(BUILD_DOC_DIR)/linkcheck/output.txt."
|
||||
|
||||
.PHONY: doctest
|
||||
doctest: regen-api-doc verify-virtualenv
|
||||
doctest: verify-virtualenv
|
||||
$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILD_DOC_DIR)/doctest
|
||||
@echo "Testing of doctests in the sources finished, look at the " \
|
||||
"results in $(BUILD_DOC_DIR)/doctest/output.txt."
|
||||
|
||||
.PHONY: coverage
|
||||
coverage: regen-api-doc verify-virtualenv
|
||||
coverage: verify-virtualenv
|
||||
$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILD_DOC_DIR)/coverage
|
||||
@echo "Testing of coverage in the sources finished, look at the " \
|
||||
"results in $(BUILD_DOC_DIR)/coverage/python.txt."
|
||||
|
||||
.PHONY: xml
|
||||
xml: regen-api-doc verify-virtualenv
|
||||
xml: verify-virtualenv
|
||||
$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILD_DOC_DIR)/xml
|
||||
@echo
|
||||
@echo "Build finished. The XML files are in $(BUILD_DOC_DIR)/xml."
|
||||
|
||||
.PHONY: pseudoxml
|
||||
pseudoxml: regen-api-doc verify-virtualenv
|
||||
pseudoxml: verify-virtualenv
|
||||
$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILD_DOC_DIR)/pseudoxml
|
||||
@echo
|
||||
@echo "Build finished. The pseudo-XML files are in $(BUILD_DOC_DIR)/pseudoxml."
|
||||
|
||||
.PHONY: dummy
|
||||
dummy: regen-api-doc verify-virtualenv
|
||||
dummy: verify-virtualenv
|
||||
$(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILD_DOC_DIR)/dummy
|
||||
@echo
|
||||
@echo "Build finished. Dummy builder generates no files."
|
||||
|
||||
+3
-3
@@ -88,11 +88,11 @@ exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
|
||||
# The reST default role (used for this markup: `text`) to use for all
|
||||
# documents.
|
||||
#
|
||||
# default_role = None
|
||||
default_role = 'any'
|
||||
|
||||
# If true, '()' will be appended to :func: etc. cross-reference text.
|
||||
#
|
||||
# add_function_parentheses = True
|
||||
add_function_parentheses = True
|
||||
|
||||
# If true, the current module name will be prepended to all description
|
||||
# unit titles (such as .. function::).
|
||||
@@ -156,7 +156,7 @@ html_theme = 'alabaster'
|
||||
# Add any paths that contain custom static files (such as style sheets) here,
|
||||
# relative to this directory. They are copied after the builtin static files,
|
||||
# so a file named "default.css" will overwrite the builtin "default.css".
|
||||
html_static_path = ['_static']
|
||||
# html_static_path = []
|
||||
|
||||
# Add any extra paths that contain custom files (such as robots.txt or
|
||||
# .htaccess) here, relative to this directory. These files are copied
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
framework module
|
||||
================
|
||||
|
||||
.. automodule:: framework
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
@@ -0,0 +1,7 @@
|
||||
hook module
|
||||
===========
|
||||
|
||||
.. automodule:: hook
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
+150
-7
@@ -1,15 +1,158 @@
|
||||
.. VPP test framework documentation master file, created by
|
||||
sphinx-quickstart on Thu Oct 13 08:45:03 2016.
|
||||
You can adapt this file completely to your liking, but it should at least
|
||||
contain the root `toctree` directive.
|
||||
.. _unittest: https://docs.python.org/2/library/unittest.html
|
||||
.. _TestCase: https://docs.python.org/2/library/unittest.html#unittest.TestCase
|
||||
.. _AssertionError: https://docs.python.org/2/library/exceptions.html#exceptions.AssertionError
|
||||
.. _SkipTest: https://docs.python.org/2/library/unittest.html#unittest.SkipTest
|
||||
.. _virtualenv: http://docs.python-guide.org/en/latest/dev/virtualenvs/
|
||||
.. _scapy: http://www.secdev.org/projects/scapy/
|
||||
|
||||
Welcome to VPP test framework's documentation!
|
||||
==============================================
|
||||
.. |vtf| replace:: VPP Test Framework
|
||||
|
||||
Contents:
|
||||
|vtf|
|
||||
=====
|
||||
|
||||
Overview
|
||||
########
|
||||
|
||||
The goal of the |vtf| is to ease writing, running and debugging
|
||||
unit tests for the VPP. For this, python was chosen as a high level language
|
||||
allowing rapid development with scapy_ providing the necessary tool for creating
|
||||
and dissecting packets.
|
||||
|
||||
Anatomy of a test case
|
||||
######################
|
||||
|
||||
Python's unittest_ is used as the base framework upon which the VPP test
|
||||
framework is built. A test suite in the |vtf| constists of multiple classes
|
||||
derived from `VppTestCase`, which is itself derived from TestCase_.
|
||||
The test class defines one or more test functions, which act as test cases.
|
||||
|
||||
Function flow when running a test case is:
|
||||
|
||||
1. `setUpClass <VppTestCase.setUpClass>`:
|
||||
This function is called once for each test class, allowing a one-time test
|
||||
setup to be executed. If this functions throws an exception,
|
||||
none of the test functions are executed.
|
||||
2. `setUp <VppTestCase.setUp>`:
|
||||
The setUp function runs before each of the test functions. If this function
|
||||
throws an exception other than AssertionError_ or SkipTest_, then this is
|
||||
considered an error, not a test failure.
|
||||
3. *test_<name>*:
|
||||
This is the guts of the test case. It should execute the test scenario
|
||||
and use the various assert functions from the unittest framework to check
|
||||
necessary.
|
||||
4. `tearDown <VppTestCase.tearDown>`:
|
||||
The tearDown function is called after each test function with the purpose
|
||||
of doing partial cleanup.
|
||||
5. `tearDownClass <VppTestCase.tearDownClass>`:
|
||||
Method called once after running all of the test funnctions to perform
|
||||
the final cleanup.
|
||||
|
||||
Test temporary directory and VPP life cycle
|
||||
################################################
|
||||
|
||||
Test separation is achieved by separating the test files and vpp instances.
|
||||
Each test creates a temporary directory and it's name is used to create
|
||||
a shared memory prefix which is used to run a VPP instance.
|
||||
This way, there is no conflict between any other VPP instances running
|
||||
on the box and the test VPP. Any temporary files created by the test case
|
||||
are stored in this temporary test directory.
|
||||
|
||||
Virtual environment
|
||||
###################
|
||||
|
||||
Virtualenv_ is a python module which provides a means to crate an environment
|
||||
containing the dependencies required by the |vtf|, allowing a separation
|
||||
from any existing system-wide packages. |vtf|'s Makefile automatically
|
||||
creates a virtualenv_ inside build-root and installs the required packages
|
||||
in that environment. The environment is entered whenever executing a test
|
||||
via one of the make test targets.
|
||||
|
||||
Naming conventions
|
||||
##################
|
||||
|
||||
Most unit tests do some kind of packet manipulation - sending and receiving
|
||||
packets between VPP and virtual hosts connected to the VPP. Referring
|
||||
to the sides, addresses, etc.. is always done as if looking from the VPP side,
|
||||
thus:
|
||||
|
||||
* *local_* prefix is used for the VPP side.
|
||||
So e.g. `local_ip4 <VppInterface.local_ip4>` address is the IPv4 address
|
||||
assigned to the VPP interface.
|
||||
* *remote_* prefix is used for the virtual host side.
|
||||
So e.g. `remote_mac <VppInterface.remote_mac>` address is the MAC address
|
||||
assigned to the virtual host connected to the VPP.
|
||||
|
||||
Packet flow in the |vtf|
|
||||
########################
|
||||
|
||||
Test framework -> VPP
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|vtf| doesn't send any packets to VPP directly. Traffic is instead injected
|
||||
using packet-generator interfaces, represented by the `VppPGInterface` class.
|
||||
Packets are written into a temporary .pcap file, which is then read by the VPP
|
||||
and the packets are injected into the VPP world.
|
||||
|
||||
VPP -> test framework
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Similarly, VPP doesn't send any packets to |vtf| directly. Instead, packet
|
||||
capture feature is used to capture and write traffic to a temporary .pcap file,
|
||||
which is then read and analyzed by the |vtf|.
|
||||
|
||||
Test framework objects
|
||||
######################
|
||||
|
||||
The following objects provide VPP abstraction and provide a means to do
|
||||
common tasks easily in the test cases.
|
||||
|
||||
* `VppInterface`: abstract class representing generic VPP interface
|
||||
and contains some common functionality, which is then used by derived classes
|
||||
* `VppPGInterface`: class representing VPP packet-generator interface.
|
||||
The interface is created/destroyed when the object is created/destroyed.
|
||||
* `VppSubInterface`: VPP sub-interface abstract class, containing common
|
||||
functionality for e.g. `VppDot1QSubint` and `VppDot1ADSubint` classes
|
||||
|
||||
How VPP API/CLI is called
|
||||
#########################
|
||||
|
||||
Vpp provides python bindings in a python module called vpp-papi, which the test
|
||||
framework installs in the virtual environment. A shim layer represented by
|
||||
the `VppPapiProvider` class is built on top of the vpp-papi, serving these
|
||||
purposes:
|
||||
|
||||
1. Automatic return value checks:
|
||||
After each API is called, the return value is checked against the expected
|
||||
return value (by default 0, but can be overridden) and an exception
|
||||
is raised if the check fails.
|
||||
2. Automatic call of hooks:
|
||||
|
||||
a. `before_cli <Hook.before_cli>` and `before_api <Hook.before_api>` hooks
|
||||
are used for debug logging and stepping through the test
|
||||
b. `after_cli <Hook.after_cli>` and `after_api <Hook.after_api>` hooks
|
||||
are used for monitoring the vpp process for crashes
|
||||
3. Simplification of API calls:
|
||||
Many of the VPP APIs take a lot of parameters and by providing sane defaults
|
||||
for these, the API is much easier to use in the common case and the code is
|
||||
more readable. E.g. ip_add_del_route API takes ~25 parameters, of which
|
||||
in the common case, only 3 are needed.
|
||||
|
||||
Example: how to add a new test
|
||||
##############################
|
||||
|
||||
In this example, we will describe how to add a new test case which tests VPP...
|
||||
|
||||
1. Add a new file called...
|
||||
2. Add a setUpClass function containing...
|
||||
3. Add the test code in test...
|
||||
4. Run the test...
|
||||
|
||||
|vtf| module documentation
|
||||
##########################
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:glob:
|
||||
|
||||
modules.rst
|
||||
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
log module
|
||||
==========
|
||||
|
||||
.. automodule:: log
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
@@ -0,0 +1,24 @@
|
||||
test
|
||||
====
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 4
|
||||
|
||||
framework
|
||||
hook
|
||||
log
|
||||
run_tests
|
||||
scapy_handlers
|
||||
template_bd
|
||||
test_ip
|
||||
test_ip6
|
||||
test_l2bd
|
||||
test_l2xc
|
||||
test_lb
|
||||
test_mpls
|
||||
test_vxlan
|
||||
util
|
||||
vpp_interface
|
||||
vpp_papi_provider
|
||||
vpp_pg_interface
|
||||
vpp_sub_interface
|
||||
@@ -0,0 +1,7 @@
|
||||
run_tests module
|
||||
================
|
||||
|
||||
.. automodule:: run_tests
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
@@ -0,0 +1,22 @@
|
||||
scapy_handlers package
|
||||
======================
|
||||
|
||||
Submodules
|
||||
----------
|
||||
|
||||
scapy_handlers.vxlan module
|
||||
---------------------------
|
||||
|
||||
.. automodule:: scapy_handlers.vxlan
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
|
||||
Module contents
|
||||
---------------
|
||||
|
||||
.. automodule:: scapy_handlers
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
@@ -0,0 +1,7 @@
|
||||
template_bd module
|
||||
==================
|
||||
|
||||
.. automodule:: template_bd
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
@@ -0,0 +1,7 @@
|
||||
test_ip module
|
||||
==============
|
||||
|
||||
.. automodule:: test_ip
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
@@ -0,0 +1,7 @@
|
||||
test_ip6 module
|
||||
===============
|
||||
|
||||
.. automodule:: test_ip6
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
@@ -0,0 +1,7 @@
|
||||
test_l2bd module
|
||||
================
|
||||
|
||||
.. automodule:: test_l2bd
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
@@ -0,0 +1,7 @@
|
||||
test_l2xc module
|
||||
================
|
||||
|
||||
.. automodule:: test_l2xc
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
@@ -0,0 +1,7 @@
|
||||
test_lb module
|
||||
==============
|
||||
|
||||
.. automodule:: test_lb
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
@@ -0,0 +1,7 @@
|
||||
test_mpls module
|
||||
================
|
||||
|
||||
.. automodule:: test_mpls
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
@@ -0,0 +1,7 @@
|
||||
test_vxlan module
|
||||
=================
|
||||
|
||||
.. automodule:: test_vxlan
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
@@ -0,0 +1,7 @@
|
||||
util module
|
||||
===========
|
||||
|
||||
.. automodule:: util
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
@@ -0,0 +1,7 @@
|
||||
vpp_interface module
|
||||
====================
|
||||
|
||||
.. automodule:: vpp_interface
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
@@ -0,0 +1,7 @@
|
||||
vpp_papi_provider module
|
||||
========================
|
||||
|
||||
.. automodule:: vpp_papi_provider
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
@@ -0,0 +1,7 @@
|
||||
vpp_pg_interface module
|
||||
=======================
|
||||
|
||||
.. automodule:: vpp_pg_interface
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
@@ -0,0 +1,7 @@
|
||||
vpp_sub_interface module
|
||||
========================
|
||||
|
||||
.. automodule:: vpp_sub_interface
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
@@ -1,145 +0,0 @@
|
||||
# VPP Functional Test Infra
|
||||
|
||||
## Running VPP tests
|
||||
VPP functional tests are triggered by `make test` command run in the git vpp source directory. Following Linux environment variables are used by the current VPP functional test infrastructure:
|
||||
|
||||
- `TEST=<name>` - run only specific test identified by filename `test/test_<name>.py`
|
||||
- `V=[0|1|2]` - set verbosity level. `0` for minimal verbosity, `1` for increased verbosity, `2` for maximum verbosity. Default value is 0.
|
||||
|
||||
Example of running tests:
|
||||
|
||||
```
|
||||
~/src/vpp-test-infra$ make test V=1 TEST=vxlan
|
||||
```
|
||||
|
||||
All tests listed in `test/` directory are run by default. To run selected tests you can set variable TEST when starting tests.
|
||||
|
||||
## Overview
|
||||
The main functionality of the test framework is defined in [framework.py](test/framework.py) file. The implementation of the test framework uses classes and methods from Python module *unittest*.
|
||||
|
||||
Three main classes are defined to support the overall test automation:
|
||||
|
||||
* **class VppTestCase(unittest.TestCase)** - a sub-class of *unittest.TestCase* class. Provides methods to create and run test case. These methods can be divided into 5 groups:
|
||||
1. Methods to control test case setup and tear down:
|
||||
* *def setUpConstants(cls):*
|
||||
* *def setUpClass(cls):*
|
||||
* *def quit(cls):*
|
||||
* *def tearDownClass(cls):*
|
||||
* *def tearDown(self):*
|
||||
* *def setUp(self):*
|
||||
|
||||
2. Methods to create VPP packet generator interfaces:
|
||||
* *def create_interfaces(cls, args):*
|
||||
|
||||
3. Methods to execute VPP commands and print logs in the output (terminal for now):
|
||||
* *def log(cls, s, v=1):*
|
||||
* *def api(cls, s):*
|
||||
* *def cli(cls, v, s):*
|
||||
|
||||
4. Methods to control packet stream generation and capturing:
|
||||
* *def pg_add_stream(cls, i, pkts):*
|
||||
* *def pg_enable_capture(cls, args):*
|
||||
* *def pg_start(cls):*
|
||||
* *def pg_get_capture(cls, o):*
|
||||
|
||||
5. Methods to create and verify packets:
|
||||
* *def extend_packet(packet, size):*
|
||||
* *def add_packet_info_to_list(self, info):*
|
||||
* *def create_packet_info(self, pg_id, target_id):*
|
||||
* *def info_to_payload(info):*
|
||||
* *def payload_to_info(payload):*
|
||||
* *def get_next_packet_info(self, info):*
|
||||
* *def get_next_packet_info_for_interface(self, src_pg, info):*
|
||||
* *def get_next_packet_info_for_interface2(self, src_pg, dst_pg, info):*
|
||||
|
||||
* **class VppTestResult(unittest.TestResult)** - a sub-class of *unittest.TestResult* class. Provides methods to compile information about the tests that have succeeded and the ones that have failed. These methods can be divided into 4 groups:
|
||||
1. Processing test case result:
|
||||
* *def addSuccess(self, test):*
|
||||
* *def addFailure(self, test, err):*
|
||||
* *def addError(self, test, err):*
|
||||
|
||||
2. Processing test case description:
|
||||
* *def getDescription(self, test):*
|
||||
|
||||
3. Processing test case start and stop:
|
||||
* *def startTest(self, test):*
|
||||
* *def stopTest(self, test):*
|
||||
|
||||
4. Printing error and failure information:
|
||||
* *def printErrors(self):*
|
||||
* *def printErrorList(self, flavour, errors):*
|
||||
|
||||
* **class VppTestRunner(unittest.TextTestRunner)** - a sub-class of *unittest.TextTestRunner* class. Provides basic test runner implementation that prints results on standard error stream. Contains one method:
|
||||
* *def run(self, test):*
|
||||
|
||||
In addition [util.py] (test/util.py) file defines number of common methods useful for many test cases. All of these methods are currently contained in one class:
|
||||
|
||||
* **class Util(object)**:
|
||||
* *def resolve_arp(cls, args):*
|
||||
* *def resolve_icmpv6_nd(cls, args):*
|
||||
* *def config_ip4(cls, args):*
|
||||
* *def config_ip6(cls, args):*
|
||||
|
||||
## Interaction with VPP
|
||||
VPP is started from command line as a sub-process during the test case setup phase. Command line attributes to start VPP are stored in class variable *vpp_cmdline*.
|
||||
To get an overview of VPP command line attributes, visit section [Command-line Arguments](https://wiki.fd.io/view/VPP/Command-line_Arguments) on VPP wiki page.
|
||||
|
||||
Current VPP test infrastructure is using two ways to interact with VPP for configuration, operational status check, tracing and logging.
|
||||
|
||||
### Using API commands
|
||||
API commands are executed by VPP API test tool that is started from command line as a sub-process. Command line attributes to start VPP API test tool are stored in class variable *vpp_api_test_cmdline*.
|
||||
When executed, API command and its possible output are printed in the terminal if verbosity level is greater then 0.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
cls.api("sw_interface_set_flags pg1 admin-up")
|
||||
```
|
||||
|
||||
will print in the terminal
|
||||
|
||||
```
|
||||
API: sw_interface_set_flags pg1 admin-up
|
||||
```
|
||||
|
||||
### Using CLI commands
|
||||
CLI commands are executed via VPP API test tool by sending API command "*exec + cli_command*". It is possible to set verbosity level for executing specific CLI commands, so that the CLI command is executed only and only if its associated verbosity level is equal or lower then the verbosity level set in the system.
|
||||
|
||||
Similarly to API commands, when executed, CLI command and its possible output are printed in the terminal if verbosity level is greater then 0.
|
||||
|
||||
Example I - CLI command will be executed always (its verbosity is 0):
|
||||
|
||||
```
|
||||
cls.cli(0, "show l2fib")
|
||||
```
|
||||
|
||||
Example II - CLI command will be executed only if the verbosity level is set to 2:
|
||||
|
||||
```
|
||||
self.cli(2, "show l2fib verbose")
|
||||
```
|
||||
|
||||
## Logging
|
||||
It is possible to log some additional information in the terminal for different verbosity levels.
|
||||
|
||||
Example I - verbosity level of the log is set to default value (0):
|
||||
|
||||
```
|
||||
self.log("Verifying capture %u" % i)
|
||||
```
|
||||
|
||||
will be always printed in the terminal:
|
||||
|
||||
```
|
||||
LOG: Verifying capture 0
|
||||
```
|
||||
|
||||
Example II - the log will be printed in the terminal only if the verbosity level is set to 2:
|
||||
|
||||
```
|
||||
self.log("Got packet on port %u: src=%u (id=%u)"
|
||||
% (o, payload_info.src, payload_info.index), 2)
|
||||
```
|
||||
|
||||
---
|
||||
***END***
|
||||
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
import time
|
||||
from logging import error
|
||||
from scapy.utils import wrpcap, rdpcap
|
||||
from vpp_interface import VppInterface
|
||||
@@ -39,11 +40,27 @@ class VppPGInterface(VppInterface):
|
||||
"""CLI string to load the injected packets"""
|
||||
return self._input_cli
|
||||
|
||||
@property
|
||||
def in_history_counter(self):
|
||||
"""Self-incrementing counter used when renaming old pcap files"""
|
||||
v = self._in_history_counter
|
||||
self._in_history_counter += 1
|
||||
return v
|
||||
|
||||
@property
|
||||
def out_history_counter(self):
|
||||
"""Self-incrementing counter used when renaming old pcap files"""
|
||||
v = self._out_history_counter
|
||||
self._out_history_counter += 1
|
||||
return v
|
||||
|
||||
def post_init_setup(self):
|
||||
""" Perform post-init setup for super class and add our own setup """
|
||||
super(VppPGInterface, self).post_init_setup()
|
||||
self._out_path = self.test.tempdir + "/pg%u_out.pcap" % self.sw_if_index
|
||||
self._in_path = self.test.tempdir + "/pg%u_in.pcap" % self.sw_if_index
|
||||
self._out_file = "pg%u_out.pcap" % self.sw_if_index
|
||||
self._out_path = self.test.tempdir + "/" + self._out_file
|
||||
self._in_file = "pg%u_in.pcap" % self.sw_if_index
|
||||
self._in_path = self.test.tempdir + "/" + self._in_file
|
||||
self._capture_cli = "packet-generator capture pg%u pcap %s" % (
|
||||
self.pg_index, self.out_path)
|
||||
self._cap_name = "pcap%u" % self.sw_if_index
|
||||
@@ -52,6 +69,8 @@ class VppPGInterface(VppInterface):
|
||||
|
||||
def __init__(self, test, pg_index):
|
||||
""" Create VPP packet-generator interface """
|
||||
self._in_history_counter = 0
|
||||
self._out_history_counter = 0
|
||||
self._pg_index = pg_index
|
||||
self._test = test
|
||||
r = self.test.vapi.pg_create_interface(self.pg_index)
|
||||
@@ -61,7 +80,14 @@ class VppPGInterface(VppInterface):
|
||||
def enable_capture(self):
|
||||
""" Enable capture on this packet-generator interface"""
|
||||
try:
|
||||
os.unlink(self.out_path)
|
||||
if os.path.isfile(self.out_path):
|
||||
os.rename(self.out_path,
|
||||
"%s/history.[timestamp:%f].[%s-counter:%04d].%s" %
|
||||
(self.test.tempdir,
|
||||
time.time(),
|
||||
self.name,
|
||||
self.out_history_counter,
|
||||
self._out_file))
|
||||
except:
|
||||
pass
|
||||
# FIXME this should be an API, but no such exists atm
|
||||
@@ -75,7 +101,14 @@ class VppPGInterface(VppInterface):
|
||||
|
||||
"""
|
||||
try:
|
||||
os.remove(self.in_path)
|
||||
if os.path.isfile(self.in_path):
|
||||
os.rename(self.in_path,
|
||||
"%s/history.[timestamp:%f].[%s-counter:%04d].%s" %
|
||||
(self.test.tempdir,
|
||||
time.time(),
|
||||
self.name,
|
||||
self.in_history_counter,
|
||||
self._in_file))
|
||||
except:
|
||||
pass
|
||||
wrpcap(self.in_path, pkts)
|
||||
|
||||
Reference in New Issue
Block a user