tests docs: fix lcov code coverage report generation
- Updated/rebased version of https://gerrit.fd.io/r/c/vpp/+/34199 Type: test Change-Id: I43913ecfd11a4578bdb10c4be76253fe38d57976 Signed-off-by: Nathan Skrzypczak <nathan.skrzypczak@gmail.com> Signed-off-by: Dave Wallace <dwallacelf@gmail.com>
This commit is contained in:
Dave Wallace
committed by
Andrew Yourtchenko
Andrew Yourtchenko
parent
fdf6fbe2e7
commit
1c95e12b0d
8
.gitignore
vendored
8
.gitignore
vendored
@ -16,14 +16,8 @@
|
||||
/build-root/*.tar.xz
|
||||
/build-root/*.changes
|
||||
/build-root/rpmbuild/
|
||||
/build-root/test-doc/
|
||||
/build-root/test-cov/
|
||||
/build-root/test-coverage/
|
||||
/build-root/test/
|
||||
/test/run/
|
||||
/test/build/
|
||||
/test/coverage/
|
||||
/test/vapi_test/
|
||||
/test/doc/build/
|
||||
/build-config.mk
|
||||
/build/external/*.tar.gz
|
||||
/build/external/*.tar.xz
|
||||
|
||||
45
Makefile
45
Makefile
@ -65,7 +65,7 @@ endif
|
||||
DEB_DEPENDS = curl build-essential autoconf automake ccache
|
||||
DEB_DEPENDS += debhelper dkms git libtool libapr1-dev dh-python
|
||||
DEB_DEPENDS += libconfuse-dev git-review exuberant-ctags cscope pkg-config
|
||||
DEB_DEPENDS += lcov chrpath autoconf libnuma-dev
|
||||
DEB_DEPENDS += gcovr lcov chrpath autoconf libnuma-dev
|
||||
DEB_DEPENDS += python3-all python3-setuptools check
|
||||
DEB_DEPENDS += libffi-dev python3-ply
|
||||
DEB_DEPENDS += cmake ninja-build uuid-dev python3-jsonschema python3-yaml
|
||||
@ -426,7 +426,7 @@ rebuild-release: wipe-release build-release
|
||||
export TEST_DIR ?= $(WS_ROOT)/test
|
||||
|
||||
define test
|
||||
$(if $(filter-out $(2),retest),make -C $(BR) PLATFORM=vpp TAG=$(1) vpp-install,)
|
||||
$(if $(filter-out $(2),retest),make -C $(BR) PLATFORM=vpp TAG=$(1) CC=$(CC) vpp-install,)
|
||||
$(eval libs:=lib lib64)
|
||||
make -C test \
|
||||
VPP_BUILD_DIR=$(BR)/build-$(1)-native/vpp \
|
||||
@ -449,10 +449,28 @@ test:
|
||||
test-debug:
|
||||
$(call test,vpp_debug,test)
|
||||
|
||||
.PHONY: test-gcov
|
||||
test-gcov:
|
||||
.PHONY: test-cov
|
||||
test-cov:
|
||||
$(eval CC=gcc)
|
||||
$(eval TEST_GCOV=1)
|
||||
$(call test,vpp_gcov,cov)
|
||||
|
||||
.PHONY: test-cov-build
|
||||
test-cov-build:
|
||||
$(eval CC=gcc)
|
||||
$(eval TEST_GCOV=1)
|
||||
$(call test,vpp_gcov,test)
|
||||
|
||||
.PHONY: test-cov-prep
|
||||
test-cov-prep:
|
||||
$(eval CC=gcc)
|
||||
$(call test,vpp_gcov,cov-prep)
|
||||
|
||||
.PHONY: test-cov-post
|
||||
test-cov-post:
|
||||
$(eval CC=gcc)
|
||||
$(call test,vpp_gcov,cov-post)
|
||||
|
||||
.PHONY: test-all
|
||||
test-all:
|
||||
$(eval EXTENDED_TESTS=1)
|
||||
@ -463,6 +481,13 @@ test-all-debug:
|
||||
$(eval EXTENDED_TESTS=1)
|
||||
$(call test,vpp_debug,test)
|
||||
|
||||
.PHONY: test-all-cov
|
||||
test-all-cov:
|
||||
$(eval CC=gcc)
|
||||
$(eval TEST_GCOV=1)
|
||||
$(eval EXTENDED_TESTS=1)
|
||||
$(call test,vpp_gcov,test)
|
||||
|
||||
.PHONY: papi-wipe
|
||||
papi-wipe: test-wipe-papi
|
||||
$(call banner,"This command is deprecated. Please use 'test-wipe-papi'")
|
||||
@ -487,8 +512,10 @@ test-shell:
|
||||
test-shell-debug:
|
||||
$(call test,vpp_debug,shell)
|
||||
|
||||
.PHONY: test-shell-gcov
|
||||
test-shell-gcov:
|
||||
.PHONY: test-shell-cov
|
||||
test-shell-cov:
|
||||
$(eval CC=gcc)
|
||||
$(eval TEST_GCOV=1)
|
||||
$(call test,vpp_gcov,shell)
|
||||
|
||||
.PHONY: test-dep
|
||||
@ -505,13 +532,9 @@ test-wipe-doc:
|
||||
@echo "make test-wipe-doc is DEPRECATED"
|
||||
sleep 300
|
||||
|
||||
.PHONY: test-cov
|
||||
test-cov:
|
||||
$(eval EXTENDED_TESTS=1)
|
||||
$(call test,vpp_gcov,cov)
|
||||
|
||||
.PHONY: test-wipe-cov
|
||||
test-wipe-cov:
|
||||
$(call make,$(PLATFORM)_gcov,$(addsuffix -wipe,$(TARGETS)))
|
||||
@make -C test wipe-cov
|
||||
|
||||
.PHONY: test-wipe-all
|
||||
|
||||
45
docs/developer/build-run-debug/code_coverage.rst
Normal file
45
docs/developer/build-run-debug/code_coverage.rst
Normal file
@ -0,0 +1,45 @@
|
||||
.. _lcov_code_coverage:
|
||||
|
||||
Code coverage with lcov
|
||||
=======================
|
||||
|
||||
Prerequisites
|
||||
-------------
|
||||
|
||||
Ensure required packages are installed:
|
||||
|
||||
::
|
||||
|
||||
$ make install-deps
|
||||
|
||||
The Linux gcov and lcov tools can be fussy about gcc / g++ compiler
|
||||
versions. As of this writing, Ubuntu 22.04 gcov / lcov works with
|
||||
the latest gcc version (``11.3.0``).
|
||||
|
||||
Generate coverage for a test case
|
||||
---------------------------------
|
||||
|
||||
As a first run, in order to generate the coverage report of
|
||||
a specific plugin or test, run for example
|
||||
|
||||
::
|
||||
|
||||
$ make test-cov TEST=fib
|
||||
|
||||
Then open the file ``.build-root/test-coverage/html/index.html`` in a Chrome browser.
|
||||
|
||||
Improving test coverage
|
||||
-----------------------
|
||||
|
||||
When doing modifications on the test cases, you can run
|
||||
|
||||
::
|
||||
|
||||
# This will run the test & report the result in the coverage data
|
||||
$ make test-cov-build TEST=fib
|
||||
|
||||
# This will generate the html report with the current coverage data
|
||||
$ make test-cov-post
|
||||
|
||||
# To reset the coverage data use
|
||||
$ make test-cov-prep
|
||||
@ -12,3 +12,4 @@ Build, Run & Debug
|
||||
testing_vpp
|
||||
gdb_examples
|
||||
cross_compile_macos
|
||||
code_coverage
|
||||
|
||||
@ -7,7 +7,6 @@ VPP extra tools
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
lcov
|
||||
snap
|
||||
strongswan
|
||||
vpp_config
|
||||
|
||||
@ -1 +0,0 @@
|
||||
../../../extras/lcov/README.rst
|
||||
@ -397,7 +397,6 @@ Gerrit
|
||||
gethostbyname
|
||||
gettingsources
|
||||
gettingstarted
|
||||
ggcov
|
||||
gid
|
||||
Gigabit
|
||||
GigE
|
||||
|
||||
@ -1,48 +0,0 @@
|
||||
.. _lcov_code_coverage:
|
||||
|
||||
Code coverage with lcov
|
||||
=======================
|
||||
|
||||
Prerequisites
|
||||
-------------
|
||||
|
||||
The Linux gcov and lcov tools are fussy about gcc / g++ compiler
|
||||
versions. As of this writing, Ubuntu 18.04 gcov / lcov work with these
|
||||
toolchain versions:
|
||||
|
||||
$ gcc –version gcc (Ubuntu 7.4.0-1ubuntu1~18.04.1) 7.4.0 $ g++ –version
|
||||
g++ (Ubuntu 8.3.0-6ubuntu1~18.04.1) 8.3.0
|
||||
|
||||
Refer to
|
||||
https://askubuntu.com/questions/26498/how-to-choose-the-default-gcc-and-g-version
|
||||
for information on how to install multiple gcc / g++ versions, and
|
||||
switch between them.
|
||||
|
||||
You’ll need to install the following additional packages:
|
||||
|
||||
$ sudo apt-get install gcovr ggcov lcov
|
||||
|
||||
Compile an instrumented vpp image
|
||||
---------------------------------
|
||||
|
||||
Two ways:
|
||||
|
||||
$ cd $ make test-gcov $ ## interrupt compilation after building the
|
||||
image
|
||||
|
||||
or $ cd /build-root $ make PLATFORM=vpp TAG=vpp_gcov vpp-install
|
||||
|
||||
Initialize the lcov database
|
||||
----------------------------
|
||||
|
||||
$ cd $ ./extras/lcov/lcov_prep $ make test-gcov or make TEST=my_test
|
||||
test-gcov $ # repeat or vary as desired to increase reported coverage $
|
||||
# Generate the report: $ ./extras/lcov/lcov_post
|
||||
|
||||
You can run vpp manually, do anything you like. Results are cumulative
|
||||
until you re-run the “prep” script.
|
||||
|
||||
Look at the results
|
||||
-------------------
|
||||
|
||||
Point a browser at file:////build-root/html/index.html
|
||||
@ -1,12 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
cd build-root
|
||||
rm -rf html
|
||||
mkdir html
|
||||
lcov --no-checksum --directory . --capture --output-file out.info
|
||||
lcov --remove out.info \
|
||||
"/usr/include/*" "*/build-root/*" "/opt/*" "/usr/lib/*" \
|
||||
-o filtered.info
|
||||
genhtml filtered.info -o html
|
||||
@ -1,7 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
cd build-root
|
||||
rm -rf html
|
||||
lcov --capture --initial --directory . --output-file lcov.out
|
||||
@ -127,6 +127,7 @@ elseif (${CMAKE_BUILD_TYPE_LC} MATCHES "coverity")
|
||||
elseif (${CMAKE_BUILD_TYPE_LC} MATCHES "gcov")
|
||||
add_compile_options(-O0 -fprofile-arcs -ftest-coverage)
|
||||
add_compile_definitions(CLIB_DEBUG CLIB_GCOV)
|
||||
link_libraries(gcov)
|
||||
endif()
|
||||
|
||||
set(BUILD_TYPES release debug coverity gcov)
|
||||
|
||||
@ -145,17 +145,6 @@ unix_signal_handler (int signum, siginfo_t * si, ucontext_t * uc)
|
||||
break;
|
||||
}
|
||||
|
||||
#ifdef CLIB_GCOV
|
||||
/*
|
||||
* Test framework sends SIGTERM, so we need to flush the
|
||||
* code coverage stats here.
|
||||
*/
|
||||
{
|
||||
void __gcov_flush (void);
|
||||
__gcov_flush ();
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Null terminate. */
|
||||
vec_add1 (syslog_msg, 0);
|
||||
|
||||
|
||||
@ -80,7 +80,7 @@ PIP_TOOLS_VERSION=6.6.0
|
||||
PIP_SETUPTOOLS_VERSION=62.1.0
|
||||
PYTHON_DEPENDS=requirements-$(PYTHON_VERSION).txt
|
||||
SCAPY_SOURCE=$(shell find $(VENV_PATH)/lib/python* -name site-packages)
|
||||
BUILD_COV_DIR=$(TEST_BR)/coverage
|
||||
BUILD_COV_DIR=$(BR)/test-coverage
|
||||
|
||||
PIP_TOOLS_INSTALL_DONE=$(VENV_RUN_DIR)/pip-tools-install-$(PYTHON_VERSION)-$(PIP_TOOLS_VERSION).done
|
||||
PIP_INSTALL_DONE=$(VENV_RUN_DIR)/pip-install-$(PYTHON_VERSION)-$(PIP_VERSION).done
|
||||
@ -326,19 +326,36 @@ wipe: reset
|
||||
$(BUILD_COV_DIR):
|
||||
@mkdir -p $@
|
||||
|
||||
.PHONY: cov
|
||||
cov: wipe-cov test-dep ext $(BUILD_COV_DIR)
|
||||
.PHONY: cov-prep
|
||||
cov-prep: test-dep
|
||||
@lcov --zerocounters --directory $(VPP_BUILD_DIR)
|
||||
@test -z "$(EXTERN_COV_DIR)" || lcov --zerocounters --directory $(EXTERN_COV_DIR)
|
||||
$(call retest-func)
|
||||
@lcov --capture --directory $(VPP_BUILD_DIR) --output-file $(BUILD_COV_DIR)/coverage.info
|
||||
@test -z "$(EXTERN_COV_DIR)" || lcov --capture --directory $(EXTERN_COV_DIR) --output-file $(BUILD_COV_DIR)/extern-coverage.info
|
||||
@genhtml $(BUILD_COV_DIR)/coverage.info --output-directory $(BUILD_COV_DIR)/html
|
||||
@test -z "$(EXTERN_COV_DIR)" || genhtml $(BUILD_COV_DIR)/extern-coverage.info --output-directory $(BUILD_COV_DIR)/extern-html
|
||||
|
||||
.PHONY: cov-post
|
||||
cov-post: wipe-cov $(BUILD_COV_DIR)
|
||||
@lcov --capture \
|
||||
--directory $(VPP_BUILD_DIR) \
|
||||
--output-file $(BUILD_COV_DIR)/coverage.info
|
||||
@test -z "$(EXTERN_COV_DIR)" || \
|
||||
lcov --capture \
|
||||
--directory $(EXTERN_COV_DIR) \
|
||||
--output-file $(BUILD_COV_DIR)/extern-coverage.info
|
||||
@lcov --remove $(BUILD_COV_DIR)/coverage.info \
|
||||
"/usr/include/*" "*/build-root/*" "/opt/*" "/usr/lib/*" \
|
||||
-o $(BUILD_COV_DIR)/coverage-filtered.info
|
||||
@genhtml $(BUILD_COV_DIR)/coverage-filtered.info \
|
||||
--output-directory $(BUILD_COV_DIR)/html
|
||||
@test -z "$(EXTERN_COV_DIR)" || \
|
||||
genhtml $(BUILD_COV_DIR)/extern-coverage.info \
|
||||
--output-directory $(BUILD_COV_DIR)/extern-html
|
||||
@echo
|
||||
@echo "Build finished. Code coverage report is in $(BUILD_COV_DIR)/html/index.html"
|
||||
@test -z "$(EXTERN_COV_DIR)" || echo "Code coverage report for out-of-tree objects is in $(BUILD_COV_DIR)/extern-html/index.html"
|
||||
|
||||
.PHONY: cov
|
||||
cov:
|
||||
make -C . cov-prep test cov-post
|
||||
|
||||
.PHONY: wipe-cov
|
||||
wipe-cov: wipe
|
||||
@rm -rf $(BUILD_COV_DIR)
|
||||
@ -388,14 +405,17 @@ help:
|
||||
@echo ""
|
||||
@echo " test - build and run (basic) functional tests"
|
||||
@echo " test-debug - build and run (basic) functional tests (debug build)"
|
||||
@echo " test-cov - generate code coverage report for functional tests"
|
||||
@echo " test-cov-prep - coverage phase #1 : prepare lcov"
|
||||
@echo " test-cov-build - coverage phase #2 : build gcov image & run tests against it (use TEST=)"
|
||||
@echo " test-cov-post - coverage phase #3 : generate lcov html report"
|
||||
@echo " test-all - build and run functional and extended tests"
|
||||
@echo " test-all-debug - build and run functional and extended tests (debug build)"
|
||||
@echo " test-all-cov - generate code coverage report for functional and extended tests"
|
||||
@echo " retest - run functional tests"
|
||||
@echo " retest-debug - run functional tests (debug build)"
|
||||
@echo " retest-all - run functional and extended tests"
|
||||
@echo " retest-all-debug - run functional and extended tests (debug build)"
|
||||
@echo " test-cov - generate code coverage report for test framework"
|
||||
@echo " test-gcov - build and run functional tests (gcov build)"
|
||||
@echo " test-wipe - wipe (temporary) files generated by unit tests"
|
||||
@echo " test-wipe-cov - wipe code coverage report for test framework"
|
||||
@echo " test-wipe-papi - rebuild vpp_papi sources"
|
||||
@ -556,6 +576,10 @@ help:
|
||||
@echo " enable debugging of the test framework itself (expert)"
|
||||
@echo " (default: no)"
|
||||
@echo ""
|
||||
@echo " TEST_GCOV=[1|y|yes]"
|
||||
@echo " enable tests specifically designed soley for code coverage"
|
||||
@echo " (default: no)"
|
||||
@echo ""
|
||||
@echo " API_FUZZ=[1|y|yes]"
|
||||
@echo " enable VPP api fuzz testing"
|
||||
@echo " (default: no)"
|
||||
@ -569,7 +593,3 @@ help:
|
||||
@echo " test-start-vpp-in-gdb - start VPP in gdb (release)"
|
||||
@echo " test-start-vpp-debug-in-gdb - start VPP in gdb (debug)"
|
||||
@echo ""
|
||||
@echo "Creating test code coverage report:"
|
||||
@echo ""
|
||||
@echo " test-cov - generate code coverage report for test framework"
|
||||
@echo " test-wipe-cov - wipe code coverage report for test framework"
|
||||
|
||||
Reference in New Issue
Block a user