make test: add test framework debugging

New option TEST_DEBUG=1 turns on test framework debugging, which
currently consists of printing difference in allocated objects/memory
and also creates reference graphs for any unfreed VppPapiProvider/VPP
objects - these take a lot of memory and thus should be freed regularly.

Change-Id: I29db0c1341009d4b5c5df9222d14f3095883fd0f
Signed-off-by: Klement Sekera <ksekera@cisco.com>

test/Makefile

@@ -46,8 +46,15 @@ ifneq ($(EXTERN_TESTS),)
 UNITTEST_EXTRA_OPTS=$(UNITTEST_FAILFAST_OPTS) -d $(EXTERN_TESTS)
 endif
 
+ifeq ($(TEST_DEBUG),1)
+VPP_PYTHON_PREFIX:=$(VPP_PYTHON_PREFIX)/debug
+PYTHON_EXTRA_DEPENDS=objgraph pympler
+else
+PYTHON_EXTRA_DEPENDS=
+endif
+
 PYTHON_VENV_PATH=$(VPP_PYTHON_PREFIX)/virtualenv
-PYTHON_DEPENDS=faulthandler six scapy==2.3.3 pexpect pycrypto subprocess32 cffi git+https://github.com/klement/py-lispnetworking@setup
+PYTHON_DEPENDS=$(PYTHON_EXTRA_DEPENDS) faulthandler six scapy==2.3.3 pexpect pycrypto subprocess32 cffi git+https://github.com/klement/py-lispnetworking@setup
 SCAPY_SOURCE=$(shell find $(PYTHON_VENV_PATH) -name site-packages)
 BUILD_COV_DIR=$(BR)/test-cov
 
@@ -243,6 +250,8 @@ help:
 	@echo " EXTERN_PLUGINS=<path>- path to out-of-tree plugins to be loaded by vpp under test"
 	@echo " EXTERN_COV_DIR=<path>- path to out-of-tree prefix, where source, object and .gcda files can be found for coverage report"
 	@echo ""
+	@echo " TEST_DEBUG=1         - turn on debugging of the test framework itself (expert)"
+	@echo ""
 	@echo "Creating test documentation"
 	@echo " test-doc            - generate documentation for test framework"
 	@echo " test-wipe-doc       - wipe documentation for test framework"

test/debug_internal.py

@@ -0,0 +1,39 @@
+import gc
+import pprint
+import vpp_papi
+from vpp_papi_provider import VppPapiProvider
+import objgraph
+from pympler import tracker
+tr = tracker.SummaryTracker()
+
+"""
+  Internal debug module
+
+  The module provides functions for debugging test framework
+"""
+
+
+def on_tear_down_class(cls):
+    gc.collect()
+    tr.print_diff()
+    objects = gc.get_objects()
+    counter = 0
+    with open(cls.tempdir + '/python_objects.txt', 'w') as f:
+        interesting = [
+            o for o in objects
+            if isinstance(o, (VppPapiProvider, vpp_papi.VPP))]
+        del objects
+        gc.collect()
+        for o in interesting:
+            objgraph.show_backrefs([o], max_depth=5,
+                                   filename="%s/%s.png" %
+                                   (cls.tempdir, counter))
+            counter += 1
+            refs = gc.get_referrers(o)
+            pp = pprint.PrettyPrinter(indent=2)
+            f.write("%s\n" % pp.pformat(o))
+            for r in refs:
+                try:
+                    f.write("%s\n" % pp.pformat(r))
+                except:
+                    f.write("%s\n" % type(r))

test/framework.py

@@ -32,6 +32,12 @@ if os.name == 'posix' and sys.version_info[0] < 3:
 else:
     import subprocess
 
+debug_framework = False
+if os.getenv('TEST_DEBUG', "0") == "1":
+    debug_framework = True
+    import debug_internal
+
+
 """
   Test framework module.
 
@@ -453,6 +459,9 @@ class VppTestCase(unittest.TestCase):
         """ Perform final cleanup after running all tests in this test-case """
         cls.quit()
         cls.file_handler.close()
+        cls.reset_packet_infos()
+        if debug_framework:
+            debug_internal.on_tear_down_class(cls)
 
     def tearDown(self):
         """ Show various debug prints after each test """