tests: more options for decoding pcaps

Introduce "none", "all" and "failed" options for --decode-pcaps
parameter. Keep "failed" as default to be consistent with current
behaviour. Add missing documentation to test/Makefile and passthrough to
Makefile.

Rationale: running tshark binary takes about 100-150ms and if there are
thousands of pcap files, it takes minutes to decode them. This might not
be desirable if rerunning the tests repeatedly during development.

Type: improvement
Change-Id: Ie033521d51d18b9d499b9bc40fe6eff21c94622d
Signed-off-by: Klement Sekera <klement.sekera@gmail.com>

Makefile

@@ -473,6 +473,7 @@ define test
 	  VPP_BIN=$(BR)/install-$(1)-native/vpp/bin/vpp \
 	  VPP_INSTALL_PATH=$(BR)/install-$(1)-native/ \
 	  EXTENDED_TESTS=$(EXTENDED_TESTS) \
+	  DECODE_PCAPS=$(DECODE_PCAPS) \
 	  TEST_GCOV=$(TEST_GCOV) \
 	  PYTHON=$(PYTHON) \
 	  OS_ID=$(OS_ID) \

test/Makefile

@@ -255,8 +255,8 @@ ARG17=--extern-apidir=$(EXTERN_APIDIR)
 endif
 
 ARG18=
-ifneq ($(findstring $(DECODE_PCAPS),1 y yes),)
-ARG18=--decode-pcaps
+ifneq ($(DECODE_PCAPS),)
+ARG18=--decode-pcaps=$(DECODE_PCAPS)
 endif
 
 ifneq ($(findstring $(API_PRELOAD),1 y yes),)
@@ -654,6 +654,10 @@ help:
 	@echo "       random seed used by test framework"
 	@echo "       (default: time.time())"
 	@echo ""
+	@echo "   DECODE_PCAPS=[all|failed|none]"
+	@echo "       decode pcap files using tshark - all, only failed or none"
+	@echo "       (default: failed)"
+	@echo ""
 	@echo "Starting VPP in GDB for use with DEBUG=attach:"
 	@echo ""
 	@echo " test-start-vpp-in-gdb       - start VPP in gdb (release)"

test/asf/asfframework.py

@@ -1161,16 +1161,13 @@ class VppTestResult(unittest.TestResult):
         self.runner = runner
         self.printed = []
 
-    def decodePcapFiles(self, test, when_configured=False):
-        if when_configured == False or config.decode_pcaps == True:
-            if hasattr(test, "pg_interfaces") and len(test.pg_interfaces) > 0:
-                testcase_dir = os.path.dirname(test.pg_interfaces[0].out_path)
-                test.pg_interfaces[0].decode_pcap_files(
-                    testcase_dir, f"suite{test.__class__.__name__}"
-                )
-                test.pg_interfaces[0].decode_pcap_files(
-                    testcase_dir, test._testMethodName
-                )
+    def decodePcapFiles(self, test):
+        if hasattr(test, "pg_interfaces") and len(test.pg_interfaces) > 0:
+            testcase_dir = os.path.dirname(test.pg_interfaces[0].out_path)
+            test.pg_interfaces[0].decode_pcap_files(
+                testcase_dir, f"suite{test.__class__.__name__}"
+            )
+            test.pg_interfaces[0].decode_pcap_files(testcase_dir, test._testMethodName)
 
     def addSuccess(self, test):
         """
@@ -1180,7 +1177,8 @@ class VppTestResult(unittest.TestResult):
 
         """
         self.log_result("addSuccess", test)
-        self.decodePcapFiles(test, when_configured=True)
+        if "all" == config.decode_pcaps:
+            self.decodePcapFiles(test)
         unittest.TestResult.addSuccess(self, test)
         self.result_string = colorize("OK", GREEN)
         self.result_code = TestResultCode.PASS
@@ -1188,7 +1186,8 @@ class VppTestResult(unittest.TestResult):
 
     def addExpectedFailure(self, test, err):
         self.log_result("addExpectedFailure", test, err)
-        self.decodePcapFiles(test)
+        if "none" != config.decode_pcaps:
+            self.decodePcapFiles(test)
         super().addExpectedFailure(test, err)
         self.result_string = colorize("FAIL", GREEN)
         self.result_code = TestResultCode.EXPECTED_FAIL
@@ -1196,7 +1195,8 @@ class VppTestResult(unittest.TestResult):
 
     def addUnexpectedSuccess(self, test):
         self.log_result("addUnexpectedSuccess", test)
-        self.decodePcapFiles(test, when_configured=True)
+        if "none" != config.decode_pcaps:
+            self.decodePcapFiles(test)
         super().addUnexpectedSuccess(test)
         self.result_string = colorize("OK", RED)
         self.result_code = TestResultCode.UNEXPECTED_PASS
@@ -1282,7 +1282,9 @@ class VppTestResult(unittest.TestResult):
             error_type_str = colorize("ERROR", RED)
         else:
             raise Exception(f"Unexpected result code {result_code}")
-        self.decodePcapFiles(test)
+
+        if "none" != config.decode_pcaps:
+            self.decodePcapFiles(test)
 
         unittest_fn(self, test, err)
         if self.current_test_case_info:

test/config.py

@@ -409,10 +409,11 @@ parser.add_argument(
     "/var/run/user/${uid}/vpp.",
 )
 
-default_decode_pcaps = False
+default_decode_pcaps = "failed"
 parser.add_argument(
     "--decode-pcaps",
-    action="store_true",
+    action="store",
+    choices=["none", "failed", "all"],
     default=default_decode_pcaps,
     help=f"if set, decode all pcap files from a test run (default: {default_decode_pcaps})",
 )