tests: refactor asf framework code

- Make framework.py classes a subset of asfframework.py classes
- Remove all packet related code from asfframework.py
- Add test class and test case set up debug output to log
- Repatriate packet tests from asf to test directory
- Remove non-packet related code from framework.py and
  inherit them from asfframework.py classes
- Clean up unused import variables
- Re-enable BFD tests on Ubuntu 22.04 and fix
  intermittent test failures in echo_looped_back
  testcases (where # control packets verified but
  not guaranteed to be received during test)
- Re-enable Wireguard tests on Ubuntu 22.04 and fix
  intermittent test failures in handshake ratelimiting
  testcases and event testcase
- Run Wiregard testcase suites solo
- Improve debug output in log.txt
- Increase VCL/LDP post sleep timeout to allow iperf server
  to finish cleanly.
- Fix pcap history files to be sorted by suite and testcase
  and ensure order/timestamp is correct based on creation
  in the testcase.
- Decode pcap files for each suite and testcase for all
  errors or if configured via comandline option / env var
- Improve vpp corefile detection to allow complete corefile
  generation
- Disable vm vpp interfaces testcases on debian11
- Clean up failed unittest dir when retrying failed testcases
  and unify testname directory and failed linknames into
  framwork functions

Type: test

Change-Id: I0764f79ea5bb639d278bf635ed2408d4d5220e1e
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>

Makefile

@@ -81,6 +81,7 @@ DEB_DEPENDS += iperf3 # for 'make test TEST=vcl'
 DEB_DEPENDS += nasm
 DEB_DEPENDS += iperf ethtool  # for 'make test TEST=vm_vpp_interfaces'
 DEB_DEPENDS += libpcap-dev
+DEB_DEPENDS += tshark
 
 LIBFFI=libffi6 # works on all but 20.04 and debian-testing
 
@@ -197,6 +198,7 @@ ifneq ($(wildcard $(STARTUP_DIR)/startup.conf),)
 endif
 
 ifeq ($(findstring y,$(UNATTENDED)),y)
+DEBIAN_FRONTEND=noninteractive
 CONFIRM=-y
 FORCE=--allow-downgrades --allow-remove-essential --allow-change-held-packages
 endif

test/Makefile

@@ -74,7 +74,7 @@ V=0
 endif
 
 PYTHON_VERSION=$(shell $(PYTHON_INTERP) -c 'import sys; print(sys.version_info.major)')
-PIP_VERSION=23.2.1
+PIP_VERSION=23.3.1
 # Keep in sync with requirements.txt
 PIP_TOOLS_VERSION=7.3.0
 PIP_SETUPTOOLS_VERSION=68.1.0
@@ -254,6 +254,11 @@ ifneq ($(EXTERN_APIDIR),)
 ARG17=--extern-apidir=$(EXTERN_APIDIR)
 endif
 
+ARG18=
+ifneq ($(findstring $(DECODE_PCAPS),1 y yes),)
+ARG18=--decode-pcaps
+endif
+
 EXC_PLUGINS_ARG=
 ifneq ($(VPP_EXCLUDED_PLUGINS),)
 # convert the comma-separated list into N invocations of the argument to exclude a plugin
@@ -262,7 +267,7 @@ endif
 
 
 
-EXTRA_ARGS=$(ARG0) $(ARG1) $(ARG2) $(ARG3) $(ARG4) $(ARG5) $(ARG6) $(ARG7) $(ARG8) $(ARG9) $(ARG10) $(ARG11) $(ARG12) $(ARG13) $(ARG14) $(ARG15) $(ARG16) $(ARG17)
+EXTRA_ARGS=$(ARG0) $(ARG1) $(ARG2) $(ARG3) $(ARG4) $(ARG5) $(ARG6) $(ARG7) $(ARG8) $(ARG9) $(ARG10) $(ARG11) $(ARG12) $(ARG13) $(ARG14) $(ARG15) $(ARG16) $(ARG17) $(ARG18)
 
 RUN_TESTS_ARGS=--failed-dir=$(FAILED_DIR) --verbose=$(V) --jobs=$(TEST_JOBS) --filter=$(TEST) --retries=$(RETRIES) --venv-dir=$(VENV_PATH) --vpp-ws-dir=$(WS_ROOT) --vpp-tag=$(TAG) --rnd-seed=$(RND_SEED) --vpp-worker-count="$(VPP_WORKER_COUNT)" --keep-pcaps $(PLUGIN_PATH_ARGS) $(EXC_PLUGINS_ARG) $(TEST_PLUGIN_PATH_ARGS) $(EXTRA_ARGS)
 RUN_SCRIPT_ARGS=--python-opts=$(PYTHON_OPTS)

test/asf/test_adl.py

@@ -2,11 +2,10 @@
 
 import unittest
 
-from asfframework import VppTestCase, VppTestRunner
-from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath
+from asfframework import VppAsfTestCase, VppTestRunner
 
 
-class TestAdl(VppTestCase):
+class TestAdl(VppAsfTestCase):
     """Allow/Deny Plugin Unit Test Cases"""
 
     @classmethod

test/asf/test_api_client.py

@@ -2,11 +2,10 @@
 
 import unittest
 
-from asfframework import VppTestCase, VppTestRunner
-from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath
+from asfframework import VppAsfTestCase, VppTestRunner
 
 
-class TestAPIClient(VppTestCase):
+class TestAPIClient(VppAsfTestCase):
     """API Internal client Test Cases"""
 
     def test_client_unittest(self):

test/asf/test_api_trace.py

@@ -1,12 +1,10 @@
-import os
 import unittest
-from asfframework import VppTestCase, VppTestRunner
-from vpp_papi import VppEnum
+from asfframework import VppAsfTestCase, VppTestRunner
 import json
 import shutil
 
 
-class TestJsonApiTrace(VppTestCase):
+class TestJsonApiTrace(VppAsfTestCase):
     """JSON API trace related tests"""
 
     @classmethod

test/asf/test_bihash.py

@@ -3,11 +3,10 @@
 import unittest
 
 from config import config
-from asfframework import VppTestCase, VppTestRunner
-from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath
+from asfframework import VppAsfTestCase, VppTestRunner
 
 
-class TestBihash(VppTestCase):
+class TestBihash(VppAsfTestCase):
     """Bihash Test Cases"""
 
     @classmethod

test/asf/test_buffers.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 
-from asfframework import VppTestCase
+from asfframework import VppAsfTestCase
 
 
-class TestBuffers(VppTestCase):
+class TestBuffers(VppAsfTestCase):
     """Buffer C Unit Tests"""
 
     @classmethod

test/asf/test_cli.py

@@ -1,16 +1,14 @@
 #!/usr/bin/env python3
 """CLI functional tests"""
 
-import datetime
-import time
 import unittest
 
 from vpp_papi import VPPIOError
 
-from asfframework import VppTestCase, VppTestRunner
+from asfframework import VppAsfTestCase, VppTestRunner
 
 
-class TestCLI(VppTestCase):
+class TestCLI(VppAsfTestCase):
     """CLI Test Case"""
 
     maxDiff = None
@@ -50,7 +48,7 @@ class TestCLI(VppTestCase):
         self.assertEqual(rv.retval, 0)
 
 
-class TestCLIExtendedVapiTimeout(VppTestCase):
+class TestCLIExtendedVapiTimeout(VppAsfTestCase):
     maxDiff = None
 
     @classmethod

test/asf/test_counters.py

@@ -1,11 +1,10 @@
 #!/usr/bin/env python3
 
-from asfframework import VppTestCase
-from asfframework import tag_fixme_vpp_workers
+from asfframework import VppAsfTestCase, tag_fixme_vpp_workers
 
 
 @tag_fixme_vpp_workers
-class TestCounters(VppTestCase):
+class TestCounters(VppAsfTestCase):
     """Counters C Unit Tests"""
 
     @classmethod

test/asf/test_crypto.py

@@ -2,10 +2,10 @@
 
 import unittest
 
-from asfframework import VppTestCase, VppTestRunner
+from asfframework import VppAsfTestCase, VppTestRunner
 
 
-class TestCrypto(VppTestCase):
+class TestCrypto(VppAsfTestCase):
     """Crypto Test Case"""
 
     @classmethod

test/asf/test_endian.py

@@ -12,13 +12,13 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-import asfframework
+from asfframework import VppAsfTestCase
 import vpp_papi_provider
 
 F64_ONE = 1.0
 
 
-class TestEndian(asfframework.VppTestCase):
+class TestEndian(VppAsfTestCase):
     """TestEndian"""
 
     def test_f64_endian_value(self):

test/asf/test_fib.py

@@ -2,12 +2,11 @@
 
 import unittest
 
-from asfframework import tag_fixme_vpp_workers
-from asfframework import VppTestCase, VppTestRunner
+from asfframework import VppAsfTestCase, VppTestRunner, tag_fixme_vpp_workers
 
 
 @tag_fixme_vpp_workers
-class TestFIB(VppTestCase):
+class TestFIB(VppAsfTestCase):
     """FIB Test Case"""
 
     @classmethod

test/asf/test_http.py

@@ -2,15 +2,12 @@
 """ Vpp HTTP tests """
 
 import unittest
-import os
-import subprocess
 import http.client
-from asfframework import VppTestCase, VppTestRunner, Worker
-from vpp_devices import VppTAPInterface
+from asfframework import VppAsfTestCase, VppTestRunner
 
 
 @unittest.skip("Requires root")
-class TestHttpTps(VppTestCase):
+class TestHttpTps(VppAsfTestCase):
     """HTTP test class"""
 
     @classmethod

test/asf/test_http_static.py

@@ -1,5 +1,5 @@
 from config import config
-from asfframework import VppTestCase, VppTestRunner
+from asfframework import VppAsfTestCase, VppTestRunner
 import unittest
 import subprocess
 import tempfile
@@ -15,7 +15,7 @@ from vpp_qemu_utils import (
     "http_static" in config.excluded_plugins, "Exclude HTTP Static Server plugin tests"
 )
 @unittest.skipIf(config.skip_netns_tests, "netns not available or disabled from cli")
-class TestHttpStaticVapi(VppTestCase):
+class TestHttpStaticVapi(VppAsfTestCase):
     """enable the http static server and send requests [VAPI]"""
 
     @classmethod
@@ -82,7 +82,7 @@ class TestHttpStaticVapi(VppTestCase):
     "http_static" in config.excluded_plugins, "Exclude HTTP Static Server plugin tests"
 )
 @unittest.skipIf(config.skip_netns_tests, "netns not available or disabled from cli")
-class TestHttpStaticCli(VppTestCase):
+class TestHttpStaticCli(VppAsfTestCase):
     """enable the static http server and send requests [CLI]"""
 
     @classmethod

test/asf/test_lb_api.py

@@ -12,13 +12,12 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-import asfframework
-import ipaddress
+from asfframework import VppAsfTestCase
 
 DEFAULT_VIP = "lb_vip_details(_0=978, context=12, vip=vl_api_lb_ip_addr_t(pfx=IPv6Network(u'::/0'), protocol=<vl_api_ip_proto_t.IP_API_PROTO_RESERVED: 255>, port=0), encap=<vl_api_lb_encap_type_t.LB_API_ENCAP_TYPE_GRE4: 0>, dscp=<vl_api_ip_dscp_t.IP_API_DSCP_CS0: 0>, srv_type=<vl_api_lb_srv_type_t.LB_API_SRV_TYPE_CLUSTERIP: 0>, target_port=0, flow_table_length=0)"  # noqa
 
 
-class TestLbEmptyApi(asfframework.VppTestCase):
+class TestLbEmptyApi(VppAsfTestCase):
     """TestLbEmptyApi"""
 
     def test_lb_empty_vip_dump(self):
@@ -35,7 +34,7 @@ class TestLbEmptyApi(asfframework.VppTestCase):
         self.assertEqual(rv, [], "Expected: [] Received: %r." % rv)
 
 
-class TestLbApi(asfframework.VppTestCase):
+class TestLbApi(VppAsfTestCase):
     """TestLbApi"""
 
     def test_lb_vip_dump(self):
@@ -56,7 +55,7 @@ class TestLbApi(asfframework.VppTestCase):
         self.vapi.cli("lb vip 2001::/16 del")
 
 
-class TestLbAsApi(asfframework.VppTestCase):
+class TestLbAsApi(VppAsfTestCase):
     """TestLbAsApi"""
 
     def test_lb_as_dump(self):

test/asf/test_mactime.py

@@ -3,11 +3,10 @@
 import unittest
 
 from config import config
-from asfframework import VppTestCase, VppTestRunner
-from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath
+from asfframework import VppAsfTestCase, VppTestRunner
 
 
-class TestMactime(VppTestCase):
+class TestMactime(VppAsfTestCase):
     """Mactime Unit Test Cases"""
 
     @classmethod

test/asf/test_mpcap.py

@@ -2,12 +2,11 @@
 
 import unittest
 
-from asfframework import VppTestCase, VppTestRunner
-from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath
+from asfframework import VppAsfTestCase, VppTestRunner
 import os
 
 
-class TestMpcap(VppTestCase):
+class TestMpcap(VppAsfTestCase):
     """Mpcap Unit Test Cases"""
 
     @classmethod

test/asf/test_node_variants.py

@@ -2,7 +2,7 @@
 import re
 import unittest
 import platform
-from asfframework import VppTestCase
+from asfframework import VppAsfTestCase
 
 
 def checkX86():
@@ -19,7 +19,7 @@ def skipVariant(variant):
     return checkX86() and match is not None
 
 
-class TestNodeVariant(VppTestCase):
+class TestNodeVariant(VppAsfTestCase):
     """Test Node Variants"""
 
     @classmethod

test/asf/test_offload.py

@@ -2,11 +2,10 @@
 
 import unittest
 
-from asfframework import VppTestCase, VppTestRunner
-from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath
+from asfframework import VppAsfTestCase, VppTestRunner
 
 
-class TestOffload(VppTestCase):
+class TestOffload(VppAsfTestCase):
     """Offload Unit Test Cases"""
 
     @classmethod

test/asf/test_policer.py

@@ -3,8 +3,8 @@
 
 import unittest
 
-from asfframework import VppTestCase, VppTestRunner
-from vpp_policer import VppPolicer, PolicerAction
+from asfframework import VppAsfTestCase, VppTestRunner
+from vpp_policer import VppPolicer
 
 # Default for the tests is 10s of "Green" packets at 8Mbps, ie. 10M bytes.
 # The policer helper CLI "sends" 500 byte packets, so default is 20000.
@@ -23,7 +23,7 @@ CBURST = 100000  # Committed burst in bytes
 EBURST = 200000  # Excess burst in bytes
 
 
-class TestPolicer(VppTestCase):
+class TestPolicer(VppAsfTestCase):
     """Policer Test Case"""
 
     def run_policer_test(

test/asf/test_quic.py

@@ -3,11 +3,9 @@
 
 import unittest
 import os
-import subprocess
 import signal
 from config import config
-from framework import tag_fixme_vpp_workers
-from framework import VppTestCase, VppTestRunner, Worker
+from asfframework import VppAsfTestCase, VppTestRunner, Worker, tag_fixme_vpp_workers
 from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath
 
 
@@ -53,7 +51,7 @@ class QUICAppWorker(Worker):
 
 
 @unittest.skipIf("quic" in config.excluded_plugins, "Exclude QUIC plugin tests")
-class QUICTestCase(VppTestCase):
+class QUICTestCase(VppAsfTestCase):
     """QUIC Test Case"""
 
     timeout = 20

test/asf/test_session.py

@@ -2,14 +2,17 @@
 
 import unittest
 
-from asfframework import tag_fixme_vpp_workers
-from asfframework import VppTestCase, VppTestRunner
-from asfframework import tag_run_solo
+from asfframework import (
+    VppAsfTestCase,
+    VppTestRunner,
+    tag_fixme_vpp_workers,
+    tag_run_solo,
+)
 from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath
 
 
 @tag_fixme_vpp_workers
-class TestSession(VppTestCase):
+class TestSession(VppAsfTestCase):
     """Session Test Case"""
 
     @classmethod
@@ -106,7 +109,7 @@ class TestSession(VppTestCase):
 
 
 @tag_fixme_vpp_workers
-class TestSessionUnitTests(VppTestCase):
+class TestSessionUnitTests(VppAsfTestCase):
     """Session Unit Tests Case"""
 
     @classmethod
@@ -135,7 +138,7 @@ class TestSessionUnitTests(VppTestCase):
 
 
 @tag_run_solo
-class TestSegmentManagerTests(VppTestCase):
+class TestSegmentManagerTests(VppAsfTestCase):
     """SVM Fifo Unit Tests Case"""
 
     @classmethod
@@ -162,7 +165,7 @@ class TestSegmentManagerTests(VppTestCase):
 
 
 @tag_run_solo
-class TestSvmFifoUnitTests(VppTestCase):
+class TestSvmFifoUnitTests(VppAsfTestCase):
     """SVM Fifo Unit Tests Case"""
 
     @classmethod

test/asf/test_sparse_vec.py

@@ -2,11 +2,10 @@
 
 import unittest
 
-from asfframework import VppTestCase, VppTestRunner
-from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath
+from asfframework import VppAsfTestCase, VppTestRunner
 
 
-class TestSparseVec(VppTestCase):
+class TestSparseVec(VppAsfTestCase):
     """SparseVec Test Cases"""
 
     @classmethod

test/asf/test_string.py

@@ -2,11 +2,10 @@
 
 import unittest
 
-from asfframework import VppTestCase, VppTestRunner
-from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath
+from asfframework import VppAsfTestCase, VppTestRunner
 
 
-class TestString(VppTestCase):
+class TestString(VppAsfTestCase):
     """String Test Cases"""
 
     @classmethod

test/asf/test_tap.py

@@ -1,7 +1,7 @@
 import unittest
 import os
 
-from asfframework import VppTestCase, VppTestRunner
+from asfframework import VppAsfTestCase, VppTestRunner
 from vpp_devices import VppTAPInterface
 
 
@@ -10,7 +10,7 @@ def check_tuntap_driver_access():
 
 
 @unittest.skip("Requires root")
-class TestTAP(VppTestCase):
+class TestTAP(VppAsfTestCase):
     """TAP Test Case"""
 
     def test_tap_add_del(self):

test/asf/test_tcp.py

@@ -2,11 +2,11 @@
 
 import unittest
 
-from asfframework import VppTestCase, VppTestRunner
+from asfframework import VppAsfTestCase, VppTestRunner
 from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath
 
 
-class TestTCP(VppTestCase):
+class TestTCP(VppAsfTestCase):
     """TCP Test Case"""
 
     @classmethod
@@ -93,7 +93,7 @@ class TestTCP(VppTestCase):
         ip_t10.remove_vpp_config()
 
 
-class TestTCPUnitTests(VppTestCase):
+class TestTCPUnitTests(VppAsfTestCase):
     "TCP Unit Tests"
 
     @classmethod

test/asf/test_tls.py

@@ -5,7 +5,7 @@ import os
 import re
 import subprocess
 
-from asfframework import VppTestCase, VppTestRunner
+from asfframework import VppAsfTestCase, VppTestRunner
 from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath
 
 
@@ -52,7 +52,7 @@ def checkAll():
     return ret
 
 
-class TestTLS(VppTestCase):
+class TestTLS(VppAsfTestCase):
     """TLS Qat Test Case."""
 
     @classmethod

test/asf/test_vapi.py

@@ -5,10 +5,10 @@ import unittest
 import os
 import signal
 from config import config
-from asfframework import VppTestCase, VppTestRunner, Worker
+from asfframework import VppAsfTestCase, VppTestRunner, Worker
 
 
-class VAPITestCase(VppTestCase):
+class VAPITestCase(VppAsfTestCase):
     """VAPI test"""
 
     @classmethod

test/asf/test_vcl.py

@@ -7,8 +7,8 @@ import subprocess
 import signal
 import glob
 from config import config
-from asfframework import VppTestCase, VppTestRunner, Worker
-from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath, FibPathProto
+from asfframework import VppAsfTestCase, VppTestRunner, Worker
+from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath
 
 iperf3 = "/usr/bin/iperf3"
 
@@ -58,7 +58,7 @@ class VCLAppWorker(Worker):
         super(VCLAppWorker, self).__init__(self.args, logger, env, *args, **kwargs)
 
 
-class VCLTestCase(VppTestCase):
+class VCLTestCase(VppAsfTestCase):
     """VCL Test Class"""
 
     session_startup = ["poll-main"]
@@ -84,7 +84,7 @@ class VCLTestCase(VppTestCase):
         self.timeout = 20
         self.echo_phrase = "Hello, world! Jenny is a friend of mine."
         self.pre_test_sleep = 0.3
-        self.post_test_sleep = 0.2
+        self.post_test_sleep = 1
         self.sapi_client_sock = ""
         self.sapi_server_sock = ""