hs-test: memory leak testing

add infra for memory leak testing

Type: test

Change-Id: I882e8dbb360597cdb82ad52682725f7d39b2df24
Signed-off-by: Matus Fabian <matfabia@cisco.com>

extras/hs-test/Makefile

@@ -62,6 +62,7 @@ help:
 	@echo "Make targets:"
 	@echo " test                     - run tests"
 	@echo " test-debug               - run tests (vpp debug image)"
+	@echo " test-leak                - run memory leak tests (vpp debug image)"
 	@echo " build                    - build test infra"
 	@echo " build-cov                - coverage build of VPP and Docker images"
 	@echo " build-debug              - build test infra (vpp debug image)"
@@ -143,6 +144,10 @@ test-cov: .deps.ok .build.cov.ok
 	@$(MAKE) -C ../.. test-cov-post HS_TEST=1
 	@bash ./script/compress.sh
 
+.PHONY: test-leak
+test-leak: .deps.ok .build_debug.ok
+	@bash ./hs_test.sh --test=$(TEST) --debug_build=true --leak_check=true --vppsrc=$(VPPSRC)
+
 .PHONY: build-go
 build-go:
 	go build ./tools/http_server

extras/hs-test/README.rst

@@ -307,8 +307,52 @@ It is possible to debug VPP by attaching ``gdb`` before test execution by adding
 
 If a test consists of more VPP instances then this is done for each of them.
 
+**Memory leak testing**
 
-**Eternal dependencies**
+It is possible to use VPP memory traces to diagnose if and where memory leaks happen by comparing of two traces at different point in time.
+You can do it by test like following:
+
+::
+
+    func MemLeakTest(s *NoTopoSuite) {
+    	s.SkipUnlessLeakCheck()  // test is excluded from usual test run
+    	vpp := s.GetContainerByName("vpp").VppInstance
+    	/* do your configuration here */
+    	vpp.Disconnect()  // no goVPP less noise
+    	vpp.EnableMemoryTrace()  // enable memory traces
+    	traces1, err := vpp.GetMemoryTrace()  // get first sample
+    	s.AssertNil(err, fmt.Sprint(err))
+    	vpp.Vppctl("test mem-leak")  // execute some action
+    	traces2, err := vpp.GetMemoryTrace()  // get second sample
+    	s.AssertNil(err, fmt.Sprint(err))
+    	vpp.MemLeakCheck(traces1, traces2)  // compare samples and generate report
+    }
+
+To get your memory leak report run following command:
+
+::
+
+    $ make test-leak TEST=MemLeakTest
+    ...
+    NoTopoSuiteSolo mem_leak_test.go/MemLeakTest [SOLO]
+    /home/matus/vpp/extras/hs-test/infra/suite_no_topo.go:113
+
+      Report Entries >>
+
+      SUMMARY: 112 byte(s) leaked in 1 allocation(s)
+       - /home/matus/vpp/extras/hs-test/infra/vppinstance.go:624 @ 07/19/24 15:53:33.539
+
+        leak of 112 byte(s) in 1 allocation(s) from:
+            #0 clib_mem_heap_alloc_aligned + 0x31
+            #1 _vec_alloc_internal + 0x113
+            #2 _vec_validate + 0x81
+            #3 leak_memory_fn + 0x4f
+            #4 0x7fc167815ac3
+            #5 0x7fc1678a7850
+      << Report Entries
+    ------------------------------
+
+**External dependencies**
 
 * Linux tools ``ip``, ``brctl``
 * Standalone programs ``wget``, ``iperf3`` - since these are downloaded when Docker image is made,

extras/hs-test/hs_test.sh

@@ -7,8 +7,10 @@ single_test=0
 persist_set=0
 unconfigure_set=0
 debug_set=0
+leak_check_set=0
 debug_build=
 ginkgo_args=
+tc_name=
 
 for i in "$@"
 do
@@ -74,6 +76,13 @@ case "${i}" in
             args="$args -cpu0"
         fi
         ;;
+    --leak_check=*)
+        leak_check="${i#*=}"
+        if [ "$leak_check" = "true" ]; then
+            args="$args -leak_check"
+            leak_check_set=1
+        fi
+        ;;
 esac
 done
 
@@ -97,6 +106,16 @@ if [ $single_test -eq 0 ] && [ $debug_set -eq 1 ]; then
     exit 1
 fi
 
+if [ $leak_check_set -eq 1 ]; then
+  if [ $single_test -eq 0 ]; then
+    echo "a single test has to be specified when leak_check is set"
+    exit 1
+  fi
+  ginkgo_args="--focus $tc_name"
+  sudo -E go run github.com/onsi/ginkgo/v2/ginkgo $ginkgo_args -- $args
+  exit 0
+fi
+
 mkdir -p summary
 # shellcheck disable=SC2086
 sudo -E go run github.com/onsi/ginkgo/v2/ginkgo --no-color --trace --json-report=summary/report.json $ginkgo_args -- $args

extras/hs-test/infra/container.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"github.com/docker/go-units"
 	"os"
 	"os/exec"
 	"slices"
@@ -15,7 +16,6 @@ import (
 	containerTypes "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/image"
 	"github.com/docker/docker/pkg/stdcopy"
-	"github.com/docker/go-units"
 	"github.com/edwarnicke/exechelper"
 	. "github.com/onsi/ginkgo/v2"
 )
@@ -382,6 +382,11 @@ func (c *Container) CreateFile(destFileName string, content string) error {
 	return nil
 }
 
+func (c *Container) GetFile(sourceFileName, targetFileName string) error {
+	cmd := exec.Command("docker", "cp", c.Name+":"+sourceFileName, targetFileName)
+	return cmd.Run()
+}
+
 /*
  * Executes in detached mode so that the started application can continue to run
  * without blocking execution of test

extras/hs-test/infra/hst_suite.go

@@ -35,6 +35,7 @@ var NConfiguredCpus = flag.Int("cpus", 1, "number of CPUs assigned to vpp")
 var VppSourceFileDir = flag.String("vppsrc", "", "vpp source file directory")
 var IsDebugBuild = flag.Bool("debug_build", false, "some paths are different with debug build")
 var UseCpu0 = flag.Bool("cpu0", false, "use cpu0")
+var IsLeakCheck = flag.Bool("leak_check", false, "run leak-check tests")
 var NumaAwareCpuAlloc bool
 var SuiteTimeout time.Duration
 
@@ -285,6 +286,11 @@ func (s *HstSuite) SkipUnlessExtendedTestsBuilt() {
 	}
 }
 
+func (s *HstSuite) SkipUnlessLeakCheck() {
+	if !*IsLeakCheck {
+		s.Skip("leak-check tests excluded")
+	}
+}
 func (s *HstSuite) ResetContainers() {
 	for _, container := range s.StartedContainers {
 		container.stop()

extras/hs-test/infra/vppinstance.go

@@ -2,6 +2,7 @@ package hst
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"go.fd.io/govpp/binapi/ethernet_types"
 	"io"
@@ -97,6 +98,13 @@ type VppCpuConfig struct {
 	SkipCores          int
 }
 
+type VppMemTrace struct {
+	Count     int      `json:"count"`
+	Size      int      `json:"bytes"`
+	Sample    string   `json:"sample"`
+	Traceback []string `json:"traceback"`
+}
+
 func (vpp *VppInstance) getSuite() *HstSuite {
 	return vpp.Container.Suite
 }
@@ -535,3 +543,83 @@ func (vpp *VppInstance) generateVPPCpuConfig() string {
 
 	return c.Close().ToString()
 }
+
+// EnableMemoryTrace enables memory traces of VPP main-heap
+func (vpp *VppInstance) EnableMemoryTrace() {
+	vpp.getSuite().Log(vpp.Vppctl("memory-trace on main-heap"))
+}
+
+// GetMemoryTrace dumps memory traces for analysis
+func (vpp *VppInstance) GetMemoryTrace() ([]VppMemTrace, error) {
+	var trace []VppMemTrace
+	vpp.getSuite().Log(vpp.Vppctl("save memory-trace trace.json"))
+	err := vpp.Container.GetFile("/tmp/trace.json", "/tmp/trace.json")
+	if err != nil {
+		return nil, err
+	}
+	fileBytes, err := os.ReadFile("/tmp/trace.json")
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(fileBytes, &trace)
+	if err != nil {
+		return nil, err
+	}
+	return trace, nil
+}
+
+// memTracesSuppressCli filter out CLI related samples
+func memTracesSuppressCli(traces []VppMemTrace) []VppMemTrace {
+	var filtered []VppMemTrace
+	for i := 0; i < len(traces); i++ {
+		isCli := false
+		for j := 0; j < len(traces[i].Traceback); j++ {
+			if strings.Contains(traces[i].Traceback[j], "unix_cli") {
+				isCli = true
+				break
+			}
+		}
+		if !isCli {
+			filtered = append(filtered, traces[i])
+		}
+	}
+	return filtered
+}
+
+// MemLeakCheck compares memory traces at different point in time, analyzes if memory leaks happen and produces report
+func (vpp *VppInstance) MemLeakCheck(first, second []VppMemTrace) {
+	totalBytes := 0
+	totalCounts := 0
+	trace1 := memTracesSuppressCli(first)
+	trace2 := memTracesSuppressCli(second)
+	report := ""
+	for i := 0; i < len(trace2); i++ {
+		match := false
+		for j := 0; j < len(trace1); j++ {
+			if trace1[j].Sample == trace2[i].Sample {
+				if trace2[i].Size > trace1[j].Size {
+					deltaBytes := trace2[i].Size - trace1[j].Size
+					deltaCounts := trace2[i].Count - trace1[j].Count
+					report += fmt.Sprintf("grow %d byte(s) in %d allocation(s) from:\n", deltaBytes, deltaCounts)
+					for j := 0; j < len(trace2[i].Traceback); j++ {
+						report += fmt.Sprintf("\t#%d %s\n", j, trace2[i].Traceback[j])
+					}
+					totalBytes += deltaBytes
+					totalCounts += deltaCounts
+				}
+				match = true
+				break
+			}
+		}
+		if !match {
+			report += fmt.Sprintf("\nleak of %d byte(s) in %d allocation(s) from:\n", trace2[i].Size, trace2[i].Count)
+			for j := 0; j < len(trace2[i].Traceback); j++ {
+				report += fmt.Sprintf("\t#%d %s\n", j, trace2[i].Traceback[j])
+			}
+			totalBytes += trace2[i].Size
+			totalCounts += trace2[i].Count
+		}
+	}
+	summary := fmt.Sprintf("\nSUMMARY: %d byte(s) leaked in %d allocation(s)\n", totalBytes, totalCounts)
+	AddReportEntry(summary, report)
+}

extras/hs-test/mem_leak_test.go

@@ -0,0 +1,24 @@
+package main
+
+import (
+	. "fd.io/hs-test/infra"
+	"fmt"
+)
+
+func init() {
+	RegisterNoTopoSoloTests(MemLeakTest)
+}
+
+func MemLeakTest(s *NoTopoSuite) {
+	s.SkipUnlessLeakCheck()
+	vpp := s.GetContainerByName("vpp").VppInstance
+	/* no goVPP less noise */
+	vpp.Disconnect()
+	vpp.EnableMemoryTrace()
+	traces1, err := vpp.GetMemoryTrace()
+	s.AssertNil(err, fmt.Sprint(err))
+	vpp.Vppctl("test mem-leak")
+	traces2, err := vpp.GetMemoryTrace()
+	s.AssertNil(err, fmt.Sprint(err))
+	vpp.MemLeakCheck(traces1, traces2)
+}