tests: run a test inside a QEMU VM

Use the script test/run.py to run a test named test_vm_tap
inside a QEMU VM. The run script builds out a virtual env,
launches a light weight QEMU VM, mounts host directories,
starts VPP inside the VM and runs the test. The test named
test_vm_tap, creates two tap v2 interfaces in separate Linux
namespaces and using iPerf, streams traffic between the VM
and VPP. All data files are stored in the directory named
/tmp/vpp-vm-tests. To clean up, use the make test-wipe
command.
Usage:
test/run.py --vm --debug --test test_vm_tap

Type: improvement

Change-Id: I4425dbef52acee1e5b8af5acaa169b89a2c0f171
Signed-off-by: Naveen Joy <najoy@cisco.com>

test/Makefile

@@ -310,6 +310,7 @@ reset:
 	@if [ $(FORCE_NO_WIPE) -eq "0" ] ; then rm -rf /tmp/vpp-unittest-*;  fi
 	@rm -f /tmp/api_post_mortem.*
 	@rm -rf $(FAILED_DIR)
+	@rm -rf /tmp/vpp-vm-tests
 
 .PHONY: wipe
 wipe: reset

test/scripts/run_vpp_in_vm.sh

@@ -0,0 +1,198 @@
+#!/usr/bin/env bash
+# Run VPP in a QEMU VM
+# set -o xtrace
+set -o nounset
+
+# Arguments:
+# $1:- Test Filter
+# $2:- Kernel Image
+# $3:- Test Data Directory
+# $4:- CPU Mask String (e.g. "5,6,7,8")
+# $5:- Guest MEM in Gibibytes (e.g. 2G)
+
+if [[ -z "${1:-}" ]]; then
+    echo "ERROR: A non-empty test selection is required to run
+    tests in a QEMU VM"
+    exit 1
+fi
+TEST=${1:-}
+TEST_JOBS=${TEST_JOBS:-1}
+
+# Init RAM disk image to boot the QEMU VM
+INITRD=${INITRD:-}
+
+# Ensure test dir
+TEST_DATA_DIR=${3:-"/tmp/vpp-vm-tests"}
+if [[ ! -d ${TEST_DATA_DIR} ]]; then
+    mkdir -p ${TEST_DATA_DIR}
+fi
+
+# CPU Affinity for taskset
+CPU_MASK=${4:-"5,6,7,8"}
+IFS=',' read -r -a CPU_MASK_ARRAY <<< ${CPU_MASK}
+CPUS=${#CPU_MASK_ARRAY[@]}
+
+# Guest MEM (Default 2G)
+MEM=${5:-"2G"}
+
+# Set the QEMU executable for the OS pkg.
+os_VENDOR=$(lsb_release -i -s)
+if [[ $os_VENDOR =~ (Debian|Ubuntu) ]]; then
+    os_PACKAGE="deb"
+    QEMU=${QEMU:-"qemu-system-x86_64"}
+else
+    os_PACKAGE="rpm"
+    QEMU=${QEMU:-"qemu-kvm"}
+fi
+
+# Exit if the ${QEMU} executable is not available
+if ! command -v ${QEMU} &> /dev/null; then
+    echo "Error: ${QEMU} is required, but could not be found."
+    exit 1
+fi
+
+# Download the Generic Linux Kernel, if needed
+if [[ -z "${2:-}" ]] || [[ ! -f "${2:-}" ]]; then
+    if [[ $os_PACKAGE == "deb" ]]; then
+        PWD=$(pwd)
+        cd ${TEST_DATA_DIR}
+        PKG="linux-image-$(uname -r)"
+        echo "Getting the Linux Kernel image..${PKG}"
+        apt-get download ${PKG}
+        dpkg --fsys-tarfile ${PKG}_*.deb | tar xvf - ./boot
+        KERNEL_BIN=$(ls ${TEST_DATA_DIR}/boot/vmlinuz-*-generic)
+        cd ${PWD}
+    else
+        echo "ERROR: Kernel Image selection is required for RPM pkgs."
+        exit 1
+    fi
+else
+    KERNEL_BIN=${2:-}
+fi
+
+## Create initrd with 9p drivers, if ${INITRD} is null
+DRIVERS_9P=""
+if [[ -z "${INITRD}" ]] && [[ ! -d "/etc/initramfs-tools" ]]; then
+   echo "To boot the QEMU VM, an initial RAM disk with 9p drivers is needed"
+   echo "Install the initramfs-tools package or set env var INITRD to the RAM disk path"
+   exit 1
+elif [[ -z "${INITRD}" ]]; then
+   if [[ -f "/etc/initramfs-tools/modules" ]]; then
+       DRIVERS_9P=$(grep 9p /etc/initramfs-tools/modules | awk '{print $1}' | cut -d$'\n' -f1)
+   fi
+   if [[ -z "${DRIVERS_9P}" ]]; then
+       echo "You'll need to update the file /etc/initramfs-tools/modules with the below 9p drivers"
+       echo "9p >> /etc/initramfs-tools/modules"
+       echo "9pnet >> /etc/initramfs-tools/modules"
+       echo "9pnet_virtio >> /etc/initramfs-tools/modules"
+       exit 1
+   fi
+   # Generate the initramfs image, if the we haven't generated one yet
+   if ! ls ${TEST_DATA_DIR}/boot/initrd.img-*-generic &> /dev/null; then
+       echo "Generating a bootable initramfs image in ${TEST_DATA_DIR}/boot/"
+       update-initramfs -c -k $(uname -r) -b ${TEST_DATA_DIR}/boot >/dev/null 2>&1
+       echo "Generated the INITRD image"
+   fi
+   INITRD=$(ls ${TEST_DATA_DIR}/boot/initrd.img-*-generic)
+fi
+echo "Using INITRD=${TEST_DATA_DIR}/boot/${INITRD} for booting the QEMU VM"
+
+
+## Install iperf into ${TEST_DATA_DIR}
+IPERF=${TEST_DATA_DIR}/usr/bin/iperf
+if [[ ! -x ${IPERF} ]] && [[ $os_PACKAGE == "deb" ]]; then
+    echo "Installing iperf: ${IPERF}"
+    PWD=$(pwd)
+    cd ${TEST_DATA_DIR}
+    IPRF_PKG="iperf_2.0.5+dfsg1-2_amd64.deb"
+    wget https://iperf.fr/download/ubuntu/${IPRF_PKG}
+    dpkg --fsys-tarfile ${IPRF_PKG} | tar xvf -
+    if [[ -x ${IPERF} ]]; then
+       echo "${IPERF} installed successfully"
+    else
+       echo "ERROR: iperf executable ${IPERF} installation failed"
+       exit 1
+    fi
+    cd ${PWD}
+elif [[ ! -x ${IPERF} ]] && [[ $os_PACKAGE != "deb" ]]; then
+    echo "ERROR: install iperf: ${IPERF} before running QEMU tests"
+    exit 1
+fi
+
+FAILED_DIR=${FAILED_DIR:-"/tmp/vpp-failed-unittests/"}
+if [[ ! -d ${FAILED_DIR} ]]; then
+    mkdir -p ${FAILED_DIR}
+fi
+
+HUGEPAGES=${HUGEPAGES:-256}
+
+# Ensure all required Env vars are bound to non-zero values
+EnvVarArray=("WS_ROOT=${WS_ROOT:-}"
+             "RND_SEED=${RND_SEED:-}"
+             "BR=${BR:-}"
+             "VENV_PATH=${VENV_PATH:-}"
+             "VPP_BUILD_DIR=${VPP_BUILD_DIR:-}"
+             "VPP_BIN=${VPP_BIN:-}"
+             "VPP_PLUGIN_PATH=${VPP_PLUGIN_PATH:-}"
+             "VPP_TEST_PLUGIN_PATH=${VPP_TEST_PLUGIN_PATH:-}"
+             "VPP_INSTALL_PATH=${VPP_INSTALL_PATH:-}"
+             "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}")
+
+for envVar in ${EnvVarArray[*]}; do
+    var_name=$(echo $envVar | cut -d= -f1)
+    var_val=$(echo $envVar | cut -d= -f2)
+    if [[ -z "$var_val" ]]; then
+        echo "ERROR: Env var: $var_name is not set"
+        exit 1
+    fi
+done
+
+# Boot QEMU VM and run the test
+function run_in_vm {
+    INIT=$(mktemp -p ${TEST_DATA_DIR})
+    cat > ${INIT} << _EOF_
+#!/bin/bash
+mkdir -p /dev/shm
+mount -t tmpfs -o rw,nosuid,nodev tmpfs /dev/shm
+mount -t devpts -o noexec,nosuid,gid=5,mode=0620 devpts /dev/pts || true
+mount -t tmpfs -o "noexec,nosuid,size=10%,mode=0755" tmpfs /run
+mount -t 9p /dev/vpp9p ${WS_ROOT}
+mount -t 9p tmp9p /tmp
+modprobe -a vhost_net
+env SOCKET=1 SANITY=no \
+FAILED_DIR=${FAILED_DIR} RND_SEED=${RND_SEED} BR=${BR} \
+VENV_PATH=${VENV_PATH} TEST=${TEST} TEST_JOBS=${TEST_JOBS} \
+VPP_BUILD_DIR=${VPP_BUILD_DIR} VPP_BIN=${VPP_BIN} VPP_PLUGIN_PATH=${VPP_PLUGIN_PATH} \
+VPP_TEST_PLUGIN_PATH=${VPP_TEST_PLUGIN_PATH} VPP_INSTALL_PATH=${VPP_INSTALL_PATH} \
+LD_LIBRARY_PATH=${LD_LIBRARY_PATH} TEST_DATA_DIR=${TEST_DATA_DIR} INITRD=${INITRD} \
+bash -c "${WS_ROOT}/test/scripts/run.sh --filter=${TEST} --jobs=${TEST_JOBS} --failed-dir=${FAILED_DIR} \
+--venv-dir=${VENV_PATH} --vpp-ws-dir=${WS_ROOT} --extended"
+poweroff -f
+_EOF_
+
+    chmod +x ${INIT}
+
+    sudo taskset -c ${CPU_MASK} ${QEMU} \
+                 -nodefaults \
+                 -name test_$(basename $INIT) \
+                 -chardev stdio,mux=on,id=char0 \
+                 -mon chardev=char0,mode=readline,pretty=on \
+                 -serial chardev:char0 \
+                 -machine pc,accel=kvm,usb=off,mem-merge=off \
+                 -cpu host \
+                 -smp ${CPUS},sockets=1,cores=${CPUS},threads=1 \
+                 -m ${MEM} \
+                 -no-user-config \
+                 -kernel ${KERNEL_BIN} \
+                 -initrd ${INITRD} \
+                 -fsdev local,id=root9p,path=/,security_model=none,multidevs=remap \
+                 -device virtio-9p-pci,fsdev=root9p,mount_tag=fsRoot \
+                 -virtfs local,path=${WS_ROOT},mount_tag=/dev/vpp9p,security_model=none,id=vpp9p,multidevs=remap \
+                 -virtfs local,path=/tmp,mount_tag=tmp9p,security_model=passthrough,id=tmp9p,multidevs=remap \
+                 -netdev tap,id=net0,vhost=on \
+                 -device virtio-net-pci,netdev=net0,mac=52:54:00:de:64:01 \
+                 -nographic \
+                 -append "ro root=fsRoot rootfstype=9p rootflags=trans=virtio,cache=mmap console=ttyS0 hugepages=${HUGEPAGES} init=${INIT}"
+    }
+
+run_in_vm

test/test_vm_tap.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+import unittest
+from ipaddress import ip_interface
+from vpp_qemu_utils import create_namespace
+from vpp_iperf import VppIperf
+from framework import VppTestCase, VppTestRunner
+from config import config
+
+
+class TestTapQemu(VppTestCase):
+    """Test Tap interfaces inside a QEMU VM.
+
+    Start an iPerf connection stream between QEMU and VPP via
+    tap v2 interfaces.
+
+    Linux_ns1 -- iperf_client -- tap1 -- VPP-BD -- tap2 --
+                              -- iperfServer -- Linux_ns2
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        super(TestTapQemu, cls).setUpClass()
+
+    @classmethod
+    def tearDownClass(cls):
+        super(TestTapQemu, cls).tearDownClass()
+
+    def setUp(self):
+        """Perform test setup before running QEMU tests.
+
+        1. Create a namespace for the iPerf Server & Client.
+        2. Create 2 tap interfaces in VPP & add them to each namespace.
+        3. Add the tap interfaces to a bridge-domain.
+        """
+        super(TestTapQemu, self).setUp()
+        self.client_namespace = "iprf_client_ns"
+        self.server_namespace = "iprf_server_ns"
+        self.client_ip4_prefix = "10.0.0.101/24"
+        self.server_ip4_prefix = "10.0.0.102/24"
+        create_namespace(self.client_namespace)
+        create_namespace(self.server_namespace)
+        tap1_if_idx = self.create_tap(
+            101, self.client_namespace, self.client_ip4_prefix
+        )
+        tap2_if_idx = self.create_tap(
+            102, self.server_namespace, self.server_ip4_prefix
+        )
+        self.l2_connect_interfaces(tap1_if_idx, tap2_if_idx)
+
+    def create_tap(self, id, host_namespace, host_ip4_prefix):
+        result = self.vapi.api(
+            self.vapi.papi.tap_create_v2,
+            {
+                "id": id,
+                "use_random_mac": True,
+                "host_namespace_set": True,
+                "host_namespace": host_namespace,
+                "host_if_name_set": False,
+                "host_bridge_set": False,
+                "host_mac_addr_set": False,
+                "host_ip4_prefix": ip_interface(host_ip4_prefix),
+                "host_ip4_prefix_set": True,
+            },
+        )
+        sw_if_index = result.sw_if_index
+        self.vapi.api(
+            self.vapi.papi.sw_interface_set_flags,
+            {"sw_if_index": sw_if_index, "flags": 1},
+        )
+        return sw_if_index
+
+    def dump_vpp_tap_interfaces(self):
+        return self.vapi.api(self.vapi.papi.sw_interface_tap_v2_dump, {})
+
+    def dump_bridge_domain_details(self):
+        return self.vapi.api(self.vapi.papi.bridge_domain_dump, {"bd_id": 1})
+
+    def l2_connect_interfaces(self, *sw_if_idxs):
+        for if_idx in sw_if_idxs:
+            self.vapi.api(
+                self.vapi.papi.sw_interface_set_l2_bridge,
+                {
+                    "rx_sw_if_index": if_idx,
+                    "bd_id": 1,
+                    "shg": 0,
+                    "port_type": 0,
+                    "enable": True,
+                },
+            )
+
+    @unittest.skipUnless(config.extended, "part of extended tests")
+    def test_tap_iperf(self):
+        """Start an iperf connection stream between QEMU & VPP via tap."""
+        iperf = VppIperf()
+        iperf.client_ns = self.client_namespace
+        iperf.server_ns = self.server_namespace
+        iperf.server_ip = str(ip_interface(self.server_ip4_prefix).ip)
+        iperf.start()
+
+
+if __name__ == "__main__":
+    unittest.main(testRunner=VppTestRunner)

test/vpp_iperf.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+
+# Start an iPerf connection stream between two Linux namespaces ##
+
+import subprocess
+import os
+
+
+class VppIperf:
+    """ "Create an iPerf connection stream between two namespaces.
+
+    Usage:
+    iperf = VppIperf()                   # Create the iPerf Object
+    iperf.client_ns = 'ns1'              # Client Namespace
+    iperf.server_ns = 'ns2'              # Server Namespace
+    iperf.server_ip = '10.0.0.102'       # Server IP Address
+    iperf.start()                        # Start the connection stream
+
+    Optional:
+    iperf.duration = 15   # Time to transmit for in seconds (Default=10)
+
+    ## Optionally set any iperf client & server args
+    Example:
+    # Run 4 parallel streams, write to logfile & bind to port 5202
+    iperf.client_args='-P 4 --logfile /tmp/vpp-vm-tests/vpp_iperf.log -p 5202'
+    iperf.server_args='-p 5202'
+    """
+
+    def __init__(self, server_ns=None, client_ns=None, server_ip=None):
+        self.server_ns = server_ns
+        self.client_ns = client_ns
+        self.server_ip = server_ip
+        self.duration = 10
+        self.client_args = ""
+        self.server_args = ""
+        # Set the iperf executable
+        self.iperf = os.path.join(os.getenv("TEST_DATA_DIR") or "/", "usr/bin/iperf")
+
+    def ensure_init(self):
+        if self.server_ns and self.client_ns and self.server_ip:
+            return True
+        else:
+            raise Exception(
+                "Error: Cannot Start." "iPerf object has not been initialized"
+            )
+
+    def start_iperf_server(self):
+        print("Starting iPerf Server Daemon in Namespace ", self.server_ns)
+        args = [
+            "ip",
+            "netns",
+            "exec",
+            self.server_ns,
+            self.iperf,
+            "-s",
+            "-D",
+            "-B",
+            self.server_ip,
+        ]
+        args.extend(self.server_args.split())
+        try:
+            subprocess.run(
+                args,
+                stderr=subprocess.STDOUT,
+                timeout=self.duration + 5,
+                encoding="utf-8",
+            )
+        except subprocess.TimeoutExpired as e:
+            raise Exception("Error: Timeout expired for iPerf", e.output)
+
+    def start_iperf_client(self):
+        print("Starting iPerf Client in Namespace ", self.client_ns)
+        args = [
+            "ip",
+            "netns",
+            "exec",
+            self.client_ns,
+            self.iperf,
+            "-c",
+            self.server_ip,
+            "-t",
+            str(self.duration),
+        ]
+        args.extend(self.client_args.split())
+        try:
+            subprocess.run(
+                args,
+                stderr=subprocess.STDOUT,
+                timeout=self.duration + 5,
+                encoding="utf-8",
+            )
+        except subprocess.TimeoutExpired as e:
+            raise Exception("Error: Timeout expired for iPerf", e.output)
+
+    def start(self):
+        """Run iPerf and return True if successful"""
+        self.ensure_init()
+        try:
+            self.start_iperf_server()
+        except Exception as e:
+            subprocess.run(["pkill", "iperf"])
+            raise Exception("Error starting iPerf Server", e)
+
+        try:
+            self.start_iperf_client()
+        except Exception as e:
+            raise Exception("Error starting iPerf Client", e)
+        subprocess.run(["pkill", "iperf"])
+
+
+if __name__ == "__main__":
+    # Run iPerf using default settings
+    iperf = VppIperf()
+    iperf.client_ns = "ns1"
+    iperf.server_ns = "ns2"
+    iperf.server_ip = "10.0.0.102"
+    iperf.duration = 20
+    iperf.start()

test/vpp_qemu_utils.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+
+# Utility functions for QEMU tests ##
+
+import subprocess
+
+
+def create_namespace(ns):
+    try:
+        subprocess.run(["ip", "netns", "add", ns])
+    except subprocess.CalledProcessError as e:
+        raise Exception("Error creating namespace:", e.output)
+
+
+def list_namespace(ns):
+    """List the IP address of a namespace"""
+    try:
+        subprocess.run(["ip", "netns", "exec", ns, "ip", "addr"])
+    except subprocess.CalledProcessError as e:
+        raise Exception("Error listing namespace IP:", e.output)