ligang/vpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

vmxnet3: auto bind support

Browse Source 
For creating the vmxnet3 interface, add the bind option to automatically bind
the pci to vfio-pci module which removes the need for manual bind. Manual bind
still works, should people prefer to go that route.

Change-Id: Ife75926f8755d754a08dd0ecff0f1de326ad5ba1
Signed-off-by: Steven Luong <sluong@cisco.com>
This commit is contained in:
Steven Luong
2019-03-14 09:57:09 -07:00
committed by Damjan Marion
parent a990a2e493
commit ee8ba6877e
7 changed files with 63 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/plugins/vmxnet3/README.md
+14 -7
View File
@@ -7,11 +7,11 @@ This plugin provides native PCI driver support for VMWare vmxnet3.
* This code is tested with vfio-pci driver installed with Ubuntu 18.04 which
has kernel version 4.15.0-33-generic.
* This code is tested with ESXi vSwitch version 6.0, release build 3620759.
* This driver is tested with ESXi vSwitch version 6.5/6.7 for LRO/TSO support, VMware Workstation 15 Pro (no LRO/TSO), and VMware Fusion 11 Pro (no LRO/TSO)
* Driver requires MSI-X interrupt support, which is not supported by
uio_pci_generic driver, so vfio-pci needs to be used. On systems without IOMMU
vfio driver can still be used with recent kernels which support no-iommu mode.
uio_pci_generic driver. So vfio-pci must be used. On systems without IOMMU,
vfio driver can still be used with 4.15.0-33-generic kernel (Ubuntu 18.04) which supports no-iommu mode.
##Known issues
@@ -25,20 +25,27 @@ vfio driver can still be used with recent kernels which support no-iommu mode.
sudo modprobe vfio-pci
```
2. (systems without IOMMU only) enable unsafe NOIOMMU mode
2. Make sure the interface is down
```
sudo ifconfig <if-name> down
```
Steps 3 and 4 are optional. They can be accomplished by specifying the optional keyword "bind" when creating the vmxnet3 interface.
3. (systems without IOMMU only) enable unsafe NOIOMMU mode
```
echo Y | sudo tee /sys/module/vfio/parameters/enable_unsafe_noiommu_mode
```
3. Bind interface to vfio-pci
4. Bind interface to vfio-pci
```
sudo dpdk-devbind.py --bind vfio-pci 0b:00.0
```
### Interface Creation
Interface can be dynamically created with following CLI:
Interface can be dynamically created with following CLI, with or without the bind option. If step 3 and 4 were executed, bind can be omitted.
```
create interface vmxnet3 0000:0b:00.0
create interface vmxnet3 0000:0b:00.0 bind
set int state vmxnet3-0/b/0/0 up
```
src/plugins/vmxnet3/cli.c
+3 -1
View File
@@ -44,6 +44,8 @@ vmxnet3_create_command_fn (vlib_main_t * vm, unformat_input_t * input,
;
else if (unformat (line_input, "elog"))
args.enable_elog = 1;
else if (unformat (line_input, "bind"))
args.bind = 1;
else if (unformat (line_input, "rx-queue-size %u", &args.rxq_size))
;
else if (unformat (line_input, "tx-queue-size %u", &args.txq_size))
@@ -69,7 +71,7 @@ VLIB_CLI_COMMAND (vmxnet3_create_command, static) = {
.path = "create interface vmxnet3",
.short_help = "create interface vmxnet3 <pci-address>"
"[rx-queue-size <size>] [tx-queue-size <size>]"
"[num-tx-queues <number>] [num-rx-queues <number>]",
"[num-tx-queues <number>] [num-rx-queues <number>] [bind]",
.function = vmxnet3_create_command_fn,
};
/* *INDENT-ON* */
src/plugins/vmxnet3/vmxnet3.api
+3
View File
@@ -28,6 +28,8 @@ option version = "1.0.0";
@param rxq_num - number of receive queues (optional - default is 1)
@param txq_size - transmit queue size (optional - default is 1024)
@param txq_num - number of transmit queues (optional - default is 1)
@param bind - automatically bind PCI to vfio-pci module
(optional - default is 0)
*/
define vmxnet3_create
@@ -41,6 +43,7 @@ define vmxnet3_create
u16 rxq_num;
u16 txq_size;
u16 txq_num;
u8 bind;
};
/** \brief
src/plugins/vmxnet3/vmxnet3.c
+32 -14
View File
@@ -658,9 +658,10 @@ vmxnet3_create_if (vlib_main_t * vm, vmxnet3_create_if_args_t * args)
pool_foreach (vd, vmxm->devices, ({
if (vd->pci_addr.as_u32 == args->addr.as_u32)
{
args->rv = VNET_API_ERROR_INVALID_VALUE;
args->rv = VNET_API_ERROR_ADDRESS_IN_USE;
args->error =
clib_error_return (error, "PCI address in use");
clib_error_return (error, "%U: %s", format_vlib_pci_addr,
&args->addr, "pci address in use");
vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
format_vlib_pci_addr, &args->addr, "pci address in use");
return;
@@ -668,24 +669,31 @@ vmxnet3_create_if (vlib_main_t * vm, vmxnet3_create_if_args_t * args)
}));
/* *INDENT-ON* */
pool_get (vmxm->devices, vd);
vd->num_tx_queues = args->txq_num;
vd->num_rx_queues = args->rxq_num;
vd->dev_instance = vd - vmxm->devices;
vd->per_interface_next_index = ~0;
vd->pci_addr = args->addr;
if (args->enable_elog)
vd->flags |= VMXNET3_DEVICE_F_ELOG;
if (args->bind)
{
error = vlib_pci_bind_to_uio (vm, &args->addr, (char *) "auto");
if (error)
{
args->rv = VNET_API_ERROR_INVALID_INTERFACE;
args->error =
clib_error_return (error, "%U: %s", format_vlib_pci_addr,
&args->addr,
"error encountered on binding pci device");
vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
format_vlib_pci_addr, &args->addr,
"error encountered on binding pci devicee");
return;
}
}
if ((error =
vlib_pci_device_open (vm, &args->addr, vmxnet3_pci_device_ids, &h)))
{
pool_put (vmxm->devices, vd);
args->rv = VNET_API_ERROR_INVALID_INTERFACE;
args->error =
clib_error_return (error, "pci-addr %U", format_vlib_pci_addr,
&args->addr);
clib_error_return (error, "%U: %s", format_vlib_pci_addr,
&args->addr,
"error encountered on pci device open");
vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
format_vlib_pci_addr, &args->addr,
"error encountered on pci device open");
@@ -696,6 +704,16 @@ vmxnet3_create_if (vlib_main_t * vm, vmxnet3_create_if_args_t * args)
* Do not use vmxnet3_log_error prior to this line since the macro
* references vd->pci_dev_handle
*/
pool_get (vmxm->devices, vd);
vd->num_tx_queues = args->txq_num;
vd->num_rx_queues = args->rxq_num;
vd->dev_instance = vd - vmxm->devices;
vd->per_interface_next_index = ~0;
vd->pci_addr = args->addr;
if (args->enable_elog)
vd->flags |= VMXNET3_DEVICE_F_ELOG;
vd->pci_dev_handle = h;
vd->numa_node = vlib_pci_get_numa_node (vm, h);
vd->num_intrs = vd->num_rx_queues + 1; // +1 for the event interrupt
src/plugins/vmxnet3/vmxnet3.h
+1
View File
@@ -605,6 +605,7 @@ typedef struct
u16 rxq_num;
u16 txq_size;
u16 txq_num;
u8 bind;
/* return */
i32 rv;
u32 sw_if_index;
src/plugins/vmxnet3/vmxnet3_api.c
+2
View File
@@ -72,6 +72,8 @@ vl_api_vmxnet3_create_t_handler (vl_api_vmxnet3_create_t * mp)
args.rxq_size = ntohs (mp->rxq_size);
args.txq_size = ntohs (mp->txq_size);
args.txq_num = ntohs (mp->txq_num);
args.rxq_num = ntohs (mp->rxq_num);
args.bind = mp->bind;
vmxnet3_create_if (vm, &args);
rv = args.rv;
src/plugins/vmxnet3/vmxnet3_test.c
+8 -1
View File
@@ -117,12 +117,16 @@ api_vmxnet3_create (vat_main_t * vam)
}
else if (unformat (i, "elog"))
args.enable_elog = 1;
else if (unformat (i, "bind"))
args.bind = 1;
else if (unformat (i, "rx-queue-size %u", &args.rxq_size))
;
else if (unformat (i, "tx-queue-size %u", &args.txq_size))
;
else if (unformat (i, "num-tx-queues %u", &args.txq_num))
;
else if (unformat (i, "num-rx-queues %u", &args.rxq_num))
;
else
{
clib_warning ("unknown input '%U'", format_unformat_error, i);
@@ -137,6 +141,8 @@ api_vmxnet3_create (vat_main_t * vam)
mp->rxq_size = clib_host_to_net_u16 (args.rxq_size);
mp->txq_size = clib_host_to_net_u16 (args.txq_size);
mp->txq_num = clib_host_to_net_u16 (args.txq_num);
mp->rxq_num = clib_host_to_net_u16 (args.rxq_num);
mp->bind = args.bind;
S (mp);
W (ret);
@@ -289,7 +295,8 @@ vl_api_vmxnet3_details_t_handler (vl_api_vmxnet3_details_t * mp)
*/
#define foreach_vpe_api_msg \
_(vmxnet3_create, "<pci-address> [rx-queue-size <size>] " \
"[tx-queue-size <size>] [num-tx-queues <num>]") \
"[tx-queue-size <size>] [num-tx-queues <num>]" \
"[num-rx-queues <num>] [bind]") \
_(vmxnet3_delete, "sw_if_index <sw_if_index>") \
_(vmxnet3_dump, "")