Intel Adaptive Virtual Function native device driver plugin

Change-Id: If168a9c54baaa516ecbe78de2141f11c17aa2f53
Signed-off-by: Damjan Marion <damarion@cisco.com>

src/configure.ac

@@ -211,6 +211,7 @@ AC_SUBST(AR_FLAGS)
 
 # Please keep alphabetical order
 PLUGIN_ENABLED(acl)
+PLUGIN_ENABLED(avf)
 PLUGIN_ENABLED(cdp)
 PLUGIN_ENABLED(dpdk)
 PLUGIN_ENABLED(flowprobe)

src/plugins/Makefile.am

@@ -34,6 +34,10 @@ if ENABLE_ACL_PLUGIN
 include acl.am
 endif
 
+if ENABLE_AVF_PLUGIN
+include avf.am
+endif
+
 if ENABLE_CDP_PLUGIN
 include cdp.am
 endif

src/plugins/avf.am

@@ -0,0 +1,57 @@
+# Copyright (c) 2018 Cisco Systems, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+vppplugins_LTLIBRARIES += avf_plugin.la
+
+avf_plugin_la_LIBADD =
+avf_plugin_la_SOURCES = \
+	avf/cli.c			\
+	avf/device.c			\
+	avf/format.c			\
+	avf/input.c			\
+	avf/output.c			\
+	avf/plugin.c
+
+noinst_HEADERS += avf/avf.h
+
+if CPU_X86_64
+avf_multiversioning_files = 					\
+	avf/input.c						\
+	avf/output.c
+
+if CC_SUPPORTS_AVX2
+###############################################################
+# AVX2
+###############################################################
+libavf_plugin_avx2_la_SOURCES = $(avf_multiversioning_files)
+libavf_plugin_avx2_la_CFLAGS =				\
+	$(AM_CFLAGS)  @CPU_AVX2_FLAGS@				\
+	-DCLIB_MULTIARCH_VARIANT=avx2
+noinst_LTLIBRARIES += libavf_plugin_avx2.la
+avf_plugin_la_LIBADD += libavf_plugin_avx2.la
+endif
+
+if CC_SUPPORTS_AVX512
+###############################################################
+# AVX512
+###############################################################
+libavf_plugin_avx512_la_SOURCES = $(avf_multiversioning_files)
+libavf_plugin_avx512_la_CFLAGS =				\
+	$(AM_CFLAGS) @CPU_AVX512_FLAGS@				\
+	-DCLIB_MULTIARCH_VARIANT=avx512
+noinst_LTLIBRARIES += libavf_plugin_avx512.la
+avf_plugin_la_LIBADD += libavf_plugin_avx512.la
+endif
+endif
+
+# vi:syntax=automake

src/plugins/avf/README.md

@@ -0,0 +1,94 @@
+# Intel AVF device plugin for VPP   
+
+##Overview
+This plugins provides native device support for intel Adaptive Virtual
+Function (AVF). AVF is driver specification for current and future
+Intel Virtual Function devices. AVF defines communication channel between
+Physical Funciton (PF) and VF.
+In essence, today this driver can be used only with 
+Intel XL710 / X710 / XXV710 adapters.
+
+##Prerequisites
+ * Driver requires newer i40e PF linux driver to be installed on the system,
+which supports virtualchnl interface. This code is tested with i40e driver
+version 2.4.6.
+
+* Driver requires MSI-X interrupt support, which is not supported by
+uio_pci_generic driver, so vfio-pci needs to be used. On systems without IOMMU
+vfio driver can still be used with recent kernels which support no-iommu mode.
+
+##Known issues
+This driver is still in experimental phase, however it shows very good 
+performance numbers. Following items are not implemented (yet).
+
+* Jumbo MTU support
+* Interrupt and adaptive mode
+* NUMA support
+
+## Usage
+### System setup
+
+1. load VFIO driver
+```
+sudo modprobe vfio-pci
+```
+
+2. (systems without IOMMU only) enable unsafe NOIOMMU mode
+```
+echo Y | sudo tee /sys/module/vfio/parameters/enable_unsafe_noiommu_mode
+```
+
+3. Create and bind SR-IOV virtual function(s)
+
+Following script creates VF, assigns MAC address and binds VF to vfio-pci
+```bash
+#!/bin/bash
+
+if [ $USER != "root" ] ; then
+	echo "Restarting script with sudo..."
+	sudo $0 ${*}
+	exit
+fi
+
+setup () {
+  cd /sys/bus/pci/devices/${1}
+  driver=$(basename $(readlink driver))
+  if [ "${driver}" != "i40e" ]; then
+    echo ${1} | tee driver/unbind
+    echo ${1} | tee /sys/bus/pci/drivers/i40e/bind
+  fi
+  ifname=$(basename net/*)
+  echo 0 | tee sriov_numvfs > /dev/null
+  echo 1 | tee sriov_numvfs > /dev/null
+  ip link set dev ${ifname} vf 0 mac ${2}
+  ip link show dev ${ifname}
+  vf=$(basename $(readlink virtfn0))
+  echo ${vf} | tee virtfn0/driver/unbind
+  echo vfio-pci | tee virtfn0/driver_override
+  echo ${vf} | sudo tee /sys/bus/pci/drivers/vfio-pci/bind
+  echo  | tee virtfn0/driver_override
+}
+
+# Setup one VF on PF 0000:3b:00.0 and assign MAC address
+setup 0000:3b:00.0 00:11:22:33:44:00
+# Setup one VF on PF 0000:3b:00.1 and assign MAC address
+setup 0000:3b:00.1 00:11:22:33:44:01
+```
+
+### Interface Cration
+Interfaces can be dynamically created by using following CLI:
+```
+create interface avf 0000:3b:02.0
+set int state AVF0/3b/2/0 up
+```
+
+### Interface Deletion
+Interface can be deleted with following CLI:
+```
+delete interface avf <interface name>
+```
+
+### Interface Statistics
+Interface statistics can be displayed with `sh hardware-interface <if-name>`
+command.
+

src/plugins/avf/avf.h

@@ -0,0 +1,265 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+#include <avf/virtchnl.h>
+
+#define foreach_avf_device_flags \
+  _(0, INITIALIZED, "initialized") \
+  _(1, ERROR, "error") \
+  _(2, ADMIN_UP, "admin-up") \
+  _(3, IOVA, "iova") \
+  _(4, LINK_UP, "link-up") \
+  _(5, SHARED_TXQ_LOCK, "shared-txq-lock") \
+  _(6, ELOG, "elog")
+
+enum
+{
+#define _(a, b, c) AVF_DEVICE_F_##b = (1 << a),
+  foreach_avf_device_flags
+#undef _
+};
+
+typedef struct
+{
+  u64 qword[4];
+} avf_rx_desc_t;
+
+STATIC_ASSERT_SIZEOF (avf_rx_desc_t, 32);
+
+typedef struct
+{
+  union
+  {
+    u64 qword[2];
+    u64x2 as_u64x2;
+  };
+} avf_tx_desc_t;
+
+STATIC_ASSERT_SIZEOF (avf_tx_desc_t, 16);
+
+typedef struct
+{
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+  volatile u32 *qrx_tail;
+  u16 next;
+  u16 size;
+  avf_rx_desc_t *descs;
+  u32 *bufs;
+  u16 n_bufs;
+} avf_rxq_t;
+
+typedef struct
+{
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+  volatile u32 *qtx_tail;
+  u16 next;
+  u16 size;
+  clib_spinlock_t lock;
+  avf_tx_desc_t *descs;
+  u32 *bufs;
+  u16 n_bufs;
+} avf_txq_t;
+
+typedef struct
+{
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+  u32 flags;
+  u32 per_interface_next_index;
+
+  u32 dev_instance;
+  u32 sw_if_index;
+  u32 hw_if_index;
+  vlib_pci_dev_handle_t pci_dev_handle;
+  void *bar0;
+
+  /* queues */
+  avf_rxq_t *rxqs;
+  avf_txq_t *txqs;
+
+  /* Admin queues */
+  avf_aq_desc_t *atq;
+  avf_aq_desc_t *arq;
+  void *atq_bufs;
+  void *arq_bufs;
+  u64 atq_bufs_pa;
+  u64 arq_bufs_pa;
+  u16 atq_next_slot;
+  u16 arq_next_slot;
+  virtchnl_pf_event_t *events;
+
+  u16 vsi_id;
+  u32 feature_bitmap;
+  u8 hwaddr[6];
+  u16 num_queue_pairs;
+  u16 max_vectors;
+  u16 max_mtu;
+  u32 rss_key_size;
+  u32 rss_lut_size;
+  virtchnl_link_speed_t link_speed;
+
+  /* stats */
+  virtchnl_eth_stats_t eth_stats;
+
+  /* error */
+  clib_error_t *error;
+} avf_device_t;
+
+typedef struct
+{
+  u32 bi;
+  u32 status;
+  u16 length;
+  u16 tail_length;
+  u8 ptype;
+  u8 error;
+} avf_rx_vector_entry_t;
+
+STATIC_ASSERT_SIZEOF (avf_rx_vector_entry_t, 16);
+
+#define AVF_RX_VECTOR_SZ VLIB_FRAME_SIZE
+
+enum
+{
+  AVF_PROCESS_EVENT_START = 1,
+  AVF_PROCESS_EVENT_STOP = 2,
+  AVF_PROCESS_EVENT_AQ_INT = 3,
+} avf_process_event_t;
+
+typedef struct
+{
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+  avf_rx_vector_entry_t rx_vector[AVF_RX_VECTOR_SZ];
+  u32 *to_free;
+} avf_per_thread_data_t;
+
+typedef struct
+{
+  u8 next_node;
+  i8 buffer_advance;
+  u32 flags;
+} avf_ptype_t;
+
+STATIC_ASSERT (VNET_DEVICE_INPUT_N_NEXT_NODES < 256, "too many next nodes");
+
+typedef struct
+{
+  avf_device_t *devices;
+  avf_per_thread_data_t *per_thread_data;
+  vlib_physmem_region_index_t physmem_region;
+  int physmem_region_alloc;
+
+  /* 256 element array for ptype based lookup */
+  avf_ptype_t *ptypes;
+} avf_main_t;
+
+extern avf_main_t avf_main;
+
+typedef struct
+{
+  vlib_pci_addr_t addr;
+  int enable_elog;
+  /* return */
+  int rv;
+  clib_error_t *error;
+} avf_create_if_args_t;
+
+void avf_create_if (vlib_main_t * vm, avf_create_if_args_t * args);
+void avf_delete_if (vlib_main_t * vm, avf_device_t * ad);
+
+extern vlib_node_registration_t avf_input_node;
+extern vnet_device_class_t avf_device_class;
+uword avf_interface_tx (vlib_main_t * vm, vlib_node_runtime_t * node,
+			vlib_frame_t * frame);
+
+/* format.c */
+format_function_t format_avf_device;
+format_function_t format_avf_device_name;
+format_function_t format_avf_input_trace;
+
+static inline u32
+avf_get_u32 (void *start, int offset)
+{
+  return *(u32 *) (((u8 *) start) + offset);
+}
+
+static inline u64
+avf_get_u64 (void *start, int offset)
+{
+  return *(u64 *) (((u8 *) start) + offset);
+}
+
+static inline u32
+avf_get_u32_bits (void *start, int offset, int first, int last)
+{
+  u32 value = avf_get_u32 (start, offset);
+  if ((last == 0) && (first == 31))
+    return value;
+  value >>= last;
+  value &= (1 << (first - last + 1)) - 1;
+  return value;
+}
+
+static inline u64
+avf_get_u64_bits (void *start, int offset, int first, int last)
+{
+  u64 value = avf_get_u64 (start, offset);
+  if ((last == 0) && (first == 63))
+    return value;
+  value >>= last;
+  value &= (1 << (first - last + 1)) - 1;
+  return value;
+}
+
+static inline void
+avf_set_u32 (void *start, int offset, u32 value)
+{
+  (*(u32 *) (((u8 *) start) + offset)) = value;
+}
+
+static inline void
+avf_reg_write (avf_device_t * ad, u32 addr, u32 val)
+{
+  *(volatile u32 *) ((u8 *) ad->bar0 + addr) = val;
+}
+
+static inline u32
+avf_reg_read (avf_device_t * ad, u32 addr)
+{
+  return *(volatile u32 *) (ad->bar0 + addr);
+}
+
+static inline void
+avf_reg_flush (avf_device_t * ad)
+{
+  avf_reg_read (ad, AVFGEN_RSTAT);
+  asm volatile ("":::"memory");
+}
+
+typedef struct
+{
+  u32 next_index;
+  u32 hw_if_index;
+  avf_rx_vector_entry_t rxve;
+} avf_input_trace_t;
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */

src/plugins/avf/cli.c

@@ -0,0 +1,200 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+#include <stdint.h>
+#include <net/if.h>
+#include <sys/ioctl.h>
+#include <inttypes.h>
+
+#include <vlib/vlib.h>
+#include <vlib/unix/unix.h>
+#include <vlib/pci/pci.h>
+#include <vnet/ethernet/ethernet.h>
+
+#include <avf/avf.h>
+
+static clib_error_t *
+avf_create_command_fn (vlib_main_t * vm, unformat_input_t * input,
+		       vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  avf_create_if_args_t args;
+
+  memset (&args, 0, sizeof (avf_create_if_args_t));
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "%U", unformat_vlib_pci_addr, &args.addr))
+	;
+      else if (unformat (line_input, "elog"))
+	args.enable_elog = 1;
+      else
+	return clib_error_return (0, "unknown input `%U'",
+				  format_unformat_error, input);
+    }
+  unformat_free (line_input);
+
+
+  avf_create_if (vm, &args);
+
+  return args.error;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (avf_create_command, static) = {
+  .path = "create interface avf",
+  .short_help = "create interface avf <pci-address>",
+  .function = avf_create_command_fn,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+avf_delete_command_fn (vlib_main_t * vm, unformat_input_t * input,
+		       vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  u32 sw_if_index = ~0;
+  vnet_hw_interface_t *hw;
+  avf_main_t *am = &avf_main;
+  avf_device_t *ad;
+  vnet_main_t *vnm = vnet_get_main ();
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "sw_if_index %d", &sw_if_index))
+	;
+      else if (unformat (line_input, "%U", unformat_vnet_sw_interface,
+			 vnm, &sw_if_index))
+	;
+      else
+	return clib_error_return (0, "unknown input `%U'",
+				  format_unformat_error, input);
+    }
+  unformat_free (line_input);
+
+  if (sw_if_index == ~0)
+    return clib_error_return (0,
+			      "please specify interface name or sw_if_index");
+
+  hw = vnet_get_sup_hw_interface (vnm, sw_if_index);
+  if (hw == NULL || avf_device_class.index != hw->dev_class_index)
+    return clib_error_return (0, "not a AVF interface");
+
+  ad = pool_elt_at_index (am->devices, hw->dev_instance);
+
+  avf_delete_if (vm, ad);
+
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (avf_delete_command, static) = {
+  .path = "delete interface avf",
+  .short_help = "delete interface avf "
+    "{<interface> | sw_if_index <sw_idx>}",
+  .function = avf_delete_command_fn,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+avf_test_command_fn (vlib_main_t * vm, unformat_input_t * input,
+		     vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  u32 sw_if_index = ~0;
+  vnet_hw_interface_t *hw;
+  avf_main_t *am = &avf_main;
+  avf_device_t *ad;
+  vnet_main_t *vnm = vnet_get_main ();
+  int test_irq = 0, enable_elog = 0, disable_elog = 0;
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "sw_if_index %d", &sw_if_index))
+	;
+      else if (unformat (line_input, "irq"))
+	test_irq = 1;
+      else if (unformat (line_input, "elog-on"))
+	enable_elog = 1;
+      else if (unformat (line_input, "elog-off"))
+	disable_elog = 1;
+      else if (unformat (line_input, "%U", unformat_vnet_sw_interface,
+			 vnm, &sw_if_index))
+	;
+      else
+	return clib_error_return (0, "unknown input `%U'",
+				  format_unformat_error, input);
+    }
+  unformat_free (line_input);
+
+  if (sw_if_index == ~0)
+    return clib_error_return (0,
+			      "please specify interface name or sw_if_index");
+
+  hw = vnet_get_sup_hw_interface (vnm, sw_if_index);
+  if (hw == NULL || avf_device_class.index != hw->dev_class_index)
+    return clib_error_return (0, "not a AVF interface");
+
+  ad = pool_elt_at_index (am->devices, hw->dev_instance);
+
+  if (enable_elog)
+    ad->flags |= AVF_DEVICE_F_ELOG;
+
+  if (disable_elog)
+    ad->flags &= ~AVF_DEVICE_F_ELOG;
+
+  if (test_irq)
+    avf_reg_write (ad, AVFINT_DYN_CTL0, (1 << 0) | (3 << 3) | (1 << 2));
+
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (avf_test_command, static) = {
+  .path = "test avf",
+  .short_help = "test avf [<interface> | sw_if_index <sw_idx>] [irq] "
+    "[elog-on] [elog-off]",
+  .function = avf_test_command_fn,
+};
+/* *INDENT-ON* */
+
+clib_error_t *
+avf_cli_init (vlib_main_t * vm)
+{
+  return 0;
+}
+
+VLIB_INIT_FUNCTION (avf_cli_init);
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */

src/plugins/avf/format.c

@@ -0,0 +1,145 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+#include <vlib/vlib.h>
+#include <vlib/unix/unix.h>
+#include <vlib/pci/pci.h>
+#include <vnet/ethernet/ethernet.h>
+
+#include <avf/avf.h>
+
+u8 *
+format_avf_device_name (u8 * s, va_list * args)
+{
+  u32 i = va_arg (*args, u32);
+  avf_main_t *am = &avf_main;
+  avf_device_t *ad = vec_elt_at_index (am->devices, i);
+  vlib_pci_addr_t *addr = vlib_pci_get_addr (ad->pci_dev_handle);
+
+  s = format (s, "AVF%x/%x/%x/%x",
+	      addr->domain, addr->bus, addr->slot, addr->function);
+  return s;
+}
+
+u8 *
+format_avf_device_flags (u8 * s, va_list * args)
+{
+  avf_device_t *ad = va_arg (*args, avf_device_t *);
+  u8 *t = 0;
+
+#define _(a, b, c) if (ad->flags & (1 << a)) \
+t = format (t, "%s%s", t ? " ":"", c);
+  foreach_avf_device_flags
+#undef _
+    s = format (s, "%v", t);
+  vec_free (t);
+  return s;
+}
+
+u8 *
+format_avf_vf_cap_flags (u8 * s, va_list * args)
+{
+  u32 flags = va_arg (*args, u32);
+  u8 *t = 0;
+
+#define _(a, b, c) if (flags & (1 << a)) \
+  t = format (t, "%s%s", t ? " ":"", c);
+  foreach_avf_vf_cap_flag;
+#undef _
+  s = format (s, "%v", t);
+  vec_free (t);
+  return s;
+}
+
+static u8 *
+format_virtchnl_link_speed (u8 * s, va_list * args)
+{
+  virtchnl_link_speed_t speed = va_arg (*args, virtchnl_link_speed_t);
+
+  if (speed == 0)
+    return format (s, "unknown");
+#define _(a, b, c) \
+  else if (speed == VIRTCHNL_LINK_SPEED_##b) \
+    return format (s, c);
+  foreach_virtchnl_link_speed;
+#undef _
+  return s;
+}
+
+u8 *
+format_avf_device (u8 * s, va_list * args)
+{
+  u32 i = va_arg (*args, u32);
+  avf_main_t *am = &avf_main;
+  avf_device_t *ad = vec_elt_at_index (am->devices, i);
+  u32 indent = format_get_indent (s);
+  u8 *a = 0;
+
+  s = format (s, "flags: %U", format_avf_device_flags, ad);
+  s = format (s, "\n%Uoffload features: %U", format_white_space, indent,
+	      format_avf_vf_cap_flags, ad->feature_bitmap);
+
+  s = format (s, "\n%Unum-queue-pairs %d max-vectors %u max-mtu %u "
+	      "rss-key-size %u rss-lut-size %u", format_white_space, indent,
+	      ad->num_queue_pairs, ad->max_vectors, ad->max_mtu,
+	      ad->rss_key_size, ad->rss_lut_size);
+  s = format (s, "\n%Uspeed %U", format_white_space, indent,
+	      format_virtchnl_link_speed, ad->link_speed);
+  if (ad->error)
+    s = format (s, "\n%Uerror %U", format_white_space, indent,
+		format_clib_error, ad->error);
+
+#define _(c) if (ad->eth_stats.c) \
+  a = format (a, "\n%U%-20U %u", format_white_space, indent + 2, \
+	      format_c_identifier, #c, ad->eth_stats.c);
+  foreach_virtchnl_eth_stats;
+#undef _
+  if (a)
+    s = format (s, "\n%Ustats:%v", format_white_space, indent, a);
+
+  vec_free (a);
+  return s;
+}
+
+u8 *
+format_avf_input_trace (u8 * s, va_list * args)
+{
+  vlib_main_t *vm = va_arg (*args, vlib_main_t *);
+  vlib_node_t *node = va_arg (*args, vlib_node_t *);
+  avf_input_trace_t *t = va_arg (*args, avf_input_trace_t *);
+  vnet_main_t *vnm = vnet_get_main ();
+  vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, t->hw_if_index);
+  u32 indent = format_get_indent (s);
+  avf_rx_vector_entry_t *rxve = &t->rxve;
+
+  s = format (s, "avf: %v (%d) next-node %U",
+	      hi->name, t->hw_if_index, format_vlib_next_node_name, vm,
+	      node->index, t->next_index);
+  s = format (s, "\n%Ustatus 0x%x error 0x%x ptype 0x%x length %u",
+	      format_white_space, indent + 2, rxve->status, rxve->error,
+	      rxve->ptype, rxve->length);
+
+  return s;
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */

src/plugins/avf/output.c

@@ -0,0 +1,183 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+#include <vlib/vlib.h>
+#include <vlib/unix/unix.h>
+#include <vlib/pci/pci.h>
+#include <vnet/ethernet/ethernet.h>
+#include <vnet/devices/devices.h>
+
+#include <avf/avf.h>
+
+#define AVF_TXQ_DESC_CMD(x)             (1 << (x + 4))
+#define AVF_TXQ_DESC_CMD_EOP		AVF_TXQ_DESC_CMD(0)
+#define AVF_TXQ_DESC_CMD_RS		AVF_TXQ_DESC_CMD(1)
+
+static_always_inline u8
+avf_tx_desc_get_dtyp (avf_tx_desc_t * d)
+{
+  return d->qword[1] & 0x0f;
+}
+
+uword
+CLIB_MULTIARCH_FN (avf_interface_tx) (vlib_main_t * vm,
+				      vlib_node_runtime_t * node,
+				      vlib_frame_t * frame)
+{
+  avf_main_t *am = &avf_main;
+  vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
+  avf_device_t *ad = pool_elt_at_index (am->devices, rd->dev_instance);
+  u32 thread_index = vlib_get_thread_index ();
+  u8 qid = thread_index;
+  avf_txq_t *txq = vec_elt_at_index (ad->txqs, qid % ad->num_queue_pairs);
+  avf_tx_desc_t *d0, *d1, *d2, *d3;
+  u32 *buffers = vlib_frame_args (frame);
+  u32 bi0, bi1, bi2, bi3;
+  u16 n_left = frame->n_vectors;
+  vlib_buffer_t *b0, *b1, *b2, *b3;
+  u16 mask = txq->size - 1;
+
+  clib_spinlock_lock_if_init (&txq->lock);
+
+  /* release cosumed bufs */
+  if (txq->n_bufs)
+    {
+      u16 first, slot, n_free = 0;
+      first = slot = (txq->next - txq->n_bufs) & mask;
+      d0 = txq->descs + slot;
+      while (n_free < txq->n_bufs && avf_tx_desc_get_dtyp (d0) == 0x0F)
+	{
+	  n_free++;
+	  slot = (slot + 1) & mask;
+	  d0 = txq->descs + slot;
+	}
+
+      if (n_free)
+	{
+	  txq->n_bufs -= n_free;;
+	  vlib_buffer_free_from_ring (vm, txq->bufs, first, txq->size,
+				      n_free);
+	}
+    }
+
+  while (n_left >= 7)
+    {
+      u16 slot0, slot1, slot2, slot3;
+
+      vlib_prefetch_buffer_with_index (vm, buffers[4], LOAD);
+      vlib_prefetch_buffer_with_index (vm, buffers[5], LOAD);
+      vlib_prefetch_buffer_with_index (vm, buffers[6], LOAD);
+      vlib_prefetch_buffer_with_index (vm, buffers[7], LOAD);
+
+      slot0 = txq->next;
+      slot1 = (txq->next + 1) & mask;
+      slot2 = (txq->next + 2) & mask;
+      slot3 = (txq->next + 3) & mask;
+
+      d0 = txq->descs + slot0;
+      d1 = txq->descs + slot1;
+      d2 = txq->descs + slot2;
+      d3 = txq->descs + slot3;
+
+      bi0 = buffers[0];
+      bi1 = buffers[1];
+      bi2 = buffers[2];
+      bi3 = buffers[3];
+
+      txq->bufs[slot0] = bi0;
+      txq->bufs[slot1] = bi1;
+      txq->bufs[slot2] = bi2;
+      txq->bufs[slot3] = bi3;
+      b0 = vlib_get_buffer (vm, bi0);
+      b1 = vlib_get_buffer (vm, bi1);
+      b2 = vlib_get_buffer (vm, bi2);
+      b3 = vlib_get_buffer (vm, bi3);
+
+#if 0
+      d->qword[0] = vlib_get_buffer_data_physical_address (vm, bi0) +
+	b0->current_data;
+#else
+      d0->qword[0] = pointer_to_uword (b0->data);
+      d1->qword[0] = pointer_to_uword (b1->data);
+      d2->qword[0] = pointer_to_uword (b2->data);
+      d3->qword[0] = pointer_to_uword (b3->data);
+
+#endif
+      u64 bits = AVF_TXQ_DESC_CMD_EOP | AVF_TXQ_DESC_CMD_RS;
+      d0->qword[1] = ((u64) b0->current_length) << 34 | bits;
+      d1->qword[1] = ((u64) b1->current_length) << 34 | bits;
+      d2->qword[1] = ((u64) b2->current_length) << 34 | bits;
+      d3->qword[1] = ((u64) b3->current_length) << 34 | bits;
+
+      txq->next = (txq->next + 4) & mask;
+      txq->n_bufs += 4;
+      buffers += 4;
+      n_left -= 4;
+    }
+
+  while (n_left)
+    {
+      d0 = txq->descs + txq->next;
+      bi0 = buffers[0];
+      txq->bufs[txq->next] = bi0;
+      b0 = vlib_get_buffer (vm, bi0);
+
+#if 0
+      d->qword[0] = vlib_get_buffer_data_physical_address (vm, bi0) +
+	b0->current_data;
+#else
+      d0->qword[0] = pointer_to_uword (b0->data);
+
+#endif
+      d0->qword[1] = ((u64) b0->current_length) << 34;
+      d0->qword[1] |= AVF_TXQ_DESC_CMD_EOP | AVF_TXQ_DESC_CMD_RS;
+
+      txq->next = (txq->next + 1) & mask;
+      txq->n_bufs++;
+      buffers++;
+      n_left--;
+    }
+  CLIB_MEMORY_BARRIER ();
+  *(txq->qtx_tail) = txq->next;
+
+  clib_spinlock_unlock_if_init (&txq->lock);
+
+  return frame->n_vectors - n_left;
+}
+
+#ifndef CLIB_MULTIARCH_VARIANT
+#if __x86_64__
+vlib_node_function_t __clib_weak avf_interface_tx_avx512;
+vlib_node_function_t __clib_weak avf_interface_tx_avx2;
+static void __clib_constructor
+avf_interface_tx_multiarch_select (void)
+{
+  if (avf_interface_tx_avx512 && clib_cpu_supports_avx512f ())
+    avf_device_class.tx_function = avf_interface_tx_avx512;
+  else if (avf_interface_tx_avx2 && clib_cpu_supports_avx2 ())
+    avf_device_class.tx_function = avf_interface_tx_avx2;
+}
+#endif
+#endif
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */

src/plugins/avf/plugin.c

@@ -0,0 +1,35 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+#include <vlib/vlib.h>
+#include <vnet/plugin/plugin.h>
+#include <vpp/app/version.h>
+
+/* *INDENT-OFF* */
+VLIB_PLUGIN_REGISTER () = {
+  .version = VPP_BUILD_VER,
+  .description = "Intel Adaptive Virtual Function (AVF) Device Plugin",
+};
+/* *INDENT-ON* */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */