virtio: support virtio 1.1 packed ring in vhost

virtio 1.1 defines a number of new features. Packed ring is among the most
notable and important one. It combines used, available, and descripptor rings
into one.

This patch provides experimental support for packed ring. To avoid
regression, when packed ring is configured for the interface, it is branched
to a separate RX and TX driver. Non packed ring should continue to perform
as it was before.

Packed ring is tested using qemu4.2 and ubuntu focal fossa (kernel 5.4.0-12)
on the guess VM which supports packed ring.

To configure VPP with packed ring, just add the optional keyword "packed"
when creating the vhost interface. To bring up the guest VM with packed ring,
add "packed=on" in the qemu launch command.

To facilitate troubleshooting, also added "verbose" option in
show vhost desc CLI to include displaying the indirect descriptors.

Known qemu reconnect issue -
If VPP is restarted, guest VMs also need to be restarted. The problem
is kernel virtio-net-pci keeps track of the previous available and used
indices. For virtio 1.0, these indices are in shared memory and qemu can
easily copy them to pass to the backend for reconnect. For virio 1.1, these
indices are no longer in shared memory. Qemu needs a new mechanism to retrieve
them and it is not currently implemented. So when the protocol reconnects,
qemu does not have the correct available and used indices to pass to the
backend. As a result, after the reconnect, virtio-net-pci is reading the TX
ring from the wrong position in the ring, not the same position which the
backend is writing. Similar problem exists also in the RX.

Type: feature

Signed-off-by: Steven Luong <sluong@cisco.com>
Change-Id: I5afc50b0bafab5a1de7a6dd10f399db3fafd144c

src/vat/api_format.c

@@ -12447,6 +12447,7 @@ api_create_vhost_user_if (vat_main_t * vam)
   u8 disable_indirect_desc = 0;
   u8 *tag = 0;
   u8 enable_gso = 0;
+  u8 enable_packed = 0;
   int ret;
 
   /* Shut up coverity */
@@ -12470,6 +12471,8 @@ api_create_vhost_user_if (vat_main_t * vam)
 	disable_indirect_desc = 1;
       else if (unformat (i, "gso"))
 	enable_gso = 1;
+      else if (unformat (i, "packed"))
+	enable_packed = 1;
       else if (unformat (i, "tag %s", &tag))
 	;
       else
@@ -12495,6 +12498,7 @@ api_create_vhost_user_if (vat_main_t * vam)
   mp->disable_mrg_rxbuf = disable_mrg_rxbuf;
   mp->disable_indirect_desc = disable_indirect_desc;
   mp->enable_gso = enable_gso;
+  mp->enable_packed = enable_packed;
   clib_memcpy (mp->sock_filename, file_name, vec_len (file_name));
   vec_free (file_name);
   if (custom_dev_instance != ~0)
@@ -12526,6 +12530,7 @@ api_modify_vhost_user_if (vat_main_t * vam)
   u8 sw_if_index_set = 0;
   u32 sw_if_index = (u32) ~ 0;
   u8 enable_gso = 0;
+  u8 enable_packed = 0;
   int ret;
 
   while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
@@ -12544,6 +12549,8 @@ api_modify_vhost_user_if (vat_main_t * vam)
 	is_server = 1;
       else if (unformat (i, "gso"))
 	enable_gso = 1;
+      else if (unformat (i, "packed"))
+	enable_packed = 1;
       else
 	break;
     }
@@ -12572,6 +12579,7 @@ api_modify_vhost_user_if (vat_main_t * vam)
   mp->sw_if_index = ntohl (sw_if_index);
   mp->is_server = is_server;
   mp->enable_gso = enable_gso;
+  mp->enable_packed = enable_packed;
   clib_memcpy (mp->sock_filename, file_name, vec_len (file_name));
   vec_free (file_name);
   if (custom_dev_instance != ~0)
@@ -20782,10 +20790,10 @@ _(l2_interface_vlan_tag_rewrite,                                        \
 _(create_vhost_user_if,                                                 \
         "socket <filename> [server] [renumber <dev_instance>] "         \
         "[disable_mrg_rxbuf] [disable_indirect_desc] [gso] "            \
-        "[mac <mac_address>]")                                          \
+        "[mac <mac_address>] [packed]")                                 \
 _(modify_vhost_user_if,                                                 \
         "<intfc> | sw_if_index <nn> socket <filename>\n"                \
-        "[server] [renumber <dev_instance>] [gso]")                     \
+        "[server] [renumber <dev_instance>] [gso] [packed]")            \
 _(delete_vhost_user_if, "<intfc> | sw_if_index <nn>")                   \
 _(sw_interface_vhost_user_dump, "")                                     \
 _(show_version, "")                                                     \

src/vnet/devices/virtio/FEATURE.yaml

@@ -7,7 +7,7 @@ features:
   - device mode to emulate vhost-user interface presented to VPP from the
     guest VM.
   - support multi-queue, GSO, checksum offload, indirect descriptor,
-    and jumbo frame.
+    jumbo frame, and packed ring.
 description: "Virtio v1.0 implementation"
 missing:
   - API dump filtering by sw_if_index

src/vnet/devices/virtio/vhost_user.api

@@ -13,7 +13,7 @@
  * limitations under the License.
  */
 
-option version = "4.0.0";
+option version = "4.0.1";
 
 import "vnet/interface_types.api";
 import "vnet/ethernet/ethernet_types.api";
@@ -27,6 +27,7 @@ import "vnet/devices/virtio/virtio_types.api";
     @param disable_mrg_rxbuf - disable the use of merge receive buffers
     @param disable_indirect_desc - disable the use of indirect descriptors which driver can use
     @param enable_gso - enable gso support (default 0)
+    @param enable_packed - enable packed ring support (default 0)
     @param mac_address - hardware address to use if 'use_custom_mac' is set
 */
 define create_vhost_user_if
@@ -39,6 +40,7 @@ define create_vhost_user_if
   bool disable_mrg_rxbuf;
   bool disable_indirect_desc;
   bool enable_gso;
+  bool enable_packed;
   u32 custom_dev_instance;
   bool use_custom_mac;
   vl_api_mac_address_t mac_address;
@@ -62,6 +64,7 @@ define create_vhost_user_if_reply
     @param is_server - our side is socket server
     @param sock_filename - unix socket filename, used to speak with frontend
     @param enable_gso - enable gso support (default 0)
+    @param enable_packed - enable packed ring support (default 0)
 */
 autoreply define modify_vhost_user_if
 {
@@ -72,6 +75,7 @@ autoreply define modify_vhost_user_if
   string sock_filename[256];
   bool renumber;
   bool enable_gso;
+  bool enable_packed;
   u32 custom_dev_instance;
 };
 

src/vnet/devices/virtio/vhost_user.h

@@ -25,8 +25,15 @@
 
 #define VHOST_USER_VRING_NOFD_MASK      0x100
 #define VIRTQ_DESC_F_NEXT               1
+#define VIRTQ_DESC_F_WRITE              2
 #define VIRTQ_DESC_F_INDIRECT           4
-#define VHOST_USER_REPLY_MASK       (0x1 << 2)
+
+#define VIRTQ_DESC_F_AVAIL	        (1 << 7)
+#define VIRTQ_DESC_F_USED	        (1 << 15)
+
+#define VRING_EVENT_F_ENABLE            0x0
+#define VRING_EVENT_F_DISABLE           0x1
+#define VRING_EVENT_F_DESC              0x2
 
 #define VHOST_USER_PROTOCOL_F_MQ   0
 #define VHOST_USER_PROTOCOL_F_LOG_SHMFD	1
@@ -100,8 +107,11 @@ typedef enum
  _ (VHOST_F_LOG_ALL, 26)                \
  _ (VIRTIO_F_ANY_LAYOUT, 27)            \
  _ (VIRTIO_F_INDIRECT_DESC, 28)         \
+ _ (VIRTIO_F_EVENT_IDX, 29)		\
  _ (VHOST_USER_F_PROTOCOL_FEATURES, 30) \
- _ (VIRTIO_F_VERSION_1, 32)
+ _ (VIRTIO_F_VERSION_1, 32)             \
+ _ (VIRTIO_F_RING_PACKED, 34)           \
+ _ (VIRTIO_F_IN_ORDER, 35)
 
 typedef enum
 {
@@ -130,12 +140,12 @@ int vhost_user_create_if (vnet_main_t * vnm, vlib_main_t * vm,
 			  const char *sock_filename, u8 is_server,
 			  u32 * sw_if_index, u64 feature_mask,
 			  u8 renumber, u32 custom_dev_instance, u8 * hwaddr,
-			  u8 enable_gso);
+			  u8 enable_gso, u8 enable_packed);
 int vhost_user_modify_if (vnet_main_t * vnm, vlib_main_t * vm,
 			  const char *sock_filename, u8 is_server,
 			  u32 sw_if_index, u64 feature_mask,
 			  u8 renumber, u32 custom_dev_instance,
-			  u8 enable_gso);
+			  u8 enable_gso, u8 enable_packed);
 int vhost_user_delete_if (vnet_main_t * vnm, vlib_main_t * vm,
 			  u32 sw_if_index);
 
@@ -223,6 +233,22 @@ typedef struct
     } ring[VHOST_VRING_MAX_SIZE];
 } __attribute ((packed)) vring_used_t;
 
+typedef CLIB_PACKED (struct
+{
+  u64 addr;			// packet data buffer address
+  u32 len;			// packet data buffer size
+  u16 id;			// buffer id
+  u16 flags;			// flags
+}) vring_packed_desc_t;
+
+STATIC_ASSERT_SIZEOF (vring_packed_desc_t, 16);
+
+typedef CLIB_PACKED (struct
+{
+  u16 off_wrap;
+  u16 flags;
+}) vring_desc_event_t;
+
 typedef struct
 {
   u8 flags;
@@ -260,9 +286,21 @@ typedef struct
   u16 last_avail_idx;
   u16 last_used_idx;
   u16 n_since_last_int;
-  vring_desc_t *desc;
-  vring_avail_t *avail;
-  vring_used_t *used;
+  union
+  {
+    vring_desc_t *desc;
+    vring_packed_desc_t *packed_desc;
+  };
+  union
+  {
+    vring_avail_t *avail;
+    vring_desc_event_t *avail_event;
+  };
+  union
+  {
+    vring_used_t *used;
+    vring_desc_event_t *used_event;
+  };
   uword desc_user_addr;
   uword used_user_addr;
   uword avail_user_addr;
@@ -287,6 +325,9 @@ typedef struct
    * the interface even if it is disconnected and reconnected.
    */
   i16 qid;
+
+  u16 used_wrap_counter;
+  u16 avail_wrap_counter;
 } vhost_user_vring_t;
 
 #define VHOST_USER_EVENT_START_TIMER 1
@@ -332,6 +373,10 @@ typedef struct
   u16 *per_cpu_tx_qid;
 
   u8 enable_gso;
+
+  /* Packed ring configured */
+  u8 enable_packed;
+
 } vhost_user_intf_t;
 
 typedef struct
@@ -350,7 +395,6 @@ typedef struct
   virtio_net_hdr_mrg_rxbuf_t hdr; /** Virtio header **/
 } vhost_trace_t;
 
-
 #define VHOST_USER_RX_BUFFERS_N (2 * VLIB_FRAME_SIZE + 2)
 #define VHOST_USER_COPY_ARRAY_N (4 * VLIB_FRAME_SIZE)
 
@@ -365,6 +409,9 @@ typedef struct
   /* This is here so it doesn't end-up
    * using stack or registers. */
   vhost_trace_t *current_trace;
+
+  u32 *to_next_list;
+  vlib_buffer_t **rx_buffers_pdesc;
 } vhost_cpu_t;
 
 typedef struct

src/vnet/devices/virtio/vhost_user_api.c

@@ -71,10 +71,12 @@ vl_api_create_vhost_user_if_t_handler (vl_api_create_vhost_user_if_t * mp)
     disabled_features |= (1ULL << FEAT_VIRTIO_F_INDIRECT_DESC);
 
   /*
-   * feature mask is not supported via binary API. We disable GSO feature in the
-   * feature mask. It may be enabled via enable_gso argument.
+   * GSO and PACKED are not supported by feature mask via binary API. We
+   * disable GSO and PACKED feature in the feature mask. They may be enabled
+   * explicitly via enable_gso and enable_packed argument
    */
-  disabled_features |= FEATURE_VIRTIO_NET_F_HOST_GUEST_TSO_FEATURE_BITS;
+  disabled_features |= FEATURE_VIRTIO_NET_F_HOST_GUEST_TSO_FEATURE_BITS |
+    (1ULL << FEAT_VIRTIO_F_RING_PACKED);
   features &= ~disabled_features;
 
   if (mp->use_custom_mac)
@@ -86,7 +88,7 @@ vl_api_create_vhost_user_if_t_handler (vl_api_create_vhost_user_if_t * mp)
   rv = vhost_user_create_if (vnm, vm, (char *) mp->sock_filename,
 			     mp->is_server, &sw_if_index, features,
 			     mp->renumber, ntohl (mp->custom_dev_instance),
-			     mac_p, mp->enable_gso);
+			     mac_p, mp->enable_gso, mp->enable_packed);
 
   /* Remember an interface tag for the new interface */
   if (rv == 0)
@@ -122,16 +124,18 @@ vl_api_modify_vhost_user_if_t_handler (vl_api_modify_vhost_user_if_t * mp)
   vlib_main_t *vm = vlib_get_main ();
 
   /*
-   * feature mask is not supported via binary API. We disable GSO feature in the
-   * feature mask. It may be enabled via enable_gso argument.
+   * GSO and PACKED are not supported by feature mask via binary API. We
+   * disable GSO and PACKED feature in the feature mask. They may be enabled
+   * explicitly via enable_gso and enable_packed argument
    */
-  disabled_features |= FEATURE_VIRTIO_NET_F_HOST_GUEST_TSO_FEATURE_BITS;
+  disabled_features |= FEATURE_VIRTIO_NET_F_HOST_GUEST_TSO_FEATURE_BITS |
+    (1ULL << FEAT_VIRTIO_F_RING_PACKED);
   features &= ~disabled_features;
 
   rv = vhost_user_modify_if (vnm, vm, (char *) mp->sock_filename,
 			     mp->is_server, sw_if_index, features,
 			     mp->renumber, ntohl (mp->custom_dev_instance),
-			     mp->enable_gso);
+			     mp->enable_gso, mp->enable_packed);
 
   REPLY_MACRO (VL_API_MODIFY_VHOST_USER_IF_REPLY);
 }

src/vnet/devices/virtio/vhost_user_inline.h

@@ -292,6 +292,85 @@ vhost_user_update_gso_interface_count (vhost_user_intf_t * vui, u8 add)
 	}
     }
 }
+
+static_always_inline u8
+vhost_user_packed_desc_available (vhost_user_vring_t * vring, u16 idx)
+{
+  return (((vring->packed_desc[idx].flags & VIRTQ_DESC_F_AVAIL) ==
+	   vring->avail_wrap_counter));
+}
+
+static_always_inline void
+vhost_user_advance_last_avail_idx (vhost_user_vring_t * vring)
+{
+  vring->last_avail_idx++;
+  if (PREDICT_FALSE ((vring->last_avail_idx & vring->qsz_mask) == 0))
+    vring->avail_wrap_counter ^= VIRTQ_DESC_F_AVAIL;
+}
+
+static_always_inline void
+vhost_user_advance_last_avail_table_idx (vhost_user_intf_t * vui,
+					 vhost_user_vring_t * vring,
+					 u8 chained)
+{
+  if (chained)
+    {
+      vring_packed_desc_t *desc_table = vring->packed_desc;
+
+      /* pick up the slot of the next avail idx */
+      while (desc_table[vring->last_avail_idx & vring->qsz_mask].flags &
+	     VIRTQ_DESC_F_NEXT)
+	vhost_user_advance_last_avail_idx (vring);
+    }
+
+  vhost_user_advance_last_avail_idx (vring);
+}
+
+static_always_inline void
+vhost_user_undo_advanced_last_avail_idx (vhost_user_vring_t * vring)
+{
+  if (PREDICT_FALSE ((vring->last_avail_idx & vring->qsz_mask) == 0))
+    vring->avail_wrap_counter ^= VIRTQ_DESC_F_AVAIL;
+  vring->last_avail_idx--;
+}
+
+static_always_inline void
+vhost_user_dequeue_descs (vhost_user_vring_t * rxvq,
+			  virtio_net_hdr_mrg_rxbuf_t * hdr,
+			  u16 * n_descs_processed)
+{
+  u16 i;
+
+  *n_descs_processed -= (hdr->num_buffers - 1);
+  for (i = 0; i < hdr->num_buffers - 1; i++)
+    vhost_user_undo_advanced_last_avail_idx (rxvq);
+}
+
+static_always_inline void
+vhost_user_dequeue_chained_descs (vhost_user_vring_t * rxvq,
+				  u16 * n_descs_processed)
+{
+  while (*n_descs_processed)
+    {
+      vhost_user_undo_advanced_last_avail_idx (rxvq);
+      (*n_descs_processed)--;
+    }
+}
+
+static_always_inline void
+vhost_user_advance_last_used_idx (vhost_user_vring_t * vring)
+{
+  vring->last_used_idx++;
+  if (PREDICT_FALSE ((vring->last_used_idx & vring->qsz_mask) == 0))
+    vring->used_wrap_counter ^= 1;
+}
+
+static_always_inline u64
+vhost_user_is_packed_ring_supported (vhost_user_intf_t * vui)
+{
+  return (vui->features & (1ULL << FEAT_VIRTIO_F_RING_PACKED));
+}
+
 #endif
 
 /*

src/vpp/api/custom_dump.c

@@ -1734,9 +1734,11 @@ static void *vl_api_create_vhost_user_if_t_print
   if (mp->disable_indirect_desc)
     s = format (s, "disable_indirect_desc ");
   if (mp->tag[0])
-    s = format (s, "tag %s", mp->tag);
+    s = format (s, "tag %s ", mp->tag);
   if (mp->enable_gso)
-    s = format (s, "gso");
+    s = format (s, "gso ");
+  if (mp->enable_packed)
+    s = format (s, "packed");
 
   FINISH;
 }
@@ -1755,7 +1757,9 @@ static void *vl_api_modify_vhost_user_if_t_print
   if (mp->renumber)
     s = format (s, "renumber %d ", (mp->custom_dev_instance));
   if (mp->enable_gso)
-    s = format (s, "gso");
+    s = format (s, "gso ");
+  if (mp->enable_packed)
+    s = format (s, "packed");
 
   FINISH;
 }

test/vpp_vhost_interface.py

@@ -6,8 +6,8 @@ class VppVhostInterface(VppInterface):
 
     def __init__(self, test, sock_filename, is_server=0, renumber=0,
                  disable_mrg_rxbuf=0, disable_indirect_desc=0, gso=0,
-                 custom_dev_instance=0, use_custom_mac=0, mac_address='',
-                 tag=''):
+                 packed_ring=0, custom_dev_instance=0, use_custom_mac=0,
+                 mac_address='', tag=''):
 
         """ Create VPP Vhost interface """
         super(VppVhostInterface, self).__init__(test)
@@ -17,6 +17,7 @@ class VppVhostInterface(VppInterface):
         self.disable_mrg_rxbuf = disable_mrg_rxbuf
         self.disable_indirect_desc = disable_indirect_desc
         self.gso = gso
+        self.packed_ring = packed_ring
         self.custom_dev_instance = custom_dev_instance
         self.use_custom_mac = use_custom_mac
         self.mac_address = mac_address
@@ -29,6 +30,7 @@ class VppVhostInterface(VppInterface):
                                                 self.disable_mrg_rxbuf,
                                                 self.disable_indirect_desc,
                                                 self.gso,
+                                                self.packed_ring,
                                                 self.custom_dev_instance,
                                                 self.use_custom_mac,
                                                 self.mac_address,