libmemif: zero-copy-slave mode + header space

Slave is now able to dequeue buffers from rx queue and enqueue them to tx queue
(zero-copy operation). Slave can produce buffers with headroom, which will allow adding
 encap without copy.

Change-Id: Ia189f8de1a68be787545ed46cf78d36403e7e9bf
Signed-off-by: Jakub Grajciar <jgrajcia@cisco.com>

extras/libmemif/examples/icmp_responder-epoll/main.c

@@ -282,6 +282,7 @@ int
 on_connect (memif_conn_handle_t conn, void *private_ctx)
 {
   INFO ("memif connected!");
+  memif_refill_queue (conn, 0, -1, 0);
   enable_log = 1;
   return 0;
 }
@@ -384,6 +385,7 @@ on_interrupt (memif_conn_handle_t conn, void *private_ctx, uint16_t qid)
 	}
 
       i = 0;
+      memset (c->tx_bufs, 0, sizeof (memif_buffer_t) * rx);
       err = memif_buffer_alloc (c->conn, qid, c->tx_bufs, rx, &tx, 128);
       if ((err != MEMIF_ERR_SUCCESS) && (err != MEMIF_ERR_NOBUF_RING))
 	{
@@ -404,7 +406,7 @@ on_interrupt (memif_conn_handle_t conn, void *private_ctx, uint16_t qid)
 	  tx--;
 	}
 
-      err = memif_refill_queue (c->conn, qid, rx);
+      err = memif_refill_queue (c->conn, qid, rx, 0);
       if (err != MEMIF_ERR_SUCCESS)
 	INFO ("memif_buffer_free: %s", memif_strerror (err));
       rx -= rx;
@@ -426,7 +428,7 @@ on_interrupt (memif_conn_handle_t conn, void *private_ctx, uint16_t qid)
   return 0;
 
 error:
-  err = memif_refill_queue (c->conn, qid, rx);
+  err = memif_refill_queue (c->conn, qid, rx, 0);
   if (err != MEMIF_ERR_SUCCESS)
     INFO ("memif_buffer_free: %s", memif_strerror (err));
   c->rx_buf_num -= rx;
@@ -512,7 +514,7 @@ on_interrupt0 (memif_conn_handle_t conn, void *private_ctx, uint16_t qid)
 	    }
 	  /* mark memif buffers and shared memory buffers as free */
 	  /* free processed buffers */
-	  err = memif_refill_queue (c->conn, qid, j);
+	  err = memif_refill_queue (c->conn, qid, j, 0);
 	  if (err != MEMIF_ERR_SUCCESS)
 	    INFO ("memif_buffer_free: %s", memif_strerror (err));
 	  rx -= j;
@@ -538,7 +540,7 @@ on_interrupt0 (memif_conn_handle_t conn, void *private_ctx, uint16_t qid)
   return 0;
 
 error:
-  err = memif_refill_queue (c->conn, qid, rx);
+  err = memif_refill_queue (c->conn, qid, rx, 0);
   if (err != MEMIF_ERR_SUCCESS)
     INFO ("memif_buffer_free: %s", memif_strerror (err));
   c->rx_buf_num -= rx;
@@ -595,7 +597,7 @@ on_interrupt1 (memif_conn_handle_t conn, void *private_ctx, uint16_t qid)
 	    }
 	}
 
-      err = memif_refill_queue (c->conn, qid, rx);
+      err = memif_refill_queue (c->conn, qid, rx, 0);
       if (err != MEMIF_ERR_SUCCESS)
 	INFO ("memif_buffer_free: %s", memif_strerror (err));
       c->rx_buf_num -= rx;
@@ -607,7 +609,7 @@ on_interrupt1 (memif_conn_handle_t conn, void *private_ctx, uint16_t qid)
   return 0;
 
 error:
-  err = memif_refill_queue (c->conn, qid, rx);
+  err = memif_refill_queue (c->conn, qid, rx, 0);
   if (err != MEMIF_ERR_SUCCESS)
     INFO ("memif_buffer_free: %s", memif_strerror (err));
   c->rx_buf_num -= rx;

extras/libmemif/examples/icmp_responder-mt/main.c

@@ -299,10 +299,6 @@ memif_rx_poll (void *ptr)
       data->rx_buf_num += rx;
       if (rx == 0)
 	{
-	  /* if queue is in polling mode, it's not refilled */
-	  err = memif_refill_queue (c->conn, data->qid, -1);
-	  if (err != MEMIF_ERR_SUCCESS)
-	    INFO ("memif_buffer_free: %s", memif_strerror (err));
 	  continue;
 	}
 
@@ -313,7 +309,7 @@ memif_rx_poll (void *ptr)
 
       err =
 	memif_buffer_alloc (c->conn, data->qid, data->tx_bufs,
-			    data->rx_buf_num, &tx, 128);
+			    data->rx_buf_num, &tx, 0);
       if (err != MEMIF_ERR_SUCCESS)
 	{
 	  INFO ("memif_buffer_alloc: %s", memif_strerror (err));
@@ -334,10 +330,10 @@ memif_rx_poll (void *ptr)
 	}
 
       /* mark memif buffers and shared memory buffers as free */
-      err = memif_refill_queue (c->conn, data->qid, -1);
+      err = memif_refill_queue (c->conn, data->qid, rx, 0);
       if (err != MEMIF_ERR_SUCCESS)
 	INFO ("memif_buffer_free: %s", memif_strerror (err));
-      data->rx_buf_num -= rx;
+      data->rx_buf_num -= fb;
 
       DBG ("freed %d buffers. %u/%u alloc/free buffers",
 	   fb, data->rx_buf_num, MAX_MEMIF_BUFS - data->rx_buf_num);
@@ -359,10 +355,10 @@ error:
   goto close;
 
 close:
-  err = memif_refill_queue (c->conn, data->qid, -1);
+  err = memif_refill_queue (c->conn, data->qid, rx, 0);
   if (err != MEMIF_ERR_SUCCESS)
     INFO ("memif_buffer_free: %s", memif_strerror (err));
-  data->rx_buf_num -= rx;
+  data->rx_buf_num -= fb;
   DBG ("freed %d buffers. %u/%u alloc/free buffers",
        fb, data->rx_buf_num, MAX_MEMIF_BUFS - data->rx_buf_num);
   free (data->rx_bufs);
@@ -443,7 +439,7 @@ memif_rx_interrupt (void *ptr)
 
 	  err =
 	    memif_buffer_alloc (c->conn, data->qid, data->tx_bufs,
-				data->rx_buf_num, &tx, 128);
+				data->rx_buf_num, &tx, 0);
 	  if (err != MEMIF_ERR_SUCCESS)
 	    {
 	      INFO ("memif_buffer_alloc: %s", memif_strerror (err));
@@ -464,10 +460,10 @@ memif_rx_interrupt (void *ptr)
 	    }
 
 	  /* mark memif buffers and shared memory buffers as free */
-	  err = memif_refill_queue (c->conn, data->qid, rx);
+	  err = memif_refill_queue (c->conn, data->qid, rx, 0);
 	  if (err != MEMIF_ERR_SUCCESS)
 	    INFO ("memif_buffer_free: %s", memif_strerror (err));
-	  data->rx_buf_num -= rx;
+	  data->rx_buf_num -= fb;
 
 	  DBG ("freed %d buffers. %u/%u alloc/free buffers",
 	       fb, data->rx_buf_num, MAX_MEMIF_BUFS - data->rx_buf_num);
@@ -490,10 +486,10 @@ error:
   goto close;
 
 close:
-  err = memif_refill_queue (c->conn, data->qid, rx);
+  err = memif_refill_queue (c->conn, data->qid, rx, 0);
   if (err != MEMIF_ERR_SUCCESS)
     INFO ("memif_buffer_free: %s", memif_strerror (err));
-  data->rx_buf_num -= rx;
+  data->rx_buf_num -= fb;
   DBG ("freed %d buffers. %u/%u alloc/free buffers",
        fb, data->rx_buf_num, MAX_MEMIF_BUFS - data->rx_buf_num);
   free (data->rx_bufs);

extras/libmemif/examples/icmp_responder/icmp_proto.c

@@ -37,6 +37,7 @@
 #include <net/if_arp.h>
 #include <asm/byteorder.h>
 #include <byteswap.h>
+#include <assert.h>
 
 #include <icmp_proto.h>
 
@@ -337,3 +338,188 @@ generate_packet (void *pck, uint32_t * size, uint8_t saddr[4],
 
   return 0;
 }
+
+int
+generate_packet2 (void *pck, uint32_t * size, uint8_t saddr[4],
+		  uint8_t daddr[4], uint8_t hw_daddr[6], uint32_t seq,
+		  icmpr_flow_mode_t mode)
+{
+  struct ether_header *eh;
+  struct iphdr *ip;
+  struct icmphdr *icmp;
+
+  *size = 0;
+
+  if (mode == ICMPR_FLOW_MODE_ETH)
+    {
+      eh = (struct ether_header *) pck;
+      *size += generate_eth (eh, hw_daddr);
+    }
+
+  ip = (struct iphdr *) (pck + *size);
+  *size += generate_ip (ip, saddr, daddr);
+
+  icmp = (struct icmphdr *) (pck + *size);
+  *size += generate_icmp (icmp, seq);
+
+  ((struct icmphdr *) (pck + *size - sizeof (struct icmphdr)))->checksum =
+    cksum (pck + *size - sizeof (struct icmphdr), sizeof (struct icmphdr));
+
+  ip->tot_len = __bswap_16 (*size - sizeof (struct ether_header));
+  ip->check = 0;
+  ip->check = cksum (ip, sizeof (struct iphdr));
+
+  return 0;
+}
+
+#define GET_HEADER(out,hdr,src,off) do {	\
+					out = (hdr*)(src + off); \
+					off += sizeof (hdr); \
+				} while (0)
+
+int
+resolve_packet2 (void *pck, uint32_t * size, uint8_t ip_addr[4])
+{
+  struct ether_header *eh;
+  struct ether_arp *eah;
+  struct iphdr *ip;
+  struct icmphdr *icmp;
+  uint32_t offset = 0;
+
+  if (pck == NULL)
+    return 0;
+
+  GET_HEADER (eh, struct ether_header, pck, offset);
+
+  memcpy (eh->ether_dhost, eh->ether_shost, 6);
+  memcpy (eh->ether_shost, "aaaaaa", 6);
+
+  if (eh->ether_type == 0x0608)
+    {
+      GET_HEADER (eah, struct ether_arp, pck, offset);
+      struct arphdr *arp = &eah->ea_hdr;
+
+      arp->ar_hrd = __bswap_16 (ARPHRD_ETHER);
+      arp->ar_pro = __bswap_16 (0x0800);
+
+      arp->ar_hln = 6;
+      arp->ar_pln = 4;
+
+      arp->ar_op = __bswap_16 (ARPOP_REPLY);
+
+      memcpy (eah->arp_tha, eah->arp_sha, 6);
+      memcpy (eah->arp_tpa, eah->arp_spa, 4);
+
+      memcpy (eah->arp_sha, eh->ether_shost, 6);
+      memcpy (eah->arp_spa, ip_addr, 4);
+    }
+
+  else if (eh->ether_type == 0x0008)
+    {
+      GET_HEADER (ip, struct iphdr, pck, offset);
+
+      if (ip->protocol == 1)
+	{
+	  ip->ihl = 5;
+	  ip->version = 4;
+	  ip->tos = 0;
+	  ip->tot_len = 0x0000;
+	  ip->id = 0;
+	  ip->frag_off = 0;
+	  ip->ttl = 0x40;
+	  ip->protocol = 1;
+	  ip->check = 0x0000;
+	  ip->daddr = ip->saddr;
+	  ((uint8_t *) & ip->saddr)[0] = ip_addr[0];
+	  ((uint8_t *) & ip->saddr)[1] = ip_addr[1];
+	  ((uint8_t *) & ip->saddr)[2] = ip_addr[2];
+	  ((uint8_t *) & ip->saddr)[3] = ip_addr[3];
+
+	  GET_HEADER (icmp, struct icmphdr, pck, offset);
+
+	  icmp->type = 0x00;
+	  icmp->code = 0;
+	  icmp->checksum = cksum (icmp, sizeof (struct icmphdr));
+
+	  /* rest is payload */
+	  offset = *size;
+
+	  ip->tot_len = __bswap_16 (offset - sizeof (struct ether_header));
+	  ip->check = cksum (ip, sizeof (struct iphdr));
+	}
+    }
+
+  assert (offset == *size && "unsupported protocol");
+  return 0;
+}
+
+
+int
+resolve_packet3 (void **pck_, uint32_t * size, uint8_t ip_addr[4])
+{
+  struct ether_header *eh;
+  struct ether_arp *eah;
+  struct iphdr *ip;
+  struct icmphdr *icmp;
+  int32_t offset = 0;
+  uint16_t encap_size = sizeof (struct ether_header);
+  void *pck = *pck_;
+
+  if (pck == NULL)
+    return 0;
+
+  *pck_ -= encap_size;
+  offset -= encap_size;
+
+  GET_HEADER (eh, struct ether_header, pck, offset);
+
+  uint8_t hw_daddr[6];
+  memset (hw_daddr, 0, sizeof (uint8_t) * 6);
+
+  generate_eth (eh, hw_daddr);
+
+  if (eh->ether_type == 0x0008)
+    {
+      GET_HEADER (ip, struct iphdr, pck, offset);
+
+      if (ip->protocol == 1)
+	{
+	  ip->ihl = 5;
+	  ip->version = 4;
+	  ip->tos = 0;
+	  ip->tot_len = 0x0000;
+	  ip->id = 0;
+	  ip->frag_off = 0;
+	  ip->ttl = 0x40;
+	  ip->protocol = 1;
+	  ip->check = 0x0000;
+	  ip->daddr = ip->saddr;
+	  ((uint8_t *) & ip->saddr)[0] = ip_addr[0];
+	  ((uint8_t *) & ip->saddr)[1] = ip_addr[1];
+	  ((uint8_t *) & ip->saddr)[2] = ip_addr[2];
+	  ((uint8_t *) & ip->saddr)[3] = ip_addr[3];
+
+	  GET_HEADER (icmp, struct icmphdr, pck, offset);
+
+	  icmp->type = 0x00;
+	  icmp->code = 0;
+	  icmp->checksum = cksum (icmp, sizeof (struct icmphdr));
+
+	  /* rest is payload */
+	  offset = *size;
+
+	  ip->tot_len = __bswap_16 (offset - sizeof (struct ether_header));
+	  ip->check = cksum (ip, sizeof (struct iphdr));
+	}
+    }
+
+  offset += encap_size;
+
+  assert (offset != *size &&
+	  "new packet length must be increased by encap size");
+
+  /* overwrite packet size */
+  *size = offset;
+
+  return 0;
+}

extras/libmemif/examples/icmp_responder/icmp_proto.h

@@ -18,12 +18,28 @@
 #ifndef _ICMP_PROTO_H_
 #define _ICMP_PROTO_H_
 
+typedef enum
+{
+  ICMPR_FLOW_MODE_ETH = 0,
+  ICMPR_FLOW_MODE_IP,
+} icmpr_flow_mode_t;
+
 int resolve_packet (void *in_pck, ssize_t in_size, void *out_pck,
 		    uint32_t * out_size, uint8_t ip_addr[4]);
 
+/* resolve packet in place */
+int resolve_packet2 (void *pck, uint32_t * size, uint8_t ip_addr[4]);
+
+/* resolve packet in place and add eth encap */
+int resolve_packet3 (void **pck, uint32_t * size, uint8_t ip_addr[4]);
+
 int generate_packet (void *pck, uint32_t * size, uint8_t saddr[4],
 		     uint8_t daddr[4], uint8_t hw_daddr[6], uint32_t seq);
 
+int generate_packet2 (void *pck, uint32_t * size, uint8_t saddr[4],
+		      uint8_t daddr[4], uint8_t hw_daddr[6], uint32_t seq,
+		      icmpr_flow_mode_t);
+
 int print_packet (void *pck);
 
 #endif /* _ICMP_PROTO_H_ */

extras/libmemif/examples/icmp_responder/main.c

@@ -225,7 +225,7 @@ icmpr_buffer_alloc (long n, uint16_t qid)
   int err;
   uint16_t r;
   /* set data pointer to shared memory and set buffer_len to shared mmeory buffer len */
-  err = memif_buffer_alloc (c->conn, qid, c->tx_bufs, n, &r, 128);
+  err = memif_buffer_alloc (c->conn, qid, c->tx_bufs, n, &r, 0);
   if (err != MEMIF_ERR_SUCCESS)
     {
       INFO ("memif_buffer_alloc: %s", memif_strerror (err));
@@ -311,24 +311,25 @@ on_interrupt (memif_conn_handle_t conn, void *private_ctx, uint16_t qid)
 		      &(c->tx_bufs + i)->len, c->ip_addr);
     }
 
+  uint16_t fb;
   /* mark memif buffers and shared memory buffers as free */
-  err = memif_refill_queue (c->conn, qid, rx);
-  c->rx_buf_num -= rx;
+  err = memif_refill_queue (c->conn, qid, rx, 0);
+  c->rx_buf_num -= fb;
 
   DBG ("freed %d buffers. %u/%u alloc/free buffers",
-       rx, c->rx_buf_num, MAX_MEMIF_BUFS - c->rx_buf_num);
+       fb, c->rx_buf_num, MAX_MEMIF_BUFS - c->rx_buf_num);
 
   icmpr_tx_burst (c->tx_qid);
 
   return 0;
 
 error:
-  err = memif_refill_queue (c->conn, qid, rx);
+  err = memif_refill_queue (c->conn, qid, rx, 0);
   if (err != MEMIF_ERR_SUCCESS)
     INFO ("memif_buffer_free: %s", memif_strerror (err));
-  c->rx_buf_num -= rx;
+  c->rx_buf_num -= fb;
   DBG ("freed %d buffers. %u/%u alloc/free buffers",
-       rx, c->rx_buf_num, MAX_MEMIF_BUFS - c->rx_buf_num);
+       fb, c->rx_buf_num, MAX_MEMIF_BUFS - c->rx_buf_num);
   return 0;
 }